discourse/lib/tasks/emoji.rake

# frozen_string_literal: true

require "active_support/test_case"
require "fileutils"
require "json"
require "nokogiri"
require "open-uri"
require_dependency "file_helper"

EMOJI_GROUPS_PATH ||= "lib/emoji/groups.json"

EMOJI_DB_PATH ||= "lib/emoji/db.json"

EMOJI_IMAGES_PATH ||= "public/images/emoji"

EMOJI_ORDERING_URL ||= "http://www.unicode.org/emoji/charts/emoji-ordering.html"

# Format is search pattern => associated emojis
# eg: "cry" => [ "sob" ]
# for a "cry" query should return: cry and sob
SEARCH_ALIASES ||= {
  "sad" => [ "frowning_face", "slightly_frowning_face", "sob", "crying_cat_face", "cry" ],
  "cry" => [ "sob" ]
}

# emoji aliases are actually created as images
# eg: "right_anger_bubble" => [ "anger_right" ]
# your app will physically have right_anger_bubble.png and anger_right.png
EMOJI_ALIASES ||= {
  "right_anger_bubble" => [ "anger_right" ],
  "ballot_box" => [ "ballot_box_with_ballot" ],
  "basketball_man" => [ "basketball_player", "person_with_ball" ],
  "beach_umbrella" => [ "umbrella_on_ground", "beach", "beach_with_umbrella" ],
  "parasol_on_ground" => [ "umbrella_on_ground" ],
  "bellhop_bell" => [ "bellhop" ],
  "biohazard" => [ "biohazard_sign" ],
  "bow_and_arrow" => [ "archery" ],
  "spiral_calendar" => [ "calendar_spiral", "spiral_calendar_pad" ],
  "card_file_box" => [ "card_box" ],
  "champagne" => [ "bottle_with_popping_cork" ],
  "cheese" => [ "cheese_wedge" ],
  "city_sunset" => [ "city_dusk" ],
  "couch_and_lamp" => [ "couch" ],
  "crayon" => [ "lower_left_crayon" ],
  "cricket_bat_and_ball" => [ "cricket_bat_ball" ],
  "latin_cross" => [ "cross" ],
  "dagger" => [ "dagger_knife" ],
  "desktop_computer" => [ "desktop" ],
  "card_index_dividers" => [ "dividers" ],
  "dove" => [ "dove_of_peace" ],
  "footprints" => [ "feet" ],
  "fire" => [ "flame" ],
  "black_flag" => [ "flag_black", "waving_black_flag" ],
  "cn" => [ "flag_cn" ],
  "de" => [ "flag_de" ],
  "es" => [ "flag_es" ],
  "fr" => [ "flag_fr" ],
  "uk" => [ "gb", "flag_gb" ],
  "it" => [ "flag_it" ],
  "jp" => [ "flag_jp" ],
  "kr" => [ "flag_kr" ],
  "ru" => [ "flag_ru" ],
  "us" => [ "flag_us" ],
  "white_flag" => [ "flag_white", "waving_white_flag" ],
  "plate_with_cutlery" => [ "fork_knife_plate", "fork_and_knife_with_plate" ],
  "framed_picture" => [ "frame_photo", "frame_with_picture" ],
  "hammer_and_pick" => [ "hammer_pick" ],
  "heavy_heart_exclamation" => [ "heart_exclamation", "heavy_heart_exclamation_mark_ornament" ],
  "houses" => [ "homes", "house_buildings" ],
  "hotdog" => [ "hot_dog" ],
  "derelict_house" => [ "house_abandoned", "derelict_house_building" ],
  "desert_island" => [ "island" ],
  "old_key" => [ "key2" ],
  "laughing" => [ "satisfied" ],
  "business_suit_levitating" => [ "levitate", "man_in_business_suit_levitating" ],
  "weight_lifting_man" => [ "lifter", "weight_lifter" ],
  "medal_sports" => [ "medal", "sports_medal" ],
  "metal" => [ "sign_of_the_horns" ],
  "fu" => [ "middle_finger", "reversed_hand_with_middle_finger_extended" ],
  "motorcycle" => [ "racing_motorcycle" ],
  "mountain_snow" => [ "snow_capped_mountain" ],
  "newspaper_roll" => [ "newspaper2", "rolled_up_newspaper" ],
  "spiral_notepad" => [ "notepad_spiral", "spiral_note_pad" ],
  "oil_drum" => [ "oil" ],
  "older_woman" => [ "grandma" ],
  "paintbrush" => [ "lower_left_paintbrush" ],
  "paperclips" => [ "linked_paperclips" ],
  "pause_button" => [ "double_vertical_bar" ],
  "peace_symbol" => [ "peace" ],
  "fountain_pen" => [ "pen_fountain", "lower_left_fountain_pen" ],
  "ping_pong" => [ "table_tennis" ],
  "place_of_worship" => [ "worship_symbol" ],
  "poop" => [ "shit", "hankey", "poo" ],
  "radioactive" => [ "radioactive_sign" ],
  "railway_track" => [ "railroad_track" ],
  "robot" => [ "robot_face" ],
  "skull" => [ "skeleton" ],
  "skull_and_crossbones" => [ "skull_crossbones" ],
  "speaking_head" => [ "speaking_head_in_silhouette" ],
  "male_detective" => [ "spy", "sleuth_or_spy" ],
  "thinking" => [ "thinking_face" ],
  "-1" => [ "thumbsdown" ],
  "+1" => [ "thumbsup" ],
  "cloud_with_lightning_and_rain" => [ "thunder_cloud_rain", "thunder_cloud_and_rain" ],
  "tickets" => [ "admission_tickets" ],
  "next_track_button" => [ "track_next", "next_track" ],
  "previous_track_button" => [ "track_previous", "previous_track" ],
  "unicorn" => [ "unicorn_face" ],
  "funeral_urn" => [ "urn" ],
  "sun_behind_large_cloud" => [ "white_sun_cloud", "white_sun_behind_cloud" ],
  "sun_behind_rain_cloud" => [ "white_sun_rain_cloud", "white_sun_behind_cloud_with_rain" ],
  "partly_sunny" => [ "white_sun_small_cloud", "white_sun_with_small_cloud" ],
  "open_umbrella" => [ "umbrella2" ],
  "hammer_and_wrench" => [ "tools" ],
  "face_with_thermometer" => [ "thermometer_face" ],
  "timer_clock" => [ "timer" ],
  "keycap_ten" => [ "ten" ],
  "memo" => [ "pencil" ],
  "rescue_worker_helmet" => [ "helmet_with_cross", "helmet_with_white_cross" ],
  "slightly_smiling_face" => [ "slightly_smiling", "slight_smile"],
  "construction_worker_man" => [ "construction_worker" ],
  "upside_down_face" => [ "upside_down" ],
  "money_mouth_face" => [ "money_mouth" ],
  "nerd_face" => [ "nerd" ],
  "hugs" => [ "hugging", "hugging_face" ],
  "roll_eyes" => [ "rolling_eyes", "face_with_rolling_eyes" ],
  "slightly_frowning_face" => [ "slight_frown" ],
  "frowning_face" => [ "frowning2", "white_frowning_face" ],
  "zipper_mouth_face" => [ "zipper_mouth" ],
  "face_with_head_bandage" => [ "head_bandage" ],
  "raised_hand_with_fingers_splayed" => [ "hand_splayed" ],
  "raised_hand" => [ "hand" ],
  "vulcan_salute" => [ "vulcan", "raised_hand_with_part_between_middle_and_ring_fingers" ],
  "policeman" => [ "cop" ],
  "running_man" => [ "runner" ],
  "walking_man" => [ "walking" ],
  "bowing_man" => [ "bow" ],
  "no_good_woman" => [ "no_good" ],
  "raising_hand_woman" => [ "raising_hand" ],
  "pouting_woman" => [ "person_with_pouting_face" ],
  "frowning_woman" => [ "person_frowning" ],
  "haircut_woman" => [ "haircut" ],
  "massage_woman" => [ "massage" ],
  "tshirt" => [ "shirt" ],
  "biking_man" => [ "bicyclist" ],
  "mountain_biking_man" => [ "mountain_bicyclist" ],
  "passenger_ship" => [ "cruise_ship" ],
  "motor_boat" => [ "motorboat", "boat" ],
  "flight_arrival" => [ "airplane_arriving" ],
  "flight_departure" => [ "airplane_departure" ],
  "small_airplane" => [ "airplane_small" ],
  "racing_car" => [ "race_car" ],
  "family_man_woman_boy_boy" => [ "family_man_woman_boys" ],
  "family_man_woman_girl_girl" => [ "family_man_woman_girls" ],
  "family_woman_woman_boy" => [ "family_women_boy" ],
  "family_woman_woman_girl" => [ "family_women_girl" ],
  "family_woman_woman_girl_boy" => [ "family_women_girl_boy" ],
  "family_woman_woman_boy_boy" => [ "family_women_boys" ],
  "family_woman_woman_girl_girl" => [ "family_women_girls" ],
  "family_man_man_boy" => [ "family_men_boy" ],
  "family_man_man_girl" => [ "family_men_girl" ],
  "family_man_man_girl_boy" => [ "family_men_girl_boy" ],
  "family_man_man_boy_boy" => [ "family_men_boys" ],
  "family_man_man_girl_girl" => [ "family_men_girls" ],
  "cloud_with_lightning" => [ "cloud_lightning" ],
  "tornado" => [ "cloud_tornado", "cloud_with_tornado" ],
  "cloud_with_rain" => [ "cloud_rain" ],
  "cloud_with_snow" => [ "cloud_snow" ],
  "asterisk" => [ "keycap_star" ],
  "studio_microphone" => [ "microphone2" ],
  "medal_military" => [ "military_medal" ],
  "couple_with_heart_woman_woman" => [ "female_couple_with_heart" ],
  "couple_with_heart_man_man" => [ "male_couple_with_heart" ],
  "couplekiss_woman_woman" => [ "female_couplekiss" ],
  "couplekiss_man_man" => [ "male_couplekiss" ],
  "honeybee" => [ "bee" ],
  "lion" => [ "lion_face" ],
  "artificial_satellite" => [ "satellite_orbital" ],
  "computer_mouse" => [ "mouse_three_button", "three_button_mouse" ],
  "hocho" => [ "knife" ],
  "swimming_man" => [ "swimmer" ],
  "wind_face" => [ "wind_blowing_face" ],
  "golfing_man" => [ "golfer" ],
  "facepunch" => [ "punch" ],
  "building_construction" => [ "construction_site" ],
  "family_man_woman_girl_boy" => [ "family" ],
  "ice_hockey" => [ "hockey" ],
  "snowman_with_snow" => [ "snowman2" ],
  "play_or_pause_button" => [ "play_pause" ],
  "film_projector" => [ "projector" ],
  "shopping" => [ "shopping_bags" ],
  "open_book" => [ "book" ],
  "national_park" => [ "park" ],
  "world_map" => [ "map" ],
  "pen" => [ "pen_ballpoint", "lower_left_ballpoint_pen" ],
  "email" => [ "envelope", "e-mail" ],
  "phone" => [ "telephone" ],
  "atom_symbol" => [ "atom" ],
  "mantelpiece_clock" => [ "clock" ],
  "camera_flash" => [ "camera_with_flash" ],
  "film_strip" => [ "film_frames" ],
  "balance_scale" => [ "scales" ],
  "surfing_man" => [ "surfer" ],
  "couplekiss_man_woman" => [ "couplekiss" ],
  "couple_with_heart_woman_man" => [ "couple_with_heart" ],
  "clamp" => [ "compression" ],
  "dancing_women" => [ "dancers" ],
  "blonde_man" => [ "person_with_blond_hair" ],
  "sleeping_bed" => [ "sleeping_accommodation" ],
  "om" => [ "om_symbol" ],
  "tipping_hand_woman" => [ "information_desk_person" ],
  "rowing_man" => [ "rowboat" ],
  "new_moon" => [ "moon" ],
  "oncoming_automobile" => [ "car", "automobile" ],
  "fleur_de_lis" => [ "fleur-de-lis" ],
  "face_vomiting" => [ "puke" ]
}

EMOJI_GROUPS ||= [
  {
    "name" => "smileys_&_emotion",
    "tabicon" => "grinning"
  },
  {
    "name" => "people_&_body",
    "tabicon" => "wave"
  },
  {
    "name" => "animals_&_nature",
    "tabicon" => "evergreen_tree"
  },
  {
    "name" => "food_&_drink",
    "tabicon" => "hamburger"
  },
  {
    "name" => "travel_&_places",
    "tabicon" => "airplane"
  },
  {
    "name" => "activities",
    "tabicon" => "soccer"
  },
  {
    "name" => "objects",
    "tabicon" => "eyeglasses"
  },
  {
    "name" => "symbols",
    "tabicon" => "white_check_mark"
  },
  {
    "name" => "flags",
    "tabicon" => "checkered_flag"
  }
]

FITZPATRICK_SCALE ||= [ "1f3fb", "1f3fc", "1f3fd", "1f3fe", "1f3ff" ]

DEFAULT_SET ||= "twitter"

# Replace the platform by another when downloading the image (accepts names or categories)
EMOJI_IMAGES_PATCH ||= {
  "apple" => { "snowboarder" => "twitter" },
  "emoji_one" => { "country-flag" => "twitter" },
  "windows" => { "country-flag" => "twitter" }
}

EMOJI_SETS ||= {
  "apple" => "apple",
  "google" => "google",
  "google_blob" => "google_classic",
  "facebook" => "facebook_messenger",
  "twitter" => "twitter",
  "emoji_one" => "emoji_one",
  "windows" => "win10",
}

EMOJI_DB_REPO ||= "git@github.com:jjaffeux/emoji-db.git"

EMOJI_DB_REPO_PATH ||= File.join("tmp", "emoji-db")

GENERATED_PATH ||= File.join(EMOJI_DB_REPO_PATH, "generated")

desc "update emoji images"
task "emoji:update" do
  copy_emoji_db

  json_db = open(File.join(GENERATED_PATH, "db.json")).read
  db = JSON.parse(json_db)

  write_db_json(db["emojis"], db["translations"])
  fix_incomplete_sets(db["emojis"])
  write_aliases
  groups = generate_emoji_groups(db["emojis"], db["sections"])
  write_js_groups(db["emojis"], groups)
  optimize_images(Dir.glob(File.join(Rails.root, EMOJI_IMAGES_PATH, "/**/*.png")))

  TestEmojiUpdate.run_and_summarize

  FileUtils.rm_rf(EMOJI_DB_REPO_PATH)
end

desc "test the emoji generation script"
task "emoji:test" do
  ENV['EMOJI_TEST'] = "1"
  Rake::Task["emoji:update"].invoke
end

def optimize_images(images)
  images.each do |filename|
    FileHelper.image_optim(
      allow_pngquant: true,
      strip_image_metadata: true
    ).optimize_image!(filename)
  end
end

def copy_emoji_db
  `rm -rf tmp/emoji-db && git clone #{EMOJI_DB_REPO} tmp/emoji-db`

  path = "#{EMOJI_IMAGES_PATH}/**/*"
  confirm_overwrite(path)
  puts "Cleaning emoji folder..."
  FileUtils.rm_rf(Dir.glob(path))

  EMOJI_SETS.each do |set_name, set_destination|
    origin = File.join(GENERATED_PATH, set_name)
    destination = File.join(EMOJI_IMAGES_PATH, set_destination)
    FileUtils.mv(origin, destination)
  end
end

def fix_incomplete_sets(emojis)
  emojis.each do |code, config|
    EMOJI_SETS.each do |set_name, set_destination|
      patch_set = EMOJI_SETS[EMOJI_IMAGES_PATCH.dig(set_name, config["name"])] ||
        EMOJI_SETS[EMOJI_IMAGES_PATCH.dig(set_name, config["category"])]

      if patch_set || !File.exist?(File.join(EMOJI_IMAGES_PATH, set_destination, "#{config['name']}.png"))
        origin = File.join(EMOJI_IMAGES_PATH, patch_set || EMOJI_SETS[DEFAULT_SET], config['name'])

        FileUtils.cp("#{origin}.png", File.join(EMOJI_IMAGES_PATH, set_destination, "#{config['name']}.png"))
        if File.directory?(origin)
          FileUtils.cp_r(origin, File.join(EMOJI_IMAGES_PATH, set_destination, config['name']))
        end
      end
    end
  end
end

def generate_emoji_groups(keywords, sections)
  puts "Generating groups..."

  list = open(EMOJI_ORDERING_URL).read
  doc = Nokogiri::HTML(list)
  table = doc.css("table")[0]

  EMOJI_GROUPS.map do |group|
    group["icons"] ||= []

    sub_sections = sections[group["name"]]["sub_sections"]
    sub_sections.each do |section|
      title_section = table.css("tr th a[@name='#{section}']")
      emoji_list_section = title_section.first.parent.parent.next_element
      emoji_list_section.css("a.plain img").each do |link|
        emoji_code = link.attr("title")
          .scan(/U\+(.{4,5})\b/)
          .flatten
          .map { |code| code.downcase.strip }
          .join("_")

        emoji_char = code_to_emoji(emoji_code)

        if emoji = keywords[emoji_char]
          group["icons"] << { name: emoji["name"], diversity: emoji["fitzpatrick_scale"] }
        end
      end
    end
    group.delete("sections")
    group
  end
end

def write_aliases
  EMOJI_ALIASES.each do |original, aliases|
    aliases.each do |emoji_alias|
      EMOJI_SETS.each do |set_name, set_destination|
        origin_file = File.join(EMOJI_IMAGES_PATH, set_destination, "#{original}.png")
        origin_dir = File.join(EMOJI_IMAGES_PATH, set_destination, original)
        FileUtils.cp(origin_file, File.join(EMOJI_IMAGES_PATH, set_destination, "#{emoji_alias}.png"))

        if File.directory?(origin_dir)
          FileUtils.cp_r(origin_dir, File.join(EMOJI_IMAGES_PATH, set_destination, emoji_alias))
        end
      end
    end
  end
end

def write_db_json(emojis, translations)
  puts "Writing #{EMOJI_DB_PATH}..."

  confirm_overwrite(EMOJI_DB_PATH)

  FileUtils.mkdir_p(File.expand_path("..", EMOJI_DB_PATH))

  # skin tones variations of emojis shouldn’t appear in autocomplete
  emojis_without_tones = emojis
    .select { |char, config|
                           !FITZPATRICK_SCALE.any? { |scale|
                             codepoints_to_code(char.codepoints, config["fitzpatrick_scale"])[scale]
                           }
                         }
    .map { |char, config|
    {
      "code" => codepoints_to_code(char.codepoints, config["fitzpatrick_scale"]).tr("_", "-"),
      "name" => config["name"]
    }
  }

  emoji_with_tones = emojis
    .select { |code, config| config["fitzpatrick_scale"] }
    .map { |code, config| config["name"] }

  db = {
    "emojis" => emojis_without_tones,
    "tonableEmojis" => emoji_with_tones,
    "aliases" => EMOJI_ALIASES,
    "searchAliases" => SEARCH_ALIASES,
    "translations" => translations
  }

  File.write(EMOJI_DB_PATH, JSON.pretty_generate(db))
end

def write_js_groups(emojis, groups)
  puts "Writing #{EMOJI_GROUPS_PATH}..."

  confirm_overwrite(EMOJI_GROUPS_PATH)

  template = JSON.pretty_generate(groups)
  FileUtils.mkdir_p(File.expand_path("..", EMOJI_GROUPS_PATH))
  File.write(EMOJI_GROUPS_PATH, template)
end

def code_to_emoji(code)
  code
    .split("_")
    .map { |e| e.to_i(16) }
    .pack "U*"
end

def codepoints_to_code(codepoints, fitzpatrick_scale)
  codepoints = codepoints
    .map { |c| c.to_s(16).rjust(4, "0") }
    .join("_")
    .downcase

  if !fitzpatrick_scale
    codepoints.gsub!(/_fe0f$/, "")
  end

  codepoints
end

def confirm_overwrite(path)
  return if ENV['EMOJI_TEST']

  STDOUT.puts("[!] You are about to overwrite #{path}, are you sure? [CTRL+c] to cancel, [ENTER] to continue")
  STDIN.gets.chomp
end

class TestEmojiUpdate < MiniTest::Test
  def self.run_and_summarize
    puts "Runnings tests..."
    reporter = Minitest::SummaryReporter.new
    TestEmojiUpdate.run(reporter)
    puts reporter.to_s
  end

  def image_path(style, name)
    File.join("public", "images", "emoji", style, "#{name}.png")
  end

  def test_code_to_emoji
    assert_equal "😎", code_to_emoji("1f60e")
  end

  def test_codepoints_to_code
    assert_equal "1f6b5_200d_2640", codepoints_to_code([128693, 8205, 9792, 65039], false)
  end

  def test_codepoints_to_code_with_scale
    assert_equal "1f6b5_200d_2640_fe0f", codepoints_to_code([128693, 8205, 9792, 65039], true)
  end

  def test_groups_js_es6_creation
    assert File.exists?(EMOJI_GROUPS_PATH)
    assert File.size?(EMOJI_GROUPS_PATH)
  end

  def test_db_json_creation
    assert File.exists?(EMOJI_DB_PATH)
    assert File.size?(EMOJI_DB_PATH)
  end

  def test_alias_creation
    original_image = image_path("apple", "right_anger_bubble")
    alias_image = image_path("apple", "anger_right")

    assert_equal File.size(original_image), File.size(alias_image)
  end

  def test_cell_index_patch
    original_image = image_path("apple", "snowboarder")
    alias_image = image_path("twitter", "snowboarder")

    assert_equal File.size(original_image), File.size(alias_image)
  end

  def test_scales
    original_image = image_path("apple", "blonde_woman")
    assert File.exists?(original_image)
    assert File.size?(original_image)

    (2..6).each do |scale|
      image = image_path("apple", "blonde_woman/#{scale}")
      assert File.exists?(image)
      assert File.size?(image)
    end
  end

  def test_default_set
    original_image = image_path("twitter", "snowboarder")
    alias_image = image_path("apple", "snowboarder")
    assert_equal File.size(original_image), File.size(alias_image)

    original_image = image_path("twitter", "macau")
    alias_image = image_path("emoji_one", "macau")
    assert_equal File.size(original_image), File.size(alias_image)
  end
end