discourse/lib/tasks/emoji.rake

548 lines
18 KiB
Ruby
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# frozen_string_literal: true
require "active_support/test_case"
require "fileutils"
require "json"
require "nokogiri"
require "open-uri"
require "file_helper"
EMOJI_GROUPS_PATH ||= "lib/emoji/groups.json"
EMOJI_DB_PATH ||= "lib/emoji/db.json"
EMOJI_IMAGES_PATH ||= "public/images/emoji"
EMOJI_ORDERING_URL ||= "http://www.unicode.org/emoji/charts/emoji-ordering.html"
# emoji aliases are actually created as images
# eg: "right_anger_bubble" => [ "anger_right" ]
# your app will physically have right_anger_bubble.png and anger_right.png
EMOJI_ALIASES ||= {
"right_anger_bubble" => ["anger_right"],
"ballot_box" => ["ballot_box_with_ballot"],
"basketball_man" => %w[basketball_player person_with_ball],
"beach_umbrella" => %w[umbrella_on_ground beach beach_with_umbrella],
"parasol_on_ground" => ["umbrella_on_ground"],
"bellhop_bell" => ["bellhop"],
"biohazard" => ["biohazard_sign"],
"bow_and_arrow" => ["archery"],
"spiral_calendar" => %w[calendar_spiral spiral_calendar_pad],
"card_file_box" => ["card_box"],
"champagne" => ["bottle_with_popping_cork"],
"cheese" => ["cheese_wedge"],
"city_sunset" => ["city_dusk"],
"couch_and_lamp" => ["couch"],
"crayon" => ["lower_left_crayon"],
"cricket_bat_and_ball" => ["cricket_bat_ball"],
"latin_cross" => ["cross"],
"dagger" => ["dagger_knife"],
"desktop_computer" => ["desktop"],
"card_index_dividers" => ["dividers"],
"dove" => ["dove_of_peace"],
"footprints" => ["feet"],
"fire" => ["flame"],
"black_flag" => %w[flag_black waving_black_flag],
"cn" => ["flag_cn"],
"de" => ["flag_de"],
"es" => ["flag_es"],
"fr" => ["flag_fr"],
"uk" => %w[gb flag_gb],
"it" => ["flag_it"],
"jp" => ["flag_jp"],
"kr" => ["flag_kr"],
"ru" => ["flag_ru"],
"us" => ["flag_us"],
"white_flag" => %w[flag_white waving_white_flag],
"plate_with_cutlery" => %w[fork_knife_plate fork_and_knife_with_plate],
"framed_picture" => %w[frame_photo frame_with_picture],
"hammer_and_pick" => ["hammer_pick"],
"heavy_heart_exclamation" => %w[heart_exclamation heavy_heart_exclamation_mark_ornament],
"houses" => %w[homes house_buildings],
"hotdog" => ["hot_dog"],
"derelict_house" => %w[house_abandoned derelict_house_building],
"desert_island" => ["island"],
"old_key" => ["key2"],
"laughing" => ["satisfied"],
"business_suit_levitating" => %w[levitate man_in_business_suit_levitating],
"weight_lifting_man" => %w[lifter weight_lifter],
"medal_sports" => %w[medal sports_medal],
"metal" => ["sign_of_the_horns"],
"fu" => %w[middle_finger reversed_hand_with_middle_finger_extended],
"motorcycle" => ["racing_motorcycle"],
"mountain_snow" => ["snow_capped_mountain"],
"newspaper_roll" => %w[newspaper2 rolled_up_newspaper],
"spiral_notepad" => %w[notepad_spiral spiral_note_pad],
"oil_drum" => ["oil"],
"older_woman" => ["grandma"],
"paintbrush" => ["lower_left_paintbrush"],
"paperclips" => ["linked_paperclips"],
"pause_button" => ["double_vertical_bar"],
"peace_symbol" => ["peace"],
"fountain_pen" => %w[pen_fountain lower_left_fountain_pen],
"ping_pong" => ["table_tennis"],
"place_of_worship" => ["worship_symbol"],
"poop" => %w[shit hankey poo],
"radioactive" => ["radioactive_sign"],
"railway_track" => ["railroad_track"],
"robot" => ["robot_face"],
"skull" => ["skeleton"],
"skull_and_crossbones" => ["skull_crossbones"],
"speaking_head" => ["speaking_head_in_silhouette"],
"male_detective" => %w[spy sleuth_or_spy],
"thinking" => ["thinking_face"],
"-1" => ["thumbsdown"],
"+1" => ["thumbsup"],
"cloud_with_lightning_and_rain" => %w[thunder_cloud_rain thunder_cloud_and_rain],
"tickets" => ["admission_tickets"],
"next_track_button" => %w[track_next next_track],
"previous_track_button" => %w[track_previous previous_track],
"unicorn" => ["unicorn_face"],
"funeral_urn" => ["urn"],
"sun_behind_large_cloud" => %w[white_sun_cloud white_sun_behind_cloud],
"sun_behind_rain_cloud" => %w[white_sun_rain_cloud white_sun_behind_cloud_with_rain],
"partly_sunny" => %w[white_sun_small_cloud white_sun_with_small_cloud],
"open_umbrella" => ["umbrella2"],
"hammer_and_wrench" => ["tools"],
"face_with_thermometer" => ["thermometer_face"],
"timer_clock" => ["timer"],
"keycap_ten" => ["ten"],
"memo" => ["pencil"],
"rescue_worker_helmet" => %w[helmet_with_cross helmet_with_white_cross],
"slightly_smiling_face" => %w[slightly_smiling slight_smile],
"construction_worker_man" => ["construction_worker"],
"upside_down_face" => ["upside_down"],
"money_mouth_face" => ["money_mouth"],
"nerd_face" => ["nerd"],
"hugs" => %w[hugging hugging_face],
"roll_eyes" => %w[rolling_eyes face_with_rolling_eyes],
"slightly_frowning_face" => %w[frowning slight_frown],
"frowning_face" => %w[frowning2 white_frowning_face],
"zipper_mouth_face" => ["zipper_mouth"],
"face_with_head_bandage" => ["head_bandage"],
"raised_hand_with_fingers_splayed" => ["hand_splayed"],
"raised_hand" => ["hand"],
"vulcan_salute" => %w[vulcan raised_hand_with_part_between_middle_and_ring_fingers],
"policeman" => ["cop"],
"running_man" => ["runner"],
"walking_man" => ["walking"],
"bowing_man" => ["bow"],
"no_good_woman" => ["no_good"],
"raising_hand_woman" => ["raising_hand"],
"pouting_woman" => ["person_with_pouting_face"],
"frowning_woman" => ["person_frowning"],
"haircut_woman" => ["haircut"],
"massage_woman" => ["massage"],
"tshirt" => ["shirt"],
"biking_man" => ["bicyclist"],
"mountain_biking_man" => ["mountain_bicyclist"],
"passenger_ship" => ["cruise_ship"],
"motor_boat" => %w[motorboat boat],
"flight_arrival" => ["airplane_arriving"],
"flight_departure" => ["airplane_departure"],
"small_airplane" => ["airplane_small"],
"racing_car" => ["race_car"],
"family_man_woman_boy_boy" => ["family_man_woman_boys"],
"family_man_woman_girl_girl" => ["family_man_woman_girls"],
"family_woman_woman_boy" => ["family_women_boy"],
"family_woman_woman_girl" => ["family_women_girl"],
"family_woman_woman_girl_boy" => ["family_women_girl_boy"],
"family_woman_woman_boy_boy" => ["family_women_boys"],
"family_woman_woman_girl_girl" => ["family_women_girls"],
"family_man_man_boy" => ["family_men_boy"],
"family_man_man_girl" => ["family_men_girl"],
"family_man_man_girl_boy" => ["family_men_girl_boy"],
"family_man_man_boy_boy" => ["family_men_boys"],
"family_man_man_girl_girl" => ["family_men_girls"],
"cloud_with_lightning" => ["cloud_lightning"],
"tornado" => %w[cloud_tornado cloud_with_tornado],
"cloud_with_rain" => ["cloud_rain"],
"cloud_with_snow" => ["cloud_snow"],
"asterisk" => ["keycap_star"],
"studio_microphone" => ["microphone2"],
"medal_military" => ["military_medal"],
"couple_with_heart_woman_woman" => ["female_couple_with_heart"],
"couple_with_heart_man_man" => ["male_couple_with_heart"],
"couplekiss_woman_woman" => ["female_couplekiss"],
"couplekiss_man_man" => ["male_couplekiss"],
"honeybee" => ["bee"],
"lion" => ["lion_face"],
"artificial_satellite" => ["satellite_orbital"],
"computer_mouse" => %w[mouse_three_button three_button_mouse],
"hocho" => ["knife"],
"swimming_man" => ["swimmer"],
"wind_face" => ["wind_blowing_face"],
"golfing_man" => ["golfer"],
"facepunch" => ["punch"],
"building_construction" => ["construction_site"],
"family_man_woman_girl_boy" => ["family"],
"ice_hockey" => ["hockey"],
"snowman_with_snow" => ["snowman2"],
"play_or_pause_button" => ["play_pause"],
"film_projector" => ["projector"],
"shopping" => ["shopping_bags"],
"open_book" => ["book"],
"national_park" => ["park"],
"world_map" => ["map"],
"pen" => %w[pen_ballpoint lower_left_ballpoint_pen],
"email" => %w[envelope e-mail],
"phone" => ["telephone"],
"atom_symbol" => ["atom"],
"mantelpiece_clock" => ["clock"],
"camera_flash" => ["camera_with_flash"],
"film_strip" => ["film_frames"],
"balance_scale" => ["scales"],
"surfing_man" => ["surfer"],
"couplekiss_man_woman" => ["couplekiss"],
"couple_with_heart_woman_man" => ["couple_with_heart"],
"clamp" => ["compression"],
"dancing_women" => ["dancers"],
"blonde_man" => ["person_with_blond_hair"],
"sleeping_bed" => ["sleeping_accommodation"],
"om" => ["om_symbol"],
"tipping_hand_woman" => ["information_desk_person"],
"rowing_man" => ["rowboat"],
"new_moon" => ["moon"],
"oncoming_automobile" => %w[car automobile],
"fleur_de_lis" => ["fleur-de-lis"],
"face_vomiting" => ["puke"],
"smile" => ["grinning_face_with_smiling_eyes"],
"frowning_with_open_mouth" => ["frowning_face_with_open_mouth"],
}
EMOJI_GROUPS ||= [
{ "name" => "smileys_&_emotion", "tabicon" => "grinning" },
{ "name" => "people_&_body", "tabicon" => "wave" },
{ "name" => "animals_&_nature", "tabicon" => "evergreen_tree" },
{ "name" => "food_&_drink", "tabicon" => "hamburger" },
{ "name" => "travel_&_places", "tabicon" => "airplane" },
{ "name" => "activities", "tabicon" => "soccer" },
{ "name" => "objects", "tabicon" => "eyeglasses" },
{ "name" => "symbols", "tabicon" => "white_check_mark" },
{ "name" => "flags", "tabicon" => "checkered_flag" },
]
FITZPATRICK_SCALE ||= %w[1f3fb 1f3fc 1f3fd 1f3fe 1f3ff]
DEFAULT_SET ||= "twitter"
# Replace the platform by another when downloading the image (accepts names or categories)
EMOJI_IMAGES_PATCH ||= {
"apple" => {
"snowboarder" => "twitter",
},
"windows" => {
"country-flag" => "twitter",
},
}
EMOJI_SETS ||= {
"apple" => "apple",
"google" => "google",
"google_blob" => "google_classic",
"facebook" => "facebook_messenger",
"twitter" => "twitter",
"windows" => "win10",
}
EMOJI_DB_REPO ||= "git@github.com:xfalcox/emoji-db.git"
EMOJI_DB_REPO_PATH ||= File.join("tmp", "emoji-db")
GENERATED_PATH ||= File.join(EMOJI_DB_REPO_PATH, "generated")
def search_aliases(emojis)
# Format is search pattern => associated emojis
# eg: "cry" => [ "sob" ]
# for a "cry" query should return: cry and sob
@aliases ||=
begin
aliases = {
"sad" => %w[frowning_face slightly_frowning_face sob crying_cat_face cry],
"cry" => ["sob"],
}
emojis.each do |_, config|
next if config["search_aliases"].blank?
config["search_aliases"].each do |name|
aliases[name] ||= []
aliases[name] << config["name"]
end
end
aliases.map { |_, names| names.uniq! }
aliases
end
end
desc "update emoji images"
task "emoji:update" do
abort("This task can't be run on production.") if Rails.env.production?
copy_emoji_db
json_db = File.read(File.join(GENERATED_PATH, "db.json"))
db = JSON.parse(json_db)
write_db_json(db["emojis"], db["translations"], search_aliases(db["emojis"]))
fix_incomplete_sets(db["emojis"])
write_aliases
groups = generate_emoji_groups(db["emojis"], db["sections"])
write_js_groups(db["emojis"], groups)
optimize_images(Dir.glob(File.join(Rails.root, EMOJI_IMAGES_PATH, "/**/*.png")))
TestEmojiUpdate.run_and_summarize
FileUtils.rm_rf(EMOJI_DB_REPO_PATH)
end
desc "test the emoji generation script"
task "emoji:test" do
ENV["EMOJI_TEST"] = "1"
Rake::Task["emoji:update"].invoke
end
def optimize_images(images)
images.each do |filename|
FileHelper.image_optim(allow_pngquant: true, strip_image_metadata: true).optimize_image!(
filename,
)
end
end
def copy_emoji_db
`rm -rf tmp/emoji-db && git clone -b unicodeorg-as-source-of-truth --depth 1 #{EMOJI_DB_REPO} tmp/emoji-db`
path = "#{EMOJI_IMAGES_PATH}/**/*"
confirm_overwrite(path)
puts "Cleaning emoji folder..."
emoji_assets = Dir.glob(path)
emoji_assets.delete_if { |x| x == "#{EMOJI_IMAGES_PATH}/emoji_one" }
FileUtils.rm_rf(emoji_assets)
EMOJI_SETS.each do |set_name, set_destination|
origin = File.join(GENERATED_PATH, set_name)
destination = File.join(EMOJI_IMAGES_PATH, set_destination)
FileUtils.mv(origin, destination)
end
end
def fix_incomplete_sets(emojis)
emojis.each do |code, config|
EMOJI_SETS.each do |set_name, set_destination|
patch_set =
EMOJI_SETS[EMOJI_IMAGES_PATCH.dig(set_name, config["name"])] ||
EMOJI_SETS[EMOJI_IMAGES_PATCH.dig(set_name, config["category"])]
if patch_set ||
!File.exist?(File.join(EMOJI_IMAGES_PATH, set_destination, "#{config["name"]}.png"))
origin = File.join(EMOJI_IMAGES_PATH, patch_set || EMOJI_SETS[DEFAULT_SET], config["name"])
FileUtils.cp(
"#{origin}.png",
File.join(EMOJI_IMAGES_PATH, set_destination, "#{config["name"]}.png"),
)
if File.directory?(origin)
FileUtils.cp_r(origin, File.join(EMOJI_IMAGES_PATH, set_destination, config["name"]))
end
end
end
end
end
def generate_emoji_groups(keywords, sections)
puts "Generating groups..."
list = URI.parse(EMOJI_ORDERING_URL).read
doc = Nokogiri.HTML5(list)
table = doc.css("table")[0]
EMOJI_GROUPS.map do |group|
group["icons"] ||= []
sub_sections = sections[group["name"]]["sub_sections"]
sub_sections.each do |section|
title_section = table.css("tr th a[@name='#{section}']")
emoji_list_section = title_section.first.parent.parent.next_element
emoji_list_section
.css("a.plain img")
.each do |link|
emoji_code =
link
.attr("title")
.scan(/U\+(.{4,5})\b/)
.flatten
.map { |code| code.downcase.strip }
.join("_")
emoji_char = code_to_emoji(emoji_code)
if emoji = keywords[emoji_char]
group["icons"] << { name: emoji["name"], diversity: emoji["fitzpatrick_scale"] }
end
end
end
group.delete("sections")
group
end
end
def write_aliases
EMOJI_ALIASES.each do |original, aliases|
aliases.each do |emoji_alias|
EMOJI_SETS.each do |set_name, set_destination|
origin_file = File.join(EMOJI_IMAGES_PATH, set_destination, "#{original}.png")
origin_dir = File.join(EMOJI_IMAGES_PATH, set_destination, original)
FileUtils.cp(
origin_file,
File.join(EMOJI_IMAGES_PATH, set_destination, "#{emoji_alias}.png"),
)
if File.directory?(origin_dir)
FileUtils.cp_r(origin_dir, File.join(EMOJI_IMAGES_PATH, set_destination, emoji_alias))
end
end
end
end
end
def write_db_json(emojis, translations, search_aliases)
puts "Writing #{EMOJI_DB_PATH}..."
confirm_overwrite(EMOJI_DB_PATH)
FileUtils.mkdir_p(File.expand_path("..", EMOJI_DB_PATH))
# skin tones variations of emojis shouldnt appear in autocomplete
emojis_without_tones =
emojis
.select do |char, config|
!FITZPATRICK_SCALE.any? do |scale|
codepoints_to_code(char.codepoints, config["fitzpatrick_scale"])[scale]
end
end
.map do |char, config|
{
"code" => codepoints_to_code(char.codepoints, config["fitzpatrick_scale"]).tr("_", "-"),
"name" => config["name"],
}
end
emoji_with_tones =
emojis
.select { |code, config| config["fitzpatrick_scale"] }
.map { |code, config| config["name"] }
db = {
"emojis" => emojis_without_tones,
"tonableEmojis" => emoji_with_tones,
"aliases" => EMOJI_ALIASES,
"searchAliases" => search_aliases,
"translations" => translations,
}
File.write(EMOJI_DB_PATH, JSON.pretty_generate(db))
end
def write_js_groups(emojis, groups)
puts "Writing #{EMOJI_GROUPS_PATH}..."
confirm_overwrite(EMOJI_GROUPS_PATH)
template = JSON.pretty_generate(groups)
FileUtils.mkdir_p(File.expand_path("..", EMOJI_GROUPS_PATH))
File.write(EMOJI_GROUPS_PATH, template)
end
def code_to_emoji(code)
code.split("_").map { |e| e.to_i(16) }.pack "U*"
end
def codepoints_to_code(codepoints, fitzpatrick_scale)
codepoints = codepoints.map { |c| c.to_s(16).rjust(4, "0") }.join("_").downcase
codepoints.gsub!(/_fe0f\z/, "") if !fitzpatrick_scale
codepoints
end
def confirm_overwrite(path)
return if ENV["EMOJI_TEST"]
STDOUT.puts(
"[!] You are about to overwrite #{path}, are you sure? [CTRL+c] to cancel, [ENTER] to continue",
)
STDIN.gets.chomp
end
class TestEmojiUpdate < MiniTest::Test
def self.run_and_summarize
puts "Runnings tests..."
reporter = Minitest::SummaryReporter.new
TestEmojiUpdate.run(reporter)
puts reporter.to_s
end
def image_path(style, name)
File.join("public", "images", "emoji", style, "#{name}.png")
end
def test_code_to_emoji
assert_equal "😎", code_to_emoji("1f60e")
end
def test_codepoints_to_code
assert_equal "1f6b5_200d_2640", codepoints_to_code([128_693, 8205, 9792, 65_039], false)
end
def test_codepoints_to_code_with_scale
assert_equal "1f6b5_200d_2640_fe0f", codepoints_to_code([128_693, 8205, 9792, 65_039], true)
end
def test_groups_js_es6_creation
assert File.exist?(EMOJI_GROUPS_PATH)
assert File.size?(EMOJI_GROUPS_PATH)
end
def test_db_json_creation
assert File.exist?(EMOJI_DB_PATH)
assert File.size?(EMOJI_DB_PATH)
end
def test_alias_creation
original_image = image_path("apple", "right_anger_bubble")
alias_image = image_path("apple", "anger_right")
assert_equal File.size(original_image), File.size(alias_image)
end
def test_cell_index_patch
original_image = image_path("apple", "snowboarder")
alias_image = image_path("twitter", "snowboarder")
assert_equal File.size(original_image), File.size(alias_image)
end
def test_scales
original_image = image_path("apple", "blonde_woman")
assert File.exist?(original_image)
assert File.size?(original_image)
(2..6).each do |scale|
image = image_path("apple", "blonde_woman/#{scale}")
assert File.exist?(image)
assert File.size?(image)
end
end
def test_default_set
original_image = image_path("twitter", "snowboarder")
alias_image = image_path("apple", "snowboarder")
assert_equal File.size(original_image), File.size(alias_image)
original_image = image_path("twitter", "macau")
alias_image = image_path("win10", "macau")
assert_equal File.size(original_image), File.size(alias_image)
end
end