DEV: replaces huge generated emoji list by a simpler regex (#11053)

Note that this commit is also fixing various mistakes in emojis.

Some of them have been fixed manually in db.json/data.js/groups.json and will need to be fixed in emoji-db gem.

app/assets/javascripts/discourse/app/templates/components/emoji-group-sections.hbs

@@ -8,7 +8,7 @@
   <div class="section-group">
     {{replace-emoji ":grinning:" (hash lazy=true)}}
     {{replace-emoji ":smiley:" (hash lazy=true)}}
-    {{replace-emoji ":smile:" (hash lazy=true)}}
+    {{replace-emoji ":grinning_face_with_smiling_eyes:" (hash lazy=true)}}
     {{replace-emoji ":grin:" (hash lazy=true)}}
     {{replace-emoji ":laughing:" (hash lazy=true)}}
     {{replace-emoji ":sweat_smile:" (hash lazy=true)}}
@@ -78,6 +78,7 @@
     {{replace-emoji ":flushed:" (hash lazy=true)}}
     {{replace-emoji ":pleading_face:" (hash lazy=true)}}
     {{replace-emoji ":frowning:" (hash lazy=true)}}
+    {{replace-emoji ":frowning_face_with_open_mouth:" (hash lazy=true)}}
     {{replace-emoji ":anguished:" (hash lazy=true)}}
     {{replace-emoji ":fearful:" (hash lazy=true)}}
     {{replace-emoji ":cold_sweat:" (hash lazy=true)}}

app/assets/javascripts/pretty-text/addon/emoji/data.js

@@ -48,8 +48,7 @@ export const emojis = [
   "rage",
   "pensive",
   "confused",
-  "slightly_frowning_face",
+  "frowning",
-  "frowning_face",
   "persevere",
   "confounded",
   "tired_face",
@@ -60,7 +59,7 @@ export const emojis = [
   "fearful",
   "cold_sweat",
   "hushed",
-  "frowning",
+  "slightly_frowning_face",
   "frowning_face_with_open_mouth",
   "anguished",
   "cry",
@@ -1829,6 +1828,7 @@ export const tonableEmojis = [
   "palms_up_together",
 ];
 export const aliases = {
+  grinning_face_with_smiling_eyes: ["smile"],
   right_anger_bubble: ["anger_right"],
   ballot_box: ["ballot_box_with_ballot"],
   basketball_man: ["basketball_player", "person_with_ball"],
@@ -1920,7 +1920,6 @@ export const aliases = {
     "white_sun_rain_cloud",
     "white_sun_behind_cloud_with_rain",
   ],
-  grinning_face_with_smiling_eyes: ["smile"],
   partly_sunny: ["white_sun_small_cloud", "white_sun_with_small_cloud"],
   open_umbrella: ["umbrella2"],
   hammer_and_wrench: ["tools"],
@@ -1936,8 +1935,8 @@ export const aliases = {
   nerd_face: ["nerd"],
   hugs: ["hugging", "hugging_face"],
   roll_eyes: ["rolling_eyes", "face_with_rolling_eyes"],
-  slightly_frowning_face: ["slight_frown"],
+  slightly_frowning_face: ["slight_frown", "slightly_frowning"],
-  frowning_face: ["frowning2", "white_frowning_face"],
+  frowning: ["frowning2", "white_frowning_face", "frowning_face"],
   zipper_mouth_face: ["zipper_mouth"],
   face_with_head_bandage: ["head_bandage"],
   raised_hand_with_fingers_splayed: ["hand_splayed"],
@@ -2083,7 +2082,7 @@ export const replacements = {
   "😊": "blush",
   "🙂": "slightly_smiling_face",
   "🙃": "upside_down_face",
-  "☺": "slight_smile",
+  "☺": "relaxed",
   "😋": "yum",
   "😌": "relieved",
   "😍": "heart_eyes",
@@ -2115,8 +2114,7 @@ export const replacements = {
   "😡": "rage",
   "😔": "pensive",
   "😕": "confused",
-  "☹️": "slightly_frowning_face",
+  "🙁": "slightly_frowning_face",
-  "🙁": "frowning",
   "😣": "persevere",
   "😖": "confounded",
   "😫": "tired_face",
@@ -3708,7 +3706,7 @@ export const replacements = {
   "🖌": "paintbrush",
   "🔍": "mag",
   "🔎": "mag_right",
-  "❤": "heart",
+  "❤️": "heart",
   "💛": "yellow_heart",
   "💚": "green_heart",
   "💙": "blue_heart",
@@ -3944,7 +3942,6 @@ export const replacements = {
   "🀄": "mahjong",
   "♠": "spades",
   "♣": "clubs",
-  "♥": "heart",
   "♦": "diamonds",
   "🎴": "flower_playing_cards",
   "💭": "thought_balloon",
@@ -4539,19 +4536,18 @@ export const replacements = {
   "👨‍🦼": "man_in_motorized_wheelchair",
   "👩‍🦯": "woman_with_probing_cane",
   "👨‍🦯": "man_with_probing_cane",
-  "🧎‍♀": "woman_kneeling",
+  "🧎‍♀️": "woman_kneeling",
-  "🧎‍♂": "man_kneeling",
+  "🧎‍♂️": "man_kneeling",
-  "🧍‍♂": "man_standing",
+  "🧍‍♂️": "man_standing",
-  "🧍‍♀": "woman_standing",
+  "🧍‍♀️": "woman_standing",
-  "🧏‍♀": "deaf_woman",
+  "🧏‍♀️": "deaf_woman",
-  "🧏‍♂": "deaf_man",
+  "🧏‍♂️": "deaf_man",
   "🦻": "hear_with_hearing_aid",
   "🦿": "mechanical_leg",
   "🦾": "mechanical_arm",
   "🤍": "white_heart",
   "🤎": "brown_heart",
-  "🏳️‍⚧": "transgender_flag",
+  "🏳️‍⚧️": "transgender_flag",
-  "☹": "frowning",
+  "☹️": "frowning",
   "☻": "slight_smile",
-  "♡": "heart",
 };

lib/emoji/db.json

@@ -60,6 +60,10 @@
       "code": "263a",
       "name": "relaxed"
     },
+    {
+      "code": "263b",
+      "name": "slight_smile"
+    },
     {
       "code": "1f60b",
       "name": "yum"
@@ -185,12 +189,12 @@
       "name": "confused"
     },
     {
-      "code": "2639-fe0f",
+      "code": "2639",
-      "name": "slightly_frowning_face"
+      "name": "frowning"
     },
     {
       "code": "1f641",
-      "name": "frowning_face"
+      "name": "slightly_frowning_face"
     },
     {
       "code": "1f623",
@@ -232,10 +236,6 @@
       "code": "1f62f",
       "name": "hushed"
     },
-    {
-      "code": "1f641",
-      "name": "frowning"
-    },
     {
       "code": "1f626",
       "name": "frowning_face_with_open_mouth"
@@ -3664,6 +3664,10 @@
       "code": "2764",
       "name": "heart"
     },
+    {
+      "code": "2661",
+      "name": "white_heart_suit"
+    },
     {
       "code": "1f49b",
       "name": "yellow_heart"

lib/emoji/groups.json

@@ -263,10 +263,6 @@
         "name": "slightly_frowning_face",
         "diversity": false
       },
-      {
-        "name": "frowning_face",
-        "diversity": false
-      },
       {
         "name": "open_mouth",
         "diversity": false

lib/pretty_text/shims.js

@@ -1,7 +1,7 @@
 __PrettyText = require("pretty-text/pretty-text").default;
 __buildOptions = require("pretty-text/pretty-text").buildOptions;
 __performEmojiUnescape = require("pretty-text/emoji").performEmojiUnescape;
-__buildReplacementsList = require("pretty-text/emoji").buildReplacementsList;
+__emojiReplacementRegex = require("pretty-text/emoji").emojiReplacementRegex;
 __performEmojiEscape = require("pretty-text/emoji").performEmojiEscape;
 __resetTranslationTree = require("pretty-text/engines/discourse-markdown/emoji")
   .resetTranslationTree;
@@ -28,18 +28,34 @@ __utils = require("discourse/lib/utilities");
 __emojiUnicodeReplacer = null;
 
 __setUnicode = function (replacements) {
-  let unicodeRegexp = new RegExp(__buildReplacementsList(replacements), "g");
+  const regexp = new RegExp(__emojiReplacementRegex, "g");
 
   __emojiUnicodeReplacer = function (text) {
-    unicodeRegexp.lastIndex = 0;
+    regexp.lastIndex = 0;
+
     let m;
-    while ((m = unicodeRegexp.exec(text)) !== null) {
+    while ((m = regexp.exec(text)) !== null) {
-      let replacement = ":" + replacements[m[0]] + ":";
+      let match = m[0];
+
+      let replacement = replacements[match];
+
+      if (!replacement) {
+        // if we can't find replacement for an emoji match
+        // attempts to look for the same without trailing variation selector
+        match = match.replace(/\ufe0f$/g, "");
+        replacement = replacements[match];
+      }
+
+      if (!replacement) {
+        continue;
+      }
+
+      replacement = ":" + replacement + ":";
       const before = text.charAt(m.index - 1);
       if (!/\B/.test(before)) {
         replacement = "\u200b" + replacement;
       }
-      text = text.replace(m[0], replacement);
+      text = text.replace(match, replacement);
     }
 
     // fixes Safari VARIATION SELECTOR-16 issue with some emojis
@@ -53,7 +69,9 @@ __setUnicode = function (replacements) {
 __paths = {};
 
 function __getURLNoCDN(url) {
-  if (!url) return url;
+  if (!url) {
+    return url;
+  }
 
   // if it's a non relative URL, return it.
   if (url !== "/" && !/^\/[^\/]/.test(url)) {
@@ -63,7 +81,9 @@ function __getURLNoCDN(url) {
   if (url.indexOf(__paths.baseUri) !== -1) {
     return url;
   }
-  if (url[0] !== "/") url = "/" + url;
+  if (url[0] !== "/") {
+    url = "/" + url;
+  }
 
   return __paths.baseUri + url;
 }

spec/components/pretty_text_spec.rb

@@ -112,6 +112,7 @@ describe PrettyText do
 
         it "adds an only-emoji class when a line has only one emoji" do
           md = <<~MD
+            ☹️
             foo 😀
             foo 😀 bar
             :smile_cat:
@@ -131,7 +132,8 @@ describe PrettyText do
           MD
 
           html = <<~HTML
-            <p>foo <img src="/images/emoji/twitter/grinning.png?v=#{Emoji::EMOJI_VERSION}" title=":grinning:" class="emoji" alt=":grinning:"><br>
+            <p><img src="/images/emoji/twitter/frowning.png?v=#{Emoji::EMOJI_VERSION}" title=":frowning:" class="emoji only-emoji" alt=":frowning:"><br>
+            foo <img src="/images/emoji/twitter/grinning.png?v=#{Emoji::EMOJI_VERSION}" title=":grinning:" class="emoji" alt=":grinning:"><br>
             foo <img src="/images/emoji/twitter/grinning.png?v=#{Emoji::EMOJI_VERSION}" title=":grinning:" class="emoji" alt=":grinning:"> bar<br>
             <img src="/images/emoji/twitter/smile_cat.png?v=#{Emoji::EMOJI_VERSION}" title=":smile_cat:" class="emoji only-emoji" alt=":smile_cat:"><br>
             <img src="/images/emoji/twitter/smile_cat.png?v=#{Emoji::EMOJI_VERSION}" title=":smile_cat:" class="emoji only-emoji" alt=":smile_cat:"> <img src="/images/emoji/twitter/smile_cat.png?v=#{Emoji::EMOJI_VERSION}" title=":smile_cat:" class="emoji only-emoji" alt=":smile_cat:"><br>
@@ -1109,7 +1111,12 @@ describe PrettyText do
     end
 
     it "replaces some glyphs that are not in the emoji range" do
+      expect(PrettyText.cook("☹")).to match(/\:frowning\:/)
       expect(PrettyText.cook("☺")).to match(/\:relaxed\:/)
+      expect(PrettyText.cook("☻")).to match(/\:slight_smile\:/)
+      expect(PrettyText.cook("♡")).to match(/\:heart\:/)
+      expect(PrettyText.cook("❤")).to match(/\:heart\:/)
+      expect(PrettyText.cook("❤️")).to match(/\:heart\:/) # in emoji range but ensure it works along others
     end
 
     it "replaces digits" do