From 0eeedf307a8f2a8e1ccd5b24dafbf5a7fd20e51e Mon Sep 17 00:00:00 2001 From: Joffrey JAFFEUX Date: Thu, 22 Apr 2021 08:43:06 +0200 Subject: [PATCH] DEV: replaces huge generated emoji list by a simpler regex (#11053) Note that this commit is also fixing various mistakes in emojis. Some of them have been fixed manually in db.json/data.js/groups.json and will need to be fixed in emoji-db gem. --- .../components/emoji-group-sections.hbs | 3 +- .../javascripts/pretty-text/addon/emoji.js | 70 ++++++------------- .../pretty-text/addon/emoji/data.js | 36 +++++----- lib/emoji/db.json | 18 +++-- lib/emoji/groups.json | 4 -- lib/pretty_text/shims.js | 36 +++++++--- spec/components/pretty_text_spec.rb | 9 ++- 7 files changed, 86 insertions(+), 90 deletions(-) diff --git a/app/assets/javascripts/discourse/app/templates/components/emoji-group-sections.hbs b/app/assets/javascripts/discourse/app/templates/components/emoji-group-sections.hbs index 581c54d34f6..82ada7fe25f 100644 --- a/app/assets/javascripts/discourse/app/templates/components/emoji-group-sections.hbs +++ b/app/assets/javascripts/discourse/app/templates/components/emoji-group-sections.hbs @@ -8,7 +8,7 @@
{{replace-emoji ":grinning:" (hash lazy=true)}} {{replace-emoji ":smiley:" (hash lazy=true)}} - {{replace-emoji ":smile:" (hash lazy=true)}} + {{replace-emoji ":grinning_face_with_smiling_eyes:" (hash lazy=true)}} {{replace-emoji ":grin:" (hash lazy=true)}} {{replace-emoji ":laughing:" (hash lazy=true)}} {{replace-emoji ":sweat_smile:" (hash lazy=true)}} @@ -78,6 +78,7 @@ {{replace-emoji ":flushed:" (hash lazy=true)}} {{replace-emoji ":pleading_face:" (hash lazy=true)}} {{replace-emoji ":frowning:" (hash lazy=true)}} + {{replace-emoji ":frowning_face_with_open_mouth:" (hash lazy=true)}} {{replace-emoji ":anguished:" (hash lazy=true)}} {{replace-emoji ":fearful:" (hash lazy=true)}} {{replace-emoji ":cold_sweat:" (hash lazy=true)}} diff --git a/app/assets/javascripts/pretty-text/addon/emoji.js b/app/assets/javascripts/pretty-text/addon/emoji.js index 2237fe851d1..3ff60de061d 100644 --- a/app/assets/javascripts/pretty-text/addon/emoji.js +++ b/app/assets/javascripts/pretty-text/addon/emoji.js @@ -21,45 +21,12 @@ export function extendedEmojiList() { const emojiHash = {}; -export function buildReplacementsList(emojiReplacements) { - return Object.keys(emojiReplacements) - .sort() - .reverse() - .map((emoji) => { - return emoji - .split("") - .map((chr) => { - return "\\u" + chr.charCodeAt(0).toString(16).padStart(4, "0"); - }) - .join(""); - }) - .join("|"); -} +// https://github.com/mathiasbynens/emoji-regex/blob/main/text.js +export const emojiReplacementRegex = + "\uD83C\uDFF4\uDB40\uDC67\uDB40\uDC62(?:\uDB40\uDC77\uDB40\uDC6C\uDB40\uDC73|\uDB40\uDC73\uDB40\uDC63\uDB40\uDC74|\uDB40\uDC65\uDB40\uDC6E\uDB40\uDC67)\uDB40\uDC7F|(?:\uD83E\uDDD1\uD83C\uDFFF\u200D\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D)?\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFF\u200D\uD83E\uDD1D\u200D(?:\uD83D[\uDC68\uDC69]))(?:\uD83C[\uDFFB-\uDFFE])|(?:\uD83E\uDDD1\uD83C\uDFFE\u200D\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D)?\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFE\u200D\uD83E\uDD1D\u200D(?:\uD83D[\uDC68\uDC69]))(?:\uD83C[\uDFFB-\uDFFD\uDFFF])|(?:\uD83E\uDDD1\uD83C\uDFFD\u200D\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D)?\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFD\u200D\uD83E\uDD1D\u200D(?:\uD83D[\uDC68\uDC69]))(?:\uD83C[\uDFFB\uDFFC\uDFFE\uDFFF])|(?:\uD83E\uDDD1\uD83C\uDFFC\u200D\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D)?\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFC\u200D\uD83E\uDD1D\u200D(?:\uD83D[\uDC68\uDC69]))(?:\uD83C[\uDFFB\uDFFD-\uDFFF])|(?:\uD83E\uDDD1\uD83C\uDFFB\u200D\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D)?\uD83E\uDDD1|\uD83D\uDC69\uD83C\uDFFB\u200D\uD83E\uDD1D\u200D(?:\uD83D[\uDC68\uDC69]))(?:\uD83C[\uDFFC-\uDFFF])|\uD83D\uDC68(?:\uD83C\uDFFB(?:\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFF])|\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFF]))|\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFC-\uDFFF])|[\u2695\u2696\u2708]\uFE0F|\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD]))?|(?:\uD83C[\uDFFC-\uDFFF])\u200D\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFF])|\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFF]))|\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D)?\uD83D\uDC68|(?:\uD83D[\uDC68\uDC69])\u200D(?:\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67]))|\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67])|\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFE])|\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFE\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB-\uDFFD\uDFFF])|\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFC\uDFFE\uDFFF])|\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFC\u200D(?:\uD83E\uDD1D\u200D\uD83D\uDC68(?:\uD83C[\uDFFB\uDFFD-\uDFFF])|\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|(?:\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708]|\u200D[\u2695\u2696\u2708])\uFE0F|\u200D(?:(?:\uD83D[\uDC68\uDC69])\u200D(?:\uD83D[\uDC66\uDC67])|\uD83D[\uDC66\uDC67])|\uD83C\uDFFF|\uD83C\uDFFE|\uD83C\uDFFD|\uD83C\uDFFC)?|(?:\uD83D\uDC69(?:\uD83C\uDFFB\u200D\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D(?:\uD83D[\uDC68\uDC69])|\uD83D[\uDC68\uDC69])|(?:\uD83C[\uDFFC-\uDFFF])\u200D\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D(?:\uD83D[\uDC68\uDC69])|\uD83D[\uDC68\uDC69]))|\uD83E\uDDD1(?:\uD83C[\uDFFB-\uDFFF])\u200D\uD83E\uDD1D\u200D\uD83E\uDDD1)(?:\uD83C[\uDFFB-\uDFFF])|\uD83D\uDC69\u200D\uD83D\uDC69\u200D(?:\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67]))|\uD83D\uDC69(?:\u200D(?:\u2764\uFE0F\u200D(?:\uD83D\uDC8B\u200D(?:\uD83D[\uDC68\uDC69])|\uD83D[\uDC68\uDC69])|\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFE\u200D(?:\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFC\u200D(?:\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFB\u200D(?:\uD83C[\uDF3E\uDF73\uDF7C\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD]))|\uD83E\uDDD1(?:\u200D(?:\uD83E\uDD1D\u200D\uD83E\uDDD1|\uD83C[\uDF3E\uDF73\uDF7C\uDF84\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFF\u200D(?:\uD83C[\uDF3E\uDF73\uDF7C\uDF84\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFE\u200D(?:\uD83C[\uDF3E\uDF73\uDF7C\uDF84\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFD\u200D(?:\uD83C[\uDF3E\uDF73\uDF7C\uDF84\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFC\u200D(?:\uD83C[\uDF3E\uDF73\uDF7C\uDF84\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD])|\uD83C\uDFFB\u200D(?:\uD83C[\uDF3E\uDF73\uDF7C\uDF84\uDF93\uDFA4\uDFA8\uDFEB\uDFED]|\uD83D[\uDCBB\uDCBC\uDD27\uDD2C\uDE80\uDE92]|\uD83E[\uDDAF-\uDDB3\uDDBC\uDDBD]))|\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66|\uD83D\uDC69\u200D\uD83D\uDC69\u200D(?:\uD83D[\uDC66\uDC67])|\uD83D\uDC69\u200D\uD83D\uDC67\u200D(?:\uD83D[\uDC66\uDC67])|(?:\uD83D\uDC41\uFE0F\u200D\uD83D\uDDE8|\uD83E\uDDD1(?:\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708]|\uD83C\uDFFB\u200D[\u2695\u2696\u2708]|\u200D[\u2695\u2696\u2708])|\uD83D\uDC69(?:\uD83C\uDFFF\u200D[\u2695\u2696\u2708]|\uD83C\uDFFE\u200D[\u2695\u2696\u2708]|\uD83C\uDFFD\u200D[\u2695\u2696\u2708]|\uD83C\uDFFC\u200D[\u2695\u2696\u2708]|\uD83C\uDFFB\u200D[\u2695\u2696\u2708]|\u200D[\u2695\u2696\u2708])|\uD83D\uDE36\u200D\uD83C\uDF2B|\uD83C\uDFF3\uFE0F\u200D\u26A7|\uD83D\uDC3B\u200D\u2744|(?:(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC70\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD35\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD4\uDDD6-\uDDDD])(?:\uD83C[\uDFFB-\uDFFF])|\uD83D\uDC6F|\uD83E[\uDD3C\uDDDE\uDDDF])\u200D[\u2640\u2642]|(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)(?:\uFE0F|\uD83C[\uDFFB-\uDFFF])\u200D[\u2640\u2642]|\uD83C\uDFF4\u200D\u2620|(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC70\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD35\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD4\uDDD6-\uDDDD])\u200D[\u2640\u2642]|[\xA9\xAE\u203C\u2049\u2122\u2139\u2194-\u2199\u21A9\u21AA\u2328\u23CF\u23ED-\u23EF\u23F1\u23F2\u23F8-\u23FA\u24C2\u25AA\u25AB\u25B6\u25C0\u25FB\u25FC\u2600-\u2604\u260E\u2611\u2618\u2620\u2622\u2623\u2626\u262A\u262E\u262F\u2638-\u263A\u2640\u2642\u265F\u2660\u2663\u2665\u2666\u2668\u267B\u267E\u2692\u2694-\u2697\u2699\u269B\u269C\u26A0\u26A7\u26B0\u26B1\u26C8\u26CF\u26D1\u26D3\u26E9\u26F0\u26F1\u26F4\u26F7\u26F8\u2702\u2708\u2709\u270F\u2712\u2714\u2716\u271D\u2721\u2733\u2734\u2744\u2747\u2763\u27A1\u2934\u2935\u2B05-\u2B07\u3030\u303D\u3297\u3299]|\uD83C[\uDD70\uDD71\uDD7E\uDD7F\uDE02\uDE37\uDF21\uDF24-\uDF2C\uDF36\uDF7D\uDF96\uDF97\uDF99-\uDF9B\uDF9E\uDF9F\uDFCD\uDFCE\uDFD4-\uDFDF\uDFF5\uDFF7]|\uD83D[\uDC3F\uDCFD\uDD49\uDD4A\uDD6F\uDD70\uDD73\uDD76-\uDD79\uDD87\uDD8A-\uDD8D\uDDA5\uDDA8\uDDB1\uDDB2\uDDBC\uDDC2-\uDDC4\uDDD1-\uDDD3\uDDDC-\uDDDE\uDDE1\uDDE3\uDDE8\uDDEF\uDDF3\uDDFA\uDECB\uDECD-\uDECF\uDEE0-\uDEE5\uDEE9\uDEF0\uDEF3])\uFE0F|\uD83C\uDFF3\uFE0F\u200D\uD83C\uDF08|\uD83D\uDC69\u200D\uD83D\uDC67|\uD83D\uDC69\u200D\uD83D\uDC66|\uD83D\uDE35\u200D\uD83D\uDCAB|\uD83D\uDE2E\u200D\uD83D\uDCA8|\uD83D\uDC15\u200D\uD83E\uDDBA|\uD83E\uDDD1(?:\uD83C\uDFFF|\uD83C\uDFFE|\uD83C\uDFFD|\uD83C\uDFFC|\uD83C\uDFFB)?|\uD83D\uDC69(?:\uD83C\uDFFF|\uD83C\uDFFE|\uD83C\uDFFD|\uD83C\uDFFC|\uD83C\uDFFB)?|\uD83C\uDDFD\uD83C\uDDF0|\uD83C\uDDF6\uD83C\uDDE6|\uD83C\uDDF4\uD83C\uDDF2|\uD83D\uDC08\u200D\u2B1B|\u2764\uFE0F\u200D(?:\uD83D\uDD25|\uD83E\uDE79)|\uD83D\uDC41\uFE0F|\uD83C\uDFF3\uFE0F|\uD83C\uDDFF(?:\uD83C[\uDDE6\uDDF2\uDDFC])|\uD83C\uDDFE(?:\uD83C[\uDDEA\uDDF9])|\uD83C\uDDFC(?:\uD83C[\uDDEB\uDDF8])|\uD83C\uDDFB(?:\uD83C[\uDDE6\uDDE8\uDDEA\uDDEC\uDDEE\uDDF3\uDDFA])|\uD83C\uDDFA(?:\uD83C[\uDDE6\uDDEC\uDDF2\uDDF3\uDDF8\uDDFE\uDDFF])|\uD83C\uDDF9(?:\uD83C[\uDDE6\uDDE8\uDDE9\uDDEB-\uDDED\uDDEF-\uDDF4\uDDF7\uDDF9\uDDFB\uDDFC\uDDFF])|\uD83C\uDDF8(?:\uD83C[\uDDE6-\uDDEA\uDDEC-\uDDF4\uDDF7-\uDDF9\uDDFB\uDDFD-\uDDFF])|\uD83C\uDDF7(?:\uD83C[\uDDEA\uDDF4\uDDF8\uDDFA\uDDFC])|\uD83C\uDDF5(?:\uD83C[\uDDE6\uDDEA-\uDDED\uDDF0-\uDDF3\uDDF7-\uDDF9\uDDFC\uDDFE])|\uD83C\uDDF3(?:\uD83C[\uDDE6\uDDE8\uDDEA-\uDDEC\uDDEE\uDDF1\uDDF4\uDDF5\uDDF7\uDDFA\uDDFF])|\uD83C\uDDF2(?:\uD83C[\uDDE6\uDDE8-\uDDED\uDDF0-\uDDFF])|\uD83C\uDDF1(?:\uD83C[\uDDE6-\uDDE8\uDDEE\uDDF0\uDDF7-\uDDFB\uDDFE])|\uD83C\uDDF0(?:\uD83C[\uDDEA\uDDEC-\uDDEE\uDDF2\uDDF3\uDDF5\uDDF7\uDDFC\uDDFE\uDDFF])|\uD83C\uDDEF(?:\uD83C[\uDDEA\uDDF2\uDDF4\uDDF5])|\uD83C\uDDEE(?:\uD83C[\uDDE8-\uDDEA\uDDF1-\uDDF4\uDDF6-\uDDF9])|\uD83C\uDDED(?:\uD83C[\uDDF0\uDDF2\uDDF3\uDDF7\uDDF9\uDDFA])|\uD83C\uDDEC(?:\uD83C[\uDDE6\uDDE7\uDDE9-\uDDEE\uDDF1-\uDDF3\uDDF5-\uDDFA\uDDFC\uDDFE])|\uD83C\uDDEB(?:\uD83C[\uDDEE-\uDDF0\uDDF2\uDDF4\uDDF7])|\uD83C\uDDEA(?:\uD83C[\uDDE6\uDDE8\uDDEA\uDDEC\uDDED\uDDF7-\uDDFA])|\uD83C\uDDE9(?:\uD83C[\uDDEA\uDDEC\uDDEF\uDDF0\uDDF2\uDDF4\uDDFF])|\uD83C\uDDE8(?:\uD83C[\uDDE6\uDDE8\uDDE9\uDDEB-\uDDEE\uDDF0-\uDDF5\uDDF7\uDDFA-\uDDFF])|\uD83C\uDDE7(?:\uD83C[\uDDE6\uDDE7\uDDE9-\uDDEF\uDDF1-\uDDF4\uDDF6-\uDDF9\uDDFB\uDDFC\uDDFE\uDDFF])|\uD83C\uDDE6(?:\uD83C[\uDDE8-\uDDEC\uDDEE\uDDF1\uDDF2\uDDF4\uDDF6-\uDDFA\uDDFC\uDDFD\uDDFF])|[#*0-9]\uFE0F\u20E3|\u2764\uFE0F|(?:\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC70\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD35\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD4\uDDD6-\uDDDD])(?:\uD83C[\uDFFB-\uDFFF])|(?:\u26F9|\uD83C[\uDFCB\uDFCC]|\uD83D\uDD75)(?:\uFE0F|\uD83C[\uDFFB-\uDFFF])|\uD83C\uDFF4|(?:[\u270A\u270B]|\uD83C[\uDF85\uDFC2\uDFC7]|\uD83D[\uDC42\uDC43\uDC46-\uDC50\uDC66\uDC67\uDC6B-\uDC6D\uDC72\uDC74-\uDC76\uDC78\uDC7C\uDC83\uDC85\uDC8F\uDC91\uDCAA\uDD7A\uDD95\uDD96\uDE4C\uDE4F\uDEC0\uDECC]|\uD83E[\uDD0C\uDD0F\uDD18-\uDD1C\uDD1E\uDD1F\uDD30-\uDD34\uDD36\uDD77\uDDB5\uDDB6\uDDBB\uDDD2\uDDD3\uDDD5])(?:\uD83C[\uDFFB-\uDFFF])|(?:[\u261D\u270C\u270D]|\uD83D[\uDD74\uDD90])(?:\uFE0F|\uD83C[\uDFFB-\uDFFF])|[\u270A\u270B]|\uD83C[\uDF85\uDFC2\uDFC7]|\uD83D[\uDC08\uDC15\uDC3B\uDC42\uDC43\uDC46-\uDC50\uDC66\uDC67\uDC6B-\uDC6D\uDC72\uDC74-\uDC76\uDC78\uDC7C\uDC83\uDC85\uDC8F\uDC91\uDCAA\uDD7A\uDD95\uDD96\uDE2E\uDE35\uDE36\uDE4C\uDE4F\uDEC0\uDECC]|\uD83E[\uDD0C\uDD0F\uDD18-\uDD1C\uDD1E\uDD1F\uDD30-\uDD34\uDD36\uDD77\uDDB5\uDDB6\uDDBB\uDDD2\uDDD3\uDDD5]|\uD83C[\uDFC3\uDFC4\uDFCA]|\uD83D[\uDC6E\uDC70\uDC71\uDC73\uDC77\uDC81\uDC82\uDC86\uDC87\uDE45-\uDE47\uDE4B\uDE4D\uDE4E\uDEA3\uDEB4-\uDEB6]|\uD83E[\uDD26\uDD35\uDD37-\uDD39\uDD3D\uDD3E\uDDB8\uDDB9\uDDCD-\uDDCF\uDDD4\uDDD6-\uDDDD]|\uD83D\uDC6F|\uD83E[\uDD3C\uDDDE\uDDDF]|[\u231A\u231B\u23E9-\u23EC\u23F0\u23F3\u25FD\u25FE\u2614\u2615\u2648-\u2653\u267F\u2693\u26A1\u26AA\u26AB\u26BD\u26BE\u26C4\u26C5\u26CE\u26D4\u26EA\u26F2\u26F3\u26F5\u26FA\u26FD\u2705\u2728\u274C\u274E\u2753-\u2755\u2757\u2795-\u2797\u27B0\u27BF\u2B1B\u2B1C\u2B50\u2B55]|\uD83C[\uDC04\uDCCF\uDD8E\uDD91-\uDD9A\uDE01\uDE1A\uDE2F\uDE32-\uDE36\uDE38-\uDE3A\uDE50\uDE51\uDF00-\uDF20\uDF2D-\uDF35\uDF37-\uDF7C\uDF7E-\uDF84\uDF86-\uDF93\uDFA0-\uDFC1\uDFC5\uDFC6\uDFC8\uDFC9\uDFCF-\uDFD3\uDFE0-\uDFF0\uDFF8-\uDFFF]|\uD83D[\uDC00-\uDC07\uDC09-\uDC14\uDC16-\uDC3A\uDC3C-\uDC3E\uDC40\uDC44\uDC45\uDC51-\uDC65\uDC6A\uDC79-\uDC7B\uDC7D-\uDC80\uDC84\uDC88-\uDC8E\uDC90\uDC92-\uDCA9\uDCAB-\uDCFC\uDCFF-\uDD3D\uDD4B-\uDD4E\uDD50-\uDD67\uDDA4\uDDFB-\uDE2D\uDE2F-\uDE34\uDE37-\uDE44\uDE48-\uDE4A\uDE80-\uDEA2\uDEA4-\uDEB3\uDEB7-\uDEBF\uDEC1-\uDEC5\uDED0-\uDED2\uDED5-\uDED7\uDEEB\uDEEC\uDEF4-\uDEFC\uDFE0-\uDFEB]|\uD83E[\uDD0D\uDD0E\uDD10-\uDD17\uDD1D\uDD20-\uDD25\uDD27-\uDD2F\uDD3A\uDD3F-\uDD45\uDD47-\uDD76\uDD78\uDD7A-\uDDB4\uDDB7\uDDBA\uDDBC-\uDDCB\uDDD0\uDDE0-\uDDFF\uDE70-\uDE74\uDE78-\uDE7A\uDE80-\uDE86\uDE90-\uDEA8\uDEB0-\uDEB6\uDEC0-\uDEC2\uDED0-\uDED6]|(?:[#*0-9\xA9\xAE\u203C\u2049\u2122\u2139\u2194-\u2199\u21A9\u21AA\u231A\u231B\u2328\u23CF\u23E9-\u23F3\u23F8-\u23FA\u24C2\u25AA\u25AB\u25B6\u25C0\u25FB-\u25FE\u2600-\u2604\u260E\u2611\u2614\u2615\u2618\u261D\u2620\u2622\u2623\u2626\u262A\u262E\u262F\u2638-\u263A\u2640\u2642\u2648-\u2653\u265F\u2660\u2663\u2665\u2666\u2668\u267B\u267E\u267F\u2692-\u2697\u2699\u269B\u269C\u26A0\u26A1\u26A7\u26AA\u26AB\u26B0\u26B1\u26BD\u26BE\u26C4\u26C5\u26C8\u26CE\u26CF\u26D1\u26D3\u26D4\u26E9\u26EA\u26F0-\u26F5\u26F7-\u26FA\u26FD\u2702\u2705\u2708-\u270D\u270F\u2712\u2714\u2716\u271D\u2721\u2728\u2733\u2734\u2744\u2747\u274C\u274E\u2753-\u2755\u2757\u2763\u2764\u2795-\u2797\u27A1\u27B0\u27BF\u2934\u2935\u2B05-\u2B07\u2B1B\u2B1C\u2B50\u2B55\u3030\u303D\u3297\u3299]|\uD83C[\uDC04\uDCCF\uDD70\uDD71\uDD7E\uDD7F\uDD8E\uDD91-\uDD9A\uDDE6-\uDDFF\uDE01\uDE02\uDE1A\uDE2F\uDE32-\uDE3A\uDE50\uDE51\uDF00-\uDF21\uDF24-\uDF93\uDF96\uDF97\uDF99-\uDF9B\uDF9E-\uDFF0\uDFF3-\uDFF5\uDFF7-\uDFFF]|\uD83D[\uDC00-\uDCFD\uDCFF-\uDD3D\uDD49-\uDD4E\uDD50-\uDD67\uDD6F\uDD70\uDD73-\uDD7A\uDD87\uDD8A-\uDD8D\uDD90\uDD95\uDD96\uDDA4\uDDA5\uDDA8\uDDB1\uDDB2\uDDBC\uDDC2-\uDDC4\uDDD1-\uDDD3\uDDDC-\uDDDE\uDDE1\uDDE3\uDDE8\uDDEF\uDDF3\uDDFA-\uDE4F\uDE80-\uDEC5\uDECB-\uDED2\uDED5-\uDED7\uDEE0-\uDEE5\uDEE9\uDEEB\uDEEC\uDEF0\uDEF3-\uDEFC\uDFE0-\uDFEB]|\uD83E[\uDD0C-\uDD3A\uDD3C-\uDD45\uDD47-\uDD78\uDD7A-\uDDCB\uDDCD-\uDDFF\uDE70-\uDE74\uDE78-\uDE7A\uDE80-\uDE86\uDE90-\uDEA8\uDEB0-\uDEB6\uDEC0-\uDEC2\uDED0-\uDED6]|â˜ģ|♡)\uFE0F?"; -let replacementListCache; -const unicodeRegexpCache = {}; - -function replacementList() { - if (replacementListCache === undefined) { - replacementListCache = buildReplacementsList(replacements); - } - - return replacementListCache; -} - -function unicodeRegexp(inlineEmoji) { - if (unicodeRegexpCache[inlineEmoji] === undefined) { - const emojiExpression = inlineEmoji - ? "|:[^\\s:]+(?::t\\d)?:?" - : "|\\B:[^\\s:]+(?::t\\d)?:?\\B"; - - unicodeRegexpCache[inlineEmoji] = new RegExp( - replacementList() + emojiExpression, - "g" - ); - } - - return unicodeRegexpCache[inlineEmoji]; +function textEmojiRegex(inlineEmoji) { + return inlineEmoji ? /:[^\s:]+(?::t\d)?:?/g : /\B:[^\s:]+(?::t\d)?:?\B/g; } // add all default emojis @@ -83,7 +50,7 @@ function isReplacableInlineEmoji(string, index, inlineEmoji) { return ( beforeEmoji.length === 0 || /(?:\s|[>.,\/#!$%^&*;:{}=\-_`~()])$/.test(beforeEmoji) || - new RegExp(`(?:${replacementList()})$`).test(beforeEmoji) + new RegExp(`(?:${emojiReplacementRegex})$`, "g").test(beforeEmoji) ); } @@ -92,18 +59,17 @@ export function performEmojiUnescape(string, opts) { return; } - const inlineEmoji = opts.inlineEmoji; - const regexp = unicodeRegexp(inlineEmoji); const allTranslations = Object.assign( {}, translations, opts.customEmojiTranslation || {} ); - return string.replace(regexp, (m, index) => { + const replacementFunction = (m, index) => { const isEmoticon = opts.enableEmojiShortcuts && !!allTranslations[m]; const isUnicodeEmoticon = !!replacements[m]; let emojiVal; + if (isEmoticon) { emojiVal = allTranslations[m]; } else if (isUnicodeEmoticon) { @@ -123,7 +89,7 @@ export function performEmojiUnescape(string, opts) { const isReplacable = (isEmoticon || hasEndingColon || isUnicodeEmoticon) && - isReplacableInlineEmoji(string, index, inlineEmoji); + isReplacableInlineEmoji(string, index, opts.inlineEmoji); return url && isReplacable ? `` : m; - }); + }; + + return string + .replace(new RegExp(emojiReplacementRegex, "g"), replacementFunction) + .replace(textEmojiRegex(opts.inlineEmoji), replacementFunction); } export function performEmojiEscape(string, opts) { - const inlineEmoji = opts.inlineEmoji; - const regexp = unicodeRegexp(inlineEmoji); const allTranslations = Object.assign( {}, translations, opts.customEmojiTranslation || {} ); - return string.replace(regexp, (m, index) => { - if (isReplacableInlineEmoji(string, index, inlineEmoji)) { + const replacementFunction = (m, index) => { + if (isReplacableInlineEmoji(string, index, opts.inlineEmoji)) { if (!!allTranslations[m]) { return opts.emojiShortcuts ? `:${allTranslations[m]}:` : m; } else if (!!replacements[m]) { @@ -154,7 +122,11 @@ export function performEmojiEscape(string, opts) { } return m; - }); + }; + + return string + .replace(new RegExp(emojiReplacementRegex, "g"), replacementFunction) + .replace(textEmojiRegex(opts.inlineEmoji), replacementFunction); } export function isCustomEmoji(code, opts) { diff --git a/app/assets/javascripts/pretty-text/addon/emoji/data.js b/app/assets/javascripts/pretty-text/addon/emoji/data.js index f23158ec227..aaf4f4da147 100644 --- a/app/assets/javascripts/pretty-text/addon/emoji/data.js +++ b/app/assets/javascripts/pretty-text/addon/emoji/data.js @@ -48,8 +48,7 @@ export const emojis = [ "rage", "pensive", "confused", - "slightly_frowning_face", - "frowning_face", + "frowning", "persevere", "confounded", "tired_face", @@ -60,7 +59,7 @@ export const emojis = [ "fearful", "cold_sweat", "hushed", - "frowning", + "slightly_frowning_face", "frowning_face_with_open_mouth", "anguished", "cry", @@ -1829,6 +1828,7 @@ export const tonableEmojis = [ "palms_up_together", ]; export const aliases = { + grinning_face_with_smiling_eyes: ["smile"], right_anger_bubble: ["anger_right"], ballot_box: ["ballot_box_with_ballot"], basketball_man: ["basketball_player", "person_with_ball"], @@ -1920,7 +1920,6 @@ export const aliases = { "white_sun_rain_cloud", "white_sun_behind_cloud_with_rain", ], - grinning_face_with_smiling_eyes: ["smile"], partly_sunny: ["white_sun_small_cloud", "white_sun_with_small_cloud"], open_umbrella: ["umbrella2"], hammer_and_wrench: ["tools"], @@ -1936,8 +1935,8 @@ export const aliases = { nerd_face: ["nerd"], hugs: ["hugging", "hugging_face"], roll_eyes: ["rolling_eyes", "face_with_rolling_eyes"], - slightly_frowning_face: ["slight_frown"], - frowning_face: ["frowning2", "white_frowning_face"], + slightly_frowning_face: ["slight_frown", "slightly_frowning"], + frowning: ["frowning2", "white_frowning_face", "frowning_face"], zipper_mouth_face: ["zipper_mouth"], face_with_head_bandage: ["head_bandage"], raised_hand_with_fingers_splayed: ["hand_splayed"], @@ -2083,7 +2082,7 @@ export const replacements = { "😊": "blush", "🙂": "slightly_smiling_face", "🙃": "upside_down_face", - "â˜ē": "slight_smile", + "â˜ē": "relaxed", "😋": "yum", "😌": "relieved", "😍": "heart_eyes", @@ -2115,8 +2114,7 @@ export const replacements = { "😡": "rage", "😔": "pensive", "😕": "confused", - "☚ī¸": "slightly_frowning_face", - "🙁": "frowning", + "🙁": "slightly_frowning_face", "đŸ˜Ŗ": "persevere", "😖": "confounded", "đŸ˜Ģ": "tired_face", @@ -3708,7 +3706,7 @@ export const replacements = { "🖌": "paintbrush", "🔍": "mag", "🔎": "mag_right", - "❤": "heart", + "❤ī¸": "heart", "💛": "yellow_heart", "💚": "green_heart", "💙": "blue_heart", @@ -3944,7 +3942,6 @@ export const replacements = { "🀄": "mahjong", "♠": "spades", "â™Ŗ": "clubs", - "â™Ĩ": "heart", "â™Ļ": "diamonds", "🎴": "flower_playing_cards", "💭": "thought_balloon", @@ -4539,19 +4536,18 @@ export const replacements = { "👨‍đŸĻŧ": "man_in_motorized_wheelchair", "👩‍đŸĻ¯": "woman_with_probing_cane", "👨‍đŸĻ¯": "man_with_probing_cane", - "🧎‍♀": "woman_kneeling", - "🧎‍♂": "man_kneeling", - "🧍‍♂": "man_standing", - "🧍‍♀": "woman_standing", - "🧏‍♀": "deaf_woman", - "🧏‍♂": "deaf_man", + "🧎‍♀ī¸": "woman_kneeling", + "🧎‍♂ī¸": "man_kneeling", + "🧍‍♂ī¸": "man_standing", + "🧍‍♀ī¸": "woman_standing", + "🧏‍♀ī¸": "deaf_woman", + "🧏‍♂ī¸": "deaf_man", "đŸĻģ": "hear_with_hearing_aid", "đŸĻŋ": "mechanical_leg", "đŸĻž": "mechanical_arm", "🤍": "white_heart", "🤎": "brown_heart", - "đŸŗī¸â€âš§": "transgender_flag", - "☚": "frowning", + "đŸŗī¸â€âš§ī¸": "transgender_flag", + "☚ī¸": "frowning", "â˜ģ": "slight_smile", - "♡": "heart", }; diff --git a/lib/emoji/db.json b/lib/emoji/db.json index 5f7c778fd2d..3e3ee9235d1 100644 --- a/lib/emoji/db.json +++ b/lib/emoji/db.json @@ -60,6 +60,10 @@ "code": "263a", "name": "relaxed" }, + { + "code": "263b", + "name": "slight_smile" + }, { "code": "1f60b", "name": "yum" @@ -185,12 +189,12 @@ "name": "confused" }, { - "code": "2639-fe0f", - "name": "slightly_frowning_face" + "code": "2639", + "name": "frowning" }, { "code": "1f641", - "name": "frowning_face" + "name": "slightly_frowning_face" }, { "code": "1f623", @@ -232,10 +236,6 @@ "code": "1f62f", "name": "hushed" }, - { - "code": "1f641", - "name": "frowning" - }, { "code": "1f626", "name": "frowning_face_with_open_mouth" @@ -3664,6 +3664,10 @@ "code": "2764", "name": "heart" }, + { + "code": "2661", + "name": "white_heart_suit" + }, { "code": "1f49b", "name": "yellow_heart" diff --git a/lib/emoji/groups.json b/lib/emoji/groups.json index 0c18098ddaa..c8604627f44 100644 --- a/lib/emoji/groups.json +++ b/lib/emoji/groups.json @@ -263,10 +263,6 @@ "name": "slightly_frowning_face", "diversity": false }, - { - "name": "frowning_face", - "diversity": false - }, { "name": "open_mouth", "diversity": false diff --git a/lib/pretty_text/shims.js b/lib/pretty_text/shims.js index 4c9de85dfc8..bc281c23bc2 100644 --- a/lib/pretty_text/shims.js +++ b/lib/pretty_text/shims.js @@ -1,7 +1,7 @@ __PrettyText = require("pretty-text/pretty-text").default; __buildOptions = require("pretty-text/pretty-text").buildOptions; __performEmojiUnescape = require("pretty-text/emoji").performEmojiUnescape; -__buildReplacementsList = require("pretty-text/emoji").buildReplacementsList; +__emojiReplacementRegex = require("pretty-text/emoji").emojiReplacementRegex; __performEmojiEscape = require("pretty-text/emoji").performEmojiEscape; __resetTranslationTree = require("pretty-text/engines/discourse-markdown/emoji") .resetTranslationTree; @@ -28,18 +28,34 @@ __utils = require("discourse/lib/utilities"); __emojiUnicodeReplacer = null; __setUnicode = function (replacements) { - let unicodeRegexp = new RegExp(__buildReplacementsList(replacements), "g"); + const regexp = new RegExp(__emojiReplacementRegex, "g"); __emojiUnicodeReplacer = function (text) { - unicodeRegexp.lastIndex = 0; + regexp.lastIndex = 0; + let m; - while ((m = unicodeRegexp.exec(text)) !== null) { - let replacement = ":" + replacements[m[0]] + ":"; + while ((m = regexp.exec(text)) !== null) { + let match = m[0]; + + let replacement = replacements[match]; + + if (!replacement) { + // if we can't find replacement for an emoji match + // attempts to look for the same without trailing variation selector + match = match.replace(/\ufe0f$/g, ""); + replacement = replacements[match]; + } + + if (!replacement) { + continue; + } + + replacement = ":" + replacement + ":"; const before = text.charAt(m.index - 1); if (!/\B/.test(before)) { replacement = "\u200b" + replacement; } - text = text.replace(m[0], replacement); + text = text.replace(match, replacement); } // fixes Safari VARIATION SELECTOR-16 issue with some emojis @@ -53,7 +69,9 @@ __setUnicode = function (replacements) { __paths = {}; function __getURLNoCDN(url) { - if (!url) return url; + if (!url) { + return url; + } // if it's a non relative URL, return it. if (url !== "/" && !/^\/[^\/]/.test(url)) { @@ -63,7 +81,9 @@ function __getURLNoCDN(url) { if (url.indexOf(__paths.baseUri) !== -1) { return url; } - if (url[0] !== "/") url = "/" + url; + if (url[0] !== "/") { + url = "/" + url; + } return __paths.baseUri + url; } diff --git a/spec/components/pretty_text_spec.rb b/spec/components/pretty_text_spec.rb index 3baaa24ba73..93441ba4dd4 100644 --- a/spec/components/pretty_text_spec.rb +++ b/spec/components/pretty_text_spec.rb @@ -112,6 +112,7 @@ describe PrettyText do it "adds an only-emoji class when a line has only one emoji" do md = <<~MD + ☚ī¸ foo 😀 foo 😀 bar :smile_cat: @@ -131,7 +132,8 @@ describe PrettyText do MD html = <<~HTML -

foo :grinning:
+

:frowning:
+ foo :grinning:
foo :grinning: bar
:smile_cat:
:smile_cat: :smile_cat:
@@ -1109,7 +1111,12 @@ describe PrettyText do end it "replaces some glyphs that are not in the emoji range" do + expect(PrettyText.cook("☚")).to match(/\:frowning\:/) expect(PrettyText.cook("â˜ē")).to match(/\:relaxed\:/) + expect(PrettyText.cook("â˜ģ")).to match(/\:slight_smile\:/) + expect(PrettyText.cook("♡")).to match(/\:heart\:/) + expect(PrettyText.cook("❤")).to match(/\:heart\:/) + expect(PrettyText.cook("❤ī¸")).to match(/\:heart\:/) # in emoji range but ensure it works along others end it "replaces digits" do