FIX: Better emoji escaping for topic title

This commit also puts emojiVersion in its own erb file.

app/assets/javascripts/pretty-text-bundle.js

@@ -2,6 +2,7 @@
 //= require ./pretty-text/guid
 //= require ./pretty-text/censored-words
 //= require ./pretty-text/emoji/data
+//= require ./pretty-text/emoji/version
 //= require ./pretty-text/emoji
 //= require ./pretty-text/engines/discourse-markdown-it
 //= require xss.min

app/assets/javascripts/pretty-text/emoji.js.es6

@@ -0,0 +1,175 @@
+import {
+  emojis,
+  aliases,
+  searchAliases,
+  translations,
+  tonableEmojis,
+  replacements
+} from "pretty-text/emoji/data";
+import { IMAGE_VERSION } from "pretty-text/emoji/version";
+
+const extendedEmoji = {};
+
+export function registerEmoji(code, url) {
+  code = code.toLowerCase();
+  extendedEmoji[code] = url;
+}
+
+export function extendedEmojiList() {
+  return extendedEmoji;
+}
+
+const emojiHash = {};
+
+// add all default emojis
+emojis.forEach(code => (emojiHash[code] = true));
+
+// and their aliases
+const aliasHash = {};
+Object.keys(aliases).forEach(name => {
+  aliases[name].forEach(alias => (aliasHash[alias] = name));
+});
+
+export function performEmojiUnescape(string, opts) {
+  if (!string) {
+    return;
+  }
+
+  return string.replace(/[\u1000-\uFFFF]+|\B:[^\s:]+(?::t\d)?:?\B/g, m => {
+    const isEmoticon = !!translations[m];
+    const isUnicodeEmoticon = !!replacements[m];
+    const emojiVal = isEmoticon
+      ? translations[m]
+      : isUnicodeEmoticon
+      ? replacements[m]
+      : m.slice(1, m.length - 1);
+    const hasEndingColon = m.lastIndexOf(":") === m.length - 1;
+    const url = buildEmojiUrl(emojiVal, opts);
+    const classes = isCustomEmoji(emojiVal, opts)
+      ? "emoji emoji-custom"
+      : "emoji";
+
+    return url && (isEmoticon || hasEndingColon || isUnicodeEmoticon)
+      ? `<img src='${url}' ${
+          opts.skipTitle ? "" : `title='${emojiVal}'`
+        } alt='${emojiVal}' class='${classes}'>`
+      : m;
+  });
+
+  return string;
+}
+
+export function performEmojiEscape(string) {
+  if (!string) {
+    return;
+  }
+
+  return string.replace(/[\u1000-\uFFFF]+|\B:[^\s:]+(?::t\d)?:?\B/g, m => {
+    if (!!translations[m]) {
+      return ":" + translations[m] + ":";
+    } else if (!!replacements[m]) {
+      return ":" + replacements[m] + ":";
+    } else {
+      return m;
+    }
+  });
+
+  return string;
+}
+
+export function isCustomEmoji(code, opts) {
+  code = code.toLowerCase();
+  if (extendedEmoji.hasOwnProperty(code)) return true;
+  if (opts && opts.customEmoji && opts.customEmoji.hasOwnProperty(code))
+    return true;
+  return false;
+}
+
+export function buildEmojiUrl(code, opts) {
+  let url;
+  code = String(code).toLowerCase();
+  if (extendedEmoji.hasOwnProperty(code)) {
+    url = extendedEmoji[code];
+  }
+
+  if (opts && opts.customEmoji && opts.customEmoji[code]) {
+    url = opts.customEmoji[code];
+  }
+
+  const noToneMatch = code.match(/([^:]+):?/);
+  if (
+    noToneMatch &&
+    !url &&
+    (emojiHash.hasOwnProperty(noToneMatch[1]) ||
+      aliasHash.hasOwnProperty(noToneMatch[1]))
+  ) {
+    url = opts.getURL(
+      `/images/emoji/${opts.emojiSet}/${code.replace(/:t/, "/")}.png`
+    );
+  }
+
+  if (url) {
+    url = url + "?v=" + IMAGE_VERSION;
+  }
+
+  return url;
+}
+
+export function emojiExists(code) {
+  code = code.toLowerCase();
+  return !!(
+    extendedEmoji.hasOwnProperty(code) ||
+    emojiHash.hasOwnProperty(code) ||
+    aliasHash.hasOwnProperty(code)
+  );
+}
+
+let toSearch;
+export function emojiSearch(term, options) {
+  const maxResults = (options && options["maxResults"]) || -1;
+  if (maxResults === 0) {
+    return [];
+  }
+
+  toSearch =
+    toSearch ||
+    _.union(_.keys(emojiHash), _.keys(extendedEmoji), _.keys(aliasHash)).sort();
+
+  const results = [];
+
+  function addResult(t) {
+    const val = aliasHash[t] || t;
+    if (results.indexOf(val) === -1) {
+      results.push(val);
+    }
+  }
+
+  // if term matches from beginning
+  for (let i = 0; i < toSearch.length; i++) {
+    const item = toSearch[i];
+    if (item.indexOf(term) === 0) addResult(item);
+  }
+
+  if (searchAliases[term]) {
+    results.push.apply(results, searchAliases[term]);
+  }
+
+  for (let i = 0; i < toSearch.length; i++) {
+    const item = toSearch[i];
+    if (item.indexOf(term) > 0) addResult(item);
+  }
+
+  if (maxResults === -1) {
+    return results;
+  } else {
+    return results.slice(0, maxResults);
+  }
+}
+
+export function isSkinTonableEmoji(term) {
+  const match = _.compact(term.split(":"))[0];
+  if (match) {
+    return tonableEmojis.indexOf(match) !== -1;
+  }
+  return false;
+}

app/assets/javascripts/pretty-text/emoji.js.es6.erb

@@ -1,127 +0,0 @@
-import { emojis, aliases, searchAliases, translations, tonableEmojis } from 'pretty-text/emoji/data';
-
-// bump up this number to expire all emojis
-export const IMAGE_VERSION = "<%= Emoji::EMOJI_VERSION %>";
-
-const extendedEmoji = {};
-
-export function registerEmoji(code, url) {
-  code = code.toLowerCase();
-  extendedEmoji[code] = url;
-}
-
-export function extendedEmojiList() {
-  return extendedEmoji;
-}
-
-const emojiHash = {};
-
-// add all default emojis
-emojis.forEach(code => emojiHash[code] = true);
-
-// and their aliases
-const aliasHash = {};
-Object.keys(aliases).forEach(name => {
-  aliases[name].forEach(alias => aliasHash[alias] = name);
-});
-
-export function performEmojiUnescape(string, opts) {
-  if (!string) { return; }
-
-  // this can be further improved by supporting matches of emoticons that don't begin with a colon
-  if (string.indexOf(":") >= 0) {
-    return string.replace(/\B:[^\s:]+(?::t\d)?:?\B/g, m => {
-      const isEmoticon = !!translations[m];
-      const emojiVal = isEmoticon ? translations[m] : m.slice(1, m.length - 1);
-      const hasEndingColon = m.lastIndexOf(":") === m.length - 1;
-      const url = buildEmojiUrl(emojiVal, opts);
-      const classes = isCustomEmoji(emojiVal, opts) ? "emoji emoji-custom" : "emoji";
-
-      return url && (isEmoticon || hasEndingColon) ?
-             `<img src='${url}' ${opts.skipTitle ? '' : `title='${emojiVal}'`} alt='${emojiVal}' class='${classes}'>` : m;
-    });
-  }
-
-  return string;
-}
-
-export function isCustomEmoji(code, opts) {
-  code = code.toLowerCase();
-  if (extendedEmoji.hasOwnProperty(code)) return true;
-  if (opts && opts.customEmoji && opts.customEmoji.hasOwnProperty(code)) return true;
-  return false;
-}
-
-export function buildEmojiUrl(code, opts) {
-  let url;
-  code = String(code).toLowerCase();
-  if (extendedEmoji.hasOwnProperty(code)) {
-    url = extendedEmoji[code];
-  }
-
-  if (opts && opts.customEmoji && opts.customEmoji[code]) {
-    url = opts.customEmoji[code];
-  }
-
-  const noToneMatch = code.match(/([^:]+):?/);
-  if (noToneMatch && !url && (emojiHash.hasOwnProperty(noToneMatch[1]) || aliasHash.hasOwnProperty(noToneMatch[1]))) {
-    url = opts.getURL(`/images/emoji/${opts.emojiSet}/${code.replace(/:t/, '/')}.png`);
-  }
-
-  if (url) {
-    url = url + "?v=" + IMAGE_VERSION;
-  }
-
-  return url;
-}
-
-export function emojiExists(code) {
-  code = code.toLowerCase();
-  return !!(extendedEmoji.hasOwnProperty(code) || emojiHash.hasOwnProperty(code) || aliasHash.hasOwnProperty(code));
-};
-
-let toSearch;
-export function emojiSearch(term, options) {
-  const maxResults = (options && options["maxResults"]) || -1;
-  if (maxResults === 0) { return []; }
-
-  toSearch = toSearch || _.union(_.keys(emojiHash), _.keys(extendedEmoji), _.keys(aliasHash)).sort();
-
-  const results = [];
-
-  function addResult(t) {
-    const val = aliasHash[t] || t;
-    if (results.indexOf(val) === -1) {
-      results.push(val);
-    }
-  }
-
-  // if term matches from beginning
-  for (let i=0; i<toSearch.length; i++) {
-    const item = toSearch[i];
-    if (item.indexOf(term) === 0) addResult(item);
-  }
-
-  if (searchAliases[term]) {
-    results.push.apply(results, searchAliases[term]);
-  }
-
-  for (let i=0; i<toSearch.length; i++) {
-    const item = toSearch[i];
-    if (item.indexOf(term) > 0) addResult(item);
-  }
-
-  if (maxResults === -1) {
-    return results;
-  } else {
-    return results.slice(0, maxResults);
-  }
-};
-
-export function isSkinTonableEmoji(term) {
-  const match = _.compact(term.split(":"))[0];
-  if (match) {
-    return tonableEmojis.indexOf(match) !== -1;
-  }
-  return false;
-}

app/assets/javascripts/pretty-text/emoji/data.js.es6.erb

@@ -3,3 +3,4 @@ export const tonableEmojis = <%= Emoji.tonable_emojis.flatten.inspect %>;
 export const aliases = <%= Emoji.aliases.inspect.gsub("=>", ":") %>;
 export const searchAliases = <%= Emoji.search_aliases.inspect.gsub("=>", ":") %>;
 export const translations = <%= Emoji.translations.inspect.gsub("=>", ":") %>;
+export const replacements = <%= Emoji.unicode_replacements_json %>;

app/assets/javascripts/pretty-text/emoji/version.js.es6.erb

@@ -0,0 +1,2 @@
+// bump up this number to expire all emojis
+export const IMAGE_VERSION = "<%= Emoji::EMOJI_VERSION %>";

app/models/emoji.rb

@@ -157,13 +157,7 @@ class Emoji
   end
 
   def self.unicode_unescape(string)
-    string.each_char.map do |c|
-      if str = unicode_replacements[c]
-        ":#{str}:"
-      else
-        c
-      end
-    end.join
+    PrettyText.escape_emoji(string)
   end
 
   def self.gsub_emoji_to_unicode(str)

lib/pretty_text.rb

@@ -227,7 +227,7 @@ module PrettyText
   end
 
   def self.unescape_emoji(title)
-    return title unless SiteSetting.enable_emoji?
+    return title unless SiteSetting.enable_emoji? && title
 
     set = SiteSetting.emoji_set.inspect
     custom = Emoji.custom.map { |e| [e.name, e.url] }.to_h.to_json
@@ -239,6 +239,16 @@ module PrettyText
     end
   end
 
+  def self.escape_emoji(title)
+    return unless title
+
+    protect do
+      v8.eval(<<~JS)
+        __performEmojiEscape(#{title.inspect});
+      JS
+    end
+  end
+
   def self.cook(text, opts = {})
     options = opts.dup
 

lib/pretty_text/shims.js

@@ -1,6 +1,7 @@
 __PrettyText = require("pretty-text/pretty-text").default;
 __buildOptions = require("pretty-text/pretty-text").buildOptions;
 __performEmojiUnescape = require("pretty-text/emoji").performEmojiUnescape;
+__performEmojiEscape = require("pretty-text/emoji").performEmojiEscape;
 
 __utils = require("discourse/lib/utilities");
 

spec/models/topic_spec.rb

@@ -279,6 +279,7 @@ describe Topic do
     let(:topic_image) { build_topic_with_title("Topic with <img src='something'> image in its title") }
     let(:topic_script) { build_topic_with_title("Topic with <script>alert('title')</script> script in its title") }
     let(:topic_emoji) { build_topic_with_title("I 💖 candy alot") }
+    let(:topic_modifier_emoji) { build_topic_with_title("I 👨‍🌾 candy alot") }
 
     it "escapes script contents" do
       expect(topic_script.fancy_title).to eq("Topic with &lt;script&gt;alert(&lsquo;title&rsquo;)&lt;/script&gt; script in its title")
@@ -288,6 +289,10 @@ describe Topic do
       expect(topic_emoji.fancy_title).to eq("I :sparkling_heart: candy alot")
     end
 
+    it "keeps combined emojis" do
+      expect(topic_modifier_emoji.fancy_title).to eq("I :man_farmer: candy alot")
+    end
+
     it "escapes bold contents" do
       expect(topic_bold.fancy_title).to eq("Topic with &lt;b&gt;bold&lt;/b&gt; text in its title")
     end

test/javascripts/acceptance/emoji-picker-test.js.es6

@@ -1,5 +1,5 @@
 import { acceptance } from "helpers/qunit-helpers";
-import { IMAGE_VERSION as v } from "pretty-text/emoji";
+import { IMAGE_VERSION as v } from "pretty-text/emoji/version";
 import { resetCache } from "discourse/components/emoji-picker";
 
 acceptance("EmojiPicker", {

test/javascripts/acceptance/emoji-test.js.es6

@@ -1,4 +1,4 @@
-import { IMAGE_VERSION as v } from "pretty-text/emoji";
+import { IMAGE_VERSION as v } from "pretty-text/emoji/version";
 import { acceptance } from "helpers/qunit-helpers";
 
 acceptance("Emoji", { loggedIn: true });

test/javascripts/acceptance/topic-test.js.es6

@@ -1,5 +1,5 @@
 import { acceptance } from "helpers/qunit-helpers";
-import { IMAGE_VERSION as v } from "pretty-text/emoji";
+import { IMAGE_VERSION as v } from "pretty-text/emoji/version";
 
 acceptance("Topic", {
   loggedIn: true,
@@ -121,6 +121,23 @@ QUnit.test("Updating the topic title with emojis", async assert => {
   );
 });
 
+QUnit.test("Updating the topic title with unicode emojis", async assert => {
+  await visit("/t/internationalization-localization/280");
+  await click("#topic-title .d-icon-pencil-alt");
+
+  await fillIn("#edit-title", "emojis title 👨‍🌾");
+
+  await click("#topic-title .submit-edit");
+
+  assert.equal(
+    find(".fancy-title")
+      .html()
+      .trim(),
+    `emojis title <img src="/images/emoji/emoji_one/man_farmer.png?v=${v}" title="man_farmer" alt="man_farmer" class="emoji">`,
+    "it displays the new title with escaped unicode emojis"
+  );
+});
+
 acceptance("Topic featured links", {
   loggedIn: true,
   settings: {

test/javascripts/lib/emoji-test.js.es6

@@ -1,4 +1,5 @@
-import { emojiSearch, IMAGE_VERSION as v } from "pretty-text/emoji";
+import { emojiSearch } from "pretty-text/emoji";
+import { IMAGE_VERSION as v } from "pretty-text/emoji/version";
 import { emojiUnescape } from "discourse/lib/text";
 
 QUnit.module("lib:emoji");

test/javascripts/lib/pretty-text-test.js.es6

@@ -1,7 +1,7 @@
 import Quote from "discourse/lib/quote";
 import Post from "discourse/models/post";
 import { default as PrettyText, buildOptions } from "pretty-text/pretty-text";
-import { IMAGE_VERSION as v } from "pretty-text/emoji";
+import { IMAGE_VERSION as v } from "pretty-text/emoji/version";
 import { INLINE_ONEBOX_LOADING_CSS_CLASS } from "pretty-text/inline-oneboxer";
 
 QUnit.module("lib:pretty-text");

test/javascripts/models/topic-test.js.es6

@@ -1,4 +1,4 @@
-import { IMAGE_VERSION as v } from "pretty-text/emoji";
+import { IMAGE_VERSION as v } from "pretty-text/emoji/version";
 
 QUnit.module("model:topic");