FEATURE: add support for multilingual mathjax (#84)

Chinese / Korean / Japanese / Thai and Arabic use different punctuations.

assets/javascripts/lib/discourse-markdown/discourse-math.js

@@ -3,6 +3,28 @@
 //
 //
 
+const additionalPunctuation = [
+  // Chinese and Japanese punctuation
+  0x3001, // 、
+  0x3002, // 。
+
+  // Full-width punctuation used in East Asian languages
+  0xff0c, // ，
+  0xff1a, // ：
+  0xff1b, // ；
+  0xff0e, // ．
+  0xff1f, // ？
+  0xff01, // ！
+
+  // Arabic punctuation
+  0x060c, // ،
+  0x061b, // ؛
+  0x061f, // ؟
+
+  // Thai punctuation
+  0x0e2f, // ฯ
+];
+
 function isSafeBoundary(character_code, delimiter_code, md) {
   if (character_code === delimiter_code) {
     return false;
@@ -20,6 +42,10 @@ function isSafeBoundary(character_code, delimiter_code, md) {
     return true;
   }
 
+  if (additionalPunctuation.includes(character_code)) {
+    return true;
+  }
+
   return false;
 }
 

spec/pretty_text_spec.rb

@@ -38,6 +38,30 @@ describe PrettyText do
       expect(PrettyText.cook "($ +500 $)").to include("math")
     end
 
+    it "can handle inline math with Chinese punctuation" do
+      cooked = PrettyText.cook("这是一个测试，$a^2 + b^2 = c^2$，这是另一个测试。")
+      html = '<p>这是一个测试，<span class="math">a^2 + b^2 = c^2</span>，这是另一个测试。</p>'
+      expect(cooked).to eq(html)
+    end
+
+    it "can handle inline math with Japanese punctuation" do
+      cooked = PrettyText.cook("これはテストです、$a^2 + b^2 = c^2$、これもテストです。")
+      html = '<p>これはテストです、<span class="math">a^2 + b^2 = c^2</span>、これもテストです。</p>'
+      expect(cooked).to eq(html)
+    end
+
+    it "can handle inline math with Arabic punctuation" do
+      cooked = PrettyText.cook("هذا اختبار،$a^2 + b^2 = c^2$،هذا اختبار آخر.")
+      html = '<p>هذا اختبار،<span class="math">a^2 + b^2 = c^2</span>،هذا اختبار آخر.</p>'
+      expect(cooked).to eq(html)
+    end
+
+    it "can handle block math with Chinese punctuation" do
+      cooked = PrettyText.cook("$$\na^2 + b^2 = c^2\n$$")
+      html = "<div class=\"math\">\na^2 + b^2 = c^2\n</div>"
+      expect(cooked.strip).to eq(html.strip)
+    end
+
     it "can handle inline math" do
       cooked = PrettyText.cook <<~MD
         I like