FIX: Don't diplay character reference in HTML diffs (#4204)

* FIX: Don't diplay character reference in HTML diffs

Before this change, HTML escaping was done before splitting text into
tokens, so token splitter saw literals like "'", and split them as
it was normal text into parts into ["&", "#", "39", ";"]. This caused
diff to display character references, as those tokens used separate
HTML tags to display their insertion/deletion status.

* Avoid making one element arrays while generating diffs

lib/discourse_diff.rb

@@ -261,8 +261,7 @@ class DiscourseDiff
     end
 
     def characters(string)
-      string = CGI::escapeHTML(string)
-      @tokens.concat string.scan(/(\W|\w+[ \t]*)/).flatten
+      @tokens.concat string.scan(/\W|\w+[ \t]*/).map { |x| CGI::escapeHTML(x) }
     end
 
   end

spec/components/discourse_diff_spec.rb

@@ -49,6 +49,12 @@ describe DiscourseDiff do
       expect(DiscourseDiff.new(before, after).inline_html).to eq("<div class=\"inline-diff\"><p class=\"diff-del\">this is the first paragraph</p><p>this is the second paragraph</p></div>")
     end
 
+    it "does not break diff on character references" do
+      before = "<p>'</p>"
+      after = "<p></p>"
+      expect(DiscourseDiff.new(before, after).inline_html).to eq("<div class=\"inline-diff\"><p><del>&#39;</del></p></div>")
+    end
+
   end
 
   describe "side_by_side_html" do
@@ -86,6 +92,12 @@ describe DiscourseDiff do
       expect(DiscourseDiff.new(before, after).side_by_side_html).to eq("<div class=\"span8\"><p class=\"diff-del\">this is the first paragraph</p><p>this is the second paragraph</p></div><div class=\"span8 offset1\"><p>this is the second paragraph</p></div>")
     end
 
+    it "does not break diff on character references" do
+      before = "<p>'</p>"
+      after = "<p></p>"
+      expect(DiscourseDiff.new(before, after).side_by_side_html).to eq("<div class=\"span8\"><p><del>&#39;</del></p></div><div class=\"span8 offset1\"><p></p></div>")
+    end
+
   end
 
   describe "side_by_side_markdown" do