Feature: CommonMark support

This adds the markdown.it engine to Discourse.
https://github.com/markdown-it/markdown-it

As the migration is going to take a while the new engine is default
disabled. To enable it you must change the hidden site setting:
enable_experimental_markdown_it.

This commit is a squash of many other commits, it also includes some
improvements to autospec (ability to run plugins), and a dev dependency
on the og gem for html normalization.

Gemfile

@@ -75,6 +75,10 @@ gem 'discourse_image_optim', require: 'image_optim'
 gem 'multi_json'
 gem 'mustache'
 gem 'nokogiri'
+
+# this may end up deprecating nokogiri
+gem 'oga', require: false
+
 gem 'omniauth'
 gem 'omniauth-openid'
 gem 'openid-redis-store'

Gemfile.lock

@@ -42,7 +42,9 @@ GEM
     annotate (2.7.2)
       activerecord (>= 3.2, < 6.0)
       rake (>= 10.4, < 13.0)
+    ansi (1.5.0)
     arel (6.0.4)
+    ast (2.3.0)
     aws-sdk (2.5.3)
       aws-sdk-resources (= 2.5.3)
     aws-sdk-core (2.5.3)
@@ -181,6 +183,9 @@ GEM
       multi_json (~> 1.3)
       multi_xml (~> 0.5)
       rack (>= 1.2, < 3)
+    oga (2.10)
+      ast
+      ruby-ll (~> 2.1)
     oj (3.1.0)
     omniauth (1.6.1)
       hashie (>= 3.4.6, < 3.6.0)
@@ -311,6 +316,9 @@ GEM
       rspec-support (~> 3.6.0)
     rspec-support (3.6.0)
     rtlit (0.0.5)
+    ruby-ll (2.1.2)
+      ansi
+      ast
     ruby-openid (2.7.0)
     ruby-readability (0.7.0)
       guess_html_encoding (>= 0.0.4)
@@ -428,6 +436,7 @@ DEPENDENCIES
   multi_json
   mustache
   nokogiri
+  oga
   oj
   omniauth
   omniauth-facebook

app/assets/javascripts/discourse/components/d-editor.js.es6

@@ -6,7 +6,7 @@ import { categoryHashtagTriggerRule } from 'discourse/lib/category-hashtags';
 import { TAG_HASHTAG_POSTFIX } from 'discourse/lib/tag-hashtags';
 import { search as searchCategoryTag  } from 'discourse/lib/category-tag-search';
 import { SEPARATOR } from 'discourse/lib/category-hashtags';
-import { cook } from 'discourse/lib/text';
+import { cookAsync } from 'discourse/lib/text';
 import { translations } from 'pretty-text/emoji/data';
 import { emojiSearch, isSkinTonableEmoji } from 'pretty-text/emoji';
 import { emojiUrlFor } from 'discourse/lib/text';
@@ -279,14 +279,14 @@ export default Ember.Component.extend({
     const value = this.get('value');
     const markdownOptions = this.get('markdownOptions') || {};
 
-    markdownOptions.siteSettings = this.siteSettings;
+    cookAsync(value, markdownOptions).then(cooked => {
-
+      this.set('preview', cooked);
-    this.set('preview', cook(value));
+      Ember.run.scheduleOnce('afterRender', () => {
-    Ember.run.scheduleOnce('afterRender', () => {
+        if (this._state !== "inDOM") { return; }
-      if (this._state !== "inDOM") { return; }
+        const $preview = this.$('.d-editor-preview');
-      const $preview = this.$('.d-editor-preview');
+        if ($preview.length === 0) return;
-      if ($preview.length === 0) return;
+        this.sendAction('previewUpdated', $preview);
-      this.sendAction('previewUpdated', $preview);
+      });
     });
   },
 

app/assets/javascripts/discourse/lib/text.js.es6

@@ -2,22 +2,37 @@ import { default as PrettyText, buildOptions } from 'pretty-text/pretty-text';
 import { performEmojiUnescape, buildEmojiUrl } from 'pretty-text/emoji';
 import WhiteLister from 'pretty-text/white-lister';
 import { sanitize as textSanitize } from 'pretty-text/sanitizer';
+import loadScript from 'discourse/lib/load-script';
 
-function getOpts() {
+function getOpts(opts) {
   const siteSettings = Discourse.__container__.lookup('site-settings:main');
 
-  return buildOptions({
+  opts = _.merge({
     getURL: Discourse.getURLWithCDN,
     currentUser: Discourse.__container__.lookup('current-user:main'),
     siteSettings
-  });
+  }, opts);
+
+  return buildOptions(opts);
 }
 
 // Use this to easily create a pretty text instance with proper options
-export function cook(text) {
+export function cook(text, options) {
-  return new Handlebars.SafeString(new PrettyText(getOpts()).cook(text));
+  return new Handlebars.SafeString(new PrettyText(getOpts(options)).cook(text));
 }
 
+// everything should eventually move to async API and this should be renamed
+// cook
+export function cookAsync(text, options) {
+  if (Discourse.MarkdownItURL) {
+    return loadScript(Discourse.MarkdownItURL)
+      .then(()=>cook(text, options));
+  } else {
+    return Ember.RSVP.Promise.resolve(cook(text));
+  }
+}
+
+
 export function sanitize(text) {
   return textSanitize(text, new WhiteLister(getOpts()));
 }

app/assets/javascripts/markdown-it-bundle.js

@@ -0,0 +1,11 @@
+//= require markdown-it.js
+//= require ./pretty-text/engines/markdown-it/helpers
+//= require ./pretty-text/engines/markdown-it/mentions
+//= require ./pretty-text/engines/markdown-it/quotes
+//= require ./pretty-text/engines/markdown-it/emoji
+//= require ./pretty-text/engines/markdown-it/onebox
+//= require ./pretty-text/engines/markdown-it/bbcode-block
+//= require ./pretty-text/engines/markdown-it/bbcode-inline
+//= require ./pretty-text/engines/markdown-it/code
+//= require ./pretty-text/engines/markdown-it/category-hashtag
+//= require ./pretty-text/engines/markdown-it/censored

app/assets/javascripts/pretty-text-bundle.js

@@ -4,6 +4,7 @@
 //= require ./pretty-text/emoji/data
 //= require ./pretty-text/emoji
 //= require ./pretty-text/engines/discourse-markdown
+//= require ./pretty-text/engines/discourse-markdown-it
 //= require_tree ./pretty-text/engines/discourse-markdown
 //= require xss.min
 //= require better_markdown.js

app/assets/javascripts/pretty-text/censored-words.js.es6

@@ -2,9 +2,11 @@ function escapeRegexp(text) {
   return text.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
 }
 
-export function censor(text, censoredWords, censoredPattern) {
+export function censorFn(censoredWords, censoredPattern, replacementLetter) {
-  let patterns = [],
+
-      originalText = text;
+  let patterns = [];
+
+  replacementLetter = replacementLetter || "■";
 
   if (censoredWords && censoredWords.length) {
     patterns = censoredWords.split("|").map(t => `(${escapeRegexp(t)})`);
@@ -21,19 +23,35 @@ export function censor(text, censoredWords, censoredPattern) {
       censorRegexp = new RegExp("(\\b(?:" + patterns.join("|") + ")\\b)(?![^\\(]*\\))", "ig");
 
       if (censorRegexp) {
-        let m = censorRegexp.exec(text);
 
-        while (m && m[0]) {
+        return function(text) {
-          if (m[0].length > originalText.length) { return originalText; } // regex is dangerous
+          let original = text;
-          const replacement = new Array(m[0].length+1).join('■');
+
-          text = text.replace(new RegExp(`(\\b${escapeRegexp(m[0])}\\b)(?![^\\(]*\\))`, "ig"), replacement);
+          try {
-          m = censorRegexp.exec(text);
+            let m = censorRegexp.exec(text);
-        }
+
+            while (m && m[0]) {
+              if (m[0].length > original.length) { return original; } // regex is dangerous
+              const replacement = new Array(m[0].length+1).join(replacementLetter);
+              text = text.replace(new RegExp(`(\\b${escapeRegexp(m[0])}\\b)(?![^\\(]*\\))`, "ig"), replacement);
+              m = censorRegexp.exec(text);
+            }
+
+            return text;
+          } catch (e) {
+            return original;
+          }
+        };
+
       }
     } catch(e) {
-      return originalText;
+      // fall through
     }
   }
 
-  return text;
+  return function(t){ return t;};
+}
+
+export function censor(text, censoredWords, censoredPattern, replacementLetter) {
+  return censorFn(censoredWords, censoredPattern, replacementLetter)(text);
 }

app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6

@@ -0,0 +1,139 @@
+import { default as WhiteLister, whiteListFeature } from 'pretty-text/white-lister';
+import { sanitize } from 'pretty-text/sanitizer';
+
+function deprecate(feature, name){
+  return function() {
+    if (console && console.log) {
+      console.log(feature + ': ' + name + ' is deprecated, please use the new markdown it APIs');
+    }
+  };
+}
+
+function createHelper(featureName, opts, optionCallbacks, pluginCallbacks, getOptions) {
+  let helper = {};
+  helper.markdownIt = true;
+  helper.whiteList = info => whiteListFeature(featureName, info);
+  helper.registerInline = deprecate(featureName,'registerInline');
+  helper.replaceBlock = deprecate(featureName,'replaceBlock');
+  helper.addPreProcessor = deprecate(featureName,'addPreProcessor');
+  helper.inlineReplace = deprecate(featureName,'inlineReplace');
+  helper.postProcessTag = deprecate(featureName,'postProcessTag');
+  helper.inlineRegexp = deprecate(featureName,'inlineRegexp');
+  helper.inlineBetween = deprecate(featureName,'inlineBetween');
+  helper.postProcessText = deprecate(featureName,'postProcessText');
+  helper.onParseNode = deprecate(featureName,'onParseNode');
+  helper.registerBlock = deprecate(featureName,'registerBlock');
+  // hack to allow moving of getOptions
+  helper.getOptions = () => getOptions.f();
+
+  helper.registerOptions = function(callback){
+    optionCallbacks.push([featureName, callback]);
+  };
+
+  helper.registerPlugin = function(callback){
+    pluginCallbacks.push([featureName, callback]);
+  };
+
+  return helper;
+}
+
+// TODO we may just use a proper ruler from markdown it... this is a basic proxy
+class Ruler {
+  constructor() {
+    this.rules = [];
+  }
+
+  getRules() {
+    return this.rules;
+  }
+
+  push(name, rule) {
+    this.rules.push({name, rule});
+  }
+}
+
+// block bb code ruler for parsing of quotes / code / polls
+function setupBlockBBCode(md) {
+  md.block.bbcode_ruler = new Ruler();
+}
+
+export function setup(opts, siteSettings, state) {
+  if (opts.setup) {
+    return;
+  }
+
+  opts.markdownIt = true;
+
+  let optionCallbacks = [];
+  let pluginCallbacks = [];
+
+  // ideally I would like to change the top level API a bit, but in the mean time this will do
+  let getOptions = {
+    f: () => opts
+  };
+
+  const check = /discourse-markdown\/|markdown-it\//;
+  let features = [];
+
+  Object.keys(require._eak_seen).forEach(entry => {
+    if (check.test(entry)) {
+      const module = require(entry);
+      if (module && module.setup) {
+
+        const featureName = entry.split('/').reverse()[0];
+        features.push(featureName);
+        module.setup(createHelper(featureName, opts, optionCallbacks, pluginCallbacks, getOptions));
+      }
+    }
+  });
+
+  optionCallbacks.forEach(([,callback])=>{
+    callback(opts, siteSettings, state);
+  });
+
+  // enable all features by default
+  features.forEach(feature => {
+    if (!opts.features.hasOwnProperty(feature)) {
+      opts.features[feature] = true;
+    }
+  });
+
+  let copy = {};
+  Object.keys(opts).forEach(entry => {
+    copy[entry] = opts[entry];
+    delete opts[entry];
+  });
+
+  opts.discourse = copy;
+  getOptions.f = () => opts.discourse;
+
+  opts.engine = window.markdownit({
+    discourse: opts.discourse,
+    html: true,
+    breaks: opts.discourse.features.newline,
+    xhtmlOut: false,
+    linkify: true,
+    typographer: false
+  });
+
+  setupBlockBBCode(opts.engine);
+
+  pluginCallbacks.forEach(([feature, callback])=>{
+    if (opts.discourse.features[feature]) {
+      opts.engine.use(callback);
+    }
+  });
+
+  // top level markdown it notifier
+  opts.markdownIt = true;
+  opts.setup = true;
+
+  if (!opts.discourse.sanitizer) {
+    opts.sanitizer = opts.discourse.sanitizer = (!!opts.discourse.sanitize) ? sanitize : a=>a;
+  }
+}
+
+export function cook(raw, opts) {
+  const whiteLister = new WhiteLister(opts.discourse);
+  return opts.discourse.sanitizer(opts.engine.render(raw), whiteLister).trim();
+}

app/assets/javascripts/pretty-text/engines/discourse-markdown.js.es6

@@ -385,14 +385,25 @@ export function cook(raw, opts) {
   currentOpts = opts;
 
   hoisted = {};
-  raw = hoistCodeBlocksAndSpans(raw);
 
-  preProcessors.forEach(p => raw = p(raw));
+  if (!currentOpts.enableExperimentalMarkdownIt) {
+    raw = hoistCodeBlocksAndSpans(raw);
+    preProcessors.forEach(p => raw = p(raw));
+  }
 
   const whiteLister = new WhiteLister(opts);
 
-  const tree = parser.toHTMLTree(raw, 'Discourse');
+  let result;
-  let result = opts.sanitizer(parser.renderJsonML(parseTree(tree, opts)), whiteLister);
+
+  if (currentOpts.enableExperimentalMarkdownIt) {
+    result = opts.sanitizer(
+        require('pretty-text/engines/markdown-it/instance').default(opts).render(raw),
+        whiteLister
+    );
+  } else {
+    const tree = parser.toHTMLTree(raw, 'Discourse');
+    result = opts.sanitizer(parser.renderJsonML(parseTree(tree, opts)), whiteLister);
+  }
 
   // If we hoisted out anything, put it back
   const keys = Object.keys(hoisted);

app/assets/javascripts/pretty-text/engines/discourse-markdown/auto-link.js.es6

@@ -21,6 +21,7 @@ const urlReplacerArgs = {
 };
 
 export function setup(helper) {
+  if (helper.markdownIt) { return; }
   helper.inlineRegexp(_.merge({start: 'http'}, urlReplacerArgs));
   helper.inlineRegexp(_.merge({start: 'www'}, urlReplacerArgs));
 }

app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode.js.es6

@@ -102,6 +102,8 @@ export function builders(helper) {
 
 export function setup(helper) {
 
+  if (helper.markdownIt) { return; }
+
   helper.whiteList(['span.bbcode-b', 'span.bbcode-i', 'span.bbcode-u', 'span.bbcode-s']);
 
   const { replaceBBCode, rawBBCode, removeEmptyLines, replaceBBCodeParamsRaw } = builders(helper);

app/assets/javascripts/pretty-text/engines/discourse-markdown/bold-italics.js.es6

@@ -35,6 +35,8 @@ function unhoist(obj,from,to){
 
 export function setup(helper) {
 
+  if (helper.markdownIt) { return; }
+
   function replaceMarkdown(match, tag) {
     const hash = guid();
 

app/assets/javascripts/pretty-text/engines/discourse-markdown/category-hashtag.js.es6

@@ -1,4 +1,7 @@
 export function setup(helper) {
+
+  if (helper.markdownIt) { return; }
+
   helper.inlineRegexp({
     start: '#',
     matcher: /^#([\w-:]{1,101})/i,

app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6

@@ -8,6 +8,9 @@ registerOption((siteSettings, opts) => {
 });
 
 export function setup(helper) {
+
+  if (helper.markdownIt) { return; }
+
   helper.addPreProcessor(text => {
     const options = helper.getOptions();
     return censor(text, options.censoredWords, options.censoredPattern);

app/assets/javascripts/pretty-text/engines/discourse-markdown/code.js.es6

@@ -21,6 +21,8 @@ registerOption((siteSettings, opts) => {
 
 export function setup(helper) {
 
+  if (helper.markdownIt) { return; }
+
   helper.whiteList({
     custom(tag, name, value) {
       if (tag === 'code' && name === 'class') {

app/assets/javascripts/pretty-text/engines/discourse-markdown/emoji.js.es6

@@ -35,6 +35,8 @@ registerOption((siteSettings, opts, state) => {
 
 export function setup(helper) {
 
+  if (helper.markdownIt) { return; }
+
   helper.whiteList('img.emoji');
 
   function imageFor(code) {

app/assets/javascripts/pretty-text/engines/discourse-markdown/html.js.es6

@@ -21,6 +21,8 @@ function splitAtLast(tag, block, next, first) {
 
 export function setup(helper) {
 
+  if (helper.markdownIt) { return; }
+
   // If a row begins with HTML tags, don't parse it.
   helper.registerBlock('html', function(block, next) {
     let split, pos;

app/assets/javascripts/pretty-text/engines/discourse-markdown/mentions.js.es6

@@ -5,6 +5,8 @@
 **/
 export function setup(helper) {
 
+  if (helper.markdownIt) { return; }
+
   // We have to prune @mentions that are within links.
   helper.onParseNode(event => {
     const node = event.node,

app/assets/javascripts/pretty-text/engines/discourse-markdown/newline.js.es6

@@ -2,6 +2,9 @@
 // in the tree, replace any new lines with `br`s.
 
 export function setup(helper) {
+
+  if (helper.markdownIt) { return; }
+
   helper.postProcessText((text, event) => {
     const { options, insideCounts } = event;
     if (options.traditionalMarkdownLinebreaks || (insideCounts.pre > 0)) { return; }

app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6

@@ -25,6 +25,9 @@ function isOnOneLine(link, parent) {
 
 //  We only onebox stuff that is on its own line.
 export function setup(helper) {
+
+  if (helper.markdownIt) { return; }
+
   helper.onParseNode(event => {
     const node = event.node,
     path = event.path;

app/assets/javascripts/pretty-text/engines/discourse-markdown/quote.js.es6

@@ -9,6 +9,9 @@ registerOption((siteSettings, opts) => {
 
 
 export function setup(helper) {
+
+  if (helper.markdownIt) { return; }
+
   register(helper, 'quote', {noWrap: true, singlePara: true}, (contents, bbParams, options) => {
 
     const params = {'class': 'quote'};

app/assets/javascripts/pretty-text/engines/discourse-markdown/table.js.es6

@@ -18,6 +18,8 @@ registerOption((siteSettings, opts) => {
 
 export function setup(helper) {
 
+  if (helper.markdownIt) { return; }
+
   helper.whiteList(['table', 'table.md-table', 'tbody', 'thead', 'tr', 'th', 'td']);
 
   helper.replaceBlock({

app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-block.js.es6

@@ -0,0 +1,200 @@
+// parse a tag [test a=1 b=2] to a data structure
+// {tag: "test", attrs={a: "1", b: "2"}
+export function parseBBCodeTag(src, start, max) {
+
+  let i;
+  let tag;
+  let attrs = {};
+  let closed = false;
+  let length = 0;
+  let closingTag = false;
+
+  // closing tag
+  if (src.charCodeAt(start+1) === 47) {
+    closingTag = true;
+    start += 1;
+  }
+
+  for (i=start+1;i<max;i++) {
+    let letter = src[i];
+    if (!( (letter >= 'a' && letter <= 'z') ||
+         (letter >= 'A' && letter <= 'Z'))) {
+      break;
+    }
+  }
+
+  tag = src.slice(start+1, i);
+
+  if (!tag) {
+    return;
+  }
+
+  if (closingTag) {
+    if (src[i] === ']') {
+      return {tag, length: tag.length+3, closing: true};
+    }
+    return;
+  }
+
+  for (;i<max;i++) {
+    let letter = src[i];
+
+    if (letter === ']') {
+      closed = true;
+      break;
+    }
+  }
+
+  if (closed) {
+    length = i;
+
+    let raw = src.slice(start+tag.length+1, i);
+
+    // trivial parser that is going to have to be rewritten at some point
+    if (raw) {
+
+      // reading a key 0, reading a val = 1
+      let readingKey = true;
+      let startSplit = 0;
+      let key;
+
+      for(i=0; i<raw.length; i++) {
+        if (raw[i] === '=' || i === (raw.length-1)) {
+          // one more offset to allow room to capture last
+          if (raw[i] !== '=' || i === (raw.length-1)) {
+            i+=1;
+          }
+
+          let cur = raw.slice(startSplit, i).trim();
+          if (readingKey) {
+            key =  cur || '_default';
+          } else {
+            let val = raw.slice(startSplit, i).trim();
+            if (val && val.length > 0) {
+              val = val.replace(/^["'](.*)["']$/, '$1');
+              attrs[key] = val;
+            }
+          }
+          readingKey = !readingKey;
+          startSplit = i+1;
+        }
+      }
+    }
+
+    tag = tag.toLowerCase();
+
+    return {tag, attrs, length};
+  }
+}
+
+function applyBBCode(state, startLine, endLine, silent, md) {
+
+  var i, pos, nextLine,
+      old_parent, old_line_max, rule,
+      auto_closed = false,
+      start = state.bMarks[startLine] + state.tShift[startLine],
+      initial = start,
+      max = state.eMarks[startLine];
+
+
+  // [ === 91
+  if (91 !== state.src.charCodeAt(start)) { return false; }
+
+  let info = parseBBCodeTag(state.src, start, max);
+
+  if (!info) {
+    return false;
+  }
+
+  let rules = md.block.bbcode_ruler.getRules();
+
+  for(i=0;i<rules.length;i++) {
+    let r = rules[i].rule;
+
+    if (r.tag === info.tag) {
+      rule = r;
+      break;
+    }
+  }
+
+  if (!rule) { return false; }
+
+  // Since start is found, we can report success here in validation mode
+  if (silent) { return true; }
+
+  // Search for the end of the block
+  nextLine = startLine;
+
+  for (;;) {
+    nextLine++;
+    if (nextLine >= endLine) {
+      // unclosed block should be autoclosed by end of document.
+      // also block seems to be autoclosed by end of parent
+      break;
+    }
+
+    start = state.bMarks[nextLine] + state.tShift[nextLine];
+    max = state.eMarks[nextLine];
+
+    if (start < max && state.sCount[nextLine] < state.blkIndent) {
+      // non-empty line with negative indent should stop the list:
+      // - ```
+      //  test
+      break;
+    }
+
+
+    // bbcode close [ === 91
+    if (91 !== state.src.charCodeAt(start)) { continue; }
+
+    if (state.sCount[nextLine] - state.blkIndent >= 4) {
+      // closing fence should be indented less than 4 spaces
+      continue;
+    }
+
+
+    if (state.src.slice(start+2, max-1) !== rule.tag) { continue; }
+
+    if (pos < max) { continue; }
+
+    // found!
+    auto_closed = true;
+    break;
+  }
+
+  old_parent = state.parentType;
+  old_line_max = state.lineMax;
+
+  // this will prevent lazy continuations from ever going past our end marker
+  state.lineMax = nextLine;
+
+  rule.before.call(this, state, info.attrs, md, state.src.slice(initial, initial + info.length + 1));
+
+  let lastToken = state.tokens[state.tokens.length-1];
+  lastToken.map    = [ startLine, nextLine ];
+
+  state.md.block.tokenize(state, startLine + 1, nextLine);
+
+  rule.after.call(this, state, lastToken, md);
+
+  lastToken = state.tokens[state.tokens.length-1];
+
+  state.parentType = old_parent;
+
+  state.lineMax = old_line_max;
+  state.line = nextLine + (auto_closed ? 1 : 0);
+
+  return true;
+}
+
+
+
+export function setup(helper) {
+  if (!helper.markdownIt) { return; }
+
+  helper.registerPlugin(md => {
+    md.block.ruler.after('fence', 'bbcode', (state, startLine, endLine, silent)=> {
+      return applyBBCode(state, startLine, endLine, silent, md);
+    });
+  });
+}

app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-inline.js.es6

@@ -0,0 +1,109 @@
+import { parseBBCodeTag } from 'pretty-text/engines/markdown-it/bbcode-block';
+
+const rules = {
+  'b': {tag: 'span', 'class': 'bbcode-b'},
+  'i': {tag: 'span', 'class': 'bbcode-i'},
+  'u': {tag: 'span', 'class': 'bbcode-u'},
+  's': {tag: 'span', 'class': 'bbcode-s'}
+};
+
+function tokanizeBBCode(state, silent) {
+
+  let pos = state.pos;
+
+  // 91 = [
+  if (silent || state.src.charCodeAt(pos) !== 91) {
+    return false;
+  }
+
+  const tagInfo = parseBBCodeTag(state.src, pos, state.posMax);
+
+  if (!tagInfo) {
+    return false;
+  }
+
+  const rule = rules[tagInfo.tag];
+  if (!rule) {
+    return false;
+  }
+
+  tagInfo.rule = rule;
+
+  let token = state.push('text', '' , 0);
+  token.content = state.src.slice(pos, pos+tagInfo.length);
+
+  state.delimiters.push({
+    bbInfo: tagInfo,
+    marker: 'bb' + tagInfo.tag,
+    open: !tagInfo.closing,
+    close: !!tagInfo.closing,
+    token: state.tokens.length - 1,
+    level: state.level,
+    end: -1,
+    jump: 0
+  });
+
+  state.pos = pos + tagInfo.length;
+  return true;
+}
+
+function processBBCode(state, silent) {
+  let i,
+      startDelim,
+      endDelim,
+      token,
+      tagInfo,
+      delimiters = state.delimiters,
+      max = delimiters.length;
+
+  if (silent) {
+    return;
+  }
+
+  for (i=0; i<max-1; i++) {
+    startDelim = delimiters[i];
+    tagInfo = startDelim.bbInfo;
+
+    if (!tagInfo) {
+      continue;
+    }
+
+    if (startDelim.end === -1) {
+      continue;
+    }
+
+    endDelim = delimiters[startDelim.end];
+
+    token = state.tokens[startDelim.token];
+    token.type = 'bbcode_' + tagInfo.tag + '_open';
+    token.attrs = [['class', tagInfo.rule['class']]];
+    token.tag = tagInfo.rule.tag;
+    token.nesting = 1;
+    token.markup = token.content;
+    token.content = '';
+
+    token = state.tokens[endDelim.token];
+    token.type = 'bbcode_' + tagInfo.tag + '_close';
+    token.tag = tagInfo.rule.tag;
+    token.nesting = -1;
+    token.markup = token.content;
+    token.content = '';
+  }
+  return false;
+}
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.whiteList(['span.bbcode-b', 'span.bbcode-i', 'span.bbcode-u', 'span.bbcode-s']);
+
+  helper.registerOptions(opts => {
+    opts.features['bbcode-inline'] = true;
+  });
+
+  helper.registerPlugin(md => {
+    md.inline.ruler.push('bbcode-inline', tokanizeBBCode);
+    md.inline.ruler2.before('text_collapse', 'bbcode-inline', processBBCode);
+  });
+}

app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6

@@ -0,0 +1,58 @@
+import { inlineRegexRule } from 'pretty-text/engines/markdown-it/helpers';
+
+function emitter(matches, state) {
+    const options = state.md.options.discourse;
+    const [hashtag, slug] = matches;
+    const categoryHashtagLookup = options.categoryHashtagLookup;
+    const result = categoryHashtagLookup && categoryHashtagLookup(slug);
+
+    let token;
+
+    if (result) {
+      token = state.push('link_open', 'a', 1);
+      token.attrs = [['class', 'hashtag'], ['href', result[0]]];
+      token.block = false;
+
+      token = state.push('text', '', 0);
+      token.content = '#';
+
+      token = state.push('span_open', 'span', 1);
+      token.block = false;
+
+      token = state.push('text', '', 0);
+      token.content = result[1];
+
+      state.push('span_close', 'span', -1);
+
+      state.push('link_close', 'a', -1);
+    } else {
+
+      token = state.push('span_open', 'span', 1);
+      token.attrs = [['class', 'hashtag']];
+
+      token = state.push('text', '', 0);
+      token.content = hashtag;
+
+      token = state.push('span_close', 'span', -1);
+    }
+
+    return true;
+}
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerPlugin(md=>{
+
+    const rule = inlineRegexRule(md, {
+      start: '#',
+      matcher: /^#([\w-:]{1,101})/i,
+      skipInLink: true,
+      maxLength: 102,
+      emitter: emitter
+    });
+
+    md.inline.ruler.push('category-hashtag', rule);
+  });
+}

app/assets/javascripts/pretty-text/engines/markdown-it/censored.js.es6

@@ -0,0 +1,44 @@
+import { censorFn } from 'pretty-text/censored-words';
+
+function recurse(tokens, apply) {
+  let i;
+  for(i=0;i<tokens.length;i++) {
+    apply(tokens[i]);
+    if (tokens[i].children) {
+      recurse(tokens[i].children, apply);
+    }
+  }
+}
+
+function censorTree(state, censor) {
+  if (!state.tokens) {
+    return;
+  }
+
+  recurse(state.tokens, token => {
+    if (token.content) {
+      token.content = censor(token.content);
+    }
+  });
+}
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerOptions((opts, siteSettings) => {
+    opts.censoredWords = siteSettings.censored_words;
+    opts.censoredPattern = siteSettings.censored_pattern;
+  });
+
+  helper.registerPlugin(md => {
+    const words = md.options.discourse.censoredWords;
+    const patterns = md.options.discourse.censoredPattern;
+
+    if ((words && words.length > 0) || (patterns && patterns.length > 0)) {
+      const replacement = String.fromCharCode(9632);
+      const censor = censorFn(words, patterns, replacement);
+      md.core.ruler.push('censored', state => censorTree(state, censor));
+    }
+  });
+}

app/assets/javascripts/pretty-text/engines/markdown-it/code.js.es6

@@ -0,0 +1,51 @@
+// we need a custom renderer for code blocks cause we have a slightly non compliant
+// format with special handling for text and so on
+
+const TEXT_CODE_CLASSES = ["text", "pre", "plain"];
+
+
+function render(tokens, idx, options, env, slf, md) {
+  let token = tokens[idx],
+      info = token.info ? md.utils.unescapeAll(token.info) : '',
+      langName = md.options.discourse.defaultCodeLang,
+      className,
+      escapedContent = md.utils.escapeHtml(token.content);
+
+  if (info) {
+    // strip off any additional languages
+    info = info.split(/\s+/g)[0];
+  }
+
+  const acceptableCodeClasses = md.options.discourse.acceptableCodeClasses;
+  if (acceptableCodeClasses && info && acceptableCodeClasses.indexOf(info) !== -1) {
+    langName = info;
+  }
+
+  className = TEXT_CODE_CLASSES.indexOf(langName) !== -1 ? 'lang-nohighlight' : 'lang-' + langName;
+
+  return `<pre><code class='${className}'>${escapedContent}</code></pre>\n`;
+}
+
+export function setup(helper) {
+  if (!helper.markdownIt) { return; }
+
+  helper.registerOptions((opts, siteSettings) => {
+    opts.defaultCodeLang = siteSettings.default_code_lang;
+    opts.acceptableCodeClasses = (siteSettings.highlighted_languages || "").split("|").concat(['auto', 'nohighlight']);
+  });
+
+  helper.whiteList({
+    custom(tag, name, value) {
+      if (tag === 'code' && name === 'class') {
+        const m = /^lang\-(.+)$/.exec(value);
+        if (m) {
+          return helper.getOptions().acceptableCodeClasses.indexOf(m[1]) !== -1;
+        }
+      }
+    }
+  });
+
+  helper.registerPlugin(md=>{
+    md.renderer.rules.fence = (tokens,idx,options,env,slf)=>render(tokens,idx,options,env,slf,md);
+  });
+}

app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6

@@ -0,0 +1,241 @@
+import { buildEmojiUrl, isCustomEmoji } from 'pretty-text/emoji';
+import { translations } from 'pretty-text/emoji/data';
+import { textReplace } from 'pretty-text/engines/markdown-it/helpers';
+
+const MAX_NAME_LENGTH = 60;
+
+let translationTree = null;
+
+// This allows us to efficiently search for aliases
+// We build a data structure that allows us to quickly
+// search through our N next chars to see if any match
+// one of our alias emojis.
+//
+function buildTranslationTree() {
+  let tree = [];
+  let lastNode;
+
+  Object.keys(translations).forEach(function(key){
+    let i;
+    let node = tree;
+
+    for(i=0;i<key.length;i++) {
+      let code = key.charCodeAt(i);
+      let j;
+
+      let found = false;
+
+      for (j=0;j<node.length;j++){
+        if (node[j][0] === code) {
+          node = node[j][1];
+          found = true;
+          break;
+        }
+      }
+
+      if (!found) {
+        // token, children, value
+        let tmp = [code, []];
+        node.push(tmp);
+        lastNode = tmp;
+        node = tmp[1];
+      }
+    }
+
+    lastNode[1] = translations[key];
+  });
+
+  return tree;
+}
+
+
+function imageFor(code, opts) {
+  code = code.toLowerCase();
+  const url = buildEmojiUrl(code, opts);
+  if (url) {
+    const title = `:${code}:`;
+    const classes = isCustomEmoji(code, opts) ? "emoji emoji-custom" : "emoji";
+    return {url, title, classes};
+  }
+}
+
+function getEmojiName(content, pos, state) {
+
+  if (content.charCodeAt(pos) !== 58) {
+    return;
+  }
+
+  if (pos > 0) {
+    let prev = content.charCodeAt(pos-1);
+    if (!state.md.utils.isSpace(prev) && !state.md.utils.isPunctChar(String.fromCharCode(prev))) {
+      return;
+    }
+  }
+
+  pos++;
+  if (content.charCodeAt(pos) === 58) {
+    return;
+  }
+
+  let length = 0;
+  while(length < MAX_NAME_LENGTH) {
+    length++;
+
+    if (content.charCodeAt(pos+length) === 58) {
+      // check for t2-t6
+      if (content.substr(pos+length+1, 3).match(/t[2-6]:/)) {
+        length += 3;
+      }
+      break;
+    }
+
+    if (pos+length > content.length) {
+      return;
+    }
+  }
+
+  if (length === MAX_NAME_LENGTH) {
+    return;
+  }
+
+  return content.substr(pos, length);
+}
+
+// straight forward :smile: to emoji image
+function getEmojiTokenByName(name, state) {
+
+  let info;
+  if (info = imageFor(name, state.md.options.discourse)) {
+    let token = new state.Token('emoji', 'img', 0);
+    token.attrs = [['src', info.url],
+                   ['title', info.title],
+                   ['class', info.classes],
+                   ['alt', info.title]];
+
+    return token;
+  }
+}
+
+function getEmojiTokenByTranslation(content, pos, state) {
+
+  translationTree = translationTree || buildTranslationTree();
+
+  let currentTree = translationTree;
+
+  let i;
+  let search = true;
+  let found = false;
+  let start = pos;
+
+  while(search) {
+
+    search = false;
+    let code = content.charCodeAt(pos);
+
+    for (i=0;i<currentTree.length;i++) {
+      if(currentTree[i][0] === code) {
+        currentTree = currentTree[i][1];
+        pos++;
+        search = true;
+        if (typeof currentTree === "string") {
+          found = currentTree;
+        }
+        break;
+      }
+    }
+  }
+
+  if (!found) {
+    return;
+  }
+
+  // quick boundary check
+  if (start > 0) {
+    let leading = content.charAt(start-1);
+    if (!state.md.utils.isSpace(leading.charCodeAt(0)) && !state.md.utils.isPunctChar(leading)) {
+      return;
+    }
+  }
+
+  // check trailing for punct or space
+  if (pos < content.length) {
+    let trailing = content.charCodeAt(pos);
+    if (!state.md.utils.isSpace(trailing)){
+      return;
+    }
+  }
+
+  let token = getEmojiTokenByName(found, state);
+  if (token) {
+    return { pos, token };
+  }
+}
+
+function applyEmoji(content, state) {
+  let i;
+  let result = null;
+  let contentToken = null;
+
+  let start = 0;
+
+  let endToken = content.length;
+
+  for (i=0; i<content.length-1; i++) {
+    let offset = 0;
+    let emojiName = getEmojiName(content,i,state);
+    let token = null;
+
+    if (emojiName) {
+      token = getEmojiTokenByName(emojiName, state);
+      if (token) {
+        offset = emojiName.length+2;
+      }
+    }
+
+    if (!token) {
+      // handle aliases (note: we can't do this in inline cause ; is not a split point)
+      //
+      let info = getEmojiTokenByTranslation(content, i, state);
+
+      if (info) {
+        offset = info.pos - i;
+        token = info.token;
+      }
+    }
+
+    if (token) {
+      result = result || [];
+      if (i-start>0) {
+        contentToken = new state.Token('text', '', 0);
+        contentToken.content = content.slice(start,i);
+        result.push(contentToken);
+      }
+
+      result.push(token);
+      endToken = start = i + offset;
+    }
+  }
+
+  if (endToken < content.length) {
+    contentToken = new state.Token('text', '', 0);
+    contentToken.content = content.slice(endToken);
+    result.push(contentToken);
+  }
+
+  return result;
+}
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerOptions((opts, siteSettings, state)=>{
+    opts.features.emoji = !!siteSettings.enable_emoji;
+    opts.emojiSet = siteSettings.emoji_set || "";
+    opts.customEmoji = state.customEmoji;
+  });
+
+  helper.registerPlugin((md)=>{
+    md.core.ruler.push('emoji', state => textReplace(state, applyEmoji));
+  });
+}

app/assets/javascripts/pretty-text/engines/markdown-it/helpers.js.es6

@@ -0,0 +1,106 @@
+// since the markdown.it interface is a bit on the verbose side
+// we can keep some general patterns here
+
+
+export default null;
+
+// creates a rule suitable for inline parsing and replacement
+//
+// example:
+// const rule = inlineRegexRule(md, {
+//   start: '#',
+//   matcher: /^#([\w-:]{1,101})/i,
+//   emitter: emitter
+// });
+export function inlineRegexRule(md, options) {
+
+  const start = options.start.charCodeAt(0);
+  const maxLength = (options.maxLength || 500) + 1;
+
+  return function(state) {
+    const pos = state.pos;
+
+    if (state.src.charCodeAt(pos) !== start) {
+      return false;
+    }
+
+    // test prev
+    if (pos > 0) {
+      let prev = state.src.charCodeAt(pos-1);
+      if (!md.utils.isSpace(prev) && !md.utils.isPunctChar(String.fromCharCode(prev))) {
+        return false;
+      }
+    }
+
+    // skip if in a link
+    if (options.skipInLink && state.tokens) {
+      let last = state.tokens[state.tokens.length-1];
+      if (last) {
+        if (last.type === 'link_open') {
+          return false;
+        }
+        if (last.type === 'html_inline' && last.content.substr(0,2) === "<a") {
+          return false;
+        }
+      }
+    }
+
+
+    const substr = state.src.slice(pos, Math.min(pos + maxLength,state.posMax));
+
+    const matches = options.matcher.exec(substr);
+    if (!matches) {
+      return false;
+    }
+
+    // got to test trailing boundary
+    const finalPos = pos+matches[0].length;
+    if (finalPos < state.posMax) {
+      const trailing = state.src.charCodeAt(finalPos);
+      if (!md.utils.isSpace(trailing) && !md.utils.isPunctChar(String.fromCharCode(trailing))) {
+        return false;
+      }
+    }
+
+    if (options.emitter(matches, state)) {
+      state.pos = Math.min(state.posMax, finalPos);
+      return true;
+    }
+
+    return false;
+
+  };
+}
+
+// based off https://github.com/markdown-it/markdown-it-emoji/blob/master/dist/markdown-it-emoji.js
+//
+export function textReplace(state, callback) {
+  var i, j, l, tokens, token,
+        blockTokens = state.tokens,
+        autolinkLevel = 0;
+
+  for (j = 0, l = blockTokens.length; j < l; j++) {
+    if (blockTokens[j].type !== 'inline') { continue; }
+    tokens = blockTokens[j].children;
+
+    // We scan from the end, to keep position when new tags added.
+    // Use reversed logic in links start/end match
+    for (i = tokens.length - 1; i >= 0; i--) {
+      token = tokens[i];
+
+      if (token.type === 'link_open' || token.type === 'link_close') {
+        if (token.info === 'auto') { autolinkLevel -= token.nesting; }
+      }
+
+      if (token.type === 'text' && autolinkLevel === 0) {
+        let split;
+        if(split = callback(token.content, state)) {
+          // replace current node
+          blockTokens[j].children = tokens = state.md.utils.arrayReplaceAt(
+            tokens, i, split
+          );
+        }
+      }
+    }
+  }
+}

app/assets/javascripts/pretty-text/engines/markdown-it/mentions.js.es6

@@ -0,0 +1,88 @@
+const regex = /^(\w[\w.-]{0,59})\b/i;
+
+function applyMentions(state, silent, isSpace, isPunctChar, mentionLookup, getURL) {
+
+  let pos = state.pos;
+
+  // 64 = @
+  if (silent || state.src.charCodeAt(pos) !== 64) {
+    return false;
+  }
+
+  if (pos > 0) {
+    let prev = state.src.charCodeAt(pos-1);
+    if (!isSpace(prev) && !isPunctChar(String.fromCharCode(prev))) {
+      return false;
+    }
+  }
+
+  // skip if in a link
+  if (state.tokens) {
+    let last = state.tokens[state.tokens.length-1];
+    if (last) {
+      if (last.type === 'link_open') {
+        return false;
+      }
+      if (last.type === 'html_inline' && last.content.substr(0,2) === "<a") {
+        return false;
+      }
+    }
+  }
+
+  let maxMention = state.src.substr(pos+1, 60);
+
+  let matches = maxMention.match(regex);
+
+  if (!matches) {
+    return false;
+  }
+
+  let username = matches[1];
+
+  let type = mentionLookup && mentionLookup(username);
+
+  let tag = 'a';
+  let className = 'mention';
+  let href = null;
+
+  if (type === 'user') {
+    href = getURL('/u/') + username.toLowerCase();
+  } else if (type === 'group') {
+    href = getURL('/groups/') + username;
+    className = 'mention-group';
+  } else {
+    tag = 'span';
+  }
+
+  let token = state.push('mention_open', tag, 1);
+  token.attrs = [['class', className]];
+  if (href) {
+    token.attrs.push(['href', href]);
+  }
+
+  token = state.push('text', '', 0);
+  token.content = '@'+username;
+
+  state.push('mention_close', tag, -1);
+
+  state.pos = pos + username.length + 1;
+
+  return true;
+}
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerPlugin(md => {
+    md.inline.ruler.push('mentions', (state,silent)=> applyMentions(
+          state,
+          silent,
+          md.utils.isSpace,
+          md.utils.isPunctChar,
+          md.options.discourse.mentionLookup,
+          md.options.discourse.getURL
+    ));
+  });
+}
+

app/assets/javascripts/pretty-text/engines/markdown-it/onebox.js.es6

@@ -0,0 +1,66 @@
+function applyOnebox(state, silent) {
+  if (silent || !state.tokens || state.tokens.length < 3) {
+    return;
+  }
+
+  let i;
+  for(i=1;i<state.tokens.length;i++) {
+    let token = state.tokens[i];
+
+    let prev = state.tokens[i-1];
+    let prevAccepted =  prev.type === "paragraph_open" && prev.level === 0;
+
+    if (token.type === "inline" && prevAccepted) {
+      let j;
+      for(j=0;j<token.children.length;j++){
+        let child = token.children[j];
+
+        if (child.type === "link_open") {
+
+          // look behind for soft or hard break
+          if (j > 0 && token.children[j-1].tag !== 'br') {
+            continue;
+          }
+
+          // look ahead for soft or hard break
+          let text = token.children[j+1];
+          let close = token.children[j+2];
+          let lookahead = token.children[j+3];
+
+          if (lookahead && lookahead.tag !== 'br') {
+            continue;
+          }
+
+          // check attrs only include a href
+          let attrs = child["attrs"];
+
+          if (!attrs || attrs.length !== 1 || attrs[0][0] !== "href") {
+            continue;
+          }
+
+          // check text matches href
+          if (text.type !== "text" || attrs[0][1] !== text.content) {
+            continue;
+          }
+
+          if (!close || close.type !== "link_close") {
+            continue;
+          }
+
+          // decorate...
+          attrs.push(["class", "onebox"]);
+        }
+      }
+    }
+  }
+}
+
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerPlugin(md => {
+    md.core.ruler.after('linkify', 'onebox', applyOnebox);
+  });
+}

app/assets/javascripts/pretty-text/engines/markdown-it/quotes.js.es6

@@ -0,0 +1,134 @@
+import { performEmojiUnescape } from 'pretty-text/emoji';
+
+const rule = {
+  tag: 'quote',
+
+  before: function(state, attrs, md) {
+
+    let options = md.options.discourse;
+
+    let quoteInfo = attrs['_default'];
+    let username, postNumber, topicId, avatarImg, full;
+
+    if (quoteInfo) {
+      let split = quoteInfo.split(/\,\s*/);
+      username = split[0];
+
+      let i;
+      for(i=1;i<split.length;i++) {
+        if (split[i].indexOf("post:") === 0) {
+          postNumber = parseInt(split[i].substr(5),10);
+          continue;
+        }
+
+        if (split[i].indexOf("topic:") === 0) {
+          topicId = parseInt(split[i].substr(6),10);
+          continue;
+        }
+
+        if (split[i].indexOf(/full:\s*true/) === 0) {
+          full = true;
+          continue;
+        }
+      }
+    }
+
+
+    let token    = state.push('bbcode_open', 'aside', 1);
+    token.attrs  = [['class', 'quote']];
+
+    if (postNumber) {
+      token.attrs.push(['data-post', postNumber]);
+    }
+
+    if (topicId) {
+      token.attrs.push(['data-topic', topicId]);
+    }
+
+    if (full) {
+      token.attrs.push(['data-full', 'true']);
+    }
+
+    if (options.lookupAvatarByPostNumber) {
+      // client-side, we can retrieve the avatar from the post
+      avatarImg = options.lookupAvatarByPostNumber(postNumber, topicId);
+    } else if (options.lookupAvatar) {
+      // server-side, we need to lookup the avatar from the username
+      avatarImg = options.lookupAvatar(username);
+    }
+
+    if (username) {
+      let offTopicQuote = options.topicId &&
+                          postNumber &&
+                          options.getTopicInfo &&
+                          topicId !== options.topicId;
+
+      // on topic quote
+      token = state.push('quote_header_open', 'div', 1);
+      token.attrs = [['class', 'title']];
+
+      token = state.push('quote_controls_open', 'div', 1);
+      token.attrs = [['class', 'quote-controls']];
+
+      token = state.push('quote_controls_close', 'div', -1);
+
+      if (avatarImg) {
+        token = state.push('html_inline', '', 0);
+        token.content = avatarImg;
+      }
+
+      if (offTopicQuote) {
+        const topicInfo = options.getTopicInfo(topicId);
+        if (topicInfo) {
+          var href = topicInfo.href;
+          if (postNumber > 0) { href += "/" + postNumber; }
+
+          let title = topicInfo.title;
+
+
+          if (options.enableEmoji) {
+            title = performEmojiUnescape(topicInfo.title, {
+              getURL: options.getURL, emojiSet: options.emojiSet
+            });
+          }
+
+          token = state.push('link_open', 'a', 1);
+          token.attrs = [[ 'href', href ]];
+          token.block = false;
+
+          token = state.push('html_inline', '', 0);
+          token.content = title;
+
+          token = state.push('link_close', 'a', -1);
+          token.block = false;
+        }
+      } else {
+        token = state.push('text', '', 0);
+        token.content = ` ${username}:`;
+      }
+
+      token = state.push('quote_header_close', 'div', -1);
+    }
+
+    token        = state.push('bbcode_open', 'blockquote', 1);
+  },
+
+  after: function(state) {
+    state.push('bbcode_close', 'blockquote', -1);
+    state.push('bbcode_close', 'aside', -1);
+  }
+};
+
+export function setup(helper) {
+
+  if (!helper.markdownIt) { return; }
+
+  helper.registerOptions((opts, siteSettings) => {
+    opts.enableEmoji = siteSettings.enable_emoji;
+    opts.emojiSet = siteSettings.emoji_set;
+  });
+
+  helper.registerPlugin(md=>{
+     md.block.bbcode_ruler.push('quotes', rule);
+  });
+}

app/assets/javascripts/pretty-text/pretty-text.js.es6

@@ -1,4 +1,5 @@
 import { cook, setup } from 'pretty-text/engines/discourse-markdown';
+import { cook as cookIt, setup as setupIt } from 'pretty-text/engines/discourse-markdown-it';
 import { sanitize } from 'pretty-text/sanitizer';
 import WhiteLister from 'pretty-text/white-lister';
 
@@ -10,8 +11,6 @@ export function registerOption(fn) {
 }
 
 export function buildOptions(state) {
-  setup();
-
   const {
     siteSettings,
     getURL,
@@ -21,9 +20,14 @@ export function buildOptions(state) {
     categoryHashtagLookup,
     userId,
     getCurrentUser,
-    currentUser
+    currentUser,
+    lookupAvatarByPostNumber
   } = state;
 
+  if (!siteSettings.enable_experimental_markdown_it) {
+    setup();
+  }
+
   const features = {
     'bold-italics': true,
     'auto-link': true,
@@ -33,7 +37,7 @@ export function buildOptions(state) {
     'html': true,
     'category-hashtag': true,
     'onebox': true,
-    'newline': true
+    'newline': !siteSettings.traditional_markdown_linebreaks
   };
 
   const options = {
@@ -47,11 +51,18 @@ export function buildOptions(state) {
     userId,
     getCurrentUser,
     currentUser,
+    lookupAvatarByPostNumber,
     mentionLookup: state.mentionLookup,
-    allowedHrefSchemes: siteSettings.allowed_href_schemes ? siteSettings.allowed_href_schemes.split('|') : null
+    allowedHrefSchemes: siteSettings.allowed_href_schemes ? siteSettings.allowed_href_schemes.split('|') : null,
+    markdownIt: siteSettings.enable_experimental_markdown_it
   };
 
-  _registerFns.forEach(fn => fn(siteSettings, options, state));
+  if (siteSettings.enable_experimental_markdown_it) {
+    setupIt(options, siteSettings, state);
+  } else {
+    // TODO deprecate this
+    _registerFns.forEach(fn => fn(siteSettings, options, state));
+  }
 
   return options;
 }
@@ -61,13 +72,22 @@ export default class {
     this.opts = opts || {};
     this.opts.features = this.opts.features || {};
     this.opts.sanitizer = (!!this.opts.sanitize) ? (this.opts.sanitizer || sanitize) : identity;
-    setup();
+    // We used to do a failsafe call to setup here
+    // under new engine we always expect setup to be called by buildOptions.
+    // setup();
   }
 
   cook(raw) {
     if (!raw || raw.length === 0) { return ""; }
 
-    const result = cook(raw, this.opts);
+    let result;
+
+    if (this.opts.markdownIt) {
+      result = cookIt(raw, this.opts);
+    } else {
+      result = cook(raw, this.opts);
+    }
+
     return result ? result : "";
   }
 

app/assets/javascripts/pretty-text/white-lister.js.es6

@@ -155,6 +155,7 @@ whiteListFeature('default', [
   'kbd',
   'li',
   'ol',
+  'ol[start]',
   'p',
   'pre',
   's',

app/views/common/_discourse_javascript.html.erb

@@ -17,12 +17,6 @@
     <% end %>
     window.onerror(e && e.message, null,null,null,e);
   });
-
-<% if Rails.env.development? || Rails.env.test? %>
-  //Ember.ENV.RAISE_ON_DEPRECATION = true
-  //Ember.LOG_STACKTRACE_ON_DEPRECATION = true
-<% end %>
-
 </script>
 
 <script>
@@ -48,19 +42,22 @@
     Discourse.Environment = '<%= Rails.env %>';
     Discourse.SiteSettings = ps.get('siteSettings');
     Discourse.LetterAvatarVersion = '<%= LetterAvatar.version %>';
+    <%- if SiteSetting.enable_experimental_markdown_it %>
+    Discourse.MarkdownItURL = '<%= asset_url('markdown-it-bundle.js') %>';
+    <%- end %>
     I18n.defaultLocale = '<%= SiteSetting.default_locale %>';
     Discourse.start();
     Discourse.set('assetVersion','<%= Discourse.assets_digest %>');
     Discourse.Session.currentProp("disableCustomCSS", <%= loading_admin? %>);
     <%- if params["safe_mode"] %>
-        Discourse.Session.currentProp("safe_mode", <%= normalized_safe_mode.inspect.html_safe %>);
+    Discourse.Session.currentProp("safe_mode", <%= normalized_safe_mode.inspect.html_safe %>);
     <%- end %>
     Discourse.HighlightJSPath = <%= HighlightJs.path.inspect.html_safe %>;
     <%- if SiteSetting.enable_s3_uploads %>
       <%- if SiteSetting.s3_cdn_url.present? %>
-        Discourse.S3CDN = '<%= SiteSetting.s3_cdn_url %>';
+    Discourse.S3CDN = '<%= SiteSetting.s3_cdn_url %>';
       <%- end %>
-      Discourse.S3BaseUrl = '<%= Discourse.store.absolute_base_url %>';
+    Discourse.S3BaseUrl = '<%= Discourse.store.absolute_base_url %>';
     <%- end %>
   })();
 </script>

bin/notify_file_change

@@ -19,7 +19,7 @@ fi
 # 3. Add the following to your .vimrc
 #
 # function s:notify_file_change()
-#   let root = RailsRoot()
+#   let root = rails#app().path()
 #   let notify = root . "/bin/notify_file_change"
 #   if executable(notify)
 #     if executable('socat')

config/application.rb

@@ -83,6 +83,7 @@ module Discourse
                                  browser-update.js break_string.js ember_jquery.js
                                  pretty-text-bundle.js wizard-application.js
                                  wizard-vendor.js plugin.js plugin-third-party.js
+                                 markdown-it-bundle.js
                                  }
 
     # Precompile all available locales

config/site_settings.yml

@@ -490,6 +490,10 @@ posting:
   delete_removed_posts_after:
     client: true
     default: 24
+  enable_experimental_markdown_it:
+    hidden: true
+    client: true
+    default: false
   traditional_markdown_linebreaks:
     client: true
     default: false

lib/autospec/manager.rb

@@ -295,8 +295,10 @@ class Autospec::Manager
         if @queue.first && @queue.first[0] == "focus"
           focus = @queue.shift
           @queue.unshift([file, spec, runner])
-          if focus[1].include?(spec) || file != spec
+          unless spec.include?(":") && focus[1].include?(spec.split(":")[0])
-            @queue.unshift(focus)
+            if focus[1].include?(spec) || file != spec
+              @queue.unshift(focus)
+            end
           end
         else
           @queue.unshift([file, spec, runner])

lib/autospec/qunit_runner.rb

@@ -159,7 +159,12 @@ module Autospec
         if m = /moduleFor\(['"]([^'"]+)/i.match(line)
           return m[1]
         end
+        if m = /moduleForComponent\(['"]([^"']+)/i.match(line)
+          return m[1]
+        end
       end
+
+      nil
     end
 
   end

lib/autospec/rspec_runner.rb

@@ -19,6 +19,10 @@ module Autospec
 
     watch(%r{^spec/fabricators/.+_fabricator\.rb$})     { "spec" }
 
+    watch(%r{^app/assets/javascripts/pretty-text/.*\.js\.es6$}) { "spec/components/pretty_text_spec.rb"}
+
+    watch(%r{^plugins/.*/spec/.*\.rb})
+
     RELOADERS = Set.new
     def self.reload(pattern); RELOADERS << pattern; end
     def reloaders; RELOADERS; end

lib/autospec/simple_runner.rb

@@ -13,7 +13,12 @@ module Autospec
               "-f", "Autospec::Formatter", specs.split].flatten.join(" ")
       # launch rspec
       Dir.chdir(Rails.root) do
-        @pid = Process.spawn({"RAILS_ENV" => "test"}, "bin/rspec #{args}")
+        env = {"RAILS_ENV" => "test"}
+        if specs.split(' ').any?{|s| s =~ /^(.\/)?plugins/}
+          env["LOAD_PLUGINS"] = "1"
+          puts "Loading plugins while running specs"
+        end
+        @pid = Process.spawn(env, "bin/rspec #{args}")
         _, status = Process.wait2(@pid)
         status.exitstatus
       end

lib/html_normalize.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+#
+# this class is used to normalize html output for internal comparisons in specs
+#
+require 'oga'
+
+class HtmlNormalize
+
+  def self.normalize(html)
+    parsed = Oga.parse_html(html.strip, strict: true)
+    if parsed.children.length != 1
+      puts parsed.children.count
+      raise "expecting a single child"
+    end
+    new(parsed.children.first).format
+  end
+
+  SELF_CLOSE = Set.new(%w{area base br col command embed hr img input keygen line meta param source track wbr})
+
+  BLOCK = Set.new(%w{
+    html
+    body
+    aside
+    p
+    h1 h2 h3 h4 h5 h6
+    ol ul
+    address
+    blockquote
+    dl
+    div
+    fieldset
+    form
+    hr
+    noscript
+    table
+    pre
+  })
+
+  def initialize(doc)
+    @doc = doc
+  end
+
+  def format
+    buffer = String.new
+    dump_node(@doc, 0, buffer)
+    buffer.strip!
+    buffer
+  end
+
+  def inline?(node)
+    Oga::XML::Text === node || !BLOCK.include?(node.name.downcase)
+  end
+
+  def dump_node(node, indent=0, buffer)
+
+    if Oga::XML::Text === node
+      if node.parent&.name
+        buffer << node.text
+      end
+      return
+    end
+
+    name = node.name.downcase
+
+    block = BLOCK.include?(name)
+
+    buffer << " " * indent * 2 if block
+
+    buffer << "<" << name
+
+    attrs = node&.attributes
+    if (attrs && attrs.length > 0)
+      attrs.sort!{|x,y| x.name <=> y.name}
+      attrs.each do |a|
+        buffer << " "
+        buffer << a.name
+        buffer << "='"
+        buffer << a.value
+        buffer << "'"
+      end
+    end
+
+    buffer << ">"
+
+    if block
+      buffer << "\n"
+    end
+
+    children = node.children
+    children = trim(children) if block
+
+    inline_buffer = nil
+
+    children&.each do |child|
+      if block && inline?(child)
+        inline_buffer ||= String.new
+        dump_node(child, indent+1, inline_buffer)
+      else
+        if inline_buffer
+          buffer << " " * (indent+1) * 2
+          buffer << inline_buffer.strip
+          inline_buffer = nil
+        else
+          dump_node(child, indent+1, buffer)
+        end
+      end
+    end
+
+    if inline_buffer
+      buffer << " " * (indent+1) * 2
+      buffer << inline_buffer.strip
+      inline_buffer = nil
+    end
+
+    if block
+      buffer << "\n" unless buffer[-1] == "\n"
+      buffer << " " * indent * 2
+    end
+
+    unless SELF_CLOSE.include?(name)
+      buffer << "</" << name
+      buffer << ">\n"
+    end
+  end
+
+  def trim(nodes)
+    start = 0
+    finish = nodes.length
+
+    nodes.each do |n|
+      if Oga::XML::Text === n && n.text.blank?
+        start += 1
+      else
+        break
+      end
+    end
+
+    nodes.reverse_each do |n|
+      if Oga::XML::Text === n && n.text.blank?
+        finish -= 1
+      else
+        break
+      end
+    end
+
+    nodes[start...finish]
+  end
+
+
+end

lib/pretty_text.rb

@@ -50,19 +50,10 @@ module PrettyText
     end
   end
 
-  def self.create_es6_context
+  def self.ctx_load_manifest(ctx, name)
-    ctx = MiniRacer::Context.new(timeout: 15000)
+    manifest = File.read("#{Rails.root}/app/assets/javascripts/#{name}")
-
-    ctx.eval("window = {}; window.devicePixelRatio = 2;") # hack to make code think stuff is retina
-
-    if Rails.env.development? || Rails.env.test?
-      ctx.attach("console.log", proc { |l| p l })
-    end
-
-    ctx_load(ctx, "#{Rails.root}/app/assets/javascripts/discourse-loader.js")
-    ctx_load(ctx, "vendor/assets/javascripts/lodash.js")
-    manifest = File.read("#{Rails.root}/app/assets/javascripts/pretty-text-bundle.js")
     root_path = "#{Rails.root}/app/assets/javascripts/"
+
     manifest.each_line do |l|
       l = l.chomp
       if l =~ /\/\/= require (\.\/)?(.*)$/
@@ -74,6 +65,22 @@ module PrettyText
         end
       end
     end
+  end
+
+  def self.create_es6_context
+    ctx = MiniRacer::Context.new(timeout: 15000)
+
+    ctx.eval("window = {}; window.devicePixelRatio = 2;") # hack to make code think stuff is retina
+
+    if Rails.env.development? || Rails.env.test?
+      ctx.attach("console.log", proc { |l| p l })
+    end
+
+    ctx_load(ctx, "#{Rails.root}/app/assets/javascripts/discourse-loader.js")
+    ctx_load(ctx, "vendor/assets/javascripts/lodash.js")
+    ctx_load_manifest(ctx, "pretty-text-bundle.js")
+
+    root_path = "#{Rails.root}/app/assets/javascripts/"
 
     apply_es6_file(ctx, root_path, "discourse/lib/utilities")
 
@@ -140,52 +147,62 @@ module PrettyText
         paths[:S3BaseUrl] = Discourse.store.absolute_base_url
       end
 
-      context.eval("__optInput = {};")
+      if SiteSetting.enable_experimental_markdown_it
-      context.eval("__optInput.siteSettings = #{SiteSetting.client_settings_json};")
+        unless context.eval("window.markdownit")
-      context.eval("__paths = #{paths.to_json};")
+          ctx_load_manifest(context, "markdown-it-bundle.js")
-
+        end
-      if opts[:topicId]
-        context.eval("__optInput.topicId = #{opts[:topicId].to_i};")
       end
 
-      context.eval("__optInput.userId = #{opts[:user_id].to_i};") if opts[:user_id]
-
-      context.eval("__optInput.getURL = __getURL;")
-      context.eval("__optInput.getCurrentUser = __getCurrentUser;")
-      context.eval("__optInput.lookupAvatar = __lookupAvatar;")
-      context.eval("__optInput.getTopicInfo = __getTopicInfo;")
-      context.eval("__optInput.categoryHashtagLookup = __categoryLookup;")
-      context.eval("__optInput.mentionLookup = __mentionLookup;")
-
       custom_emoji = {}
       Emoji.custom.map { |e| custom_emoji[e.name] = e.url }
-      context.eval("__optInput.customEmoji = #{custom_emoji.to_json};")
 
-      context.eval('__textOptions = __buildOptions(__optInput);')
+      buffer = <<~JS
+        __optInput = {};
+        __optInput.siteSettings = #{SiteSetting.client_settings_json};
+        __paths = #{paths.to_json};
+        __optInput.getURL = __getURL;
+        __optInput.getCurrentUser = __getCurrentUser;
+        __optInput.lookupAvatar = __lookupAvatar;
+        __optInput.getTopicInfo = __getTopicInfo;
+        __optInput.categoryHashtagLookup = __categoryLookup;
+        __optInput.mentionLookup = __mentionLookup;
+        __optInput.customEmoji = #{custom_emoji.to_json};
+      JS
+
+      if opts[:topicId]
+        buffer << "__optInput.topicId = #{opts[:topicId].to_i};\n"
+      end
+
+      if opts[:user_id]
+        buffer << "__optInput.userId = #{opts[:user_id].to_i};\n"
+      end
+
+      buffer << "__textOptions = __buildOptions(__optInput);\n"
 
       # Be careful disabling sanitization. We allow for custom emails
       if opts[:sanitize] == false
-        context.eval('__textOptions.sanitize = false;')
+        buffer << ('__textOptions.sanitize = false;')
       end
 
-      opts = context.eval("__pt = new __PrettyText(__textOptions);")
+      buffer << ("__pt = new __PrettyText(__textOptions);")
+      opts = context.eval(buffer)
 
       DiscourseEvent.trigger(:markdown_context, context)
       baked = context.eval("__pt.cook(#{text.inspect})")
     end
 
-    if baked.blank? && !(opts || {})[:skip_blank_test]
+    # if baked.blank? && !(opts || {})[:skip_blank_test]
-      # we may have a js engine issue
+    #   # we may have a js engine issue
-      test = markdown("a", skip_blank_test: true)
+    #   test = markdown("a", skip_blank_test: true)
-      if test.blank?
+    #   if test.blank?
-        Rails.logger.warn("Markdown engine appears to have crashed, resetting context")
+    #     Rails.logger.warn("Markdown engine appears to have crashed, resetting context")
-        reset_context
+    #     reset_context
-        opts ||= {}
+    #     opts ||= {}
-        opts = opts.dup
+    #     opts = opts.dup
-        opts[:skip_blank_test] = true
+    #     opts[:skip_blank_test] = true
-        baked = markdown(text, opts)
+    #     baked = markdown(text, opts)
-      end
+    #   end
-    end
+    # end
 
     baked
   end

plugins/discourse-details/assets/javascripts/lib/discourse-markdown/details.js.es6

@@ -20,6 +20,23 @@ registerOption((siteSettings, opts) => {
   opts.features.details = true;
 });
 
+const rule = {
+  tag: 'details',
+  before: function(state, attrs) {
+    state.push('bbcode_open', 'details', 1);
+    state.push('bbcode_open', 'summary', 1);
+
+    let token = state.push('text', '', 0);
+    token.content = attrs['_default'] || '';
+
+    state.push('bbcode_close', 'summary', -1);
+  },
+
+  after: function(state) {
+    state.push('bbcode_close', 'details', -1);
+  }
+};
+
 export function setup(helper) {
   helper.whiteList([
     'summary',
@@ -29,5 +46,11 @@ export function setup(helper) {
     'details.elided'
   ]);
 
-  helper.addPreProcessor(text => replaceDetails(text));
+  if (helper.markdownIt) {
+    helper.registerPlugin(md => {
+      md.block.bbcode_ruler.push('details', rule);
+    });
+  } else {
+    helper.addPreProcessor(text => replaceDetails(text));
+  }
 }

plugins/poll/assets/javascripts/lib/discourse-markdown/poll.js.es6

@@ -14,6 +14,274 @@ registerOption((siteSettings, opts) => {
   opts.pollMaximumOptions = siteSettings.poll_maximum_options;
 });
 
+function getHelpText(count, min, max) {
+
+  // default values
+  if (isNaN(min) || min < 1) { min = 1; }
+  if (isNaN(max) || max > count) { max = count; }
+
+  // add some help text
+  let help;
+
+  if (max > 0) {
+    if (min === max) {
+      if (min > 1) {
+        help = I18n.t("poll.multiple.help.x_options", { count: min });
+      }
+    } else if (min > 1) {
+      if (max < count) {
+        help = I18n.t("poll.multiple.help.between_min_and_max_options", { min: min, max: max });
+      } else {
+        help = I18n.t("poll.multiple.help.at_least_min_options", { count: min });
+      }
+    } else if (max <= count) {
+      help = I18n.t("poll.multiple.help.up_to_max_options", { count: max });
+    }
+  }
+
+  return help;
+}
+
+function replaceToken(tokens, target, list) {
+  let pos = tokens.indexOf(target);
+  tokens.splice(pos, 1, ...list);
+  list[0].map = target.map;
+}
+
+// analyzes the block to that we have poll options
+function getListItems(tokens, startToken) {
+
+  let i = tokens.length-1;
+  let listItems = [];
+  let buffer = [];
+
+  for(;tokens[i]!==startToken;i--) {
+    if (i === 0) {
+      return;
+    }
+
+    let token = tokens[i];
+    if (token.level === 0) {
+      if (token.tag !== 'ol' && token.tag !== 'ul') {
+        return;
+      }
+    }
+
+    if (token.level === 1 && token.nesting === 1) {
+      if (token.tag === 'li') {
+        listItems.push([token, buffer.reverse().join(' ')]);
+      } else {
+        return;
+      }
+    }
+
+    if (token.level === 1 && token.nesting === 1 && token.tag === 'li') {
+      buffer = [];
+    } else {
+      if (token.type === 'text' || token.type === 'inline') {
+        buffer.push(token.content);
+      }
+    }
+  }
+
+  return listItems.reverse();
+}
+
+function invalidPoll(state, tag) {
+  let token = state.push('text', '', 0);
+  token.content = '[/' + tag + ']';
+}
+
+const rule = {
+  tag: 'poll',
+
+  before: function(state, attrs, md, raw){
+    let token = state.push('text', '', 0);
+    token.content = raw;
+    token.bbcode_attrs = attrs;
+    token.bbcode_type = 'poll_open';
+  },
+
+  after: function(state, openToken, md, raw) {
+
+    let items = getListItems(state.tokens, openToken);
+
+    const attrs = openToken.bbcode_attrs;
+
+    // default poll attributes
+    const attributes = [["class", "poll"]];
+    attributes.push([DATA_PREFIX + "status", "open"]);
+
+    WHITELISTED_ATTRIBUTES.forEach(name => {
+      if (attrs[name]) {
+        attributes[DATA_PREFIX + name] = attrs[name];
+      }
+    });
+
+    if (!attrs.name) {
+      attributes.push([DATA_PREFIX + "name", DEFAULT_POLL_NAME]);
+    }
+
+    // we might need these values later...
+    let min = parseInt(attributes[DATA_PREFIX + "min"], 10);
+    let max = parseInt(attributes[DATA_PREFIX + "max"], 10);
+    let step = parseInt(attributes[DATA_PREFIX + "step"], 10);
+
+    let header = [];
+
+    let token = new state.Token('poll_open', 'div', 1);
+    token.block = true;
+    token.attrs = attributes;
+    header.push(token);
+
+    token = new state.Token('poll_open', 'div', 1);
+    token.block = true;
+    header.push(token);
+
+    token = new state.Token('poll_open', 'div', 1);
+    token.attrs = [['class', 'poll-container']];
+
+    header.push(token);
+
+    // generate the options when the type is "number"
+    if (attributes[DATA_PREFIX + "type"] === "number") {
+      // default values
+      if (isNaN(min)) { min = 1; }
+      if (isNaN(max)) { max = md.options.discourse.pollMaximumOptions; }
+      if (isNaN(step)) { step = 1; }
+
+      if (items.length > 0) {
+        return invalidPoll(state, raw);
+      }
+
+      // dynamically generate options
+      token = new state.Token('bullet_list_open', 'ul', 1);
+      header.push(token);
+
+      for (let o = min; o <= max; o += step) {
+        token = new state.Token('list_item_open', '', 1);
+        items.push([token,  String(o)]);
+        header.push(token);
+
+        token = new state.Token('text', '', 0);
+        token.content = String(o);
+        header.push(token);
+
+        token = new state.Token('list_item_close', '', -1);
+        header.push(token);
+      }
+      token = new state.Token('bullet_item_close', '', -1);
+      header.push(token);
+    }
+
+    if (items.length < 2) {
+      return invalidPoll(state, raw);
+    }
+
+    // flag items so we add hashes
+    for (let o = 0; o < items.length; o++) {
+      token = items[o][0];
+      let text = items[o][1];
+
+      token.attrs = token.attrs || [];
+      let md5Hash = md5(JSON.stringify([text]));
+      token.attrs.push([DATA_PREFIX + 'option-id', md5Hash]);
+    }
+
+    replaceToken(state.tokens, openToken, header);
+
+    state.push('poll_close', 'div', -1);
+
+    token = state.push('poll_open', 'div', 1);
+    token.attrs = [['class', 'poll-info']];
+
+    state.push('paragraph_open', 'p', 1);
+
+    token = state.push('span_open', 'span', 1);
+    token.block = false;
+    token.attrs = [['class', 'info-number']];
+    token = state.push('text', '', 0);
+    token.content = '0';
+    state.push('span_close', 'span', -1);
+
+    token = state.push('span_open', 'span', 1);
+    token.block = false;
+    token.attrs = [['class', 'info-text']];
+    token = state.push('text', '', 0);
+    token.content = I18n.t("poll.voters", { count: 0 });
+    state.push('span_close', 'span', -1);
+
+    state.push('paragraph_close', 'p', -1);
+
+    // multiple help text
+    if (attributes[DATA_PREFIX + "type"] === "multiple") {
+      let help = getHelpText(items.length, min, max);
+      if (help) {
+        state.push('paragraph_open', 'p', 1);
+        token = state.push('html_inline', '', 0);
+        token.content = help;
+        state.push('paragraph_close', 'p', -1);
+      }
+    }
+
+    if (attributes[DATA_PREFIX + 'public'] === 'true') {
+      state.push('paragraph_open', 'p', 1);
+      token = state.push('text', '', 0);
+      token.content = I18n.t('poll.public.title');
+      state.push('paragraph_close', 'p', -1);
+    }
+
+    state.push('poll_close', 'div', -1);
+    state.push('poll_close', 'div', -1);
+
+    token = state.push('poll_open', 'div', 1);
+    token.attrs = [['class', 'poll-buttons']];
+
+    if (attributes[DATA_PREFIX + 'type'] === 'multiple') {
+      token = state.push('link_open', 'a', 1);
+      token.block = false;
+      token.attrs = [
+        ['class', 'button cast-votes'],
+        ['title', I18n.t('poll.cast-votes.title')]
+      ];
+
+      token = state.push('text', '', 0);
+      token.content = I18n.t('poll.cast-votes.label');
+
+      state.push('link_close', 'a', -1);
+    }
+
+    token = state.push('link_open', 'a', 1);
+    token.block = false;
+    token.attrs = [
+      ['class', 'button toggle-results'],
+      ['title', I18n.t('poll.show-results.title')]
+    ];
+
+    token = state.push('text', '', 0);
+    token.content = I18n.t("poll.show-results.label");
+
+    state.push('link_close', 'a', -1);
+
+    state.push('poll_close', 'div', -1);
+    state.push('poll_close', 'div', -1);
+  }
+};
+
+function newApiInit(helper) {
+  helper.registerOptions((opts, siteSettings) => {
+    const currentUser = (opts.getCurrentUser && opts.getCurrentUser(opts.userId)) || opts.currentUser;
+    const staff = currentUser && currentUser.staff;
+
+    opts.features.poll = !!siteSettings.poll_enabled || staff;
+    opts.pollMaximumOptions = siteSettings.poll_maximum_options;
+  });
+
+  helper.registerPlugin(md => {
+    md.block.bbcode_ruler.push('poll', rule);
+  });
+}
+
 export function setup(helper) {
   helper.whiteList([
     'div.poll',
@@ -28,6 +296,11 @@ export function setup(helper) {
     'li[data-*]'
   ]);
 
+  if (helper.markdownIt) {
+    newApiInit(helper);
+    return;
+  }
+
   helper.replaceBlock({
     start: /\[poll((?:\s+\w+=[^\s\]]+)*)\]([\s\S]*)/igm,
     stop: /\[\/poll\]/igm,

plugins/poll/spec/lib/pretty_text_spec.rb

@@ -0,0 +1,101 @@
+require 'rails_helper'
+require 'html_normalize'
+
+describe PrettyText do
+
+  def n(html)
+    HtmlNormalize.normalize(html)
+  end
+
+  context 'markdown it' do
+    before do
+      SiteSetting.enable_experimental_markdown_it = true
+    end
+
+    it 'works correctly for new vs old engine with trivial cases' do
+      md = <<~MD
+        [poll]
+        1. test 1
+        2. test 2
+        [/poll]
+      MD
+
+      new_engine = n(PrettyText.cook(md))
+
+      SiteSetting.enable_experimental_markdown_it = false
+      old_engine = n(PrettyText.cook(md))
+
+      expect(new_engine).to eq(old_engine)
+    end
+
+    it 'does not break poll options when going from loose to tight' do
+      md = <<~MD
+        [poll type=multiple]
+        1. test 1 :) <b>test</b>
+        2. test 2
+        [/poll]
+      MD
+
+      tight_cooked = PrettyText.cook(md)
+
+      md = <<~MD
+        [poll type=multiple]
+
+        1. test 1 :) <b>test</b>
+
+        2. test 2
+
+        [/poll]
+      MD
+
+      loose_cooked = PrettyText.cook(md)
+
+      tight_hashes = tight_cooked.scan(/data-poll-option-id=['"]([^'"]+)/)
+      loose_hashes = loose_cooked.scan(/data-poll-option-id=['"]([^'"]+)/)
+
+      expect(tight_hashes).to eq(loose_hashes)
+    end
+
+    it 'can correctly cook polls' do
+      md = <<~MD
+        [poll type=multiple]
+        1. test 1 :) <b>test</b>
+        2. test 2
+        [/poll]
+      MD
+
+      cooked = PrettyText.cook md
+
+      expected = <<~MD
+        <div class="poll" data-poll-status="open" data-poll-name="poll">
+        <div>
+        <div class="poll-container">
+        <ol>
+        <li data-poll-option-id='b6475cbf6acb8676b20c60582cfc487a'>test 1 <img alt=':slight_smile:' class='emoji' src='/images/emoji/emoji_one/slight_smile.png?v=5' title=':slight_smile:'> <b>test</b>
+        </li>
+        <li data-poll-option-id='7158af352698eb1443d709818df097d4'>test 2</li>
+        </li>
+        </ol>
+        </div>
+        <div class="poll-info">
+        <p>
+        <span class="info-number">0</span>
+        <span class="info-text">voters</span>
+        </p>
+        <p>
+        Choose up to <strong>2</strong> options</p>
+        </div>
+        </div>
+        <div class="poll-buttons">
+        <a title="Cast your votes">Vote now!</a>
+        <a title="Display the poll results">Show results</a>
+        </div>
+        </div>
+      MD
+
+      # note, hashes should remain stable even if emoji changes cause text content is hashed
+      expect(n cooked).to eq(n expected)
+
+    end
+  end
+end

script/benchmarks/markdown/.gitignore

@@ -0,0 +1 @@
+tmp/*

script/benchmarks/markdown/bench.rb

@@ -0,0 +1,54 @@
+require 'benchmark/ips'
+require File.expand_path('../../../../config/environment', __FILE__)
+
+
+tests = [
+  ["tiny post", "**hello**"],
+  ["giant post", File.read("giant_post.md")],
+  ["most features", File.read("most_features.md")],
+  ["lots of mentions", File.read("lots_of_mentions.md")]
+]
+
+SiteSetting.enable_experimental_markdown_it = true
+PrettyText.cook("")
+PrettyText.v8.eval("window.commonmark = window.markdownit('commonmark')")
+
+# Benchmark.ips do |x|
+#   x.report("markdown") do
+#     PrettyText.markdown("x")
+#   end
+#
+#   x.report("cook") do
+#     PrettyText.cook("y")
+#   end
+# end
+#
+# exit
+
+Benchmark.ips do |x|
+  [true,false].each do |sanitize|
+    {
+      "markdown js" =>
+        lambda{SiteSetting.enable_experimental_markdown_it = false},
+
+      "markdown it" =>
+        lambda{SiteSetting.enable_experimental_markdown_it = true}
+    }.each do |name, before|
+      before.call
+
+      tests.each do |test, text|
+        x.report("#{name} #{test} sanitize: #{sanitize}") do
+          PrettyText.markdown(text, sanitize: sanitize)
+        end
+      end
+    end
+  end
+
+
+  tests.each do |test, text|
+    x.report("markdown it no extensions commonmark #{test}") do
+      PrettyText.v8.eval("window.commonmark.render(#{text.inspect})")
+    end
+  end
+end
+

script/benchmarks/markdown/giant_post.md

@@ -0,0 +1,583 @@
+    [discourse]# ./launcher bootstrap app
+    which: no docker.io in (/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin)
+    
+    WARNING: We are about to start downloading the Discourse base image
+    This process may take anywhere between a few minutes to an hour, depending on your network speed
+    
+    Please be patient
+    
+    Unable to find image 'samsaffron/discourse:1.0.13' locally
+    1.0.13: Pulling from samsaffron/discourse
+    ........
+    
+    Fast-forward
+     .travis.yml                                        |   3 -
+     Gemfile                                            |  25 +-
+     Gemfile.lock                                       | 298 ++++++-----
+     README.md                                          |  23 +-
+     .../admin/components/embedding-setting.js.es6      |   5 +
+     .../screened_ip_address_form_component.js          |  23 +-
+     .../admin/controllers/admin-site-settings.js.es6   |  28 +-
+     .../admin/controllers/admin-user-badges.js.es6     |   4 +-
+     .../javascripts/admin/models/admin-user.js.es6     |  12 +-
+     .../javascripts/admin/models/staff_action_log.js   |   3 +-
+     app/assets/javascripts/admin/templates/admin.hbs   |   2 +-
+     .../templates/components/embedding-setting.hbs     |   2 +-
+     .../admin/templates/components/site-setting.hbs    |   2 +-
+     .../javascripts/admin/templates/dashboard.hbs      |   2 +-
+     .../javascripts/admin/templates/embedding.hbs      |   9 +-
+     .../admin/templates/modal/admin_agree_flag.hbs     |   6 +-
+     .../admin/templates/modal/admin_delete_flag.hbs    |   2 +-
+     .../javascripts/admin/templates/plugins-index.hbs  |  19 +-
+     .../admin/templates/site-settings-category.hbs     |   2 +-
+     .../javascripts/admin/templates/site-settings.hbs  |   6 +-
+     .../javascripts/admin/templates/site-text-edit.hbs |   2 +-
+     .../javascripts/admin/templates/user-index.hbs     |  36 +-
+     app/assets/javascripts/discourse.js                |   2 +-
+     .../javascripts/discourse/adapters/rest.js.es6     |  14 +-
+     .../discourse/components/actions-summary.js.es6    |   4 +-
+     .../discourse/components/category-chooser.js.es6   |   2 +-
+     .../discourse/components/d-editor-modal.js.es6     |  52 ++
+     .../discourse/components/d-editor.js.es6           | 263 ++++++++++
+     .../javascripts/discourse/components/d-link.js.es6 |   7 +-
+     .../discourse/components/date-picker.js.es6        |  22 +-
+     .../components/desktop-notification-config.js.es6  |  60 ++-
+     .../discourse/components/image-uploader.js.es6     |   9 +-
+     .../discourse/components/menu-panel.js.es6         |   3 +-
+     .../discourse/components/notification-item.js.es6  |  10 +-
+     .../discourse/components/pagedown-editor.js.es6    |  23 -
+     .../discourse/components/post-gutter.js.es6        |  53 +-
+     .../discourse/components/post-menu.js.es6          |   9 +-
+     .../components/private-message-map.js.es6          |   2 +-
+     .../discourse/components/who-liked.js.es6          |   2 +-
+     .../discourse/controllers/change-owner.js.es6      |   5 +-
+     .../discourse/controllers/composer.js.es6          |   4 +-
+     .../controllers/discovery-sortable.js.es6          |   6 +-
+     .../discourse/controllers/discovery/topics.js.es6  |  11 +-
+     .../discourse/controllers/edit-category.js.es6     |  17 +-
+     .../discourse/controllers/full-page-search.js.es6  |   2 +-
+     .../javascripts/discourse/controllers/login.js.es6 |  37 +-
+     .../discourse/controllers/quote-button.js.es6      |  38 +-
+     .../javascripts/discourse/controllers/topic.js.es6 |  52 +-
+     .../discourse/controllers/user-card.js.es6         |   1 +
+     .../javascripts/discourse/controllers/user.js.es6  |  58 ++-
+     .../javascripts/discourse/dialects/dialect.js      |   2 +-
+     .../discourse/dialects/quote_dialect.js            |  33 +-
+     .../discourse/initializers/enable-emoji.js.es6     |  10 +-
+     .../discourse/initializers/load-all-helpers.js.es6 |  17 +-
+     .../subscribe-user-notifications.js.es6            |  12 +-
+     .../javascripts/discourse/lib/Markdown.Editor.js   |   3 +-
+     .../javascripts/discourse/lib/autocomplete.js.es6  |  15 +-
+     .../discourse/lib/desktop-notifications.js.es6     |  11 +-
+     .../discourse/lib/emoji/emoji-groups.js.es6        |  57 ++
+     .../discourse/lib/emoji/emoji-toolbar.js.es6       | 259 ++++------
+     .../discourse/lib/key-value-store.js.es6           |  16 +-
+     .../discourse/lib/keyboard-shortcuts.js.es6        |  15 +-
+     .../javascripts/discourse/models/post.js.es6       |  12 +-
+     .../javascripts/discourse/models/topic-list.js.es6 |  56 +-
+     .../javascripts/discourse/models/topic.js.es6      |   2 +
+     .../javascripts/discourse/models/user.js.es6       |   4 +
+     .../pre-initializers/dynamic-route-builders.js.es6 |  47 +-
+     .../pre-initializers/sniff-capabilities.js.es6     |  21 +-
+     .../discourse/routes/app-route-map.js.es6          |  31 +-
+     .../discourse/routes/badges-show.js.es6            |   2 +-
+     .../discourse/routes/build-category-route.js.es6   |  44 +-
+     .../discourse/routes/build-static-route.js.es6     |  18 +
+     .../discourse/routes/build-topic-route.js.es6      |  20 +-
+     .../javascripts/discourse/routes/discovery.js.es6  |   4 +
+     .../discourse/routes/forgot-password.js.es6        |  27 +-
+     .../javascripts/discourse/routes/login.js.es6      |  31 +-
+     .../templates/components/category-unread.hbs       |   2 +-
+     .../templates/components/d-editor-modal.hbs        |   7 +
+     .../discourse/templates/components/d-editor.hbs    |  32 ++
+     .../templates/components/edit-category-images.hbs  |   2 +-
+     .../components/edit-category-topic-template.hbs    |   2 +-
+     .../discourse/templates/components/home-logo.hbs   |   3 -
+     .../templates/components/pagedown-editor.hbs       |   4 -
+     .../discourse/templates/components/user-menu.hbs   |   4 +-
+     .../javascripts/discourse/templates/composer.hbs   |   4 +-
+     .../discourse/templates/discovery/topics.hbs       |  23 +-
+     .../templates/emoji-selector-autocomplete.raw.hbs  |  10 +-
+     .../javascripts/discourse/templates/header.hbs     |   2 +-
+     .../discourse/templates/login-preferences.hbs      |   8 +
+     .../templates/mobile/discovery/topics.hbs          |  17 +-
+     .../templates/mobile/list/topic_list_item.raw.hbs  |  49 +-
+     .../discourse/templates/modal/create-account.hbs   |  17 +-
+     .../discourse/templates/modal/dismiss-read.hbs     |  10 +
+     .../discourse/templates/navigation/category.hbs    |   7 +-
+     .../javascripts/discourse/templates/post.hbs       |  13 +-
+     .../discourse/templates/queued-posts.hbs           |   2 +-
+     .../javascripts/discourse/templates/user/about.hbs |   2 +-
+     .../discourse/templates/user/preferences.hbs       |   3 +-
+     .../javascripts/discourse/templates/user/user.hbs  |  13 +-
+     .../discourse/views/cloaked-collection.js.es6      |   3 +
+     .../javascripts/discourse/views/composer.js.es6    |  94 ++--
+     .../discourse/views/embedded-post.js.es6           |   2 +
+     app/assets/javascripts/discourse/views/post.js.es6 |  13 +-
+     .../discourse/views/quote-button.js.es6            |  26 +-
+     .../discourse/views/topic-entrance.js.es6          |  21 +-
+     .../discourse/views/upload-selector.js.es6         |   6 +-
+     app/assets/javascripts/main_include.js             |   6 +-
+     app/assets/stylesheets/common.scss                 |   1 +
+     .../stylesheets/common/admin/admin_base.scss       |   8 -
+     app/assets/stylesheets/common/base/compose.scss    |  12 +-
+     app/assets/stylesheets/common/base/discourse.scss  |  18 -
+     app/assets/stylesheets/common/base/emoji.scss      |   6 +-
+     app/assets/stylesheets/common/base/header.scss     |   6 -
+     app/assets/stylesheets/common/base/modal.scss      |   4 -
+     app/assets/stylesheets/common/base/onebox.scss     |   2 +-
+     .../stylesheets/common/base/topic-admin-menu.scss  |   2 +-
+     app/assets/stylesheets/common/base/topic-post.scss |   2 -
+     app/assets/stylesheets/common/base/topic.scss      |   8 +-
+     app/assets/stylesheets/common/base/upload.scss     |   8 +-
+     .../stylesheets/common/components/badges.css.scss  |   8 +-
+     app/assets/stylesheets/common/d-editor.scss        |  94 ++++
+     app/assets/stylesheets/desktop/compose.scss        |  10 +-
+     app/assets/stylesheets/desktop/discourse.scss      |   2 +-
+     app/assets/stylesheets/desktop/topic-post.scss     |   8 +-
+     app/assets/stylesheets/desktop/topic.scss          |   5 +-
+     app/assets/stylesheets/desktop/upload.scss         |   2 +-
+     app/assets/stylesheets/desktop/user-card.scss      |   2 +-
+     app/assets/stylesheets/desktop/user.scss           |   8 -
+     app/assets/stylesheets/mobile.scss                 |   1 +
+     app/assets/stylesheets/mobile/alert.scss           |   2 +
+     app/assets/stylesheets/mobile/banner.scss          |   4 +-
+     app/assets/stylesheets/mobile/discourse.scss       |   3 +-
+     app/assets/stylesheets/mobile/emoji.scss           |   3 +
+     app/assets/stylesheets/mobile/header.scss          |   4 +-
+     app/assets/stylesheets/mobile/topic-list.scss      |  32 +-
+     app/assets/stylesheets/mobile/topic-post.scss      |  16 +-
+     app/assets/stylesheets/mobile/topic.scss           |   3 +-
+     app/assets/stylesheets/mobile/user.scss            |   8 +-
+     app/controllers/admin/diagnostics_controller.rb    |  13 +
+     app/controllers/admin/email_controller.rb          |   6 +
+     app/controllers/admin/embedding_controller.rb      |   4 +
+     app/controllers/application_controller.rb          |  15 +-
+     app/controllers/categories_controller.rb           |  32 +-
+     app/controllers/list_controller.rb                 |  29 +-
+     app/controllers/manifest_json_controller.rb        |  15 +
+     app/controllers/permalinks_controller.rb           |   2 +-
+     app/controllers/post_action_users_controller.rb    |  22 +
+     app/controllers/post_actions_controller.rb         |  13 +-
+     app/controllers/posts_controller.rb                |  30 +-
+     app/controllers/robots_txt_controller.rb           |   9 +-
+     app/controllers/topics_controller.rb               |   2 +-
+     .../users/omniauth_callbacks_controller.rb         |  25 +-
+     app/controllers/users_controller.rb                |  26 +-
+     app/helpers/application_helper.rb                  |  22 +-
+     app/jobs/regular/post_alert.rb                     |   2 +-
+     app/jobs/regular/process_post.rb                   |   4 +-
+     app/jobs/scheduled/periodical_updates.rb           |   4 +-
+     app/mailers/user_notifications.rb                  |   1 +
+     app/models/admin_dashboard_data.rb                 |   1 +
+     app/models/anon_site_json_cache_observer.rb        |  12 +
+     app/models/badge.rb                                |  28 +-
+     app/models/category.rb                             |  19 +-
+     app/models/category_group.rb                       |   2 +
+     app/models/color_scheme.rb                         |  22 +-
+     app/models/group.rb                                |   7 +
+     app/models/permalink.rb                            |   2 +-
+     app/models/post.rb                                 |  21 +-
+     app/models/post_action.rb                          |   6 +-
+     app/models/post_action_type.rb                     |  10 +
+     app/models/post_analyzer.rb                        |   2 +-
+     app/models/report.rb                               |  13 +-
+     app/models/screened_ip_address.rb                  |   1 +
+     app/models/site.rb                                 |  63 ++-
+     app/models/topic.rb                                |  65 ++-
+     app/models/topic_featured_users.rb                 |   4 +-
+     app/models/topic_link.rb                           |  14 +-
+     app/models/topic_link_click.rb                     |   2 +-
+     app/models/topic_tracking_state.rb                 |  16 +-
+     app/models/topic_user.rb                           |  15 -
+     app/models/upload.rb                               |   5 +-
+     app/models/user.rb                                 |   4 +-
+     app/models/user_history.rb                         |  13 +-
+     app/models/user_profile.rb                         |   2 +
+     app/models/user_profile_view.rb                    |  47 ++
+     app/serializers/admin_detailed_user_serializer.rb  |   2 +-
+     app/serializers/application_serializer.rb          |  25 +
+     app/serializers/badge_serializer.rb                |  15 +-
+     app/serializers/basic_post_serializer.rb           |   6 +-
+     app/serializers/post_action_user_serializer.rb     |   7 +-
+     app/serializers/post_serializer.rb                 |   2 +-
+     app/serializers/site_serializer.rb                 |  26 +-
+     app/serializers/topic_view_serializer.rb           |   8 +-
+     app/serializers/user_history_serializer.rb         |   1 +
+     app/serializers/user_serializer.rb                 |   7 +-
+     app/services/badge_granter.rb                      |  10 +-
+     app/services/post_alerter.rb                       |   3 +-
+     app/services/random_topic_selector.rb              |  16 +-
+     app/services/staff_action_logger.rb                |  62 +++
+     app/views/common/_discourse_javascript.html.erb    |   5 +
+     app/views/layouts/_head.html.erb                   |  11 +-
+     app/views/layouts/application.html.erb             |   1 +
+     app/views/list/list.erb                            |   3 +-
+     app/views/posts/latest.rss.erb                     |   2 +-
+     app/views/topics/plain.html.erb                    |   4 +-
+     app/views/topics/show.html.erb                     |   4 +-
+     app/views/user_notifications/digest.html.erb       |   4 +-
+     .../users/omniauth_callbacks/complete.html.erb     |   4 +-
+     app/views/users/show.html.erb                      |   4 +-
+     config/application.rb                              |   5 +-
+     config/database.yml                                |   2 +
+     config/discourse_defaults.conf                     |   8 +-
+     config/environments/production.rb                  |   2 +-
+     config/environments/profile.rb                     |   2 +-
+     config/environments/test.rb                        |   2 +-
+     config/initializers/04-message_bus.rb              |   2 +-
+     config/initializers/i18n.rb                        |  15 +-
+     config/locales/client.ar.yml                       |  45 +-
+     config/locales/client.bs_BA.yml                    |  28 +-
+     config/locales/client.cs.yml                       |   9 -
+     config/locales/client.da.yml                       | 168 +++++-
+     config/locales/client.de.yml                       |  65 ++-
+     config/locales/client.en.yml                       |  42 +-
+     config/locales/client.es.yml                       |  35 +-
+     config/locales/client.fa_IR.yml                    |   9 -
+     config/locales/client.fi.yml                       | 125 ++++-
+     config/locales/client.fr.yml                       |  56 +-
+     config/locales/client.he.yml                       | 575 +++++++++++----------
+     config/locales/client.it.yml                       | 133 +++--
+     config/locales/client.ja.yml                       |   9 -
+     config/locales/client.ko.yml                       | 496 +++++++++---------
+     config/locales/client.nb_NO.yml                    |  16 +-
+     config/locales/client.nl.yml                       |  43 +-
+     config/locales/client.pl_PL.yml                    |  66 ++-
+     config/locales/client.pt.yml                       |  37 +-
+     config/locales/client.pt_BR.yml                    | 215 +++++++-
+     config/locales/client.ro.yml                       |   8 -
+     config/locales/client.ru.yml                       | 172 ++++--
+     config/locales/client.sq.yml                       |   9 -
+     config/locales/client.sv.yml                       |   9 -
+     config/locales/client.te.yml                       |   8 -
+     config/locales/client.tr_TR.yml                    |  11 +-
+     config/locales/client.uk.yml                       |   6 -
+     config/locales/client.zh_CN.yml                    |  22 +-
+     config/locales/client.zh_TW.yml                    |  11 +-
+     config/locales/server.ar.yml                       |  61 ++-
+     config/locales/server.bs_BA.yml                    |   2 -
+     config/locales/server.cs.yml                       |   1 -
+     config/locales/server.da.yml                       | 107 +++-
+     config/locales/server.de.yml                       | 238 ++++++++-
+     config/locales/server.en.yml                       |  37 +-
+     config/locales/server.es.yml                       | 134 ++++-
+     config/locales/server.fa_IR.yml                    |   2 -
+     config/locales/server.fi.yml                       |  85 ++-
+     config/locales/server.fr.yml                       |   4 -
+     config/locales/server.he.yml                       |  21 +-
+     config/locales/server.it.yml                       |  44 +-
+     config/locales/server.ja.yml                       |   2 -
+     config/locales/server.ko.yml                       |   2 -
+     config/locales/server.nb_NO.yml                    |   1 -
+     config/locales/server.nl.yml                       |   9 +-
+     config/locales/server.pl_PL.yml                    | 236 ++++++++-
+     config/locales/server.pt.yml                       |  31 +-
+     config/locales/server.pt_BR.yml                    |  14 +-
+     config/locales/server.ru.yml                       |  48 +-
+     config/locales/server.sq.yml                       |   2 -
+     config/locales/server.sv.yml                       |   3 +-
+     config/locales/server.te.yml                       |   1 -
+     config/locales/server.tr_TR.yml                    |   4 -
+     config/locales/server.zh_CN.yml                    |  23 +-
+     config/locales/server.zh_TW.yml                    |  38 +-
+     config/routes.rb                                   |   7 +-
+     config/site_settings.yml                           |  26 +-
+     .../20150914021445_create_user_profile_views.rb    |  15 +
+     .../20150914034541_add_views_to_user_profile.rb    |   5 +
+     ...0917071017_add_category_id_to_user_histories.rb |   6 +
+     .../20150924022040_add_fancy_title_to_topic.rb     |   5 +
+     .../20150925000915_exclude_whispers_from_badges.rb |  22 +
+     docs/INSTALL-cloud.md                              |  59 +--
+     docs/VAGRANT.md                                    |   2 +-
+     lib/cooked_post_processor.rb                       |  27 +-
+     lib/discourse.rb                                   |   6 +-
+     lib/discourse_redis.rb                             |   7 +-
+     lib/edit_rate_limiter.rb                           |   6 +-
+     lib/email.rb                                       |   9 +-
+     lib/email/message_builder.rb                       |   2 +-
+     lib/email/renderer.rb                              |   4 +-
+     lib/email/sender.rb                                |   4 +-
+     lib/email/styles.rb                                |  14 +-
+     lib/file_store/base_store.rb                       |   6 +-
+     lib/freedom_patches/i18n_fallbacks.rb              |   2 +
+     lib/freedom_patches/pool_drainer.rb                |  11 +-
+     lib/guardian.rb                                    |   2 +-
+     lib/guardian/category_guardian.rb                  |   7 +-
+     lib/guardian/post_guardian.rb                      |   2 +-
+     lib/html_prettify.rb                               | 407 +++++++++++++++
+     lib/onebox/engine/discourse_local_onebox.rb        |  11 +-
+     lib/oneboxer.rb                                    |  20 +-
+     lib/plugin/auth_provider.rb                        |  21 +-
+     lib/plugin/instance.rb                             |  10 +-
+     lib/post_creator.rb                                |  21 +-
+     lib/post_revisor.rb                                |  10 +-
+     lib/pretty_text.rb                                 |  21 +
+     lib/rate_limiter.rb                                |  11 +-
+     lib/rate_limiter/limit_exceeded.rb                 |  25 +-
+     lib/search.rb                                      |   2 +-
+     lib/site_setting_extension.rb                      |   1 +
+     lib/tasks/assets.rake                              |  27 +-
+     lib/tasks/db.rake                                  |   2 +
+     lib/tasks/posts.rake                               |  20 +-
+     lib/topic_creator.rb                               |   4 +-
+     lib/topic_query.rb                                 |   9 +-
+     lib/topic_view.rb                                  |   6 +-
+     lib/version.rb                                     |   4 +-
+     plugins/poll/config/locales/client.ar.yml          |  74 ++-
+     plugins/poll/config/locales/client.bs_BA.yml       |  13 +-
+     plugins/poll/config/locales/client.cs.yml          |   3 -
+     plugins/poll/config/locales/client.da.yml          |  14 +-
+     plugins/poll/config/locales/client.de.yml          |  12 +-
+     plugins/poll/config/locales/client.en.yml          |  12 +-
+     plugins/poll/config/locales/client.es.yml          |  12 +-
+     plugins/poll/config/locales/client.fa_IR.yml       |   3 -
+     plugins/poll/config/locales/client.fi.yml          |  12 +-
+     plugins/poll/config/locales/client.fr.yml          |   3 -
+     plugins/poll/config/locales/client.he.yml          |  12 +-
+     plugins/poll/config/locales/client.id.yml          |   3 -
+     plugins/poll/config/locales/client.it.yml          |  12 +-
+     plugins/poll/config/locales/client.ja.yml          |   3 -
+     plugins/poll/config/locales/client.ko.yml          |   3 -
+     plugins/poll/config/locales/client.nb_NO.yml       |   3 -
+     plugins/poll/config/locales/client.nl.yml          |   9 +-
+     plugins/poll/config/locales/client.pl_PL.yml       |  15 +-
+     plugins/poll/config/locales/client.pt.yml          |  12 +-
+     plugins/poll/config/locales/client.pt_BR.yml       |  18 +-
+     plugins/poll/config/locales/client.ro.yml          |   3 -
+     plugins/poll/config/locales/client.ru.yml          |  46 +-
+     plugins/poll/config/locales/client.sq.yml          |   3 -
+     plugins/poll/config/locales/client.sv.yml          |   3 -
+     plugins/poll/config/locales/client.tr_TR.yml       |   3 -
+     plugins/poll/config/locales/client.zh_CN.yml       |   9 +-
+     plugins/poll/config/locales/server.ar.yml          |  54 +-
+     plugins/poll/config/locales/server.bs_BA.yml       |   5 +-
+     plugins/poll/config/locales/server.cs.yml          |   2 -
+     plugins/poll/config/locales/server.da.yml          |   8 +-
+     plugins/poll/config/locales/server.de.yml          |   8 +-
+     plugins/poll/config/locales/server.en.yml          |  11 +-
+     plugins/poll/config/locales/server.es.yml          |   8 +-
+     plugins/poll/config/locales/server.fa_IR.yml       |   2 -
+     plugins/poll/config/locales/server.fi.yml          |   8 +-
+     plugins/poll/config/locales/server.fr.yml          |   2 -
+     plugins/poll/config/locales/server.he.yml          |   8 +-
+     plugins/poll/config/locales/server.it.yml          |   8 +-
+     plugins/poll/config/locales/server.ja.yml          |   4 +-
+     plugins/poll/config/locales/server.ko.yml          |   2 -
+     plugins/poll/config/locales/server.nb_NO.yml       |   2 -
+     plugins/poll/config/locales/server.nl.yml          |   8 +-
+     plugins/poll/config/locales/server.pl_PL.yml       |  10 +-
+     plugins/poll/config/locales/server.pt.yml          |   8 +-
+     plugins/poll/config/locales/server.pt_BR.yml       |  11 +-
+     plugins/poll/config/locales/server.ru.yml          |  52 +-
+     plugins/poll/config/locales/server.sq.yml          |   2 -
+     plugins/poll/config/locales/server.sv.yml          |   2 -
+     plugins/poll/config/locales/server.tr_TR.yml       |   2 -
+     plugins/poll/config/locales/server.zh_CN.yml       |   6 +-
+     .../db/migrate/20151016163051_merge_polls_votes.rb |  20 +
+     plugins/poll/plugin.rb                             |  62 ++-
+     .../poll/spec/controllers/posts_controller_spec.rb |  57 +-
+     public/403.ar.html                                 |   4 +-
+     public/403.it.html                                 |   2 +-
+     public/403.zh_CN.html                              |   2 +-
+     public/500.zh_CN.html                              |   2 +-
+     public/images/welcome/reply-post-2x.png            | Bin 549 -> 430 bytes
+     .../welcome/topic-notification-control-2x.png      | Bin 39580 -> 50219 bytes
+     public/javascripts/pikaday.js                      |   6 +-
+     script/import_scripts/base.rb                      |  21 +-
+     script/import_scripts/lithium.rb                   |  22 +-
+     script/import_scripts/mbox.rb                      |  52 +-
+     script/import_scripts/mybb.rb                      |  38 +-
+     .../import_scripts/phpbb3/database/database_3_0.rb |   2 +-
+     .../phpbb3/database/database_base.rb               |   2 +-
+     script/import_scripts/phpbb3/importer.rb           |   8 +-
+     .../phpbb3/importers/message_importer.rb           |  11 +-
+     .../phpbb3/importers/post_importer.rb              |   4 +
+     .../phpbb3/importers/user_importer.rb              |  11 +-
+     script/import_scripts/vbulletin.rb                 |   4 +-
+     spec/components/cooked_post_processor_spec.rb      |  23 +-
+     spec/components/email/receiver_spec.rb             |   8 +-
+     spec/components/email/sender_spec.rb               |   5 +-
+     spec/components/guardian_spec.rb                   |  20 +-
+     spec/components/html_prettify_spec.rb              |  30 ++
+     .../onebox/engine/discourse_local_onebox_spec.rb   |   7 +-
+     spec/components/post_creator_spec.rb               |  36 +-
+     spec/components/pretty_text_spec.rb                |  15 +-
+     spec/components/topic_creator_spec.rb              |  33 +-
+     spec/controllers/categories_controller_spec.rb     |  13 +
+     spec/controllers/manifest_json_controller_spec.rb  |  12 +
+     spec/controllers/permalinks_controller_spec.rb     |  10 +
+     .../post_action_users_controller_spec.rb           |  34 ++
+     spec/controllers/post_actions_controller_spec.rb   |  35 --
+     spec/controllers/posts_controller_spec.rb          |  41 +-
+     spec/controllers/session_controller_spec.rb        |   1 +
+     spec/controllers/users_controller_spec.rb          |  30 +-
+     spec/fabricators/category_group_fabricator.rb      |   5 +
+     spec/fabricators/post_fabricator.rb                |  17 +-
+     spec/fixtures/emails/paragraphs.cooked             |   2 +-
+     spec/helpers/i18n_fallbacks_spec.rb                |  52 ++
+     spec/models/category_spec.rb                       |   9 +
+     spec/models/color_scheme_spec.rb                   |   9 +-
+     spec/models/post_action_spec.rb                    |  16 +
+     spec/models/screened_ip_address_spec.rb            |  61 ++-
+     spec/models/site_spec.rb                           |   3 +
+     spec/models/topic_link_spec.rb                     |   2 +-
+     spec/models/topic_spec.rb                          |  21 +-
+     spec/models/topic_tracking_state_spec.rb           |  16 -
+     spec/models/user_email_observer_spec.rb            |  43 +-
+     spec/models/user_profile_view_spec.rb              |  39 ++
+     spec/models/user_spec.rb                           |  14 +-
+     spec/services/post_alerter_spec.rb                 |  14 +
+     spec/services/staff_action_logger_spec.rb          |  77 +++
+     .../acceptance/category-edit-test.js.es6           |   2 +-
+     .../controllers/admin-user-badges-test.js.es6      |  11 +-
+     test/javascripts/components/d-editor-test.js.es6   | 461 +++++++++++++++++
+     test/javascripts/components/d-link-test.js.es6     |   4 -
+     test/javascripts/helpers/component-test.js.es6     |  12 +-
+     test/javascripts/lib/discourse-test.js.es6         |   7 +
+     test/javascripts/lib/markdown-test.js.es6          |   4 +
+     test/javascripts/models/post-stream-test.js.es6    |   2 +-
+     test/javascripts/test_helper.js                    |   3 +
+     test/stylesheets/test_helper.css                   |   8 +
+     .../lib/discourse_imgur/locale/server.ar.yml       |   4 +-
+     vendor/gems/rails_multisite/.gitignore             |  17 -
+     vendor/gems/rails_multisite/Gemfile                |  14 -
+     vendor/gems/rails_multisite/Guardfile              |   9 -
+     vendor/gems/rails_multisite/LICENSE                |  22 -
+     vendor/gems/rails_multisite/README.md              |  29 --
+     vendor/gems/rails_multisite/Rakefile               |   7 -
+     vendor/gems/rails_multisite/lib/rails_multisite.rb |   3 -
+     .../lib/rails_multisite/connection_management.rb   | 190 -------
+     .../rails_multisite/lib/rails_multisite/railtie.rb |  23 -
+     .../rails_multisite/lib/rails_multisite/version.rb |   3 -
+     vendor/gems/rails_multisite/lib/tasks/db.rake      |  31 --
+     .../gems/rails_multisite/lib/tasks/generators.rake |  26 -
+     .../gems/rails_multisite/rails_multisite.gemspec   |  20 -
+     .../spec/connection_management_rack_spec.rb        |  47 --
+     .../spec/connection_management_spec.rb             |  99 ----
+     .../rails_multisite/spec/fixtures/database.yml     |   6 -
+     .../gems/rails_multisite/spec/fixtures/two_dbs.yml |   6 -
+     vendor/gems/rails_multisite/spec/spec_helper.rb    |  38 --
+     456 files changed, 7519 insertions(+), 3458 deletions(-)
+     create mode 100644 app/assets/javascripts/discourse/components/d-editor-modal.js.es6
+     create mode 100644 app/assets/javascripts/discourse/components/d-editor.js.es6
+     delete mode 100644 app/assets/javascripts/discourse/components/pagedown-editor.js.es6
+     create mode 100644 app/assets/javascripts/discourse/lib/emoji/emoji-groups.js.es6
+     create mode 100644 app/assets/javascripts/discourse/routes/build-static-route.js.es6
+     create mode 100644 app/assets/javascripts/discourse/templates/components/d-editor-modal.hbs
+     create mode 100644 app/assets/javascripts/discourse/templates/components/d-editor.hbs
+     delete mode 100644 app/assets/javascripts/discourse/templates/components/pagedown-editor.hbs
+     create mode 100644 app/assets/javascripts/discourse/templates/login-preferences.hbs
+     create mode 100644 app/assets/javascripts/discourse/templates/modal/dismiss-read.hbs
+     create mode 100644 app/assets/stylesheets/common/d-editor.scss
+     create mode 100644 app/assets/stylesheets/mobile/emoji.scss
+     create mode 100644 app/controllers/manifest_json_controller.rb
+     create mode 100644 app/controllers/post_action_users_controller.rb
+     create mode 100644 app/models/anon_site_json_cache_observer.rb
+     create mode 100644 app/models/user_profile_view.rb
+     create mode 100644 db/migrate/20150914021445_create_user_profile_views.rb
+     create mode 100644 db/migrate/20150914034541_add_views_to_user_profile.rb
+     create mode 100644 db/migrate/20150917071017_add_category_id_to_user_histories.rb
+     create mode 100644 db/migrate/20150924022040_add_fancy_title_to_topic.rb
+     create mode 100644 db/migrate/20150925000915_exclude_whispers_from_badges.rb
+     create mode 100644 lib/html_prettify.rb
+     create mode 100644 plugins/poll/db/migrate/20151016163051_merge_polls_votes.rb
+     create mode 100644 spec/components/html_prettify_spec.rb
+     create mode 100644 spec/controllers/manifest_json_controller_spec.rb
+     create mode 100644 spec/controllers/post_action_users_controller_spec.rb
+     create mode 100644 spec/fabricators/category_group_fabricator.rb
+     create mode 100644 spec/helpers/i18n_fallbacks_spec.rb
+     create mode 100644 spec/models/user_profile_view_spec.rb
+     create mode 100644 test/javascripts/components/d-editor-test.js.es6
+     create mode 100644 test/javascripts/lib/discourse-test.js.es6
+     delete mode 100644 vendor/gems/rails_multisite/.gitignore
+     delete mode 100644 vendor/gems/rails_multisite/Gemfile
+     delete mode 100644 vendor/gems/rails_multisite/Guardfile
+     delete mode 100644 vendor/gems/rails_multisite/LICENSE
+     delete mode 100644 vendor/gems/rails_multisite/README.md
+     delete mode 100755 vendor/gems/rails_multisite/Rakefile
+     delete mode 100644 vendor/gems/rails_multisite/lib/rails_multisite.rb
+     delete mode 100644 vendor/gems/rails_multisite/lib/rails_multisite/connection_management.rb
+     delete mode 100644 vendor/gems/rails_multisite/lib/rails_multisite/railtie.rb
+     delete mode 100644 vendor/gems/rails_multisite/lib/rails_multisite/version.rb
+     delete mode 100644 vendor/gems/rails_multisite/lib/tasks/db.rake
+     delete mode 100644 vendor/gems/rails_multisite/lib/tasks/generators.rake
+     delete mode 100644 vendor/gems/rails_multisite/rails_multisite.gemspec
+     delete mode 100644 vendor/gems/rails_multisite/spec/connection_management_rack_spec.rb
+     delete mode 100644 vendor/gems/rails_multisite/spec/connection_management_spec.rb
+     delete mode 100644 vendor/gems/rails_multisite/spec/fixtures/database.yml
+     delete mode 100644 vendor/gems/rails_multisite/spec/fixtures/two_dbs.yml
+     delete mode 100644 vendor/gems/rails_multisite/spec/spec_helper.rb
+    
+    I, [2015-10-23T15:53:58.134756 #42]  INFO -- : > cd /var/www/discourse && git fetch origin tests-passed
+    From https://github.com/discourse/discourse
+     * branch            tests-passed -> FETCH_HEAD
+    I, [2015-10-23T15:54:04.068910 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.069344 #42]  INFO -- : > cd /var/www/discourse && git checkout tests-passed
+    Switched to a new branch 'tests-passed'
+    I, [2015-10-23T15:54:04.200168 #42]  INFO -- : Branch tests-passed set up to track remote branch tests-passed from origin.
+    
+    I, [2015-10-23T15:54:04.200518 #42]  INFO -- : > cd /var/www/discourse && mkdir -p tmp/pids
+    I, [2015-10-23T15:54:04.205068 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.205515 #42]  INFO -- : > cd /var/www/discourse && mkdir -p tmp/sockets
+    I, [2015-10-23T15:54:04.209201 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.209397 #42]  INFO -- : > cd /var/www/discourse && touch tmp/.gitkeep
+    I, [2015-10-23T15:54:04.213544 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.213795 #42]  INFO -- : > cd /var/www/discourse && mkdir -p                    /shared/log/rails
+    I, [2015-10-23T15:54:04.217537 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.217765 #42]  INFO -- : > cd /var/www/discourse && bash -c "touch -a           /shared/log/rails/{production,production_errors,unicorn.stdout,unicorn.stderr}.log"
+    I, [2015-10-23T15:54:04.222814 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.223049 #42]  INFO -- : > cd /var/www/discourse && bash -c "ln    -s           /shared/log/rails/{production,production_errors,unicorn.stdout,unicorn.stderr}.log /var/www/discourse/log"
+    I, [2015-10-23T15:54:04.229000 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.229490 #42]  INFO -- : > cd /var/www/discourse && bash -c "mkdir -p           /shared/{uploads,backups}"
+    I, [2015-10-23T15:54:04.236051 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.236497 #42]  INFO -- : > cd /var/www/discourse && bash -c "ln    -s           /shared/{uploads,backups} /var/www/discourse/public"
+    I, [2015-10-23T15:54:04.242560 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.242977 #42]  INFO -- : > cd /var/www/discourse && chown -R discourse:www-data /shared/log/rails /shared/uploads /shared/backups
+    I, [2015-10-23T15:54:04.249820 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.250574 #42]  INFO -- : Replacing # redis with sv start redis || exit 1 in /etc/service/unicorn/run
+    I, [2015-10-23T15:54:04.254889 #42]  INFO -- : > cd /var/www/discourse/plugins && mkdir -p plugins
+    I, [2015-10-23T15:54:04.261446 #42]  INFO -- :
+    I, [2015-10-23T15:54:04.261965 #42]  INFO -- : > cd /var/www/discourse/plugins && git clone https://github.com/discourse/docker_manager.git
+    Cloning into 'docker_manager'...
+    I, [2015-10-23T15:54:06.823958 #42]  INFO -- :
+    I, [2015-10-23T15:54:06.826947 #42]  INFO -- : > cp /var/www/discourse/config/nginx.sample.conf /etc/nginx/conf.d/discourse.conf
+    I, [2015-10-23T15:54:06.834182 #42]  INFO -- :
+    I, [2015-10-23T15:54:06.834792 #42]  INFO -- : > rm /etc/nginx/sites-enabled/default
+    I, [2015-10-23T15:54:06.839072 #42]  INFO -- :
+    I, [2015-10-23T15:54:06.839549 #42]  INFO -- : > mkdir -p /var/nginx/cache
+    I, [2015-10-23T15:54:06.843773 #42]  INFO -- :
+    I, [2015-10-23T15:54:06.845326 #42]  INFO -- : Replacing pid /run/nginx.pid; with daemon off; in /etc/nginx/nginx.conf
+    I, [2015-10-23T15:54:06.846966 #42]  INFO -- : Replacing (?m-ix:upstream[^\}]+\}) with upstream discourse { server 127.0.0.1:3000; } in /etc/nginx/conf.d/discourse.conf
+    I, [2015-10-23T15:54:06.848224 #42]  INFO -- : Replacing (?-mix:server_name.+$) with server_name _ ; in /etc/nginx/conf.d/discourse.conf
+    I, [2015-10-23T15:54:06.849445 #42]  INFO -- : Replacing (?-mix:client_max_body_size.+$) with client_max_body_size $upload_size ; in /etc/nginx/conf.d/discourse.conf
+    I, [2015-10-23T15:54:06.851315 #42]  INFO -- : > echo "done configuring web"
+    I, [2015-10-23T15:54:06.856465 #42]  INFO -- : done configuring web
+    
+    I, [2015-10-23T15:54:06.858840 #42]  INFO -- : > cd /var/www/discourse && gem update bundler
+    ERROR:  While executing gem ... (Gem::RemoteFetcher::FetchError)
+        hostname "rubygems.global.ssl.fastly.net" does not match the server certificate (https://rubygems.global.ssl.fastly.net/specs.4.8.gz)
+    I, [2015-10-23T15:54:27.329080 #42]  INFO -- : Updating installed gems
+    
+    I, [2015-10-23T15:54:27.330007 #42]  INFO -- : Terminating async processes
+    I, [2015-10-23T15:54:27.330217 #42]  INFO -- : Sending INT to HOME=/var/lib/postgresql USER=postgres exec chpst -u postgres:postgres:ssl-cert -U postgres:postgres:ssl-cert /usr/lib/postgresql/9.3/bin/postmaster -D /etc/postgresql/9.3/main pid: 112
+    2015-10-23 15:54:27 UTC [112-2] LOG:  received fast shutdown request
+    2015-10-23 15:54:27 UTC [112-3] LOG:  aborting any active transactions
+    2015-10-23 15:54:27 UTC [119-2] LOG:  autovacuum launcher shutting down
+    2015-10-23 15:54:27 UTC [116-1] LOG:  shutting down
+    I, [2015-10-23T15:54:27.330394 #42]  INFO -- : Sending TERM to exec chpst -u redis -U redis /usr/bin/redis-server /etc/redis/redis.conf pid: 240
+    240:signal-handler (1445615667) Received SIGTERM scheduling shutdown...
+    240:M 23 Oct 15:54:27.359 # User requested shutdown...
+    240:M 23 Oct 15:54:27.359 * Saving the final RDB snapshot before exiting.
+    240:M 23 Oct 15:54:27.371 * DB saved on disk
+    240:M 23 Oct 15:54:27.371 # Redis is now ready to exit, bye bye...
+    2015-10-23 15:54:31 UTC [116-2] LOG:  database system is shut down
+    
+    
+    FAILED
+    --------------------
+    RuntimeError: cd /var/www/discourse && gem update bundler failed with return #<Process::Status: pid 335 exit 1>
+    Location of failure: /pups/lib/pups/exec_command.rb:105:in `spawn'
+    exec failed with the params {"cd"=>"$home", "hook"=>"web", "cmd"=>["gem update bundler", "chown -R discourse $home"]}
+    035a935af484328809d3399e2bfca421f5de3165b113fedc7c8dfe76dd7a07f1
+    ** FAILED TO BOOTSTRAP ** please scroll up and look for earlier error messages, there may be more than one

script/benchmarks/markdown/lots_of_mentions.md

@@ -0,0 +1,84 @@
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog
+@sam @bob @bill @jane @marco @fred @figs @fogs @jog

script/benchmarks/markdown/most_features.md

@@ -0,0 +1,201 @@
+Based off https://markdown-it.github.io/ with every feature we support...
+
+
+# h1 Heading 8-)
+## h2 Heading
+### h3 Heading
+#### h4 Heading
+##### h5 Heading
+###### h6 Heading
+
+
+## Horizontal Rules
+
+___
+
+---
+
+***
+
+
+## Emphasis
+
+**This is bold text**
+
+__This is bold text__
+
+*This is italic text*
+
+_This is italic text_
+
+~~Strikethrough~~
+
+
+## Blockquotes
+
+
+> Blockquotes can also be nested...
+>> ...by using additional greater-than signs right next to each other...
+> > > ...or with spaces between arrows.
+
+
+## Lists
+
+Unordered
+
++ Create a list by starting a line with `+`, `-`, or `*`
++ Sub-lists are made by indenting 2 spaces:
+  - Marker character change forces new list start:
+    * Ac tristique libero volutpat at
+    + Facilisis in pretium nisl aliquet
+    - Nulla volutpat aliquam velit
++ Very easy!
+
+Ordered
+
+1. Lorem ipsum dolor sit amet
+2. Consectetur adipiscing elit
+3. Integer molestie lorem at massa
+
+
+1. You can use sequential numbers...
+1. ...or keep all the numbers as `1.`
+
+Start numbering with offset:
+
+57. foo
+1. bar
+
+
+## Code
+
+Inline `code`
+
+Indented code
+
+    // Some comments
+    line 1 of code
+    line 2 of code
+    line 3 of code
+
+
+Block code "fences"
+
+```
+Sample text here...
+```
+
+Syntax highlighting
+
+``` js
+var foo = function (bar) {
+  return bar++;
+};
+
+console.log(foo(5));
+```
+
+
+```text
+var foo = function (bar) {
+  return bar++;
+};
+
+console.log(foo(5));
+```
+
+## Tables
+
+| Option | Description |
+| ------ | ----------- |
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+Right aligned columns
+
+| Option | Description |
+| ------:| -----------:|
+| data   | path to data files to supply the data that will be passed into templates. |
+| engine | engine to be used for processing templates. Handlebars is the default. |
+| ext    | extension to be used for dest files. |
+
+
+## Links
+
+[link text](http://dev.nodeca.com)
+
+[link with title](http://nodeca.github.io/pica/demo/ "title text!")
+
+Autoconverted link https://github.com/discourse 
+
+
+## Images
+
+![Minion](/uploads/default/original/1X/f038dc6544a178b470b3014e92377b4dc996b991.png)
+![](/uploads/default/original/1X/974402975b9ec316057a9e331bbade74d225bc46.jpg)
+
+Like links, Images also have a footnote style syntax
+
+![Alt text][id]
+
+With a reference later in the document defining the URL location:
+
+[id]: /uploads/default/original/1X/7bd599f0af2da1f370ea7b49ebec6e73c32d722b.jpg  "The Dojocat"
+
+
+## Plugins
+
+The killer feature of `markdown-it` is very effective support of
+[syntax plugins](https://www.npmjs.org/browse/keyword/markdown-it-plugin).
+
+
+### [Emojies](https://github.com/markdown-it/markdown-it-emoji)
+
+> Classic markup: :wink:  :cry: :laughing: :yum: :surfing_woman:t4:   
+>
+> Shortcuts (emoticons) ;) :) 
+
+see [how to change output](https://github.com/markdown-it/markdown-it-emoji#change-output) with twemoji.
+
+
+### Polls
+
+[poll type=number min=1 max=20 step=1 public=true]
+[/poll]
+
+[details=Summary]This is a spoiler[/details]
+
+
+Multiline spoiler
+
+[details=Summary]
+
+This is a spoiler
+
+[/details]
+
+
+### Mentions
+Mentions ... @sam 
+
+
+### Categories
+
+ #site-feedback
+
+### Inline bbcode
+
+Hello [code]I am code[/code] 
+
+
+
+## A few paragraphs of **bacon**
+
+Bacon ipsum dolor amet boudin ham hock burgdoggen, strip steak leberkas corned beef pork chop rump short loin porchetta shank venison andouille spare ribs turkey. Boudin tri-tip picanha chicken, porchetta beef ribs hamburger leberkas shankle flank pork spare ribs cupim biltong. Meatball pig leberkas sirloin beef tenderloin tongue picanha ham biltong ribeye chicken. Ham beef chuck frankfurter bresaola pig. Beef turkey ground round kevin pork belly doner jowl. Chicken burgdoggen shankle brisket short ribs capicola beef pancetta.
+
+Rump t-bone beef ribs, cupim pork loin bresaola drumstick frankfurter capicola. Doner pastrami shank ribeye turkey ham hock meatloaf sirloin biltong pig ball tip beef ribs short loin shoulder. Meatball ribeye pastrami shank strip steak porchetta burgdoggen jowl short ribs sausage tail beef landjaeger capicola swine. Alcatra pork chop pork loin turkey, rump tenderloin landjaeger meatball swine ham hock strip steak sirloin. Strip steak drumstick tenderloin ground round, tongue ball tip t-bone tri-tip. Tenderloin doner boudin, sausage beef filet mignon short ribs.
+
+Meatloaf pork loin pork belly porchetta landjaeger frankfurter fatback chicken. Short loin boudin bacon pastrami ball tip. Chicken burgdoggen bresaola chuck porchetta. Swine spare ribs cupim, shoulder rump boudin shank pork belly porchetta chicken pancetta beef meatloaf. Prosciutto shoulder hamburger, pig corned beef picanha filet mignon shankle t-bone jowl rump. Tri-tip pork burgdoggen flank salami short loin cow fatback pig ball tip kielbasa venison ham hock.
+
+Flank jowl pastrami beef swine pork loin. Tail strip steak leberkas t-bone sausage, bresaola rump pastrami meatloaf short ribs prosciutto bacon cupim cow. Beef ribs shoulder ham hock beef meatloaf. Doner sausage porchetta, tongue pork chop jerky boudin meatball shoulder hamburger ribeye beef ribs. Pastrami turkey flank tri-tip, sausage ball tip rump ground round shankle.

spec/components/html_normalize_spec.rb

@@ -0,0 +1,69 @@
+require 'rails_helper'
+require 'html_normalize'
+
+describe HtmlNormalize do
+
+  def n(html)
+    HtmlNormalize.normalize(html)
+  end
+
+  it "handles self closing tags" do
+
+    source = <<-HTML
+<div>
+  <span><img src='testing'>
+  boo</span>
+</div>
+HTML
+    expect(n source).to eq(source.strip)
+  end
+
+  it "Can handle aside" do
+
+      source = <<~HTML
+        <aside class="quote" data-topic="2" data-post="1">
+          <div class="title">
+          <div class="quote-controls"></div>
+          <a href="http://test.localhost/t/this-is-a-test-topic-slight-smile/x/2">This is a test topic <img src="/images/emoji/emoji_one/slight_smile.png?v=5" title="slight_smile" alt="slight_smile" class="emoji"></a></div>
+          <blockquote>
+          <p>ddd</p>
+          </blockquote></aside>
+HTML
+      expected = <<~HTML
+        <aside class="quote" data-post="1" data-topic="2">
+          <div class="title">
+          <div class="quote-controls"></div>
+          <a href="http://test.localhost/t/this-is-a-test-topic-slight-smile/x/2">This is a test topic <img src="/images/emoji/emoji_one/slight_smile.png?v=5" title="slight_smile" alt="slight_smile" class="emoji"></a>
+          </div>
+          <blockquote>
+            <p>ddd</p>
+          </blockquote>
+        </aside>
+HTML
+
+      expect(n expected).to eq(n source)
+  end
+
+  it "Can normalize attributes" do
+
+    source = "<a class='a b' name='sam'>b</a>"
+    same = "<a name='sam' class='a b' >b</a>"
+
+    expect(n source).to eq(n same)
+  end
+
+  it "Can indent divs nicely" do
+    source = "<div> <div><div>hello world</div> </div>     </div>"
+    expected = <<~HTML
+        <div>
+          <div>
+            <div>
+              hello world
+            </div>
+          </div>
+        </div>
+HTML
+
+    expect(n source).to eq(expected.strip)
+  end
+end

spec/components/pretty_text_spec.rb

@@ -1,8 +1,17 @@
 require 'rails_helper'
 require 'pretty_text'
+require 'html_normalize'
 
 describe PrettyText do
 
+  def n(html)
+    HtmlNormalize.normalize(html)
+  end
+
+  def cook(*args)
+    n(PrettyText.cook(*args))
+  end
+
   let(:wrapped_image) { "<div class=\"lightbox-wrapper\"><a href=\"//localhost:3000/uploads/default/4399/33691397e78b4d75.png\" class=\"lightbox\" title=\"Screen Shot 2014-04-14 at 9.47.10 PM.png\"><img src=\"//localhost:3000/uploads/default/_optimized/bd9/b20/bbbcd6a0c0_655x500.png\" width=\"655\" height=\"500\"><div class=\"meta\">\n<span class=\"filename\">Screen Shot 2014-04-14 at 9.47.10 PM.png</span><span class=\"informations\">966x737 1.47 MB</span><span class=\"expand\"></span>\n</div></a></div>" }
   let(:wrapped_image_excerpt) {  }
 
@@ -489,4 +498,146 @@ HTML
     end
   end
 
+  context "markdown it" do
+
+    before do
+      SiteSetting.enable_experimental_markdown_it = true
+    end
+
+    # it "replaces skin toned emoji" do
+    #   expect(PrettyText.cook("hello 👱🏿‍♀️")).to eq("<p>hello <img src=\"/images/emoji/emoji_one/blonde_woman/6.png?v=5\" title=\":blonde_woman:t6:\" class=\"emoji\" alt=\":blonde_woman:t6:\"></p>")
+    #   expect(PrettyText.cook("hello 👩‍🎤")).to eq("<p>hello <img src=\"/images/emoji/emoji_one/woman_singer.png?v=5\" title=\":woman_singer:\" class=\"emoji\" alt=\":woman_singer:\"></p>")
+    #   expect(PrettyText.cook("hello 👩🏾‍🎓")).to eq("<p>hello <img src=\"/images/emoji/emoji_one/woman_student/5.png?v=5\" title=\":woman_student:t5:\" class=\"emoji\" alt=\":woman_student:t5:\"></p>")
+    #   expect(PrettyText.cook("hello 🤷‍♀️")).to eq("<p>hello <img src=\"/images/emoji/emoji_one/woman_shrugging.png?v=5\" title=\":woman_shrugging:\" class=\"emoji\" alt=\":woman_shrugging:\"></p>")
+    # end
+    #
+
+    it "produces tag links" do
+      Fabricate(:topic, {tags: [Fabricate(:tag, name: 'known')]})
+      expect(PrettyText.cook("x #unknown::tag #known::tag")).to match_html("<p>x <span class=\"hashtag\">#unknown::tag</span> <a class=\"hashtag\" href=\"http://test.localhost/tags/known\">#<span>known</span></a></p>")
+    end
+
+    it "can handle mixed lists" do
+      # known bug in old md engine
+      cooked = PrettyText.cook("* a\n\n1. b")
+      expect(cooked).to match_html("<ul>\n<li>a</li>\n</ul><ol>\n<li>b</li>\n</ol>")
+    end
+
+    it "can handle traditional vs non traditional newlines" do
+      SiteSetting.traditional_markdown_linebreaks = true
+      expect(PrettyText.cook("1\n2")).to match_html "<p>1 2</p>"
+
+      SiteSetting.traditional_markdown_linebreaks = false
+      expect(PrettyText.cook("1\n2")).to match_html "<p>1<br>\n2</p>"
+    end
+
+    it "can handle mentions" do
+      Fabricate(:user, username: "sam")
+      expect(PrettyText.cook("hi @sam! hi")).to match_html '<p>hi <a class="mention" href="/u/sam">@sam</a>! hi</p>'
+    end
+
+    it "can handle mentions inside a hyperlink" do
+    expect(PrettyText.cook("<a> @inner</a> ")).to match_html '<p><a> @inner</a></p>'
+    end
+
+    it "can handle mentions inside a hyperlink" do
+      expect(PrettyText.cook("[link @inner](http://site.com)")).to match_html '<p><a href="http://site.com" rel="nofollow noopener">link @inner</a></p>'
+    end
+
+    it "can handle a list of mentions" do
+      expect(PrettyText.cook("@a,@b")).to match_html('<p><span class="mention">@a</span>,<span class="mention">@b</span></p>')
+    end
+
+    it "can handle emoji by name" do
+
+      expected = <<HTML
+<p><img src="/images/emoji/emoji_one/smile.png?v=5\" title=":smile:" class="emoji" alt=":smile:"><img src="/images/emoji/emoji_one/sunny.png?v=5" title=":sunny:" class="emoji" alt=":sunny:"></p>
+HTML
+      expect(PrettyText.cook(":smile::sunny:")).to eq(expected.strip)
+    end
+
+    it "handles emoji boundaries correctly" do
+      cooked = PrettyText.cook("a,:man:t2:,b")
+      expected = '<p>a,<img src="/images/emoji/emoji_one/man/2.png?v=5" title=":man:t2:" class="emoji" alt=":man:t2:">,b</p>'
+      expect(cooked).to match(expected.strip)
+    end
+
+    it "can handle emoji by translation" do
+      expected = '<p><img src="/images/emoji/emoji_one/wink.png?v=5" title=":wink:" class="emoji" alt=":wink:"></p>'
+      expect(PrettyText.cook(";)")).to eq(expected)
+    end
+
+    it "can handle multiple emojis by translation" do
+      cooked = PrettyText.cook(":) ;) :)")
+      expect(cooked.split("img").length-1).to eq(3)
+    end
+
+    it "handles emoji boundries correctly" do
+      expect(PrettyText.cook(",:)")).to include("emoji")
+      expect(PrettyText.cook(":-)\n")).to include("emoji")
+      expect(PrettyText.cook("a :)")).to include("emoji")
+      expect(PrettyText.cook(":),")).not_to include("emoji")
+      expect(PrettyText.cook("abcde ^:;-P")).to include("emoji")
+    end
+
+
+    it 'can include code class correctly' do
+      expect(PrettyText.cook("```cpp\ncpp\n```")).to match_html("<pre><code class='lang-cpp'>cpp\n</code></pre>")
+    end
+
+    it 'indents code correctly' do
+      code = "X\n```\n\n    #\n    x\n```"
+      cooked = PrettyText.cook(code)
+      expect(cooked).to match_html("<p>X</p>\n<pre><code class=\"lang-auto\">\n    #\n    x\n</code></pre>")
+    end
+
+    it 'can censor words correctly' do
+      SiteSetting.censored_words = 'apple|banana'
+      expect(PrettyText.cook('yay banana yay')).not_to include('banana')
+      expect(PrettyText.cook('yay `banana` yay')).not_to include('banana')
+      expect(PrettyText.cook("yay \n\n```\nbanana\n````\n yay")).not_to include('banana')
+      expect(PrettyText.cook("# banana")).not_to include('banana')
+      expect(PrettyText.cook("# banana")).to include("\u25a0\u25a0")
+    end
+
+    it 'handles onebox correctly' do
+      # we expect 2 oneboxes
+      expect(PrettyText.cook("http://a.com\nhttp://b.com").split("onebox").length).to eq(3)
+      expect(PrettyText.cook("http://a.com\n\nhttp://b.com").split("onebox").length).to eq(3)
+
+      expect(PrettyText.cook("a\nhttp://a.com")).to include('onebox')
+      expect(PrettyText.cook("> http://a.com")).not_to include('onebox')
+      expect(PrettyText.cook("a\nhttp://a.com a")).not_to include('onebox')
+      expect(PrettyText.cook("a\nhttp://a.com\na")).to include('onebox')
+      expect(PrettyText.cook("http://a.com")).to include('onebox')
+      expect(PrettyText.cook("a.com")).not_to include('onebox')
+      expect(PrettyText.cook("http://a.com ")).to include('onebox')
+      expect(PrettyText.cook("http://a.com a")).not_to include('onebox')
+      expect(PrettyText.cook("- http://a.com")).not_to include('onebox')
+    end
+
+    it "can handle bbcode" do
+      expect(PrettyText.cook("a[b]b[/b]c")).to eq('<p>a<span class="bbcode-b">b</span>c</p>')
+      expect(PrettyText.cook("a[i]b[/i]c")).to eq('<p>a<span class="bbcode-i">b</span>c</p>')
+    end
+
+    it "do off topic quoting with emoji unescape" do
+
+      topic = Fabricate(:topic, title: "this is a test topic :slight_smile:")
+      expected = <<HTML
+<aside class="quote" data-topic="#{topic.id}" data-post="2">
+<div class="title">
+  <div class="quote-controls"></div>
+  <a href="http://test.localhost/t/this-is-a-test-topic-slight-smile/#{topic.id}/2">This is a test topic <img src="/images/emoji/emoji_one/slight_smile.png?v=5" title="slight_smile" alt="slight_smile" class="emoji"></a>
+</div>
+<blockquote>
+  <p>ddd</p>
+</blockquote>
+</aside>
+HTML
+
+      expect(cook("[quote=\"EvilTrout, post:2, topic:#{topic.id}\"]\nddd\n[/quote]", topic_id: 1)).to eq(n(expected))
+    end
+  end
+
 end

test/javascripts/components/d-editor-test.js.es6

@@ -21,7 +21,7 @@ componentTest('preview sanitizes HTML', {
   template: '{{d-editor value=value}}',
 
   test(assert) {
-    this.set('value', `"><svg onload="prompt(/xss/)"></svg>`);
+    fillIn('.d-editor-input', `"><svg onload="prompt(/xss/)"></svg>`);
     andThen(() => {
       assert.equal(this.$('.d-editor-preview').html().trim(), '<p>\"&gt;</p>');
     });

test/javascripts/test_helper.js

@@ -22,6 +22,7 @@
 //= require vendor
 //= require ember-shim
 //= require pretty-text-bundle
+//= require markdown-it-bundle
 //= require application
 //= require plugin
 //= require htmlparser.js