FIX: medium URL with @ incorrectly handled as mention

Also:

- remove unused code
- rename bbcode_ruler to bbcode.ruler
- add md.core.textPostProcess.ruler to apply at end of chain (excluding links)

app/assets/javascripts/markdown-it-bundle.js

@@ -12,4 +12,5 @@
 //= require ./pretty-text/engines/discourse-markdown/table
 //= require ./pretty-text/engines/discourse-markdown/paragraph
 //= require ./pretty-text/engines/discourse-markdown/newline
-//= require ./pretty-text/engines/discourse-markdown/html_img
+//= require ./pretty-text/engines/discourse-markdown/html-img
+//= require ./pretty-text/engines/discourse-markdown/text-post-process

app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6

@@ -71,11 +71,16 @@ class Ruler {
 
 // block bb code ruler for parsing of quotes / code / polls
 function setupBlockBBCode(md) {
-  md.block.bbcode_ruler = new Ruler();
+  md.block.bbcode = { ruler: new Ruler() };
 }
 
 function setupInlineBBCode(md) {
-  md.inline.bbcode_ruler = new Ruler();
+  md.inline.bbcode = { ruler: new Ruler() };
+}
+
+function setupTextPostProcessRuler(md) {
+  const TextPostProcessRuler = requirejs('pretty-text/engines/discourse-markdown/text-post-process').TextPostProcessRuler;
+  md.core.textPostProcess = { ruler: new TextPostProcessRuler() };
 }
 
 function renderHoisted(tokens, idx, options) {
@@ -217,6 +222,7 @@ export function setup(opts, siteSettings, state) {
   setupImageDimensions(opts.engine);
   setupBlockBBCode(opts.engine);
   setupInlineBBCode(opts.engine);
+  setupTextPostProcessRuler(opts.engine);
 
   pluginCallbacks.forEach(([feature, callback])=>{
     if (opts.discourse.features[feature]) {

app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-block.js.es6

@@ -120,7 +120,7 @@ function applyBBCode(state, startLine, endLine, silent, md) {
     return false;
   }
 
-  let ruleInfo = md.block.bbcode_ruler.getRuleForTag(info.tag);
+  let ruleInfo = md.block.bbcode.ruler.getRuleForTag(info.tag);
   if (!ruleInfo) { return false; }
 
   rule = ruleInfo.rule;
@@ -249,11 +249,8 @@ function applyBBCode(state, startLine, endLine, silent, md) {
 }
 
 export function setup(helper) {
-  if (!helper.markdownIt) { return; }
-
-
   helper.registerPlugin(md => {
-    const ruler = md.block.bbcode_ruler;
+    const ruler = md.block.bbcode.ruler;
 
     ruler.push('code', {
       tag: 'code',

app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-inline.js.es6

@@ -141,9 +141,6 @@ function processBBCode(state, silent) {
 }
 
 export function setup(helper) {
-
-  if (!helper.markdownIt) { return; }
-
   helper.whiteList(['span.bbcode-b', 'span.bbcode-i', 'span.bbcode-u', 'span.bbcode-s']);
 
   helper.registerOptions(opts => {
@@ -151,7 +148,7 @@ export function setup(helper) {
   });
 
   helper.registerPlugin(md => {
-    const ruler = md.inline.bbcode_ruler;
+    const ruler = md.inline.bbcode.ruler;
 
     md.inline.ruler.push('bbcode-inline', (state,silent) => tokanizeBBCode(state,silent,ruler));
     md.inline.ruler2.before('text_collapse', 'bbcode-inline', processBBCode);

app/assets/javascripts/pretty-text/engines/discourse-markdown/category-hashtag.js.es6

@@ -1,6 +1,6 @@
-function addHashtag(buffer, matches, state) {
+function addHashtag(buffer, match, state) {
   const options = state.md.options.discourse;
-  const [hashtag, slug] = matches;
+  const slug = match.slice(1);
   const categoryHashtagLookup = options.categoryHashtagLookup;
   const result = categoryHashtagLookup && categoryHashtagLookup(slug);
 
@@ -34,7 +34,7 @@ function addHashtag(buffer, matches, state) {
     buffer.push(token);
 
     token = new state.Token('text', '', 0);
-    token.content = hashtag;
+    token.content = match;
     buffer.push(token);
 
     token = new state.Token('span_close', 'span', -1);
@@ -42,63 +42,14 @@ function addHashtag(buffer, matches, state) {
   }
 }
 
-const REGEX = /#([\w-:]{1,101})/gi;
-
-function allowedBoundary(content, index, utils) {
-  let code = content.charCodeAt(index);
-  return (utils.isWhiteSpace(code) || utils.isPunctChar(String.fromCharCode(code)));
-}
-
-function applyHashtag(content, state) {
-  let result = null,
-      match,
-      pos = 0;
-
-  while (match = REGEX.exec(content)) {
-    // check boundary
-    if (match.index > 0) {
-      if (!allowedBoundary(content, match.index-1, state.md.utils)) {
-        continue;
-      }
-    }
-
-    // check forward boundary as well
-    if (match.index + match[0].length < content.length) {
-      if (!allowedBoundary(content, match.index + match[0].length, state.md.utils)) {
-        continue;
-      }
-    }
-
-    if (match.index > pos) {
-      result = result || [];
-      let token = new state.Token('text', '', 0);
-      token.content = content.slice(pos, match.index);
-      result.push(token);
-    }
-
-    result = result || [];
-    addHashtag(result, match, state);
-
-    pos = match.index + match[0].length;
-  }
-
-  if (result && pos < content.length) {
-    let token = new state.Token('text', '', 0);
-    token.content = content.slice(pos);
-    result.push(token);
-  }
-
-  return result;
-}
-
 export function setup(helper) {
-
-  if (!helper.markdownIt) { return; }
-
   helper.registerPlugin(md=>{
 
-    md.core.ruler.push('category-hashtag', state => md.options.discourse.helpers.textReplace(
-      state, applyHashtag, true /* skip all links */
-    ));
+    const rule = {
+      matcher: /#[\w-:]{1,101}/,
+      onMatch: addHashtag
+    };
+
+    md.core.textPostProcess.ruler.push('category-hashtag', rule);
   });
 }

app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6

@@ -23,9 +23,6 @@ function censorTree(state, censor) {
 }
 
 export function setup(helper) {
-
-  if (!helper.markdownIt) { return; }
-
   helper.registerOptions((opts, siteSettings) => {
     opts.censoredWords = siteSettings.censored_words;
     opts.censoredPattern = siteSettings.censored_pattern;

app/assets/javascripts/pretty-text/engines/discourse-markdown/code.js.es6

@@ -27,8 +27,6 @@ function render(tokens, idx, options, env, slf, md) {
 }
 
 export function setup(helper) {
-  if (!helper.markdownIt) { return; }
-
   helper.registerOptions((opts, siteSettings) => {
     opts.defaultCodeLang = siteSettings.default_code_lang;
     opts.acceptableCodeClasses = (siteSettings.highlighted_languages || "").split("|").concat(['auto', 'nohighlight']);

app/assets/javascripts/pretty-text/engines/discourse-markdown/emoji.js.es6

@@ -229,9 +229,6 @@ function applyEmoji(content, state, emojiUnicodeReplacer) {
 }
 
 export function setup(helper) {
-
-  if (!helper.markdownIt) { return; }
-
   helper.registerOptions((opts, siteSettings, state)=>{
     opts.features.emoji = !!siteSettings.enable_emoji;
     opts.emojiSet = siteSettings.emoji_set || "";

app/assets/javascripts/pretty-text/engines/discourse-markdown/html-img.js.es6

@@ -65,9 +65,6 @@ function rule(state, startLine, endLine) {
 
 
 export function setup(helper) {
-
-  if (!helper.markdownIt) { return; }
-
   helper.registerPlugin(md=>{
      md.block.ruler.before('html_block', 'html_img', rule, {alt: ['fence']});
   });

app/assets/javascripts/pretty-text/engines/discourse-markdown/mentions.js.es6

@@ -1,43 +1,7 @@
-const regex = /^(\w[\w.-]{0,59})\b/i;
-
-function applyMentions(state, silent, isWhiteSpace, isPunctChar, mentionLookup, getURL) {
-
-  let pos = state.pos;
-
-  // 64 = @
-  if (silent || state.src.charCodeAt(pos) !== 64) {
-    return false;
-  }
-
-  if (pos > 0) {
-    let prev = state.src.charCodeAt(pos-1);
-    if (!isWhiteSpace(prev) && !isPunctChar(String.fromCharCode(prev))) {
-      return false;
-    }
-  }
-
-  // skip if in a link
-  if (state.tokens) {
-    let last = state.tokens[state.tokens.length-1];
-    if (last) {
-      if (last.type === 'link_open') {
-        return false;
-      }
-      if (last.type === 'html_inline' && last.content.substr(0,2) === "<a") {
-        return false;
-      }
-    }
-  }
-
-  let maxMention = state.src.substr(pos+1, 60);
-
-  let matches = maxMention.match(regex);
-
-  if (!matches) {
-    return false;
-  }
-
-  let username = matches[1];
+function addMention(buffer, match, state) {
+  let username = match.slice(1);
+  let mentionLookup = state.md.options.discourse.mentionLookup;
+  let getURL = state.md.options.discourse.getURL;
 
   let type = mentionLookup && mentionLookup(username);
 
@@ -54,35 +18,31 @@ function applyMentions(state, silent, isWhiteSpace, isPunctChar, mentionLookup,
     tag = 'span';
   }
 
-  let token = state.push('mention_open', tag, 1);
+  let token = new state.Token('mention_open', tag, 1);
   token.attrs = [['class', className]];
   if (href) {
     token.attrs.push(['href', href]);
   }
 
-  token = state.push('text', '', 0);
+  buffer.push(token);
+
+  token = new state.Token('text', '', 0);
   token.content = '@'+username;
 
-  state.push('mention_close', tag, -1);
+  buffer.push(token);
 
-  state.pos = pos + username.length + 1;
-
-  return true;
+  token = new state.Token('mention_close', tag, -1);
+  buffer.push(token);
 }
 
 export function setup(helper) {
-
-  if (!helper.markdownIt) { return; }
-
   helper.registerPlugin(md => {
-    md.inline.ruler.push('mentions', (state,silent)=> applyMentions(
-          state,
-          silent,
-          md.utils.isWhiteSpace,
-          md.utils.isPunctChar,
-          md.options.discourse.mentionLookup,
-          md.options.discourse.getURL
-    ));
+
+    const rule = {
+      matcher: /@\w[\w.-]{0,59}/,
+      onMatch: addMention
+    };
+
+    md.core.textPostProcess.ruler.push('mentions', rule);
   });
 }
-

app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6

@@ -86,9 +86,6 @@ function applyOnebox(state, silent) {
 
 
 export function setup(helper) {
-
-  if (!helper.markdownIt) { return; }
-
   helper.registerPlugin(md => {
     md.core.ruler.after('linkify', 'onebox', applyOnebox);
   });

app/assets/javascripts/pretty-text/engines/discourse-markdown/quotes.js.es6

@@ -121,14 +121,12 @@ const rule = {
 
 export function setup(helper) {
 
-  if (!helper.markdownIt) { return; }
-
   helper.registerOptions((opts, siteSettings) => {
     opts.enableEmoji = siteSettings.enable_emoji;
     opts.emojiSet = siteSettings.emoji_set;
   });
 
   helper.registerPlugin(md=>{
-     md.block.bbcode_ruler.push('quotes', rule);
+     md.block.bbcode.ruler.push('quotes', rule);
   });
 }

app/assets/javascripts/pretty-text/engines/discourse-markdown/table.js.es6

@@ -1,7 +1,4 @@
 export function setup(helper) {
-
-  if (!helper.markdownIt) { return; }
-
   // this is built in now
   // TODO: sanitizer needs fixing, does not properly support this yet
 

app/assets/javascripts/pretty-text/engines/discourse-markdown/text-post-process.js.es6

@@ -0,0 +1,100 @@
+
+export class TextPostProcessRuler {
+  constructor() {
+    this.rules = [];
+  }
+
+  getRules() {
+    return this.rules;
+  }
+
+  // TODO error handling
+  getMatcher() {
+    if (this.matcher) { return this.matcher; }
+
+    this.matcher = new RegExp(this.rules.map((r) =>
+      "(" + r.rule.matcher.toString().slice(1,-1) + ")"
+    ).join('|'), 'g');
+
+    return this.matcher;
+  }
+
+  applyRule(buffer, match, state) {
+    let i;
+    for(i=0; i<this.rules.length; i++) {
+      if (match[i+1]) {
+        this.rules[i].rule.onMatch(buffer, match[i+1], state);
+        break;
+      }
+    }
+  }
+
+  // TODO validate inputs
+  push(name, rule) {
+    this.rules.push({name, rule});
+    this.matcher = null;
+  }
+
+}
+
+function allowedBoundary(content, index, utils) {
+  let code = content.charCodeAt(index);
+  return (utils.isWhiteSpace(code) || utils.isPunctChar(String.fromCharCode(code)));
+}
+
+function textPostProcess(content, state, ruler) {
+  let result = null,
+      match,
+      pos = 0;
+
+  const matcher = ruler.getMatcher();
+
+  while (match = matcher.exec(content)) {
+    // check boundary
+    if (match.index > 0) {
+      if (!allowedBoundary(content, match.index-1, state.md.utils)) {
+        continue;
+      }
+    }
+
+    // check forward boundary as well
+    if (match.index + match[0].length < content.length) {
+      if (!allowedBoundary(content, match.index + match[0].length, state.md.utils)) {
+        continue;
+      }
+    }
+
+    if (match.index > pos) {
+      result = result || [];
+      let token = new state.Token('text', '', 0);
+      token.content = content.slice(pos, match.index);
+      result.push(token);
+    }
+
+    result = result || [];
+
+    ruler.applyRule(result, match, state);
+
+    pos = match.index + match[0].length;
+  }
+
+  if (result && pos < content.length) {
+    let token = new state.Token('text', '', 0);
+    token.content = content.slice(pos);
+    result.push(token);
+  }
+
+  return result;
+}
+
+
+export function setup(helper) {
+  helper.registerPlugin(md => {
+    const ruler = md.core.textPostProcess.ruler;
+    const replacer = (content, state) => textPostProcess(content, state, ruler);
+
+    md.core.ruler.push('text-post-process', state =>
+      md.options.discourse.helpers.textReplace(state, replacer, true)
+    );
+  });
+}

plugins/discourse-details/assets/javascripts/lib/discourse-markdown/details.js.es6

@@ -25,6 +25,6 @@ export function setup(helper) {
   ]);
 
   helper.registerPlugin(md => {
-    md.block.bbcode_ruler.push('details', rule);
+    md.block.bbcode.ruler.push('details', rule);
   });
 }

plugins/poll/assets/javascripts/lib/discourse-markdown/poll.js.es6

@@ -295,7 +295,7 @@ function newApiInit(helper) {
   });
 
   helper.registerPlugin(md => {
-    md.block.bbcode_ruler.push('poll', rule);
+    md.block.bbcode.ruler.push('poll', rule);
   });
 }
 

spec/components/pretty_text_spec.rb

@@ -175,6 +175,10 @@ describe PrettyText do
       )
     end
 
+    it 'should not treat a medium link as a mention' do
+      expect(PrettyText.cook(". http://test/@sam")).not_to include('mention')
+    end
+
   end
 
   describe "code fences" do