DEV: remove markdown-it-bundle and custom build code (#23859)

With Embroider, we can rely on async `import()` to do the splitting
for us.

This commit extracts from `pretty-text` all the parts that are
meant to be loaded async into a new `discourse-markdown-it` package
that is also a V2 addon (meaning that all files are presumed unused
until they are imported, aka "static").

Mostly I tried to keep the very discourse specific stuff (accessing
site settings and loading plugin features) inside discourse proper,
while the new package aims to have some resembalance of a general
purpose library, a MarkdownIt++ if you will. It is far from perfect
because of how all the "options" stuff work but I think it's a good
start for more refactorings (clearing up the interfaces) to happen
later.

With this, pretty-text and app/lib/text are mostly a kitchen sink
of loosely related text processing utilities.

After the refactor, a lot more code related to setting up the
engine are now loaded lazily, which should be a pretty nice win. I
also noticed that we are currently pulling in the `xss` library at
initial load to power the "sanitize" stuff, but I suspect with a
similar refactoring effort those usages can be removed too. (See
also #23790).

This PR does not attempt to fix the sanitize issue, but I think it
sets things up on the right trajectory for that to happen later.

Co-authored-by: David Taylor <david@taylorhq.com>

app/assets/javascripts/discourse-markdown-it/addon-main.cjs

@@ -0,0 +1,4 @@
+'use strict';
+
+const { addonV1Shim } = require('@embroider/addon-shim');
+module.exports = addonV1Shim(__dirname);

app/assets/javascripts/discourse-markdown-it/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "discourse-markdown-it",
+  "version": "1.0.0",
+  "private": true,
+  "description": "Discourse's markdown-it features",
+  "author": "Discourse <team@discourse.org>",
+  "license": "GPL-2.0-only",
+  "keywords": [
+    "ember-addon"
+  ],
+  "exports": {
+    ".": "./src/index.js",
+    "./*": "./src/*.js",
+    "./addon-main.js": "./addon-main.cjs"
+  },
+  "files": [
+    "addon-main.cjs",
+    "src"
+  ],
+  "dependencies": {
+    "@embroider/addon-shim": "^1.0.0",
+    "discourse-common": "1.0.0",
+    "ember-auto-import": "^2.6.3",
+    "markdown-it": "^13.0.2"
+  },
+  "peerDependencies": {
+    "discourse-i18n": "1.0.0",
+    "pretty-text": "1.0.0",
+    "xss": "*"
+  },
+  "engines": {
+    "node": "16.* || >= 18",
+    "npm": "please-use-yarn",
+    "yarn": ">= 1.21.1"
+  },
+  "ember": {
+    "edition": "octane"
+  },
+  "ember-addon": {
+    "version": 2,
+    "type": "addon",
+    "main": "addon-main.cjs",
+    "app-js": {}
+  }
+}

app/assets/javascripts/discourse-markdown-it/src/engine.js

@@ -0,0 +1,348 @@
+import markdownit from "markdown-it";
+import AllowLister from "pretty-text/allow-lister";
+import guid from "pretty-text/guid";
+import { sanitize } from "pretty-text/sanitizer";
+import { TextPostProcessRuler } from "./features/text-post-process";
+
+// note, this will mutate options due to the way the API is designed
+// may need a refactor
+export default function makeEngine(
+  options,
+  markdownItOptions,
+  markdownItRules
+) {
+  const engine = makeMarkdownIt(markdownItOptions, markdownItRules);
+
+  const quotes =
+    options.discourse.limitedSiteSettings.markdownTypographerQuotationMarks;
+
+  if (quotes) {
+    engine.options.quotes = quotes.split("|");
+  }
+
+  const tlds = options.discourse.limitedSiteSettings.markdownLinkifyTlds || "";
+  engine.linkify.tlds(tlds.split("|"));
+
+  setupUrlDecoding(engine);
+  setupHoister(engine);
+  setupImageAndPlayableMediaRenderer(engine);
+  setupAttachments(engine);
+  setupBlockBBCode(engine);
+  setupInlineBBCode(engine);
+  setupTextPostProcessRuler(engine);
+
+  options.engine = engine;
+
+  for (const [feature, callback] of options.pluginCallbacks) {
+    if (options.discourse.features[feature]) {
+      if (callback === null || callback === undefined) {
+        // eslint-disable-next-line no-console
+        console.log("BAD MARKDOWN CALLBACK FOUND");
+        // eslint-disable-next-line no-console
+        console.log(`FEATURE IS: ${feature}`);
+      }
+      engine.use(callback);
+    }
+  }
+
+  // top level markdown it notifier
+  options.markdownIt = true;
+  options.setup = true;
+
+  if (!options.discourse.sanitizer || !options.sanitizer) {
+    const allowLister = new AllowLister(options.discourse);
+
+    options.allowListed.forEach(([feature, info]) => {
+      allowLister.allowListFeature(feature, info);
+    });
+
+    options.sanitizer = options.discourse.sanitizer = !!options.discourse
+      .sanitize
+      ? (a) => sanitize(a, allowLister)
+      : (a) => a;
+  }
+}
+
+export function cook(raw, options) {
+  // we still have to hoist html_raw nodes so they bypass the allowlister
+  // this is the case for oneboxes and also certain plugins that require
+  // raw HTML rendering within markdown bbcode rules
+  options.discourse.hoisted ??= {};
+
+  const rendered = options.engine.render(raw);
+  let cooked = options.discourse.sanitizer(rendered).trim();
+
+  // opts.discourse.hoisted guid keys will be deleted within here to
+  // keep the object empty
+  cooked = unhoistForCooked(options.discourse.hoisted, cooked);
+
+  return cooked;
+}
+
+function makeMarkdownIt(markdownItOptions, markdownItRules) {
+  if (markdownItRules) {
+    // Preset for "zero", https://github.com/markdown-it/markdown-it/blob/master/lib/presets/zero.js
+    return markdownit("zero", markdownItOptions).enable(markdownItRules);
+  } else {
+    return markdownit(markdownItOptions);
+  }
+}
+
+function setupUrlDecoding(engine) {
+  // this fixed a subtle issue where %20 is decoded as space in
+  // automatic urls
+  engine.utils.lib.mdurl.decode.defaultChars = ";/?:@&=+$,# ";
+}
+
+// hoists html_raw tokens out of the render flow and replaces them
+// with a GUID. this GUID is then replaced with the final raw HTML
+// content in unhoistForCooked
+function renderHoisted(tokens, idx, options) {
+  const content = tokens[idx].content;
+  if (content && content.length > 0) {
+    let id = guid();
+    options.discourse.hoisted[id] = content;
+    return id;
+  } else {
+    return "";
+  }
+}
+
+function unhoistForCooked(hoisted, cooked) {
+  const keys = Object.keys(hoisted);
+  if (keys.length) {
+    let found = true;
+
+    const unhoist = function (key) {
+      cooked = cooked.replace(new RegExp(key, "g"), function () {
+        found = true;
+        return hoisted[key];
+      });
+      delete hoisted[key];
+    };
+
+    while (found) {
+      found = false;
+      keys.forEach(unhoist);
+    }
+  }
+
+  return cooked;
+}
+
+// html_raw tokens, funnily enough, render raw HTML via renderHoisted and
+// unhoistForCooked
+function setupHoister(engine) {
+  engine.renderer.rules.html_raw = renderHoisted;
+}
+
+// exported for test only
+export function extractDataAttribute(str) {
+  let sep = str.indexOf("=");
+  if (sep === -1) {
+    return null;
+  }
+
+  const key = `data-${str.slice(0, sep)}`.toLowerCase();
+  if (!/^[A-Za-z]+[\w\-\:\.]*$/.test(key)) {
+    return null;
+  }
+
+  const value = str.slice(sep + 1);
+  return [key, value];
+}
+
+// videoHTML and audioHTML follow the same HTML syntax
+// as oneboxer.rb when dealing with these formats
+function videoHTML(token) {
+  const src = token.attrGet("src");
+  const origSrc = token.attrGet("data-orig-src");
+  const dataOrigSrcAttr = origSrc !== null ? `data-orig-src="${origSrc}"` : "";
+  return `<div class="video-placeholder-container" data-video-src="${src}" ${dataOrigSrcAttr}>
+  </div>`;
+}
+
+function audioHTML(token) {
+  const src = token.attrGet("src");
+  const origSrc = token.attrGet("data-orig-src");
+  const dataOrigSrcAttr = origSrc !== null ? `data-orig-src="${origSrc}"` : "";
+  return `<audio preload="metadata" controls>
+    <source src="${src}" ${dataOrigSrcAttr}>
+    <a href="${src}">${src}</a>
+  </audio>`;
+}
+
+const IMG_SIZE_REGEX =
+  /^([1-9]+[0-9]*)x([1-9]+[0-9]*)(\s*,\s*(x?)([1-9][0-9]{0,2}?)([%x]?))?$/;
+function renderImageOrPlayableMedia(tokens, idx, options, env, slf) {
+  const token = tokens[idx];
+  const alt = slf.renderInlineAsText(token.children, options, env);
+  const split = alt.split("|");
+  const altSplit = [split[0]];
+
+  // markdown-it supports returning HTML instead of continuing to render the current token
+  // see https://github.com/markdown-it/markdown-it/blob/master/docs/architecture.md#renderer
+  // handles |video and |audio alt transformations for image tags
+  if (split[1] === "video") {
+    if (
+      options.discourse.previewing &&
+      !options.discourse.limitedSiteSettings.enableDiffhtmlPreview
+    ) {
+      return `<div class="onebox-placeholder-container">
+        <span class="placeholder-icon video"></span>
+      </div>`;
+    } else {
+      return videoHTML(token);
+    }
+  } else if (split[1] === "audio") {
+    return audioHTML(token);
+  }
+
+  // parsing ![myimage|500x300]() or ![myimage|75%]() or ![myimage|500x300, 75%]
+  for (let i = 1, match, data; i < split.length; ++i) {
+    if ((match = split[i].match(IMG_SIZE_REGEX)) && match[1] && match[2]) {
+      let width = match[1];
+      let height = match[2];
+
+      // calculate using percentage
+      if (match[5] && match[6] && match[6] === "%") {
+        let percent = parseFloat(match[5]) / 100.0;
+        width = parseInt(width * percent, 10);
+        height = parseInt(height * percent, 10);
+      }
+
+      // calculate using only given width
+      if (match[5] && match[6] && match[6] === "x") {
+        let wr = parseFloat(match[5]) / width;
+        width = parseInt(match[5], 10);
+        height = parseInt(height * wr, 10);
+      }
+
+      // calculate using only given height
+      if (match[5] && match[4] && match[4] === "x" && !match[6]) {
+        let hr = parseFloat(match[5]) / height;
+        height = parseInt(match[5], 10);
+        width = parseInt(width * hr, 10);
+      }
+
+      if (token.attrIndex("width") === -1) {
+        token.attrs.push(["width", width]);
+      }
+
+      if (token.attrIndex("height") === -1) {
+        token.attrs.push(["height", height]);
+      }
+
+      if (
+        options.discourse.previewing &&
+        match[6] !== "x" &&
+        match[4] !== "x"
+      ) {
+        token.attrs.push(["class", "resizable"]);
+      }
+    } else if ((data = extractDataAttribute(split[i]))) {
+      token.attrs.push(data);
+    } else if (split[i] === "thumbnail") {
+      token.attrs.push(["data-thumbnail", "true"]);
+    } else {
+      altSplit.push(split[i]);
+    }
+  }
+
+  const altValue = altSplit.join("|").trim();
+  if (altValue === "") {
+    token.attrSet("role", "presentation");
+  } else {
+    token.attrSet("alt", altValue);
+  }
+
+  return slf.renderToken(tokens, idx, options);
+}
+
+// we have taken over the ![]() syntax in markdown to
+// be able to render a video or audio URL as well as the
+// image using |video and |audio in the text inside []
+function setupImageAndPlayableMediaRenderer(engine) {
+  engine.renderer.rules.image = renderImageOrPlayableMedia;
+}
+
+// discourse-encrypt wants this?
+export const ATTACHMENT_CSS_CLASS = "attachment";
+
+function renderAttachment(tokens, idx, options, env, slf) {
+  const linkToken = tokens[idx];
+  const textToken = tokens[idx + 1];
+
+  const split = textToken.content.split("|");
+  const contentSplit = [];
+
+  for (let i = 0, data; i < split.length; ++i) {
+    if (split[i] === ATTACHMENT_CSS_CLASS) {
+      linkToken.attrs.unshift(["class", split[i]]);
+    } else if ((data = extractDataAttribute(split[i]))) {
+      linkToken.attrs.push(data);
+    } else {
+      contentSplit.push(split[i]);
+    }
+  }
+
+  if (contentSplit.length > 0) {
+    textToken.content = contentSplit.join("|");
+  }
+
+  return slf.renderToken(tokens, idx, options);
+}
+
+function setupAttachments(engine) {
+  engine.renderer.rules.link_open = renderAttachment;
+}
+
+// TODO we may just use a proper ruler from markdown it... this is a basic proxy
+class Ruler {
+  constructor() {
+    this.rules = [];
+  }
+
+  getRules() {
+    return this.rules;
+  }
+
+  getRuleForTag(tag) {
+    this.ensureCache();
+    if (this.cache.hasOwnProperty(tag)) {
+      return this.cache[tag];
+    }
+  }
+
+  ensureCache() {
+    if (this.cache) {
+      return;
+    }
+
+    this.cache = {};
+    for (let i = this.rules.length - 1; i >= 0; i--) {
+      let info = this.rules[i];
+      this.cache[info.rule.tag] = info;
+    }
+  }
+
+  push(name, rule) {
+    this.rules.push({ name, rule });
+    this.cache = null;
+  }
+}
+
+// block bb code ruler for parsing of quotes / code / polls
+function setupBlockBBCode(engine) {
+  engine.block.bbcode = { ruler: new Ruler() };
+}
+
+// inline bbcode ruler for parsing of spoiler tags, discourse-chart etc
+function setupInlineBBCode(engine) {
+  engine.inline.bbcode = { ruler: new Ruler() };
+}
+
+// rule for text replacement via regex, used for @mentions, category hashtags, etc.
+function setupTextPostProcessRuler(engine) {
+  engine.core.textPostProcess = { ruler: new TextPostProcessRuler() };
+}

app/assets/javascripts/discourse-markdown-it/src/features/bbcode-inline.js

@@ -1,4 +1,4 @@
-import { parseBBCodeTag } from "pretty-text/engines/discourse-markdown/bbcode-block";
+import { parseBBCodeTag } from "./bbcode-block";
 
 function tokenizeBBCode(state, silent, ruler) {
   let pos = state.pos;

app/assets/javascripts/discourse-markdown-it/src/features/d-wrap.js

@@ -1,4 +1,4 @@
-import { parseBBCodeTag } from "pretty-text/engines/discourse-markdown/bbcode-block";
+import { parseBBCodeTag } from "./bbcode-block";
 
 const WRAP_CLASS = "d-wrap";
 

app/assets/javascripts/discourse-markdown-it/src/features/index.js

@@ -0,0 +1,49 @@
+import * as anchor from "./anchor";
+import * as bbcodeBlock from "./bbcode-block";
+import * as bbcodeInline from "./bbcode-inline";
+import * as censored from "./censored";
+import * as code from "./code";
+import * as customTypographerReplacements from "./custom-typographer-replacements";
+import * as dWrap from "./d-wrap";
+import * as emoji from "./emoji";
+import * as hashtagAutocomplete from "./hashtag-autocomplete";
+import * as htmlImg from "./html-img";
+import * as imageControls from "./image-controls";
+import * as imageGrid from "./image-grid";
+import * as mentions from "./mentions";
+import * as newline from "./newline";
+import * as onebox from "./onebox";
+import * as paragraph from "./paragraph";
+import * as quotes from "./quotes";
+import * as table from "./table";
+import * as textPostProcess from "./text-post-process";
+import * as uploadProtocol from "./upload-protocol";
+import * as watchedWords from "./watched-words";
+
+export default [
+  feature("watched-words", watchedWords),
+  feature("upload-protocol", uploadProtocol),
+  feature("text-post-process", textPostProcess),
+  feature("table", table),
+  feature("quotes", quotes),
+  feature("paragraph", paragraph),
+  feature("onebox", onebox),
+  feature("newline", newline),
+  feature("mentions", mentions),
+  feature("image-grid", imageGrid),
+  feature("image-controls", imageControls),
+  feature("html-img", htmlImg),
+  feature("hashtag-autocomplete", hashtagAutocomplete),
+  feature("emoji", emoji),
+  feature("d-wrap", dWrap),
+  feature("custom-typographer-replacements", customTypographerReplacements),
+  feature("code", code),
+  feature("censored", censored),
+  feature("bbcode-inline", bbcodeInline),
+  feature("bbcode-block", bbcodeBlock),
+  feature("anchor", anchor),
+];
+
+function feature(id, { setup, priority = 0 }) {
+  return { id, setup, priority };
+}

app/assets/javascripts/discourse-markdown-it/src/index.js

@@ -0,0 +1,74 @@
+import { cook as cookIt } from "./engine";
+import DEFAULT_FEATURES from "./features";
+import buildOptions from "./options";
+import setup from "./setup";
+
+function NOOP(ident) {
+  return ident;
+}
+
+export default class DiscourseMarkdownIt {
+  static withDefaultFeatures() {
+    return this.withFeatures(DEFAULT_FEATURES);
+  }
+
+  static withCustomFeatures(features) {
+    return this.withFeatures([...DEFAULT_FEATURES, ...features]);
+  }
+
+  static withFeatures(features) {
+    const withOptions = (options) => this.withOptions(features, options);
+    return { withOptions };
+  }
+
+  static withOptions(features, rawOptions) {
+    const { options, siteSettings, state } = buildOptions(rawOptions);
+
+    // note, this will mutate options due to the way the API is designed
+    // may need a refactor
+    setup(features, options, siteSettings, state);
+
+    return new DiscourseMarkdownIt(options);
+  }
+
+  static minimal() {
+    return this.withFeatures([]).withOptions({ siteSettings: {} });
+  }
+
+  constructor(options) {
+    if (!options.setup) {
+      throw new Error(
+        "Cannot construct DiscourseMarkdownIt from raw options, " +
+          "use DiscourseMarkdownIt.withOptions() instead"
+      );
+    }
+
+    this.options = options;
+  }
+
+  disableSanitizer() {
+    this.options.sanitizer = this.options.discourse.sanitizer = NOOP;
+  }
+
+  cook(raw) {
+    if (!raw || raw.length === 0) {
+      return "";
+    }
+
+    let result;
+    result = cookIt(raw, this.options);
+    return result ? result : "";
+  }
+
+  parse(markdown, env = {}) {
+    return this.options.engine.parse(markdown, env);
+  }
+
+  sanitize(html) {
+    return this.options.sanitizer(html).trim();
+  }
+
+  get linkify() {
+    return this.options.engine.linkify;
+  }
+}

app/assets/javascripts/discourse-markdown-it/src/options.js

@@ -0,0 +1,83 @@
+import { deepMerge } from "discourse-common/lib/object";
+
+// the options are passed here and must be explicitly allowed with
+// the const options & state below
+export default function buildOptions(state) {
+  const {
+    siteSettings,
+    getURL,
+    lookupAvatar,
+    lookupPrimaryUserGroup,
+    getTopicInfo,
+    topicId,
+    forceQuoteLink,
+    userId,
+    getCurrentUser,
+    currentUser,
+    lookupAvatarByPostNumber,
+    lookupPrimaryUserGroupByPostNumber,
+    formatUsername,
+    emojiUnicodeReplacer,
+    lookupUploadUrls,
+    previewing,
+    censoredRegexp,
+    disableEmojis,
+    customEmojiTranslation,
+    watchedWordsReplace,
+    watchedWordsLink,
+    emojiDenyList,
+    featuresOverride,
+    markdownItRules,
+    additionalOptions,
+    hashtagTypesInPriorityOrder,
+    hashtagIcons,
+    hashtagLookup,
+  } = state;
+
+  let features = {};
+
+  if (state.features) {
+    features = deepMerge(features, state.features);
+  }
+
+  const options = {
+    sanitize: true,
+    getURL,
+    features,
+    lookupAvatar,
+    lookupPrimaryUserGroup,
+    getTopicInfo,
+    topicId,
+    forceQuoteLink,
+    userId,
+    getCurrentUser,
+    currentUser,
+    lookupAvatarByPostNumber,
+    lookupPrimaryUserGroupByPostNumber,
+    formatUsername,
+    emojiUnicodeReplacer,
+    lookupUploadUrls,
+    censoredRegexp,
+    customEmojiTranslation,
+    allowedHrefSchemes: siteSettings.allowed_href_schemes
+      ? siteSettings.allowed_href_schemes.split("|")
+      : null,
+    allowedIframes: siteSettings.allowed_iframes
+      ? siteSettings.allowed_iframes.split("|")
+      : [],
+    markdownIt: true,
+    previewing,
+    disableEmojis,
+    watchedWordsReplace,
+    watchedWordsLink,
+    emojiDenyList,
+    featuresOverride,
+    markdownItRules,
+    additionalOptions,
+    hashtagTypesInPriorityOrder,
+    hashtagIcons,
+    hashtagLookup,
+  };
+
+  return { options, siteSettings, state };
+}

app/assets/javascripts/discourse-markdown-it/src/setup.js

@@ -0,0 +1,359 @@
+import { textReplace } from "pretty-text/text-replace";
+import deprecated from "discourse-common/lib/deprecated";
+import { cloneJSON } from "discourse-common/lib/object";
+import makeEngine, { cook } from "./engine";
+
+// note, this will mutate options due to the way the API is designed
+// may need a refactor
+export default function setupIt(features, options, siteSettings, state) {
+  Setup.run(features, options, siteSettings, state);
+}
+
+class Setup {
+  static run(features, options, siteSettings, state) {
+    if (options.setup) {
+      // Already setup
+      return;
+    }
+
+    const setup = new Setup(options);
+
+    features.sort((a, b) => a.priority - b.priority);
+
+    for (const feature of features) {
+      setup.#setupFeature(feature.id, feature.setup);
+    }
+
+    for (const entry of Object.entries(state.allowListed ?? {})) {
+      setup.allowList(entry);
+    }
+
+    setup.#runOptionsCallbacks(siteSettings, state);
+
+    setup.#enableMarkdownFeatures();
+
+    setup.#finalizeGetOptions(siteSettings);
+
+    setup.#makeEngine();
+
+    setup.#buildCookFunctions();
+  }
+
+  #context;
+  #options;
+
+  #allowListed = [];
+  #customMarkdownCookFunctionCallbacks = [];
+  #loadedFeatures = [];
+  #optionCallbacks = [];
+  #pluginCallbacks = [];
+
+  constructor(options) {
+    options.markdownIt = true;
+
+    this.#options = options;
+
+    // hack to allow moving of getOptions – see #finalizeGetOptions
+    this.#context = { options };
+  }
+
+  allowList(entry) {
+    this.#allowListed.push(entry);
+  }
+
+  registerOptions(entry) {
+    this.#optionCallbacks.push(entry);
+  }
+
+  registerPlugin(entry) {
+    this.#pluginCallbacks.push(entry);
+  }
+
+  buildCookFunction(entry) {
+    this.#customMarkdownCookFunctionCallbacks.push(entry);
+  }
+
+  #setupFeature(featureName, callback) {
+    // When we provide the API object to the setup callback, we expect them to
+    // make use of it synchronously. However, it is possible that the could
+    // close over the API object, intentionally or unintentionally, and cause
+    // memory leaks or unexpectedly call API methods at a later time with
+    // unpredictable results. This make sure to "gut" the API object after the
+    // callback is executed so that it cannot leak memory or be used later.
+    let loaned = this;
+
+    const doSetup = (methodName, ...args) => {
+      if (loaned === null) {
+        throw new Error(
+          `${featureName}: ${methodName} can only be called during setup()!`
+        );
+      }
+
+      if (loaned[methodName]) {
+        return loaned[methodName](...args);
+      }
+    };
+
+    callback(new API(featureName, this.#context, doSetup));
+
+    this.#loadedFeatures.push(featureName);
+
+    // revoke access to the Setup object
+    loaned = null;
+  }
+
+  #runOptionsCallbacks(siteSettings, state) {
+    this.#drain(this.#optionCallbacks, ([, callback]) =>
+      callback(this.#options, siteSettings, state)
+    );
+  }
+
+  #enableMarkdownFeatures({ features, featuresOverride } = this.#options) {
+    // TODO: `options.features` could in theory contain additional keys for
+    // features that aren't loaded. The way the previous code was written
+    // incidentally means we would iterate over a super set of both. To be
+    // pedantic we kept that behavior here, but I'm not sure if that's really
+    // necessary.
+    const allFeatures = new Set([
+      ...this.#drain(this.#loadedFeatures),
+      ...Object.keys(features),
+    ]);
+
+    if (featuresOverride) {
+      for (const feature of allFeatures) {
+        features[feature] = featuresOverride.includes(feature);
+      }
+    } else {
+      // enable all features by default
+      for (let feature of allFeatures) {
+        features[feature] ??= true;
+      }
+    }
+  }
+
+  #finalizeGetOptions(siteSettings) {
+    // This is weird but essentially we want to remove `options.*` in-place
+    // into `options.discourse.*`, then, we want to change `context.options`
+    // to point at `options.discourse`. This ensures features that held onto
+    // the API object during setup will continue to get the right stuff when
+    // they call `getOptions()`.
+    const options = this.#options;
+    const discourse = {};
+
+    for (const [key, value] of Object.entries(options)) {
+      discourse[key] = value;
+      delete options[key];
+    }
+
+    discourse.helpers = { textReplace };
+
+    discourse.limitedSiteSettings = {
+      secureUploads: siteSettings.secure_uploads,
+      enableDiffhtmlPreview: siteSettings.enable_diffhtml_preview,
+      traditionalMarkdownLinebreaks:
+        siteSettings.traditional_markdown_linebreaks,
+      enableMarkdownLinkify: siteSettings.enable_markdown_linkify,
+      enableMarkdownTypographer: siteSettings.enable_markdown_typographer,
+      markdownTypographerQuotationMarks:
+        siteSettings.markdown_typographer_quotation_marks,
+      markdownLinkifyTlds: siteSettings.markdown_linkify_tlds,
+    };
+
+    this.#context.options = options.discourse = discourse;
+  }
+
+  #makeEngine() {
+    const options = this.#options;
+    const { discourse } = options;
+    const { markdownItRules, limitedSiteSettings } = discourse;
+    const {
+      enableMarkdownLinkify,
+      enableMarkdownTypographer,
+      traditionalMarkdownLinebreaks,
+    } = limitedSiteSettings;
+
+    options.allowListed = this.#drain(this.#allowListed);
+    options.pluginCallbacks = this.#drain(this.#pluginCallbacks);
+
+    const markdownItOptions = {
+      discourse,
+      html: true,
+      breaks: !traditionalMarkdownLinebreaks,
+      xhtmlOut: false,
+      linkify: enableMarkdownLinkify,
+      typographer: enableMarkdownTypographer,
+    };
+
+    makeEngine(options, markdownItOptions, markdownItRules);
+  }
+
+  #buildCookFunctions() {
+    const options = this.#options;
+
+    // the callback argument we pass to the callbacks
+    let callbackArg = (engineOptions, afterBuild) =>
+      afterBuild(this.#buildCookFunction(engineOptions, options));
+
+    this.#drain(this.#customMarkdownCookFunctionCallbacks, ([, callback]) => {
+      callback(options, callbackArg);
+    });
+  }
+
+  #buildCookFunction(engineOptions, defaultOptions) {
+    // everything except the engine for opts can just point to the other
+    // opts references, they do not change and we don't need to worry about
+    // mutating them. note that this may need to be updated when additional
+    // opts are added to the pipeline
+    const options = {};
+    options.allowListed = defaultOptions.allowListed;
+    options.pluginCallbacks = defaultOptions.pluginCallbacks;
+    options.sanitizer = defaultOptions.sanitizer;
+
+    // everything from the discourse part of defaultOptions can be cloned except
+    // the features, because these can be a limited subset and we don't want to
+    // change the original object reference
+    const features = cloneJSON(defaultOptions.discourse.features);
+    options.discourse = {
+      ...defaultOptions.discourse,
+      features,
+    };
+
+    this.#enableMarkdownFeatures({
+      features,
+      featuresOverride: engineOptions.featuresOverride,
+    });
+
+    const markdownItOptions = {
+      discourse: options.discourse,
+      html: defaultOptions.engine.options.html,
+      breaks: defaultOptions.engine.options.breaks,
+      xhtmlOut: defaultOptions.engine.options.xhtmlOut,
+      linkify: defaultOptions.engine.options.linkify,
+      typographer: defaultOptions.engine.options.typographer,
+    };
+
+    makeEngine(options, markdownItOptions, engineOptions.markdownItRules);
+
+    return function customCookFunction(raw) {
+      return cook(raw, options);
+    };
+  }
+
+  #drain(items, callback) {
+    if (callback) {
+      let item = items.shift();
+
+      while (item) {
+        callback(item);
+        item = items.shift();
+      }
+    } else {
+      const cloned = [...items];
+      items.length = 0;
+      return cloned;
+    }
+  }
+}
+
+class API {
+  #name;
+  #context;
+  #setup;
+  #deprecate;
+
+  constructor(featureName, context, setup) {
+    this.#name = featureName;
+    this.#context = context;
+    this.#setup = setup;
+    this.#deprecate = (methodName, ...args) => {
+      if (window.console && window.console.log) {
+        window.console.log(
+          featureName +
+            ": " +
+            methodName +
+            " is deprecated, please use the new markdown it APIs"
+        );
+      }
+
+      return setup(methodName, ...args);
+    };
+  }
+
+  get markdownIt() {
+    return true;
+  }
+
+  // this the only method we expect to be called post-setup()
+  getOptions() {
+    return this.#context.options;
+  }
+
+  allowList(info) {
+    this.#setup("allowList", [this.#name, info]);
+  }
+
+  whiteList(info) {
+    deprecated("`whiteList` has been replaced with `allowList`", {
+      since: "2.6.0.beta.4",
+      dropFrom: "2.7.0",
+      id: "discourse.markdown-it.whitelist",
+    });
+
+    this.allowList(info);
+  }
+
+  registerOptions(callback) {
+    this.#setup("registerOptions", [this.#name, callback]);
+  }
+
+  registerPlugin(callback) {
+    this.#setup("registerPlugin", [this.#name, callback]);
+  }
+
+  buildCookFunction(callback) {
+    this.#setup("buildCookFunction", [this.#name, callback]);
+  }
+
+  // deprecate methods – "deprecate" is a bit of a misnomer here since the
+  // methods don't actually do anything anymore
+
+  registerInline() {
+    this.#deprecate("registerInline");
+  }
+
+  replaceBlock() {
+    this.#deprecate("replaceBlock");
+  }
+
+  addPreProcessor() {
+    this.#deprecate("addPreProcessor");
+  }
+
+  inlineReplace() {
+    this.#deprecate("inlineReplace");
+  }
+
+  postProcessTag() {
+    this.#deprecate("postProcessTag");
+  }
+
+  inlineRegexp() {
+    this.#deprecate("inlineRegexp");
+  }
+
+  inlineBetween() {
+    this.#deprecate("inlineBetween");
+  }
+
+  postProcessText() {
+    this.#deprecate("postProcessText");
+  }
+
+  onParseNode() {
+    this.#deprecate("onParseNode");
+  }
+
+  registerBlock() {
+    this.#deprecate("registerBlock");
+  }
+}

app/assets/javascripts/discourse/app/lib/text.js

@@ -1,44 +1,17 @@
-import { htmlSafe } from "@ember/template";
 import AllowLister from "pretty-text/allow-lister";
 import { buildEmojiUrl, performEmojiUnescape } from "pretty-text/emoji";
-import PrettyText, { buildOptions } from "pretty-text/pretty-text";
 import { sanitize as textSanitize } from "pretty-text/sanitizer";
-import { Promise } from "rsvp";
-import loadScript from "discourse/lib/load-script";
-import { MentionsParser } from "discourse/lib/mentions-parser";
-import { formatUsername } from "discourse/lib/utilities";
-import Session from "discourse/models/session";
 import deprecated from "discourse-common/lib/deprecated";
 import { getURLWithCDN } from "discourse-common/lib/get-url";
 import { helperContext } from "discourse-common/lib/helpers";
 
-function getOpts(opts) {
-  let context = helperContext();
-
-  opts = Object.assign(
-    {
-      getURL: getURLWithCDN,
-      currentUser: context.currentUser,
-      censoredRegexp: context.site.censored_regexp,
-      customEmojiTranslation: context.site.custom_emoji_translation,
-      emojiDenyList: context.site.denied_emojis,
-      siteSettings: context.siteSettings,
-      formatUsername,
-      watchedWordsReplace: context.site.watched_words_replace,
-      watchedWordsLink: context.site.watched_words_link,
-      additionalOptions: context.site.markdown_additional_options,
-    },
-    opts
-  );
-
-  return buildOptions(opts);
+async function withEngine(name, ...args) {
+  const engine = await import("discourse/static/markdown-it");
+  return engine[name](...args);
 }
 
-export function cook(text, options) {
-  return loadMarkdownIt().then(() => {
-    const cooked = createPrettyText(options).cook(text);
-    return htmlSafe(cooked);
-  });
+export async function cook(text, options) {
+  return await withEngine("cook", text, options);
 }
 
 // todo drop this function after migrating everything to cook()
@@ -48,66 +21,38 @@ export function cookAsync(text, options) {
     dropFrom: "3.2.0.beta5",
     id: "discourse.text.cook-async",
   });
+
   return cook(text, options);
 }
 
-// Warm up pretty text with a set of options and return a function
-// which can be used to cook without rebuilding pretty-text every time
-export function generateCookFunction(options) {
-  return loadMarkdownIt().then(() => {
-    const prettyText = createPrettyText(options);
-    return (text) => prettyText.cook(text);
-  });
+// Warm up the engine with a set of options and return a function
+// which can be used to cook without rebuilding the engine every time
+export async function generateCookFunction(options) {
+  return await withEngine("generateCookFunction", options);
 }
 
-export function generateLinkifyFunction(options) {
-  return loadMarkdownIt().then(() => {
-    const prettyText = createPrettyText(options);
-    return prettyText.opts.engine.linkify;
-  });
+export async function generateLinkifyFunction(options) {
+  return await withEngine("generateLinkifyFunction", options);
 }
 
+// TODO: this one is special, it attempts to do something even without
+// the engine loaded. Currently, this is what is forcing the xss library
+// to be included on initial page load. The API/behavior also seems a bit
+// different than the async version.
 export function sanitize(text, options) {
   return textSanitize(text, new AllowLister(options));
 }
 
-export function sanitizeAsync(text, options) {
-  return loadMarkdownIt().then(() => {
-    return createPrettyText(options).sanitize(text);
-  });
+export async function sanitizeAsync(text, options) {
+  return await withEngine("sanitize", text, options);
 }
 
-export function parseAsync(md, options = {}, env = {}) {
-  return loadMarkdownIt().then(() => {
-    return createPrettyText(options).parse(md, env);
-  });
+export async function parseAsync(md, options = {}, env = {}) {
+  return await withEngine("parse", md, options, env);
 }
 
 export async function parseMentions(markdown, options) {
-  await loadMarkdownIt();
-  const prettyText = createPrettyText(options);
-  const mentionsParser = new MentionsParser(prettyText);
-  return mentionsParser.parse(markdown);
-}
-
-function loadMarkdownIt() {
-  return new Promise((resolve) => {
-    let markdownItURL = Session.currentProp("markdownItURL");
-    if (markdownItURL) {
-      loadScript(markdownItURL)
-        .then(() => resolve())
-        .catch((e) => {
-          // eslint-disable-next-line no-console
-          console.error(e);
-        });
-    } else {
-      resolve();
-    }
-  });
-}
-
-function createPrettyText(options) {
-  return new PrettyText(getOpts(options));
+  return await withEngine("parseMentions", markdown, options);
 }
 
 function emojiOptions() {

app/assets/javascripts/discourse/app/static/markdown-it/features.js

@@ -0,0 +1,25 @@
+export default function loadPluginFeatures() {
+  const features = [];
+
+  for (let moduleName of Object.keys(requirejs.entries)) {
+    if (moduleName.startsWith("discourse/plugins/")) {
+      // all of the modules under discourse-markdown or markdown-it
+      // directories are considered additional markdown "features" which
+      // may define their own rules
+      if (
+        moduleName.includes("/discourse-markdown/") ||
+        moduleName.includes("/markdown-it/")
+      ) {
+        const module = requirejs(moduleName);
+
+        if (module && module.setup) {
+          const id = moduleName.split("/").reverse()[0];
+          const { setup, priority = 0 } = module;
+          features.unshift({ id, setup, priority });
+        }
+      }
+    }
+  }
+
+  return features;
+}

app/assets/javascripts/discourse/app/static/markdown-it/index.js

@@ -0,0 +1,60 @@
+import { htmlSafe } from "@ember/template";
+import { importSync } from "@embroider/macros";
+import loaderShim from "discourse-common/lib/loader-shim";
+import DiscourseMarkdownIt from "discourse-markdown-it";
+import loadPluginFeatures from "./features";
+import MentionsParser from "./mentions-parser";
+import buildOptions from "./options";
+
+// Shims the `parseBBCodeTag` utility function back to its old location. For
+// now, there is no deprecation with this as we don't have a new location for
+// them to import from (well, we do, but we don't want to expose the new code
+// to loader.js and we want to make sure the code is loaded lazily).
+//
+// TODO: find a new home for this – the code is rather small so we could just
+// throw it into the synchronous pretty-text package and call it good, but we
+// should probably look into why plugins are needing to call this utility in
+// the first place, and provide better infrastructure for registering bbcode
+// additions instead.
+loaderShim("pretty-text/engines/discourse-markdown/bbcode-block", () =>
+  importSync("./parse-bbcode-tag")
+);
+
+function buildEngine(options) {
+  return DiscourseMarkdownIt.withCustomFeatures(
+    loadPluginFeatures()
+  ).withOptions(buildOptions(options));
+}
+
+// Use this to easily create an instance with proper options
+export function cook(text, options) {
+  return htmlSafe(buildEngine(options).cook(text));
+}
+
+// Warm up the engine with a set of options and return a function
+// which can be used to cook without rebuilding the engine every time
+export function generateCookFunction(options) {
+  const engine = buildEngine(options);
+  return (text) => engine.cook(text);
+}
+
+export function generateLinkifyFunction(options) {
+  const engine = buildEngine(options);
+  return engine.linkify;
+}
+
+export function sanitize(text, options) {
+  const engine = buildEngine(options);
+  return engine.sanitize(text);
+}
+
+export function parse(md, options = {}, env = {}) {
+  const engine = buildEngine(options);
+  return engine.parse(md, env);
+}
+
+export function parseMentions(markdown, options) {
+  const engine = buildEngine(options);
+  const mentionsParser = new MentionsParser(engine);
+  return mentionsParser.parse(markdown);
+}

app/assets/javascripts/discourse/app/static/markdown-it/mentions-parser.js

@@ -1,10 +1,10 @@
-export class MentionsParser {
-  constructor(prettyText) {
-    this.prettyText = prettyText;
+export default class MentionsParser {
+  constructor(engine) {
+    this.engine = engine;
   }
 
   parse(markdown) {
-    const tokens = this.prettyText.parse(markdown);
+    const tokens = this.engine.parse(markdown);
     const mentions = this.#parse(tokens);
     return [...new Set(mentions)];
   }

app/assets/javascripts/discourse/app/static/markdown-it/options.js

@@ -0,0 +1,21 @@
+import { formatUsername } from "discourse/lib/utilities";
+import { getURLWithCDN } from "discourse-common/lib/get-url";
+import { helperContext } from "discourse-common/lib/helpers";
+
+export default function buildOptions(options) {
+  let context = helperContext();
+
+  return {
+    getURL: getURLWithCDN,
+    currentUser: context.currentUser,
+    censoredRegexp: context.site.censored_regexp,
+    customEmojiTranslation: context.site.custom_emoji_translation,
+    emojiDenyList: context.site.denied_emojis,
+    siteSettings: context.siteSettings,
+    formatUsername,
+    watchedWordsReplace: context.site.watched_words_replace,
+    watchedWordsLink: context.site.watched_words_link,
+    additionalOptions: context.site.markdown_additional_options,
+    ...options,
+  };
+}

app/assets/javascripts/discourse/app/static/markdown-it/parse-bbcode-tag.js

@@ -0,0 +1 @@
+export { parseBBCodeTag } from "discourse-markdown-it/features/bbcode-block";

app/assets/javascripts/discourse/ember-cli-build.js

@@ -93,10 +93,6 @@ module.exports = function (defaults) {
 
   const wizardTree = app.project.findAddonByName("wizard").treeForAddonBundle();
 
-  const markdownItBundleTree = app.project
-    .findAddonByName("pretty-text")
-    .treeForMarkdownItBundle();
-
   const testStylesheetTree = mergeTrees([
     discourseScss(`${discourseRoot}/app/assets/stylesheets`, "qunit.scss"),
     discourseScss(
@@ -126,19 +122,19 @@ module.exports = function (defaults) {
       inputFiles: ["**/*.js"],
       outputFile: `assets/wizard.js`,
     }),
-    concat(markdownItBundleTree, {
-      inputFiles: ["**/*.js"],
-      outputFile: `assets/markdown-it-bundle.js`,
-    }),
     generateScriptsTree(app),
     discoursePluginsTree,
     testStylesheetTree,
   ];
 
   const appTree = compatBuild(app, Webpack, {
+    staticAppPaths: ["static"],
     packagerOptions: {
       webpackConfig: {
         devtool: "source-map",
+        output: {
+          publicPath: "auto",
+        },
         externals: [
           function ({ request }, callback) {
             if (
@@ -147,8 +143,6 @@ module.exports = function (defaults) {
               (request === "jquery" ||
                 request.startsWith("admin/") ||
                 request.startsWith("wizard/") ||
-                (request.startsWith("pretty-text/engines/") &&
-                  request !== "pretty-text/engines/discourse-markdown-it") ||
                 request.startsWith("discourse/plugins/") ||
                 request.startsWith("discourse/theme-"))
             ) {

app/assets/javascripts/discourse/package.json

@@ -84,6 +84,7 @@
     "dialog-holder": "1.0.0",
     "discourse-common": "1.0.0",
     "discourse-i18n": "1.0.0",
+    "discourse-markdown-it": "1.0.0",
     "discourse-plugins": "1.0.0",
     "ember-auto-import": "^2.6.3",
     "ember-buffered-proxy": "^2.1.1",

app/assets/javascripts/discourse/tests/index.html

@@ -61,7 +61,6 @@
 
     <script src="{{rootURL}}assets/test-i18n.js" data-embroider-ignore></script>
     <script src="{{rootURL}}assets/test-site-settings.js" data-embroider-ignore></script>
-    <script src="{{rootURL}}assets/markdown-it-bundle.js" data-embroider-ignore></script>
     <script src="{{rootURL}}assets/admin.js" data-embroider-ignore></script>
     <script src="{{rootURL}}assets/wizard.js" data-embroider-ignore></script>
 

app/assets/javascripts/discourse/tests/test-helper.js

@@ -1 +1 @@
-// We don't currently use this file, but it is require'd by ember-cli's test bundle
+import "discourse/static/markdown-it";

app/assets/javascripts/discourse/tests/unit/lib/build-quote-test.js

@@ -1,8 +1,8 @@
 import { getOwner } from "@ember/application";
 import { setupTest } from "ember-qunit";
-import PrettyText from "pretty-text/pretty-text";
 import { module, test } from "qunit";
 import { buildQuote } from "discourse/lib/quote";
+import DiscourseMarkdownIt from "discourse-markdown-it";
 
 module("Unit | Utility | build-quote", function (hooks) {
   setupTest(hooks);
@@ -60,7 +60,7 @@ module("Unit | Utility | build-quote", function (hooks) {
   test("quoting a quote", function (assert) {
     const store = getOwner(this).lookup("service:store");
     const post = store.createRecord("post", {
-      cooked: new PrettyText().cook(
+      cooked: DiscourseMarkdownIt.minimal().cook(
         '[quote="sam, post:1, topic:1, full:true"]\nhello\n[/quote]\n*Test*'
       ),
       username: "eviltrout",

app/assets/javascripts/discourse/tests/unit/lib/parse-bbcode-tag-test.js

@@ -1,6 +1,6 @@
 import { setupTest } from "ember-qunit";
-import { parseBBCodeTag } from "pretty-text/engines/discourse-markdown/bbcode-block";
 import { module, test } from "qunit";
+import { parseBBCodeTag } from "discourse-markdown-it/features/bbcode-block";
 
 module("Unit | Utility | parseBBCodeTag", function (hooks) {
   setupTest(hooks);

app/assets/javascripts/discourse/tests/unit/lib/pretty-text-test.js

@@ -1,14 +1,14 @@
 import { setupTest } from "ember-qunit";
 import { registerEmoji } from "pretty-text/emoji";
 import { IMAGE_VERSION as v } from "pretty-text/emoji/version";
-import { extractDataAttribute } from "pretty-text/engines/discourse-markdown-it";
 import {
   applyCachedInlineOnebox,
   deleteCachedInlineOnebox,
 } from "pretty-text/inline-oneboxer";
-import PrettyText, { buildOptions } from "pretty-text/pretty-text";
 import QUnit, { module, test } from "qunit";
 import { deepMerge } from "discourse-common/lib/object";
+import DiscourseMarkdownIt from "discourse-markdown-it";
+import { extractDataAttribute } from "discourse-markdown-it/engine";
 
 const rawOpts = {
   siteSettings: {
@@ -27,10 +27,12 @@ const rawOpts = {
   getURL: (url) => url,
 };
 
-const defaultOpts = buildOptions(rawOpts);
+function build(options = rawOpts) {
+  return DiscourseMarkdownIt.withDefaultFeatures().withOptions(options);
+}
 
 QUnit.assert.cooked = function (input, expected, message) {
-  const actual = new PrettyText(defaultOpts).cook(input);
+  const actual = build().cook(input);
   this.pushResult({
     result: actual === expected.replace(/\/>/g, ">"),
     actual,
@@ -41,7 +43,7 @@ QUnit.assert.cooked = function (input, expected, message) {
 
 QUnit.assert.cookedOptions = function (input, opts, expected, message) {
   const merged = deepMerge({}, rawOpts, opts);
-  const actual = new PrettyText(buildOptions(merged)).cook(input);
+  const actual = build(merged).cook(input);
   this.pushResult({
     result: actual === expected,
     actual,
@@ -59,12 +61,12 @@ module("Unit | Utility | pretty-text", function (hooks) {
 
   test("buildOptions", function (assert) {
     assert.ok(
-      buildOptions({ siteSettings: { enable_emoji: true } }).discourse.features
+      build({ siteSettings: { enable_emoji: true } }).options.discourse.features
         .emoji,
       "emoji enabled"
     );
     assert.ok(
-      !buildOptions({ siteSettings: { enable_emoji: false } }).discourse
+      !build({ siteSettings: { enable_emoji: false } }).options.discourse
         .features.emoji,
       "emoji disabled"
     );
@@ -733,7 +735,7 @@ eviltrout</p>
 
   test("Oneboxing", function (assert) {
     function matches(input, regexp) {
-      return new PrettyText(defaultOpts).cook(input).match(regexp);
+      return build().cook(input).match(regexp);
     }
 
     assert.ok(
@@ -1338,7 +1340,7 @@ var bar = 'bar';
   });
 
   test("quotes with trailing formatting", function (assert) {
-    const result = new PrettyText(defaultOpts).cook(
+    const result = build().cook(
       '[quote="EvilTrout, post:123, topic:456, full:true"]\nhello\n[/quote]\n*Test*'
     );
     assert.strictEqual(

app/assets/javascripts/discourse/tests/unit/lib/sanitizer-test.js

@@ -1,38 +1,44 @@
 import { setupTest } from "ember-qunit";
 import AllowLister from "pretty-text/allow-lister";
-import PrettyText, { buildOptions } from "pretty-text/pretty-text";
 import { hrefAllowed, sanitize } from "pretty-text/sanitizer";
 import { module, test } from "qunit";
+import DiscourseMarkdownIt from "discourse-markdown-it";
+
+function build(options) {
+  return DiscourseMarkdownIt.withDefaultFeatures().withOptions(options);
+}
 
 module("Unit | Utility | sanitizer", function (hooks) {
   setupTest(hooks);
 
   test("sanitize", function (assert) {
-    const pt = new PrettyText(
-      buildOptions({
-        siteSettings: {
-          allowed_iframes:
-            "https://www.google.com/maps/embed?|https://www.openstreetmap.org/export/embed.html?",
-        },
-      })
-    );
+    const engine = build({
+      siteSettings: {
+        allowed_iframes:
+          "https://www.google.com/maps/embed?|https://www.openstreetmap.org/export/embed.html?",
+      },
+    });
     const cooked = (input, expected, text) =>
-      assert.strictEqual(pt.cook(input), expected.replace(/\/>/g, ">"), text);
+      assert.strictEqual(
+        engine.cook(input),
+        expected.replace(/\/>/g, ">"),
+        text
+      );
 
     assert.strictEqual(
-      pt.sanitize('<i class="fa-bug fa-spin">bug</i>'),
+      engine.sanitize('<i class="fa-bug fa-spin">bug</i>'),
       "<i>bug</i>"
     );
     assert.strictEqual(
-      pt.sanitize("<div><script>alert('hi');</script></div>"),
+      engine.sanitize("<div><script>alert('hi');</script></div>"),
       "<div></div>"
     );
     assert.strictEqual(
-      pt.sanitize("<div><p class=\"funky\" wrong='1'>hello</p></div>"),
+      engine.sanitize("<div><p class=\"funky\" wrong='1'>hello</p></div>"),
       "<div><p>hello</p></div>"
     );
-    assert.strictEqual(pt.sanitize("<3 <3"), "&lt;3 &lt;3");
-    assert.strictEqual(pt.sanitize("<_<"), "&lt;_&lt;");
+    assert.strictEqual(engine.sanitize("<3 <3"), "&lt;3 &lt;3");
+    assert.strictEqual(engine.sanitize("<_<"), "&lt;_&lt;");
 
     cooked(
       "hello<script>alert(42)</script>",
@@ -71,10 +77,16 @@ module("Unit | Utility | sanitizer", function (hooks) {
       "it allows iframe to OpenStreetMap"
     );
 
-    assert.strictEqual(pt.sanitize("<textarea>hullo</textarea>"), "hullo");
-    assert.strictEqual(pt.sanitize("<button>press me!</button>"), "press me!");
-    assert.strictEqual(pt.sanitize("<canvas>draw me!</canvas>"), "draw me!");
-    assert.strictEqual(pt.sanitize("<progress>hello"), "hello");
+    assert.strictEqual(engine.sanitize("<textarea>hullo</textarea>"), "hullo");
+    assert.strictEqual(
+      engine.sanitize("<button>press me!</button>"),
+      "press me!"
+    );
+    assert.strictEqual(
+      engine.sanitize("<canvas>draw me!</canvas>"),
+      "draw me!"
+    );
+    assert.strictEqual(engine.sanitize("<progress>hello"), "hello");
 
     cooked(
       "[the answer](javascript:alert(42))",
@@ -148,62 +160,62 @@ module("Unit | Utility | sanitizer", function (hooks) {
   });
 
   test("ids on headings", function (assert) {
-    const pt = new PrettyText(buildOptions({ siteSettings: {} }));
+    const engine = build({ siteSettings: {} });
     assert.strictEqual(
-      pt.sanitize("<h3>Test Heading</h3>"),
+      engine.sanitize("<h3>Test Heading</h3>"),
       "<h3>Test Heading</h3>"
     );
     assert.strictEqual(
-      pt.sanitize(`<h1 id="heading--test">Test Heading</h1>`),
+      engine.sanitize(`<h1 id="heading--test">Test Heading</h1>`),
       `<h1 id="heading--test">Test Heading</h1>`
     );
     assert.strictEqual(
-      pt.sanitize(`<h2 id="heading--cool">Test Heading</h2>`),
+      engine.sanitize(`<h2 id="heading--cool">Test Heading</h2>`),
       `<h2 id="heading--cool">Test Heading</h2>`
     );
     assert.strictEqual(
-      pt.sanitize(`<h3 id="heading--dashed-name">Test Heading</h3>`),
+      engine.sanitize(`<h3 id="heading--dashed-name">Test Heading</h3>`),
       `<h3 id="heading--dashed-name">Test Heading</h3>`
     );
     assert.strictEqual(
-      pt.sanitize(`<h4 id="heading--underscored_name">Test Heading</h4>`),
+      engine.sanitize(`<h4 id="heading--underscored_name">Test Heading</h4>`),
       `<h4 id="heading--underscored_name">Test Heading</h4>`
     );
     assert.strictEqual(
-      pt.sanitize(`<h5 id="heading--trout">Test Heading</h5>`),
+      engine.sanitize(`<h5 id="heading--trout">Test Heading</h5>`),
       `<h5 id="heading--trout">Test Heading</h5>`
     );
     assert.strictEqual(
-      pt.sanitize(`<h6 id="heading--discourse">Test Heading</h6>`),
+      engine.sanitize(`<h6 id="heading--discourse">Test Heading</h6>`),
       `<h6 id="heading--discourse">Test Heading</h6>`
     );
   });
 
   test("autoplay videos must be muted", function (assert) {
-    let pt = new PrettyText(buildOptions({ siteSettings: {} }));
+    let engine = build({ siteSettings: {} });
     assert.ok(
-      pt
+      engine
         .sanitize(
           `<p>Hey</p><video autoplay src="http://example.com/music.mp4"/>`
         )
         .match(/muted/)
     );
     assert.ok(
-      pt
+      engine
         .sanitize(
           `<p>Hey</p><video autoplay><source src="http://example.com/music.mp4" type="audio/mpeg"></video>`
         )
         .match(/muted/)
     );
     assert.ok(
-      pt
+      engine
         .sanitize(
           `<p>Hey</p><video autoplay muted><source src="http://example.com/music.mp4" type="audio/mpeg"></video>`
         )
         .match(/muted/)
     );
     assert.notOk(
-      pt
+      engine
         .sanitize(
           `<p>Hey</p><video><source src="http://example.com/music.mp4" type="audio/mpeg"></video>`
         )
@@ -212,29 +224,29 @@ module("Unit | Utility | sanitizer", function (hooks) {
   });
 
   test("poorly formed ids on headings", function (assert) {
-    let pt = new PrettyText(buildOptions({ siteSettings: {} }));
+    let engine = build({ siteSettings: {} });
     assert.strictEqual(
-      pt.sanitize(`<h1 id="evil-trout">Test Heading</h1>`),
+      engine.sanitize(`<h1 id="evil-trout">Test Heading</h1>`),
       `<h1>Test Heading</h1>`
     );
     assert.strictEqual(
-      pt.sanitize(`<h1 id="heading--">Test Heading</h1>`),
+      engine.sanitize(`<h1 id="heading--">Test Heading</h1>`),
       `<h1>Test Heading</h1>`
     );
     assert.strictEqual(
-      pt.sanitize(`<h1 id="heading--with space">Test Heading</h1>`),
+      engine.sanitize(`<h1 id="heading--with space">Test Heading</h1>`),
       `<h1>Test Heading</h1>`
     );
     assert.strictEqual(
-      pt.sanitize(`<h1 id="heading--with*char">Test Heading</h1>`),
+      engine.sanitize(`<h1 id="heading--with*char">Test Heading</h1>`),
       `<h1>Test Heading</h1>`
     );
     assert.strictEqual(
-      pt.sanitize(`<h1 id="heading--">Test Heading</h1>`),
+      engine.sanitize(`<h1 id="heading--">Test Heading</h1>`),
       `<h1>Test Heading</h1>`
     );
     assert.strictEqual(
-      pt.sanitize(`<h1 id="test-heading--cool">Test Heading</h1>`),
+      engine.sanitize(`<h1 id="test-heading--cool">Test Heading</h1>`),
       `<h1>Test Heading</h1>`
     );
   });

app/assets/javascripts/package.json

@@ -12,6 +12,7 @@
     "discourse-common",
     "discourse-hbr",
     "discourse-i18n",
+    "discourse-markdown-it",
     "discourse-plugins",
     "discourse-widget-hbs",
     "ember-cli-progress-ci",

app/assets/javascripts/pretty-text/addon/engines/discourse-markdown-it.js

@@ -1,601 +0,0 @@
-import AllowLister from "pretty-text/allow-lister";
-import guid from "pretty-text/guid";
-import { sanitize } from "pretty-text/sanitizer";
-import deprecated from "discourse-common/lib/deprecated";
-import { cloneJSON } from "discourse-common/lib/object";
-
-export const ATTACHMENT_CSS_CLASS = "attachment";
-
-function deprecate(feature, name) {
-  return function () {
-    if (window.console && window.console.log) {
-      window.console.log(
-        feature +
-          ": " +
-          name +
-          " is deprecated, please use the new markdown it APIs"
-      );
-    }
-  };
-}
-
-// We have some custom extensions and extension points for markdown-it, including
-// the helper (passed in via setup) that has registerOptions, registerPlugin etc.
-// as well as our postProcessText rule to replace text with a regex.
-//
-// Take a look at https://meta.discourse.org/t/developers-guide-to-markdown-extensions/66023
-// for more detailed information.
-function createHelper(
-  featureName,
-  opts,
-  optionCallbacks,
-  pluginCallbacks,
-  customMarkdownCookFnCallbacks,
-  getOptions,
-  allowListed
-) {
-  let helper = {};
-  helper.markdownIt = true;
-  helper.allowList = (info) => allowListed.push([featureName, info]);
-  helper.whiteList = (info) => {
-    deprecated("`whiteList` has been replaced with `allowList`", {
-      since: "2.6.0.beta.4",
-      dropFrom: "2.7.0",
-      id: "discourse.markdown-it.whitelist",
-    });
-    helper.allowList(info);
-  };
-
-  helper.registerInline = deprecate(featureName, "registerInline");
-  helper.replaceBlock = deprecate(featureName, "replaceBlock");
-  helper.addPreProcessor = deprecate(featureName, "addPreProcessor");
-  helper.inlineReplace = deprecate(featureName, "inlineReplace");
-  helper.postProcessTag = deprecate(featureName, "postProcessTag");
-  helper.inlineRegexp = deprecate(featureName, "inlineRegexp");
-  helper.inlineBetween = deprecate(featureName, "inlineBetween");
-  helper.postProcessText = deprecate(featureName, "postProcessText");
-  helper.onParseNode = deprecate(featureName, "onParseNode");
-  helper.registerBlock = deprecate(featureName, "registerBlock");
-  // hack to allow moving of getOptions
-  helper.getOptions = () => getOptions.f();
-
-  helper.registerOptions = (callback) => {
-    optionCallbacks.push([featureName, callback]);
-  };
-
-  helper.registerPlugin = (callback) => {
-    pluginCallbacks.push([featureName, callback]);
-  };
-
-  helper.buildCookFunction = (callback) => {
-    customMarkdownCookFnCallbacks.push([featureName, callback]);
-  };
-
-  return helper;
-}
-
-// TODO we may just use a proper ruler from markdown it... this is a basic proxy
-class Ruler {
-  constructor() {
-    this.rules = [];
-  }
-
-  getRules() {
-    return this.rules;
-  }
-
-  getRuleForTag(tag) {
-    this.ensureCache();
-    if (this.cache.hasOwnProperty(tag)) {
-      return this.cache[tag];
-    }
-  }
-
-  ensureCache() {
-    if (this.cache) {
-      return;
-    }
-
-    this.cache = {};
-    for (let i = this.rules.length - 1; i >= 0; i--) {
-      let info = this.rules[i];
-      this.cache[info.rule.tag] = info;
-    }
-  }
-
-  push(name, rule) {
-    this.rules.push({ name, rule });
-    this.cache = null;
-  }
-}
-
-// block bb code ruler for parsing of quotes / code / polls
-function setupBlockBBCode(md) {
-  md.block.bbcode = { ruler: new Ruler() };
-}
-
-// inline bbcode ruler for parsing of spoiler tags, discourse-chart etc
-function setupInlineBBCode(md) {
-  md.inline.bbcode = { ruler: new Ruler() };
-}
-
-// rule for text replacement via regex, used for @mentions, category hashtags, etc.
-function setupTextPostProcessRuler(md) {
-  const TextPostProcessRuler = requirejs(
-    "pretty-text/engines/discourse-markdown/text-post-process"
-  ).TextPostProcessRuler;
-  md.core.textPostProcess = { ruler: new TextPostProcessRuler() };
-}
-
-// hoists html_raw tokens out of the render flow and replaces them
-// with a GUID. this GUID is then replaced with the final raw HTML
-// content in unhoistForCooked
-function renderHoisted(tokens, idx, options) {
-  const content = tokens[idx].content;
-  if (content && content.length > 0) {
-    let id = guid();
-    options.discourse.hoisted[id] = content;
-    return id;
-  } else {
-    return "";
-  }
-}
-
-function setupUrlDecoding(md) {
-  // this fixed a subtle issue where %20 is decoded as space in
-  // automatic urls
-  md.utils.lib.mdurl.decode.defaultChars = ";/?:@&=+$,# ";
-}
-
-// html_raw tokens, funnily enough, render raw HTML via renderHoisted and
-// unhoistForCooked
-function setupHoister(md) {
-  md.renderer.rules.html_raw = renderHoisted;
-}
-
-// videoHTML and audioHTML follow the same HTML syntax
-// as oneboxer.rb when dealing with these formats
-function videoHTML(token) {
-  const src = token.attrGet("src");
-  const origSrc = token.attrGet("data-orig-src");
-  const dataOrigSrcAttr = origSrc !== null ? `data-orig-src="${origSrc}"` : "";
-  return `<div class="video-placeholder-container" data-video-src="${src}" ${dataOrigSrcAttr}>
-  </div>`;
-}
-
-function audioHTML(token) {
-  const src = token.attrGet("src");
-  const origSrc = token.attrGet("data-orig-src");
-  const dataOrigSrcAttr = origSrc !== null ? `data-orig-src="${origSrc}"` : "";
-  return `<audio preload="metadata" controls>
-    <source src="${src}" ${dataOrigSrcAttr}>
-    <a href="${src}">${src}</a>
-  </audio>`;
-}
-
-const IMG_SIZE_REGEX =
-  /^([1-9]+[0-9]*)x([1-9]+[0-9]*)(\s*,\s*(x?)([1-9][0-9]{0,2}?)([%x]?))?$/;
-function renderImageOrPlayableMedia(tokens, idx, options, env, slf) {
-  const token = tokens[idx];
-  const alt = slf.renderInlineAsText(token.children, options, env);
-  const split = alt.split("|");
-  const altSplit = [split[0]];
-
-  // markdown-it supports returning HTML instead of continuing to render the current token
-  // see https://github.com/markdown-it/markdown-it/blob/master/docs/architecture.md#renderer
-  // handles |video and |audio alt transformations for image tags
-  if (split[1] === "video") {
-    if (
-      options.discourse.previewing &&
-      options.discourse.limitedSiteSettings.enableDiffhtmlPreview
-    ) {
-      const src = token.attrGet("src");
-      const origSrc = token.attrGet("data-orig-src");
-      const dataOrigSrcAttr =
-        origSrc !== null ? `data-orig-src="${origSrc}"` : "";
-      return `<div class="video-container">
-        <video width="100%" height="100%" preload="metadata" controls>
-          <source src="${src}" ${dataOrigSrcAttr}>
-          <a href="${src}">${src}</a>
-        </video>
-        </div>`;
-    } else {
-      if (options.discourse.previewing) {
-        return `<div class="onebox-placeholder-container">
-        <span class="placeholder-icon video"></span>
-      </div>`;
-      } else {
-        return videoHTML(token);
-      }
-    }
-  } else if (split[1] === "audio") {
-    return audioHTML(token);
-  }
-
-  // parsing ![myimage|500x300]() or ![myimage|75%]() or ![myimage|500x300, 75%]
-  for (let i = 1, match, data; i < split.length; ++i) {
-    if ((match = split[i].match(IMG_SIZE_REGEX)) && match[1] && match[2]) {
-      let width = match[1];
-      let height = match[2];
-
-      // calculate using percentage
-      if (match[5] && match[6] && match[6] === "%") {
-        let percent = parseFloat(match[5]) / 100.0;
-        width = parseInt(width * percent, 10);
-        height = parseInt(height * percent, 10);
-      }
-
-      // calculate using only given width
-      if (match[5] && match[6] && match[6] === "x") {
-        let wr = parseFloat(match[5]) / width;
-        width = parseInt(match[5], 10);
-        height = parseInt(height * wr, 10);
-      }
-
-      // calculate using only given height
-      if (match[5] && match[4] && match[4] === "x" && !match[6]) {
-        let hr = parseFloat(match[5]) / height;
-        height = parseInt(match[5], 10);
-        width = parseInt(width * hr, 10);
-      }
-
-      if (token.attrIndex("width") === -1) {
-        token.attrs.push(["width", width]);
-      }
-
-      if (token.attrIndex("height") === -1) {
-        token.attrs.push(["height", height]);
-      }
-
-      if (
-        options.discourse.previewing &&
-        match[6] !== "x" &&
-        match[4] !== "x"
-      ) {
-        token.attrs.push(["class", "resizable"]);
-      }
-    } else if ((data = extractDataAttribute(split[i]))) {
-      token.attrs.push(data);
-    } else if (split[i] === "thumbnail") {
-      token.attrs.push(["data-thumbnail", "true"]);
-    } else {
-      altSplit.push(split[i]);
-    }
-  }
-
-  const altValue = altSplit.join("|").trim();
-  if (altValue === "") {
-    token.attrSet("role", "presentation");
-  } else {
-    token.attrSet("alt", altValue);
-  }
-
-  return slf.renderToken(tokens, idx, options);
-}
-
-// we have taken over the ![]() syntax in markdown to
-// be able to render a video or audio URL as well as the
-// image using |video and |audio in the text inside []
-function setupImageAndPlayableMediaRenderer(md) {
-  md.renderer.rules.image = renderImageOrPlayableMedia;
-}
-
-function renderAttachment(tokens, idx, options, env, slf) {
-  const linkToken = tokens[idx];
-  const textToken = tokens[idx + 1];
-
-  const split = textToken.content.split("|");
-  const contentSplit = [];
-
-  for (let i = 0, data; i < split.length; ++i) {
-    if (split[i] === ATTACHMENT_CSS_CLASS) {
-      linkToken.attrs.unshift(["class", split[i]]);
-    } else if ((data = extractDataAttribute(split[i]))) {
-      linkToken.attrs.push(data);
-    } else {
-      contentSplit.push(split[i]);
-    }
-  }
-
-  if (contentSplit.length > 0) {
-    textToken.content = contentSplit.join("|");
-  }
-
-  return slf.renderToken(tokens, idx, options);
-}
-
-function setupAttachments(md) {
-  md.renderer.rules.link_open = renderAttachment;
-}
-
-function buildCustomMarkdownCookFunction(engineOpts, defaultEngineOpts) {
-  // everything except the engine for opts can just point to the other
-  // opts references, they do not change and we don't need to worry about
-  // mutating them. note that this may need to be updated when additional
-  // opts are added to the pipeline
-  const newOpts = {};
-  newOpts.allowListed = defaultEngineOpts.allowListed;
-  newOpts.pluginCallbacks = defaultEngineOpts.pluginCallbacks;
-  newOpts.sanitizer = defaultEngineOpts.sanitizer;
-  newOpts.discourse = {};
-  const featureConfig = cloneJSON(defaultEngineOpts.discourse.features);
-
-  // everything from the discourse part of defaultEngineOpts can be cloned except
-  // the features, because these can be a limited subset and we don't want to
-  // change the original object reference
-  for (const [key, value] of Object.entries(defaultEngineOpts.discourse)) {
-    if (key !== "features") {
-      newOpts.discourse[key] = value;
-    }
-  }
-
-  if (engineOpts.featuresOverride !== undefined) {
-    overrideMarkdownFeatures(featureConfig, engineOpts.featuresOverride);
-  }
-  newOpts.discourse.features = featureConfig;
-
-  const markdownitOpts = {
-    discourse: newOpts.discourse,
-    html: defaultEngineOpts.engine.options.html,
-    breaks: defaultEngineOpts.engine.options.breaks,
-    xhtmlOut: defaultEngineOpts.engine.options.xhtmlOut,
-    linkify: defaultEngineOpts.engine.options.linkify,
-    typographer: defaultEngineOpts.engine.options.typographer,
-  };
-  newOpts.engine = createMarkdownItEngineWithOpts(
-    markdownitOpts,
-    engineOpts.markdownItRules
-  );
-
-  // we have to do this to make sure plugin callbacks, allow list, and helper
-  // functions are all set up correctly for the new engine
-  setupMarkdownEngine(newOpts, featureConfig);
-
-  // we don't need the whole engine as a consumer, just a cook function
-  // will do
-  return function customRenderFn(contentToRender) {
-    return cook(contentToRender, newOpts);
-  };
-}
-
-function createMarkdownItEngineWithOpts(markdownitOpts, ruleOverrides) {
-  if (ruleOverrides !== undefined) {
-    // Preset for "zero", https://github.com/markdown-it/markdown-it/blob/master/lib/presets/zero.js
-    return globalThis.markdownit("zero", markdownitOpts).enable(ruleOverrides);
-  }
-  return globalThis.markdownit(markdownitOpts);
-}
-
-function overrideMarkdownFeatures(features, featureOverrides) {
-  if (featureOverrides !== undefined) {
-    Object.keys(features).forEach((feature) => {
-      features[feature] = featureOverrides.includes(feature);
-    });
-  }
-}
-
-function setupMarkdownEngine(opts, featureConfig) {
-  const quotation_marks =
-    opts.discourse.limitedSiteSettings.markdownTypographerQuotationMarks;
-  if (quotation_marks) {
-    opts.engine.options.quotes = quotation_marks.split("|");
-  }
-
-  opts.engine.linkify.tlds(
-    (opts.discourse.limitedSiteSettings.markdownLinkifyTlds || "").split("|")
-  );
-
-  setupUrlDecoding(opts.engine);
-  setupHoister(opts.engine);
-  setupImageAndPlayableMediaRenderer(opts.engine);
-  setupAttachments(opts.engine);
-  setupBlockBBCode(opts.engine);
-  setupInlineBBCode(opts.engine);
-  setupTextPostProcessRuler(opts.engine);
-
-  opts.pluginCallbacks.forEach(([feature, callback]) => {
-    if (featureConfig[feature]) {
-      if (callback === null || callback === undefined) {
-        // eslint-disable-next-line no-console
-        console.log("BAD MARKDOWN CALLBACK FOUND");
-        // eslint-disable-next-line no-console
-        console.log(`FEATURE IS: ${feature}`);
-      }
-      opts.engine.use(callback);
-    }
-  });
-
-  // top level markdown it notifier
-  opts.markdownIt = true;
-  opts.setup = true;
-
-  if (!opts.discourse.sanitizer || !opts.sanitizer) {
-    const allowLister = new AllowLister(opts.discourse);
-
-    opts.allowListed.forEach(([feature, info]) => {
-      allowLister.allowListFeature(feature, info);
-    });
-
-    opts.sanitizer = opts.discourse.sanitizer = !!opts.discourse.sanitize
-      ? (a) => sanitize(a, allowLister)
-      : (a) => a;
-  }
-}
-
-function unhoistForCooked(hoisted, cooked) {
-  const keys = Object.keys(hoisted);
-  if (keys.length) {
-    let found = true;
-
-    const unhoist = function (key) {
-      cooked = cooked.replace(new RegExp(key, "g"), function () {
-        found = true;
-        return hoisted[key];
-      });
-      delete hoisted[key];
-    };
-
-    while (found) {
-      found = false;
-      keys.forEach(unhoist);
-    }
-  }
-
-  return cooked;
-}
-
-export function extractDataAttribute(str) {
-  let sep = str.indexOf("=");
-  if (sep === -1) {
-    return null;
-  }
-
-  const key = `data-${str.slice(0, sep)}`.toLowerCase();
-  if (!/^[A-Za-z]+[\w\-\:\.]*$/.test(key)) {
-    return null;
-  }
-
-  const value = str.slice(sep + 1);
-  return [key, value];
-}
-
-let Helpers;
-
-export function setup(opts, siteSettings, state) {
-  if (opts.setup) {
-    return;
-  }
-
-  // we got to require this late cause bundle is not loaded in pretty-text
-  Helpers =
-    Helpers || requirejs("pretty-text/engines/discourse-markdown/helpers");
-
-  opts.markdownIt = true;
-
-  let optionCallbacks = [];
-  let pluginCallbacks = [];
-  let customMarkdownCookFnCallbacks = [];
-
-  // ideally I would like to change the top level API a bit, but in the mean time this will do
-  let getOptions = {
-    f: () => opts,
-  };
-
-  const check = /discourse-markdown\/|markdown-it\//;
-  let features = [];
-  let allowListed = [];
-
-  // all of the modules under discourse-markdown or markdown-it
-  // directories are considered additional markdown "features" which
-  // may define their own rules
-  Object.keys(require.entries).forEach((entry) => {
-    if (check.test(entry)) {
-      const module = requirejs(entry);
-      if (module && module.setup) {
-        const id = entry.split("/").reverse()[0];
-        let priority = module.priority || 0;
-        features.unshift({ id, setup: module.setup, priority });
-      }
-    }
-  });
-
-  features
-    .sort((a, b) => a.priority - b.priority)
-    .forEach((markdownFeature) => {
-      markdownFeature.setup(
-        createHelper(
-          markdownFeature.id,
-          opts,
-          optionCallbacks,
-          pluginCallbacks,
-          customMarkdownCookFnCallbacks,
-          getOptions,
-          allowListed
-        )
-      );
-    });
-
-  Object.entries(state.allowListed || {}).forEach((entry) => {
-    allowListed.push(entry);
-  });
-
-  optionCallbacks.forEach(([, callback]) => {
-    callback(opts, siteSettings, state);
-  });
-
-  // enable all features by default
-  features.forEach((feature) => {
-    if (!opts.features.hasOwnProperty(feature.id)) {
-      opts.features[feature.id] = true;
-    }
-  });
-
-  if (opts.featuresOverride !== undefined) {
-    overrideMarkdownFeatures(opts.features, opts.featuresOverride);
-  }
-
-  let copy = {};
-  Object.keys(opts).forEach((entry) => {
-    copy[entry] = opts[entry];
-    delete opts[entry];
-  });
-
-  copy.helpers = {
-    textReplace: Helpers.textReplace,
-  };
-
-  opts.discourse = copy;
-  getOptions.f = () => opts.discourse;
-
-  opts.discourse.limitedSiteSettings = {
-    secureUploads: siteSettings.secure_uploads,
-    enableDiffhtmlPreview: siteSettings.enable_diffhtml_preview,
-    traditionalMarkdownLinebreaks: siteSettings.traditional_markdown_linebreaks,
-    enableMarkdownLinkify: siteSettings.enable_markdown_linkify,
-    enableMarkdownTypographer: siteSettings.enable_markdown_typographer,
-    markdownTypographerQuotationMarks:
-      siteSettings.markdown_typographer_quotation_marks,
-    markdownLinkifyTlds: siteSettings.markdown_linkify_tlds,
-  };
-
-  const markdownitOpts = {
-    discourse: opts.discourse,
-    html: true,
-    breaks: !opts.discourse.limitedSiteSettings.traditionalMarkdownLinebreaks,
-    xhtmlOut: false,
-    linkify: opts.discourse.limitedSiteSettings.enableMarkdownLinkify,
-    typographer: opts.discourse.limitedSiteSettings.enableMarkdownTypographer,
-  };
-
-  opts.engine = createMarkdownItEngineWithOpts(
-    markdownitOpts,
-    opts.discourse.markdownItRules
-  );
-
-  opts.pluginCallbacks = pluginCallbacks;
-  opts.allowListed = allowListed;
-
-  setupMarkdownEngine(opts, opts.discourse.features);
-
-  customMarkdownCookFnCallbacks.forEach(([, callback]) => {
-    callback(opts, (engineOpts, afterBuild) =>
-      afterBuild(buildCustomMarkdownCookFunction(engineOpts, opts))
-    );
-  });
-}
-
-export function cook(raw, opts) {
-  // we still have to hoist html_raw nodes so they bypass the allowlister
-  // this is the case for oneboxes and also certain plugins that require
-  // raw HTML rendering within markdown bbcode rules
-  opts.discourse.hoisted ??= {};
-
-  const rendered = opts.engine.render(raw);
-  let cooked = opts.discourse.sanitizer(rendered).trim();
-
-  // opts.discourse.hoisted guid keys will be deleted within here to
-  // keep the object empty
-  cooked = unhoistForCooked(opts.discourse.hoisted, cooked);
-
-  return cooked;
-}

app/assets/javascripts/pretty-text/addon/pretty-text.js

@@ -1,9 +1,4 @@
-import {
-  cook as cookIt,
-  setup as setupIt,
-} from "pretty-text/engines/discourse-markdown-it";
 import deprecated from "discourse-common/lib/deprecated";
-import { deepMerge } from "discourse-common/lib/object";
 
 export function registerOption() {
   deprecated(
@@ -15,121 +10,3 @@ export function registerOption() {
     }
   );
 }
-
-// see also: __optInput in PrettyText#cook and PrettyText#markdown,
-// the options are passed here and must be explicitly allowed with
-// the const options & state below
-export function buildOptions(state) {
-  const {
-    siteSettings,
-    getURL,
-    lookupAvatar,
-    lookupPrimaryUserGroup,
-    getTopicInfo,
-    topicId,
-    forceQuoteLink,
-    userId,
-    getCurrentUser,
-    currentUser,
-    lookupAvatarByPostNumber,
-    lookupPrimaryUserGroupByPostNumber,
-    formatUsername,
-    emojiUnicodeReplacer,
-    lookupUploadUrls,
-    previewing,
-    censoredRegexp,
-    disableEmojis,
-    customEmojiTranslation,
-    watchedWordsReplace,
-    watchedWordsLink,
-    emojiDenyList,
-    featuresOverride,
-    markdownItRules,
-    additionalOptions,
-    hashtagTypesInPriorityOrder,
-    hashtagIcons,
-    hashtagLookup,
-  } = state;
-
-  let features = {};
-
-  if (state.features) {
-    features = deepMerge(features, state.features);
-  }
-
-  const options = {
-    sanitize: true,
-    getURL,
-    features,
-    lookupAvatar,
-    lookupPrimaryUserGroup,
-    getTopicInfo,
-    topicId,
-    forceQuoteLink,
-    userId,
-    getCurrentUser,
-    currentUser,
-    lookupAvatarByPostNumber,
-    lookupPrimaryUserGroupByPostNumber,
-    formatUsername,
-    emojiUnicodeReplacer,
-    lookupUploadUrls,
-    censoredRegexp,
-    customEmojiTranslation,
-    allowedHrefSchemes: siteSettings.allowed_href_schemes
-      ? siteSettings.allowed_href_schemes.split("|")
-      : null,
-    allowedIframes: siteSettings.allowed_iframes
-      ? siteSettings.allowed_iframes.split("|")
-      : [],
-    markdownIt: true,
-    previewing,
-    disableEmojis,
-    watchedWordsReplace,
-    watchedWordsLink,
-    emojiDenyList,
-    featuresOverride,
-    markdownItRules,
-    additionalOptions,
-    hashtagTypesInPriorityOrder,
-    hashtagIcons,
-    hashtagLookup,
-  };
-
-  // note, this will mutate options due to the way the API is designed
-  // may need a refactor
-  setupIt(options, siteSettings, state);
-
-  return options;
-}
-
-export default class {
-  constructor(opts) {
-    if (!opts) {
-      opts = buildOptions({ siteSettings: {} });
-    }
-    this.opts = opts;
-  }
-
-  disableSanitizer() {
-    this.opts.sanitizer = this.opts.discourse.sanitizer = (ident) => ident;
-  }
-
-  cook(raw) {
-    if (!raw || raw.length === 0) {
-      return "";
-    }
-
-    let result;
-    result = cookIt(raw, this.opts);
-    return result ? result : "";
-  }
-
-  parse(markdown, env = {}) {
-    return this.opts.engine.parse(markdown, env);
-  }
-
-  sanitize(html) {
-    return this.opts.sanitizer(html).trim();
-  }
-}

app/assets/javascripts/pretty-text/addon/text-replace.js

@@ -1,8 +1,6 @@
 // since the markdown.it interface is a bit on the verbose side
 // we can keep some general patterns here
 
-export default null;
-
 // creates a rule suitable for inline parsing and replacement
 //
 // example:

app/assets/javascripts/pretty-text/index.js

@@ -1,44 +1,8 @@
 "use strict";
 
-const Funnel = require("broccoli-funnel");
-const mergeTrees = require("broccoli-merge-trees");
-const path = require("path");
-
 module.exports = {
   name: require("./package").name,
 
-  // custom method to produce the tree for markdown-it-bundle.js
-  // called by ember-cli-build.js in discourse core
-  //
-  // code in here is only needed by the editor and we do not want them included
-  // into the main addon/vendor bundle; instead, it'll be included via a script
-  // tag as needed
-  treeForMarkdownItBundle() {
-    return mergeTrees([this._treeForEngines(), this._treeForMarkdownIt()]);
-  },
-
-  // treat the JS code in /engines like any other JS code in the /addon folder
-  _treeForEngines() {
-    let enginesTreePath = path.resolve(this.root, "engines");
-    let enginesTree = this.treeGenerator(enginesTreePath);
-
-    // we started at /engines, if we just call treeForAddon, the modules will
-    // be under pretty-text/*, but we want pretty-text/engines/*
-    let namespacedTree = new Funnel(enginesTree, {
-      destDir: "engines",
-    });
-
-    return this.treeForAddon.call(this, namespacedTree);
-  },
-
-  _treeForMarkdownIt() {
-    let markdownIt = require.resolve("markdown-it/dist/markdown-it.js");
-
-    return new Funnel(path.dirname(markdownIt), {
-      files: ["markdown-it.js"],
-    });
-  },
-
   isDevelopingAddon() {
     return true;
   },

app/assets/javascripts/pretty-text/package.json

@@ -1,7 +1,7 @@
 {
   "name": "pretty-text",
   "version": "1.0.0",
-  "description": "Discourse's text rendering pipeline",
+  "description": "Discourse's text rendering utilities",
   "author": "Discourse",
   "license": "GPL-2.0-only",
   "keywords": [
@@ -57,7 +57,6 @@
     "ember-source": "~3.28.12",
     "ember-source-channel-url": "^3.0.0",
     "loader.js": "^4.7.0",
-    "markdown-it": "^13.0.2",
     "webpack": "^5.89.0"
   },
   "engines": {

app/helpers/application_helper.rb

@@ -692,7 +692,6 @@ module ApplicationHelper
       base_uri: Discourse.base_path,
       environment: Rails.env,
       letter_avatar_version: LetterAvatar.version,
-      markdown_it_url: script_asset_path("markdown-it-bundle"),
       service_worker_url: "service-worker.js",
       default_locale: SiteSetting.default_locale,
       asset_version: Discourse.assets_digest,

lib/pretty_text.rb

@@ -28,50 +28,24 @@ module PrettyText
     Rails.root
   end
 
-  def self.find_file(root, filename)
-    return filename if File.file?("#{root}#{filename}")
-
-    es6_name = "#{filename}.js.es6"
-    return es6_name if File.file?("#{root}#{es6_name}")
-
-    js_name = "#{filename}.js"
-    return js_name if File.file?("#{root}#{js_name}")
-
-    erb_name = "#{filename}.js.es6.erb"
-    return erb_name if File.file?("#{root}#{erb_name}")
-
-    erb_name = "#{filename}.js.erb"
-    erb_name if File.file?("#{root}#{erb_name}")
+  def self.apply_es6_file(ctx:, path:, module_name:)
+    source = File.read(path)
+    transpiler = DiscourseJsProcessor::Transpiler.new
+    transpiled = transpiler.perform(source, nil, module_name)
+    ctx.eval(transpiled, filename: module_name)
   end
 
-  def self.apply_es6_file(ctx, root_path, part_name)
-    filename = find_file(root_path, part_name)
-    if filename
-      source = File.read("#{root_path}#{filename}")
-      source = ERB.new(source).result(binding) if filename =~ /\.erb\z/
-
-      transpiler = DiscourseJsProcessor::Transpiler.new
-      transpiled = transpiler.perform(source, "#{Rails.root}/app/assets/javascripts/", part_name)
-      ctx.eval(transpiled)
-    else
-      # Look for vendored stuff
-      vendor_root = "#{Rails.root}/vendor/assets/javascripts/"
-      filename = find_file(vendor_root, part_name)
-      ctx.eval(File.read("#{vendor_root}#{filename}")) if filename
-    end
-  end
-
-  def self.ctx_load_directory(ctx, path)
-    root_path = "#{Rails.root}/app/assets/javascripts/"
-    Dir["#{root_path}#{path}/**/*"].sort.each do |f|
-      apply_es6_file(ctx, root_path, f.sub(root_path, "").sub(/\.js(.es6)?\z/, ""))
+  def self.ctx_load_directory(ctx:, base_path:, module_prefix:)
+    Dir["**/*.js", base: base_path].sort.each do |f|
+      module_name = "#{module_prefix}#{f.delete_suffix(".js")}"
+      apply_es6_file(ctx: ctx, path: File.join(base_path, f), module_name: module_name)
     end
   end
 
   def self.create_es6_context
     ctx = MiniRacer::Context.new(timeout: 25_000, ensure_gc_after_idle: 2000)
 
-    ctx.eval("window = {}; window.devicePixelRatio = 2;") # hack to make code think stuff is retina
+    ctx.eval("window = globalThis; window.devicePixelRatio = 2;") # hack to make code think stuff is retina
 
     ctx.attach("rails.logger.info", proc { |err| Rails.logger.info(err.to_s) })
     ctx.attach("rails.logger.warn", proc { |err| Rails.logger.warn(err.to_s) })
@@ -91,22 +65,40 @@ module PrettyText
       ctx.attach("__helpers.#{method}", PrettyText::Helpers.method(method))
     end
 
-    root_path = "#{Rails.root}/app/assets/javascripts/"
-    ctx_load(ctx, "#{root_path}/node_modules/loader.js/dist/loader/loader.js")
-    ctx_load(ctx, "#{root_path}/handlebars-shim.js")
-    ctx_load(ctx, "#{root_path}/node_modules/xss/dist/xss.js")
+    root_path = "#{Rails.root}/app/assets/javascripts"
+    ctx.load("#{root_path}/node_modules/loader.js/dist/loader/loader.js")
+    ctx.load("#{root_path}/node_modules/markdown-it/dist/markdown-it.js")
+    ctx.load("#{root_path}/handlebars-shim.js")
+    ctx.load("#{root_path}/node_modules/xss/dist/xss.js")
     ctx.load("#{Rails.root}/lib/pretty_text/vendor-shims.js")
-    ctx_load_directory(ctx, "pretty-text/addon")
-    ctx_load_directory(ctx, "pretty-text/engines/discourse-markdown")
-    ctx_load(ctx, "#{root_path}/node_modules/markdown-it/dist/markdown-it.js")
 
-    apply_es6_file(ctx, root_path, "discourse-common/addon/lib/get-url")
-    apply_es6_file(ctx, root_path, "discourse-common/addon/lib/object")
-    apply_es6_file(ctx, root_path, "discourse-common/addon/lib/deprecated")
-    apply_es6_file(ctx, root_path, "discourse-common/addon/lib/escape")
-    apply_es6_file(ctx, root_path, "discourse-common/addon/lib/avatar-utils")
-    apply_es6_file(ctx, root_path, "discourse-common/addon/utils/watched-words")
-    apply_es6_file(ctx, root_path, "discourse/app/lib/to-markdown")
+    ctx_load_directory(
+      ctx: ctx,
+      base_path: "#{root_path}/pretty-text/addon",
+      module_prefix: "pretty-text/",
+    )
+    ctx_load_directory(
+      ctx: ctx,
+      base_path: "#{root_path}/discourse-markdown-it/src",
+      module_prefix: "discourse-markdown-it/",
+    )
+
+    %w[
+      discourse-common/addon/lib/get-url
+      discourse-common/addon/lib/object
+      discourse-common/addon/lib/deprecated
+      discourse-common/addon/lib/escape
+      discourse-common/addon/lib/avatar-utils
+      discourse-common/addon/utils/watched-words
+      discourse/app/lib/to-markdown
+      discourse/app/static/markdown-it/features
+    ].each do |f|
+      apply_es6_file(
+        ctx: ctx,
+        path: "#{root_path}/#{f}.js",
+        module_name: f.sub("/addon/", "/").sub("/app/", "/"),
+      )
+    end
 
     ctx.load("#{Rails.root}/lib/pretty_text/shims.js")
     ctx.eval("__setUnicode(#{Emoji.unicode_replacements_json})")
@@ -116,10 +108,13 @@ module PrettyText
       to_load << a if File.file?(a) && a =~ /discourse-markdown/
     end
     to_load.uniq.each do |f|
-      if f =~ %r{\A.+assets/javascripts/}
-        root = Regexp.last_match[0]
-        apply_es6_file(ctx, root, f.sub(root, "").sub(/\.js(\.es6)?\z/, ""))
-      end
+      plugin_name = f[%r{/plugins/([^/]+)/}, 1]
+      module_name = f[%r{/assets/javascripts/(.+)\.}, 1]
+      apply_es6_file(
+        ctx: ctx,
+        path: f,
+        module_name: "discourse/plugins/#{plugin_name}/#{module_name}",
+      )
     end
 
     DiscoursePluginRegistry.vendored_core_pretty_text.each { |vpt| ctx.eval(File.read(vpt)) }
@@ -227,8 +222,8 @@ module PrettyText
       buffer << "__optInput.hashtagTypesInPriorityOrder = [#{hashtag_types_as_js}];\n"
       buffer << "__optInput.hashtagIcons = #{HashtagAutocompleteService.data_source_icon_map.to_json};\n"
 
-      buffer << "__textOptions = __buildOptions(__optInput);\n"
-      buffer << ("__pt = new __PrettyText(__textOptions);")
+      buffer << "__pluginFeatures = __loadPluginFeatures();"
+      buffer << "__pt = __DiscourseMarkdownIt.withCustomFeatures(__pluginFeatures).withOptions(__optInput);"
 
       # Be careful disabling sanitization. We allow for custom emails
       buffer << ("__pt.disableSanitizer();") if opts[:sanitize] == false
@@ -666,10 +661,6 @@ module PrettyText
     rval
   end
 
-  def self.ctx_load(ctx, *files)
-    files.each { |file| ctx.load(app_root + file) }
-  end
-
   private
 
   USER_TYPE ||= "user"

lib/pretty_text/shims.js

@@ -1,11 +1,3 @@
-__PrettyText = require("pretty-text/pretty-text").default;
-__buildOptions = require("pretty-text/pretty-text").buildOptions;
-__performEmojiUnescape = require("pretty-text/emoji").performEmojiUnescape;
-__emojiReplacementRegex = require("pretty-text/emoji").emojiReplacementRegex;
-__performEmojiEscape = require("pretty-text/emoji").performEmojiEscape;
-__resetTranslationTree =
-  require("pretty-text/engines/discourse-markdown/emoji").resetTranslationTree;
-
 I18n = {
   t(a, b) {
     return __helpers.t(a, b);
@@ -28,6 +20,13 @@ define("discourse-common/lib/helpers", ["exports"], function (exports) {
   };
 });
 
+define("pretty-text/engines/discourse-markdown/bbcode-block", [
+  "exports",
+  "discourse-markdown-it/features/bbcode-block",
+], function (exports, { parseBBCodeTag }) {
+  exports.parseBBCodeTag = parseBBCodeTag;
+});
+
 __emojiUnicodeReplacer = null;
 
 __setUnicode = function (replacements) {
@@ -132,3 +131,12 @@ function __lookupPrimaryUserGroup(username) {
 function __getCurrentUser(userId) {
   return __helpers.get_current_user(userId);
 }
+
+__DiscourseMarkdownIt = require("discourse-markdown-it").default;
+__buildOptions = require("discourse-markdown-it/options").default;
+__performEmojiUnescape = require("pretty-text/emoji").performEmojiUnescape;
+__emojiReplacementRegex = require("pretty-text/emoji").emojiReplacementRegex;
+__performEmojiEscape = require("pretty-text/emoji").performEmojiEscape;
+__resetTranslationTree =
+  require("discourse-markdown-it/features/emoji").resetTranslationTree;
+__loadPluginFeatures = require("discourse/static/markdown-it/features").default;

lib/pretty_text/vendor-shims.js

@@ -19,3 +19,7 @@ define("discourse-common/lib/loader-shim", ["exports", "require"], function (
 define("xss", ["exports"], function (__exports__) {
   __exports__.default = window.filterXSS;
 });
+
+define("markdown-it", ["exports"], function (exports) {
+  exports.default = window.markdownit;
+});

plugins/discourse-details/test/javascripts/lib/details-cooked-test.js

@@ -1,7 +1,7 @@
-import PrettyText, { buildOptions } from "pretty-text/pretty-text";
 import { module, test } from "qunit";
+import { cook } from "discourse/lib/text";
 
-const defaultOpts = buildOptions({
+const opts = {
   siteSettings: {
     enable_emoji: true,
     emoji_set: "twitter",
@@ -10,24 +10,21 @@ const defaultOpts = buildOptions({
   },
   censoredWords: "shucks|whiz|whizzer",
   getURL: (url) => url,
-});
+};
 
 module("lib:details-cooked-test", function () {
-  test("details", function (assert) {
-    const cooked = (input, expected, text) => {
-      assert.strictEqual(
-        new PrettyText(defaultOpts).cook(input),
-        expected.replace(/\/>/g, ">"),
-        text
-      );
+  test("details", async function (assert) {
+    const testCooked = async (input, expected, text) => {
+      const cooked = (await cook(input, opts)).toString();
+      assert.strictEqual(cooked, expected, text);
     };
-    cooked(
+    await testCooked(
       `<details><summary>Info</summary>coucou</details>`,
       `<details><summary>Info</summary>coucou</details>`,
       "manual HTML for details"
     );
 
-    cooked(
+    await testCooked(
       "[details=testing]\ntest\n[/details]",
       `<details>
 <summary>