DEV: prosemirror

app/assets/javascripts/discourse/app/static/markdown-it/index.js

@@ -4,7 +4,7 @@ import loadPluginFeatures from "./features";
 import MentionsParser from "./mentions-parser";
 import buildOptions from "./options";
 
-function buildEngine(options) {
+export function buildEngine(options) {
   return DiscourseMarkdownIt.withCustomFeatures(
     loadPluginFeatures()
   ).withOptions(buildOptions(options));

app/assets/javascripts/discourse/app/static/prosemirror/components/prosemirror-editor.gjs

@@ -13,12 +13,14 @@ import {
 } from "discourse/lib/composer/rich-editor-extensions";
 import * as ProsemirrorModel from "prosemirror-model";
 import * as ProsemirrorView from "prosemirror-view";
+import * as ProsemirrorState from "prosemirror-state";
+import * as ProsemirrorHistory from "prosemirror-history";
+import * as ProsemirrorTransform from "prosemirror-transform";
 import { createHighlight } from "../plugins/code-highlight";
 import { baseKeymap } from "prosemirror-commands";
 import { dropCursor } from "prosemirror-dropcursor";
 import { history } from "prosemirror-history";
 import { keymap } from "prosemirror-keymap";
-import * as ProsemirrorState from "prosemirror-state";
 import { EditorState, Plugin } from "prosemirror-state";
 import { EditorView } from "prosemirror-view";
 import { bind } from "discourse-common/utils/decorators";
@@ -29,12 +31,16 @@ import { convertToMarkdown } from "../lib/serializer";
 import { buildInputRules } from "../plugins/inputrules";
 import { buildKeymap } from "../plugins/keymap";
 import placeholder from "../plugins/placeholder";
+import { gapCursor } from "prosemirror-gapcursor";
 
 export default class ProsemirrorEditor extends Component {
   @service appEvents;
   @service menu;
   @service siteSettings;
+  @service dialog;
+
   @tracked rootElement;
+
   editorContainerId = guidFor(this);
   schema = createSchema();
   view;
@@ -59,18 +65,11 @@ export default class ProsemirrorEditor extends Component {
       keymap(buildKeymap(this.schema, keymapFromArgs)),
       keymap(baseKeymap),
       dropCursor({ color: "var(--primary)" }),
+      gapCursor(),
       history(),
       placeholder(this.args.placeholder),
       createHighlight(),
-      ...getPlugins().map((plugin) =>
+      ...getPlugins().flatMap(processPlugin),
-        typeof plugin === "function"
-          ? plugin({
-              ...ProsemirrorState,
-              ...ProsemirrorModel,
-              ...ProsemirrorView,
-            })
-          : new Plugin(plugin)
-      ),
     ];
 
     this.state = EditorState.create({
@@ -126,9 +125,14 @@ export default class ProsemirrorEditor extends Component {
 
   @bind
   convertFromValue() {
-    const doc = convertFromMarkdown(this.schema, this.args.value);
+    let doc;
-
+    try {
-    // console.log("Resulting doc:", doc);
+      doc = convertFromMarkdown(this.schema, this.args.value);
+    } catch (e) {
+      console.error(e);
+      this.dialog.alert(e.message);
+      return;
+    }
 
     const tr = this.state.tr
       .replaceWith(0, this.state.doc.content.size, doc.content)
@@ -151,3 +155,19 @@ export default class ProsemirrorEditor extends Component {
     </div>
   </template>
 }
+
+function processPlugin(plugin) {
+  if (typeof plugin === "function") {
+    return plugin({
+      ...ProsemirrorState,
+      ...ProsemirrorModel,
+      ...ProsemirrorView,
+      ...ProsemirrorHistory,
+      ...ProsemirrorTransform,
+    });
+  }
+  if (plugin instanceof Array) {
+    return plugin.map(processPlugin);
+  }
+  return new Plugin(plugin);
+}

app/assets/javascripts/discourse/app/static/prosemirror/extensions/code-lang-selector.js

@@ -33,8 +33,6 @@ class CodeBlockWithLangSelectorNodeView {
 
     this.dom.appendChild(select);
 
-    // TODO(renato): leaving with the keyboard to before the node doesn't work
-
     const code = document.createElement("code");
     this.dom.appendChild(document.createElement("pre")).appendChild(code);
     this.contentDOM = code;

app/assets/javascripts/discourse/app/static/prosemirror/extensions/emoji.js

@@ -1,7 +1,9 @@
 import { buildEmojiUrl, emojiExists, isCustomEmoji } from "pretty-text/emoji";
+import { translations } from "pretty-text/emoji/data";
 import { emojiOptions } from "discourse/lib/text";
+import { isBoundary } from "discourse/static/prosemirror/lib/markdown-it";
+import escapeRegExp from "discourse-common/utils/escape-regexp";
 
-// TODO(renato): we need to avoid the invalid text:emoji: state (reminder to use isPunctChar to avoid deleting the space)
 export default {
   nodeSpec: {
     emoji: {
@@ -52,6 +54,18 @@ export default {
       },
       options: { undoable: false },
     },
+    {
+      match: new RegExp(
+        `(?<=^|\\W)(${Object.keys(translations).map(escapeRegExp).join("|")})$`
+      ),
+      handler: (state, match, start, end) => {
+        return state.tr.replaceWith(
+          start,
+          end,
+          state.schema.nodes.emoji.create({ code: translations[match[1]] })
+        );
+      },
+    },
   ],
 
   parse: {
@@ -64,7 +78,11 @@ export default {
   },
 
   serializeNode: {
-    emoji: (state, node) => {
+    emoji(state, node) {
+      if (!isBoundary(state.out, state.out.length - 1)) {
+        state.write(" ");
+      }
+
       state.write(`:${node.attrs.code}:`);
     },
   },

app/assets/javascripts/discourse/app/static/prosemirror/extensions/hashtag.js

@@ -1,3 +1,5 @@
+import { isBoundary } from "discourse/static/prosemirror/lib/markdown-it";
+
 export default {
   nodeSpec: {
     hashtag: {
@@ -30,7 +32,7 @@ export default {
 
   inputRules: [
     {
-      match: /(?<=^|\W)#([\u00C0-\u1FFF\u2C00-\uD7FF\w:-]{1,101}) $/,
+      match: /(?<=^|\W)#([\u00C0-\u1FFF\u2C00-\uD7FF\w:-]{1,101})\s$/,
       handler: (state, match, start, end) =>
         state.selection.$from.nodeBefore?.type !== state.schema.nodes.hashtag &&
         state.tr.replaceWith(start, end, [
@@ -42,7 +44,7 @@ export default {
   ],
 
   parse: {
-    span: (state, token, tokens, i) => {
+    span(state, token, tokens, i) {
       if (token.attrGet("class") === "hashtag-raw") {
         state.openNode(state.schema.nodes.hashtag, {
           name: tokens[i + 1].content.slice(1),
@@ -53,8 +55,18 @@ export default {
   },
 
   serializeNode: {
-    hashtag: (state, node) => {
+    hashtag(state, node, parent, index) {
+      if (!isBoundary(state.out, state.out.length - 1)) {
+        state.write(" ");
+      }
+
       state.write(`#${node.attrs.name}`);
+
+      const nextSibling =
+        parent.childCount > index + 1 ? parent.child(index + 1) : null;
+      if (nextSibling?.isText && !isBoundary(nextSibling.text, 0)) {
+        state.write(" ");
+      }
     },
   },
 };

app/assets/javascripts/discourse/app/static/prosemirror/extensions/html-block.js

@@ -1,6 +1,47 @@
 export default {
   nodeSpec: {
-    // TODO(renato): html_block should be like a passthrough code block
+    html_block: {
-    html_block: { block: "paragraph", noCloseToken: true },
+      attrs: {
+        content: { default: "" },
+      },
+      group: "block",
+      content: "block*",
+      // it's too broad to be automatically parsed
+      parseDOM: [],
+      toDOM: (node) => {
+        const dom = document.createElement("template");
+        dom.innerHTML = node.attrs.content;
+        return dom.content.firstChild;
+      },
+    },
+  },
+  parse: {
+    // TODO(renato): should html_block be like a passthrough code block?
+    html_block: (state, token) => {
+      const openMatch = token.content.match(
+        /^<([a-zA-Z][a-zA-Z0-9-]*)(?:\s[^>]*)?>.*/
+      );
+      const closeMatch = token.content.match(
+        /^<\/([a-zA-Z][a-zA-Z0-9-]*)>\s*$/
+      );
+
+      if (openMatch) {
+        state.openNode(state.schema.nodes.html_block, {
+          content: openMatch[0],
+        });
+
+        return;
+      }
+
+      if (closeMatch) {
+        state.closeNode();
+      }
+    },
+  },
+  serializeNode: {
+    html_block: (state, node) => {
+      state.write(node.attrs.content);
+      state.renderContent(node);
+    },
   },
 };

app/assets/javascripts/discourse/app/static/prosemirror/extensions/html-inline.js

@@ -28,6 +28,7 @@ export default {
     html_inline: {
       group: "inline",
       inline: true,
+      isolating: true,
       content: "inline*",
       attrs: { tag: {} },
       parseDOM: ALLOWED_INLINE.map((tag) => ({ tag })),
@@ -37,8 +38,8 @@ export default {
   parse: {
     // TODO(renato): it breaks if it's missing an end tag
     html_inline: (state, token) => {
-      const openMatch = token.content.match(/^<([a-z]+)>$/u);
+      const openMatch = token.content.match(/^<([a-z]+)>$/);
-      const closeMatch = token.content.match(/^<\/([a-z]+)>$/u);
+      const closeMatch = token.content.match(/^<\/([a-z]+)>$/);
 
       if (openMatch) {
         const tagName = openMatch[1];

app/assets/javascripts/discourse/app/static/prosemirror/extensions/index.js

@@ -18,8 +18,8 @@ import typographerReplacements from "./typographer-replacements";
 import underline from "./underline";
 
 const defaultExtensions = [
+  // emoji before image
   emoji,
-  // image must be after emoji
   image,
   hashtag,
   mention,
@@ -27,16 +27,17 @@ const defaultExtensions = [
   underline,
   htmlInline,
   htmlBlock,
+  // onebox before link
+  onebox,
   link,
   heading,
   codeBlock,
   quote,
-  onebox,
   trailingParagraph,
   typographerReplacements,
   markdownPaste,
 
-  // table must  be last
+  // table last
   table,
 ];
 

app/assets/javascripts/discourse/app/static/prosemirror/extensions/link.js

@@ -1,25 +1,114 @@
-export default {
+import { getLinkify } from "../lib/markdown-it";
-  inputRules: [
+
-    // []() replacement
+const markdownUrlInputRule = ({ schema, markInputRule }) =>
-    ({ schema, markInputRule }) =>
   markInputRule(
     /\[([^\]]+)]\(([^)\s]+)(?:\s+[“"']([^“"']+)[”"'])?\)$/,
     schema.marks.link,
     (match) => {
       return { href: match[2], title: match[3] };
     }
-      ),
+  );
-    // TODO(renato): auto-linkify when typing (https://github.com/markdown-it/markdown-it/blob/master/lib/rules_inline/autolink.mjs)
+
+export default {
+  markSpec: {
+    link: {
+      attrs: {
+        href: {},
+        title: { default: null },
+        autoLink: { default: null },
+      },
+      inclusive: false,
+      parseDOM: [
+        {
+          tag: "a[href]",
+          getAttrs(dom) {
+            return {
+              href: dom.getAttribute("href"),
+              title: dom.getAttribute("title"),
+            };
+          },
+        },
       ],
-  plugins: ({ Plugin, Slice, Fragment }) =>
+      toDOM(node) {
+        return ["a", { href: node.attrs.href, title: node.attrs.title }];
+      },
+    },
+  },
+  parse: {
+    link: {
+      mark: "link",
+      getAttrs: (tok) => ({
+        href: tok.attrGet("href"),
+        title: tok.attrGet("title") || null,
+        autoLink: tok.markup === "autolink",
+      }),
+    },
+  },
+  inputRules: [markdownUrlInputRule],
+  plugins: ({
+    Plugin,
+    Slice,
+    Fragment,
+    undoDepth,
+    ReplaceAroundStep,
+    ReplaceStep,
+    AddMarkStep,
+    RemoveMarkStep,
+  }) =>
     new Plugin({
+      // Auto-linkify typed URLs
+      appendTransaction: (transactions, prevState, state) => {
+        const isUndo = undoDepth(prevState) - undoDepth(state) === 1;
+        if (isUndo) {
+          return;
+        }
+
+        const docChanged = transactions.some(
+          (transaction) => transaction.docChanged
+        );
+        if (!docChanged) {
+          return;
+        }
+
+        const composedTransaction = composeSteps(transactions, prevState);
+        const changes = getChangedRanges(
+          composedTransaction,
+          [ReplaceAroundStep, ReplaceStep],
+          [AddMarkStep, ReplaceAroundStep, ReplaceStep, RemoveMarkStep]
+        );
+        const { mapping } = composedTransaction;
+        const { tr, doc } = state;
+
+        for (const { prevFrom, prevTo, from, to } of changes) {
+          findTextBlocksInRange(doc, { from, to }).forEach(
+            ({ text, positionStart }) => {
+              const matches = getLinkify().match(text);
+              if (!matches) {
+                return;
+              }
+
+              for (const match of matches) {
+                const { index, lastIndex, raw } = match;
+                const start = positionStart + index;
+                const end = positionStart + lastIndex + 1;
+                const href = raw;
+                // TODO not ready yet
+                // tr.setMeta("autolinking", true).addMark(
+                //   start,
+                //   end,
+                //   state.schema.marks.link.create({ href })
+                // );
+              }
+            }
+          );
+        }
+
+        return tr;
+      },
       props: {
         // Auto-linkify plain-text pasted URLs
-        // TODO(renato): URLs copied from HTML will go through the regular HTML parsing
-        //  it would be nice to auto-linkify them too
         clipboardTextParser(text, $context, plain, view) {
-          // TODO(renato): a less naive regex, reuse existing
+          if (view.state.selection.empty || !getLinkify().test(text)) {
-          if (!text.match(/^https?:\/\//) || view.state.selection.empty) {
             return;
           }
 
@@ -34,6 +123,119 @@ export default {
           ]);
           return new Slice(Fragment.from(textNode), 0, 0);
         },
+
+        // Auto-linkify rich content with a single text node that is a URL
+        transformPasted(paste, view) {
+          if (
+            paste.content.childCount === 1 &&
+            paste.content.firstChild.isText &&
+            !paste.content.firstChild.marks.some(
+              (mark) => mark.type.name === "link"
+            )
+          ) {
+            const matches = linkify.match(paste.content.firstChild.text);
+            const isFullMatch =
+              matches &&
+              matches.length === 1 &&
+              matches[0].raw === paste.content.firstChild.text;
+
+            if (!isFullMatch) {
+              return paste;
+            }
+
+            const marks = view.state.selection.$head.marks();
+            const originalText = view.state.doc.textBetween(
+              view.state.selection.from,
+              view.state.selection.to
+            );
+            const textNode = view.state.schema.text(originalText, [
+              ...marks,
+              view.state.schema.marks.link.create({
+                href: paste.content.firstChild.text,
+              }),
+            ]);
+            paste = new Slice(Fragment.from(textNode), 0, 0);
+          }
+          return paste;
+        },
       },
     }),
 };
+
+function composeSteps(transactions, prevState) {
+  const { tr } = prevState;
+
+  transactions.forEach((transaction) => {
+    transaction.steps.forEach((step) => {
+      tr.step(step);
+    });
+  });
+
+  return tr;
+}
+
+function getChangedRanges(tr, replaceTypes, rangeTypes) {
+  const ranges = [];
+  const { steps, mapping } = tr;
+  const inverseMapping = mapping.invert();
+
+  steps.forEach((step, i) => {
+    if (!isValidStep(step, replaceTypes)) {
+      return;
+    }
+
+    const rawRanges = [];
+    const stepMap = step.getMap();
+    const mappingSlice = mapping.slice(i);
+
+    if (stepMap.ranges.length === 0 && isValidStep(step, rangeTypes)) {
+      const { from, to } = step;
+      rawRanges.push({ from, to });
+    } else {
+      stepMap.forEach((from, to) => {
+        rawRanges.push({ from, to });
+      });
+    }
+
+    rawRanges.forEach((range) => {
+      const from = mappingSlice.map(range.from, -1);
+      const to = mappingSlice.map(range.to);
+
+      ranges.push({
+        from,
+        to,
+        prevFrom: inverseMapping.map(from, -1),
+        prevTo: inverseMapping.map(to),
+      });
+    });
+  });
+
+  return ranges.sort((a, z) => a.from - z.from);
+}
+
+function isValidStep(step, types) {
+  return types.some((type) => step instanceof type);
+}
+
+function findTextBlocksInRange(doc, range) {
+  const nodesWithPos = [];
+
+  // define a placeholder for leaf nodes to calculate link position
+  doc.nodesBetween(range.from, range.to, (node, pos) => {
+    if (!node.isTextblock || !node.type.allowsMarkType("link")) {
+      return;
+    }
+
+    nodesWithPos.push({ node, pos });
+  });
+
+  return nodesWithPos.map((textBlock) => ({
+    text: doc.textBetween(
+      textBlock.pos,
+      textBlock.pos + textBlock.node.nodeSize,
+      undefined,
+      " "
+    ),
+    positionStart: textBlock.pos,
+  }));
+}

app/assets/javascripts/discourse/app/static/prosemirror/extensions/mention.js

@@ -1,5 +1,3 @@
-// TODO(renato): similar to emoji, avoid joining anything@mentions, as it's invalid markdown
-
 import { mentionRegex } from "pretty-text/mentions";
 
 export default {

app/assets/javascripts/discourse/app/static/prosemirror/extensions/onebox.js

@@ -1,13 +1,19 @@
-import { cachedInlineOnebox } from "pretty-text/inline-oneboxer";
+import {
+  applyCachedInlineOnebox,
+  cachedInlineOnebox,
+} from "pretty-text/inline-oneboxer";
+import { addToLoadingQueue, loadNext } from "pretty-text/oneboxer";
 import { lookupCache } from "pretty-text/oneboxer-cache";
+import { ajax } from "discourse/lib/ajax";
+import { isBoundary } from "discourse/static/prosemirror/lib/markdown-it";
+import escapeRegExp from "discourse-common/utils/escape-regexp";
 
 export default {
   nodeSpec: {
     onebox: {
       attrs: { url: {}, html: {} },
-      selectable: false,
+      selectable: true,
-      group: "inline",
+      group: "block",
-      inline: true,
       atom: true,
       draggable: true,
       parseDOM: [
@@ -19,18 +25,58 @@ export default {
         },
       ],
       toDOM(node) {
-        // const dom = document.createElement("aside");
+        const dom = document.createElement("div");
-        // dom.outerHTML = node.attrs.html;
+        dom.classList.add("onebox-wrapper");
-
+        dom.innerHTML = node.attrs.html;
-        // TODO(renato): revisit?
+        return dom;
-        return new DOMParser().parseFromString(node.attrs.html, "text/html")
+      },
-          .body.firstChild;
+    },
+    onebox_inline: {
+      attrs: { url: {}, title: {} },
+      inline: true,
+      group: "inline",
+      selectable: true,
+      atom: true,
+      draggable: true,
+      parseDOM: [
+        {
+          // TODO link marks are still processed before this when pasting
+          tag: "a.inline-onebox",
+          getAttrs(dom) {
+            return { url: dom.getAttribute("href"), title: dom.textContent };
+          },
+        },
+      ],
+      toDOM(node) {
+        return [
+          "a",
+          {
+            class: "inline-onebox",
+            href: node.attrs.url,
+            contentEditable: false,
+          },
+          node.attrs.title,
+        ];
       },
     },
   },
   serializeNode: {
     onebox(state, node) {
-      state.write(node.attrs.url);
+      state.ensureNewLine();
+      state.write(`${node.attrs.url}\n\n`);
+    },
+    onebox_inline(state, node, parent, index) {
+      if (!isBoundary(state.out, state.out.length - 1)) {
+        state.write(" ");
+      }
+
+      state.text(node.attrs.url);
+
+      const nextSibling =
+        parent.childCount > index + 1 ? parent.child(index + 1) : null;
+      if (nextSibling?.isText && !isBoundary(nextSibling.text, 0)) {
+        state.write(" ");
+      }
     },
   },
 
@@ -38,37 +84,102 @@ export default {
     const plugin = new Plugin({
       state: {
         init() {
-          return [];
+          return { full: {}, inline: {} };
         },
         apply(tr, value) {
-          // TODO(renato)
+          const updated = { full: [], inline: [] };
-          return value;
+
+          // we shouldn't check all descendants, but only the ones that have changed
+          // it's a problem in other plugins too where we need to optimize
+          tr.doc.descendants((node, pos) => {
+            // if node has the link mark
+            const link = node.marks.find((mark) => mark.type.name === "link");
+            if (
+              !tr.getMeta("autolinking") &&
+              !link?.attrs.autoLink &&
+              link?.attrs.href === node.textContent
+            ) {
+              const resolvedPos = tr.doc.resolve(pos);
+
+              const isAtRoot = resolvedPos.depth === 1;
+
+              const parent = resolvedPos.parent;
+              const index = resolvedPos.index();
+              const prev = index > 0 ? parent.child(index - 1) : null;
+              const next =
+                index < parent.childCount - 1 ? parent.child(index + 1) : null;
+
+              const isAlone =
+                (!prev || prev.type.name === "hard_break") &&
+                (!next || next.type.name === "hard_break");
+
+              const isInline = !isAtRoot || !isAlone;
+
+              const obj = isInline ? updated.inline : updated.full;
+
+              obj[node.textContent] ??= [];
+              obj[node.textContent].push(pos);
+            }
+          });
+
+          return updated;
         },
       },
 
       view() {
         return {
-          update(view, prevState) {
+          async update(view, prevState) {
             if (prevState.doc.eq(view.state.doc)) {
               return;
             }
 
-            // console.log("discourse", view.props.discourse);
+            const { full, inline } = plugin.getState(view.state);
 
-            const unresolvedLinks = plugin.getState(view.state);
+            for (const [url, list] of Object.entries(full)) {
+              const html = await loadFullOnebox(url, view.props.discourse);
 
-            // console.log(unresolvedLinks);
+              // naive check that this is not a <a href="url">url</a> onebox response
+              if (
+                new RegExp(
+                  `<a href=["']${escapeRegExp(url)}["'].*>${escapeRegExp(
+                    url
+                  )}</a>`
+                ).test(html)
+              ) {
+                continue;
+              }
 
-            for (const unresolved of unresolvedLinks) {
+              const tr = view.state.tr;
-              const isInline = unresolved.isInline;
+              for (const pos of list) {
-              // console.log(isInline, cachedInlineOnebox(unresolved.text));
+                const node = tr.doc.nodeAt(pos);
+                tr.replaceWith(
+                  pos - 1,
+                  pos + node.nodeSize,
+                  view.state.schema.nodes.onebox.create({ url, html })
+                );
+              }
+              tr.setMeta("addToHistory", false);
+              view.dispatch(tr);
+            }
 
-              const className = isInline
+            const inlineOneboxes = await loadInlineOneboxes(
-                ? "onebox-loading"
+              Object.keys(inline),
-                : "inline-onebox-loading";
+              view.props.discourse
+            );
 
-              if (!isInline) {
+            for (const [url, onebox] of Object.entries(inlineOneboxes)) {
-                // console.log(lookupCache(unresolved.text));
+              for (const pos of inline[url]) {
+                const tr = view.state.tr;
+                tr.replaceWith(
+                  pos,
+                  pos + tr.doc.nodeAt(pos).nodeSize,
+                  view.state.schema.nodes.onebox_inline.create({
+                    url,
+                    title: onebox.title,
+                  })
+                );
+                tr.setMeta("addToHistory", false);
+                view.dispatch(tr);
               }
             }
           },
@@ -80,18 +191,45 @@ export default {
   },
 };
 
-function isValidUrl(text) {
+async function loadInlineOneboxes(urls, { categoryId, topicId }) {
-  try {
+  const allOneboxes = {};
-    new URL(text); // If it can be parsed as a URL, it's valid.
+
-    return true;
+  const uncachedUrls = [];
-  } catch {
+  for (const url of urls) {
-    return false;
+    const cached = cachedInlineOnebox(url);
+    if (cached) {
+      allOneboxes[url] = cached;
+    } else {
+      uncachedUrls.push(url);
     }
   }
 
-function isNodeInline(state, pos) {
+  if (uncachedUrls.length === 0) {
-  const resolvedPos = state.doc.resolve(pos);
+    return allOneboxes;
-  const parent = resolvedPos.parent;
+  }
-
+
-  return parent.childCount !== 1;
+  const { "inline-oneboxes": oneboxes } = await ajax("/inline-onebox", {
+    data: { urls: uncachedUrls, categoryId, topicId },
+  });
+
+  oneboxes.forEach((onebox) => {
+    if (onebox.title) {
+      applyCachedInlineOnebox(onebox.url, onebox);
+      allOneboxes[onebox.url] = onebox;
+    }
+  });
+
+  return allOneboxes;
+}
+
+async function loadFullOnebox(url, { categoryId, topicId }) {
+  const cached = lookupCache(url);
+  if (cached) {
+    return cached;
+  }
+
+  return new Promise((onResolve) => {
+    addToLoadingQueue({ url, categoryId, topicId, onResolve });
+    loadNext(ajax);
+  });
 }

app/assets/javascripts/discourse/app/static/prosemirror/extensions/table.js

@@ -17,6 +17,8 @@ export default {
       group: "block",
       tableRole: "table",
       isolating: true,
+      selectable: true,
+      draggable: true,
       parseDOM: [{ tag: "table" }],
       toDOM() {
         return ["table", 0];
@@ -124,10 +126,13 @@ export default {
     table(state, node) {
       state.flushClose(1);
 
-      let headerBuffer = state.delim && state.atBlank() ? state.delim : "";
+      let headerBuffer = state.delim;
       const prevInTable = state.inTable;
       state.inTable = true;
 
+      // leading newline, it seems to have issues in a line just below a > blockquote otherwise
+      state.out += "\n";
+
       // group is table_head or table_body
       node.forEach((group, groupOffset, groupIndex) => {
         group.forEach((row) => {

app/assets/javascripts/discourse/app/static/prosemirror/extensions/trailing-paragraph.js

@@ -16,16 +16,27 @@ export default {
       },
       state: {
         init(_, state) {
-          return state.doc.lastChild.type !== state.schema.nodes.paragraph;
+          return !isLastChildEmptyParagraph(state);
         },
         apply(tr, value) {
           if (!tr.docChanged) {
             return value;
           }
 
-          return tr.doc.lastChild.type !== tr.doc.type.schema.nodes.paragraph;
+          return !isLastChildEmptyParagraph(tr);
         },
       },
     });
   },
 };
+
+function isLastChildEmptyParagraph(state) {
+  const { doc } = state;
+  const lastChild = doc.lastChild;
+
+  return (
+    lastChild.type.name === "paragraph" &&
+    lastChild.nodeSize === 2 &&
+    lastChild.content.size === 0
+  );
+}

app/assets/javascripts/discourse/app/static/prosemirror/lib/markdown-it.js

@@ -0,0 +1,24 @@
+import { buildEngine } from "discourse/static/markdown-it";
+import loadPluginFeatures from "discourse/static/markdown-it/features";
+import defaultFeatures from "discourse-markdown-it/features/index";
+
+let engine;
+
+function getEngine() {
+  engine ??= buildEngine({
+    featuresOverride: [...defaultFeatures, ...loadPluginFeatures()]
+      .map(({ id }) => id)
+      // Avoid oneboxing when parsing, we'll handle that separately
+      .filter((id) => id !== "onebox"),
+  });
+
+  return engine;
+}
+
+export const parse = (text) => getEngine().parse(text);
+
+export const getLinkify = () => getEngine().linkify;
+
+export const isBoundary = (str, index) =>
+  getEngine().options.engine.utils.isWhiteSpace(str.charCodeAt(index)) ||
+  getEngine().options.engine.utils.isMdAsciiPunct(str.charCodeAt(index));

app/assets/javascripts/discourse/app/static/prosemirror/lib/parser.js

@@ -1,8 +1,6 @@
 import { defaultMarkdownParser, MarkdownParser } from "prosemirror-markdown";
 import { getParsers } from "discourse/lib/composer/rich-editor-extensions";
-import { parse as markdownItParse } from "discourse/static/markdown-it";
+import { parse } from "./markdown-it";
-import loadPluginFeatures from "discourse/static/markdown-it/features";
-import defaultFeatures from "discourse-markdown-it/features/index";
 
 // TODO(renato): We need a workaround for this parsing issue:
 //   https://github.com/ProseMirror/prosemirror-markdown/issues/82
@@ -19,9 +17,9 @@ const postParseTokens = {
   softbreak: (state) => state.addNode(state.schema.nodes.hard_break),
 };
 
-let parseOptions;
+let initialized;
-function initializeParser() {
+function loadCustomParsers() {
-  if (parseOptions) {
+  if (initialized) {
     return;
   }
 
@@ -33,18 +31,13 @@ function initializeParser() {
     }
   }
 
-  const featuresOverride = [...defaultFeatures, ...loadPluginFeatures()]
+  initialized = true;
-    .map(({ id }) => id)
-    // Avoid oneboxing when parsing, we'll handle that separately
-    .filter((id) => id !== "onebox");
-
-  parseOptions = { featuresOverride };
 }
 
 export function convertFromMarkdown(schema, text) {
-  initializeParser();
+  loadCustomParsers();
 
-  const tokens = markdownItParse(text, parseOptions);
+  const tokens = parse(text);
 
   console.log("Converting tokens", tokens);
 

app/assets/javascripts/discourse/app/static/prosemirror/plugins/keymap.js

@@ -1,8 +1,6 @@
 import {
   chainCommands,
   exitCode,
-  joinDown,
-  joinUp,
   lift,
   selectParentNode,
   setBlockType,
@@ -48,8 +46,6 @@ export function buildKeymap(schema, initialKeymap = {}, suppressKeys) {
     bind("Mod-y", redo);
   }
 
-  bind("Alt-ArrowUp", joinUp);
-  bind("Alt-ArrowDown", joinDown);
   bind("Mod-BracketLeft", lift);
   bind("Escape", selectParentNode);
 

app/assets/javascripts/discourse/package.json

@@ -36,6 +36,7 @@
     "pretty-text": "workspace:1.0.0",
     "prosemirror-commands": "^1.6.0",
     "prosemirror-dropcursor": "^1.8.1",
+    "prosemirror-gapcursor": "^1.3.2",
     "prosemirror-highlight": "^0.11.0",
     "prosemirror-history": "^1.4.1",
     "prosemirror-inputrules": "^1.4.0",
@@ -44,6 +45,7 @@
     "prosemirror-model": "^1.23.0",
     "prosemirror-schema-list": "^1.4.1",
     "prosemirror-state": "^1.4.3",
+    "prosemirror-transform": "^1.10.2",
     "prosemirror-view": "^1.34.3"
   },
   "devDependencies": {

app/assets/javascripts/pretty-text/addon/oneboxer.js

@@ -54,7 +54,7 @@ function _handleLoadingOneboxImages() {
   this.removeEventListener("load", _handleLoadingOneboxImages);
 }
 
-function loadNext(ajax) {
+export function loadNext(ajax) {
   if (loadingQueue.length === 0) {
     timeout = null;
     return;
@@ -62,7 +62,8 @@ function loadNext(ajax) {
 
   let timeoutMs = 150;
   let removeLoading = true;
-  const { url, refresh, elem, categoryId, topicId } = loadingQueue.shift();
+  const { url, refresh, elem, categoryId, topicId, onResolve } =
+    loadingQueue.shift();
 
   // Retrieve the onebox
   return ajax("/onebox", {
@@ -78,6 +79,7 @@ function loadNext(ajax) {
       (template) => {
         const node = domFromString(template)[0];
         setLocalCache(normalize(url), node);
+        onResolve?.(template);
         elem.replaceWith(node);
         applySquareGenericOnebox(node);
       },
@@ -155,3 +157,17 @@ export function load({
     timeout = timeout || discourseLater(() => loadNext(ajax), 150);
   }
 }
+
+export function addToLoadingQueue({
+  url,
+  elem = {
+    replaceWith() {},
+    classList: { remove() {}, add() {} },
+    dataset: {},
+  },
+  categoryId,
+  topicId,
+  onResolve,
+}) {
+  loadingQueue.push({ url, elem, categoryId, topicId, onResolve });
+}

app/assets/stylesheets/common/rich-editor/rich-editor.scss

@@ -17,6 +17,16 @@
     pointer-events: none;
   }
 
+  > div:first-child,
+  > details:first-child {
+    // This is hacky, but helps having the leading gapcursor at the right position
+    &.ProseMirror-gapcursor {
+      position: relative;
+      display: block;
+    }
+    margin-top: 0.5rem;
+  }
+
   h1,
   h2,
   h3,
@@ -43,7 +53,6 @@
 
   img {
     display: inline-block;
-    margin: 0 auto;
     max-width: 100%;
 
     &[data-placeholder="true"] {
@@ -119,6 +128,14 @@
     display: inline;
     padding-top: 0.2rem;
   }
+
+  .onebox-wrapper {
+    white-space: normal;
+
+    a {
+      pointer-events: all;
+    }
+  }
 }
 
 .d-editor__code-block {
@@ -199,8 +216,38 @@ li.ProseMirror-selectednode:after {
 
 /* Protect against generic img rules */
 
-img.ProseMirror-separator {
+.ProseMirror-separator {
   display: inline !important;
   border: none !important;
   margin: 0 !important;
 }
+
+/*
+ Everything below was copied from prosemirror-gapcursor/style/gapcursor.css
+ */
+
+.ProseMirror-gapcursor {
+  display: none;
+  pointer-events: none;
+  position: absolute;
+}
+
+.ProseMirror-gapcursor:after {
+  content: "";
+  display: block;
+  position: absolute;
+  top: -2px;
+  width: 20px;
+  border-top: 1px solid var(--primary);
+  animation: ProseMirror-cursor-blink 1.1s steps(2, start) infinite;
+}
+
+@keyframes ProseMirror-cursor-blink {
+  to {
+    visibility: hidden;
+  }
+}
+
+.ProseMirror-focused .ProseMirror-gapcursor {
+  display: block;
+}

plugins/discourse-details/assets/javascripts/lib/rich-editor-extension.js

@@ -1,10 +1,14 @@
 export default {
   nodeSpec: {
     details: {
+      allowGapCursor: true,
       attrs: { open: { default: true } },
       content: "summary block+",
       group: "block",
+      draggable: true,
+      selectable: true,
       defining: true,
+      isolating: true,
       parseDOM: [{ tag: "details" }],
       toDOM: (node) => ["details", { open: node.attrs.open || undefined }, 0],
     },
@@ -36,6 +40,9 @@ export default {
     },
     summary(state, node, parent) {
       state.write('[details="');
+      if (node.content.childCount === 0) {
+        state.text(" ");
+      }
       node.content.forEach(
         (child) =>
           child.text &&

plugins/footnote/assets/javascripts/lib/rich-editor-extension.js

@@ -1 +1,42 @@
-export default {};
+export default {
+  nodeSpec: {
+    footnote: {
+      attrs: { id: {} },
+      group: "group",
+      content: "group*",
+      atom: true,
+      draggable: true,
+      selectable: false,
+      parseDOM: [
+        {
+          tag: "span.footnote",
+          preserveWhitespace: "full",
+          getAttrs: (dom) => {
+            return { id: dom.getAttribute("data-id") };
+          },
+        },
+      ],
+      toDOM: (node) => {
+        return ["span", { class: "footnote", "data-id": node.attrs.id }, [0]];
+      },
+    },
+  },
+  parse: {
+    footnote_block: { ignore: true },
+    footnote: {
+      ignore: true,
+      // block: "footnote",
+      // getAttrs: (token, tokens, i) => ({ id: token.meta.id }),
+    },
+    footnote_anchor: { ignore: true, noCloseToken: true },
+    footnote_ref: {
+      node: "footnote",
+      getAttrs: (token, tokens, i) => ({ id: token.meta.id }),
+    },
+  },
+  serializeNode: {
+    footnote: (state, node) => {
+      state.write(`^[${node.attrs.id}] `);
+    },
+  },
+};

plugins/poll/assets/javascripts/lib/rich-editor-extension.js

@@ -2,19 +2,22 @@ export default {
   nodeSpec: {
     poll: {
       attrs: {
-        type: {},
+        type: { default: null },
-        results: {},
+        results: { default: null },
-        public: {},
+        public: { default: null },
         name: {},
-        chartType: {},
+        chartType: { default: null },
         close: { default: null },
         groups: { default: null },
         max: { default: null },
         min: { default: null },
       },
-      content: "poll_container poll_info",
+      content: "heading? bullet_list poll_info?",
       group: "block",
       draggable: true,
+      selectable: true,
+      isolating: true,
+      defining: true,
       parseDOM: [
         {
           tag: "div.poll",
@@ -48,17 +51,10 @@ export default {
         0,
       ],
     },
-    poll_container: {
-      content: "heading? bullet_list",
-      group: "block",
-      parseDOM: [{ tag: "div.poll-container" }],
-      toDOM: () => ["div", { class: "poll-container" }, 0],
-    },
     poll_info: {
       content: "inline*",
-      group: "block",
-      atom: true,
       selectable: false,
+      isolating: true,
       parseDOM: [{ tag: "div.poll-info" }],
       toDOM: () => ["div", { class: "poll-info", contentEditable: false }, 0],
     },
@@ -78,13 +74,16 @@ export default {
         min: token.attrGet("data-poll-min"),
       }),
     },
-    poll_container: { block: "poll_container" },
+    poll_container: { ignore: true },
     poll_title: { block: "heading" },
     poll_info: { block: "poll_info" },
     poll_info_counts: { ignore: true },
     poll_info_counts_count: { ignore: true },
     poll_info_number: { ignore: true },
-    poll_info_label: { ignore: true },
+    poll_info_label_open(state) {
+      state.addText(" ");
+    },
+    poll_info_label_close() {},
   },
   serializeNode: {
     poll(state, node) {
@@ -96,9 +95,6 @@ export default {
       state.renderContent(node);
       state.write("[/poll]\n\n");
     },
-    poll_container(state, node) {
-      state.renderContent(node);
-    },
     poll_info() {},
   },
 };

plugins/poll/assets/stylesheets/common/poll.scss

@@ -496,6 +496,7 @@ div.poll-outer {
 
 .d-editor__editable {
   .poll {
+    margin-bottom: 1rem;
     ul {
       list-style-type: none;
       padding: 0;

plugins/spoiler-alert/assets/javascripts/lib/rich-editor-extension.js

@@ -1,4 +1,4 @@
-const INLINE_NODES = ["inline_spoiler", "spoiler"];
+const SPOILER_NODES = ["inline_spoiler", "spoiler"];
 
 export default {
   nodeSpec: {
@@ -6,7 +6,6 @@ export default {
       attrs: { blurred: { default: true } },
       group: "block",
       content: "block+",
-      defining: true,
       parseDOM: [{ tag: "div.spoiled" }],
       toDOM: (node) => [
         "div",
@@ -51,8 +50,8 @@ export default {
   },
   plugins: {
     props: {
-      handleClickOn(view, pos, node, nodePos, event, direct) {
+      handleClickOn(view, pos, node, nodePos) {
-        if (INLINE_NODES.includes(node.type.name)) {
+        if (SPOILER_NODES.includes(node.type.name)) {
           view.dispatch(
             view.state.tr.setNodeMarkup(nodePos, null, {
               blurred: !node.attrs.blurred,

pnpm-lock.yaml

@@ -356,6 +356,9 @@ importers:
       prosemirror-state:
         specifier: ^1.4.3
         version: 1.4.3
+      prosemirror-transform:
+        specifier: ^1.10.2
+        version: 1.10.2
       prosemirror-view:
         specifier: ^1.34.3
         version: 1.37.1
@@ -542,7 +545,7 @@ importers:
         version: 3.0.1(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0)))
       ember-modifier:
         specifier: ^4.2.0
-        version: 4.2.0(@babel/core@7.26.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0)))
+        version: 4.2.0(@babel/core@7.26.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5))
       ember-on-resize-modifier:
         specifier: ^2.0.2
         version: 2.0.2(@babel/core@7.26.0)(@glint/template@1.5.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0)))(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0))
@@ -783,7 +786,7 @@ importers:
         version: 4.2.0
       ember-this-fallback:
         specifier: ^0.4.0
-        version: 0.4.0(patch_hash=znalyv6akdxlqfpmxunrdi3osa)(ember-cli-htmlbars@6.3.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0)))
+        version: 0.4.0(patch_hash=znalyv6akdxlqfpmxunrdi3osa)(ember-cli-htmlbars@6.3.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5))
     devDependencies:
       ember-cli:
         specifier: ~6.0.1
@@ -1126,7 +1129,7 @@ importers:
         version: 5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0))
       ember-this-fallback:
         specifier: ^0.4.0
-        version: 0.4.0(patch_hash=znalyv6akdxlqfpmxunrdi3osa)(ember-cli-htmlbars@6.3.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0)))
+        version: 0.4.0(patch_hash=znalyv6akdxlqfpmxunrdi3osa)(ember-cli-htmlbars@6.3.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5))
       handlebars:
         specifier: ^4.7.8
         version: 4.7.8
@@ -10118,7 +10121,7 @@ snapshots:
       '@glint/template': 1.5.0
     optionalDependencies:
       ember-cli-htmlbars: 6.3.0
-      ember-modifier: 4.2.0(@babel/core@7.26.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0)))
+      ember-modifier: 4.2.0(@babel/core@7.26.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5))
 
   '@glint/environment-ember-template-imports@1.5.0(@glint/environment-ember-loose@1.5.0(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(ember-cli-htmlbars@6.3.0)(ember-modifier@4.2.0(@babel/core@7.26.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5))))(@glint/template@1.5.0)':
     dependencies:
@@ -12817,7 +12820,7 @@ snapshots:
       - '@babel/core'
       - supports-color
 
-  ember-modifier@4.2.0(@babel/core@7.26.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0))):
+  ember-modifier@4.2.0(@babel/core@7.26.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)):
     dependencies:
       '@embroider/addon-shim': 1.9.0
       decorator-transforms: 2.3.0(@babel/core@7.26.0)
@@ -12834,7 +12837,7 @@ snapshots:
       ember-auto-import: 2.10.0(@glint/template@1.5.0)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0))
       ember-cli-babel: 7.26.11
       ember-cli-htmlbars: 5.7.2
-      ember-modifier: 4.2.0(@babel/core@7.26.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0)))
+      ember-modifier: 4.2.0(@babel/core@7.26.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5))
       ember-resize-observer-service: 1.1.0
     transitivePeerDependencies:
       - '@babel/core'
@@ -13012,7 +13015,7 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  ember-this-fallback@0.4.0(patch_hash=znalyv6akdxlqfpmxunrdi3osa)(ember-cli-htmlbars@6.3.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)(webpack@5.97.1(@swc/core@1.10.1)(esbuild@0.24.0))):
+  ember-this-fallback@0.4.0(patch_hash=znalyv6akdxlqfpmxunrdi3osa)(ember-cli-htmlbars@6.3.0)(ember-source@5.12.0(patch_hash=xx7mvsb7nmshqkkqhmf45r3hse)(@glimmer/component@1.1.2(@babel/core@7.26.0))(@glint/template@1.5.0)(rsvp@4.8.5)):
     dependencies:
       '@glimmer/syntax': 0.84.3
       babel-plugin-ember-template-compilation: 2.2.5