discourse/lib/html_to_markdown.rb

# frozen_string_literal: true

require "securerandom"

class HtmlToMarkdown
  def initialize(html, opts = {})
    @opts = opts
    @within_html_block = false

    # we're only interested in <body>
    @doc = Nokogiri.HTML5(html).at("body")

    remove_not_allowed!(@doc)
    remove_hidden!(@doc)
    hoist_line_breaks!(@doc)
    remove_whitespaces!(@doc)
  end

  def to_markdown
    traverse(@doc).gsub(/\n{2,}/, "\n\n").strip
  end

  private

  def strip_newlines(string)
    string.gsub(/\n/, " ")&.squeeze(" ")
  end

  def remove_not_allowed!(doc)
    allowed = Set.new(@opts[:additional_allowed_tags] || [])

    HtmlToMarkdown.private_instance_methods.each do |m|
      if tag = m.to_s[/^visit_(.+)/, 1]
        allowed << tag
      end
    end

    @doc.traverse { |node| node.remove if !allowed.include?(node.name) }
  end

  def remove_hidden!(doc)
    @doc.css("[hidden]").remove
    @doc.css("img[width]").each { |n| n.remove if n["width"].to_i <= 0 }
    @doc.css("img[height]").each { |n| n.remove if n["height"].to_i <= 0 }
  end

  # When there's a <br> inside an inline element, split the inline element around the <br>
  def hoist_line_breaks!(doc)
    klass = "_" + SecureRandom.hex
    doc.css("br").each { |br| br.add_class(klass) }

    loop do
      changed = false

      doc
        .css("br.#{klass}")
        .each do |br|
          parent = br.parent

          if block?(parent)
            br.remove_class(klass)
          else
            before, after = parent.children.slice_when { |n| n == br }.to_a

            if before.size > 1
              b = doc.document.create_element(parent.name)
              before[0...-1].each { |c| b.add_child(c) }
              parent.previous = b if b.inner_html.present?
            end

            if after.present?
              a = doc.document.create_element(parent.name)
              after.each { |c| a.add_child(c) }
              parent.next = a if a.inner_html.present?
            end

            parent.replace(br)

            changed = true
          end
        end

      break if !changed
    end
  end

  # Removes most of the unnecessary white spaces for better markdown conversion
  # Loosely based on the CSS' White Space Processing Rules (https://www.w3.org/TR/css-text-3/#white-space-rules)
  def remove_whitespaces!(node)
    return true if "pre" == node.name

    node
      .children
      .chunk { |n| is_inline?(n) }
      .each do |inline, nodes|
        if inline
          collapse_spaces!(nodes) && remove_trailing_space!(nodes)
        else
          nodes.each { |n| remove_whitespaces!(n) }
        end
      end
  end

  def is_inline?(node)
    node.text? ||
      ("br" != node.name && node.description&.inline? && node.children.all? { |n| is_inline?(n) })
  end

  def collapse_spaces!(nodes, was_space = true)
    nodes.each do |node|
      if node.text?
        text = String.new

        node.text.chars.each do |c|
          if c[/[[:space:]]/]
            text << " " if !was_space
            was_space = true
          else
            text << c
            was_space = false
          end
        end

        node.content = text
      else
        node.children.each { |n| was_space = collapse_spaces!([n], was_space) }
      end
    end

    was_space
  end

  def remove_trailing_space!(nodes)
    last = nodes[-1]

    if last.text?
      last.content = last.content[0...-1] if last.content[-1] == " "
    elsif last.children.present?
      remove_trailing_space!(last.children)
    end
  end

  def traverse(node, within_html_block: false)
    within_html_block_changed = false
    if within_html_block
      within_html_block_changed = true
      @within_html_block = true
    end

    text = node.children.map { |n| visit(n) }.join
    @within_html_block = false if within_html_block_changed
    text
  end

  def visit(node)
    visitor = "visit_#{node.name}"
    send(visitor, node) if respond_to?(visitor, true)
  end

  ALLOWED_IMG_SRCS = %w[http:// https:// www.]

  def allowed_hrefs
    @allowed_hrefs ||=
      begin
        hrefs = SiteSetting.allowed_href_schemes.split("|").map { |scheme| "#{scheme}:" }.to_set
        ALLOWED_IMG_SRCS.each { |src| hrefs << src }
        hrefs << "mailto:"
        hrefs.to_a
      end
  end

  def visit_a(node)
    if node["href"].present? && node["href"].start_with?(*allowed_hrefs)
      "[#{traverse(node)}](#{node["href"]})"
    else
      traverse(node)
    end
  end

  def visit_img(node)
    return if node["src"].blank?

    node["alt"] = strip_newlines(node["alt"]) if node["alt"].present?
    node["title"] = strip_newlines(node["title"]) if node["title"].present?

    if @opts[:keep_img_tags]
      node.to_html
    elsif @opts[:keep_cid_imgs] && node["src"].start_with?("cid:")
      node.to_html
    elsif node["src"].start_with?(*ALLOWED_IMG_SRCS)
      width = node["width"].to_i
      height = node["height"].to_i
      dimensions = "|#{width}x#{height}" if width > 0 && height > 0
      "![#{node["alt"] || node["title"]}#{dimensions}](#{node["src"]})"
    end
  end

  ALLOWED = %w[kbd del ins small big sub sup dl dd dt mark]
  ALLOWED.each do |tag|
    define_method("visit_#{tag}") do |node|
      "<#{tag}>#{traverse(node, within_html_block: true)}</#{tag}>"
    end
  end

  def visit_blockquote(node)
    text = traverse(node)
    text.strip!
    text.gsub!(/\n{2,}/, "\n\n")
    text.gsub!(/^/, "> ")
    "\n\n#{text}\n\n"
  end

  BLOCKS = %w[div tr]
  BLOCKS.each do |tag|
    define_method("visit_#{tag}") do |node|
      prefix = block?(node.previous_element) ? "" : "\n"
      "#{prefix}#{traverse(node)}\n"
    end
  end

  def visit_p(node)
    "\n\n#{traverse(node)}\n\n"
  end

  TRAVERSABLES = %w[aside font span thead tbody tfoot u center]
  TRAVERSABLES.each { |tag| define_method("visit_#{tag}") { |node| traverse(node) } }

  def visit_tt(node)
    "`#{traverse(node)}`"
  end

  def visit_code(node)
    node.ancestors("pre").present? ? traverse(node) : visit_tt(node)
  end

  def visit_pre(node)
    text = traverse(node)
    fence = text["`"] ? "~~~" : "```"
    code = node.at("code")
    code_class = code ? code["class"] : ""
    lang = code_class ? code_class[/lang-(\w+)/, 1] : ""
    "\n\n#{fence}#{lang}\n#{traverse(node)}\n#{fence}\n\n"
  end

  def visit_br(node)
    "\n"
  end

  def visit_hr(node)
    "\n\n---\n\n"
  end

  def visit_abbr(node)
    title = node["title"].presence
    attributes = { title: } if title
    create_element("abbr", traverse(node, within_html_block: true), attributes).to_html
  end

  def visit_acronym(node)
    visit_abbr(node)
  end

  (1..6).each { |n| define_method("visit_h#{n}") { |node| "#{"#" * n} #{traverse(node)}" } }

  def visit_table(node)
    if (rows = extract_rows(node))
      headers = rows[0].css("td, th")
      text = "| " + headers.map { |td| traverse(td).gsub(/\n/, "<br>") }.join(" | ") + " |\n"
      text << "| " + (["-"] * headers.size).join(" | ") + " |\n"
      rows[1..-1].each do |row|
        text << "| " + row.css("td").map { |td| traverse(td).gsub(/\n/, "<br>") }.join(" | ") +
          " |\n"
      end
      "\n\n#{text}\n\n"
    else
      "<table>\n#{traverse(node, within_html_block: true)}</table>"
    end
  end

  def extract_rows(table)
    return if table.ancestors("table").present?
    return if (rows = table.css("tr")).empty?
    headers_count = rows[0].css("td, th").size
    return if rows[1..-1].any? { |row| row.css("td").size != headers_count }
    rows
  end

  def visit_tr(node)
    text = traverse(node)
    @within_html_block ? "<tr>\n#{text}</tr>\n" : text
  end

  TABLE_CELLS = %w[th td]
  TABLE_CELLS.each do |tag|
    define_method("visit_#{tag}") do |node|
      text = traverse(node)
      if @within_html_block
        element = create_element(tag, "\n\n#{text}\n\n")
        node.attribute_nodes.each do |a|
          element[a.name] = a.value if %w[rowspan colspan].include?(a.name)
        end
        "#{element.to_html}\n"
      else
        text
      end
    end
  end

  LISTS = %w[ul ol]
  LISTS.each do |tag|
    define_method("visit_#{tag}") do |node|
      prefix = block?(node.previous_element) ? "" : "\n"
      suffix = node.ancestors("ul, ol, li").size > 0 ? "" : "\n"
      "#{prefix}#{traverse(node)}#{suffix}"
    end
  end

  def visit_li(node)
    text = traverse(node)

    lists = node.ancestors("ul, ol")
    marker = "ol" == lists[0]&.name ? "1. " : "- "
    indent = (" " * marker.size) * [1, lists.size].max
    suffix = node == node.parent.elements[-1] ? "" : "\n"

    text.gsub!(/\n{2,}/, "\n\n")
    text.gsub!(/^(?!\s*$)/, indent)
    text.lstrip!

    "#{marker}#{text}#{suffix}"
  end

  EMPHASES = %w[i em]
  EMPHASES.each do |tag|
    define_method("visit_#{tag}") do |node|
      text = traverse(node)

      return "" if text.empty?
      return " " if text.blank?
      return "<#{tag}>#{text}</#{tag}>" if text["\n"] || (text["*"] && text["_"])

      prefix = text[0][" "]
      suffix = text[-1][" "] if text.size > 1
      wrap = text["*"] ? "_" : "*"

      "#{prefix}#{wrap}#{text.strip}#{wrap}#{suffix}"
    end
  end

  STRONGS = %w[b strong]
  STRONGS.each do |tag|
    define_method("visit_#{tag}") do |node|
      text = traverse(node)

      return "" if text.empty?
      return " " if text.blank?
      return "<#{tag}>#{text}</#{tag}>" if text["\n"] || (text["*"] && text["_"])

      prefix = text[0][" "]
      suffix = text[-1][" "] if text.size > 1
      wrap = text["*"] ? "__" : "**"

      "#{prefix}#{wrap}#{text.strip}#{wrap}#{suffix}"
    end
  end

  STRIKES = %w[s strike]
  STRIKES.each do |tag|
    define_method("visit_#{tag}") do |node|
      text = traverse(node)

      return "" if text.empty?
      return " " if text.blank?
      return "<#{tag}>#{text}</#{tag}>" if text["\n"] || text["~~"]

      prefix = text[0][" "]
      suffix = text[-1][" "] if text.size > 1

      "#{prefix}~~#{text.strip}~~#{suffix}"
    end
  end

  def visit_text(node)
    if @within_html_block
      node.to_html
    else
      node.text
    end
  end

  HTML5_BLOCK_ELEMENTS ||= %w[
    article
    aside
    details
    dialog
    figcaption
    figure
    footer
    header
    main
    nav
    section
  ]
  def block?(node)
    return false if !node
    node.description&.block? || HTML5_BLOCK_ELEMENTS.include?(node.name)
  end

  def fragment_document
    @fragment_document ||= Nokogiri::HTML5::DocumentFragment.parse("").document
  end

  def create_element(tag, inner_html = nil, attributes = {})
    element = fragment_document.create_element(tag, nil, attributes)
    element.inner_html = inner_html if inner_html
    element
  end
end