# frozen_string_literal: true require "securerandom" class HtmlToMarkdown def initialize(html, opts = {}) @opts = opts @within_html_block = false # we're only interested in @doc = Nokogiri.HTML5(html).at("body") remove_not_allowed!(@doc) remove_hidden!(@doc) hoist_line_breaks!(@doc) remove_whitespaces!(@doc) end def to_markdown traverse(@doc).gsub(/\n{2,}/, "\n\n").strip end private def strip_newlines(string) string.gsub(/\n/, " ")&.squeeze(" ") end def remove_not_allowed!(doc) allowed = Set.new(@opts[:additional_allowed_tags] || []) HtmlToMarkdown.private_instance_methods.each do |m| if tag = m.to_s[/^visit_(.+)/, 1] allowed << tag end end @doc.traverse { |node| node.remove if !allowed.include?(node.name) } end def remove_hidden!(doc) @doc.css("[hidden]").remove @doc.css("img[width]").each { |n| n.remove if n["width"].to_i <= 0 } @doc.css("img[height]").each { |n| n.remove if n["height"].to_i <= 0 } end # When there's a
inside an inline element, split the inline element around the
def hoist_line_breaks!(doc) klass = "_" + SecureRandom.hex doc.css("br").each { |br| br.add_class(klass) } loop do changed = false doc .css("br.#{klass}") .each do |br| parent = br.parent if block?(parent) br.remove_class(klass) else before, after = parent.children.slice_when { |n| n == br }.to_a if before.size > 1 b = doc.document.create_element(parent.name) before[0...-1].each { |c| b.add_child(c) } parent.previous = b if b.inner_html.present? end if after.present? a = doc.document.create_element(parent.name) after.each { |c| a.add_child(c) } parent.next = a if a.inner_html.present? end parent.replace(br) changed = true end end break if !changed end end # Removes most of the unnecessary white spaces for better markdown conversion # Loosely based on the CSS' White Space Processing Rules (https://www.w3.org/TR/css-text-3/#white-space-rules) def remove_whitespaces!(node) return true if "pre" == node.name node .children .chunk { |n| is_inline?(n) } .each do |inline, nodes| if inline collapse_spaces!(nodes) && remove_trailing_space!(nodes) else nodes.each { |n| remove_whitespaces!(n) } end end end def is_inline?(node) node.text? || ("br" != node.name && node.description&.inline? && node.children.all? { |n| is_inline?(n) }) end def collapse_spaces!(nodes, was_space = true) nodes.each do |node| if node.text? text = String.new node.text.chars.each do |c| if c[/[[:space:]]/] text << " " if !was_space was_space = true else text << c was_space = false end end node.content = text else node.children.each { |n| was_space = collapse_spaces!([n], was_space) } end end was_space end def remove_trailing_space!(nodes) last = nodes[-1] if last.text? last.content = last.content[0...-1] if last.content[-1] == " " elsif last.children.present? remove_trailing_space!(last.children) end end def traverse(node, within_html_block: false) within_html_block_changed = false if within_html_block within_html_block_changed = true @within_html_block = true end text = node.children.map { |n| visit(n) }.join @within_html_block = false if within_html_block_changed text end def visit(node) visitor = "visit_#{node.name}" send(visitor, node) if respond_to?(visitor, true) end ALLOWED_IMG_SRCS = %w[http:// https:// www.] def allowed_hrefs @allowed_hrefs ||= begin hrefs = SiteSetting.allowed_href_schemes.split("|").map { |scheme| "#{scheme}:" }.to_set ALLOWED_IMG_SRCS.each { |src| hrefs << src } hrefs << "mailto:" hrefs.to_a end end def visit_a(node) if node["href"].present? && node["href"].start_with?(*allowed_hrefs) "[#{traverse(node)}](#{node["href"]})" else traverse(node) end end def visit_img(node) return if node["src"].blank? node["alt"] = strip_newlines(node["alt"]) if node["alt"].present? node["title"] = strip_newlines(node["title"]) if node["title"].present? if @opts[:keep_img_tags] node.to_html elsif @opts[:keep_cid_imgs] && node["src"].start_with?("cid:") node.to_html elsif node["src"].start_with?(*ALLOWED_IMG_SRCS) width = node["width"].to_i height = node["height"].to_i dimensions = "|#{width}x#{height}" if width > 0 && height > 0 "![#{node["alt"] || node["title"]}#{dimensions}](#{node["src"]})" end end ALLOWED = %w[kbd del ins small big sub sup dl dd dt mark] ALLOWED.each do |tag| define_method("visit_#{tag}") do |node| "<#{tag}>#{traverse(node, within_html_block: true)}" end end def visit_blockquote(node) text = traverse(node) text.strip! text.gsub!(/\n{2,}/, "\n\n") text.gsub!(/^/, "> ") "\n\n#{text}\n\n" end BLOCKS = %w[div tr] BLOCKS.each do |tag| define_method("visit_#{tag}") do |node| prefix = block?(node.previous_element) ? "" : "\n" "#{prefix}#{traverse(node)}\n" end end def visit_p(node) "\n\n#{traverse(node)}\n\n" end TRAVERSABLES = %w[aside font span thead tbody tfoot u center] TRAVERSABLES.each { |tag| define_method("visit_#{tag}") { |node| traverse(node) } } def visit_tt(node) "`#{traverse(node)}`" end def visit_code(node) node.ancestors("pre").present? ? traverse(node) : visit_tt(node) end def visit_pre(node) text = traverse(node) fence = text["`"] ? "~~~" : "```" code = node.at("code") code_class = code ? code["class"] : "" lang = code_class ? code_class[/lang-(\w+)/, 1] : "" "\n\n#{fence}#{lang}\n#{traverse(node)}\n#{fence}\n\n" end def visit_br(node) "\n" end def visit_hr(node) "\n\n---\n\n" end def visit_abbr(node) title = node["title"].presence attributes = { title: } if title create_element("abbr", traverse(node, within_html_block: true), attributes).to_html end def visit_acronym(node) visit_abbr(node) end (1..6).each { |n| define_method("visit_h#{n}") { |node| "#{"#" * n} #{traverse(node)}" } } def visit_table(node) if (rows = extract_rows(node)) headers = rows[0].css("td, th") text = "| " + headers.map { |td| traverse(td).gsub(/\n/, "
") }.join(" | ") + " |\n" text << "| " + (["-"] * headers.size).join(" | ") + " |\n" rows[1..-1].each do |row| text << "| " + row.css("td").map { |td| traverse(td).gsub(/\n/, "
") }.join(" | ") + " |\n" end "\n\n#{text}\n\n" else "\n#{traverse(node, within_html_block: true)}

" end end def extract_rows(table) return if table.ancestors("table").present? return if (rows = table.css("tr")).empty? headers_count = rows[0].css("td, th").size return if rows[1..-1].any? { |row| row.css("td").size != headers_count } rows end def visit_tr(node) text = traverse(node) @within_html_block ? "\n#{text}\n" : text end TABLE_CELLS = %w[th td] TABLE_CELLS.each do |tag| define_method("visit_#{tag}") do |node| text = traverse(node) if @within_html_block element = create_element(tag, "\n\n#{text}\n\n") node.attribute_nodes.each do |a| element[a.name] = a.value if %w[rowspan colspan].include?(a.name) end "#{element.to_html}\n" else text end end end LISTS = %w[ul ol] LISTS.each do |tag| define_method("visit_#{tag}") do |node| prefix = block?(node.previous_element) ? "" : "\n" suffix = node.ancestors("ul, ol, li").size > 0 ? "" : "\n" "#{prefix}#{traverse(node)}#{suffix}" end end def visit_li(node) text = traverse(node) lists = node.ancestors("ul, ol") marker = "ol" == lists[0]&.name ? "1. " : "- " indent = (" " * marker.size) * [1, lists.size].max suffix = node == node.parent.elements[-1] ? "" : "\n" text.gsub!(/\n{2,}/, "\n\n") text.gsub!(/^(?!\s*$)/, indent) text.lstrip! "#{marker}#{text}#{suffix}" end EMPHASES = %w[i em] EMPHASES.each do |tag| define_method("visit_#{tag}") do |node| text = traverse(node) return "" if text.empty? return " " if text.blank? return "<#{tag}>#{text}" if text["\n"] || (text["*"] && text["_"]) prefix = text[0][" "] suffix = text[-1][" "] if text.size > 1 wrap = text["*"] ? "_" : "*" "#{prefix}#{wrap}#{text.strip}#{wrap}#{suffix}" end end STRONGS = %w[b strong] STRONGS.each do |tag| define_method("visit_#{tag}") do |node| text = traverse(node) return "" if text.empty? return " " if text.blank? return "<#{tag}>#{text}" if text["\n"] || (text["*"] && text["_"]) prefix = text[0][" "] suffix = text[-1][" "] if text.size > 1 wrap = text["*"] ? "__" : "**" "#{prefix}#{wrap}#{text.strip}#{wrap}#{suffix}" end end STRIKES = %w[s strike] STRIKES.each do |tag| define_method("visit_#{tag}") do |node| text = traverse(node) return "" if text.empty? return " " if text.blank? return "<#{tag}>#{text}" if text["\n"] || text["~~"] prefix = text[0][" "] suffix = text[-1][" "] if text.size > 1 "#{prefix}~~#{text.strip}~~#{suffix}" end end def visit_text(node) if @within_html_block node.to_html else node.text end end HTML5_BLOCK_ELEMENTS ||= %w[ article aside details dialog figcaption figure footer header main nav section ] def block?(node) return false if !node node.description&.block? || HTML5_BLOCK_ELEMENTS.include?(node.name) end def fragment_document @fragment_document ||= Nokogiri::HTML5::DocumentFragment.parse("").document end def create_element(tag, inner_html = nil, attributes = {}) element = fragment_document.create_element(tag, nil, attributes) element.inner_html = inner_html if inner_html element end end