DEV: Modernise highlightjs loading (#24197)

- Remove vendored copy
- Update Rails implementation to look for language definitions in node_modules
- Use webpack-based dynamic import for hljs core
- Use browser-native dynamic import for site-specific language bundle (and fallback to webpack-based dynamic import in tests)
- Simplify markdown implementation to allow all languages into the `lang-{blah}` className
- Now that all languages are passed through, resolve aliases at runtime to avoid the need for the pre-built `highlightjs-aliases` index

lib/highlight_js.rb

@@ -1,45 +1,7 @@
 # frozen_string_literal: true
 
 module HighlightJs
-  HIGHLIGHTJS_DIR ||= "#{Rails.root}/vendor/assets/javascripts/highlightjs/"
-  BUNDLED_LANGS = %w[
-    bash
-    c
-    cpp
-    csharp
-    css
-    diff
-    go
-    graphql
-    ini
-    java
-    javascript
-    json
-    kotlin
-    less
-    lua
-    makefile
-    xml
-    markdown
-    objectivec
-    perl
-    php
-    php-template
-    plaintext
-    python
-    python-repl
-    r
-    ruby
-    rust
-    scss
-    shell
-    sql
-    swift
-    typescript
-    vbnet
-    wasm
-    yaml
-  ]
+  HIGHLIGHTJS_DIR ||= "#{Rails.root}/app/assets/javascripts/node_modules/@highlightjs/cdn-assets/"
 
   def self.languages
     langs = Dir.glob(HIGHLIGHTJS_DIR + "languages/*.js").map { |path| File.basename(path)[0..-8] }
@@ -48,22 +10,36 @@ module HighlightJs
   end
 
   def self.bundle(langs)
-    result = File.read(HIGHLIGHTJS_DIR + "highlight.min.js")
-    (langs - BUNDLED_LANGS).each do |lang|
-      begin
-        result << "\n" << File.read(HIGHLIGHTJS_DIR + "languages/#{lang}.min.js")
+    lang_js =
+      langs.filter_map do |lang|
+        File.read(HIGHLIGHTJS_DIR + "languages/#{lang}.min.js")
       rescue Errno::ENOENT
         # no file, don't care
       end
-    end
 
-    result
+    <<~JS
+      export default function registerLanguages(hljs) {
+        #{lang_js.join("\n")}
+      }
+    JS
+  end
+
+  def self.cache
+    @lang_string_cache ||= {}
   end
 
   def self.version(lang_string)
-    (@lang_string_cache ||= {})[lang_string] ||= Digest::SHA1.hexdigest(
-      bundle lang_string.split("|")
-    )
+    cache_info = cache[RailsMultisite::ConnectionManagement.current_db]
+
+    return cache_info[:digest] if cache_info&.[](:lang_string) == lang_string
+
+    cache_info = {
+      lang_string: lang_string,
+      digest: Digest::SHA1.hexdigest(bundle(lang_string.split("|"))),
+    }
+
+    cache[RailsMultisite::ConnectionManagement.current_db] = cache_info
+    cache_info[:digest]
   end
 
   def self.path

lib/tasks/javascript.rake

@@ -76,7 +76,6 @@ def dependencies
     { source: "diffhtml/dist/diffhtml.min.js", public: true },
     { source: "magnific-popup/dist/jquery.magnific-popup.min.js", public: true },
     { source: "pikaday/pikaday.js", public: true },
-    { source: "@highlightjs/cdn-assets/.", destination: "highlightjs" },
     { source: "moment/moment.js" },
     { source: "moment/locale/.", destination: "moment-locale" },
     {
@@ -317,32 +316,6 @@ task "javascript:update_constants" => :environment do
     export const replacements = #{Emoji.unicode_replacements_json};
   JS
 
-  langs = []
-  Dir
-    .glob("vendor/assets/javascripts/highlightjs/languages/*.min.js")
-    .each { |f| langs << File.basename(f, ".min.js") }
-  bundle = HighlightJs.bundle(langs)
-
-  ctx = MiniRacer::Context.new
-  hljs_aliases = ctx.eval(<<~JS)
-    #{bundle}
-
-    let aliases = {};
-    hljs.listLanguages().forEach((lang) => {
-      if (hljs.getLanguage(lang).aliases) {
-        aliases[lang] = hljs.getLanguage(lang).aliases;
-      }
-    });
-
-    aliases;
-  JS
-
-  write_template("pretty-text/addon/highlightjs-aliases.js", task_name, <<~JS)
-    export const HLJS_ALIASES = #{hljs_aliases.to_json};
-  JS
-
-  ctx.dispose
-
   write_template("pretty-text/addon/emoji/version.js", task_name, <<~JS)
     export const IMAGE_VERSION = "#{Emoji::EMOJI_VERSION}";
   JS
@@ -380,16 +353,6 @@ task "javascript:update" => "clean_up" do
       filename = f[:source].split("/").last
     end
 
-    if src.include? "highlightjs"
-      puts "Cleanup highlightjs styles and install smaller test bundle"
-      system("rm -rf node_modules/@highlightjs/cdn-assets/styles")
-
-      # We don't need every language for tests
-      langs = %w[javascript sql ruby]
-      test_bundle_dest = "vendor/assets/javascripts/highlightjs/highlight-test-bundle.min.js"
-      File.write(test_bundle_dest, HighlightJs.bundle(langs))
-    end
-
     if f[:public_root]
       dest = "#{public_root}/#{filename}"
     elsif f[:public]