DEV: perform theme extra_js compilation all together

Previously, compiling theme 'extra_js' was done with a number of steps. Each theme_field would be compiled into its own value_baked column, and then the JavascriptCache content would be built by concatenating all of those compiled values.

This commit streamlines things by removing the value_baked step. The raw value of all extra_js theme_fields are passed directly to the ThemeJavascriptCompiler, and then the result is stored in the JavascriptCache.

In itself, this commit should not cause any behavior change. It is designed to open the door to more advanced compilation features which have interdependencies between different source files (e.g. template colocation, sourcemaps).

app/models/theme.rb

@@ -118,12 +118,12 @@ class Theme < ActiveRecord::Base
     all_extra_js = theme_fields
       .where(target_id: Theme.targets[:extra_js])
       .order(:name, :id)
-      .pluck(:value_baked)
+      .pluck(:name, :value)
-      .join("\n")
+      .to_h
 
     if all_extra_js.present?
       js_compiler = ThemeJavascriptCompiler.new(id, name)
-      js_compiler.append_raw_script(all_extra_js)
+      js_compiler.append_tree(all_extra_js)
       settings_hash = build_settings_hash
       js_compiler.prepend_settings(settings_hash) if settings_hash.present?
       javascript_cache || build_javascript_cache
@@ -707,14 +707,17 @@ class Theme < ActiveRecord::Base
   end
 
   def baked_js_tests_with_digest
-    content = theme_fields
+    tests_tree = theme_fields
       .where(target_id: Theme.targets[:tests_js])
       .order(name: :asc)
-      .each(&:ensure_baked!)
+      .pluck(:name, :value)
-      .map(&:value_baked)
+      .to_h
-      .join("\n")
 
-    return [nil, nil] if content.blank?
+    return [nil, nil] if tests_tree.blank?
+
+    compiler = ThemeJavascriptCompiler.new(id, name)
+    compiler.append_tree(tests_tree, for_tests: true)
+    content = compiler.content
 
     content = <<~JS + content
       (function() {

app/models/theme_field.rb

@@ -153,30 +153,6 @@ class ThemeField < ActiveRecord::Base
     [doc.to_s, errors&.join("\n")]
   end
 
-  def process_extra_js(content)
-    errors = []
-
-    js_compiler = ThemeJavascriptCompiler.new(theme_id, theme.name)
-    filename, extension = name.split(".", 2)
-    filename = "test/#{filename}" if js_tests_field?
-    begin
-      case extension
-      when "js.es6", "js"
-        js_compiler.append_module(content, filename, include_variables: true)
-      when "hbs"
-        js_compiler.append_ember_template(filename, content)
-      when "hbr", "raw.hbs"
-        js_compiler.append_raw_template(filename.sub("discourse/templates/", ""), content)
-      else
-        raise ThemeJavascriptCompiler::CompileError.new(I18n.t("themes.compile_error.unrecognized_extension", extension: extension))
-      end
-    rescue ThemeJavascriptCompiler::CompileError => ex
-      errors << ex.message
-    end
-
-    [js_compiler.content, errors&.join("\n")]
-  end
-
   def validate_svg_sprite_xml
     upload = Upload.find(self.upload_id) rescue nil
 
@@ -377,8 +353,8 @@ class ThemeField < ActiveRecord::Base
       self.compiler_version = Theme.compiler_version
       DB.after_commit { CSP::Extension.clear_theme_extensions_cache! }
     elsif extra_js_field? || js_tests_field?
-      self.value_baked, self.error = process_extra_js(self.value)
+      self.error = nil
-      self.error = nil unless self.error.present?
+      self.value_baked = "baked"
       self.compiler_version = Theme.compiler_version
     elsif basic_scss_field?
       ensure_scss_compiles!

config/locales/server.en.yml

@@ -68,8 +68,6 @@ en:
     bad_color_scheme: "Can not update theme, invalid color palette"
     other_error: "Something went wrong updating theme"
     ember_selector_error: "Sorry – using #ember or .ember-view CSS selectors is not permitted, because these names are dynamically generated at runtime and will change over time, eventually resulting in broken CSS. Try a different selector."
-    compile_error:
-      unrecognized_extension: "Unrecognized file extension: %{extension}"
     import_error:
       generic: An error occurred while importing that theme
       upload: "Error creating upload asset: %{name}. %{errors}"

lib/theme_javascript_compiler.rb

@@ -25,6 +25,38 @@ class ThemeJavascriptCompiler
     JS
   end
 
+  def append_tree(tree, for_tests: false)
+    root_name = "discourse"
+
+    # Replace legacy extensions
+    tree.transform_keys! do |filename|
+      if filename.ends_with? ".js.es6"
+        filename.sub(/\.js\.es6\z/, ".js")
+      elsif filename.ends_with? ".raw.hbs"
+        filename.sub(/\.raw\.hbs\z/, ".hbr")
+      else
+        filename
+      end
+    end
+
+    # Transpile and write to output
+    tree.each_pair do |filename, content|
+      module_name, extension = filename.split(".", 2)
+      module_name = "test/#{module_name}" if for_tests
+      if extension == "js"
+        append_module(content, module_name)
+      elsif extension == "hbs"
+        append_ember_template(module_name, content)
+      elsif extension == "hbr"
+        append_raw_template(module_name.sub("discourse/templates/", ""), content)
+      else
+        append_js_error("unknown file extension '#{extension}' (#{filename})")
+      end
+    rescue CompileError => e
+      append_js_error "#{e.message} (#{filename})"
+    end
+  end
+
   def append_ember_template(name, hbs_template)
     name = "/#{name}" if !name.start_with?("/")
     module_name = "discourse/theme-#{@theme_id}#{name}"
@@ -93,7 +125,8 @@ class ThemeJavascriptCompiler
   end
 
   def append_js_error(message)
-    @content << "console.error('Theme Transpilation Error:', #{message.inspect});"
+    message = "[THEME #{@theme_id} '#{@theme_name}'] Compile error: #{message}"
+    append_raw_script "console.error(#{message.to_json});"
   end
 
   private

spec/lib/theme_javascript_compiler_spec.rb

@@ -2,7 +2,6 @@
 
 RSpec.describe ThemeJavascriptCompiler do
   let(:compiler) { ThemeJavascriptCompiler.new(1, 'marks') }
-  let(:theme_id) { 22 }
 
   describe "#append_raw_template" do
     it 'uses the correct template paths' do
@@ -72,4 +71,20 @@ RSpec.describe ThemeJavascriptCompiler do
       end.to raise_error(ThemeJavascriptCompiler::CompileError, /Parse error on line 1/)
     end
   end
+
+  describe "#append_tree" do
+    it "can handle multiple modules" do
+      compiler.append_tree(
+        {
+          "discourse/components/mycomponent.js" => <<~JS,
+            import Component from "@glimmer/component";
+            export default class MyComponent extends Component {}
+          JS
+          "discourse/templates/components/mycomponent.hbs" => "{{my-component-template}}"
+        }
+      )
+      expect(compiler.content).to include('define("discourse/theme-1/components/mycomponent"')
+      expect(compiler.content).to include('define("discourse/theme-1/discourse/templates/components/mycomponent"')
+    end
+  end
 end

spec/models/theme_field_spec.rb

@@ -111,7 +111,7 @@ HTML
     field.ensure_baked!
     expect(field.error).not_to eq(nil)
     expect(field.value_baked).to include("<script defer=\"\" src=\"#{field.javascript_cache.url}\" data-theme-id=\"1\"></script>")
-    expect(field.javascript_cache.content).to include("Theme Transpilation Error:")
+    expect(field.javascript_cache.content).to include("[THEME 1 'Default'] Compile error")
 
     field.update!(value: '')
     field.ensure_baked!
@@ -183,15 +183,9 @@ HTML
     theme.save!
 
     js_field.reload
-    expect(js_field.value_baked).to include("if ('define' in window) {")
+    expect(js_field.value_baked).to eq("baked")
-    expect(js_field.value_baked).to include("define(\"discourse/theme-#{theme.id}/controllers/discovery\"")
+    expect(js_field.value_baked).to eq("baked")
-    expect(js_field.value_baked).to include("console.log('hello from .js.es6');")
+    expect(js_field.value_baked).to eq("baked")
-
-    expect(hbs_field.reload.value_baked).to include("define(\"discourse/theme-#{theme.id}/discourse/templates/discovery\", [\"exports\", \"@ember/template-factory\"]")
-    expect(raw_hbs_field.reload.value_baked).to include('addRawTemplate("discovery"')
-    expect(hbr_field.reload.value_baked).to include('addRawTemplate("other_discovery"')
-    expect(unknown_field.reload.value_baked).to eq("")
-    expect(unknown_field.reload.error).to eq(I18n.t("themes.compile_error.unrecognized_extension", extension: "blah"))
 
     # All together
     expect(theme.javascript_cache.content).to include("define(\"discourse/theme-#{theme.id}/discourse/templates/discovery\", [\"exports\", \"@ember/template-factory\"]")
@@ -199,6 +193,7 @@ HTML
     expect(theme.javascript_cache.content).to include("define(\"discourse/theme-#{theme.id}/controllers/discovery\"")
     expect(theme.javascript_cache.content).to include("define(\"discourse/theme-#{theme.id}/controllers/discovery-2\"")
     expect(theme.javascript_cache.content).to include("const settings =")
+    expect(theme.javascript_cache.content).to include("[THEME #{theme.id} '#{theme.name}'] Compile error: unknown file extension 'blah' (discourse/controllers/discovery.blah)")
   end
 
   def create_upload_theme_field!(name)