DEV: Use DiscourseJsProcessor for theme template compilation (#18135)

Previously we were relying on a highly-customized version of the unmaintained Barber gem for theme template compilation. This commit switches us to use our own DiscourseJsProcessor, which makes use of more modern patterns and will be easier to maintain going forward.

In summary:
- Refactors DiscourseJsProcessor to move multiline JS heredocs into a companion `discourse-js-processor.js` file
- Use MiniRacer's `.call` method to avoid manually escaping JS strings
- Move Theme template AST transformers into DiscourseJsProcessor, and formalise interface for extending RawHandlebars AST transformations
- Update Ember template compilation to use a babel-based approach, just like Ember CLI. This gives each template its own ES6 module rather than directly assigning `Ember.TEMPLATES` values
- Improve testing of template compilation (and move some tests from `theme_javascript_compiler_spec.rb` to `discourse_js_processor_spec.rb`

app/assets/javascripts/discourse-common/addon/lib/raw-handlebars.js

@@ -73,9 +73,10 @@ if (Handlebars.Compiler) {
     RawHandlebars.JavaScriptCompiler;
   RawHandlebars.JavaScriptCompiler.prototype.namespace = "RawHandlebars";
 
-  RawHandlebars.precompile = function (value, asObject) {
+  RawHandlebars.precompile = function (value, asObject, { plugins = [] } = {}) {
     let ast = Handlebars.parse(value);
     replaceGet(ast);
+    plugins.forEach((plugin) => plugin(ast));
 
     let options = {
       knownHelpers: {
@@ -96,9 +97,10 @@ if (Handlebars.Compiler) {
     );
   };
 
-  RawHandlebars.compile = function (string) {
+  RawHandlebars.compile = function (string, { plugins = [] } = {}) {
     let ast = Handlebars.parse(string);
     replaceGet(ast);
+    plugins.forEach((plugin) => plugin(ast));
 
     // this forces us to rewrite helpers
     let options = { data: true, stringParams: true };

app/assets/javascripts/discourse-js-processor.js

@@ -0,0 +1,110 @@
+/* global Babel:true */
+
+// This is executed in mini_racer to provide the JS logic for lib/discourse_js_processor.rb
+
+const makeEmberTemplateCompilerPlugin =
+  require("babel-plugin-ember-template-compilation").default;
+const precompile = require("ember-template-compiler").precompile;
+const Handlebars = require("handlebars").default;
+
+function manipulateAstNodeForTheme(node, themeId) {
+  // Magically add theme id as the first param for each of these helpers)
+  if (
+    node.path.parts &&
+    ["theme-i18n", "theme-prefix", "theme-setting"].includes(node.path.parts[0])
+  ) {
+    if (node.params.length === 1) {
+      node.params.unshift({
+        type: "NumberLiteral",
+        value: themeId,
+        original: themeId,
+        loc: { start: {}, end: {} },
+      });
+    }
+  }
+}
+
+function buildEmberTemplateManipulatorPlugin(themeId) {
+  return function () {
+    return {
+      name: "theme-template-manipulator",
+      visitor: {
+        SubExpression: (node) => manipulateAstNodeForTheme(node, themeId),
+        MustacheStatement: (node) => manipulateAstNodeForTheme(node, themeId),
+      },
+    };
+  };
+}
+
+function buildTemplateCompilerBabelPlugins({ themeId }) {
+  let compileFunction = precompile;
+  if (themeId) {
+    compileFunction = (src, opts) => {
+      return precompile(src, {
+        ...opts,
+        plugins: {
+          ast: [buildEmberTemplateManipulatorPlugin(themeId)],
+        },
+      });
+    };
+  }
+
+  return [
+    require("widget-hbs-compiler").WidgetHbsCompiler,
+    [
+      makeEmberTemplateCompilerPlugin(() => compileFunction),
+      { enableLegacyModules: ["ember-cli-htmlbars"] },
+    ],
+  ];
+}
+
+function buildThemeRawHbsTemplateManipulatorPlugin(themeId) {
+  return function (ast) {
+    ["SubExpression", "MustacheStatement"].forEach((pass) => {
+      let visitor = new Handlebars.Visitor();
+      visitor.mutating = true;
+      visitor[pass] = (node) => manipulateAstNodeForTheme(node, themeId);
+      visitor.accept(ast);
+    });
+  };
+}
+
+exports.compileRawTemplate = function (source, themeId) {
+  try {
+    const RawHandlebars = require("raw-handlebars").default;
+    const plugins = [];
+    if (themeId) {
+      plugins.push(buildThemeRawHbsTemplateManipulatorPlugin(themeId));
+    }
+    return RawHandlebars.precompile(source, false, { plugins }).toString();
+  } catch (error) {
+    // Workaround for https://github.com/rubyjs/mini_racer/issues/262
+    error.message = JSON.stringify(error.message);
+    throw error;
+  }
+};
+
+exports.transpile = function (
+  source,
+  { moduleId, filename, skipModule, themeId, commonPlugins } = {}
+) {
+  const plugins = [];
+  plugins.push(...buildTemplateCompilerBabelPlugins({ themeId }));
+  if (moduleId && !skipModule) {
+    plugins.push(["transform-modules-amd", { noInterop: true }]);
+  }
+  plugins.push(...commonPlugins);
+
+  try {
+    return Babel.transform(source, {
+      moduleId,
+      filename,
+      ast: false,
+      plugins,
+    }).code;
+  } catch (error) {
+    // Workaround for https://github.com/rubyjs/mini_racer/issues/262
+    error.message = JSON.stringify(error.message);
+    throw error;
+  }
+};

app/assets/javascripts/discourse/public/assets/scripts/discourse-boot.js

@@ -10,10 +10,12 @@
   const discoursePrefixLength = discoursePrefix.length;
 
   const pluginRegex = /^discourse\/plugins\/([^\/]+)\//;
+  const themeRegex = /^discourse\/theme-([^\/]+)\//;
 
   Object.keys(requirejs.entries).forEach(function (key) {
     let templateKey;
     let pluginName;
+    let themeId;
     if (key.startsWith(discoursePrefix)) {
       templateKey = key.slice(discoursePrefixLength);
     } else if (key.startsWith(adminPrefix) || key.startsWith(wizardPrefix)) {
@@ -28,6 +30,16 @@
       templateKey = key.slice(`discourse/plugins/${pluginName}/`.length);
       templateKey = templateKey.replace("discourse/templates/", "");
       templateKey = `javascripts/${templateKey}`;
+    } else if (
+      (themeId = key.match(themeRegex)?.[1]) &&
+      key.includes("/templates/")
+    ) {
+      // And likewise for themes - this mimics the old logic
+      templateKey = key.slice(`discourse/theme-${themeId}/`.length);
+      templateKey = templateKey.replace("discourse/templates/", "");
+      if (!templateKey.startsWith("javascripts/")) {
+        templateKey = `javascripts/${templateKey}`;
+      }
     }
 
     if (templateKey) {

app/models/theme.rb

@@ -6,7 +6,7 @@ require 'json_schemer'
 class Theme < ActiveRecord::Base
   include GlobalPath
 
-  BASE_COMPILER_VERSION = 60
+  BASE_COMPILER_VERSION = 61
 
   attr_accessor :child_components
 

app/models/theme_field.rb

@@ -164,7 +164,7 @@ class ThemeField < ActiveRecord::Base
       when "js.es6", "js"
         js_compiler.append_module(content, filename, include_variables: true)
       when "hbs"
-        js_compiler.append_ember_template(filename.sub("discourse/templates/", ""), content)
+        js_compiler.append_ember_template(filename, content)
       when "hbr", "raw.hbs"
         js_compiler.append_raw_template(filename.sub("discourse/templates/", ""), content)
       else

lib/discourse_js_processor.rb

@@ -3,6 +3,7 @@ require 'execjs'
 require 'mini_racer'
 
 class DiscourseJsProcessor
+  class TranspileError < StandardError; end
 
   DISCOURSE_COMMON_BABEL_PLUGINS = [
     'proposal-optional-chaining',
@@ -55,9 +56,9 @@ class DiscourseJsProcessor
     { data: data }
   end
 
-  def self.transpile(data, root_path, logical_path)
+  def self.transpile(data, root_path, logical_path, theme_id: nil)
     transpiler = Transpiler.new(skip_module: skip_module?(data))
-    transpiler.perform(data, root_path, logical_path)
+    transpiler.perform(data, root_path, logical_path, theme_id: theme_id)
   end
 
   def self.should_transpile?(filename)
@@ -114,7 +115,7 @@ class DiscourseJsProcessor
       contents = File.read("#{Rails.root}/app/assets/javascripts/#{path}")
       if wrap_in_module
         contents = <<~JS
-          define(#{wrap_in_module.to_json}, ["exports", "require"], function(exports, require){
+          define(#{wrap_in_module.to_json}, ["exports", "require", "module"], function(exports, require, module){
             #{contents}
           });
         JS
@@ -138,7 +139,6 @@ class DiscourseJsProcessor
           warn: function(...args){ rails.logger.warn(console.prefix + args.join(" ")); },
           error: function(...args){ rails.logger.error(console.prefix + args.join(" ")); }
         };
-        const DISCOURSE_COMMON_BABEL_PLUGINS = #{DISCOURSE_COMMON_BABEL_PLUGINS.to_json};
       JS
 
       # define/require support
@@ -146,6 +146,11 @@ class DiscourseJsProcessor
 
       # Babel
       load_file_in_context(ctx, "node_modules/@babel/standalone/babel.js")
+      ctx.eval <<~JS
+        globalThis.rawBabelTransform = function(){
+          return Babel.transform(...arguments).code;
+        }
+      JS
 
       # Template Compiler
       load_file_in_context(ctx, "node_modules/ember-source/dist/ember-template-compiler.js")
@@ -160,28 +165,37 @@ class DiscourseJsProcessor
           #{widget_hbs_compiler_source}
         });
       JS
-      widget_hbs_compiler_transpiled = ctx.eval <<~JS
+      widget_hbs_compiler_transpiled = ctx.call("rawBabelTransform", widget_hbs_compiler_source, {
-        Babel.transform(
-          #{widget_hbs_compiler_source.to_json},
-          {
         ast: false,
         moduleId: 'widget-hbs-compiler',
-            plugins: [
+        plugins: DISCOURSE_COMMON_BABEL_PLUGINS
-              ...DISCOURSE_COMMON_BABEL_PLUGINS
+      })
-            ]
-          }
-        ).code
-      JS
       ctx.eval(widget_hbs_compiler_transpiled, filename: "widget-hbs-compiler.js")
 
-      # Prepare template compiler plugins
+      # Raw HBS compiler
-      ctx.eval <<~JS
+      load_file_in_context(ctx, "node_modules/handlebars/dist/handlebars.js", wrap_in_module: "handlebars")
-        const makeEmberTemplateCompilerPlugin = require("babel-plugin-ember-template-compilation").default;
+
-        const precompile = require("ember-template-compiler").precompile;
+      raw_hbs_transpiled = ctx.call(
-        const DISCOURSE_TEMPLATE_COMPILER_PLUGINS = [
+        "rawBabelTransform",
-          require("widget-hbs-compiler").WidgetHbsCompiler,
+        File.read("#{Rails.root}/app/assets/javascripts/discourse-common/addon/lib/raw-handlebars.js"),
-          [makeEmberTemplateCompilerPlugin(() => precompile), { enableLegacyModules: ["ember-cli-htmlbars"] }],
+        {
+          ast: false,
+          moduleId: "raw-handlebars",
+          plugins: [
+            ['transform-modules-amd', { noInterop: true }],
+            *DISCOURSE_COMMON_BABEL_PLUGINS
           ]
+        }
+      )
+      ctx.eval(raw_hbs_transpiled, filename: "raw-handlebars.js")
+
+      # Theme template AST transformation plugins
+      load_file_in_context(ctx, "discourse-js-processor.js", wrap_in_module: "discourse-js-processor")
+
+      # Make interfaces available via `v8.call`
+      ctx.eval <<~JS
+        globalThis.compileRawTemplate = require('discourse-js-processor').compileRawTemplate;
+        globalThis.transpile = require('discourse-js-processor').transpile;
       JS
 
       ctx
@@ -204,59 +218,41 @@ class DiscourseJsProcessor
       @ctx
     end
 
+    def self.v8_call(*args, **kwargs)
+      mutex.synchronize do
+        v8.call(*args, **kwargs)
+      end
+    rescue MiniRacer::RuntimeError => e
+      message = e.message
+      begin
+        # Workaround for https://github.com/rubyjs/mini_racer/issues/262
+        possible_encoded_message = message.delete_prefix("Error: ")
+        decoded = JSON.parse("{\"value\": #{possible_encoded_message}}")["value"]
+        message = "Error: #{decoded}"
+      rescue JSON::ParserError
+        message = e.message
+      end
+      transpile_error = TranspileError.new(message)
+      transpile_error.set_backtrace(e.backtrace)
+      raise transpile_error
+    end
+
     def initialize(skip_module: false)
       @skip_module = skip_module
     end
 
-    def perform(source, root_path = nil, logical_path = nil)
+    def perform(source, root_path = nil, logical_path = nil, theme_id: nil)
-      klass = self.class
+      self.class.v8_call(
-      klass.mutex.synchronize do
+        "transpile",
-        klass.v8.eval("console.prefix = 'BABEL: babel-eval: ';")
-        transpiled = babel_source(
         source,
-          module_name: module_name(root_path, logical_path),
+        {
-          filename: logical_path
+          skip_module: @skip_module,
+          moduleId: module_name(root_path, logical_path),
+          filename: logical_path || 'unknown',
+          themeId: theme_id,
+          commonPlugins: DISCOURSE_COMMON_BABEL_PLUGINS
+        }
       )
-        @output = klass.v8.eval(transpiled)
-      end
-    end
-
-    def babel_source(source, opts = nil)
-      opts ||= {}
-
-      js_source = ::JSON.generate(source, quirks_mode: true)
-
-      if opts[:module_name] && !@skip_module
-        filename = opts[:filename] || 'unknown'
-        <<~JS
-          Babel.transform(
-            #{js_source},
-            {
-              moduleId: '#{opts[:module_name]}',
-              filename: '#{filename}',
-              ast: false,
-              plugins: [
-                ...DISCOURSE_TEMPLATE_COMPILER_PLUGINS,
-                ['transform-modules-amd', {noInterop: true}],
-                ...DISCOURSE_COMMON_BABEL_PLUGINS
-              ]
-            }
-          ).code
-        JS
-      else
-        <<~JS
-          Babel.transform(
-            #{js_source},
-            {
-              ast: false,
-              plugins: [
-                ...DISCOURSE_TEMPLATE_COMPILER_PLUGINS,
-                ...DISCOURSE_COMMON_BABEL_PLUGINS
-              ]
-            }
-          ).code
-        JS
-      end
     end
 
     def module_name(root_path, logical_path)
@@ -275,5 +271,9 @@ class DiscourseJsProcessor
       path || logical_path&.gsub('app/', '')&.gsub('addon/', '')&.gsub('admin/addon', 'admin')
     end
 
+    def compile_raw_template(source, theme_id: nil)
+      self.class.v8_call("compileRawTemplate", source, theme_id)
+    end
+
   end
 end

lib/theme_javascript_compiler.rb

@@ -2,75 +2,7 @@
 
 class ThemeJavascriptCompiler
 
-  module PrecompilerExtension
+  COLOCATED_CONNECTOR_REGEX = /\A(?<prefix>.*)\/connectors\/(?<outlet>[^\/]+)\/(?<name>[^\/\.]+)\z/
-    def initialize(theme_id)
-      super()
-      @theme_id = theme_id
-    end
-
-    def discourse_node_manipulator
-      <<~JS
-      function manipulateNode(node) {
-        // Magically add theme id as the first param for each of these helpers)
-        if (node.path.parts && ["theme-i18n", "theme-prefix", "theme-setting"].includes(node.path.parts[0])) {
-          if(node.params.length === 1){
-            node.params.unshift({
-              type: "NumberLiteral",
-              value: #{@theme_id},
-              original: #{@theme_id},
-              loc: { start: {}, end: {} }
-            })
-          }
-        }
-      }
-      JS
-    end
-
-    def source
-      [super, discourse_node_manipulator, discourse_extension].join("\n")
-    end
-  end
-
-  class RawTemplatePrecompiler < Barber::Precompiler
-    include PrecompilerExtension
-
-    def discourse_extension
-      <<~JS
-        let _superCompile = Handlebars.Compiler.prototype.compile;
-        Handlebars.Compiler.prototype.compile = function(program, options) {
-          [
-            "SubExpression",
-            "MustacheStatement"
-          ].forEach((pass) => {
-            let visitor = new Handlebars.Visitor();
-            visitor.mutating = true;
-            visitor[pass] = manipulateNode;
-            visitor.accept(program);
-          })
-
-          return _superCompile.apply(this, arguments);
-        };
-      JS
-    end
-  end
-
-  class EmberTemplatePrecompiler < Barber::Ember::Precompiler
-    include PrecompilerExtension
-
-    def discourse_extension
-      <<~JS
-        module.exports.registerPlugin('ast', function() {
-          return {
-            name: 'theme-template-manipulator',
-            visitor: {
-              SubExpression: manipulateNode,
-              MustacheStatement: manipulateNode
-            }
-          }
-        });
-      JS
-    end
-  end
 
   class CompileError < StandardError
   end
@@ -93,25 +25,30 @@ class ThemeJavascriptCompiler
     JS
   end
 
-  # TODO Error handling for handlebars templates
   def append_ember_template(name, hbs_template)
-    if !name.start_with?("javascripts/")
+    name = "/#{name}" if !name.start_with?("/")
-      prefix = "javascripts"
+    module_name = "discourse/theme-#{@theme_id}#{name}"
-      prefix += "/" if !name.start_with?("/")
+
-      name = prefix + name
+    # Some themes are colocating connector JS under `/connectors`. Move template to /templates to avoid module name clash
+    if (match = COLOCATED_CONNECTOR_REGEX.match(module_name)) && !match[:prefix].end_with?("/templates")
+      module_name = "#{match[:prefix]}/templates/connectors/#{match[:outlet]}/#{match[:name]}"
     end
-    name = name.inspect
+
-    compiled = EmberTemplatePrecompiler.new(@theme_id).compile(hbs_template)
+    # Mimics the ember-cli implementation
-    # the `'Ember' in window` check is needed for no_ember pages
+    # https://github.com/ember-cli/ember-cli-htmlbars/blob/d5aa14b3/lib/template-compiler-plugin.js#L18-L26
-    content << <<~JS
+    script = <<~JS
-      (function() {
+      import { hbs } from 'ember-cli-htmlbars';
-        if ('Ember' in window) {
+      export default hbs(#{hbs_template.to_json}, { moduleName: #{module_name.to_json} });
-          Ember.TEMPLATES[#{name}] = Ember.HTMLBars.template(#{compiled});
-        }
-      })();
     JS
-  rescue Barber::PrecompilerError => e
+
-    raise CompileError.new e.instance_variable_get(:@error) # e.message contains the entire template, which could be very long
+    template_module = DiscourseJsProcessor.transpile(script, "", module_name, theme_id: @theme_id)
+    content << <<~JS
+      if ('define' in window) {
+      #{template_module}
+      }
+    JS
+  rescue MiniRacer::RuntimeError, DiscourseJsProcessor::TranspileError => ex
+    raise CompileError.new ex.message
   end
 
   def raw_template_name(name)
@@ -120,7 +57,7 @@ class ThemeJavascriptCompiler
   end
 
   def append_raw_template(name, hbs_template)
-    compiled = RawTemplatePrecompiler.new(@theme_id).compile(hbs_template)
+    compiled = DiscourseJsProcessor::Transpiler.new.compile_raw_template(hbs_template, theme_id: @theme_id)
     @content << <<~JS
       (function() {
         const addRawTemplate = requirejs('discourse-common/lib/raw-templates').addRawTemplate;
@@ -128,8 +65,8 @@ class ThemeJavascriptCompiler
         addRawTemplate(#{raw_template_name(name)}, template);
       })();
     JS
-  rescue Barber::PrecompilerError => e
+  rescue MiniRacer::RuntimeError, DiscourseJsProcessor::TranspileError => ex
-    raise CompileError.new e.instance_variable_get(:@error) # e.message contains the entire template, which could be very long
+    raise CompileError.new ex.message
   end
 
   def append_raw_script(script)
@@ -138,6 +75,12 @@ class ThemeJavascriptCompiler
 
   def append_module(script, name, include_variables: true)
     name = "discourse/theme-#{@theme_id}/#{name.gsub(/^discourse\//, '')}"
+
+    # Some themes are colocating connector JS under `/templates/connectors`. Move out of templates to avoid module name clash
+    if (match = COLOCATED_CONNECTOR_REGEX.match(name)) && match[:prefix].end_with?("/templates")
+      name = "#{match[:prefix].delete_suffix("/templates")}/connectors/#{match[:outlet]}/#{match[:name]}"
+    end
+
     script = "#{theme_settings}#{script}" if include_variables
     transpiler = DiscourseJsProcessor::Transpiler.new
     @content << <<~JS
@@ -145,7 +88,7 @@ class ThemeJavascriptCompiler
       #{transpiler.perform(script, "", name).strip}
       }
     JS
-  rescue MiniRacer::RuntimeError => ex
+  rescue MiniRacer::RuntimeError, DiscourseJsProcessor::TranspileError => ex
     raise CompileError.new ex.message
   end
 

spec/lib/discourse_js_processor_spec.rb

@@ -78,4 +78,111 @@ RSpec.describe DiscourseJsProcessor do
       });
     JS
   end
+
+  describe "Raw template theme transformations" do
+    # For the raw templates, we can easily render them serverside, so let's do that
+
+    let(:compiler) { DiscourseJsProcessor::Transpiler.new }
+    let(:theme_id) { 22 }
+
+    let(:helpers) {
+      <<~JS
+      Handlebars.registerHelper('theme-prefix', function(themeId, string) {
+        return `theme_translations.${themeId}.${string}`
+      })
+      Handlebars.registerHelper('theme-i18n', function(themeId, string) {
+        return `translated(theme_translations.${themeId}.${string})`
+      })
+      Handlebars.registerHelper('theme-setting', function(themeId, string) {
+        return `setting(${themeId}:${string})`
+      })
+      Handlebars.registerHelper('dummy-helper', function(string) {
+        return `dummy(${string})`
+      })
+      JS
+    }
+
+    let(:mini_racer) {
+      ctx = MiniRacer::Context.new
+      ctx.eval(File.open("#{Rails.root}/app/assets/javascripts/node_modules/handlebars/dist/handlebars.js").read)
+      ctx.eval(helpers)
+      ctx
+    }
+
+    def render(template)
+      compiled = compiler.compile_raw_template(template, theme_id: theme_id)
+      mini_racer.eval "Handlebars.template(#{compiled.squish})({})"
+    end
+
+    it 'adds the theme id to the helpers' do
+      # Works normally
+      expect(render("{{theme-prefix 'translation_key'}}")).
+        to eq('theme_translations.22.translation_key')
+      expect(render("{{theme-i18n 'translation_key'}}")).
+        to eq('translated(theme_translations.22.translation_key)')
+      expect(render("{{theme-setting 'setting_key'}}")).
+        to eq('setting(22:setting_key)')
+
+      # Works when used inside other statements
+      expect(render("{{dummy-helper (theme-prefix 'translation_key')}}")).
+        to eq('dummy(theme_translations.22.translation_key)')
+    end
+
+    it "doesn't duplicate number parameter inside {{each}}" do
+      expect(compiler.compile_raw_template("{{#each item as |test test2|}}{{theme-setting 'setting_key'}}{{/each}}", theme_id: theme_id)).
+        to include('{"name":"theme-setting","hash":{},"hashTypes":{},"hashContexts":{},"types":["NumberLiteral","StringLiteral"]')
+      # Fail would be if theme-setting is defined with types:["NumberLiteral","NumberLiteral","StringLiteral"]
+    end
+  end
+
+  describe "Ember template transformations" do
+    # For the Ember (Glimmer) templates, serverside rendering is not trivial,
+    # so we compile the expected result with the standard compiler and compare to the theme compiler
+    let(:theme_id) { 22 }
+
+    def theme_compile(template)
+      script = <<~JS
+        import { hbs } from 'ember-cli-htmlbars';
+        export default hbs(#{template.to_json});
+      JS
+      result = DiscourseJsProcessor.transpile(script, "", "theme/blah", theme_id: theme_id)
+      result.gsub(/\/\*(.*)\*\//m, "/* (js comment stripped) */")
+    end
+
+    def standard_compile(template)
+      script = <<~JS
+        import { hbs } from 'ember-cli-htmlbars';
+        export default hbs(#{template.to_json});
+      JS
+      result = DiscourseJsProcessor.transpile(script, "", "theme/blah")
+      result.gsub(/\/\*(.*)\*\//m, "/* (js comment stripped) */")
+    end
+
+    it 'adds the theme id to the helpers' do
+      expect(
+        theme_compile "{{theme-prefix 'translation_key'}}"
+      ).to eq(
+        standard_compile "{{theme-prefix #{theme_id} 'translation_key'}}"
+      )
+
+      expect(
+        theme_compile "{{theme-i18n 'translation_key'}}"
+      ).to eq(
+        standard_compile "{{theme-i18n #{theme_id} 'translation_key'}}"
+      )
+
+      expect(
+        theme_compile "{{theme-setting 'setting_key'}}"
+      ).to eq(
+        standard_compile "{{theme-setting #{theme_id} 'setting_key'}}"
+      )
+
+      # Works when used inside other statements
+      expect(
+        theme_compile "{{dummy-helper (theme-prefix 'translation_key')}}"
+      ).to eq(
+        standard_compile "{{dummy-helper (theme-prefix #{theme_id} 'translation_key')}}"
+      )
+    end
+  end
 end

spec/lib/theme_javascript_compiler_spec.rb

@@ -1,112 +1,10 @@
 # frozen_string_literal: true
 
 RSpec.describe ThemeJavascriptCompiler do
-
+  let(:compiler) { ThemeJavascriptCompiler.new(1, 'marks') }
   let(:theme_id) { 22 }
 
-  describe ThemeJavascriptCompiler::RawTemplatePrecompiler do
-    # For the raw templates, we can easily render them serverside, so let's do that
-
-    let(:compiler) { described_class.new(theme_id) }
-
-    let(:helpers) {
-      <<~JS
-      Handlebars.registerHelper('theme-prefix', function(themeId, string) {
-        return `theme_translations.${themeId}.${string}`
-      })
-      Handlebars.registerHelper('theme-i18n', function(themeId, string) {
-        return `translated(theme_translations.${themeId}.${string})`
-      })
-      Handlebars.registerHelper('theme-setting', function(themeId, string) {
-        return `setting(${themeId}:${string})`
-      })
-      Handlebars.registerHelper('dummy-helper', function(string) {
-        return `dummy(${string})`
-      })
-      JS
-    }
-
-    let(:mini_racer) {
-      ctx = MiniRacer::Context.new
-      ctx.eval(File.open("#{Rails.root}/app/assets/javascripts/node_modules/handlebars/dist/handlebars.js").read)
-      ctx.eval(helpers)
-      ctx
-    }
-
-    def render(template)
-      compiled = compiler.compile(template)
-      mini_racer.eval "Handlebars.template(#{compiled.squish})({})"
-    end
-
-    it 'adds the theme id to the helpers' do
-      # Works normally
-      expect(render("{{theme-prefix 'translation_key'}}")).
-        to eq('theme_translations.22.translation_key')
-      expect(render("{{theme-i18n 'translation_key'}}")).
-        to eq('translated(theme_translations.22.translation_key)')
-      expect(render("{{theme-setting 'setting_key'}}")).
-        to eq('setting(22:setting_key)')
-
-      # Works when used inside other statements
-      expect(render("{{dummy-helper (theme-prefix 'translation_key')}}")).
-        to eq('dummy(theme_translations.22.translation_key)')
-    end
-
-    it "doesn't duplicate number parameter inside {{each}}" do
-      expect(compiler.compile("{{#each item as |test test2|}}{{theme-setting 'setting_key'}}{{/each}}")).
-        to include('{"name":"theme-setting","hash":{},"hashTypes":{},"hashContexts":{},"types":["NumberLiteral","StringLiteral"]')
-      # Fail would be if theme-setting is defined with types:["NumberLiteral","NumberLiteral","StringLiteral"]
-    end
-  end
-
-  describe ThemeJavascriptCompiler::EmberTemplatePrecompiler do
-    # For the Ember (Glimmer) templates, serverside rendering is not trivial,
-    # so we compile the expected result with the standard compiler and compare to the theme compiler
-    let(:standard_compiler) { Barber::Ember::Precompiler.new }
-    let(:theme_compiler) { described_class.new(theme_id) }
-
-    def theme_compile(template)
-      compiled = theme_compiler.compile(template)
-      data = JSON.parse(compiled)
-      JSON.parse(data["block"])
-    end
-
-    def standard_compile(template)
-      compiled = standard_compiler.compile(template)
-      data = JSON.parse(compiled)
-      JSON.parse(data["block"])
-    end
-
-    it 'adds the theme id to the helpers' do
-      expect(
-        theme_compile "{{theme-prefix 'translation_key'}}"
-      ).to eq(
-        standard_compile "{{theme-prefix #{theme_id} 'translation_key'}}"
-      )
-
-      expect(
-        theme_compile "{{theme-i18n 'translation_key'}}"
-      ).to eq(
-        standard_compile "{{theme-i18n #{theme_id} 'translation_key'}}"
-      )
-
-      expect(
-        theme_compile "{{theme-setting 'setting_key'}}"
-      ).to eq(
-        standard_compile "{{theme-setting #{theme_id} 'setting_key'}}"
-      )
-
-      # # Works when used inside other statements
-      expect(
-        theme_compile "{{dummy-helper (theme-prefix 'translation_key')}}"
-      ).to eq(
-        standard_compile "{{dummy-helper (theme-prefix #{theme_id} 'translation_key')}}"
-      )
-    end
-  end
-
   describe "#append_raw_template" do
-    let(:compiler) { ThemeJavascriptCompiler.new(1, 'marks') }
     it 'uses the correct template paths' do
       template = "<h1>hello</h1>"
       name = "/path/to/templates1"
@@ -124,16 +22,54 @@ RSpec.describe ThemeJavascriptCompiler do
   end
 
   describe "#append_ember_template" do
-    let(:compiler) { ThemeJavascriptCompiler.new(1, 'marks') }
+    it 'maintains module names so that discourse-boot.js can correct them' do
-    it 'prepends `javascripts/` to template name if it is not prepended' do
       compiler.append_ember_template("/connectors/blah-1", "{{var}}")
-      expect(compiler.content.to_s).to include('Ember.TEMPLATES["javascripts/connectors/blah-1"]')
+      expect(compiler.content.to_s).to include("define(\"discourse/theme-1/connectors/blah-1\", [\"exports\", \"@ember/template-factory\"]")
 
       compiler.append_ember_template("connectors/blah-2", "{{var}}")
-      expect(compiler.content.to_s).to include('Ember.TEMPLATES["javascripts/connectors/blah-2"]')
+      expect(compiler.content.to_s).to include("define(\"discourse/theme-1/connectors/blah-2\", [\"exports\", \"@ember/template-factory\"]")
 
       compiler.append_ember_template("javascripts/connectors/blah-3", "{{var}}")
-      expect(compiler.content.to_s).to include('Ember.TEMPLATES["javascripts/connectors/blah-3"]')
+      expect(compiler.content.to_s).to include("define(\"discourse/theme-1/javascripts/connectors/blah-3\", [\"exports\", \"@ember/template-factory\"]")
+    end
+  end
+
+  describe "connector module name handling" do
+    it 'separates colocated connectors to avoid module name clash' do
+      # Colocated under `/connectors`
+      compiler = ThemeJavascriptCompiler.new(1, 'marks')
+      compiler.append_ember_template("connectors/outlet/blah-1", "{{var}}")
+      compiler.append_module("console.log('test')", "connectors/outlet/blah-1")
+      expect(compiler.content.to_s).to include("discourse/theme-1/connectors/outlet/blah-1")
+      expect(compiler.content.to_s).to include("discourse/theme-1/templates/connectors/outlet/blah-1")
+
+      # Colocated under `/templates/connectors`
+      compiler = ThemeJavascriptCompiler.new(1, 'marks')
+      compiler.append_ember_template("templates/connectors/outlet/blah-1", "{{var}}")
+      compiler.append_module("console.log('test')", "templates/connectors/outlet/blah-1")
+      expect(compiler.content.to_s).to include("discourse/theme-1/connectors/outlet/blah-1")
+      expect(compiler.content.to_s).to include("discourse/theme-1/templates/connectors/outlet/blah-1")
+
+      # Not colocated
+      compiler = ThemeJavascriptCompiler.new(1, 'marks')
+      compiler.append_ember_template("templates/connectors/outlet/blah-1", "{{var}}")
+      compiler.append_module("console.log('test')", "connectors/outlet/blah-1")
+      expect(compiler.content.to_s).to include("discourse/theme-1/connectors/outlet/blah-1")
+      expect(compiler.content.to_s).to include("discourse/theme-1/templates/connectors/outlet/blah-1")
+    end
+  end
+
+  describe "error handling" do
+    it "handles syntax errors in raw templates" do
+      expect do
+        compiler.append_raw_template("sometemplate.hbr", "{{invalidtemplate")
+      end.to raise_error(ThemeJavascriptCompiler::CompileError, /Parse error on line 1/)
+    end
+
+    it "handles syntax errors in ember templates" do
+      expect do
+        compiler.append_ember_template("sometemplate", "{{invalidtemplate")
+      end.to raise_error(ThemeJavascriptCompiler::CompileError, /Parse error on line 1/)
     end
   end
 end

spec/models/theme_field_spec.rb

@@ -186,14 +186,14 @@ HTML
     expect(js_field.value_baked).to include("define(\"discourse/theme-#{theme.id}/controllers/discovery\"")
     expect(js_field.value_baked).to include("console.log('hello from .js.es6');")
 
-    expect(hbs_field.reload.value_baked).to include('Ember.TEMPLATES["javascripts/discovery"]')
+    expect(hbs_field.reload.value_baked).to include("define(\"discourse/theme-#{theme.id}/discourse/templates/discovery\", [\"exports\", \"@ember/template-factory\"]")
     expect(raw_hbs_field.reload.value_baked).to include('addRawTemplate("discovery"')
     expect(hbr_field.reload.value_baked).to include('addRawTemplate("other_discovery"')
     expect(unknown_field.reload.value_baked).to eq("")
     expect(unknown_field.reload.error).to eq(I18n.t("themes.compile_error.unrecognized_extension", extension: "blah"))
 
     # All together
-    expect(theme.javascript_cache.content).to include('Ember.TEMPLATES["javascripts/discovery"]')
+    expect(theme.javascript_cache.content).to include("define(\"discourse/theme-#{theme.id}/discourse/templates/discovery\", [\"exports\", \"@ember/template-factory\"]")
     expect(theme.javascript_cache.content).to include('addRawTemplate("discovery"')
     expect(theme.javascript_cache.content).to include("define(\"discourse/theme-#{theme.id}/controllers/discovery\"")
     expect(theme.javascript_cache.content).to include("define(\"discourse/theme-#{theme.id}/controllers/discovery-2\"")

spec/models/theme_spec.rb

@@ -135,7 +135,7 @@ HTML
     baked = Theme.lookup_field(theme.id, :mobile, "header")
 
     expect(baked).to include(field.javascript_cache.url)
-    expect(field.javascript_cache.content).to include('HTMLBars')
+    expect(field.javascript_cache.content).to include('@ember/template-factory')
     expect(field.javascript_cache.content).to include('raw-handlebars')
   end