DEV: Allow Ember CLI assets to be used by development Rails app (#16511)

Previously, accessing the Rails app directly in development mode would give you assets from our 'legacy' Ember asset pipeline. The only way to run with Ember CLI assets was to run ember-cli as a proxy. This was quite limiting when working on things which are bypassed when using the ember-cli proxy (e.g. changes to `application.html.erb`). Also, since `ember-auto-import` introduced chunking, visiting `/theme-qunit` under Ember CLI was failing to include all necessary chunks.

This commit teaches Sprockets about our Ember CLI assets so that they can be used in development mode, and are automatically collected up under `/public/assets` during `assets:precompile`. As a bonus, this allows us to remove all the custom manifest modification from `assets:precompile`.

The key changes are:
- Introduce a shared `EmberCli.enabled?` helper
- When ember-cli is enabled, add ember-cli `/dist/assets` as the top-priority Rails asset directory
- Have ember-cli output a `chunks.json` manifest, and teach `preload_script` to read it and append the correct chunks to their associated `afterFile`
- Remove most custom ember-cli logic from the `assets:precompile` step. Instead, rely on Rails to take care of pulling the 'precompiled' assets into the `public/assets` directory. Move the 'renaming' logic to runtime, so it can be used in development mode as well.
- Remove fingerprinting from `ember-cli-build`, and allow Rails to take care of things

Long-term, we may want to replace Sprockets with the lighter-weight Propshaft. The changes made in this commit have been made with that long-term goal in mind.

tldr: when you visit the rails app directly, you'll now be served the current ember-cli assets. To keep these up-to-date make sure either `ember serve`, or `ember build --watch` is running. If you really want to load the old non-ember-cli assets, then you should start the server with `EMBER_CLI_PROD_ASSETS=0`. (the legacy asset pipeline will be removed very soon)
This commit is contained in:
David Taylor 2022-04-21 16:26:34 +01:00 committed by GitHub
parent e8f8a7fc91
commit 22a7905f2d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 132 additions and 162 deletions

View File

@ -8,7 +8,6 @@ const prettyTextEngine = require("./lib/pretty-text-engine");
const { createI18nTree } = require("./lib/translation-plugin");
const discourseScss = require("./lib/discourse-scss");
const funnel = require("broccoli-funnel");
const AssetRev = require("broccoli-asset-rev");
module.exports = function (defaults) {
let discourseRoot = resolve("../../../..");
@ -32,8 +31,7 @@ module.exports = function (defaults) {
forbidEval: true,
},
fingerprint: {
// Disabled here, but handled manually below when in production mode.
// This is so we can apply a single AssetRev operation over the application and our additional trees
// Handled by Rails asset pipeline
enabled: false,
},
SRI: {
@ -119,7 +117,7 @@ module.exports = function (defaults) {
"/app/assets/javascripts/discourse/public/assets/scripts/module-shims.js"
);
const mergedTree = mergeTrees([
return mergeTrees([
createI18nTree(discourseRoot, vendorJs),
app.toTree(),
funnel(`${discourseRoot}/public/javascripts`, { destDir: "javascripts" }),
@ -137,17 +135,4 @@ module.exports = function (defaults) {
inputFiles: [`discourse-boot.js`],
}),
]);
if (isProduction) {
return new AssetRev(mergedTree, {
exclude: [
"javascripts/**/*",
"assets/test-i18n*",
"assets/highlightjs",
"assets/testem.css",
],
});
} else {
return mergedTree;
}
};

View File

@ -37,6 +37,7 @@
"discourse-hbr": "^1.0.0",
"discourse-widget-hbs": "^1.0.0",
"ember-auto-import": "^2.2.4",
"ember-auto-import-chunks-json-generator": "^1.0.0",
"ember-buffered-proxy": "^2.0.0-beta.0",
"ember-cached-decorator-polyfill": "^0.1.4",
"ember-cli": "~3.25.3",

View File

@ -5943,6 +5943,16 @@ elliptic@^6.5.3:
minimalistic-assert "^1.0.1"
minimalistic-crypto-utils "^1.0.1"
ember-auto-import-chunks-json-generator@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/ember-auto-import-chunks-json-generator/-/ember-auto-import-chunks-json-generator-1.0.0.tgz#c2ca9380854f3cfefc67e3a1bd7f7c7a124f45ec"
integrity sha512-4itBcVrK/oAjbS8rycgRFS/f07rGtRgliHQU4BLXGKFm5qPrB6D75H76yCY/tE9+R+yIBV8cfdkVhJcPR1jtwQ==
dependencies:
broccoli-merge-trees "^4.2.0"
broccoli-plugin "^4.0.7"
ember-cli-babel "^7.26.6"
ember-cli-htmlbars "^5.7.1"
ember-auto-import@^1.10.1:
version "1.11.2"
resolved "https://registry.yarnpkg.com/ember-auto-import/-/ember-auto-import-1.11.2.tgz#b6e9a0dddd88a10692830ffa4f5dfd8c137c8919"

View File

@ -12,28 +12,15 @@ class QunitController < ApplicationController
request.headers["HTTP_X_DISCOURSE_EMBER_CLI"] == "true"
end
# only used in test / dev
# only used in non-ember-cli test / dev
def index
raise Discourse::NotFound.new if is_ember_cli_proxy?
raise Discourse::NotFound.new if is_ember_cli_proxy? || EmberCli.enabled?
raise Discourse::InvalidAccess.new if Rails.env.production?
end
def theme
raise Discourse::NotFound.new if !can_see_theme_qunit?
@is_proxied = is_ember_cli_proxy?
@legacy_ember = if Rails.env.production?
ENV['EMBER_CLI_PROD_ASSETS'] == "0"
else
!@is_proxied
end
# In production mode all bundles use `application`
@app_bundle = "application"
if Rails.env.development? && @is_proxied
@app_bundle = "discourse"
end
param_key = nil
@suggested_themes = nil
if (id = get_param(:id)).present?

View File

@ -135,27 +135,21 @@ module ApplicationHelper
path
end
def preload_vendor_scripts
scripts = ["vendor"]
def preload_script(script)
script = EmberCli.transform_name(script)
if ENV["EMBER_CLI_PROD_ASSETS"] != "0"
@@vendor_chunks ||= begin
all_assets = ActionController::Base.helpers.assets_manifest.assets
all_assets.keys.filter_map { |name| name[/\A(chunk\..*)\.js\z/, 1] }
end
scripts.push(*@@vendor_chunks)
scripts = [script]
if EmberCli.enabled? && chunks = EmberCli.script_chunks[script]
scripts.push(*chunks)
end
scripts.map do |name|
preload_script(name)
path = script_asset_path(name)
preload_script_url(path)
end.join("\n").html_safe
end
def preload_script(script)
path = script_asset_path(script)
preload_script_url(path)
end
def preload_script_url(url)
<<~HTML.html_safe
<link rel="preload" href="#{url}" as="script">

View File

@ -3,32 +3,27 @@
module QunitHelper
def vendor_theme_tests
return preload_script("vendor-theme-tests") if @legacy_ember
preload_vendor_scripts
if EmberCli.enabled?
preload_script("vendor")
else
preload_script("vendor-theme-tests")
end
end
def support_bundles
result = []
if Rails.env.production? || @legacy_ember
result << preload_script("discourse/tests/test-support-rails")
result << preload_script("discourse/tests/test-helpers-rails")
else
result << preload_script("test-support")
result << preload_script("test-helpers")
end
result.join("\n").html_safe
result = [
preload_script("discourse/tests/test-support-rails"),
preload_script("discourse/tests/test-helpers-rails")
].join("\n").html_safe
end
def boot_bundles
result = []
if @legacy_ember
result << preload_script("discourse/tests/test_starter")
elsif @is_proxied
if EmberCli.enabled?
result << preload_script("scripts/discourse-test-listen-boot")
result << preload_script("scripts/discourse-boot")
else
result << preload_script("discourse-test-listen-boot")
result << preload_script("discourse-boot")
result << preload_script("discourse/tests/test_starter")
end
result.join("\n").html_safe
end

View File

@ -29,7 +29,7 @@
<%- if ExtraLocalesController.client_overrides_exist? %>
<%= preload_script_url ExtraLocalesController.url('overrides') %>
<%- end %>
<%= preload_vendor_scripts %>
<%= preload_script "vendor" %>
<%= preload_script "application" %>
<%- Discourse.find_plugin_js_assets(include_official: allow_plugins?, include_unofficial: allow_third_party_plugins?, request: request).each do |file| %>
<%= preload_script file %>

View File

@ -6,7 +6,7 @@
<%= discourse_stylesheet_link_tag(:desktop, theme_id: nil) %>
<%= discourse_stylesheet_link_tag(:test_helper, theme_id: nil) %>
<%= preload_script "locales/#{I18n.locale}" %>
<%= preload_vendor_scripts %>
<%= preload_script "vendor" %>
<%= preload_script "application" %>
<%= preload_script "admin" %>
<%= preload_script "discourse/tests/test-support-rails" %>

View File

@ -8,7 +8,7 @@
<%= discourse_stylesheet_link_tag(:test_helper, theme_id: nil) %>
<%= preload_script "locales/#{I18n.locale}" %>
<%= vendor_theme_tests %>
<%= preload_script @app_bundle %>
<%= preload_script "application" %>
<%= preload_script "admin" %>
<%= preload_script "discourse/tests/active-plugins" %>
<%= preload_script "admin-plugins" %>
@ -31,7 +31,7 @@
</head>
<body>
<%- if !@suggested_themes %>
<%- if @legacy_ember %>
<%- if !EmberCli.enabled? %>
<div id="qunit"></div>
<div id="qunit-fixture"></div>
<%- end %>

View File

@ -10,7 +10,7 @@
</div>
<%- content_for(:no_ember_head) do %>
<%= preload_vendor_scripts %>
<%= preload_script "vendor" %>
<%= render_google_universal_analytics_code %>
<%= tag.meta id: 'data-activate-account', data: { path: path('/session/hp') } %>
<%- end %>

View File

@ -55,6 +55,8 @@ require 'pry-rails' if Rails.env.development?
require 'discourse_fonts'
require_relative '../lib/ember_cli'
if defined?(Bundler)
bundler_groups = [:default]
@ -178,8 +180,17 @@ module Discourse
discourse/tests/test_starter.js
}
if ENV['EMBER_CLI_PROD_ASSETS'] == "0"
if EmberCli.enabled?
config.assets.precompile += %w{
discourse.js
test-support.js
test-helpers.js
scripts/discourse-test-listen-boot
scripts/discourse-boot
}
else
config.assets.precompile += %w{
application.js
discourse/tests/test-support-rails.js
discourse/tests/test-helpers-rails.js
vendor-theme-tests.js
@ -290,6 +301,13 @@ module Discourse
Sprockets.register_mime_type 'application/javascript', extensions: ['.js', '.es6', '.js.es6'], charset: :unicode
Sprockets.register_postprocessor 'application/javascript', DiscourseJsProcessor
if EmberCli.enabled?
Discourse::Application.initializer :prepend_ember_assets do |app|
# Needs to be in its own initializer so it runs after the append_assets_path initializer defined by Sprockets
app.config.assets.paths.unshift "#{app.config.root}/app/assets/javascripts/discourse/dist/assets"
end
end
require 'discourse_redis'
require 'logster/redis_store'
# Use redis for our cache

View File

@ -990,6 +990,9 @@ module Discourse
},
Thread.new {
SvgSprite.core_svgs
},
Thread.new {
EmberCli.script_chunks
}
].each(&:join)
ensure

View File

@ -8,6 +8,10 @@ class DiscourseJsProcessor
@@plugin_transpile_paths ||= Set.new
end
def self.ember_cli?(filename)
filename.include?("/app/assets/javascripts/discourse/dist/")
end
def self.call(input)
root_path = input[:load_path] || ''
logical_path = (input[:filename] || '').sub(root_path, '').gsub(/\.(js|es6).*$/, '').sub(/^\//, '')
@ -18,7 +22,7 @@ class DiscourseJsProcessor
end
# add sourceURL until we can do proper source maps
unless Rails.env.production?
if !Rails.env.production? && !ember_cli?(input[:filename])
plugin_name = root_path[/\/plugins\/([\w-]+)\/assets/, 1]
source_url = if plugin_name
"plugins/#{plugin_name}/assets/javascripts/#{logical_path}"
@ -40,6 +44,9 @@ class DiscourseJsProcessor
def self.should_transpile?(filename)
filename ||= ''
# skip ember cli
return false if ember_cli?(filename)
# es6 is always transpiled
return true if filename.end_with?(".es6") || filename.end_with?(".es6.erb")

39
lib/ember_cli.rb Normal file
View File

@ -0,0 +1,39 @@
# frozen_string_literal: true
module EmberCli
ALIASES ||= {
"application" => "discourse",
"discourse/tests/test-support-rails" => "test-support",
"discourse/tests/test-helpers-rails" => "test-helpers"
}
def self.enabled?
ENV["EMBER_CLI_PROD_ASSETS"] != "0"
end
def self.script_chunks
return @@chunk_infos if defined? @@chunk_infos
raw_chunk_infos = JSON.parse(File.read("#{Rails.configuration.root}/app/assets/javascripts/discourse/dist/chunks.json"))
chunk_infos = raw_chunk_infos["scripts"].map do |info|
logical_name = info["afterFile"][/\Aassets\/(.*)\.js\z/, 1]
chunks = info["scriptChunks"].map { |filename| filename[/\Aassets\/(.*)\.js\z/, 1] }
[logical_name, chunks]
end.to_h
@@chunk_infos = chunk_infos if Rails.env.production?
chunk_infos
rescue Errno::ENOENT
{}
end
# Some assets have changed name following the switch
# to ember-cli. When the switch is complete, we can
# drop this method and update all the references
# to use the new names
def self.transform_name(name)
return name if !enabled?
ALIASES[name] || name
end
end

View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
if !defined?(EMBER_CLI)
EMBER_CLI = ENV["EMBER_CLI_PROD_ASSETS"] != "0"
EMBER_CLI = EmberCli.enabled?
end
task 'assets:precompile:before' do
@ -52,10 +52,10 @@ task 'assets:precompile:before' do
require 'digest/sha1'
if EMBER_CLI
# Remove the assets that Ember CLI will handle for us
Rails.configuration.assets.precompile.reject! do |asset|
asset.is_a?(String) && is_ember_cli_asset?(asset)
end
# Add ember cli chunks
Rails.configuration.assets.precompile.push(
*EmberCli.script_chunks.values.flatten
)
end
end
@ -100,7 +100,7 @@ end
def is_ember_cli_asset?(name)
return false if !EMBER_CLI
%w(
application.js
discourse.js
admin.js
ember_jquery.js
pretty-text-bundle.js
@ -261,80 +261,7 @@ def copy_maxmind(from_path, to_path)
end
end
def copy_ember_cli_assets
ember_dir = "app/assets/javascripts/discourse"
ember_cli_assets = "#{ember_dir}/dist/assets/"
assets = {}
files = {}
# Copy assets and generate manifest data
log_task_duration('Copy assets and generate manifest data') {
Dir["#{ember_cli_assets}**/*"].each do |f|
if File.file?(f)
rel_file = f.sub(ember_cli_assets, "")
file_digest = Digest::SHA384.digest(File.read(f))
digest = if f =~ /\-([a-f0-9]+)\./
Regexp.last_match[1]
else
Digest.hexencode(file_digest)[0...32]
end
dest = "public/assets"
dest_sub = dest
if rel_file =~ /^([a-z\-\_]+)\//
dest_sub = "#{dest}/#{Regexp.last_match[1]}"
end
FileUtils.mkdir_p(dest_sub) unless Dir.exist?(dest_sub)
log_file = File.basename(rel_file).sub("-#{digest}", "")
# We need a few hacks here to move what Ember uses to what Rails wants
case log_file
when "discourse.js"
log_file = "application.js"
rel_file.sub!(/^discourse/, "application")
when "test-support.js"
log_file = "discourse/tests/test-support-rails.js"
rel_file = "discourse/tests/test-support-rails-#{digest}.js"
when "test-helpers.js"
log_file = "discourse/tests/test-helpers-rails.js"
rel_file = "discourse/tests/test-helpers-rails-#{digest}.js"
end
res = FileUtils.cp(f, "#{dest}/#{rel_file}")
assets[log_file] = rel_file
files[rel_file] = {
"logical_path" => log_file,
"mtime" => File.mtime(f).iso8601(9),
"size" => File.size(f),
"digest" => digest,
"integrity" => "sha384-#{Base64.encode64(file_digest).chomp}"
}
end
end
}
# Update manifest file
log_task_duration('Update manifest file') {
manifest_result = Dir["public/assets/.sprockets-manifest-*.json"]
if manifest_result && manifest_result.size == 1
json = JSON.parse(File.read(manifest_result[0]))
json['files'].merge!(files)
json['assets'].merge!(assets)
File.write(manifest_result[0], json.to_json)
end
}
end
task 'test_ember_cli_copy' do
copy_ember_cli_assets
end
task 'assets:precompile' => 'assets:precompile:before' do
copy_ember_cli_assets if EMBER_CLI
refresh_days = GlobalSetting.refresh_maxmind_db_during_precompile_days
if refresh_days.to_i > 0

View File

@ -58,6 +58,10 @@ task "qunit:test", [:timeout, :qunit_path] do |_, args|
"UNICORN_TIMEOUT" => "90",
}
if !ember_cli
env["EMBER_CLI_PROD_ASSETS"] = "0"
end
pid = Process.spawn(
env,
"#{Rails.root}/bin/unicorn",

View File

@ -33,7 +33,7 @@ describe ApplicationHelper do
helper.request.env["HTTP_ACCEPT_ENCODING"] = 'br'
link = helper.preload_script('application')
expect(link).to eq(preload_link("https://awesome.com/brotli_asset/application.js"))
expect(link).to eq(preload_link("https://awesome.com/brotli_asset/#{EmberCli.transform_name("application")}.js"))
end
context "with s3 CDN" do
@ -47,14 +47,14 @@ describe ApplicationHelper do
it "deals correctly with subfolder" do
set_subfolder "/community"
expect(helper.preload_script("application")).to include('https://s3cdn.com/assets/application.js')
expect(helper.preload_script("application")).to include("https://s3cdn.com/assets/#{EmberCli.transform_name("application")}.js")
end
it "replaces cdn URLs with s3 cdn subfolder paths" do
global_setting :s3_cdn_url, 'https://s3cdn.com/s3_subpath'
set_cdn_url "https://awesome.com"
set_subfolder "/community"
expect(helper.preload_script("application")).to include('https://s3cdn.com/s3_subpath/assets/application.js')
expect(helper.preload_script("application")).to include("https://s3cdn.com/s3_subpath/assets/#{EmberCli.transform_name("application")}.js")
end
it "returns magic brotli mangling for brotli requests" do
@ -62,26 +62,26 @@ describe ApplicationHelper do
helper.request.env["HTTP_ACCEPT_ENCODING"] = 'br'
link = helper.preload_script('application')
expect(link).to eq(preload_link("https://s3cdn.com/assets/application.br.js"))
expect(link).to eq(preload_link("https://s3cdn.com/assets/#{EmberCli.transform_name("application")}.br.js"))
end
it "gives s3 cdn if asset host is not set" do
link = helper.preload_script('application')
expect(link).to eq(preload_link("https://s3cdn.com/assets/application.js"))
expect(link).to eq(preload_link("https://s3cdn.com/assets/#{EmberCli.transform_name("application")}.js"))
end
it "can fall back to gzip compression" do
helper.request.env["HTTP_ACCEPT_ENCODING"] = 'gzip'
link = helper.preload_script('application')
expect(link).to eq(preload_link("https://s3cdn.com/assets/application.gz.js"))
expect(link).to eq(preload_link("https://s3cdn.com/assets/#{EmberCli.transform_name("application")}.gz.js"))
end
it "gives s3 cdn even if asset host is set" do
set_cdn_url "https://awesome.com"
link = helper.preload_script('application')
expect(link).to eq(preload_link("https://s3cdn.com/assets/application.js"))
expect(link).to eq(preload_link("https://s3cdn.com/assets/#{EmberCli.transform_name("application")}.js"))
end
it "gives s3 cdn but without brotli/gzip extensions for theme tests assets" do

View File

@ -10,7 +10,7 @@ describe InvitesController do
it 'shows the accept invite page' do
get "/invites/#{invite.invite_key}"
expect(response.status).to eq(200)
expect(response.body).to have_tag(:script, with: { src: '/assets/application.js' })
expect(response.body).to have_tag(:script, with: { src: "/assets/#{EmberCli.transform_name("application")}.js" })
expect(response.body).not_to include(invite.email)
expect(response.body).to_not include(I18n.t('invite.not_found_template', site_name: SiteSetting.title, base_url: Discourse.base_url))
@ -26,7 +26,7 @@ describe InvitesController do
ActionDispatch::Request.any_instance.stubs(:session).returns(authentication: { email: invite.email })
get "/invites/#{invite.invite_key}"
expect(response.status).to eq(200)
expect(response.body).to have_tag(:script, with: { src: '/assets/application.js' })
expect(response.body).to have_tag(:script, with: { src: "/assets/#{EmberCli.transform_name("application")}.js" })
expect(response.body).to include(invite.email)
expect(response.body).not_to include('i*****g@a***********e.ooo')
end

View File

@ -99,7 +99,7 @@ describe QunitController do
expect(response.body).to include("/test-support")
expect(response.body).to include("/test-helpers")
expect(response.body).to include("/assets/markdown-it-bundle.js")
expect(response.body).to include("/assets/application.js")
expect(response.body).to include("/assets/#{EmberCli.transform_name("application")}.js")
expect(response.body).to include("/assets/admin.js")
expect(response.body).to match(/\/theme-javascripts\/\h{40}\.js/)
expect(response.body).to include("/theme-javascripts/tests/#{theme.id}-")

View File

@ -2355,7 +2355,7 @@ RSpec.describe TopicsController do
body = response.body
expect(body).to have_tag(:script, src: '/assets/application.js')
expect(body).to have_tag(:script, src: "/assets/#{EmberCli.transform_name("application")}.js")
expect(body).to have_tag(:meta, with: { name: 'fragment' })
end
end
@ -2631,7 +2631,7 @@ RSpec.describe TopicsController do
body = response.body
expect(response.status).to eq(200)
expect(body).to have_tag(:script, with: { src: '/assets/application.js' })
expect(body).to have_tag(:script, with: { src: "/assets/#{EmberCli.transform_name("application")}.js" })
expect(body).to_not have_tag(:meta, with: { name: 'fragment' })
end
end
@ -2646,7 +2646,7 @@ RSpec.describe TopicsController do
body = response.body
expect(body).to have_tag(:script, with: { src: '/assets/application.js' })
expect(body).to have_tag(:script, with: { src: "/assets/#{EmberCli.transform_name("application")}.js" })
expect(body).to have_tag(:meta, with: { name: 'fragment' })
end
@ -4320,7 +4320,7 @@ RSpec.describe TopicsController do
body = response.body
expect(body).to have_tag(:script, with: { src: '/assets/application.js' })
expect(body).to have_tag(:script, with: { src: "/assets/#{EmberCli.transform_name("application")}.js" })
expect(body).to have_tag(:meta, with: { name: 'fragment' })
end
end