From a0b94dca16504521120a555d4851836ec17cbd30 Mon Sep 17 00:00:00 2001 From: David Taylor Date: Tue, 7 Nov 2023 10:24:49 +0000 Subject: [PATCH] DEV: Use WebPack stats plugin to map entrypoints to chunks (#24239) Previously, we were parsing webpack JS chunk filenames from the HTML files which ember-cli generates. This worked ok for simple entrypoints, but falls apart once we start using async imports(), which are not included in the HTML. This commit uses the stats plugin to generate an assets.json file, and updates Rails to parse it instead of the HTML. Caching on the Rails side is also improved to avoid reading from the filesystem multiple times per request in develoment. Co-authored-by: Godfrey Chan --- .../javascripts/discourse/app/index.html | 9 +--- .../javascripts/discourse/ember-cli-build.js | 41 +++++++++++++++ app/assets/javascripts/discourse/package.json | 1 + .../javascripts/discourse/tests/index.html | 15 ++---- app/assets/javascripts/yarn.lock | 5 ++ app/views/qunit/theme.html.erb | 3 +- lib/ember_cli.rb | 52 ++++++------------- spec/lib/ember_cli_spec.rb | 35 ------------- 8 files changed, 70 insertions(+), 91 deletions(-) diff --git a/app/assets/javascripts/discourse/app/index.html b/app/assets/javascripts/discourse/app/index.html index 13db87b245e..3b465595f3b 100644 --- a/app/assets/javascripts/discourse/app/index.html +++ b/app/assets/javascripts/discourse/app/index.html @@ -24,14 +24,9 @@ {{content-for "head"}} - - - + - - - - + diff --git a/app/assets/javascripts/discourse/ember-cli-build.js b/app/assets/javascripts/discourse/ember-cli-build.js index d22036ad697..4ed050884d7 100644 --- a/app/assets/javascripts/discourse/ember-cli-build.js +++ b/app/assets/javascripts/discourse/ember-cli-build.js @@ -13,6 +13,7 @@ const DeprecationSilencer = require("deprecation-silencer"); const generateWorkboxTree = require("./lib/workbox-tree-builder"); const { compatBuild } = require("@embroider/compat"); const { Webpack } = require("@embroider/webpack"); +const { StatsWriterPlugin } = require("webpack-stats-plugin"); process.env.BROCCOLI_ENABLED_MEMOIZE = true; @@ -135,6 +136,13 @@ module.exports = function (defaults) { output: { publicPath: "auto", }, + entry: { + "assets/discourse.js/features/markdown-it.js": { + import: "./static/markdown-it", + dependOn: "assets/discourse.js", + runtime: false, + }, + }, externals: [ function ({ request }, callback) { if ( @@ -175,6 +183,39 @@ module.exports = function (defaults) { }, ], }, + plugins: [ + // The server use this output to map each asset to its chunks + new StatsWriterPlugin({ + filename: "assets.json", + stats: { + all: false, + entrypoints: true, + }, + transform({ entrypoints }) { + let names = Object.keys(entrypoints); + let output = {}; + + for (let name of names.sort()) { + let assets = entrypoints[name].assets.map( + (asset) => asset.name + ); + + let parent = names.find((parentName) => + name.startsWith(parentName + "/") + ); + + if (parent) { + name = name.slice(parent.length + 1); + output[parent][name] = { assets }; + } else { + output[name] = { assets }; + } + } + + return JSON.stringify(output, null, 2); + }, + }), + ], }, }, }); diff --git a/app/assets/javascripts/discourse/package.json b/app/assets/javascripts/discourse/package.json index f2a4a5f4adc..743be342876 100644 --- a/app/assets/javascripts/discourse/package.json +++ b/app/assets/javascripts/discourse/package.json @@ -130,6 +130,7 @@ "util": "^0.12.5", "virtual-dom": "^2.1.1", "webpack": "^5.89.0", + "webpack-stats-plugin": "^1.1.3", "wizard": "1.0.0", "workbox-cacheable-response": "^7.0.0", "workbox-core": "^7.0.0", diff --git a/app/assets/javascripts/discourse/tests/index.html b/app/assets/javascripts/discourse/tests/index.html index 625754161b3..d12a6ae8004 100644 --- a/app/assets/javascripts/discourse/tests/index.html +++ b/app/assets/javascripts/discourse/tests/index.html @@ -45,19 +45,12 @@ {{content-for "body"}} {{content-for "test-body"}} - - - - - - - + - - - - + + + diff --git a/app/assets/javascripts/yarn.lock b/app/assets/javascripts/yarn.lock index 789436df5a0..b90242c503c 100644 --- a/app/assets/javascripts/yarn.lock +++ b/app/assets/javascripts/yarn.lock @@ -10789,6 +10789,11 @@ webpack-sources@^3.2.3: resolved "https://registry.yarnpkg.com/webpack-sources/-/webpack-sources-3.2.3.tgz#2d4daab8451fd4b240cc27055ff6a0c2ccea0cde" integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== +webpack-stats-plugin@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/webpack-stats-plugin/-/webpack-stats-plugin-1.1.3.tgz#ebcc36c8b468074ad737882e2043c1ce4b55d928" + integrity sha512-yUKYyy+e0iF/w31QdfioRKY+h3jDBRpthexBOWGKda99iu2l/wxYsI/XqdlP5IU58/0KB9CsJZgWNAl+/MPkRw== + webpack@^5.89.0: version "5.89.0" resolved "https://registry.yarnpkg.com/webpack/-/webpack-5.89.0.tgz#56b8bf9a34356e93a6625770006490bf3a7f32dc" diff --git a/app/views/qunit/theme.html.erb b/app/views/qunit/theme.html.erb index eff72c156ce..7a1d27396d4 100644 --- a/app/views/qunit/theme.html.erb +++ b/app/views/qunit/theme.html.erb @@ -8,7 +8,8 @@ <%- if @has_test_bundle && !@suggested_themes %> <%= preload_script "vendor" %> <%= preload_script "test-support" %> - <%= preload_script "discourse-for-tests" %> + <%= preload_script "discourse" %> + <%= preload_script "test" %> <%= preload_script "locales/#{I18n.locale}" %> <%= preload_script "admin" %> <%- Discourse.find_plugin_js_assets(include_disabled: true).each do |file| %> diff --git a/lib/ember_cli.rb b/lib/ember_cli.rb index b2e7607c9ce..d866cd2bf38 100644 --- a/lib/ember_cli.rb +++ b/lib/ember_cli.rb @@ -1,6 +1,9 @@ # frozen_string_literal: true -module EmberCli +class EmberCli < ActiveSupport::CurrentAttributes + # Cache which persists for the duration of a request + attribute :request_cached_script_chunks + def self.dist_dir "#{Rails.root}/app/assets/javascripts/discourse/dist" end @@ -10,30 +13,23 @@ module EmberCli end def self.script_chunks - return @chunk_infos if @chunk_infos + return @production_chunk_infos if @production_chunk_infos + return self.request_cached_script_chunks if self.request_cached_script_chunks - chunk_infos = {} + chunk_infos = JSON.parse(File.read("#{dist_dir}/assets.json")) - begin - test_html = File.read("#{dist_dir}/tests/index.html") - chunk_infos.merge! parse_chunks_from_html(test_html) - rescue Errno::ENOENT - # production build + chunk_infos.transform_keys! { |key| key.delete_prefix("assets/").delete_suffix(".js") } + + chunk_infos.transform_values! do |value| + value["assets"].map { |chunk| chunk.delete_prefix("assets/").delete_suffix(".js") } end - index_html = File.read("#{dist_dir}/index.html") - chunk_infos.merge! parse_chunks_from_html(index_html) - - @chunk_infos = chunk_infos if Rails.env.production? - chunk_infos + @production_chunk_infos = chunk_infos if Rails.env.production? + self.request_cached_script_chunks = chunk_infos rescue Errno::ENOENT {} end - def self.parse_source_map_path(file) - File.read("#{dist_dir}/assets/#{file}")[%r{//# sourceMappingURL=(.*)$}, 1] - end - def self.is_ember_cli_asset?(name) assets.include?(name) || script_chunks.values.flatten.include?(name.delete_suffix(".js")) end @@ -56,31 +52,13 @@ module EmberCli end end - def self.parse_chunks_from_html(html) - doc = Nokogiri::HTML5.parse(html) - - chunk_infos = {} - - doc - .css("discourse-chunked-script") - .each do |discourse_script| - entrypoint = discourse_script.attr("entrypoint") - chunk_infos[entrypoint] = discourse_script - .css("script[src]") - .map do |script| - script.attr("src").delete_prefix("#{Discourse.base_path}/assets/").delete_suffix(".js") - end - end - - chunk_infos - end - def self.has_tests? File.exist?("#{dist_dir}/tests/index.html") end def self.clear_cache! - @chunk_infos = nil + @prod_chunk_infos = nil @assets = nil + self.request_cached_script_chunks = nil end end diff --git a/spec/lib/ember_cli_spec.rb b/spec/lib/ember_cli_spec.rb index 59de34d6709..2079f9f483e 100644 --- a/spec/lib/ember_cli_spec.rb +++ b/spec/lib/ember_cli_spec.rb @@ -6,39 +6,4 @@ describe EmberCli do expect(EmberCli.ember_version).to match(/\A\d+\.\d+/) end end - - describe ".parse_chunks_from_html" do - def generate_html - <<~HTML - - - - - - - - - Hello world - - - HTML - end - - it "can parse chunks for a normal site" do - chunks = EmberCli.parse_chunks_from_html generate_html - expect(chunks["discourse"]).to eq(%w[firstchunk secondchunk]) - end - - it "can parse chunks for a subfolder site" do - set_subfolder "/discuss" - - html = generate_html - - # sanity check that our fixture is working - expect(html).to include("/discuss/assets/firstchunk.js") - - chunks = EmberCli.parse_chunks_from_html html - expect(chunks["discourse"]).to eq(%w[firstchunk secondchunk]) - end - end end