FEATURE: Theme settings migrations (#24071)

This commit introduces a new feature that allows theme developers to manage the transformation of theme settings over time. Similar to Rails migrations, the theme settings migration system enables developers to write and execute migrations for theme settings, ensuring a smooth transition when changes are required in the format or structure of setting values.

Example use cases for the theme settings migration system:

1. Renaming a theme setting.

2. Changing the data type of a theme setting (e.g., transforming a string setting containing comma-separated values into a proper list setting).

3. Altering the format of data stored in a theme setting.

All of these use cases and more are now possible while preserving theme setting values for sites that have already modified their theme settings.

Usage:

1. Create a top-level directory called `migrations` in your theme/component, and then within the `migrations` directory create another directory called `settings`.

2. Inside the `migrations/settings` directory, create a JavaScript file using the format `XXXX-some-name.js`, where `XXXX` is a unique 4-digit number, and `some-name` is a descriptor of your choice that describes the migration.

3. Within the JavaScript file, define and export (as the default) a function called `migrate`. This function will receive a `Map` object and must also return a `Map` object (it's acceptable to return the same `Map` object that the function received).

4. The `Map` object received by the `migrate` function will include settings that have been overridden or changed by site administrators. Settings that have never been changed from the default will not be included.

5. The keys and values contained in the `Map` object that the `migrate` function returns will replace all the currently changed settings of the theme.

6. Migrations are executed in numerical order based on the XXXX segment in the migration filenames. For instance, `0001-some-migration.js` will be executed before `0002-another-migration.js`.

Here's a complete example migration script that renames a setting from `setting_with_old_name` to `setting_with_new_name`:

```js
// File name: 0001-rename-setting.js

export default function migrate(settings) {
  if (settings.has("setting_with_old_name")) {
    settings.set("setting_with_new_name", settings.get("setting_with_old_name"));
  }
  return settings;
}
```

Internal topic: t/109980

app/controllers/admin/themes_controller.rb

@@ -160,6 +160,8 @@ class Admin::ThemesController < Admin::AdminController
       render_json_error I18n.t("themes.import_error.unknown_file_type"),
                         status: :unprocessable_entity
     end
+  rescue Theme::SettingsMigrationError => err
+    render_json_error err.message
   end
 
   def index
@@ -226,10 +228,14 @@ class Admin::ThemesController < Admin::AdminController
 
     @theme.remote_theme.update_remote_version if params[:theme][:remote_check]
 
-    @theme.remote_theme.update_from_remote if params[:theme][:remote_update]
+    if params[:theme][:remote_update]
+      @theme.remote_theme.update_from_remote(raise_if_theme_save_fails: false)
+    else
+      @theme.save
+    end
 
     respond_to do |format|
-      if @theme.save
+      if @theme.errors.blank?
         update_default_theme
 
         @theme = Theme.include_relations.find(@theme.id)
@@ -253,6 +259,8 @@ class Admin::ThemesController < Admin::AdminController
     end
   rescue RemoteTheme::ImportError => e
     render_json_error e.message
+  rescue Theme::SettingsMigrationError => e
+    render_json_error e.message
   end
 
   def destroy

app/models/remote_theme.rb

@@ -110,30 +110,33 @@ class RemoteTheme < ActiveRecord::Base
     remote_theme = new
     remote_theme.theme = theme
     remote_theme.remote_url = ""
-    remote_theme.update_from_remote(importer, skip_update: true)
 
-    theme.save!
+    do_update_child_components = false
+    theme.transaction do
+      remote_theme.update_from_remote(importer, skip_update: true, already_in_transaction: true)
 
-    if existing && update_components.present? && update_components != "none"
-      child_components = child_components.map { |url| ThemeStore::GitImporter.new(url.strip).url }
+      if existing && update_components.present? && update_components != "none"
+        child_components = child_components.map { |url| ThemeStore::GitImporter.new(url.strip).url }
 
-      if update_components == "sync"
-        ChildTheme
-          .joins(child_theme: :remote_theme)
-          .where("remote_themes.remote_url NOT IN (?)", child_components)
-          .delete_all
+        if update_components == "sync"
+          ChildTheme
+            .joins(child_theme: :remote_theme)
+            .where("remote_themes.remote_url NOT IN (?)", child_components)
+            .delete_all
+        end
+
+        child_components -=
+          theme
+            .child_themes
+            .joins(:remote_theme)
+            .where("remote_themes.remote_url IN (?)", child_components)
+            .pluck("remote_themes.remote_url")
+        theme.child_components = child_components
+        do_update_child_components = true
       end
-
-      child_components -=
-        theme
-          .child_themes
-          .joins(:remote_theme)
-          .where("remote_themes.remote_url IN (?)", child_components)
-          .pluck("remote_themes.remote_url")
-      theme.child_components = child_components
-      theme.update_child_components
     end
 
+    theme.update_child_components if do_update_child_components
     theme
   ensure
     begin
@@ -160,9 +163,9 @@ class RemoteTheme < ActiveRecord::Base
     remote_theme.private_key = private_key
     remote_theme.branch = branch
     remote_theme.remote_url = importer.url
+
     remote_theme.update_from_remote(importer)
 
-    theme.save!
     theme
   ensure
     begin
@@ -209,7 +212,12 @@ class RemoteTheme < ActiveRecord::Base
     end
   end
 
-  def update_from_remote(importer = nil, skip_update: false)
+  def update_from_remote(
+    importer = nil,
+    skip_update: false,
+    raise_if_theme_save_fails: true,
+    already_in_transaction: false
+  )
     cleanup = false
 
     unless importer
@@ -333,12 +341,6 @@ class RemoteTheme < ActiveRecord::Base
       updated_fields << theme.set_field(**opts.merge(value: value))
     end
 
-    theme.convert_settings
-
-    # Destroy fields that no longer exist in the remote theme
-    field_ids_to_destroy = theme.theme_fields.pluck(:id) - updated_fields.map { |tf| tf&.id }
-    ThemeField.where(id: field_ids_to_destroy).destroy_all
-
     if !skip_update
       self.remote_updated_at = Time.zone.now
       self.remote_version = importer.version
@@ -346,9 +348,28 @@ class RemoteTheme < ActiveRecord::Base
       self.commits_behind = 0
     end
 
-    update_theme_color_schemes(theme, theme_info["color_schemes"]) unless theme.component
+    transaction_block = -> do
+      # Destroy fields that no longer exist in the remote theme
+      field_ids_to_destroy = theme.theme_fields.pluck(:id) - updated_fields.map { |tf| tf&.id }
+      ThemeField.where(id: field_ids_to_destroy).destroy_all
+
+      update_theme_color_schemes(theme, theme_info["color_schemes"]) unless theme.component
+
+      self.save!
+      if raise_if_theme_save_fails
+        theme.save!
+      else
+        raise ActiveRecord::Rollback if !theme.save
+      end
+      theme.migrate_settings(start_transaction: false)
+    end
+
+    if already_in_transaction
+      transaction_block.call
+    else
+      self.transaction(&transaction_block)
+    end
 
-    self.save!
     self
   ensure
     begin

app/models/theme.rb

@@ -8,6 +8,9 @@ class Theme < ActiveRecord::Base
 
   BASE_COMPILER_VERSION = 77
 
+  class SettingsMigrationError < StandardError
+  end
+
   attr_accessor :child_components
 
   def self.cache
@@ -33,6 +36,7 @@ class Theme < ActiveRecord::Base
            through: :parent_theme_relation,
            source: :parent_theme
   has_many :color_schemes
+  has_many :theme_settings_migrations
   belongs_to :remote_theme, dependent: :destroy
   has_one :theme_modifier_set, dependent: :destroy
   has_one :theme_svg_sprite, dependent: :destroy
@@ -59,6 +63,9 @@ class Theme < ActiveRecord::Base
   has_many :builder_theme_fields,
            -> { where("name IN (?)", ThemeField.scss_fields) },
            class_name: "ThemeField"
+  has_many :migration_fields,
+           -> { where(target_id: Theme.targets[:migrations]) },
+           class_name: "ThemeField"
 
   validate :component_validations
   validate :validate_theme_fields
@@ -380,6 +387,7 @@ class Theme < ActiveRecord::Base
         extra_scss: 5,
         extra_js: 6,
         tests_js: 7,
+        migrations: 8,
       )
   end
 
@@ -828,22 +836,51 @@ class Theme < ActiveRecord::Base
     contents
   end
 
-  def convert_settings
-    settings.each do |setting|
-      setting_row = ThemeSetting.where(theme_id: self.id, name: setting.name.to_s).first
+  def migrate_settings(start_transaction: true)
+    block = -> do
+      runner = ThemeSettingsMigrationsRunner.new(self)
+      results = runner.run
 
-      if setting_row && setting_row.data_type != setting.type
-        if (
-             setting_row.data_type == ThemeSetting.types[:list] &&
-               setting.type == ThemeSetting.types[:string] && setting.json_schema.present?
-           )
-          convert_list_to_json_schema(setting_row, setting)
+      next if results.blank?
+
+      old_settings = self.theme_settings.pluck(:name)
+      self.theme_settings.destroy_all
+
+      final_result = results.last
+      final_result[:settings_after].each do |key, val|
+        self.update_setting(key.to_sym, val)
+      rescue Discourse::NotFound
+        if old_settings.include?(key)
+          final_result[:settings_after].delete(key)
         else
-          Rails.logger.warn(
-            "Theme setting type has changed but cannot be converted. \n\n #{setting.inspect}",
-          )
+          raise Theme::SettingsMigrationError.new(
+                  I18n.t(
+                    "themes.import_error.migrations.unknown_setting_returned_by_migration",
+                    name: final_result[:original_name],
+                    setting_name: key,
+                  ),
+                )
         end
       end
+
+      results.each do |res|
+        record =
+          ThemeSettingsMigration.new(
+            theme_id: self.id,
+            version: res[:version],
+            name: res[:name],
+            theme_field_id: res[:theme_field_id],
+          )
+        record.calculate_diff(res[:settings_before], res[:settings_after])
+        record.save!
+      end
+      self.save!
+    end
+
+    if start_transaction
+      self.transaction(&block)
+    else
+      block.call
     end
   end
 

app/models/theme_field.rb

@@ -1,12 +1,17 @@
 # frozen_string_literal: true
 
 class ThemeField < ActiveRecord::Base
+  MIGRATION_NAME_PART_MAX_LENGTH = 150
+
   belongs_to :upload
   has_one :javascript_cache, dependent: :destroy
   has_one :upload_reference, as: :target, dependent: :destroy
+  has_one :theme_settings_migration
 
   validates :value, { length: { maximum: 1024**2 } }
 
+  validate :migration_filename_is_valid, if: :migration_field?
+
   after_save do
     if self.type_id == ThemeField.types[:theme_upload_var] && saved_change_to_upload_id?
       UploadReference.ensure_exist!(upload_ids: [self.upload_id], target: self)
@@ -340,7 +345,7 @@ class ThemeField < ActiveRecord::Base
       types[:scss]
     elsif target.to_s == "extra_scss"
       types[:scss]
-    elsif target.to_s == "extra_js"
+    elsif %w[migrations extra_js].include?(target.to_s)
       types[:js]
     elsif target.to_s == "settings" || target.to_s == "translations"
       types[:yaml]
@@ -394,6 +399,10 @@ class ThemeField < ActiveRecord::Base
       self.name == SvgSprite.theme_sprite_variable_name
   end
 
+  def migration_field?
+    Theme.targets[:migrations] == self.target_id
+  end
+
   def ensure_baked!
     needs_baking = !self.value_baked || compiler_version != Theme.compiler_version
     return unless needs_baking
@@ -422,6 +431,9 @@ class ThemeField < ActiveRecord::Base
       self.error = validate_svg_sprite_xml
       self.value_baked = "baked"
       self.compiler_version = Theme.compiler_version
+    elsif migration_field?
+      self.value_baked = "baked"
+      self.compiler_version = Theme.compiler_version
     end
 
     if self.will_save_change_to_value_baked? || self.will_save_change_to_compiler_version? ||
@@ -603,6 +615,13 @@ class ThemeField < ActiveRecord::Base
       targets: :common,
       canonical: ->(h) { "assets/#{h[:name]}#{File.extname(h[:filename])}" },
     ),
+    ThemeFileMatcher.new(
+      regex: %r{\Amigrations/settings/(?<name>[^/]+)\.js\z},
+      names: nil,
+      types: :js,
+      targets: :migrations,
+      canonical: ->(h) { "migrations/settings/#{h[:name]}.js" },
+    ),
   ]
 
   # For now just work for standard fields
@@ -716,6 +735,28 @@ class ThemeField < ActiveRecord::Base
       true
     end
   end
+
+  def migration_filename_is_valid
+    if !name.match?(/\A\d{4}-[a-zA-Z0-9]+/)
+      self.errors.add(
+        :base,
+        I18n.t("themes.import_error.migrations.invalid_filename", filename: name),
+      )
+      return
+    end
+
+    # the 5 here is the length of the first 4 digits and the dash that follows
+    # them
+    if name.size - 5 > MIGRATION_NAME_PART_MAX_LENGTH
+      self.errors.add(
+        :base,
+        I18n.t(
+          "themes.import_error.migrations.name_too_long",
+          count: MIGRATION_NAME_PART_MAX_LENGTH,
+        ),
+      )
+    end
+  end
 end
 
 # == Schema Information

app/models/theme_settings_migration.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+class ThemeSettingsMigration < ActiveRecord::Base
+  belongs_to :theme
+  belongs_to :theme_field
+
+  validates :theme_id, presence: true
+  validates :theme_field_id, presence: true
+
+  validates :version, presence: true
+  validates :name, presence: true
+  validates :diff, presence: true
+
+  def calculate_diff(settings_before, settings_after)
+    diff = { additions: [], deletions: [] }
+
+    before_keys = settings_before.keys
+    after_keys = settings_after.keys
+
+    removed_keys = before_keys - after_keys
+    removed_keys.each { |key| diff[:deletions] << { key: key, val: settings_before[key] } }
+
+    added_keys = after_keys - before_keys
+    added_keys.each { |key| diff[:additions] << { key: key, val: settings_after[key] } }
+
+    common_keys = before_keys & after_keys
+    common_keys.each do |key|
+      if settings_before[key] != settings_after[key]
+        diff[:deletions] << { key: key, val: settings_before[key] }
+        diff[:additions] << { key: key, val: settings_after[key] }
+      end
+    end
+
+    self.diff = diff
+  end
+end
+
+# == Schema Information
+#
+# Table name: theme_settings_migrations
+#
+#  id             :bigint           not null, primary key
+#  theme_id       :integer          not null
+#  theme_field_id :integer          not null
+#  version        :integer          not null
+#  name           :string(150)      not null
+#  diff           :json             not null
+#  created_at     :datetime         not null
+#
+# Indexes
+#
+#  index_theme_settings_migrations_on_theme_field_id        (theme_field_id) UNIQUE
+#  index_theme_settings_migrations_on_theme_id_and_version  (theme_id,version) UNIQUE
+#

app/services/theme_settings_migrations_runner.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+class ThemeSettingsMigrationsRunner
+  Migration = Struct.new(:version, :name, :original_name, :code, :theme_field_id)
+
+  MIGRATION_ENTRY_POINT_JS = <<~JS
+    const migrate = require("discourse/theme/migration")?.default;
+    function main(settingsObj) {
+      if (!migrate) {
+        throw new Error("no_exported_migration_function");
+      }
+      if (typeof migrate !== "function") {
+        throw new Error("default_export_is_not_a_function");
+      }
+      const map = new Map(Object.entries(settingsObj));
+      const updatedMap = migrate(map);
+      if (!updatedMap) {
+        throw new Error("migration_function_no_returned_value");
+      }
+      if (!(updatedMap instanceof Map)) {
+        throw new Error("migration_function_wrong_return_type");
+      }
+      return Object.fromEntries(updatedMap.entries());
+    }
+  JS
+
+  private_constant :Migration, :MIGRATION_ENTRY_POINT_JS
+
+  def self.loader_js_lib_content
+    @loader_js_lib_content ||=
+      File.read(
+        File.join(
+          Rails.root,
+          "app/assets/javascripts/node_modules/loader.js/dist/loader/loader.js",
+        ),
+      )
+  end
+
+  def initialize(theme, limit: 100, timeout: 100, memory: 2.megabytes)
+    @theme = theme
+    @limit = limit
+    @timeout = timeout
+    @memory = memory
+  end
+
+  def run
+    fields = lookup_pending_migrations_fields
+
+    count = fields.count
+    return [] if count == 0
+
+    raise_error("themes.import_error.migrations.too_many_pending_migrations") if count > @limit
+
+    migrations = convert_fields_to_migrations(fields)
+    migrations.sort_by!(&:version)
+
+    current_migration_version =
+      @theme.theme_settings_migrations.order(version: :desc).pick(:version)
+    current_migration_version ||= -Float::INFINITY
+
+    current_settings = lookup_overriden_settings
+
+    migrations.map do |migration|
+      if migration.version <= current_migration_version
+        raise_error(
+          "themes.import_error.migrations.out_of_sequence",
+          name: migration.original_name,
+          current: current_migration_version,
+        )
+      end
+
+      migrated_settings = execute(migration, current_settings)
+      results = {
+        version: migration.version,
+        name: migration.name,
+        original_name: migration.original_name,
+        theme_field_id: migration.theme_field_id,
+        settings_before: current_settings,
+        settings_after: migrated_settings,
+      }
+      current_settings = migrated_settings
+      current_migration_version = migration.version
+      results
+    rescue DiscourseJsProcessor::TranspileError => error
+      raise_error(
+        "themes.import_error.migrations.syntax_error",
+        name: migration.original_name,
+        error: error.message,
+      )
+    rescue MiniRacer::V8OutOfMemoryError
+      raise_error(
+        "themes.import_error.migrations.exceeded_memory_limit",
+        name: migration.original_name,
+      )
+    rescue MiniRacer::ScriptTerminatedError
+      raise_error("themes.import_error.migrations.timed_out", name: migration.original_name)
+    rescue MiniRacer::RuntimeError => error
+      message = error.message
+      if message.include?("no_exported_migration_function")
+        raise_error(
+          "themes.import_error.migrations.no_exported_function",
+          name: migration.original_name,
+        )
+      elsif message.include?("default_export_is_not_a_function")
+        raise_error(
+          "themes.import_error.migrations.default_export_not_a_function",
+          name: migration.original_name,
+        )
+      elsif message.include?("migration_function_no_returned_value")
+        raise_error(
+          "themes.import_error.migrations.no_returned_value",
+          name: migration.original_name,
+        )
+      elsif message.include?("migration_function_wrong_return_type")
+        raise_error(
+          "themes.import_error.migrations.wrong_return_type",
+          name: migration.original_name,
+        )
+      else
+        raise_error(
+          "themes.import_error.migrations.runtime_error",
+          name: migration.original_name,
+          error: message,
+        )
+      end
+    end
+  end
+
+  private
+
+  def lookup_pending_migrations_fields
+    @theme
+      .migration_fields
+      .left_joins(:theme_settings_migration)
+      .where(theme_settings_migration: { id: nil })
+  end
+
+  def convert_fields_to_migrations(fields)
+    fields.map do |field|
+      match_data = /\A(?<version>\d{4})-(?<name>.+)/.match(field.name)
+
+      if !match_data
+        raise_error("themes.import_error.migrations.invalid_filename", filename: field.name)
+      end
+
+      version = match_data[:version].to_i
+      name = match_data[:name]
+      original_name = field.name
+
+      Migration.new(
+        version: version,
+        name: name,
+        original_name: original_name,
+        code: field.value,
+        theme_field_id: field.id,
+      )
+    end
+  end
+
+  def lookup_overriden_settings
+    hash = {}
+    @theme.theme_settings.each { |row| hash[row.name] = ThemeSettingsManager.cast_row_value(row) }
+    hash
+  end
+
+  def execute(migration, settings)
+    context = MiniRacer::Context.new(timeout: @timeout, max_memory: @memory)
+
+    context.eval(self.class.loader_js_lib_content, filename: "loader.js")
+
+    context.eval(
+      DiscourseJsProcessor.transpile(migration.code, "", "discourse/theme/migration"),
+      filename: "theme-#{@theme.id}-migration.js",
+    )
+
+    context.eval(MIGRATION_ENTRY_POINT_JS, filename: "migration-entrypoint.js")
+    context.call("main", settings)
+  ensure
+    context&.dispose
+  end
+
+  def raise_error(message_key, **i18n_args)
+    raise Theme::SettingsMigrationError.new(I18n.t(message_key, **i18n_args))
+  end
+end

app/services/themes_install_task.rb

@@ -74,8 +74,7 @@ class ThemesInstallTask
   end
 
   def update
-    @remote_theme.update_from_remote
-    @theme.save
+    @remote_theme.update_from_remote(raise_if_theme_save_fails: false)
     add_component_to_all_themes
     @remote_theme.last_error_text.nil?
   end