REFACTOR: Simplify converter steps in migration tooling (#29779)

* Remove unused `report_progress_in_percent` option from step
* Remove `use_custom_progress_increment` option from the step because we can figure it out by looking at the progress
* Introduce `StepTracker` to for logging warnings and errors and tracking step progress
* Make it easier to log warnings and errors in all methods of `Step` without the need to pass around a `stats` object

migrations/lib/common/extended_progress_bar.rb

@@ -4,14 +4,8 @@ require "ruby-progressbar"
 
 module Migrations
   class ExtendedProgressBar
-    def initialize(
-      max_progress: nil,
-      report_progress_in_percent: false,
-      use_custom_progress_increment: false
-    )
+    def initialize(max_progress: nil)
       @max_progress = max_progress
-      @report_progress_in_percent = report_progress_in_percent
-      @use_custom_progress_increment = use_custom_progress_increment
 
       @warning_count = 0
       @error_count = 0
@@ -31,16 +25,16 @@ module Migrations
       nil
     end
 
-    def update(stats)
+    def update(progress, warning_count, error_count)
       extra_information_changed = false
 
-      if stats.warning_count > 0
-        @warning_count += stats.warning_count
+      if warning_count > 0
+        @warning_count += warning_count
         extra_information_changed = true
       end
 
-      if stats.error_count > 0
-        @error_count += stats.error_count
+      if error_count > 0
+        @error_count += error_count
         extra_information_changed = true
       end
 
@@ -59,10 +53,10 @@ module Migrations
         @progressbar.format = "#{@base_format}#{@extra_information}"
       end
 
-      if @use_custom_progress_increment
-        @progressbar.progress += stats.progress
-      else
+      if progress == 1
         @progressbar.increment
+      else
+        @progressbar.progress += progress
       end
     end
 
@@ -70,9 +64,7 @@ module Migrations
 
     def setup_progressbar
       format =
-        if @report_progress_in_percent
-          I18n.t("progressbar.processed.percentage", percentage: "%J%")
-        elsif @max_progress
+        if @max_progress
           I18n.t("progressbar.processed.progress_with_max", current: "%c", max: "%C")
         else
           I18n.t("progressbar.processed.progress", current: "%c")

migrations/lib/converters/base/converter.rb

@@ -73,7 +73,7 @@ module Migrations::Converters::Base
       default_args = { settings: settings }
 
       args = default_args.merge(step_args(step_class))
-      step_class.new(args)
+      step_class.new(StepTracker.new, args)
     end
   end
 end

migrations/lib/converters/base/parallel_job.rb

@@ -4,7 +4,7 @@ module Migrations::Converters::Base
   class ParallelJob
     def initialize(step)
       @step = step
-      @stats = ProgressStats.new
+      @tracker = step.tracker
 
       @offline_connection = ::Migrations::Database::OfflineConnection.new
 
@@ -14,16 +14,16 @@ module Migrations::Converters::Base
     end
 
     def run(item)
-      @stats.reset!
+      @tracker.reset_stats!
       @offline_connection.clear!
 
       begin
-        @step.process_item(item, @stats)
+        @step.process_item(item)
       rescue StandardError => e
-        @stats.log_error("Failed to process item", exception: e, details: item)
+        @tracker.log_error("Failed to process item", exception: e, details: item)
       end
 
-      [@offline_connection.parametrized_insert_statements, @stats]
+      [@offline_connection.parametrized_insert_statements, @tracker.stats]
     end
 
     def cleanup

migrations/lib/converters/base/progress_step.rb

@@ -10,7 +10,7 @@ module Migrations::Converters::Base
       raise NotImplementedError
     end
 
-    def process_item(item, stats)
+    def process_item(item)
       raise NotImplementedError
     end
 
@@ -22,22 +22,6 @@ module Migrations::Converters::Base
       def run_in_parallel?
         @run_in_parallel == true
       end
-
-      def report_progress_in_percent(value)
-        @report_progress_in_percent = !!value
-      end
-
-      def report_progress_in_percent?
-        @report_progress_in_percent == true
-      end
-
-      def use_custom_progress_increment(value)
-        @use_custom_progress_increment = !!value
-      end
-
-      def use_custom_progress_increment?
-        @use_custom_progress_increment == true
-      end
     end
   end
 end

migrations/lib/converters/base/progress_step_executor.rb

@@ -39,7 +39,7 @@ module Migrations::Converters::Base
       with_progressbar do |progressbar|
         @step.items.each do |item|
           stats = job.run(item)
-          progressbar.update(stats)
+          progressbar.update(stats.progress, stats.warning_count, stats.error_count)
         end
       end
     end
@@ -76,11 +76,7 @@ module Migrations::Converters::Base
 
     def with_progressbar
       ::Migrations::ExtendedProgressBar
-        .new(
-          max_progress: @max_progress,
-          report_progress_in_percent: @step.class.report_progress_in_percent?,
-          use_custom_progress_increment: @step.class.use_custom_progress_increment?,
-        )
+        .new(max_progress: @max_progress)
         .run { |progressbar| yield progressbar }
     end
 
@@ -94,7 +90,7 @@ module Migrations::Converters::Base
               ::Migrations::Database::IntermediateDB.insert(sql, *parameters)
             end
 
-            progressbar.update(stats)
+            progressbar.update(stats.progress, stats.warning_count, stats.error_count)
           end
         end
       end

migrations/lib/converters/base/serial_job.rb

@@ -4,19 +4,19 @@ module Migrations::Converters::Base
   class SerialJob
     def initialize(step)
       @step = step
-      @stats = ProgressStats.new
+      @tracker = step.tracker
     end
 
     def run(item)
-      @stats.reset!
+      @tracker.reset_stats!
 
       begin
-        @step.process_item(item, @stats)
+        @step.process_item(item)
       rescue StandardError => e
-        @stats.log_error("Failed to process item", exception: e, details: item)
+        @tracker.log_error("Failed to process item", exception: e, details: item)
       end
 
-      @stats
+      @tracker.stats
     end
 
     def cleanup

migrations/lib/converters/base/step.rb

@@ -5,9 +5,18 @@ module Migrations::Converters::Base
     IntermediateDB = ::Migrations::Database::IntermediateDB
 
     attr_accessor :settings
+    attr_reader :tracker
 
-    def initialize(args = {})
-      args.each { |arg, value| instance_variable_set("@#{arg}", value) if respond_to?(arg, true) }
+    # inside of Step it might make more sense to access it as `step` instead of `tracker`
+    alias step tracker
+
+    def initialize(tracker, args = {})
+      @tracker = tracker
+
+      args.each do |arg, value|
+        setter = "#{arg}=".to_sym
+        public_send(setter, value) if respond_to?(setter, true)
+      end
     end
 
     def execute

migrations/lib/converters/base/step_stats.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Migrations::Converters::Base
+  StepStats = Struct.new(:progress, :warning_count, :error_count)
+end

migrations/lib/converters/base/step_tracker.rb

@@ -1,17 +1,22 @@
 # frozen_string_literal: true
 
 module Migrations::Converters::Base
-  class ProgressStats
-    attr_accessor :progress, :warning_count, :error_count
+  class StepTracker
+    attr_reader :stats
 
     def initialize
-      reset!
+      @stats = StepStats.new
+      reset_stats!
     end
 
-    def reset!
-      @progress = 1
-      @warning_count = 0
-      @error_count = 0
+    def reset_stats!
+      @stats.progress = 1
+      @stats.warning_count = 0
+      @stats.error_count = 0
+    end
+
+    def progress=(value)
+      @stats.progress = value
     end
 
     def log_info(message, details: nil)
@@ -19,20 +24,15 @@ module Migrations::Converters::Base
     end
 
     def log_warning(message, exception: nil, details: nil)
-      @warning_count += 1
+      @stats.warning_count += 1
       log(::Migrations::Database::IntermediateDB::LogEntry::WARNING, message, exception:, details:)
     end
 
     def log_error(message, exception: nil, details: nil)
-      @error_count += 1
+      @stats.error_count += 1
       log(::Migrations::Database::IntermediateDB::LogEntry::ERROR, message, exception:, details:)
     end
 
-    def ==(other)
-      other.is_a?(ProgressStats) && progress == other.progress &&
-        warning_count == other.warning_count && error_count == other.error_count
-    end
-
     private
 
     def log(type, message, exception: nil, details: nil)

migrations/lib/converters/base/worker.rb

@@ -4,14 +4,7 @@ require "oj"
 
 module Migrations::Converters::Base
   class Worker
-    OJ_SETTINGS = {
-      mode: :custom,
-      create_id: "^o",
-      create_additions: true,
-      cache_keys: true,
-      class_cache: true,
-      symbol_keys: true,
-    }
+    OJ_SETTINGS = { mode: :object, class_cache: true, symbol_keys: true }
 
     def initialize(index, input_queue, output_queue, job)
       @index = index

migrations/lib/converters/example/steps/step2.rb

@@ -8,11 +8,11 @@ module Migrations::Converters::Example
       [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
     end
 
-    def process_item(item, stats)
+    def process_item(item)
       sleep(0.5)
 
-      stats.warning_count += 1 if item.in?([3, 7, 9])
-      stats.error_count += 1 if item.in?([6, 10])
+      step.log_warning("Test", details: item) if item.in?([3, 7, 9])
+      step.log_error("Test", details: item) if item.in?([6, 10])
 
       IntermediateDB::LogEntry.create!(type: "info", message: "Step2 - #{item}")
     end

migrations/lib/converters/example/steps/step3.rb

@@ -12,9 +12,11 @@ module Migrations::Converters::Example
       (1..1000).map { |i| { counter: i } }
     end
 
-    def process_item(item, stats)
+    def process_item(item)
       sleep(0.5)
 
+      step.log_warning("Test", details: item) if item[:counter] > 10 && item[:counter] < 20
+
       IntermediateDB::LogEntry.create!(type: "info", message: "Step3 - #{item[:counter]}")
     end
   end