PERF: limit anonymization to 1 per cluster (#21992)

Anonymization is among the most expensive operations we can perform with
extreme potential to impact the database. To mitigate risk we only allow a
single anonymization across the entire cluster concurrently.

This commit introduces support for `cluster_concurrency 1`. When you set that on a Job it will only allow 1 concurrent execution per cluster.

spec/jobs/jobs_base_spec.rb

@@ -22,6 +22,46 @@ RSpec.describe ::Jobs::Base do
     end
   end
 
+  class ConcurrentJob < ::Jobs::Base
+    cluster_concurrency 1
+
+    def self.running?
+      @running
+    end
+
+    def self.running=(val)
+      @running = val
+    end
+
+    def execute(args)
+      self.class.running = true
+      sleep 20
+    ensure
+      self.class.running = false
+    end
+  end
+
+  it "handles job concurrency" do
+    ConcurrentJob.clear_cluster_concurrency_lock!
+
+    expect(ConcurrentJob.get_cluster_concurrency).to eq(1)
+    expect(BadJob.get_cluster_concurrency).to eq(nil)
+
+    expect(Sidekiq::Queues["default"].size).to eq(0)
+
+    thread = Thread.new { ConcurrentJob.new.perform({ "test" => 100 }) }
+
+    wait_for { ConcurrentJob.running? }
+
+    ConcurrentJob.new.perform({ "test" => 100 })
+    expect(Sidekiq::Queues["default"].size).to eq(1)
+
+    expect(Sidekiq::Queues["default"][0]["args"][0]).to eq("test" => 100)
+
+    thread.wakeup
+    thread.join
+  end
+
   it "handles correct jobs" do
     job = GoodJob.new
     job.perform({})