DEV: Support synchronous mode for uploads:sync_s3_acls rake task (#31091)

This commit updates the `uploads:sync_s3_acls` rake task to accept a
`sync` argument that would run the rake task in a synchronous manner
so that the outcome of running the rake task can easily be determined.
This commit is contained in:
Alan Guo Xiang Tan
2025-01-31 17:34:34 +08:00
committed by GitHub
parent 3978f45ca3
commit a9eefd1b48

View File

@@ -528,13 +528,8 @@ task "uploads:recover" => :environment do
end
end
task "uploads:sync_s3_acls" => :environment do
RailsMultisite::ConnectionManagement.each_connection do |db|
unless Discourse.store.external?
puts "This task only works for external storage."
exit 1
end
def sync_s3_acls(async: true, concurrency: 10)
if async
puts "CAUTION: This task may take a long time to complete! There are #{Upload.count} uploads to sync ACLs for."
puts ""
puts "-" * 30
@@ -542,6 +537,40 @@ task "uploads:sync_s3_acls" => :environment do
puts "Upload ACLs will be updated in Sidekiq jobs in batches of 100 at a time, check Sidekiq queues for SyncAclsForUploads for progress."
Upload.select(:id).find_in_batches(batch_size: 100) { |uploads| adjust_acls(uploads.map(&:id)) }
puts "", "Upload ACL sync complete!"
else
executor =
Concurrent::ThreadPoolExecutor.new(min_threads: 1, max_threads: concurrency, max_queue: 0)
Upload.find_in_batches(batch_size: 100) do |uploads|
uploads
.map do |upload|
Concurrent::Future.execute(executor:) { Discourse.store.update_upload_ACL(upload) }
end
.each(&:wait)
end
executor.shutdown
executor.wait_for_termination
end
end
task "uploads:sync_s3_acls", %i[synchronous parallel] => :environment do |_, args|
unless Discourse.store.external?
puts "This task only works for external storage."
exit 1
end
method = :sync_s3_acls
method_args = {
async: !args.key?(:synchronous) || !(args[:synchronous] == "true"),
concurrency: args[:parallel].to_i,
}
if ENV["RAILS_DB"]
send(method, method_args)
else
RailsMultisite::ConnectionManagement.each_connection { send(method, method_args) }
end
end