mirror of
https://github.com/discourse/discourse.git
synced 2025-02-25 18:55:32 -06:00
FEATURE: add uploads:s3_migration_status for looking at current status
Also a few minor cleanups and better progress reporting
This commit is contained in:
parent
59012fc0f7
commit
4f296608da
@ -393,10 +393,17 @@ task 'posts:reorder_posts', [:topic_id] => [:environment] do |_, args|
|
|||||||
end
|
end
|
||||||
|
|
||||||
def missing_uploads
|
def missing_uploads
|
||||||
|
puts "Looking for missing uploads on: #{RailsMultisite::ConnectionManagement.current_db}"
|
||||||
|
|
||||||
old_scheme_upload_count = 0
|
old_scheme_upload_count = 0
|
||||||
|
|
||||||
|
count_missing = 0
|
||||||
|
|
||||||
missing = Post.find_missing_uploads(include_local_upload: true) do |post, src, path, sha1|
|
missing = Post.find_missing_uploads(include_local_upload: true) do |post, src, path, sha1|
|
||||||
next if sha1.present?
|
next if sha1.present?
|
||||||
|
puts "Fixing missing uploads: " if count_missing == 0
|
||||||
|
count_missing += 1
|
||||||
|
print "."
|
||||||
|
|
||||||
upload_id = nil
|
upload_id = nil
|
||||||
|
|
||||||
@ -439,7 +446,6 @@ def missing_uploads
|
|||||||
upload_id
|
upload_id
|
||||||
end
|
end
|
||||||
|
|
||||||
puts "Database name: #{RailsMultisite::ConnectionManagement.current_db}"
|
|
||||||
puts "", "#{missing[:count]} post uploads are missing.", ""
|
puts "", "#{missing[:count]} post uploads are missing.", ""
|
||||||
|
|
||||||
if missing[:count] > 0
|
if missing[:count] > 0
|
||||||
|
@ -223,25 +223,46 @@ def migrate_to_s3_all_sites
|
|||||||
end
|
end
|
||||||
|
|
||||||
def migration_successful?(db, should_raise = false)
|
def migration_successful?(db, should_raise = false)
|
||||||
|
success = true
|
||||||
|
|
||||||
failure_message = "S3 migration failed for db '#{db}'."
|
failure_message = "S3 migration failed for db '#{db}'."
|
||||||
prefix = ENV["MIGRATE_TO_MULTISITE"] ? "uploads/#{db}/original/" : "original/"
|
prefix = ENV["MIGRATE_TO_MULTISITE"] ? "uploads/#{db}/original/" : "original/"
|
||||||
|
|
||||||
base_url = File.join(SiteSetting.Upload.s3_base_url, prefix)
|
base_url = File.join(SiteSetting.Upload.s3_base_url, prefix)
|
||||||
count = Upload.where("id >= 0 AND url NOT LIKE '#{base_url}%'").count
|
count = Upload.where("id >= 0 AND url NOT LIKE '#{base_url}%'").count
|
||||||
raise "#{count} of #{Upload.count} uploads are not migrated to S3. #{failure_message}" if count > 0 && should_raise
|
raise "#{count} of #{Upload.count} uploads are not migrated to S3. #{failure_message}" if count > 0 && should_raise
|
||||||
return false if count > 0
|
success &&= count == 0
|
||||||
|
|
||||||
cdn_path = SiteSetting.cdn_path("/uploads/#{db}/original").sub(/https?:/, "")
|
cdn_path = SiteSetting.cdn_path("/uploads/#{db}/original").sub(/https?:/, "")
|
||||||
count = Post.where("cooked LIKE '%#{cdn_path}%'").count
|
count = Post.where("cooked LIKE '%#{cdn_path}%'").count
|
||||||
raise "#{count} posts are not remapped to new S3 upload URL. #{failure_message}" if count > 0 && should_raise
|
raise "#{count} posts are not remapped to new S3 upload URL. #{failure_message}" if count > 0 && should_raise
|
||||||
return false if count > 0
|
success &&= count == 0
|
||||||
|
|
||||||
Rake::Task['posts:missing_uploads'].invoke('single_site')
|
Rake::Task['posts:missing_uploads'].invoke('single_site')
|
||||||
count = PostCustomField.where(name: Post::MISSING_UPLOADS).count
|
count = PostCustomField.where(name: Post::MISSING_UPLOADS).count
|
||||||
raise "rake posts:missing_uploads identified #{count} issues. #{failure_message}" if count > 0 && should_raise
|
raise "rake posts:missing_uploads identified #{count} issues. #{failure_message}" if count > 0 && should_raise
|
||||||
return false if count > 0
|
success &&= count == 0
|
||||||
|
|
||||||
return true
|
count = Post.where('baked_version <> ?', Post::BAKED_VERSION).count
|
||||||
|
if count > 0
|
||||||
|
puts "No posts require rebaking"
|
||||||
|
else
|
||||||
|
puts "#{count} posts still require rebaking and will be rebaked during regular job"
|
||||||
|
end
|
||||||
|
|
||||||
|
success
|
||||||
|
end
|
||||||
|
|
||||||
|
task "uploads:s3_migration_status" => :environment do
|
||||||
|
success = true
|
||||||
|
RailsMultisite::ConnectionManagement.each_connection do
|
||||||
|
db = RailsMultisite::ConnectionManagement.current_db
|
||||||
|
success &&= migration_successful?(db)
|
||||||
|
end
|
||||||
|
|
||||||
|
exit 1 if !success
|
||||||
|
|
||||||
|
puts "All sites appear to have uploads in order!"
|
||||||
end
|
end
|
||||||
|
|
||||||
def migrate_to_s3
|
def migrate_to_s3
|
||||||
|
Loading…
Reference in New Issue
Block a user