mirror of
https://github.com/discourse/discourse.git
synced 2025-02-25 18:55:32 -06:00
DEV: Apply syntax_tree formatting to lib/*
This commit is contained in:
@@ -1,30 +1,28 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'file_store/local_store'
|
||||
require "file_store/local_store"
|
||||
|
||||
desc 'Update each post with latest markdown'
|
||||
task 'posts:rebake' => :environment do
|
||||
ENV['RAILS_DB'] ? rebake_posts : rebake_posts_all_sites
|
||||
desc "Update each post with latest markdown"
|
||||
task "posts:rebake" => :environment do
|
||||
ENV["RAILS_DB"] ? rebake_posts : rebake_posts_all_sites
|
||||
end
|
||||
|
||||
task 'posts:rebake_uncooked_posts' => :environment do
|
||||
task "posts:rebake_uncooked_posts" => :environment do
|
||||
# rebaking uncooked posts can very quickly saturate sidekiq
|
||||
# this provides an insurance policy so you can safely run and stop
|
||||
# this rake task without worrying about your sidekiq imploding
|
||||
Jobs.run_immediately!
|
||||
|
||||
ENV['RAILS_DB'] ? rebake_uncooked_posts : rebake_uncooked_posts_all_sites
|
||||
ENV["RAILS_DB"] ? rebake_uncooked_posts : rebake_uncooked_posts_all_sites
|
||||
end
|
||||
|
||||
def rebake_uncooked_posts_all_sites
|
||||
RailsMultisite::ConnectionManagement.each_connection do |db|
|
||||
rebake_uncooked_posts
|
||||
end
|
||||
RailsMultisite::ConnectionManagement.each_connection { |db| rebake_uncooked_posts }
|
||||
end
|
||||
|
||||
def rebake_uncooked_posts
|
||||
puts "Rebaking uncooked posts on #{RailsMultisite::ConnectionManagement.current_db}"
|
||||
uncooked = Post.where('baked_version <> ? or baked_version IS NULL', Post::BAKED_VERSION)
|
||||
uncooked = Post.where("baked_version <> ? or baked_version IS NULL", Post::BAKED_VERSION)
|
||||
|
||||
rebaked = 0
|
||||
total = uncooked.count
|
||||
@@ -37,9 +35,7 @@ def rebake_uncooked_posts
|
||||
# may have been cooked in interim
|
||||
post = uncooked.where(id: id).first
|
||||
|
||||
if post
|
||||
rebake_post(post)
|
||||
end
|
||||
rebake_post(post) if post
|
||||
|
||||
print_status(rebaked += 1, total)
|
||||
end
|
||||
@@ -47,17 +43,23 @@ def rebake_uncooked_posts
|
||||
puts "", "#{rebaked} posts done!", ""
|
||||
end
|
||||
|
||||
desc 'Update each post with latest markdown and refresh oneboxes'
|
||||
task 'posts:refresh_oneboxes' => :environment do
|
||||
ENV['RAILS_DB'] ? rebake_posts(invalidate_oneboxes: true) : rebake_posts_all_sites(invalidate_oneboxes: true)
|
||||
desc "Update each post with latest markdown and refresh oneboxes"
|
||||
task "posts:refresh_oneboxes" => :environment do
|
||||
if ENV["RAILS_DB"]
|
||||
rebake_posts(invalidate_oneboxes: true)
|
||||
else
|
||||
rebake_posts_all_sites(invalidate_oneboxes: true)
|
||||
end
|
||||
end
|
||||
|
||||
desc 'Rebake all posts with a quote using a letter_avatar'
|
||||
task 'posts:fix_letter_avatars' => :environment do
|
||||
desc "Rebake all posts with a quote using a letter_avatar"
|
||||
task "posts:fix_letter_avatars" => :environment do
|
||||
next unless SiteSetting.external_system_avatars_enabled
|
||||
|
||||
search = Post.where("user_id <> -1")
|
||||
.where("raw LIKE '%/letter\_avatar/%' OR cooked LIKE '%/letter\_avatar/%'")
|
||||
search =
|
||||
Post.where("user_id <> -1").where(
|
||||
"raw LIKE '%/letter\_avatar/%' OR cooked LIKE '%/letter\_avatar/%'",
|
||||
)
|
||||
|
||||
rebaked = 0
|
||||
total = search.count
|
||||
@@ -70,9 +72,9 @@ task 'posts:fix_letter_avatars' => :environment do
|
||||
puts "", "#{rebaked} posts done!", ""
|
||||
end
|
||||
|
||||
desc 'Rebake all posts matching string/regex and optionally delay the loop'
|
||||
task 'posts:rebake_match', [:pattern, :type, :delay] => [:environment] do |_, args|
|
||||
args.with_defaults(type: 'string')
|
||||
desc "Rebake all posts matching string/regex and optionally delay the loop"
|
||||
task "posts:rebake_match", %i[pattern type delay] => [:environment] do |_, args|
|
||||
args.with_defaults(type: "string")
|
||||
pattern = args[:pattern]
|
||||
type = args[:type]&.downcase
|
||||
delay = args[:delay]&.to_i
|
||||
@@ -83,7 +85,7 @@ task 'posts:rebake_match', [:pattern, :type, :delay] => [:environment] do |_, ar
|
||||
elsif delay && delay < 1
|
||||
puts "ERROR: delay parameter should be an integer and greater than 0"
|
||||
exit 1
|
||||
elsif type != 'string' && type != 'regex'
|
||||
elsif type != "string" && type != "regex"
|
||||
puts "ERROR: Expecting rake posts:rebake_match[pattern,type] where type is string or regex"
|
||||
exit 1
|
||||
end
|
||||
@@ -103,9 +105,7 @@ task 'posts:rebake_match', [:pattern, :type, :delay] => [:environment] do |_, ar
|
||||
end
|
||||
|
||||
def rebake_posts_all_sites(opts = {})
|
||||
RailsMultisite::ConnectionManagement.each_connection do |db|
|
||||
rebake_posts(opts)
|
||||
end
|
||||
RailsMultisite::ConnectionManagement.each_connection { |db| rebake_posts(opts) }
|
||||
end
|
||||
|
||||
def rebake_posts(opts = {})
|
||||
@@ -118,13 +118,17 @@ def rebake_posts(opts = {})
|
||||
total = Post.count
|
||||
rebaked = 0
|
||||
batch = 1000
|
||||
Post.update_all('baked_version = NULL')
|
||||
Post.update_all("baked_version = NULL")
|
||||
|
||||
(0..(total - 1).abs).step(batch) do |i|
|
||||
Post.order(id: :desc).offset(i).limit(batch).each do |post|
|
||||
rebake_post(post, opts)
|
||||
print_status(rebaked += 1, total)
|
||||
end
|
||||
Post
|
||||
.order(id: :desc)
|
||||
.offset(i)
|
||||
.limit(batch)
|
||||
.each do |post|
|
||||
rebake_post(post, opts)
|
||||
print_status(rebaked += 1, total)
|
||||
end
|
||||
end
|
||||
ensure
|
||||
SiteSetting.disable_system_edit_notifications = disable_system_edit_notifications
|
||||
@@ -134,72 +138,77 @@ def rebake_posts(opts = {})
|
||||
end
|
||||
|
||||
def rebake_post(post, opts = {})
|
||||
if !opts[:priority]
|
||||
opts[:priority] = :ultra_low
|
||||
end
|
||||
opts[:priority] = :ultra_low if !opts[:priority]
|
||||
post.rebake!(**opts)
|
||||
rescue => e
|
||||
puts "", "Failed to rebake (topic_id: #{post.topic_id}, post_id: #{post.id})", e, e.backtrace.join("\n")
|
||||
puts "",
|
||||
"Failed to rebake (topic_id: #{post.topic_id}, post_id: #{post.id})",
|
||||
e,
|
||||
e.backtrace.join("\n")
|
||||
end
|
||||
|
||||
def print_status(current, max)
|
||||
print "\r%9d / %d (%5.1f%%)" % [current, max, ((current.to_f / max.to_f) * 100).round(1)]
|
||||
end
|
||||
|
||||
desc 'normalize all markdown so <pre><code> is not used and instead backticks'
|
||||
task 'posts:normalize_code' => :environment do
|
||||
lang = ENV['CODE_LANG'] || ''
|
||||
require 'import/normalize'
|
||||
desc "normalize all markdown so <pre><code> is not used and instead backticks"
|
||||
task "posts:normalize_code" => :environment do
|
||||
lang = ENV["CODE_LANG"] || ""
|
||||
require "import/normalize"
|
||||
|
||||
puts "Normalizing"
|
||||
i = 0
|
||||
Post.where("raw like '%<pre>%<code>%'").each do |p|
|
||||
normalized = Import::Normalize.normalize_code_blocks(p.raw, lang)
|
||||
if normalized != p.raw
|
||||
p.revise(Discourse.system_user, raw: normalized)
|
||||
putc "."
|
||||
i += 1
|
||||
Post
|
||||
.where("raw like '%<pre>%<code>%'")
|
||||
.each do |p|
|
||||
normalized = Import::Normalize.normalize_code_blocks(p.raw, lang)
|
||||
if normalized != p.raw
|
||||
p.revise(Discourse.system_user, raw: normalized)
|
||||
putc "."
|
||||
i += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
puts
|
||||
puts "#{i} posts normalized!"
|
||||
end
|
||||
|
||||
def remap_posts(find, type, ignore_case, replace = "")
|
||||
ignore_case = ignore_case == 'true'
|
||||
ignore_case = ignore_case == "true"
|
||||
i = 0
|
||||
|
||||
Post.raw_match(find, type).find_each do |p|
|
||||
regex =
|
||||
case type
|
||||
when 'string' then
|
||||
Regexp.new(Regexp.escape(find), ignore_case)
|
||||
when 'regex' then
|
||||
Regexp.new(find, ignore_case)
|
||||
end
|
||||
Post
|
||||
.raw_match(find, type)
|
||||
.find_each do |p|
|
||||
regex =
|
||||
case type
|
||||
when "string"
|
||||
Regexp.new(Regexp.escape(find), ignore_case)
|
||||
when "regex"
|
||||
Regexp.new(find, ignore_case)
|
||||
end
|
||||
|
||||
new_raw = p.raw.gsub(regex, replace)
|
||||
new_raw = p.raw.gsub(regex, replace)
|
||||
|
||||
if new_raw != p.raw
|
||||
begin
|
||||
p.revise(Discourse.system_user, { raw: new_raw }, bypass_bump: true, skip_revision: true)
|
||||
putc "."
|
||||
i += 1
|
||||
rescue
|
||||
puts "\nFailed to remap post (topic_id: #{p.topic_id}, post_id: #{p.id})\n"
|
||||
if new_raw != p.raw
|
||||
begin
|
||||
p.revise(Discourse.system_user, { raw: new_raw }, bypass_bump: true, skip_revision: true)
|
||||
putc "."
|
||||
i += 1
|
||||
rescue StandardError
|
||||
puts "\nFailed to remap post (topic_id: #{p.topic_id}, post_id: #{p.id})\n"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
i
|
||||
end
|
||||
|
||||
desc 'Remap all posts matching specific string'
|
||||
task 'posts:remap', [:find, :replace, :type, :ignore_case] => [:environment] do |_, args|
|
||||
require 'highline/import'
|
||||
desc "Remap all posts matching specific string"
|
||||
task "posts:remap", %i[find replace type ignore_case] => [:environment] do |_, args|
|
||||
require "highline/import"
|
||||
|
||||
args.with_defaults(type: 'string', ignore_case: 'false')
|
||||
args.with_defaults(type: "string", ignore_case: "false")
|
||||
find = args[:find]
|
||||
replace = args[:replace]
|
||||
type = args[:type]&.downcase
|
||||
@@ -211,15 +220,18 @@ task 'posts:remap', [:find, :replace, :type, :ignore_case] => [:environment] do
|
||||
elsif !replace
|
||||
puts "ERROR: Expecting rake posts:remap['find','replace']. Want to delete a word/string instead? Try rake posts:delete_word['word-to-delete']"
|
||||
exit 1
|
||||
elsif type != 'string' && type != 'regex'
|
||||
elsif type != "string" && type != "regex"
|
||||
puts "ERROR: Expecting rake posts:remap['find','replace',type] where type is string or regex"
|
||||
exit 1
|
||||
elsif ignore_case != 'true' && ignore_case != 'false'
|
||||
elsif ignore_case != "true" && ignore_case != "false"
|
||||
puts "ERROR: Expecting rake posts:remap['find','replace',type,ignore_case] where ignore_case is true or false"
|
||||
exit 1
|
||||
else
|
||||
confirm_replace = ask("Are you sure you want to replace all #{type} occurrences of '#{find}' with '#{replace}'? (Y/n)")
|
||||
exit 1 unless (confirm_replace == "" || confirm_replace.downcase == 'y')
|
||||
confirm_replace =
|
||||
ask(
|
||||
"Are you sure you want to replace all #{type} occurrences of '#{find}' with '#{replace}'? (Y/n)",
|
||||
)
|
||||
exit 1 unless (confirm_replace == "" || confirm_replace.downcase == "y")
|
||||
end
|
||||
|
||||
puts "Remapping"
|
||||
@@ -227,11 +239,11 @@ task 'posts:remap', [:find, :replace, :type, :ignore_case] => [:environment] do
|
||||
puts "", "#{total} posts remapped!", ""
|
||||
end
|
||||
|
||||
desc 'Delete occurrence of a word/string'
|
||||
task 'posts:delete_word', [:find, :type, :ignore_case] => [:environment] do |_, args|
|
||||
require 'highline/import'
|
||||
desc "Delete occurrence of a word/string"
|
||||
task "posts:delete_word", %i[find type ignore_case] => [:environment] do |_, args|
|
||||
require "highline/import"
|
||||
|
||||
args.with_defaults(type: 'string', ignore_case: 'false')
|
||||
args.with_defaults(type: "string", ignore_case: "false")
|
||||
find = args[:find]
|
||||
type = args[:type]&.downcase
|
||||
ignore_case = args[:ignore_case]&.downcase
|
||||
@@ -239,15 +251,16 @@ task 'posts:delete_word', [:find, :type, :ignore_case] => [:environment] do |_,
|
||||
if !find
|
||||
puts "ERROR: Expecting rake posts:delete_word['word-to-delete']"
|
||||
exit 1
|
||||
elsif type != 'string' && type != 'regex'
|
||||
elsif type != "string" && type != "regex"
|
||||
puts "ERROR: Expecting rake posts:delete_word[pattern, type] where type is string or regex"
|
||||
exit 1
|
||||
elsif ignore_case != 'true' && ignore_case != 'false'
|
||||
elsif ignore_case != "true" && ignore_case != "false"
|
||||
puts "ERROR: Expecting rake posts:delete_word[pattern, type,ignore_case] where ignore_case is true or false"
|
||||
exit 1
|
||||
else
|
||||
confirm_delete = ask("Are you sure you want to remove all #{type} occurrences of '#{find}'? (Y/n)")
|
||||
exit 1 unless (confirm_delete == "" || confirm_delete.downcase == 'y')
|
||||
confirm_delete =
|
||||
ask("Are you sure you want to remove all #{type} occurrences of '#{find}'? (Y/n)")
|
||||
exit 1 unless (confirm_delete == "" || confirm_delete.downcase == "y")
|
||||
end
|
||||
|
||||
puts "Processing"
|
||||
@@ -255,9 +268,8 @@ task 'posts:delete_word', [:find, :type, :ignore_case] => [:environment] do |_,
|
||||
puts "", "#{total} posts updated!", ""
|
||||
end
|
||||
|
||||
desc 'Delete all likes'
|
||||
task 'posts:delete_all_likes' => :environment do
|
||||
|
||||
desc "Delete all likes"
|
||||
task "posts:delete_all_likes" => :environment do
|
||||
post_actions = PostAction.where(post_action_type_id: PostActionType.types[:like])
|
||||
|
||||
likes_deleted = 0
|
||||
@@ -267,7 +279,7 @@ task 'posts:delete_all_likes' => :environment do
|
||||
begin
|
||||
post_action.remove_act!(Discourse.system_user)
|
||||
print_status(likes_deleted += 1, total)
|
||||
rescue
|
||||
rescue StandardError
|
||||
# skip
|
||||
end
|
||||
end
|
||||
@@ -277,8 +289,8 @@ task 'posts:delete_all_likes' => :environment do
|
||||
puts "", "#{likes_deleted} likes deleted!", ""
|
||||
end
|
||||
|
||||
desc 'Refreshes each post that was received via email'
|
||||
task 'posts:refresh_emails', [:topic_id] => [:environment] do |_, args|
|
||||
desc "Refreshes each post that was received via email"
|
||||
task "posts:refresh_emails", [:topic_id] => [:environment] do |_, args|
|
||||
posts = Post.where.not(raw_email: nil).where(via_email: true)
|
||||
posts = posts.where(topic_id: args[:topic_id]) if args[:topic_id]
|
||||
|
||||
@@ -290,12 +302,17 @@ task 'posts:refresh_emails', [:topic_id] => [:environment] do |_, args|
|
||||
receiver = Email::Receiver.new(post.raw_email)
|
||||
|
||||
body, elided = receiver.select_body
|
||||
body = receiver.add_attachments(body || '', post.user)
|
||||
body = receiver.add_attachments(body || "", post.user)
|
||||
body << Email::Receiver.elided_html(elided) if elided.present?
|
||||
|
||||
post.revise(Discourse.system_user, { raw: body, cook_method: Post.cook_methods[:regular] },
|
||||
skip_revision: true, skip_validations: true, bypass_bump: true)
|
||||
rescue
|
||||
post.revise(
|
||||
Discourse.system_user,
|
||||
{ raw: body, cook_method: Post.cook_methods[:regular] },
|
||||
skip_revision: true,
|
||||
skip_validations: true,
|
||||
bypass_bump: true,
|
||||
)
|
||||
rescue StandardError
|
||||
puts "Failed to refresh post (topic_id: #{post.topic_id}, post_id: #{post.id})"
|
||||
end
|
||||
|
||||
@@ -307,10 +324,9 @@ task 'posts:refresh_emails', [:topic_id] => [:environment] do |_, args|
|
||||
puts "", "Done. #{updated} posts updated.", ""
|
||||
end
|
||||
|
||||
desc 'Reorders all posts based on their creation_date'
|
||||
task 'posts:reorder_posts', [:topic_id] => [:environment] do |_, args|
|
||||
desc "Reorders all posts based on their creation_date"
|
||||
task "posts:reorder_posts", [:topic_id] => [:environment] do |_, args|
|
||||
Post.transaction do
|
||||
|
||||
builder = DB.build <<~SQL
|
||||
WITH ordered_posts AS (
|
||||
SELECT
|
||||
@@ -342,11 +358,11 @@ task 'posts:reorder_posts', [:topic_id] => [:environment] do |_, args|
|
||||
builder.exec
|
||||
|
||||
[
|
||||
["notifications", "post_number"],
|
||||
["post_timings", "post_number"],
|
||||
["posts", "reply_to_post_number"],
|
||||
["topic_users", "last_read_post_number"],
|
||||
["topic_users", "last_emailed_post_number"],
|
||||
%w[notifications post_number],
|
||||
%w[post_timings post_number],
|
||||
%w[posts reply_to_post_number],
|
||||
%w[topic_users last_read_post_number],
|
||||
%w[topic_users last_emailed_post_number],
|
||||
].each do |table, column|
|
||||
builder = DB.build <<~SQL
|
||||
UPDATE
|
||||
@@ -385,7 +401,6 @@ task 'posts:reorder_posts', [:topic_id] => [:environment] do |_, args|
|
||||
builder.where("topic_id = ?", args[:topic_id]) if args[:topic_id]
|
||||
builder.where("post_number < 0")
|
||||
builder.exec
|
||||
|
||||
end
|
||||
|
||||
puts "", "Done.", ""
|
||||
@@ -398,63 +413,69 @@ def missing_uploads
|
||||
|
||||
count_missing = 0
|
||||
|
||||
missing = Post.find_missing_uploads(include_local_upload: true) do |post, src, path, sha1|
|
||||
next if sha1.present?
|
||||
puts "Fixing missing uploads: " if count_missing == 0
|
||||
count_missing += 1
|
||||
missing =
|
||||
Post.find_missing_uploads(include_local_upload: true) do |post, src, path, sha1|
|
||||
next if sha1.present?
|
||||
puts "Fixing missing uploads: " if count_missing == 0
|
||||
count_missing += 1
|
||||
|
||||
upload_id = nil
|
||||
upload_id = nil
|
||||
|
||||
# recovering old scheme upload.
|
||||
local_store = FileStore::LocalStore.new
|
||||
public_path = "#{local_store.public_dir}#{path}"
|
||||
file_path = nil
|
||||
# recovering old scheme upload.
|
||||
local_store = FileStore::LocalStore.new
|
||||
public_path = "#{local_store.public_dir}#{path}"
|
||||
file_path = nil
|
||||
|
||||
if File.file?(public_path)
|
||||
file_path = public_path
|
||||
else
|
||||
tombstone_path = public_path.sub("/uploads/", "/uploads/tombstone/")
|
||||
file_path = tombstone_path if File.file?(tombstone_path)
|
||||
end
|
||||
|
||||
if file_path.present?
|
||||
if (upload = UploadCreator.new(File.open(file_path), File.basename(path)).create_for(Discourse.system_user.id)).persisted?
|
||||
upload_id = upload.id
|
||||
|
||||
post.reload
|
||||
new_raw = post.raw.dup
|
||||
new_raw = new_raw.gsub(path, upload.url)
|
||||
|
||||
PostRevisor.new(post, Topic.with_deleted.find_by(id: post.topic_id)).revise!(
|
||||
Discourse.system_user,
|
||||
{
|
||||
raw: new_raw
|
||||
},
|
||||
skip_validations: true,
|
||||
force_new_version: true,
|
||||
bypass_bump: true
|
||||
)
|
||||
|
||||
print "🆗"
|
||||
if File.file?(public_path)
|
||||
file_path = public_path
|
||||
else
|
||||
print "❌"
|
||||
tombstone_path = public_path.sub("/uploads/", "/uploads/tombstone/")
|
||||
file_path = tombstone_path if File.file?(tombstone_path)
|
||||
end
|
||||
else
|
||||
print "🚫"
|
||||
old_scheme_upload_count += 1
|
||||
end
|
||||
|
||||
upload_id
|
||||
end
|
||||
if file_path.present?
|
||||
if (
|
||||
upload =
|
||||
UploadCreator.new(File.open(file_path), File.basename(path)).create_for(
|
||||
Discourse.system_user.id,
|
||||
)
|
||||
).persisted?
|
||||
upload_id = upload.id
|
||||
|
||||
post.reload
|
||||
new_raw = post.raw.dup
|
||||
new_raw = new_raw.gsub(path, upload.url)
|
||||
|
||||
PostRevisor.new(post, Topic.with_deleted.find_by(id: post.topic_id)).revise!(
|
||||
Discourse.system_user,
|
||||
{ raw: new_raw },
|
||||
skip_validations: true,
|
||||
force_new_version: true,
|
||||
bypass_bump: true,
|
||||
)
|
||||
|
||||
print "🆗"
|
||||
else
|
||||
print "❌"
|
||||
end
|
||||
else
|
||||
print "🚫"
|
||||
old_scheme_upload_count += 1
|
||||
end
|
||||
|
||||
upload_id
|
||||
end
|
||||
|
||||
puts "", "#{missing[:count]} post uploads are missing.", ""
|
||||
|
||||
if missing[:count] > 0
|
||||
puts "#{missing[:uploads].count} uploads are missing."
|
||||
puts "#{old_scheme_upload_count} of #{missing[:uploads].count} are old scheme uploads." if old_scheme_upload_count > 0
|
||||
if old_scheme_upload_count > 0
|
||||
puts "#{old_scheme_upload_count} of #{missing[:uploads].count} are old scheme uploads."
|
||||
end
|
||||
puts "#{missing[:post_uploads].count} of #{Post.count} posts are affected.", ""
|
||||
|
||||
if ENV['GIVE_UP'] == "1"
|
||||
if ENV["GIVE_UP"] == "1"
|
||||
missing[:post_uploads].each do |id, uploads|
|
||||
post = Post.with_deleted.find_by(id: id)
|
||||
if post
|
||||
@@ -466,11 +487,9 @@ def missing_uploads
|
||||
end
|
||||
end
|
||||
|
||||
if ENV['VERBOSE'] == "1"
|
||||
if ENV["VERBOSE"] == "1"
|
||||
puts "missing uploads!"
|
||||
missing[:uploads].each do |path|
|
||||
puts "#{path}"
|
||||
end
|
||||
missing[:uploads].each { |path| puts "#{path}" }
|
||||
|
||||
if missing[:post_uploads].count > 0
|
||||
puts
|
||||
@@ -490,14 +509,12 @@ def missing_uploads
|
||||
missing[:count] == 0
|
||||
end
|
||||
|
||||
desc 'Finds missing post upload records from cooked HTML content'
|
||||
task 'posts:missing_uploads' => :environment do |_, args|
|
||||
desc "Finds missing post upload records from cooked HTML content"
|
||||
task "posts:missing_uploads" => :environment do |_, args|
|
||||
if ENV["RAILS_DB"]
|
||||
missing_uploads
|
||||
else
|
||||
RailsMultisite::ConnectionManagement.each_connection do
|
||||
missing_uploads
|
||||
end
|
||||
RailsMultisite::ConnectionManagement.each_connection { missing_uploads }
|
||||
end
|
||||
end
|
||||
|
||||
@@ -506,56 +523,55 @@ def recover_uploads_from_index(path)
|
||||
|
||||
db = RailsMultisite::ConnectionManagement.current_db
|
||||
cdn_path = SiteSetting.cdn_path("/uploads/#{db}").sub(/https?:/, "")
|
||||
Post.where("cooked LIKE ?", "%#{cdn_path}%").each do |post|
|
||||
regex = Regexp.new("((https?:)?#{Regexp.escape(cdn_path)}[^,;\\]\\>\\t\\n\\s)\"\']+)")
|
||||
uploads = []
|
||||
post.raw.scan(regex).each do |match|
|
||||
uploads << match[0]
|
||||
Post
|
||||
.where("cooked LIKE ?", "%#{cdn_path}%")
|
||||
.each do |post|
|
||||
regex = Regexp.new("((https?:)?#{Regexp.escape(cdn_path)}[^,;\\]\\>\\t\\n\\s)\"\']+)")
|
||||
uploads = []
|
||||
post.raw.scan(regex).each { |match| uploads << match[0] }
|
||||
|
||||
if uploads.length > 0
|
||||
lookup << [post.id, uploads]
|
||||
else
|
||||
print "."
|
||||
post.rebake!
|
||||
end
|
||||
end
|
||||
|
||||
if uploads.length > 0
|
||||
lookup << [post.id, uploads]
|
||||
else
|
||||
print "."
|
||||
post.rebake!
|
||||
end
|
||||
end
|
||||
|
||||
PostCustomField.where(name: Post::MISSING_UPLOADS).pluck(:post_id, :value).each do |post_id, uploads|
|
||||
uploads = JSON.parse(uploads)
|
||||
raw = Post.where(id: post_id).pluck_first(:raw)
|
||||
uploads.map! do |upload|
|
||||
orig = upload
|
||||
if raw.scan(upload).length == 0
|
||||
upload = upload.sub(SiteSetting.Upload.s3_cdn_url, SiteSetting.Upload.s3_base_url)
|
||||
end
|
||||
if raw.scan(upload).length == 0
|
||||
upload = upload.sub(SiteSetting.Upload.s3_base_url, Discourse.base_url)
|
||||
end
|
||||
if raw.scan(upload).length == 0
|
||||
upload = upload.sub(Discourse.base_url + "/", "/")
|
||||
end
|
||||
if raw.scan(upload).length == 0
|
||||
# last resort, try for sha
|
||||
sha = upload.split("/")[-1]
|
||||
sha = sha.split(".")[0]
|
||||
|
||||
if sha.length == 40 && raw.scan(sha).length == 1
|
||||
raw.match(Regexp.new("([^\"'<\\s\\n]+#{sha}[^\"'>\\s\\n]+)"))
|
||||
upload = $1
|
||||
PostCustomField
|
||||
.where(name: Post::MISSING_UPLOADS)
|
||||
.pluck(:post_id, :value)
|
||||
.each do |post_id, uploads|
|
||||
uploads = JSON.parse(uploads)
|
||||
raw = Post.where(id: post_id).pluck_first(:raw)
|
||||
uploads.map! do |upload|
|
||||
orig = upload
|
||||
if raw.scan(upload).length == 0
|
||||
upload = upload.sub(SiteSetting.Upload.s3_cdn_url, SiteSetting.Upload.s3_base_url)
|
||||
end
|
||||
if raw.scan(upload).length == 0
|
||||
upload = upload.sub(SiteSetting.Upload.s3_base_url, Discourse.base_url)
|
||||
end
|
||||
upload = upload.sub(Discourse.base_url + "/", "/") if raw.scan(upload).length == 0
|
||||
if raw.scan(upload).length == 0
|
||||
# last resort, try for sha
|
||||
sha = upload.split("/")[-1]
|
||||
sha = sha.split(".")[0]
|
||||
|
||||
if sha.length == 40 && raw.scan(sha).length == 1
|
||||
raw.match(Regexp.new("([^\"'<\\s\\n]+#{sha}[^\"'>\\s\\n]+)"))
|
||||
upload = $1
|
||||
end
|
||||
end
|
||||
if raw.scan(upload).length == 0
|
||||
puts "can not find #{orig} in\n\n#{raw}"
|
||||
upload = nil
|
||||
end
|
||||
upload
|
||||
end
|
||||
if raw.scan(upload).length == 0
|
||||
puts "can not find #{orig} in\n\n#{raw}"
|
||||
upload = nil
|
||||
end
|
||||
upload
|
||||
uploads.compact!
|
||||
lookup << [post_id, uploads] if uploads.length > 0
|
||||
end
|
||||
uploads.compact!
|
||||
if uploads.length > 0
|
||||
lookup << [post_id, uploads]
|
||||
end
|
||||
end
|
||||
|
||||
lookup.each do |post_id, uploads|
|
||||
post = Post.find(post_id)
|
||||
@@ -573,7 +589,12 @@ def recover_uploads_from_index(path)
|
||||
puts "Skipping #{url} in #{post.full_url} cause it appears to have a short file name"
|
||||
next
|
||||
end
|
||||
found = `cat #{path} | grep #{name} | grep original`.split("\n")[0] rescue nil
|
||||
found =
|
||||
begin
|
||||
`cat #{path} | grep #{name} | grep original`.split("\n")[0]
|
||||
rescue StandardError
|
||||
nil
|
||||
end
|
||||
if found.blank?
|
||||
puts "Skipping #{url} in #{post.full_url} cause it missing from index"
|
||||
next
|
||||
@@ -609,8 +630,8 @@ def recover_uploads_from_index(path)
|
||||
end
|
||||
end
|
||||
|
||||
desc 'Attempts to recover missing uploads from an index file'
|
||||
task 'posts:recover_uploads_from_index' => :environment do |_, args|
|
||||
desc "Attempts to recover missing uploads from an index file"
|
||||
task "posts:recover_uploads_from_index" => :environment do |_, args|
|
||||
path = File.expand_path(Rails.root + "public/uploads/all_the_files")
|
||||
if File.exist?(path)
|
||||
puts "Found existing index file at #{path}"
|
||||
@@ -621,14 +642,12 @@ task 'posts:recover_uploads_from_index' => :environment do |_, args|
|
||||
if RailsMultisite::ConnectionManagement.current_db != "default"
|
||||
recover_uploads_from_index(path)
|
||||
else
|
||||
RailsMultisite::ConnectionManagement.each_connection do
|
||||
recover_uploads_from_index(path)
|
||||
end
|
||||
RailsMultisite::ConnectionManagement.each_connection { recover_uploads_from_index(path) }
|
||||
end
|
||||
end
|
||||
|
||||
desc 'invalidate broken images'
|
||||
task 'posts:invalidate_broken_images' => :environment do
|
||||
desc "invalidate broken images"
|
||||
task "posts:invalidate_broken_images" => :environment do
|
||||
puts "Invalidating broken images.."
|
||||
|
||||
posts = Post.where("raw like '%<img%'")
|
||||
@@ -646,8 +665,8 @@ task 'posts:invalidate_broken_images' => :environment do
|
||||
end
|
||||
|
||||
desc "Coverts full upload URLs in `Post#raw` to short upload url"
|
||||
task 'posts:inline_uploads' => :environment do |_, args|
|
||||
if ENV['RAILS_DB']
|
||||
task "posts:inline_uploads" => :environment do |_, args|
|
||||
if ENV["RAILS_DB"]
|
||||
correct_inline_uploads
|
||||
else
|
||||
RailsMultisite::ConnectionManagement.each_connection do |db|
|
||||
@@ -662,11 +681,14 @@ def correct_inline_uploads
|
||||
dry_run = (ENV["DRY_RUN"].nil? ? true : ENV["DRY_RUN"] != "false")
|
||||
verbose = ENV["VERBOSE"]
|
||||
|
||||
scope = Post.joins(:upload_references).distinct("posts.id")
|
||||
.where(
|
||||
"raw LIKE ?",
|
||||
"%/uploads/#{RailsMultisite::ConnectionManagement.current_db}/original/%",
|
||||
)
|
||||
scope =
|
||||
Post
|
||||
.joins(:upload_references)
|
||||
.distinct("posts.id")
|
||||
.where(
|
||||
"raw LIKE ?",
|
||||
"%/uploads/#{RailsMultisite::ConnectionManagement.current_db}/original/%",
|
||||
)
|
||||
|
||||
affected_posts_count = scope.count
|
||||
fixed_count = 0
|
||||
@@ -684,20 +706,17 @@ def correct_inline_uploads
|
||||
|
||||
if post.raw != new_raw
|
||||
if !dry_run
|
||||
PostRevisor.new(post, Topic.with_deleted.find_by(id: post.topic_id))
|
||||
.revise!(
|
||||
Discourse.system_user,
|
||||
{
|
||||
raw: new_raw
|
||||
},
|
||||
skip_validations: true,
|
||||
force_new_version: true,
|
||||
bypass_bump: true
|
||||
)
|
||||
PostRevisor.new(post, Topic.with_deleted.find_by(id: post.topic_id)).revise!(
|
||||
Discourse.system_user,
|
||||
{ raw: new_raw },
|
||||
skip_validations: true,
|
||||
force_new_version: true,
|
||||
bypass_bump: true,
|
||||
)
|
||||
end
|
||||
|
||||
if verbose
|
||||
require 'diffy'
|
||||
require "diffy"
|
||||
Diffy::Diff.default_format = :color
|
||||
puts "Cooked diff for Post #{post.id}"
|
||||
puts Diffy::Diff.new(PrettyText.cook(post.raw), PrettyText.cook(new_raw), context: 1)
|
||||
@@ -713,7 +732,7 @@ def correct_inline_uploads
|
||||
putc "X"
|
||||
not_corrected_post_ids << post.id
|
||||
end
|
||||
rescue
|
||||
rescue StandardError
|
||||
putc "!"
|
||||
failed_to_correct_post_ids << post.id
|
||||
end
|
||||
@@ -730,7 +749,5 @@ def correct_inline_uploads
|
||||
puts "Ids of posts that encountered failures: #{failed_to_correct_post_ids}"
|
||||
end
|
||||
|
||||
if dry_run
|
||||
puts "Task was ran in dry run mode. Set `DRY_RUN=false` to revise affected posts"
|
||||
end
|
||||
puts "Task was ran in dry run mode. Set `DRY_RUN=false` to revise affected posts" if dry_run
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user