IntenseWebs/discourse
3
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

PERF: use url instead of file key in temporary inventory table.

Browse Source
This commit is contained in:
Vinoth Kannan
2019-06-13 22:03:58 +05:30
parent ed21128ee6
commit 35d6fff69e
2 changed files with 10 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
lib/s3_inventory.rb
View File

@@ -37,13 +37,14 @@ class S3Inventory
multisite_prefix = "uploads/#{RailsMultisite::ConnectionManagement.current_db}/" multisite_prefix = "uploads/#{RailsMultisite::ConnectionManagement.current_db}/"
ActiveRecord::Base.transaction do ActiveRecord::Base.transaction do
begin begin
connection.exec("CREATE TEMP TABLE #{table_name}(key text UNIQUE, etag text, PRIMARY KEY(etag, key))") connection.exec("CREATE TEMP TABLE #{table_name}(url text UNIQUE, etag text, PRIMARY KEY(etag, url))")
connection.copy_data("COPY #{table_name} FROM STDIN CSV") do connection.copy_data("COPY #{table_name} FROM STDIN CSV") do
files.each do |file| files.each do |file|
CSV.foreach(file[:filename][0...-3], headers: false) do |row| CSV.foreach(file[:filename][0...-3], headers: false) do |row|
key = row[CSV_KEY_INDEX] key = row[CSV_KEY_INDEX]
next if Rails.configuration.multisite && key.exclude?(multisite_prefix) next if Rails.configuration.multisite && key.exclude?(multisite_prefix)
connection.put_copy_data("#{key},#{row[CSV_ETAG_INDEX]}\n") url = File.join(Discourse.store.absolute_base_url, key)
connection.put_copy_data("#{url},#{row[CSV_ETAG_INDEX]}\n")
end end
end end
end end
@@ -53,7 +54,7 @@ class S3Inventory
SET etag = #{table_name}.etag SET etag = #{table_name}.etag
FROM #{table_name} FROM #{table_name}
WHERE #{model.table_name}.etag IS NULL WHERE #{model.table_name}.etag IS NULL
AND url ILIKE '%' || #{table_name}.key") AND #{model.table_name}.url = #{table_name}.url")
list_missing_post_uploads if type == "original" list_missing_post_uploads if type == "original"
@@ -85,11 +86,12 @@ class S3Inventory
next if sha1.blank? next if sha1.blank?
upload_id = nil upload_id = nil
result = connection.exec("SELECT * FROM #{table_name} WHERE key LIKE '%original/%/#{sha1}%'") result = connection.exec("SELECT * FROM #{table_name} WHERE url LIKE '%original/%/#{sha1}%'")
if result.count >= 1 if result.count >= 1
begin begin
key = result[0]["key"] url = result[0]["url"]
key = url.sub(/^#{Discourse.store.absolute_base_url}\//, "")
data = @s3_helper.object(key).data data = @s3_helper.object(key).data
filename = (data.content_disposition&.match(/filename=\"(.*)\"/) || [])[1] filename = (data.content_disposition&.match(/filename=\"(.*)\"/) || [])[1]
@@ -97,7 +99,7 @@ class S3Inventory
user_id: Discourse.system_user.id, user_id: Discourse.system_user.id,
original_filename: filename || File.basename(key), original_filename: filename || File.basename(key),
filesize: data.content_length, filesize: data.content_length,
url: File.join(Discourse.store.absolute_base_url, key), url: url,
sha1: sha1, sha1: sha1,
etag: result[0]["etag"] etag: result[0]["etag"]
) )

4
spec/components/s3_inventory_spec.rb
View File

@@ -82,8 +82,8 @@ describe "S3Inventory" do
it "should backfill etags to uploads table correctly" do it "should backfill etags to uploads table correctly" do
files = [ files = [
["//bucket.amazonaws.com/original/0184537a4f419224404d013414e913a4f56018f2.jpg", "defcaac0b4aca535c284e95f30d608d0"], ["#{Discourse.store.absolute_base_url}/original/0184537a4f419224404d013414e913a4f56018f2.jpg", "defcaac0b4aca535c284e95f30d608d0"],
["//bucket.amazonaws.com/original/0789fbf5490babc68326b9cec90eeb0d6590db05.png", "25c02eaceef4cb779fc17030d33f7f06"] ["#{Discourse.store.absolute_base_url}/original/0789fbf5490babc68326b9cec90eeb0d6590db05.png", "25c02eaceef4cb779fc17030d33f7f06"]
] ]
files.each { |file| Fabricate(:upload, url: file[0]) } files.each { |file| Fabricate(:upload, url: file[0]) }