IntenseWebs/discourse
3
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2025-02-25 18:55:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

FIX: Bucket name is missing in S3 inventory data path

Browse Source
This commit is contained in:
Vinoth Kannan 2019-02-06 19:16:08 +05:30
parent 15857b900a
commit ff12c4b2d4
3 changed files with 51 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
lib/s3_inventory.rb
View File

@ -10,6 +10,7 @@ class S3Inventory
CSV_KEY_INDEX ||= 1 CSV_KEY_INDEX ||= 1
CSV_ETAG_INDEX ||= 2 CSV_ETAG_INDEX ||= 2
INVENTORY_PREFIX ||= "inventory" INVENTORY_PREFIX ||= "inventory"
INVENTORY_VERSION ||= "1"
def initialize(s3_helper, type) def initialize(s3_helper, type)
@s3_helper = s3_helper @s3_helper = s3_helper
@ -96,10 +97,10 @@ class S3Inventory
"Effect": "Allow", "Effect": "Allow",
"Principal": { "Service": "s3.amazonaws.com" }, "Principal": { "Service": "s3.amazonaws.com" },
"Action": ["s3:PutObject"], "Action": ["s3:PutObject"],
"Resource": ["arn:aws:s3:::#{inventory_root_path}/*"], "Resource": ["#{inventory_path_arn}/*"],
"Condition": { "Condition": {
"ArnLike": { "ArnLike": {
"aws:SourceArn": "arn:aws:s3:::#{bucket_name}" "aws:SourceArn": bucket_arn
}, },
"StringEquals": { "StringEquals": {
"s3:x-amz-acl": "bucket-owner-full-control" "s3:x-amz-acl": "bucket-owner-full-control"
@ -134,7 +135,7 @@ class S3Inventory
{ {
destination: { destination: {
s3_bucket_destination: { s3_bucket_destination: {
bucket: "arn:aws:s3:::#{bucket_name}", bucket: bucket_arn,
prefix: destination_prefix, prefix: destination_prefix,
format: "CSV" format: "CSV"
} }
@ -163,7 +164,7 @@ class S3Inventory
def unsorted_files def unsorted_files
objects = [] objects = []
@s3_helper.list(File.join(inventory_path, "data")).each do |obj| @s3_helper.list(inventory_data_path).each do |obj|
if obj.key.match?(/\.csv\.gz$/i) if obj.key.match?(/\.csv\.gz$/i)
objects << obj objects << obj
end end
@ -174,12 +175,22 @@ class S3Inventory
log("Failed to list inventory from S3", e) log("Failed to list inventory from S3", e)
end end
def inventory_path def inventory_data_path
File.join(inventory_root_path, inventory_id) File.join(inventory_path, bucket_name, inventory_id, "data")
end end
def inventory_root_path def inventory_path_arn
File.join(bucket_name, bucket_folder_path || "", INVENTORY_PREFIX) File.join(bucket_arn, inventory_path)
end
def inventory_path
path = File.join(INVENTORY_PREFIX, INVENTORY_VERSION)
path = File.join(bucket_folder_path, path) if bucket_folder_path.present?
path
end
def bucket_arn
"arn:aws:s3:::#{bucket_name}"
end end
def log(message, ex = nil) def log(message, ex = nil)

57
spec/components/s3_inventory_spec.rb
View File

@ -16,33 +16,38 @@ describe "S3Inventory" do
SiteSetting.s3_secret_access_key = "def" SiteSetting.s3_secret_access_key = "def"
SiteSetting.enable_s3_inventory = true SiteSetting.enable_s3_inventory = true
client.stub_responses(:list_objects, client.stub_responses(:list_objects, -> (context) {
contents: [ inventory_data_path = "#{S3Inventory::INVENTORY_PREFIX}/#{S3Inventory::INVENTORY_VERSION}/bucket/original/data"
{ expect(context.params[:prefix]).to eq(inventory_data_path)
etag: "\"70ee1738b6b21e2c8a43f3a5ab0eee71\"",
key: "example1.csv.gz", {
last_modified: Time.parse("2014-11-21T19:40:05.000Z"), contents: [
owner: { {
display_name: "myname", etag: "\"70ee1738b6b21e2c8a43f3a5ab0eee71\"",
id: "12345example25102679df27bb0ae12b3f85be6f290b936c4393484be31bebcc", key: "example1.csv.gz",
last_modified: Time.parse("2014-11-21T19:40:05.000Z"),
owner: {
display_name: "myname",
id: "12345example25102679df27bb0ae12b3f85be6f290b936c4393484be31bebcc",
},
size: 11,
storage_class: "STANDARD",
}, },
size: 11, {
storage_class: "STANDARD", etag: "\"9c8af9a76df052144598c115ef33e511\"",
}, key: "example2.csv.gz",
{ last_modified: Time.parse("2013-11-15T01:10:49.000Z"),
etag: "\"9c8af9a76df052144598c115ef33e511\"", owner: {
key: "example2.csv.gz", display_name: "myname",
last_modified: Time.parse("2013-11-15T01:10:49.000Z"), id: "12345example25102679df27bb0ae12b3f85be6f290b936c4393484be31bebcc",
owner: { },
display_name: "myname", size: 713193,
id: "12345example25102679df27bb0ae12b3f85be6f290b936c4393484be31bebcc", storage_class: "STANDARD",
}, }
size: 713193, ],
storage_class: "STANDARD", next_marker: "eyJNYXJrZXIiOiBudWxsLCAiYm90b190cnVuY2F0ZV9hbW91bnQiOiAyfQ=="
} }
], })
next_marker: "eyJNYXJrZXIiOiBudWxsLCAiYm90b190cnVuY2F0ZV9hbW91bnQiOiAyfQ=="
)
end end
it "should return the latest inventory file name" do it "should return the latest inventory file name" do

2
spec/jobs/update_s3_inventory_spec.rb
View File

@ -18,7 +18,7 @@ describe Jobs::UpdateS3Inventory do
id = "original" id = "original"
@client.expects(:put_bucket_policy).with( @client.expects(:put_bucket_policy).with(
bucket: "bucket", bucket: "bucket",
policy: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"InventoryAndAnalyticsPolicy\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"s3.amazonaws.com\"},\"Action\":[\"s3:PutObject\"],\"Resource\":[\"arn:aws:s3:::bucket/inventory/*\"],\"Condition\":{\"ArnLike\":{\"aws:SourceArn\":\"arn:aws:s3:::bucket\"},\"StringEquals\":{\"s3:x-amz-acl\":\"bucket-owner-full-control\"}}}]}" policy: "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"InventoryAndAnalyticsPolicy\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"s3.amazonaws.com\"},\"Action\":[\"s3:PutObject\"],\"Resource\":[\"arn:aws:s3:::bucket/#{S3Inventory::INVENTORY_PREFIX}/#{S3Inventory::INVENTORY_VERSION}/*\"],\"Condition\":{\"ArnLike\":{\"aws:SourceArn\":\"arn:aws:s3:::bucket\"},\"StringEquals\":{\"s3:x-amz-acl\":\"bucket-owner-full-control\"}}}]}"
) )
@client.expects(:put_bucket_inventory_configuration) @client.expects(:put_bucket_inventory_configuration)
@client.expects(:put_bucket_inventory_configuration).with( @client.expects(:put_bucket_inventory_configuration).with(