IntenseWebs/discourse
3
0
Fork 0
mirror of https://github.com/discourse/discourse.git synced 2024-11-22 08:57:10 -06:00
Code Issues Packages Projects Releases Wiki Activity

FEATURE: Use S3 dualstack endpoints

Browse Source 
Allows S3 without a CDN to serve images from dualstack domains that also support ipv6
This commit is contained in:
Raul Tambre 2018-08-27 04:22:46 +03:00 committed by Sam
parent 020eba4623
commit 2271918be2
5 changed files with 26 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/models/s3_region_site_setting.rb
View File

@ -18,6 +18,7 @@ class S3RegionSiteSetting < EnumSiteSetting
'ap-southeast-1',
'ap-southeast-2',
'cn-north-1',
'cn-northwest-1',
'eu-central-1',
'eu-west-1',
'eu-west-2',

8
app/models/site_setting.rb
View File

@ -151,12 +151,10 @@ class SiteSetting < ActiveRecord::Base
# cf. http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
if SiteSetting.s3_endpoint == "https://s3.amazonaws.com"
if SiteSetting.Upload.s3_region == "us-east-1"
"//#{bucket}.s3.amazonaws.com"
elsif SiteSetting.Upload.s3_region == 'cn-north-1'
"//#{bucket}.s3.cn-north-1.amazonaws.com.cn"
if SiteSetting.Upload.s3_region == 'cn-north-1' || SiteSetting.Upload.s3_region == 'cn-northwest-1'
"//#{bucket}.s3.#{SiteSetting.Upload.s3_region}.amazonaws.com.cn"
else
"//#{bucket}.s3-#{SiteSetting.Upload.s3_region}.amazonaws.com"
"//#{bucket}.s3.dualstack.#{SiteSetting.Upload.s3_region}.amazonaws.com"
end
elsif SiteSetting.s3_force_path_style
"//#{url_basename}/#{bucket}"

29
spec/components/file_store/s3_store_spec.rb
View File

@ -48,7 +48,7 @@ describe FileStore::S3Store do
s3_object.expects(:upload_file)
expect(store.store_upload(uploaded_file, upload)).to eq(
"//s3-upload-bucket.s3-us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png"
"//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png"
)
end
@ -66,7 +66,7 @@ describe FileStore::S3Store do
s3_object.expects(:upload_file)
expect(store.store_upload(uploaded_file, upload)).to eq(
"//s3-upload-bucket.s3-us-west-1.amazonaws.com/discourse-uploads/original/1X/#{upload.sha1}.png"
"//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/discourse-uploads/original/1X/#{upload.sha1}.png"
)
end
end
@ -83,7 +83,7 @@ describe FileStore::S3Store do
s3_object.expects(:upload_file)
expect(store.store_optimized_image(optimized_image_file, optimized_image)).to eq(
"//s3-upload-bucket.s3-us-west-1.amazonaws.com/#{path}"
"//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/#{path}"
)
end
@ -102,7 +102,7 @@ describe FileStore::S3Store do
s3_object.expects(:upload_file)
expect(store.store_optimized_image(optimized_image_file, optimized_image)).to eq(
"//s3-upload-bucket.s3-us-west-1.amazonaws.com/#{path}"
"//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/#{path}"
)
end
end
@ -117,7 +117,7 @@ describe FileStore::S3Store do
s3_helper.expects(:s3_bucket).returns(s3_bucket)
upload.update!(
url: "//s3-upload-bucket.s3-us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png"
url: "//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png"
)
source = Discourse.store.get_path_for_upload(upload)
@ -143,7 +143,7 @@ describe FileStore::S3Store do
it "removes the file from s3 with the right paths" do
store.expects(:get_depth_for).with(upload.id).returns(0)
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
upload.update_attributes!(url: "//s3-upload-bucket.s3-us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png")
upload.update_attributes!(url: "//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/original/1X/#{upload.sha1}.png")
s3_object = stub
s3_bucket.expects(:object).with("tombstone/original/1X/#{upload.sha1}.png").returns(s3_object)
@ -162,7 +162,7 @@ describe FileStore::S3Store do
it "removes the file from s3 with the right paths" do
store.expects(:get_depth_for).with(upload.id).returns(0)
s3_helper.expects(:s3_bucket).returns(s3_bucket).at_least_once
upload.update_attributes!(url: "//s3-upload-bucket.s3-us-west-1.amazonaws.com/discourse-uploads/original/1X/#{upload.sha1}.png")
upload.update_attributes!(url: "//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/discourse-uploads/original/1X/#{upload.sha1}.png")
s3_object = stub
s3_bucket.expects(:object).with("discourse-uploads/tombstone/original/1X/#{upload.sha1}.png").returns(s3_object)
@ -178,7 +178,7 @@ describe FileStore::S3Store do
describe "#remove_optimized_image" do
let(:optimized_image) do
Fabricate(:optimized_image,
url: "//s3-upload-bucket.s3-us-west-1.amazonaws.com/optimized/1X/#{upload.sha1}_1_100x200.png",
url: "//s3-upload-bucket.s3.dualstack.us-west-1.amazonaws.com/optimized/1X/#{upload.sha1}_1_100x200.png",
upload: upload
)
end
@ -220,10 +220,11 @@ describe FileStore::S3Store do
describe ".has_been_uploaded?" do
it "identifies S3 uploads" do
expect(store.has_been_uploaded?("//s3-upload-bucket.s3.amazonaws.com/1337.png")).to eq(true)
expect(store.has_been_uploaded?("//s3-upload-bucket.s3.dualstack.us-east-1.amazonaws.com/1337.png")).to eq(true)
end
it "does not match other s3 urls" do
expect(store.has_been_uploaded?("//s3-upload-bucket.s3.amazonaws.com/1337.png")).to eq(false)
expect(store.has_been_uploaded?("//s3-upload-bucket.s3-us-east-1.amazonaws.com/1337.png")).to eq(false)
expect(store.has_been_uploaded?("//s3.amazonaws.com/s3-upload-bucket/1337.png")).to eq(false)
expect(store.has_been_uploaded?("//s4_upload_bucket.s3.amazonaws.com/1337.png")).to eq(false)
@ -233,19 +234,21 @@ describe FileStore::S3Store do
describe ".absolute_base_url" do
it "returns a lowercase schemaless absolute url" do
expect(store.absolute_base_url).to eq("//s3-upload-bucket.s3.amazonaws.com")
expect(store.absolute_base_url).to eq("//s3-upload-bucket.s3.dualstack.us-east-1.amazonaws.com")
end
it "uses the proper endpoint" do
SiteSetting.s3_region = "us-east-1"
expect(FileStore::S3Store.new(s3_helper).absolute_base_url).to eq("//s3-upload-bucket.s3.amazonaws.com")
expect(FileStore::S3Store.new(s3_helper).absolute_base_url).to eq("//s3-upload-bucket.s3.dualstack.us-east-1.amazonaws.com")
SiteSetting.s3_region = "us-west-2"
expect(FileStore::S3Store.new(s3_helper).absolute_base_url).to eq("//s3-upload-bucket.s3-us-west-2.amazonaws.com")
expect(FileStore::S3Store.new(s3_helper).absolute_base_url).to eq("//s3-upload-bucket.s3.dualstack.us-west-2.amazonaws.com")
SiteSetting.s3_region = "cn-north-1"
expect(FileStore::S3Store.new(s3_helper).absolute_base_url).to eq("//s3-upload-bucket.s3.cn-north-1.amazonaws.com.cn")
SiteSetting.s3_region = "cn-northwest-1"
expect(FileStore::S3Store.new(s3_helper).absolute_base_url).to eq("//s3-upload-bucket.s3.cn-northwest-1.amazonaws.com.cn")
end
end
@ -294,7 +297,7 @@ describe FileStore::S3Store do
global_setting :relative_url_root, '/community'
Discourse.stubs(:base_uri).returns("/community")
url = "//s3-upload-bucket.s3.amazonaws.com/livechat/original/gif.png"
url = "//s3-upload-bucket.s3.dualstack.us-east-1.amazonaws.com/livechat/original/gif.png"
expect(store.cdn_url(url)).to eq("https://rainbow.com/original/gif.png")
end

8
spec/requests/session_controller_spec.rb
View File

@ -620,10 +620,10 @@ RSpec.describe SessionController do
SiteSetting.s3_upload_bucket = "test"
SiteSetting.s3_cdn_url = "http://cdn.com"
stub_request(:any, /test.s3.amazonaws.com/).to_return(status: 200, body: "", headers: {})
stub_request(:any, /test.s3.dualstack.us-east-1.amazonaws.com/).to_return(status: 200, body: "", headers: {})
@user.create_user_avatar!
upload = Fabricate(:upload, url: "//test.s3.amazonaws.com/something")
upload = Fabricate(:upload, url: "//test.s3.dualstack.us-east-1.amazonaws.com/something")
Fabricate(:optimized_image,
sha1: SecureRandom.hex << "A" * 8,
@ -635,8 +635,8 @@ RSpec.describe SessionController do
@user.update_columns(uploaded_avatar_id: upload.id)
@user.user_profile.update_columns(
profile_background: "//test.s3.amazonaws.com/something",
card_background: "//test.s3.amazonaws.com/something"
profile_background: "//test.s3.dualstack.us-east-1.amazonaws.com/something",
card_background: "//test.s3.dualstack.us-east-1.amazonaws.com/something"
)
@user.reload

4
spec/requests/user_avatars_controller_spec.rb
View File

@ -77,14 +77,14 @@ describe UserAvatarsController do
GlobalSetting.expects(:cdn_url).returns("http://awesome.com/boom")
upload = Fabricate(:upload, url: "//test.s3.amazonaws.com/something")
upload = Fabricate(:upload, url: "//test.s3.dualstack.us-east-1.amazonaws.com/something")
optimized_image = Fabricate(:optimized_image,
sha1: SecureRandom.hex << "A" * 8,
upload: upload,
width: 98,
height: 98,
url: "//test.s3.amazonaws.com/something/else"
url: "//test.s3.dualstack.us-east-1.amazonaws.com/something/else"
)
user = Fabricate(:user, uploaded_avatar_id: upload.id)