DEV: Clean up S3 specs, stubs, and helpers

Extracted commonly used spec helpers into spec/support/uploads_helpers.rb, removed unused stubs and let definitions. Makes it easier to write new S3-related specs without copy and pasting setup steps from other specs.

spec/jobs/correct_missing_dualstack_urls_spec.rb

@@ -3,33 +3,30 @@
 require 'rails_helper'
 
 describe Jobs::CorrectMissingDualstackUrls do
-
   it 'corrects the urls' do
-
-    SiteSetting.s3_upload_bucket = "s3-upload-bucket"
-    SiteSetting.s3_access_key_id = "s3-access-key-id"
-    SiteSetting.s3_secret_access_key = "s3-secret-access-key"
-    SiteSetting.enable_s3_uploads = true
+    setup_s3
+    SiteSetting.s3_region = "us-east-1"
+    SiteSetting.s3_upload_bucket = "custom-bucket"
 
     # we will only correct for our base_url, random urls will be left alone
-    expect(Discourse.store.absolute_base_url).to eq('//s3-upload-bucket.s3.dualstack.us-east-1.amazonaws.com')
+    expect(Discourse.store.absolute_base_url).to eq('//custom-bucket.s3.dualstack.us-east-1.amazonaws.com')
 
     current_upload = Upload.create!(
-      url: '//s3-upload-bucket.s3-us-east-1.amazonaws.com/somewhere/a.png',
+      url: '//custom-bucket.s3-us-east-1.amazonaws.com/somewhere/a.png',
       original_filename: 'a.png',
       filesize: 100,
       user_id: -1,
     )
 
     bad_upload = Upload.create!(
-      url: '//s3-upload-bucket.s3-us-west-1.amazonaws.com/somewhere/a.png',
+      url: '//custom-bucket.s3-us-west-1.amazonaws.com/somewhere/a.png',
       original_filename: 'a.png',
       filesize: 100,
       user_id: -1,
     )
 
     current_optimized = OptimizedImage.create!(
-      url: '//s3-upload-bucket.s3-us-east-1.amazonaws.com/somewhere/a.png',
+      url: '//custom-bucket.s3-us-east-1.amazonaws.com/somewhere/a.png',
       filesize: 100,
       upload_id: current_upload.id,
       width: 100,
@@ -39,7 +36,7 @@ describe Jobs::CorrectMissingDualstackUrls do
     )
 
     bad_optimized = OptimizedImage.create!(
-      url: '//s3-upload-bucket.s3-us-west-1.amazonaws.com/somewhere/a.png',
+      url: '//custom-bucket.s3-us-west-1.amazonaws.com/somewhere/a.png',
       filesize: 100,
       upload_id: current_upload.id,
       width: 110,
@@ -51,15 +48,15 @@ describe Jobs::CorrectMissingDualstackUrls do
     Jobs::CorrectMissingDualstackUrls.new.execute_onceoff(nil)
 
     bad_upload.reload
-    expect(bad_upload.url).to eq('//s3-upload-bucket.s3-us-west-1.amazonaws.com/somewhere/a.png')
+    expect(bad_upload.url).to eq('//custom-bucket.s3-us-west-1.amazonaws.com/somewhere/a.png')
 
     current_upload.reload
-    expect(current_upload.url).to eq('//s3-upload-bucket.s3.dualstack.us-east-1.amazonaws.com/somewhere/a.png')
+    expect(current_upload.url).to eq('//custom-bucket.s3.dualstack.us-east-1.amazonaws.com/somewhere/a.png')
 
     bad_optimized.reload
-    expect(bad_optimized.url).to eq('//s3-upload-bucket.s3-us-west-1.amazonaws.com/somewhere/a.png')
+    expect(bad_optimized.url).to eq('//custom-bucket.s3-us-west-1.amazonaws.com/somewhere/a.png')
 
     current_optimized.reload
-    expect(current_optimized.url).to eq('//s3-upload-bucket.s3.dualstack.us-east-1.amazonaws.com/somewhere/a.png')
+    expect(current_optimized.url).to eq('//custom-bucket.s3.dualstack.us-east-1.amazonaws.com/somewhere/a.png')
   end
 end

spec/jobs/ensure_s3_uploads_existence_spec.rb

@@ -5,9 +5,7 @@ require 'rails_helper'
 RSpec.describe Jobs::EnsureS3UploadsExistence do
   context "S3 inventory enabled" do
     before do
-      SiteSetting.enable_s3_uploads = true
-      SiteSetting.s3_access_key_id = "abc"
-      SiteSetting.s3_secret_access_key = "def"
+      setup_s3
       SiteSetting.enable_s3_inventory = true
     end
 

spec/jobs/pull_hotlinked_images_spec.rb

@@ -186,7 +186,9 @@ describe Jobs::PullHotlinkedImages do
 
     context "when secure media enabled for an upload that has already been downloaded and exists" do
       it "doesnt redownload the secure upload" do
-        enable_secure_media
+        setup_s3
+        SiteSetting.secure_media = true
+
         upload = Fabricate(:secure_upload_s3, secure: true)
         stub_s3(upload)
         url = Upload.secure_media_url_from_upload_url(upload.url)
@@ -199,7 +201,9 @@ describe Jobs::PullHotlinkedImages do
 
       context "when the upload original_sha1 is missing" do
         it "redownloads the upload" do
-          enable_secure_media
+          setup_s3
+          SiteSetting.secure_media = true
+
           upload = Fabricate(:upload_s3, secure: true)
           stub_s3(upload)
           Upload.stubs(:signed_url_from_secure_media_url).returns(upload.url)
@@ -218,7 +222,9 @@ describe Jobs::PullHotlinkedImages do
 
       context "when the upload access_control_post is different to the current post" do
         it "redownloads the upload" do
-          enable_secure_media
+          setup_s3
+          SiteSetting.secure_media = true
+
           upload = Fabricate(:secure_upload_s3, secure: true)
           stub_s3(upload)
           Upload.stubs(:signed_url_from_secure_media_url).returns(upload.url)
@@ -395,7 +401,9 @@ describe Jobs::PullHotlinkedImages do
 
       context "when secure media enabled" do
         it 'should return false for secure-media-upload url' do
-          enable_secure_media
+          setup_s3
+          SiteSetting.secure_media = true
+
           upload = Fabricate(:upload_s3, secure: true)
           stub_s3(upload)
           url = Upload.secure_media_url_from_upload_url(upload.url)
@@ -415,12 +423,9 @@ describe Jobs::PullHotlinkedImages do
     end
 
     it "returns false for emoji when app and S3 CDNs configured" do
-      set_cdn_url "https://mydomain.cdn/test"
-      SiteSetting.s3_upload_bucket = "some-bucket-on-s3"
-      SiteSetting.s3_access_key_id = "s3-access-key-id"
-      SiteSetting.s3_secret_access_key = "s3-secret-access-key"
+      setup_s3
       SiteSetting.s3_cdn_url = "https://s3.cdn.com"
-      SiteSetting.enable_s3_uploads = true
+      set_cdn_url "https://mydomain.cdn/test"
 
       src = UrlHelper.cook_url(Emoji.url_for("testemoji.png"))
       expect(subject.should_download_image?(src)).to eq(false)
@@ -503,22 +508,8 @@ describe Jobs::PullHotlinkedImages do
     end
   end
 
-  def enable_secure_media
-    SiteSetting.enable_s3_uploads = true
-    SiteSetting.s3_upload_bucket = "s3-upload-bucket"
-    SiteSetting.s3_access_key_id = "some key"
-    SiteSetting.s3_secret_access_key = "some secrets3_region key"
-    SiteSetting.secure_media = true
-  end
-
   def stub_s3(upload)
-    stub_request(:head, "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/")
-
-    stub_request(
-      :put,
-      "https://#{SiteSetting.s3_upload_bucket}.s3.amazonaws.com/original/1X/#{upload.sha1}.#{upload.extension}?acl"
-    )
+    stub_upload(upload)
     stub_request(:get, "https:" + upload.url).to_return(status: 200, body: file_from_fixtures("smallest.png"))
-    # stub_request(:get, /#{SiteSetting.s3_upload_bucket}\.s3\.amazonaws\.com/)
   end
 end

spec/jobs/update_s3_inventory_spec.rb

@@ -5,9 +5,8 @@ require "file_store/s3_store"
 
 describe Jobs::UpdateS3Inventory do
   before do
-    SiteSetting.enable_s3_uploads = true
-    SiteSetting.s3_access_key_id = "abc"
-    SiteSetting.s3_secret_access_key = "def"
+    setup_s3
+    SiteSetting.s3_upload_bucket = "special-bucket"
     SiteSetting.enable_s3_inventory = true
 
     store = FileStore::S3Store.new
@@ -21,17 +20,17 @@ describe Jobs::UpdateS3Inventory do
     path = File.join(S3Inventory::INVENTORY_PREFIX, S3Inventory::INVENTORY_VERSION)
 
     @client.expects(:put_bucket_policy).with(
-      bucket: "bucket",
-      policy: %Q|{"Version":"2012-10-17","Statement":[{"Sid":"InventoryAndAnalyticsPolicy","Effect":"Allow","Principal":{"Service":"s3.amazonaws.com"},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::bucket/#{path}/*"],"Condition":{"ArnLike":{"aws:SourceArn":"arn:aws:s3:::bucket"},"StringEquals":{"s3:x-amz-acl":"bucket-owner-full-control"}}}]}|
+      bucket: "special-bucket",
+      policy: %Q|{"Version":"2012-10-17","Statement":[{"Sid":"InventoryAndAnalyticsPolicy","Effect":"Allow","Principal":{"Service":"s3.amazonaws.com"},"Action":["s3:PutObject"],"Resource":["arn:aws:s3:::special-bucket/#{path}/*"],"Condition":{"ArnLike":{"aws:SourceArn":"arn:aws:s3:::special-bucket"},"StringEquals":{"s3:x-amz-acl":"bucket-owner-full-control"}}}]}|
     )
     @client.expects(:put_bucket_inventory_configuration)
     @client.expects(:put_bucket_inventory_configuration).with(
-      bucket: "bucket",
+      bucket: "special-bucket",
       id: id,
       inventory_configuration: {
         destination: {
           s3_bucket_destination: {
-            bucket: "arn:aws:s3:::bucket",
+            bucket: "arn:aws:s3:::special-bucket",
             prefix: path,
             format: "CSV"
           }

spec/jobs/vacate_legacy_prefix_backups_spec.rb

@@ -18,9 +18,7 @@ describe Jobs::VacateLegacyPrefixBackups, type: :multisite do
   end
 
   before do
-    SiteSetting.enable_s3_uploads = true
-    SiteSetting.s3_access_key_id = "abc"
-    SiteSetting.s3_secret_access_key = "def"
+    setup_s3
     SiteSetting.s3_backup_bucket = bucket_name
     SiteSetting.backup_location = BackupLocationSiteSetting::S3
   end