FEATURE: Create upload_references table (#16146)

This table holds associations between uploads and other models. This can be used to prevent removing uploads that are still in use.

* DEV: Create upload_references
* DEV: Use UploadReference instead of PostUpload
* DEV: Use UploadReference for SiteSetting
* DEV: Use UploadReference for Badge
* DEV: Use UploadReference for Category
* DEV: Use UploadReference for CustomEmoji
* DEV: Use UploadReference for Group
* DEV: Use UploadReference for ThemeField
* DEV: Use UploadReference for ThemeSetting
* DEV: Use UploadReference for User
* DEV: Use UploadReference for UserAvatar
* DEV: Use UploadReference for UserExport
* DEV: Use UploadReference for UserProfile
* DEV: Add method to extract uploads from raw text
* DEV: Use UploadReference for Draft
* DEV: Use UploadReference for ReviewableQueuedPost
* DEV: Use UploadReference for UserProfile's bio_raw
* DEV: Do not copy user uploads to upload references
* DEV: Copy post uploads again after deploy
* DEV: Use created_at and updated_at from uploads table
* FIX: Check if upload site setting is empty
* DEV: Copy user uploads to upload references
* DEV: Make upload extraction less strict

spec/models/post_spec.rb

@@ -1485,17 +1485,17 @@ describe Post do
       post.link_post_uploads
 
       post.trash!
-      expect(PostUpload.count).to eq(6)
+      expect(UploadReference.count).to eq(6)
 
       post.destroy!
-      expect(PostUpload.count).to eq(0)
+      expect(UploadReference.count).to eq(0)
     end
 
     context "#link_post_uploads" do
       it "finds all the uploads in the post" do
         post.link_post_uploads
 
-        expect(PostUpload.where(post: post).pluck(:upload_id)).to contain_exactly(
+        expect(UploadReference.where(target: post).pluck(:upload_id)).to contain_exactly(
           video_upload.id,
           image_upload.id,
           audio_upload.id,
@@ -1508,13 +1508,11 @@ describe Post do
       it "cleans the reverse index up for the current post" do
         post.link_post_uploads
 
-        post_uploads_ids = post.post_uploads.pluck(:id)
+        post_uploads_ids = post.upload_references.pluck(:id)
 
         post.link_post_uploads
 
-        expect(post.reload.post_uploads.pluck(:id)).to_not contain_exactly(
-          post_uploads_ids
-        )
+        expect(post.reload.upload_references.pluck(:id)).to_not contain_exactly(post_uploads_ids)
       end
 
       context "when secure media is enabled" do
@@ -1578,7 +1576,7 @@ describe Post do
         post.link_post_uploads
         post.update_uploads_secure_status(source: "test")
 
-        expect(PostUpload.where(post: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
+        expect(UploadReference.where(target: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
           [attachment_upload.id, true],
           [image_upload.id, true]
         )
@@ -1590,7 +1588,7 @@ describe Post do
         post.link_post_uploads
         post.update_uploads_secure_status(source: "test")
 
-        expect(PostUpload.where(post: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
+        expect(UploadReference.where(target: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
           [attachment_upload.id, false],
           [image_upload.id, false]
         )
@@ -1603,7 +1601,7 @@ describe Post do
         post.link_post_uploads
         post.update_uploads_secure_status(source: "test")
 
-        expect(PostUpload.where(post: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
+        expect(UploadReference.where(target: post).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
           [attachment_upload.id, true],
           [image_upload.id, true]
         )
@@ -1618,7 +1616,7 @@ describe Post do
         pm.link_post_uploads
         pm.update_uploads_secure_status(source: "test")
 
-        expect(PostUpload.where(post: pm).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
+        expect(UploadReference.where(target: pm).joins(:upload).pluck(:upload_id, :secure)).to contain_exactly(
           [attachment_upload.id, false],
           [image_upload.id, false]
         )

spec/models/post_upload_spec.rb

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-describe PostUpload do
-
-  it { is_expected.to belong_to :post }
-  it { is_expected.to belong_to :upload }
-
-end

spec/models/upload_reference_spec.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+describe UploadReference do
+  context 'badge uploads' do
+    fab!(:upload) { Fabricate(:upload) }
+
+    it 'creates upload references' do
+      badge = nil
+      expect { badge = Fabricate(:badge, image_upload_id: upload.id) }
+        .to change { UploadReference.count }.by(1)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.upload).to eq(upload)
+      expect(upload_reference.target).to eq(badge)
+
+      expect { badge.destroy! }
+        .to change { UploadReference.count }.by(-1)
+    end
+  end
+
+  context 'category uploads' do
+    fab!(:upload1) { Fabricate(:upload) }
+    fab!(:upload2) { Fabricate(:upload) }
+
+    it 'creates upload references' do
+      category = nil
+      expect { category = Fabricate(:category, uploaded_logo_id: upload1.id, uploaded_background_id: upload2.id) }
+        .to change { UploadReference.count }.by(2)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.target).to eq(category)
+
+      expect { category.destroy! }
+        .to change { UploadReference.count }.by(-2)
+    end
+  end
+
+  context 'custom emoji uploads' do
+    fab!(:upload) { Fabricate(:upload) }
+
+    it 'creates upload references' do
+      custom_emoji = nil
+      expect { custom_emoji = CustomEmoji.create!(name: 'emoji', upload_id: upload.id) }
+        .to change { UploadReference.count }.by(1)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.target).to eq(custom_emoji)
+
+      expect { custom_emoji.destroy! }
+        .to change { UploadReference.count }.by(-1)
+    end
+  end
+
+  context 'group uploads' do
+    fab!(:upload) { Fabricate(:upload) }
+
+    it 'creates upload references' do
+      group = nil
+      expect { group = Fabricate(:group, flair_upload_id: upload.id) }
+        .to change { UploadReference.count }.by(1)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.upload).to eq(upload)
+      expect(upload_reference.target).to eq(group)
+
+      expect { group.destroy! }
+        .to change { UploadReference.count }.by(-1)
+    end
+  end
+
+  context 'post uploads' do
+    fab!(:upload) { Fabricate(:upload) }
+    fab!(:post) { Fabricate(:post, raw: "[](#{upload.short_url})") }
+
+    it 'creates upload references' do
+      expect { post.link_post_uploads }
+        .to change { UploadReference.count }.by(1)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.upload).to eq(upload)
+      expect(upload_reference.target).to eq(post)
+
+      expect { post.destroy! }
+        .to change { UploadReference.count }.by(-1)
+    end
+  end
+
+  context 'site setting uploads' do
+    let(:provider) { SiteSettings::DbProvider.new(SiteSetting) }
+    fab!(:upload) { Fabricate(:upload) }
+    fab!(:upload2) { Fabricate(:upload) }
+
+    it 'creates upload references for uploads' do
+      expect { provider.save('logo', upload.id, SiteSettings::TypeSupervisor.types[:upload]) }
+        .to change { UploadReference.count }.by(1)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.upload).to eq(upload)
+      expect(upload_reference.target).to eq(SiteSetting.find_by(name: 'logo'))
+
+      expect { provider.destroy('logo') }
+        .to change { UploadReference.count }.by(-1)
+    end
+
+    it 'creates upload references for uploaded_image_lists' do
+      expect { provider.save('selectable_avatars', "#{upload.id}|#{upload2.id}", SiteSettings::TypeSupervisor.types[:uploaded_image_list]) }
+        .to change { UploadReference.count }.by(2)
+
+      upload_references = UploadReference.all.where(target: SiteSetting.find_by(name: 'selectable_avatars'))
+      expect(upload_references.pluck(:upload_id)).to contain_exactly(upload.id, upload2.id)
+
+      expect { provider.destroy('selectable_avatars') }
+        .to change { UploadReference.count }.by(-2)
+    end
+  end
+
+  context 'theme field uploads' do
+    fab!(:upload) { Fabricate(:upload) }
+
+    it 'creates upload refererences' do
+      theme_field = nil
+      expect do
+        theme_field = ThemeField.create!(
+          theme_id: Fabricate(:theme).id,
+          target_id: 0,
+          name: 'field',
+          value: '',
+          upload: upload,
+          type_id: ThemeField.types[:theme_upload_var],
+        )
+      end.to change { UploadReference.count }.by(1)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.upload).to eq(upload)
+      expect(upload_reference.target).to eq(theme_field)
+
+      expect { theme_field.destroy! }
+        .to change { UploadReference.count }.by(-1)
+    end
+  end
+
+  context 'theme setting uploads' do
+    fab!(:upload) { Fabricate(:upload) }
+
+    it 'creates upload refererences' do
+      theme_setting = nil
+      expect do
+        theme_setting = ThemeSetting.create!(
+          name: 'field',
+          data_type: ThemeSetting.types[:upload],
+          value: upload.id,
+          theme_id: Fabricate(:theme).id,
+        )
+      end.to change { UploadReference.count }.by(1)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.upload).to eq(upload)
+      expect(upload_reference.target).to eq(theme_setting)
+
+      expect { theme_setting.destroy! }
+        .to change { UploadReference.count }.by(-1)
+    end
+  end
+
+  context 'user uploads' do
+    fab!(:upload) { Fabricate(:upload) }
+
+    it 'creates upload references' do
+      user = nil
+      expect { user = Fabricate(:user, uploaded_avatar_id: upload.id) }
+        .to change { UploadReference.count }.by(1)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.upload).to eq(upload)
+      expect(upload_reference.target).to eq(user)
+
+      expect { user.destroy! }
+        .to change { UploadReference.count }.by(-1)
+    end
+  end
+
+  context 'user avatar uploads' do
+    fab!(:upload1) { Fabricate(:upload) }
+    fab!(:upload2) { Fabricate(:upload) }
+
+    it 'creates upload references' do
+      user_avatar = nil
+      expect { user_avatar = Fabricate(:user_avatar, custom_upload_id: upload1.id, gravatar_upload_id: upload2.id) }
+        .to change { UploadReference.count }.by(2)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.target).to eq(user_avatar)
+
+      expect { user_avatar.destroy! }
+        .to change { UploadReference.count }.by(-2)
+    end
+  end
+
+  context 'user export uploads' do
+    fab!(:upload) { Fabricate(:upload) }
+
+    it 'creates upload references' do
+      user_export = nil
+      expect do
+        user_export = UserExport.create!(
+          file_name: 'export',
+          user: Fabricate(:user),
+          upload: upload,
+          topic: Fabricate(:topic),
+        )
+      end.to change { UploadReference.count }.by(1)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.upload).to eq(upload)
+      expect(upload_reference.target).to eq(user_export)
+
+      expect { user_export.destroy! }
+        .to change { UploadReference.count }.by(-1)
+    end
+  end
+
+  context 'user profile uploads' do
+    fab!(:user) { Fabricate(:user) }
+    fab!(:upload1) { Fabricate(:upload) }
+    fab!(:upload2) { Fabricate(:upload) }
+
+    it 'creates upload references' do
+      user_profile = user.user_profile
+      expect { user_profile.update!(profile_background_upload_id: upload1.id, card_background_upload_id: upload2.id) }
+        .to change { UploadReference.count }.by(2)
+
+      upload_reference = UploadReference.last
+      expect(upload_reference.target).to eq(user_profile)
+
+      expect { user_profile.destroy! }
+        .to change { UploadReference.count }.by(-2)
+    end
+  end
+end

spec/models/upload_spec.rb

@@ -517,6 +517,25 @@ describe Upload do
     end
   end
 
+  describe '.extract_upload_ids' do
+    let(:upload) { Fabricate(:upload) }
+
+    it 'works with short URLs' do
+      ids = Upload.extract_upload_ids("This URL #{upload.short_url} is an upload")
+      expect(ids).to contain_exactly(upload.id)
+    end
+
+    it 'works with SHA1s' do
+      ids = Upload.extract_upload_ids("This URL /#{upload.sha1} is an upload")
+      expect(ids).to contain_exactly(upload.id)
+    end
+
+    it 'works with Base62 hashes' do
+      ids = Upload.extract_upload_ids("This URL /#{Upload.base62_sha1(upload.sha1)} is an upload")
+      expect(ids).to contain_exactly(upload.id)
+    end
+  end
+
   def enable_secure_media
     setup_s3
     SiteSetting.secure_media = true