FEATURE: Create upload_references table (#16146)

This table holds associations between uploads and other models. This can be used to prevent removing uploads that are still in use.

* DEV: Create upload_references
* DEV: Use UploadReference instead of PostUpload
* DEV: Use UploadReference for SiteSetting
* DEV: Use UploadReference for Badge
* DEV: Use UploadReference for Category
* DEV: Use UploadReference for CustomEmoji
* DEV: Use UploadReference for Group
* DEV: Use UploadReference for ThemeField
* DEV: Use UploadReference for ThemeSetting
* DEV: Use UploadReference for User
* DEV: Use UploadReference for UserAvatar
* DEV: Use UploadReference for UserExport
* DEV: Use UploadReference for UserProfile
* DEV: Add method to extract uploads from raw text
* DEV: Use UploadReference for Draft
* DEV: Use UploadReference for ReviewableQueuedPost
* DEV: Use UploadReference for UserProfile's bio_raw
* DEV: Do not copy user uploads to upload references
* DEV: Copy post uploads again after deploy
* DEV: Use created_at and updated_at from uploads table
* FIX: Check if upload site setting is empty
* DEV: Copy user uploads to upload references
* DEV: Make upload extraction less strict

db/migrate/20220308201942_create_upload_references.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class CreateUploadReferences < ActiveRecord::Migration[6.1]
+  def change
+    create_table :upload_references do |t|
+      t.references :upload, null: false
+      t.references :target, polymorphic: true, null: false
+      t.timestamps
+    end
+
+    add_index :upload_references, [:upload_id, :target_type, :target_id], unique: true, name: 'index_upload_references_on_upload_and_target'
+  end
+end

db/migrate/20220309132719_copy_post_uploads_to_upload_references.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class CopyPostUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT post_uploads.upload_id, 'Post', post_uploads.post_id, uploads.created_at, uploads.updated_at
+      FROM post_uploads
+      JOIN uploads ON uploads.id = post_uploads.upload_id
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220330160747_copy_site_settings_uploads_to_upload_references.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class CopySiteSettingsUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      WITH site_settings_uploads AS (
+        SELECT id, unnest(string_to_array(value, '|'))::integer upload_id
+        FROM site_settings
+        WHERE data_type = 17
+        UNION
+        SELECT id, value::integer
+        FROM site_settings
+        WHERE data_type = 18 AND value != ''
+      )
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT site_settings_uploads.upload_id, 'SiteSetting', site_settings_uploads.id, uploads.created_at, uploads.updated_at
+      FROM site_settings_uploads
+      JOIN uploads ON uploads.id = site_settings_uploads.upload_id
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220330160751_copy_badges_uploads_to_upload_references.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CopyBadgesUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT badges.image_upload_id, 'Badge', badges.id, uploads.created_at, uploads.updated_at
+      FROM badges
+      JOIN uploads ON uploads.id = badges.image_upload_id
+      WHERE badges.image_upload_id IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220330160754_copy_groups_uploads_to_upload_references.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CopyGroupsUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT groups.flair_upload_id, 'Group', groups.id, uploads.created_at, uploads.updated_at
+      FROM groups
+      JOIN uploads ON uploads.id = groups.flair_upload_id
+      WHERE groups.flair_upload_id IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220330160757_copy_user_exports_uploads_to_upload_references.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class CopyUserExportsUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT user_exports.upload_id, 'UserExport', user_exports.id, uploads.created_at, uploads.updated_at
+      FROM user_exports
+      JOIN uploads ON uploads.id = user_exports.upload_id
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220330164740_copy_theme_fields_uploads_to_upload_references.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CopyThemeFieldsUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT theme_fields.upload_id, 'ThemeField', theme_fields.id, uploads.created_at, uploads.updated_at
+      FROM theme_fields
+      JOIN uploads ON uploads.id = theme_fields.upload_id
+      WHERE type_id = 2
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220404195635_copy_categories_uploads_to_upload_references.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class CopyCategoriesUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT categories.uploaded_logo_id, 'Category', categories.id, uploads.created_at, uploads.updated_at
+      FROM categories
+      JOIN uploads ON uploads.id = categories.uploaded_logo_id
+      WHERE categories.uploaded_logo_id IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT categories.uploaded_background_id, 'Category', categories.id, uploads.created_at, uploads.updated_at
+      FROM categories
+      JOIN uploads ON uploads.id = categories.uploaded_background_id
+      WHERE categories.uploaded_background_id IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220404201949_copy_custom_emojis_uploads_to_upload_references.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CopyCustomEmojisUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT custom_emojis.upload_id, 'CustomEmoji', custom_emojis.id, uploads.created_at, uploads.updated_at
+      FROM custom_emojis
+      JOIN uploads ON uploads.id = custom_emojis.upload_id
+      WHERE custom_emojis.upload_id IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220404203356_copy_user_profiles_uploads_to_upload_references.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class CopyUserProfilesUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT user_profiles.profile_background_upload_id, 'UserProfile', user_profiles.user_id, uploads.created_at, uploads.updated_at
+      FROM user_profiles
+      JOIN uploads ON uploads.id = user_profiles.profile_background_upload_id
+      WHERE user_profiles.profile_background_upload_id IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT user_profiles.card_background_upload_id, 'UserProfile', user_profiles.user_id, uploads.created_at, uploads.updated_at
+      FROM user_profiles
+      JOIN uploads ON uploads.id = user_profiles.card_background_upload_id
+      WHERE user_profiles.card_background_upload_id IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220404204439_copy_user_avatars_uploads_to_upload_references.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class CopyUserAvatarsUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT user_avatars.custom_upload_id, 'UserAvatar', user_avatars.id, uploads.created_at, uploads.updated_at
+      FROM user_avatars
+      JOIN uploads ON uploads.id = user_avatars.custom_upload_id
+      WHERE user_avatars.custom_upload_id IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT user_avatars.gravatar_upload_id, 'UserAvatar', user_avatars.id, uploads.created_at, uploads.updated_at
+      FROM user_avatars
+      JOIN uploads ON uploads.id = user_avatars.gravatar_upload_id
+      WHERE user_avatars.gravatar_upload_id IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220404212716_copy_theme_settings_uploads_to_upload_references.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CopyThemeSettingsUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT theme_settings.value::int, 'ThemeSetting', theme_settings.id, uploads.created_at, uploads.updated_at
+      FROM theme_settings
+      JOIN uploads ON uploads.id = theme_settings.value::int
+      WHERE data_type = 6 AND theme_settings.value IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

db/migrate/20220526203356_copy_user_uploads_to_upload_references.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CopyUserUploadsToUploadReferences < ActiveRecord::Migration[6.1]
+  def up
+    execute <<~SQL
+      INSERT INTO upload_references(upload_id, target_type, target_id, created_at, updated_at)
+      SELECT users.uploaded_avatar_id, 'User', users.id, uploads.created_at, uploads.updated_at
+      FROM users
+      JOIN uploads ON uploads.id = users.uploaded_avatar_id
+      WHERE users.uploaded_avatar_id IS NOT NULL
+      ON CONFLICT DO NOTHING
+    SQL
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end