diff --git a/db/post_migrate/20180917024729_remove_superfluous_columns.rb b/db/post_migrate/20180917024729_remove_superfluous_columns.rb index 128c3e810ef..a2307345d0a 100644 --- a/db/post_migrate/20180917024729_remove_superfluous_columns.rb +++ b/db/post_migrate/20180917024729_remove_superfluous_columns.rb @@ -4,24 +4,23 @@ require 'migration/column_dropper' require 'badge_posts_view_manager' class RemoveSuperfluousColumns < ActiveRecord::Migration[5.2] - def up - { - user_profiles: %i{ + DROPPED_COLUMNS ||= { + user_profiles: %i{ card_image_badge_id }, - categories: %i{ + categories: %i{ logo_url background_url suppress_from_homepage }, - groups: %i{ + groups: %i{ visible public alias_level }, - theme_fields: %i{target}, - user_stats: %i{first_topic_unread_at}, - topics: %i{ + theme_fields: %i{target}, + user_stats: %i{first_topic_unread_at}, + topics: %i{ auto_close_at auto_close_user_id auto_close_started_at @@ -35,7 +34,7 @@ class RemoveSuperfluousColumns < ActiveRecord::Migration[5.2] last_unread_at vote_count }, - users: %i{ + users: %i{ email email_always mailing_list_mode @@ -58,23 +57,27 @@ class RemoveSuperfluousColumns < ActiveRecord::Migration[5.2] silenced trust_level_locked }, - user_auth_tokens: %i{legacy}, - user_options: %i{theme_key}, - themes: %i{key}, - email_logs: %i{ + user_auth_tokens: %i{legacy}, + user_options: %i{theme_key}, + themes: %i{key}, + email_logs: %i{ topic_id reply_key skipped skipped_reason }, - }.each do |table, columns| + posts: %i{vote_count} + } + + def up + BadgePostsViewManager.drop! + + DROPPED_COLUMNS.each do |table, columns| Migration::ColumnDropper.execute_drop(table, columns) end DB.exec "DROP FUNCTION IF EXISTS first_unread_topic_for(int)" - BadgePostsViewManager.drop! - Migration::ColumnDropper.execute_drop(:posts, %i{vote_count}) BadgePostsViewManager.create! end diff --git a/db/post_migrate/20180917034056_remove_superfluous_tables.rb b/db/post_migrate/20180917034056_remove_superfluous_tables.rb index 21cba12ec4e..f6524526835 100644 --- a/db/post_migrate/20180917034056_remove_superfluous_tables.rb +++ b/db/post_migrate/20180917034056_remove_superfluous_tables.rb @@ -3,12 +3,14 @@ require 'migration/table_dropper' class RemoveSuperfluousTables < ActiveRecord::Migration[5.2] - def up - %i{ + DROPPED_TABLES ||= %i{ category_featured_users versions topic_status_updates - }.each do |table| + } + + def up + DROPPED_TABLES.each do |table| Migration::TableDropper.execute_drop(table) end end diff --git a/db/post_migrate/20181012123001_drop_group_locked_trust_level_from_user.rb b/db/post_migrate/20181012123001_drop_group_locked_trust_level_from_user.rb index b1ec42c26b9..3b3bd6eba30 100644 --- a/db/post_migrate/20181012123001_drop_group_locked_trust_level_from_user.rb +++ b/db/post_migrate/20181012123001_drop_group_locked_trust_level_from_user.rb @@ -3,8 +3,14 @@ require 'migration/column_dropper' class DropGroupLockedTrustLevelFromUser < ActiveRecord::Migration[5.2] + DROPPED_COLUMNS ||= { + posts: %i{group_locked_trust_level} + } + def up - Migration::ColumnDropper.execute_drop(:posts, %i{group_locked_trust_level}) + DROPPED_COLUMNS.each do |table, columns| + Migration::ColumnDropper.execute_drop(table, columns) + end end def down diff --git a/db/post_migrate/20190103065652_remove_uploaded_meta_id_from_category.rb b/db/post_migrate/20190103065652_remove_uploaded_meta_id_from_category.rb index 484a31c6c41..9aba22fa065 100644 --- a/db/post_migrate/20190103065652_remove_uploaded_meta_id_from_category.rb +++ b/db/post_migrate/20190103065652_remove_uploaded_meta_id_from_category.rb @@ -3,8 +3,14 @@ require 'migration/column_dropper' class RemoveUploadedMetaIdFromCategory < ActiveRecord::Migration[5.2] + DROPPED_COLUMNS ||= { + categories: %i{uploaded_meta_id} + } + def up - Migration::ColumnDropper.execute_drop(:categories, %i{uploaded_meta_id}) + DROPPED_COLUMNS.each do |table, columns| + Migration::ColumnDropper.execute_drop(table, columns) + end end def down diff --git a/db/post_migrate/20190208144706_drop_unused_auth_tables_again.rb b/db/post_migrate/20190208144706_drop_unused_auth_tables_again.rb index c82137c9e66..6fb208c6c83 100644 --- a/db/post_migrate/20190208144706_drop_unused_auth_tables_again.rb +++ b/db/post_migrate/20190208144706_drop_unused_auth_tables_again.rb @@ -3,11 +3,13 @@ require 'migration/table_dropper' class DropUnusedAuthTablesAgain < ActiveRecord::Migration[5.2] - def up - %i{ + DROPPED_TABLES ||= %i{ facebook_user_infos twitter_user_infos - }.each do |table| + } + + def up + DROPPED_TABLES.each do |table| Migration::TableDropper.execute_drop(table) end end diff --git a/db/post_migrate/20190312194528_drop_email_user_options_columns.rb b/db/post_migrate/20190312194528_drop_email_user_options_columns.rb index bb9ec958c1a..1b4203b1754 100644 --- a/db/post_migrate/20190312194528_drop_email_user_options_columns.rb +++ b/db/post_migrate/20190312194528_drop_email_user_options_columns.rb @@ -3,14 +3,16 @@ require 'migration/column_dropper' class DropEmailUserOptionsColumns < ActiveRecord::Migration[5.2] - def up - { - user_options: %i{ + DROPPED_COLUMNS ||= { + user_options: %i{ email_direct email_private_messages email_always }, - }.each do |table, columns| + } + + def up + DROPPED_COLUMNS.each do |table, columns| Migration::ColumnDropper.execute_drop(table, columns) end end diff --git a/db/post_migrate/20190716124050_remove_via_email_from_invite.rb b/db/post_migrate/20190716124050_remove_via_email_from_invite.rb index 676ea11bcac..29e91f39edb 100644 --- a/db/post_migrate/20190716124050_remove_via_email_from_invite.rb +++ b/db/post_migrate/20190716124050_remove_via_email_from_invite.rb @@ -3,8 +3,14 @@ require 'migration/column_dropper' class RemoveViaEmailFromInvite < ActiveRecord::Migration[5.2] + DROPPED_COLUMNS ||= { + invites: %i{via_email} + } + def up - Migration::ColumnDropper.execute_drop(:invites, %i{via_email}) + DROPPED_COLUMNS.each do |table, columns| + Migration::ColumnDropper.execute_drop(table, columns) + end end def down diff --git a/lib/backup_restore/restorer.rb b/lib/backup_restore/restorer.rb index 701c72e7d48..1b038a5d1e3 100644 --- a/lib/backup_restore/restorer.rb +++ b/lib/backup_restore/restorer.rb @@ -63,6 +63,7 @@ module BackupRestore validate_metadata extract_dump + create_missing_discourse_functions if !can_restore_into_different_schema? log "Cannot restore into different schema, restoring in-place" @@ -144,6 +145,7 @@ module BackupRestore @logs = [] @readonly_mode_was_enabled = Discourse.readonly_mode? + @created_functions_for_table_columns = [] end def listen_for_shutdown_signal @@ -561,8 +563,46 @@ module BackupRestore log "Something went wrong while notifying user.", ex end + def create_missing_discourse_functions + log "Creating missing functions in the discourse_functions schema" + + all_readonly_table_columns = [] + + Dir[Rails.root.join(Discourse::DB_POST_MIGRATE_PATH, "*.rb")].each do |path| + require path + class_name = File.basename(path, ".rb").sub(/^\d+_/, "").camelize + migration_class = class_name.constantize + + if migration_class.const_defined?(:DROPPED_TABLES) + migration_class::DROPPED_TABLES.each do |table_name| + all_readonly_table_columns << [table_name] + end + end + + if migration_class.const_defined?(:DROPPED_COLUMNS) + migration_class::DROPPED_COLUMNS.each do |table_name, column_names| + column_names.each do |column_name| + all_readonly_table_columns << [table_name, column_name] + end + end + end + end + + existing_function_names = Migration::BaseDropper.existing_discourse_function_names.map { |name| "#{name}()" } + + all_readonly_table_columns.each do |table_name, column_name| + function_name = Migration::BaseDropper.readonly_function_name(table_name, column_name, with_schema: false) + + if !existing_function_names.include?(function_name) + Migration::BaseDropper.create_readonly_function(table_name, column_name) + @created_functions_for_table_columns << [table_name, column_name] + end + end + end + def clean_up log "Cleaning stuff up..." + drop_created_discourse_functions remove_tmp_directory unpause_sidekiq disable_readonly_mode if Discourse.readonly_mode? @@ -590,6 +630,15 @@ module BackupRestore Stylesheet::Manager.cache.clear end + def drop_created_discourse_functions + log "Dropping function from the discourse_functions schema" + @created_functions_for_table_columns.each do |table_name, column_name| + Migration::BaseDropper.drop_readonly_function(table_name, column_name) + end + rescue => ex + log "Something went wrong while dropping functions from the discourse_functions schema", ex + end + def disable_readonly_mode return if @readonly_mode_was_enabled log "Disabling readonly mode..." diff --git a/lib/migration/base_dropper.rb b/lib/migration/base_dropper.rb index e6f397ecd0b..2ea01131e2f 100644 --- a/lib/migration/base_dropper.rb +++ b/lib/migration/base_dropper.rb @@ -22,7 +22,11 @@ module Migration SQL end - def self.readonly_function_name(table_name, column_name = nil) + def self.drop_readonly_function(table_name, column_name = nil) + DB.exec("DROP FUNCTION IF EXISTS #{readonly_function_name(table_name, column_name)} CASCADE") + end + + def self.readonly_function_name(table_name, column_name = nil, with_schema: true) function_name = [ "raise", table_name, @@ -30,12 +34,7 @@ module Migration "readonly()" ].compact.join("_") - if DB.exec(<<~SQL).to_s == '1' - SELECT schema_name - FROM information_schema.schemata - WHERE schema_name = '#{FUNCTION_SCHEMA_NAME}' - SQL - + if with_schema && function_schema_exists? "#{FUNCTION_SCHEMA_NAME}.#{function_name}" else function_name @@ -51,5 +50,21 @@ module Migration def self.readonly_trigger_name(table_name, column_name = nil) [table_name, column_name, "readonly"].compact.join("_") end + + def self.function_schema_exists? + DB.exec(<<~SQL).to_s == '1' + SELECT schema_name + FROM information_schema.schemata + WHERE schema_name = '#{FUNCTION_SCHEMA_NAME}' + SQL + end + + def self.existing_discourse_function_names + DB.query_single(<<~SQL) + SELECT routine_name + FROM information_schema.routines + WHERE routine_type = 'FUNCTION' AND specific_schema = '#{FUNCTION_SCHEMA_NAME}' + SQL + end end end diff --git a/lib/migration/column_dropper.rb b/lib/migration/column_dropper.rb index 62c8ad97f4b..754f5bf8267 100644 --- a/lib/migration/column_dropper.rb +++ b/lib/migration/column_dropper.rb @@ -28,13 +28,11 @@ module Migration end end - def self.drop_readonly(table, column) - DB.exec <<~SQL - DROP FUNCTION IF EXISTS #{BaseDropper.readonly_function_name(table, column)} CASCADE; - -- Backward compatibility for old functions created in the public - -- schema - DROP FUNCTION IF EXISTS #{BaseDropper.old_readonly_function_name(table, column)} CASCADE; - SQL + def self.drop_readonly(table_name, column_name) + BaseDropper.drop_readonly_function(table_name, column_name) + + # Backward compatibility for old functions created in the public schema + DB.exec("DROP FUNCTION IF EXISTS #{BaseDropper.old_readonly_function_name(table_name, column_name)} CASCADE") end end end diff --git a/spec/integrity/coding_style_spec.rb b/spec/integrity/coding_style_spec.rb index 96c4ff17daf..864ff1d5a62 100644 --- a/spec/integrity/coding_style_spec.rb +++ b/spec/integrity/coding_style_spec.rb @@ -32,4 +32,36 @@ describe 'Coding style' do MSG end end + + describe 'Post Migrations' do + def check_offenses(files, method_name, constant_name) + method_name_regex = /#{Regexp.escape(method_name)}/ + constant_name_regex = /#{Regexp.escape(constant_name)}/ + offenses = files.reject { |file| is_valid?(file, method_name_regex, constant_name_regex) } + + expect(offenses).to be_empty, <<~MSG + You need to use the constant #{constant_name} when you use + #{method_name} in order to help with restoring backups. + + Please take a look at existing migrations to see how to use it correctly. + + Offenses: + #{offenses.join("\n")} + MSG + end + + def is_valid?(file, method_name_regex, constant_name_regex) + contains_method_name = File.open(file).grep(method_name_regex).any? + contains_constant_name = File.open(file).grep(constant_name_regex).any? + + contains_method_name ? contains_constant_name : true + end + + it 'ensures dropped tables and columns are stored in constants' do + migration_files = list_files('db/post_migrate', '**/*.rb') + + check_offenses(migration_files, "ColumnDropper.execute_drop", "DROPPED_COLUMNS") + check_offenses(migration_files, "TableDropper.execute_drop", "DROPPED_TABLES") + end + end end