FIX: Create readonly functions during backup

Temporarily recreate already dropped functions in the discourse_functions schema in order to allow restoring of backups which still reference dropped functions.

db/post_migrate/20180917024729_remove_superfluous_columns.rb

@@ -4,24 +4,23 @@ require 'migration/column_dropper'
 require 'badge_posts_view_manager'
 
 class RemoveSuperfluousColumns < ActiveRecord::Migration[5.2]
-  def up
+  DROPPED_COLUMNS ||= {
-    {
+    user_profiles: %i{
-      user_profiles: %i{
         card_image_badge_id
       },
-      categories: %i{
+    categories: %i{
         logo_url
         background_url
         suppress_from_homepage
       },
-      groups: %i{
+    groups: %i{
         visible
         public
         alias_level
       },
-      theme_fields: %i{target},
+    theme_fields: %i{target},
-      user_stats: %i{first_topic_unread_at},
+    user_stats: %i{first_topic_unread_at},
-      topics: %i{
+    topics: %i{
         auto_close_at
         auto_close_user_id
         auto_close_started_at
@@ -35,7 +34,7 @@ class RemoveSuperfluousColumns < ActiveRecord::Migration[5.2]
         last_unread_at
         vote_count
       },
-      users: %i{
+    users: %i{
         email
         email_always
         mailing_list_mode
@@ -58,23 +57,27 @@ class RemoveSuperfluousColumns < ActiveRecord::Migration[5.2]
         silenced
         trust_level_locked
       },
-      user_auth_tokens: %i{legacy},
+    user_auth_tokens: %i{legacy},
-      user_options: %i{theme_key},
+    user_options: %i{theme_key},
-      themes: %i{key},
+    themes: %i{key},
-      email_logs: %i{
+    email_logs: %i{
         topic_id
         reply_key
         skipped
         skipped_reason
       },
-    }.each do |table, columns|
+    posts: %i{vote_count}
+  }
+
+  def up
+    BadgePostsViewManager.drop!
+
+    DROPPED_COLUMNS.each do |table, columns|
       Migration::ColumnDropper.execute_drop(table, columns)
     end
 
     DB.exec "DROP FUNCTION IF EXISTS first_unread_topic_for(int)"
 
-    BadgePostsViewManager.drop!
-    Migration::ColumnDropper.execute_drop(:posts, %i{vote_count})
     BadgePostsViewManager.create!
   end
 

db/post_migrate/20180917034056_remove_superfluous_tables.rb

@@ -3,12 +3,14 @@
 require 'migration/table_dropper'
 
 class RemoveSuperfluousTables < ActiveRecord::Migration[5.2]
-  def up
+  DROPPED_TABLES ||= %i{
-    %i{
       category_featured_users
       versions
       topic_status_updates
-    }.each do |table|
+    }
+
+  def up
+    DROPPED_TABLES.each do |table|
       Migration::TableDropper.execute_drop(table)
     end
   end

db/post_migrate/20181012123001_drop_group_locked_trust_level_from_user.rb

@@ -3,8 +3,14 @@
 require 'migration/column_dropper'
 
 class DropGroupLockedTrustLevelFromUser < ActiveRecord::Migration[5.2]
+  DROPPED_COLUMNS ||= {
+    posts: %i{group_locked_trust_level}
+  }
+
   def up
-    Migration::ColumnDropper.execute_drop(:posts, %i{group_locked_trust_level})
+    DROPPED_COLUMNS.each do |table, columns|
+      Migration::ColumnDropper.execute_drop(table, columns)
+    end
   end
 
   def down

db/post_migrate/20190103065652_remove_uploaded_meta_id_from_category.rb

@@ -3,8 +3,14 @@
 require 'migration/column_dropper'
 
 class RemoveUploadedMetaIdFromCategory < ActiveRecord::Migration[5.2]
+  DROPPED_COLUMNS ||= {
+    categories: %i{uploaded_meta_id}
+  }
+
   def up
-    Migration::ColumnDropper.execute_drop(:categories, %i{uploaded_meta_id})
+    DROPPED_COLUMNS.each do |table, columns|
+      Migration::ColumnDropper.execute_drop(table, columns)
+    end
   end
 
   def down

db/post_migrate/20190208144706_drop_unused_auth_tables_again.rb

@@ -3,11 +3,13 @@
 require 'migration/table_dropper'
 
 class DropUnusedAuthTablesAgain < ActiveRecord::Migration[5.2]
-  def up
+  DROPPED_TABLES ||= %i{
-    %i{
         facebook_user_infos
         twitter_user_infos
-      }.each do |table|
+      }
+
+  def up
+    DROPPED_TABLES.each do |table|
       Migration::TableDropper.execute_drop(table)
     end
   end

db/post_migrate/20190312194528_drop_email_user_options_columns.rb

@@ -3,14 +3,16 @@
 require 'migration/column_dropper'
 
 class DropEmailUserOptionsColumns < ActiveRecord::Migration[5.2]
-  def up
+  DROPPED_COLUMNS ||= {
-    {
+    user_options: %i{
-      user_options: %i{
         email_direct
         email_private_messages
         email_always
       },
-    }.each do |table, columns|
+  }
+
+  def up
+    DROPPED_COLUMNS.each do |table, columns|
       Migration::ColumnDropper.execute_drop(table, columns)
     end
   end

db/post_migrate/20190716124050_remove_via_email_from_invite.rb

@@ -3,8 +3,14 @@
 require 'migration/column_dropper'
 
 class RemoveViaEmailFromInvite < ActiveRecord::Migration[5.2]
+  DROPPED_COLUMNS ||= {
+    invites: %i{via_email}
+  }
+
   def up
-    Migration::ColumnDropper.execute_drop(:invites, %i{via_email})
+    DROPPED_COLUMNS.each do |table, columns|
+      Migration::ColumnDropper.execute_drop(table, columns)
+    end
   end
 
   def down

lib/backup_restore/restorer.rb

@@ -63,6 +63,7 @@ module BackupRestore
       validate_metadata
 
       extract_dump
+      create_missing_discourse_functions
 
       if !can_restore_into_different_schema?
         log "Cannot restore into different schema, restoring in-place"
@@ -144,6 +145,7 @@ module BackupRestore
 
       @logs = []
       @readonly_mode_was_enabled = Discourse.readonly_mode?
+      @created_functions_for_table_columns = []
     end
 
     def listen_for_shutdown_signal
@@ -561,8 +563,46 @@ module BackupRestore
       log "Something went wrong while notifying user.", ex
     end
 
+    def create_missing_discourse_functions
+      log "Creating missing functions in the discourse_functions schema"
+
+      all_readonly_table_columns = []
+
+      Dir[Rails.root.join(Discourse::DB_POST_MIGRATE_PATH, "*.rb")].each do |path|
+        require path
+        class_name = File.basename(path, ".rb").sub(/^\d+_/, "").camelize
+        migration_class = class_name.constantize
+
+        if migration_class.const_defined?(:DROPPED_TABLES)
+          migration_class::DROPPED_TABLES.each do |table_name|
+            all_readonly_table_columns << [table_name]
+          end
+        end
+
+        if migration_class.const_defined?(:DROPPED_COLUMNS)
+          migration_class::DROPPED_COLUMNS.each do |table_name, column_names|
+            column_names.each do |column_name|
+              all_readonly_table_columns << [table_name, column_name]
+            end
+          end
+        end
+      end
+
+      existing_function_names = Migration::BaseDropper.existing_discourse_function_names.map { |name| "#{name}()" }
+
+      all_readonly_table_columns.each do |table_name, column_name|
+        function_name = Migration::BaseDropper.readonly_function_name(table_name, column_name, with_schema: false)
+
+        if !existing_function_names.include?(function_name)
+          Migration::BaseDropper.create_readonly_function(table_name, column_name)
+          @created_functions_for_table_columns << [table_name, column_name]
+        end
+      end
+    end
+
     def clean_up
       log "Cleaning stuff up..."
+      drop_created_discourse_functions
       remove_tmp_directory
       unpause_sidekiq
       disable_readonly_mode if Discourse.readonly_mode?
@@ -590,6 +630,15 @@ module BackupRestore
       Stylesheet::Manager.cache.clear
     end
 
+    def drop_created_discourse_functions
+      log "Dropping function from the discourse_functions schema"
+      @created_functions_for_table_columns.each do |table_name, column_name|
+        Migration::BaseDropper.drop_readonly_function(table_name, column_name)
+      end
+    rescue => ex
+      log "Something went wrong while dropping functions from the discourse_functions schema", ex
+    end
+
     def disable_readonly_mode
       return if @readonly_mode_was_enabled
       log "Disabling readonly mode..."

lib/migration/base_dropper.rb

@@ -22,7 +22,11 @@ module Migration
       SQL
     end
 
-    def self.readonly_function_name(table_name, column_name = nil)
+    def self.drop_readonly_function(table_name, column_name = nil)
+      DB.exec("DROP FUNCTION IF EXISTS #{readonly_function_name(table_name, column_name)} CASCADE")
+    end
+
+    def self.readonly_function_name(table_name, column_name = nil, with_schema: true)
       function_name = [
         "raise",
         table_name,
@@ -30,12 +34,7 @@ module Migration
         "readonly()"
       ].compact.join("_")
 
-      if DB.exec(<<~SQL).to_s == '1'
+      if with_schema && function_schema_exists?
-         SELECT schema_name
-         FROM information_schema.schemata
-         WHERE schema_name = '#{FUNCTION_SCHEMA_NAME}'
-         SQL
-
         "#{FUNCTION_SCHEMA_NAME}.#{function_name}"
       else
         function_name
@@ -51,5 +50,21 @@ module Migration
     def self.readonly_trigger_name(table_name, column_name = nil)
       [table_name, column_name, "readonly"].compact.join("_")
     end
+
+    def self.function_schema_exists?
+      DB.exec(<<~SQL).to_s == '1'
+        SELECT schema_name
+        FROM information_schema.schemata
+        WHERE schema_name = '#{FUNCTION_SCHEMA_NAME}'
+      SQL
+    end
+
+    def self.existing_discourse_function_names
+      DB.query_single(<<~SQL)
+        SELECT routine_name
+        FROM information_schema.routines
+        WHERE routine_type = 'FUNCTION' AND specific_schema = '#{FUNCTION_SCHEMA_NAME}'
+      SQL
+    end
   end
 end

lib/migration/column_dropper.rb

@@ -28,13 +28,11 @@ module Migration
       end
     end
 
-    def self.drop_readonly(table, column)
+    def self.drop_readonly(table_name, column_name)
-      DB.exec <<~SQL
+      BaseDropper.drop_readonly_function(table_name, column_name)
-        DROP FUNCTION IF EXISTS #{BaseDropper.readonly_function_name(table, column)} CASCADE;
+
-        -- Backward compatibility for old functions created in the public
+      # Backward compatibility for old functions created in the public schema
-        -- schema
+      DB.exec("DROP FUNCTION IF EXISTS #{BaseDropper.old_readonly_function_name(table_name, column_name)} CASCADE")
-        DROP FUNCTION IF EXISTS #{BaseDropper.old_readonly_function_name(table, column)} CASCADE;
-      SQL
     end
   end
 end

spec/integrity/coding_style_spec.rb

@@ -32,4 +32,36 @@ describe 'Coding style' do
       MSG
     end
   end
+
+  describe 'Post Migrations' do
+    def check_offenses(files, method_name, constant_name)
+      method_name_regex = /#{Regexp.escape(method_name)}/
+      constant_name_regex = /#{Regexp.escape(constant_name)}/
+      offenses = files.reject { |file| is_valid?(file, method_name_regex, constant_name_regex) }
+
+      expect(offenses).to be_empty, <<~MSG
+        You need to use the constant #{constant_name} when you use
+        #{method_name} in order to help with restoring backups.
+
+        Please take a look at existing migrations to see how to use it correctly.
+
+        Offenses:
+        #{offenses.join("\n")}
+      MSG
+    end
+
+    def is_valid?(file, method_name_regex, constant_name_regex)
+      contains_method_name = File.open(file).grep(method_name_regex).any?
+      contains_constant_name = File.open(file).grep(constant_name_regex).any?
+
+      contains_method_name ? contains_constant_name : true
+    end
+
+    it 'ensures dropped tables and columns are stored in constants' do
+      migration_files = list_files('db/post_migrate', '**/*.rb')
+
+      check_offenses(migration_files, "ColumnDropper.execute_drop", "DROPPED_COLUMNS")
+      check_offenses(migration_files, "TableDropper.execute_drop", "DROPPED_TABLES")
+    end
+  end
 end