Merge pull request #4360 from tgxworld/support_new_pg_dump_format

Changes to backup and restore

config/locales/server.en.yml

@@ -1885,9 +1885,16 @@ en:
 
     backup_succeeded:
       subject_template: "Backup completed successfully"
-      text_body_template: "The backup was successful.
+      text_body_template: |
+        The backup was successful.
 
-      Visit the [admin > backup section](%{base_url}/admin/backups) to download your new backup."
+        Visit the [admin > backup section](%{base_url}/admin/backups) to download your new backup."
+
+        Here's the log:
+
+        ```text
+        %{logs}
+        ```
 
     backup_failed:
       subject_template: "Backup failed"
@@ -1896,13 +1903,20 @@ en:
 
         Here's the log:
 
-        ```
+        ```text
         %{logs}
         ```
 
     restore_succeeded:
       subject_template: "Restore completed successfully"
-      text_body_template: "The restore was successful."
+      text_body_template: |
+        The restore was successful.
+
+        Here's the log:
+
+        ```text
+        %{logs}
+        ```
 
     restore_failed:
       subject_template: "Restore failed"
@@ -1911,7 +1925,7 @@ en:
 
         Here's the log:
 
-        ```
+        ```text
         %{logs}
         ```
 

config/routes.rb

@@ -9,7 +9,7 @@ require_dependency "permalink_constraint"
 # and makes Guard not work properly.
 USERNAME_ROUTE_FORMAT = /[\w.\-]+/ unless defined? USERNAME_ROUTE_FORMAT
 
-BACKUP_ROUTE_FORMAT = /.+\.(tar\.gz|tgz)/i unless defined? BACKUP_ROUTE_FORMAT
+BACKUP_ROUTE_FORMAT = /.+\.(sql\.gz|tar\.gz|tgz)/i unless defined? BACKUP_ROUTE_FORMAT
 
 Discourse::Application.routes.draw do
 

lib/backup_restore/backup_restore.rb

@@ -6,6 +6,7 @@ module BackupRestore
 
   class OperationRunningError < RuntimeError; end
 
+  VERSION_PREFIX = "v".freeze
   DUMP_FILE = "dump.sql"
   METADATA_FILE = "meta.json"
   LOGS_CHANNEL = "/admin/backups/logs"

lib/backup_restore/backuper.rb

@@ -28,8 +28,6 @@ module BackupRestore
       ensure_directory_exists(@tmp_directory)
       ensure_directory_exists(@archive_directory)
 
-      write_metadata
-
       ### READ-ONLY / START ###
       enable_readonly_mode
 
@@ -43,7 +41,7 @@ module BackupRestore
 
       log "Finalizing backup..."
 
-      create_archive
+      @with_uploads ? create_archive : move_dump_backup
 
       after_create_hook
     rescue SystemExit
@@ -54,7 +52,7 @@ module BackupRestore
       @success = false
     else
       @success = true
-      "#{@archive_basename}.tar.gz"
+      @backup_filename
     ensure
       begin
         notify_user
@@ -84,9 +82,16 @@ module BackupRestore
       @timestamp = Time.now.strftime("%Y-%m-%d-%H%M%S")
       @tmp_directory = File.join(Rails.root, "tmp", "backups", @current_db, @timestamp)
       @dump_filename = "#{File.join(@tmp_directory, BackupRestore::DUMP_FILE)}.gz"
-      @meta_filename = File.join(@tmp_directory, BackupRestore::METADATA_FILE)
       @archive_directory = File.join(Rails.root, "public", "backups", @current_db)
-      @archive_basename = File.join(@archive_directory, "#{SiteSetting.title.parameterize}-#{@timestamp}")
+      @archive_basename = File.join(@archive_directory, "#{SiteSetting.title.parameterize}-#{@timestamp}-#{BackupRestore::VERSION_PREFIX}#{BackupRestore.current_version}")
+
+      @backup_filename =
+        if @with_uploads
+          "#{File.basename(@archive_basename)}.tar.gz"
+        else
+          "#{File.basename(@archive_basename)}.sql.gz"
+        end
+
       @logs = []
       @readonly_mode_was_enabled = Discourse.readonly_mode? || !SiteSetting.readonly_mode_during_backup
     end
@@ -137,15 +142,6 @@ module BackupRestore
       false
     end
 
-    def write_metadata
-      log "Writing metadata to '#{@meta_filename}'..."
-      metadata = {
-        source: "discourse",
-        version: BackupRestore.current_version
-      }
-      File.write(@meta_filename, metadata.to_json)
-    end
-
     def dump_public_schema
       log "Dumping the public schema of the database..."
 
@@ -199,8 +195,19 @@ module BackupRestore
       ].join(" ")
     end
 
+    def move_dump_backup
+      log "Finalizing database dump file: #{@backup_filename}"
+
+      execute_command(
+        "mv #{@dump_filename} #{File.join(@archive_directory, @backup_filename)}",
+        "Failed to move database dump file."
+      )
+
+      remove_tmp_directory
+    end
+
     def create_archive
-      log "Creating archive: #{File.basename(@archive_basename)}.tar.gz"
+      log "Creating archive: #{@backup_filename}"
 
       tar_filename = "#{@archive_basename}.tar"
 
@@ -219,23 +226,17 @@ module BackupRestore
         )
       end
 
-      log "Archiving metadata..."
+      upload_directory = "uploads/" + @current_db
-      FileUtils.cd(File.dirname(@meta_filename)) do
-        execute_command(
-          "tar --append --dereference --file #{tar_filename} #{File.basename(@meta_filename)}",
-          "Failed to archive metadata."
-        )
-      end
 
-      if @with_uploads
+      log "Archiving uploads..."
-        upload_directory = "uploads/" + @current_db
+      FileUtils.cd(File.join(Rails.root, "public")) do
-
+        if File.directory?(upload_directory)
-        log "Archiving uploads..."
-        FileUtils.cd(File.join(Rails.root, "public")) do
           execute_command(
             "tar --append --dereference --file #{tar_filename} #{upload_directory}",
             "Failed to archive uploads."
           )
+        else
+          log "No uploads found, skipping archiving uploads..."
         end
       end
 
@@ -247,7 +248,7 @@ module BackupRestore
 
     def after_create_hook
       log "Executing the after_create_hook for the backup..."
-      backup = Backup.create_from_filename("#{File.basename(@archive_basename)}.tar.gz")
+      backup = Backup.create_from_filename(@backup_filename)
       backup.after_create_hook
     end
 
@@ -259,9 +260,9 @@ module BackupRestore
     def notify_user
       log "Notifying '#{@user.username}' of the end of the backup..."
       if @success
-        SystemMessage.create_from_system_user(@user, :backup_succeeded)
+        SystemMessage.create_from_system_user(@user, :backup_succeeded, logs: pretty_logs(@logs))
       else
-        SystemMessage.create_from_system_user(@user, :backup_failed, logs: @logs.join("\n"))
+        SystemMessage.create_from_system_user(@user, :backup_failed, logs: pretty_logs(@logs))
       end
     end
 

lib/backup_restore/restorer.rb

@@ -110,13 +110,18 @@ module BackupRestore
       @archive_filename = File.join(@tmp_directory, @filename)
       @tar_filename = @archive_filename[0...-3]
       @meta_filename = File.join(@tmp_directory, BackupRestore::METADATA_FILE)
+      @is_archive = !(@filename =~ /.sql.gz$/)
 
       # For backwards compatibility
       @dump_filename =
-        if system("tar --list --file #{@source_filename} #{BackupRestore::DUMP_FILE}")
+        if @is_archive
-          File.join(@tmp_directory, BackupRestore::DUMP_FILE)
+          if system("tar --list --file #{@source_filename} #{BackupRestore::DUMP_FILE}")
+            File.join(@tmp_directory, BackupRestore::DUMP_FILE)
+          else
+            File.join(@tmp_directory, "#{BackupRestore::DUMP_FILE}.gz")
+          end
         else
-          File.join(@tmp_directory, "#{BackupRestore::DUMP_FILE}.gz")
+          File.join(@tmp_directory, @filename)
         end
 
       @logs = []
@@ -175,7 +180,10 @@ module BackupRestore
     end
 
     def unzip_archive
+      return unless @is_archive
+
       log "Unzipping archive, this may take a while..."
+
       FileUtils.cd(@tmp_directory) do
         execute_command("gzip --decompress '#{@archive_filename}'", "Failed to unzip archive.")
       end
@@ -184,14 +192,23 @@ module BackupRestore
     def extract_metadata
       log "Extracting metadata file..."
 
-      FileUtils.cd(@tmp_directory) do
+      @metadata =
-        execute_command(
+        if system("tar --list --file #{@source_filename} #{BackupRestore::METADATA_FILE}")
-          "tar --extract --file '#{@tar_filename}' #{BackupRestore::METADATA_FILE}",
+          FileUtils.cd(@tmp_directory) do
-          "Failed to extract metadata file."
+            execute_command(
-        )
+              "tar --extract --file '#{@tar_filename}' #{BackupRestore::METADATA_FILE}",
-      end
+              "Failed to extract metadata file."
+            )
+          end
 
-      @metadata = Oj.load_file(@meta_filename)
+          Oj.load_file(@meta_filename)
+        else
+          if @filename =~ /-#{BackupRestore::VERSION_PREFIX}(\d{14})/
+            { "version" => Regexp.last_match[1].to_i }
+          else
+            raise "Migration version is missing from the filename."
+          end
+        end
     end
 
     def validate_metadata
@@ -204,6 +221,8 @@ module BackupRestore
     end
 
     def extract_dump
+      return unless @is_archive
+
       log "Extracting dump file..."
 
       FileUtils.cd(@tmp_directory) do
@@ -334,7 +353,7 @@ module BackupRestore
     end
 
     def extract_uploads
-      if `tar --list --file '#{@tar_filename}' | grep 'uploads/'`.present?
+      if system("tar --list --file '#{@tar_filename}' 'uploads'")
         log "Extracting uploads..."
         FileUtils.cd(File.join(Rails.root, "public")) do
           execute_command(
@@ -359,9 +378,9 @@ module BackupRestore
       if user = User.find_by(email: @user_info[:email])
         log "Notifying '#{user.username}' of the end of the restore..."
         if @success
-          SystemMessage.create_from_system_user(user, :restore_succeeded)
+          SystemMessage.create_from_system_user(user, :restore_succeeded, logs: pretty_logs(@logs))
         else
-          SystemMessage.create_from_system_user(user, :restore_failed, logs: @logs.join("\n"))
+          SystemMessage.create_from_system_user(user, :restore_failed, logs: pretty_logs(@logs))
         end
       else
         log "Could not send notification to '#{@user_info[:username]}' (#{@user_info[:email]}), because the user does not exists..."

lib/backup_restore/utils.rb

@@ -10,5 +10,9 @@ module BackupRestore
 
       output
     end
+
+    def pretty_logs(logs)
+      logs.join("\n")
+    end
   end
 end