From 469a2c36edf63923eadcb2b673f734bc592b81cd Mon Sep 17 00:00:00 2001 From: Gerhard Schlager Date: Wed, 19 Sep 2018 20:35:43 +0200 Subject: [PATCH] FIX: Always unpause Sidekiq after backup and restore * Logs exceptions during the cleanup phase, but doesn't stop executing subsequent cleanup tasks. * Notifies the user at the end of the cleanup phase, so that the log contains possible errors during that phase. --- lib/backup_restore/backuper.rb | 37 +++++++++++++++++++++------------- lib/backup_restore/restorer.rb | 27 ++++++++++++++----------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/lib/backup_restore/backuper.rb b/lib/backup_restore/backuper.rb index 50b52e08c3d..e7fdc04ed6d 100644 --- a/lib/backup_restore/backuper.rb +++ b/lib/backup_restore/backuper.rb @@ -54,13 +54,10 @@ module BackupRestore @success = true File.join(@archive_directory, @backup_filename) ensure - begin - notify_user - remove_old - clean_up - rescue => ex - Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n")) - end + remove_old + clean_up + notify_user + log "Finished!" @success ? log("[SUCCESS]") : log("[FAILED]") end @@ -255,6 +252,8 @@ module BackupRestore def remove_old log "Removing old backups..." Backup.remove_old + rescue => ex + log "Something went wrong while removing old backups.", ex end def notify_user @@ -270,6 +269,8 @@ module BackupRestore end post + rescue => ex + log "Something went wrong while notifying user.", ex end def clean_up @@ -279,42 +280,49 @@ module BackupRestore disable_readonly_mode if Discourse.readonly_mode? mark_backup_as_not_running refresh_disk_space - log "Finished!" end def refresh_disk_space - log "Refreshing disk cache..." + log "Refreshing disk stats..." DiskSpace.reset_cached_stats + rescue => ex + log "Something went wrong while refreshing disk stats.", ex end def remove_tar_leftovers log "Removing '.tar' leftovers..." Dir["#{@archive_directory}/*.tar"].each { |filename| File.delete(filename) } + rescue => ex + log "Something went wrong while removing '.tar' leftovers.", ex end def remove_tmp_directory log "Removing tmp '#{@tmp_directory}' directory..." FileUtils.rm_rf(@tmp_directory) if Dir[@tmp_directory].present? - rescue - log "Something went wrong while removing the following tmp directory: #{@tmp_directory}" + rescue => ex + log "Something went wrong while removing the following tmp directory: #{@tmp_directory}", ex end def unpause_sidekiq log "Unpausing sidekiq..." Sidekiq.unpause! - rescue - log "Something went wrong while unpausing Sidekiq." + rescue => ex + log "Something went wrong while unpausing Sidekiq.", ex end def disable_readonly_mode return if @readonly_mode_was_enabled log "Disabling readonly mode..." Discourse.disable_readonly_mode + rescue => ex + log "Something went wrong while disabling readonly mode.", ex end def mark_backup_as_not_running log "Marking backup as finished..." BackupRestore.mark_as_not_running! + rescue => ex + log "Something went wrong while marking backup as finished.", ex end def ensure_directory_exists(directory) @@ -322,11 +330,12 @@ module BackupRestore FileUtils.mkdir_p(directory) end - def log(message) + def log(message, ex = nil) timestamp = Time.now.strftime("%Y-%m-%d %H:%M:%S") puts(message) publish_log(message, timestamp) save_log(message, timestamp) + Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n")) if ex end def publish_log(message, timestamp) diff --git a/lib/backup_restore/restorer.rb b/lib/backup_restore/restorer.rb index a546a3072c2..e8a98f0581c 100644 --- a/lib/backup_restore/restorer.rb +++ b/lib/backup_restore/restorer.rb @@ -103,12 +103,9 @@ module BackupRestore else @success = true ensure - begin - notify_user - clean_up - rescue => ex - Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n")) - end + clean_up + notify_user + log "Finished!" @success ? log("[SUCCESS]") : log("[FAILED]") end @@ -459,6 +456,8 @@ module BackupRestore else log "Could not send notification to '#{@user_info[:username]}' (#{@user_info[:email]}), because the user does not exists..." end + rescue => ex + log "Something went wrong while notifying user.", ex end def clean_up @@ -467,32 +466,35 @@ module BackupRestore unpause_sidekiq disable_readonly_mode if Discourse.readonly_mode? mark_restore_as_not_running - log "Finished!" end def remove_tmp_directory log "Removing tmp '#{@tmp_directory}' directory..." FileUtils.rm_rf(@tmp_directory) if Dir[@tmp_directory].present? - rescue - log "Something went wrong while removing the following tmp directory: #{@tmp_directory}" + rescue => ex + log "Something went wrong while removing the following tmp directory: #{@tmp_directory}", ex end def unpause_sidekiq log "Unpausing sidekiq..." Sidekiq.unpause! - rescue - log "Something went wrong while unpausing Sidekiq." + rescue => ex + log "Something went wrong while unpausing Sidekiq.", ex end def disable_readonly_mode return if @readonly_mode_was_enabled log "Disabling readonly mode..." Discourse.disable_readonly_mode + rescue => ex + log "Something went wrong while disabling readonly mode.", ex end def mark_restore_as_not_running log "Marking restore as finished..." BackupRestore.mark_as_not_running! + rescue => ex + log "Something went wrong while marking restore as finished.", ex end def ensure_directory_exists(directory) @@ -500,11 +502,12 @@ module BackupRestore FileUtils.mkdir_p(directory) end - def log(message) + def log(message, ex = nil) timestamp = Time.now.strftime("%Y-%m-%d %H:%M:%S") puts(message) publish_log(message, timestamp) save_log(message, timestamp) + Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n")) if ex end def publish_log(message, timestamp)