FIX: Always unpause Sidekiq after backup and restore

* Logs exceptions during the cleanup phase, but doesn't stop executing subsequent cleanup tasks.
* Notifies the user at the end of the cleanup phase, so that the log contains possible errors during that phase.
This commit is contained in:
Gerhard Schlager 2018-09-19 20:35:43 +02:00
parent 681262ddce
commit 469a2c36ed
2 changed files with 38 additions and 26 deletions

View File

@ -54,13 +54,10 @@ module BackupRestore
@success = true
File.join(@archive_directory, @backup_filename)
ensure
begin
notify_user
remove_old
clean_up
rescue => ex
Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n"))
end
remove_old
clean_up
notify_user
log "Finished!"
@success ? log("[SUCCESS]") : log("[FAILED]")
end
@ -255,6 +252,8 @@ module BackupRestore
def remove_old
log "Removing old backups..."
Backup.remove_old
rescue => ex
log "Something went wrong while removing old backups.", ex
end
def notify_user
@ -270,6 +269,8 @@ module BackupRestore
end
post
rescue => ex
log "Something went wrong while notifying user.", ex
end
def clean_up
@ -279,42 +280,49 @@ module BackupRestore
disable_readonly_mode if Discourse.readonly_mode?
mark_backup_as_not_running
refresh_disk_space
log "Finished!"
end
def refresh_disk_space
log "Refreshing disk cache..."
log "Refreshing disk stats..."
DiskSpace.reset_cached_stats
rescue => ex
log "Something went wrong while refreshing disk stats.", ex
end
def remove_tar_leftovers
log "Removing '.tar' leftovers..."
Dir["#{@archive_directory}/*.tar"].each { |filename| File.delete(filename) }
rescue => ex
log "Something went wrong while removing '.tar' leftovers.", ex
end
def remove_tmp_directory
log "Removing tmp '#{@tmp_directory}' directory..."
FileUtils.rm_rf(@tmp_directory) if Dir[@tmp_directory].present?
rescue
log "Something went wrong while removing the following tmp directory: #{@tmp_directory}"
rescue => ex
log "Something went wrong while removing the following tmp directory: #{@tmp_directory}", ex
end
def unpause_sidekiq
log "Unpausing sidekiq..."
Sidekiq.unpause!
rescue
log "Something went wrong while unpausing Sidekiq."
rescue => ex
log "Something went wrong while unpausing Sidekiq.", ex
end
def disable_readonly_mode
return if @readonly_mode_was_enabled
log "Disabling readonly mode..."
Discourse.disable_readonly_mode
rescue => ex
log "Something went wrong while disabling readonly mode.", ex
end
def mark_backup_as_not_running
log "Marking backup as finished..."
BackupRestore.mark_as_not_running!
rescue => ex
log "Something went wrong while marking backup as finished.", ex
end
def ensure_directory_exists(directory)
@ -322,11 +330,12 @@ module BackupRestore
FileUtils.mkdir_p(directory)
end
def log(message)
def log(message, ex = nil)
timestamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
puts(message)
publish_log(message, timestamp)
save_log(message, timestamp)
Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n")) if ex
end
def publish_log(message, timestamp)

View File

@ -103,12 +103,9 @@ module BackupRestore
else
@success = true
ensure
begin
notify_user
clean_up
rescue => ex
Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n"))
end
clean_up
notify_user
log "Finished!"
@success ? log("[SUCCESS]") : log("[FAILED]")
end
@ -459,6 +456,8 @@ module BackupRestore
else
log "Could not send notification to '#{@user_info[:username]}' (#{@user_info[:email]}), because the user does not exists..."
end
rescue => ex
log "Something went wrong while notifying user.", ex
end
def clean_up
@ -467,32 +466,35 @@ module BackupRestore
unpause_sidekiq
disable_readonly_mode if Discourse.readonly_mode?
mark_restore_as_not_running
log "Finished!"
end
def remove_tmp_directory
log "Removing tmp '#{@tmp_directory}' directory..."
FileUtils.rm_rf(@tmp_directory) if Dir[@tmp_directory].present?
rescue
log "Something went wrong while removing the following tmp directory: #{@tmp_directory}"
rescue => ex
log "Something went wrong while removing the following tmp directory: #{@tmp_directory}", ex
end
def unpause_sidekiq
log "Unpausing sidekiq..."
Sidekiq.unpause!
rescue
log "Something went wrong while unpausing Sidekiq."
rescue => ex
log "Something went wrong while unpausing Sidekiq.", ex
end
def disable_readonly_mode
return if @readonly_mode_was_enabled
log "Disabling readonly mode..."
Discourse.disable_readonly_mode
rescue => ex
log "Something went wrong while disabling readonly mode.", ex
end
def mark_restore_as_not_running
log "Marking restore as finished..."
BackupRestore.mark_as_not_running!
rescue => ex
log "Something went wrong while marking restore as finished.", ex
end
def ensure_directory_exists(directory)
@ -500,11 +502,12 @@ module BackupRestore
FileUtils.mkdir_p(directory)
end
def log(message)
def log(message, ex = nil)
timestamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
puts(message)
publish_log(message, timestamp)
save_log(message, timestamp)
Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n")) if ex
end
def publish_log(message, timestamp)