BUGFIX: better resiliency in the backup/restore processes

lib/backup_restore.rb

@@ -33,10 +33,10 @@ module BackupRestore
   end
 
   def self.mark_as_running!
-    # TODO: for more safety, it should acquire a lock
-    #       and raise an exception if already running!
+    # TODO: for extra safety, it should acquire a lock and raise an exception if already running
     $redis.set(running_key, "1")
     save_start_logs_message_id
+    keep_it_running
   end
 
   def self.is_operation_running?
@@ -80,14 +80,14 @@ module BackupRestore
     <<-SQL
       DO $$DECLARE row record;
       BEGIN
-        -- create "destination" schema if it does not exists already
-        -- NOTE: DROP & CREATE SCHEMA is easier, but we don't wont to drop the public schema
-        -- ortherwise extensions (like hstore & pg_trgm) won't work anymore
+        -- create <destination> schema if it does not exists already
+        -- NOTE: DROP & CREATE SCHEMA is easier, but we don't want to drop the public schema
+        -- ortherwise extensions (like hstore & pg_trgm) won't work anymore...
         IF NOT EXISTS(SELECT 1 FROM pg_namespace WHERE nspname = '#{destination}')
         THEN
           CREATE SCHEMA #{destination};
         END IF;
-        -- move all "source" tables to "destination" schema
+        -- move all <source> tables to <destination> schema
         FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = '#{source}'
         LOOP
           EXECUTE 'DROP TABLE IF EXISTS #{destination}.' || quote_ident(row.tablename) || ' CASCADE;';
@@ -117,6 +117,17 @@ module BackupRestore
     "backup_restore_operation_is_running"
   end
 
+  def self.keep_it_running
+    # extend the expiry by 1 minute every 30 seconds
+    Thread.new do
+      # this thread will be killed when the fork dies
+      while true
+        $redis.expire(running_key, 1.minute)
+        sleep 30.seconds
+      end
+    end
+  end
+
   def self.shutdown_signal_key
     "backup_restore_operation_should_shutdown"
   end

lib/export/exporter.rb

@@ -53,7 +53,7 @@ module Export
       @success = true
       "#{@archive_basename}.tar.gz"
     ensure
-      notify_user
+      notify_user rescue nil
       clean_up
       @success ? log("[SUCCESS]") : log("[FAILED]")
     end
@@ -293,6 +293,8 @@ module Export
     def unpause_sidekiq
       log "Unpausing sidekiq..."
       Sidekiq.unpause!
+    rescue
+      log "Something went wrong while unpausing Sidekiq."
     end
 
     def disable_readonly_mode

lib/import/importer.rb

@@ -63,7 +63,7 @@ module Import
     else
       @success = true
     ensure
-      notify_user
+      notify_user rescue nil
       clean_up
       @success ? log("[SUCCESS]") : log("[FAILED]")
     end
@@ -312,6 +312,8 @@ module Import
     def unpause_sidekiq
       log "Unpausing sidekiq..."
       Sidekiq.unpause!
+    rescue
+      log "Something went wrong while unpausing Sidekiq."
     end
 
     def disable_readonly_mode