From f2dc59d61fe48b80b36f4fb13df4b59fde515705 Mon Sep 17 00:00:00 2001
From: Gerhard Schlager <mail@gerhard-schlager.at>
Date: Mon, 1 Jul 2019 20:38:36 +0200
Subject: [PATCH] FEATURE: Add hidden setting to include S3 uploads in backups

---
 config/site_settings.yml       |  4 +++
 lib/backup_restore/backuper.rb | 64 ++++++++++++++++++++++++++++++----
 lib/file_store/s3_store.rb     |  4 +++
 3 files changed, 65 insertions(+), 7 deletions(-)

diff --git a/config/site_settings.yml b/config/site_settings.yml
index 071412314dd..1a9fbb94dd1 100644
--- a/config/site_settings.yml
+++ b/config/site_settings.yml
@@ -1632,6 +1632,10 @@ backups:
   include_thumbnails_in_backups:
     default: false
     shadowed_by_global: true
+  include_s3_uploads_in_backups:
+    default: false
+    shadowed_by_global: true
+    hidden: true
 
 search:
   min_search_term_length:
diff --git a/lib/backup_restore/backuper.rb b/lib/backup_restore/backuper.rb
index 51a12410c0a..4c2b9f2aaad 100644
--- a/lib/backup_restore/backuper.rb
+++ b/lib/backup_restore/backuper.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "mini_mime"
+require "file_store/s3_store"
 
 module BackupRestore
 
@@ -234,9 +235,25 @@ module BackupRestore
         )
       end
 
+      if SiteSetting.Upload.enable_s3_uploads
+        add_remote_uploads_to_archive(tar_filename)
+      else
+        add_local_uploads_to_archive(tar_filename)
+      end
+
+      remove_tmp_directory
+
+      log "Gzipping archive, this may take a while..."
+      Discourse::Utils.execute_command(
+        'gzip', "-#{SiteSetting.backup_gzip_compression_level_for_uploads}", tar_filename,
+        failure_message: "Failed to gzip archive."
+      )
+    end
+
+    def add_local_uploads_to_archive(tar_filename)
+      log "Archiving uploads..."
       upload_directory = "uploads/" + @current_db
 
-      log "Archiving uploads..."
       FileUtils.cd(File.join(Rails.root, "public")) do
         if File.directory?(upload_directory)
           exclude_optimized = SiteSetting.include_thumbnails_in_backups ? '' : "--exclude=#{upload_directory}/optimized"
@@ -249,14 +266,47 @@ module BackupRestore
           log "No uploads found, skipping archiving uploads..."
         end
       end
+    end
 
-      remove_tmp_directory
+    def add_remote_uploads_to_archive(tar_filename)
+      if !SiteSetting.include_s3_uploads_in_backups
+        log "Skipping uploads stored on S3."
+        return
+      end
 
-      log "Gzipping archive, this may take a while..."
-      Discourse::Utils.execute_command(
-        'gzip', "-#{SiteSetting.backup_gzip_compression_level_for_uploads}", tar_filename,
-        failure_message: "Failed to gzip archive."
-      )
+      log "Downloading uploads from S3. This may take a while..."
+
+      store = FileStore::S3Store.new
+      upload_directory = File.join("uploads", @current_db)
+      count = 0
+
+      FileUtils.cd(@tmp_directory) do
+        Upload.find_each do |upload|
+          next if upload.local?
+          filename = File.join(@tmp_directory, upload_directory, store.get_path_for_upload(upload))
+
+          begin
+            FileUtils.mkdir_p(File.dirname(filename))
+            store.download_file(upload, filename)
+          rescue StandardError => ex
+            log "Failed to download file with upload ID #{upload.id} from S3", ex
+          end
+
+          if File.exists?(filename)
+            Discourse::Utils.execute_command(
+              'tar', '--append', '--file', tar_filename, upload_directory,
+              failure_message: "Failed to add #{upload.original_filename} to archive.", success_status_codes: [0, 1]
+            )
+
+            File.delete(filename)
+          end
+
+          count += 1
+          log "#{count} files have already been downloaded. Still downloading..." if count % 500 == 0
+        end
+      end
+
+      log "No uploads found, skipping archiving uploads..." if count == 0
     end
 
     def upload_archive
diff --git a/lib/file_store/s3_store.rb b/lib/file_store/s3_store.rb
index feb3ca8d2b1..9b27c7e2dbf 100644
--- a/lib/file_store/s3_store.rb
+++ b/lib/file_store/s3_store.rb
@@ -167,6 +167,10 @@ module FileStore
       end
     end
 
+    def download_file(upload, destination_path)
+      @s3_helper.download_file(get_upload_key(upload), destination_path)
+    end
+
     private
 
     def get_upload_key(upload)