Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DEV: new S3 backup layout 

Currently, with $S3_BACKUP_BUCKET of "bucket/backups", multisite backups
end up in "bucket/backups/backups/dbname/" and single-site will be in
"bucket/backups/".

Both _should_ be in "bucket/backups/dbname/"

- remove MULTISITE_PREFIX,
- always include dbname,
- method to move to the new prefix
- job to call the method
This commit is contained in:
Andrew Schleifer 2020-05-07 12:58:53 +08:00
parent 07b728c5e5
commit caffd002b9
3 changed files with 41 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/jobs/onceoff/vacate_legacy_prefix_backups.rb Normal file
View File

@ -0,0 +1,9 @@
# frozen_string_literal: true
module Jobs
class VacateLegacyPrefixBackups < ::Jobs::Onceoff
def execute_onceoff(args)
BackupRestore::S3BackupStore.create.vacate_legacy_prefix if SiteSetting.backup_location == BackupLocationSiteSetting::S3
end
end
end

27
lib/backup_restore/s3_backup_store.rb
View File

@ -3,12 +3,11 @@
module BackupRestore
class S3BackupStore < BackupStore
UPLOAD_URL_EXPIRES_AFTER_SECONDS ||= 21_600 # 6 hours
MULTISITE_PREFIX = "backups"
def initialize(opts = {})
s3_options = S3Helper.s3_options(SiteSetting)
s3_options.merge!(opts[:s3_options]) if opts[:s3_options]
@s3_helper = S3Helper.new(s3_bucket_name_with_prefix, '', s3_options)
@s3_options = S3Helper.s3_options(SiteSetting)
@s3_options.merge!(opts[:s3_options]) if opts[:s3_options]
@s3_helper = S3Helper.new(s3_bucket_name_with_prefix, '', @s3_options)
end
def remote?
@ -52,6 +51,20 @@ module BackupRestore
raise StorageError
end
def vacate_legacy_prefix
legacy_s3_helper = S3Helper.new(s3_bucket_name_with_legacy_prefix, '', @s3_options)
legacy_keys = legacy_s3_helper.list.map { |o| o.key }
legacy_keys.each do |legacy_key|
bucket, prefix = s3_bucket_name_with_prefix.split('/', 2)
@s3_helper.s3_client.copy_object({
copy_source: File.join(bucket, legacy_key),
bucket: bucket,
key: File.join(prefix, legacy_key.split('/').last)
})
legacy_s3_helper.remove(legacy_key, false)
end
end
private
def unsorted_files
@ -99,8 +112,12 @@ module BackupRestore
end
def s3_bucket_name_with_prefix
File.join(SiteSetting.s3_backup_bucket, RailsMultisite::ConnectionManagement.current_db)
end
def s3_bucket_name_with_legacy_prefix
if Rails.configuration.multisite
File.join(SiteSetting.s3_backup_bucket, MULTISITE_PREFIX, RailsMultisite::ConnectionManagement.current_db)
File.join(SiteSetting.s3_backup_bucket, "backups", RailsMultisite::ConnectionManagement.current_db)
else
SiteSetting.s3_backup_bucket
end

25
spec/lib/backup_restore/s3_backup_store_spec.rb
View File

@ -13,7 +13,7 @@ describe BackupRestore::S3BackupStore do
@objects = []
def expected_prefix
Rails.configuration.multisite ? "backups/#{RailsMultisite::ConnectionManagement.current_db}/" : ""
"#{RailsMultisite::ConnectionManagement.current_db}/"
end
def check_context(context)
@ -102,26 +102,21 @@ describe BackupRestore::S3BackupStore do
def objects_with_prefix(context)
prefix = context.params[:prefix]
if prefix.blank?
@objects.reject { |obj| obj[:key].include?("backups/") }
else
@objects.select { |obj| obj[:key].start_with?(prefix) }
end
@objects.select { |obj| obj[:key].start_with?(prefix) }
end
def create_backups
@objects.clear
@objects << { key: "b.tar.gz", size: 17, last_modified: Time.parse("2018-09-13T15:10:00Z") }
@objects << { key: "a.tgz", size: 29, last_modified: Time.parse("2018-02-11T09:27:00Z") }
@objects << { key: "r.sql.gz", size: 11, last_modified: Time.parse("2017-12-20T03:48:00Z") }
@objects << { key: "no-backup.txt", size: 12, last_modified: Time.parse("2018-09-05T14:27:00Z") }
@objects << { key: "subfolder/c.tar.gz", size: 23, last_modified: Time.parse("2019-01-24T18:44:00Z") }
@objects << { key: "default/b.tar.gz", size: 17, last_modified: Time.parse("2018-09-13T15:10:00Z") }
@objects << { key: "default/a.tgz", size: 29, last_modified: Time.parse("2018-02-11T09:27:00Z") }
@objects << { key: "default/r.sql.gz", size: 11, last_modified: Time.parse("2017-12-20T03:48:00Z") }
@objects << { key: "default/no-backup.txt", size: 12, last_modified: Time.parse("2018-09-05T14:27:00Z") }
@objects << { key: "default/subfolder/c.tar.gz", size: 23, last_modified: Time.parse("2019-01-24T18:44:00Z") }
@objects << { key: "backups/second/multi-2.tar.gz", size: 19, last_modified: Time.parse("2018-11-27T03:16:54Z") }
@objects << { key: "backups/second/multi-1.tar.gz", size: 22, last_modified: Time.parse("2018-11-26T03:17:09Z") }
@objects << { key: "backups/second/subfolder/multi-3.tar.gz", size: 23, last_modified: Time.parse("2019-01-24T18:44:00Z") }
@objects << { key: "second/multi-2.tar.gz", size: 19, last_modified: Time.parse("2018-11-27T03:16:54Z") }
@objects << { key: "second/multi-1.tar.gz", size: 22, last_modified: Time.parse("2018-11-26T03:17:09Z") }
@objects << { key: "second/subfolder/multi-3.tar.gz", size: 23, last_modified: Time.parse("2019-01-24T18:44:00Z") }
end
def remove_backups