From a65a9a85d5aa0b180cafbb603478bee451867c5d Mon Sep 17 00:00:00 2001 From: Gerhard Schlager Date: Tue, 9 Jul 2019 13:57:31 +0200 Subject: [PATCH] FEATURE: Remap uploads during restore when S3 or CDN changes In order for this to work the Backuper stores a couple of site settings in the new backup_metadata table, because the old setting values might not be available on restore anymore. --- app/models/backup_metadata.rb | 16 +++++++ .../20190704133453_create_backup_metadata.rb | 10 ++++ lib/backup_restore/backuper.rb | 12 +++++ lib/backup_restore/restorer.rb | 47 ++++++++++++++++--- 4 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 app/models/backup_metadata.rb create mode 100644 db/migrate/20190704133453_create_backup_metadata.rb diff --git a/app/models/backup_metadata.rb b/app/models/backup_metadata.rb new file mode 100644 index 00000000000..6c273997739 --- /dev/null +++ b/app/models/backup_metadata.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class BackupMetadata < ActiveRecord::Base + def self.value_for(name) + where(name: name).pluck(:value).first + end +end + +# == Schema Information +# +# Table name: backup_metadata +# +# id :bigint not null, primary key +# name :string not null +# value :string +# diff --git a/db/migrate/20190704133453_create_backup_metadata.rb b/db/migrate/20190704133453_create_backup_metadata.rb new file mode 100644 index 00000000000..b24fd3d0725 --- /dev/null +++ b/db/migrate/20190704133453_create_backup_metadata.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class CreateBackupMetadata < ActiveRecord::Migration[5.2] + def change + create_table :backup_metadata do |t| + t.string :name, null: false + t.string :value + end + end +end diff --git a/lib/backup_restore/backuper.rb b/lib/backup_restore/backuper.rb index 4c2b9f2aaad..c813d87f91a 100644 --- a/lib/backup_restore/backuper.rb +++ b/lib/backup_restore/backuper.rb @@ -32,6 +32,8 @@ module BackupRestore ensure_directory_exists(@tmp_directory) ensure_directory_exists(@archive_directory) + update_metadata + ### READ-ONLY / START ### enable_readonly_mode @@ -117,6 +119,16 @@ module BackupRestore BackupRestore.mark_as_running! end + def update_metadata + log "Updating metadata..." + BackupMetadata.delete_all + BackupMetadata.create!(name: "base_url", value: Discourse.base_url) + BackupMetadata.create!(name: "cdn_url", value: Discourse.asset_host) + BackupMetadata.create!(name: "s3_base_url", value: SiteSetting.Upload.enable_s3_uploads ? SiteSetting.Upload.s3_base_url : nil) + BackupMetadata.create!(name: "s3_cdn_url", value: SiteSetting.Upload.enable_s3_uploads ? SiteSetting.Upload.s3_cdn_url : nil) + BackupMetadata.create!(name: "db_name", value: RailsMultisite::ConnectionManagement.current_db) + end + def enable_readonly_mode return if @readonly_mode_was_enabled log "Enabling readonly mode..." diff --git a/lib/backup_restore/restorer.rb b/lib/backup_restore/restorer.rb index a5ea6a03163..9586e4d32b9 100644 --- a/lib/backup_restore/restorer.rb +++ b/lib/backup_restore/restorer.rb @@ -434,7 +434,7 @@ module BackupRestore FileUtils.mkdir_p("uploads") tmp_uploads_path = Dir.glob(File.join(@tmp_directory, "uploads", "*")).first - previous_db_name = File.basename(tmp_uploads_path) + previous_db_name = BackupMetadata.value_for("db_name") || File.basename(tmp_uploads_path) current_db_name = RailsMultisite::ConnectionManagement.current_db optimized_images_exist = File.exist?(File.join(tmp_uploads_path, 'optimized')) @@ -443,10 +443,7 @@ module BackupRestore failure_message: "Failed to restore uploads." ) - if previous_db_name != current_db_name - log "Remapping uploads..." - DbHelper.remap("uploads/#{previous_db_name}", "uploads/#{current_db_name}") - end + remap_uploads(previous_db_name, current_db_name) if SiteSetting.Upload.enable_s3_uploads migrate_to_s3 @@ -458,6 +455,45 @@ module BackupRestore end end + def remap_uploads(previous_db_name, current_db_name) + log "Remapping uploads..." + + if (old_base_url = BackupMetadata.value_for("base_url")) && old_base_url != Discourse.base_url + DbHelper.remap(old_base_url, Discourse.base_url) + end + + current_s3_base_url = SiteSetting.Upload.enable_s3_uploads ? SiteSetting.Upload.s3_base_url : nil + if (old_s3_base_url = BackupMetadata.value_for("s3_base_url")) && old_base_url != current_s3_base_url + DbHelper.remap("#{old_s3_base_url}/", "/uploads/#{current_db_name}/") + end + + current_s3_cdn_url = SiteSetting.Upload.enable_s3_uploads ? SiteSetting.Upload.s3_cdn_url : nil + if (old_s3_cdn_url = BackupMetadata.value_for("s3_cdn_url")) && old_s3_cdn_url != current_s3_cdn_url + base_url = SiteSetting.Upload.enable_s3_uploads ? SiteSetting.Upload.s3_cdn_url : Discourse.base_url + DbHelper.remap("#{old_s3_cdn_url}/", UrlHelper.schemaless("#{base_url}/uploads/#{current_db_name}/")) + + old_host = URI.parse(old_s3_cdn_url).host + new_host = URI.parse(base_url.presence || Discourse.base_url).host + DbHelper.remap(old_host, new_host) + end + + if (old_cdn_url = BackupMetadata.value_for("cdn_url")) && old_cdn_url != Discourse.asset_host + base_url = SiteSetting.Upload.enable_s3_uploads ? SiteSetting.Upload.s3_base_url : Discourse.base_url + DbHelper.remap("#{old_cdn_url}/", UrlHelper.schemaless("#{base_url}/")) + + old_host = URI.parse(old_cdn_url).host + new_host = URI.parse(base_url.presence || Discourse.base_url).host + DbHelper.remap(old_host, new_host) + end + + if previous_db_name != current_db_name + DbHelper.remap("uploads/#{previous_db_name}", "uploads/#{current_db_name}") + end + + rescue => ex + log "Something went wrong while remapping uploads.", ex + end + def migrate_to_s3 log "Migrating uploads to S3..." ENV["SKIP_FAILED"] = "1" @@ -589,5 +625,4 @@ module BackupRestore end end - end