473 lines
14 KiB
Ruby
473 lines
14 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require "mini_mime"
|
|
require "file_store/s3_store"
|
|
|
|
module BackupRestore
|
|
class Backuper
|
|
attr_reader :success, :store
|
|
|
|
def initialize(user_id, opts = {})
|
|
@user_id = user_id
|
|
@client_id = opts[:client_id]
|
|
@publish_to_message_bus = opts[:publish_to_message_bus] || false
|
|
@with_uploads = opts[:with_uploads] == false ? false : include_uploads?
|
|
@filename_override = opts[:filename]
|
|
@ticket = opts[:ticket]
|
|
|
|
initialize_state
|
|
end
|
|
|
|
def run
|
|
ensure_no_operation_is_running
|
|
ensure_we_have_a_user
|
|
|
|
log "[STARTED]"
|
|
log "'#{@user.username}' has started the backup!"
|
|
|
|
mark_backup_as_running
|
|
|
|
listen_for_shutdown_signal
|
|
|
|
ensure_directory_exists(@tmp_directory)
|
|
ensure_directory_exists(@archive_directory)
|
|
|
|
update_metadata
|
|
dump_public_schema
|
|
|
|
log "Finalizing backup..."
|
|
|
|
@with_uploads ? create_archive : move_dump_backup
|
|
upload_archive
|
|
|
|
after_create_hook
|
|
rescue SystemExit
|
|
log "Backup process was cancelled!"
|
|
rescue Exception => ex
|
|
log "EXCEPTION: " + ex.message
|
|
log ex.backtrace.join("\n")
|
|
else
|
|
@success = true
|
|
@backup_filename
|
|
ensure
|
|
delete_old
|
|
delete_prior_to_n_days
|
|
clean_up
|
|
notify_user
|
|
log "Finished!"
|
|
publish_completion
|
|
end
|
|
|
|
protected
|
|
|
|
def ensure_no_operation_is_running
|
|
raise BackupRestore::OperationRunningError if BackupRestore.is_operation_running?
|
|
end
|
|
|
|
def ensure_we_have_a_user
|
|
raise Discourse::InvalidParameters.new(:user_id) unless @user
|
|
end
|
|
|
|
def get_parameterized_title
|
|
SiteSetting.title.parameterize.presence || "discourse"
|
|
end
|
|
|
|
def initialize_state
|
|
@success = false
|
|
@user = User.find_by(id: @user_id)
|
|
@logs = []
|
|
@store = BackupRestore::BackupStore.create
|
|
@current_db = RailsMultisite::ConnectionManagement.current_db
|
|
@timestamp = Time.now.strftime("%Y-%m-%d-%H%M%S")
|
|
@tmp_directory = File.join(Rails.root, "tmp", "backups", @current_db, @timestamp)
|
|
@dump_filename = File.join(@tmp_directory, BackupRestore::DUMP_FILE)
|
|
@archive_directory = BackupRestore::LocalBackupStore.base_directory(db: @current_db)
|
|
filename = @filename_override || "#{get_parameterized_title}-#{@timestamp}"
|
|
@archive_basename =
|
|
File.join(
|
|
@archive_directory,
|
|
"#{filename}-#{BackupRestore::VERSION_PREFIX}#{BackupRestore.current_version}",
|
|
)
|
|
|
|
@backup_filename =
|
|
if @with_uploads
|
|
"#{File.basename(@archive_basename)}.tar.gz"
|
|
else
|
|
"#{File.basename(@archive_basename)}.sql.gz"
|
|
end
|
|
end
|
|
|
|
def listen_for_shutdown_signal
|
|
BackupRestore.clear_shutdown_signal!
|
|
|
|
Thread.new do
|
|
Thread.current.name = "shutdown_wait"
|
|
|
|
RailsMultisite::ConnectionManagement.with_connection(@current_db) do
|
|
while BackupRestore.is_operation_running?
|
|
exit if BackupRestore.should_shutdown?
|
|
sleep 0.1
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def mark_backup_as_running
|
|
log "Marking backup as running..."
|
|
BackupRestore.mark_as_running!
|
|
end
|
|
|
|
def update_metadata
|
|
log "Updating metadata..."
|
|
BackupMetadata.delete_all
|
|
BackupMetadata.create!(name: "base_url", value: Discourse.base_url)
|
|
BackupMetadata.create!(name: "cdn_url", value: Discourse.asset_host)
|
|
BackupMetadata.create!(
|
|
name: "s3_base_url",
|
|
value: SiteSetting.Upload.enable_s3_uploads ? SiteSetting.Upload.s3_base_url : nil,
|
|
)
|
|
BackupMetadata.create!(
|
|
name: "s3_cdn_url",
|
|
value: SiteSetting.Upload.enable_s3_uploads ? SiteSetting.Upload.s3_cdn_url : nil,
|
|
)
|
|
BackupMetadata.create!(
|
|
name: "db_name",
|
|
value: RailsMultisite::ConnectionManagement.current_db,
|
|
)
|
|
BackupMetadata.create!(name: "multisite", value: Rails.configuration.multisite)
|
|
end
|
|
|
|
def dump_public_schema
|
|
log "Dumping the public schema of the database..."
|
|
|
|
logs = Queue.new
|
|
pg_dump_running = true
|
|
|
|
Thread.new do
|
|
RailsMultisite::ConnectionManagement.establish_connection(db: @current_db)
|
|
while pg_dump_running
|
|
message = logs.pop.strip
|
|
log(message) unless message.blank?
|
|
end
|
|
end
|
|
|
|
IO.popen("#{pg_dump_command} 2>&1") do |pipe|
|
|
begin
|
|
while line = pipe.readline
|
|
logs << line
|
|
end
|
|
rescue EOFError
|
|
# finished reading...
|
|
ensure
|
|
pg_dump_running = false
|
|
logs << ""
|
|
end
|
|
end
|
|
|
|
raise "pg_dump failed" unless $?.success?
|
|
end
|
|
|
|
def pg_dump_command
|
|
db_conf = BackupRestore.database_configuration
|
|
|
|
password_argument = "PGPASSWORD='#{db_conf.password}'" if db_conf.password.present?
|
|
host_argument = "--host=#{db_conf.host}" if db_conf.host.present?
|
|
port_argument = "--port=#{db_conf.port}" if db_conf.port.present?
|
|
username_argument = "--username=#{db_conf.username}" if db_conf.username.present?
|
|
|
|
[
|
|
password_argument, # pass the password to pg_dump (if any)
|
|
"pg_dump", # the pg_dump command
|
|
"--schema=public", # only public schema
|
|
"-T public.pg_*", # exclude tables and views whose name starts with "pg_"
|
|
"--file='#{@dump_filename}'", # output to the dump.sql file
|
|
"--no-owner", # do not output commands to set ownership of objects
|
|
"--no-privileges", # prevent dumping of access privileges
|
|
"--verbose", # specifies verbose mode
|
|
"--compress=4", # Compression level of 4
|
|
host_argument, # the hostname to connect to (if any)
|
|
port_argument, # the port to connect to (if any)
|
|
username_argument, # the username to connect as (if any)
|
|
db_conf.database, # the name of the database to dump
|
|
].join(" ")
|
|
end
|
|
|
|
def move_dump_backup
|
|
log "Finalizing database dump file: #{@backup_filename}"
|
|
|
|
archive_filename = File.join(@archive_directory, @backup_filename)
|
|
|
|
Discourse::Utils.execute_command(
|
|
"mv",
|
|
@dump_filename,
|
|
archive_filename,
|
|
failure_message: "Failed to move database dump file.",
|
|
)
|
|
|
|
remove_tmp_directory
|
|
end
|
|
|
|
def create_archive
|
|
log "Creating archive: #{@backup_filename}"
|
|
|
|
tar_filename = "#{@archive_basename}.tar"
|
|
|
|
log "Making sure archive does not already exist..."
|
|
Discourse::Utils.execute_command("rm", "-f", tar_filename)
|
|
Discourse::Utils.execute_command("rm", "-f", "#{tar_filename}.gz")
|
|
|
|
log "Creating empty archive..."
|
|
Discourse::Utils.execute_command(
|
|
"tar",
|
|
"--create",
|
|
"--file",
|
|
tar_filename,
|
|
"--files-from",
|
|
"/dev/null",
|
|
)
|
|
|
|
log "Archiving data dump..."
|
|
Discourse::Utils.execute_command(
|
|
"tar",
|
|
"--append",
|
|
"--dereference",
|
|
"--file",
|
|
tar_filename,
|
|
File.basename(@dump_filename),
|
|
failure_message: "Failed to archive data dump.",
|
|
chdir: File.dirname(@dump_filename),
|
|
)
|
|
|
|
add_local_uploads_to_archive(tar_filename)
|
|
add_remote_uploads_to_archive(tar_filename) if SiteSetting.Upload.enable_s3_uploads
|
|
|
|
remove_tmp_directory
|
|
|
|
log "Gzipping archive, this may take a while..."
|
|
Discourse::Utils.execute_command(
|
|
"gzip",
|
|
"-#{SiteSetting.backup_gzip_compression_level_for_uploads}",
|
|
tar_filename,
|
|
failure_message: "Failed to gzip archive.",
|
|
)
|
|
end
|
|
|
|
def include_uploads?
|
|
has_local_uploads? || SiteSetting.include_s3_uploads_in_backups
|
|
end
|
|
|
|
def local_uploads_directory
|
|
@local_uploads_directory ||= File.join(Rails.root, "public", Discourse.store.upload_path)
|
|
end
|
|
|
|
def has_local_uploads?
|
|
File.directory?(local_uploads_directory) && !Dir.empty?(local_uploads_directory)
|
|
end
|
|
|
|
def add_local_uploads_to_archive(tar_filename)
|
|
log "Archiving uploads..."
|
|
|
|
if has_local_uploads?
|
|
upload_directory = Discourse.store.upload_path
|
|
|
|
if SiteSetting.include_thumbnails_in_backups
|
|
exclude_optimized = ""
|
|
else
|
|
optimized_path = File.join(upload_directory, "optimized")
|
|
exclude_optimized = "--exclude=#{optimized_path}"
|
|
end
|
|
|
|
Discourse::Utils.execute_command(
|
|
"tar",
|
|
"--append",
|
|
"--dereference",
|
|
exclude_optimized,
|
|
"--file",
|
|
tar_filename,
|
|
upload_directory,
|
|
failure_message: "Failed to archive uploads.",
|
|
success_status_codes: [0, 1],
|
|
chdir: File.join(Rails.root, "public"),
|
|
)
|
|
else
|
|
log "No local uploads found. Skipping archiving of local uploads..."
|
|
end
|
|
end
|
|
|
|
def add_remote_uploads_to_archive(tar_filename)
|
|
if !SiteSetting.include_s3_uploads_in_backups
|
|
log "Skipping uploads stored on S3."
|
|
return
|
|
end
|
|
|
|
log "Downloading uploads from S3. This may take a while..."
|
|
|
|
store = FileStore::S3Store.new
|
|
upload_directory = Discourse.store.upload_path
|
|
count = 0
|
|
|
|
Upload.find_each do |upload|
|
|
next if upload.local?
|
|
filename = File.join(@tmp_directory, upload_directory, store.get_path_for_upload(upload))
|
|
|
|
begin
|
|
FileUtils.mkdir_p(File.dirname(filename))
|
|
store.download_file(upload, filename)
|
|
rescue StandardError => ex
|
|
log "Failed to download file with upload ID #{upload.id} from S3", ex
|
|
end
|
|
|
|
count += 1
|
|
log "#{count} files have already been downloaded. Still downloading..." if count % 500 == 0
|
|
end
|
|
|
|
log "Appending uploads to archive..."
|
|
Discourse::Utils.execute_command(
|
|
"tar",
|
|
"--append",
|
|
"--file",
|
|
tar_filename,
|
|
upload_directory,
|
|
failure_message: "Failed to append uploads to archive.",
|
|
success_status_codes: [0, 1],
|
|
chdir: @tmp_directory,
|
|
)
|
|
|
|
log "No uploads found on S3. Skipping archiving of uploads stored on S3..." if count == 0
|
|
end
|
|
|
|
def upload_archive
|
|
return unless store.remote?
|
|
|
|
log "Uploading archive..."
|
|
content_type = MiniMime.lookup_by_filename(@backup_filename).content_type
|
|
archive_path = File.join(@archive_directory, @backup_filename)
|
|
store.upload_file(@backup_filename, archive_path, content_type)
|
|
end
|
|
|
|
def after_create_hook
|
|
log "Executing the after_create_hook for the backup..."
|
|
DiscourseEvent.trigger(:backup_created)
|
|
end
|
|
|
|
def delete_old
|
|
return if Rails.env.development?
|
|
|
|
log "Deleting old backups..."
|
|
store.delete_old
|
|
rescue => ex
|
|
log "Something went wrong while deleting old backups.", ex
|
|
end
|
|
|
|
def delete_prior_to_n_days
|
|
return if Rails.env.development?
|
|
|
|
log "Deleting backups prior to n days..."
|
|
store.delete_prior_to_n_days
|
|
rescue => ex
|
|
log "Something went wrong while deleting backups prior to n days....", ex
|
|
end
|
|
|
|
def notify_user
|
|
return if success && @user.id == Discourse::SYSTEM_USER_ID
|
|
|
|
log "Notifying '#{@user.username}' of the end of the backup..."
|
|
status = success ? :backup_succeeded : :backup_failed
|
|
|
|
logs = Discourse::Utils.logs_markdown(@logs, user: @user)
|
|
post = SystemMessage.create_from_system_user(@user, status, logs: logs)
|
|
|
|
post.topic.invite_group(@user, Group[:admins]) if @user.id == Discourse::SYSTEM_USER_ID
|
|
rescue => ex
|
|
log "Something went wrong while notifying user.", ex
|
|
end
|
|
|
|
def clean_up
|
|
log "Cleaning stuff up..."
|
|
delete_uploaded_archive
|
|
remove_tar_leftovers
|
|
mark_backup_as_not_running
|
|
refresh_disk_space if success
|
|
end
|
|
|
|
def delete_uploaded_archive
|
|
return unless store.remote?
|
|
|
|
archive_path = File.join(@archive_directory, @backup_filename)
|
|
|
|
if File.exist?(archive_path)
|
|
log "Removing archive from local storage..."
|
|
File.delete(archive_path)
|
|
end
|
|
rescue => ex
|
|
log "Something went wrong while deleting uploaded archive from local storage.", ex
|
|
end
|
|
|
|
def refresh_disk_space
|
|
log "Refreshing disk stats..."
|
|
store.reset_cache
|
|
rescue => ex
|
|
log "Something went wrong while refreshing disk stats.", ex
|
|
end
|
|
|
|
def remove_tar_leftovers
|
|
log "Removing '.tar' leftovers..."
|
|
Dir["#{@archive_directory}/*.tar"].each { |filename| File.delete(filename) }
|
|
rescue => ex
|
|
log "Something went wrong while removing '.tar' leftovers.", ex
|
|
end
|
|
|
|
def remove_tmp_directory
|
|
log "Removing tmp '#{@tmp_directory}' directory..."
|
|
FileUtils.rm_rf(@tmp_directory) if Dir[@tmp_directory].present?
|
|
rescue => ex
|
|
log "Something went wrong while removing the following tmp directory: #{@tmp_directory}", ex
|
|
end
|
|
|
|
def mark_backup_as_not_running
|
|
log "Marking backup as finished..."
|
|
BackupRestore.mark_as_not_running!
|
|
rescue => ex
|
|
log "Something went wrong while marking backup as finished.", ex
|
|
end
|
|
|
|
def ensure_directory_exists(directory)
|
|
log "Making sure '#{directory}' exists..."
|
|
FileUtils.mkdir_p(directory)
|
|
end
|
|
|
|
def log(message, ex = nil)
|
|
timestamp = Time.now.strftime("%Y-%m-%d %H:%M:%S")
|
|
puts(message) if !Rails.env.test?
|
|
publish_log(message, timestamp)
|
|
save_log(message, timestamp)
|
|
Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n")) if ex
|
|
end
|
|
|
|
def publish_log(message, timestamp)
|
|
return unless @publish_to_message_bus
|
|
data = { timestamp: timestamp, operation: "backup", message: message }
|
|
MessageBus.publish(
|
|
BackupRestore::LOGS_CHANNEL,
|
|
data,
|
|
user_ids: [@user_id],
|
|
client_ids: [@client_id],
|
|
)
|
|
end
|
|
|
|
def save_log(message, timestamp)
|
|
@logs << "[#{timestamp}] #{message}"
|
|
end
|
|
|
|
def publish_completion
|
|
if success
|
|
log("[SUCCESS]")
|
|
DiscourseEvent.trigger(:backup_complete, logs: @logs, ticket: @ticket)
|
|
else
|
|
log("[FAILED]")
|
|
DiscourseEvent.trigger(:backup_failed, logs: @logs, ticket: @ticket)
|
|
end
|
|
end
|
|
end
|
|
end
|