Discource-C
/
discourse
mirror of https://github.com/discourse/discourse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4341 from tgxworld/pipe_dump_to_gzip 

FEATURE: Work with compressed version of `pg_dump` during backup and …
This commit is contained in:
Guo Xiang Tan 2016-07-26 10:43:39 +08:00 committed by GitHub
parent b5fbff947b 76e57ddef3
commit b335ccbd59
4 changed files with 89 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/backup_restore/backup_restore.rb
View File

@ -1,3 +1,4 @@
require "backup_restore/utils"
require "backup_restore/backuper"
require "backup_restore/restorer"

64
lib/backup_restore/backuper.rb
View File

@ -1,6 +1,7 @@
module BackupRestore
class Backuper
include BackupRestore::Utils
attr_reader :success
@ -42,8 +43,6 @@ module BackupRestore
log "Finalizing backup..."
update_dump
create_archive
after_create_hook
@ -84,7 +83,7 @@ module BackupRestore
@current_db = RailsMultisite::ConnectionManagement.current_db
@timestamp = Time.now.strftime("%Y-%m-%d-%H%M%S")
@tmp_directory = File.join(Rails.root, "tmp", "backups", @current_db, @timestamp)
@dump_filename = File.join(@tmp_directory, BackupRestore::DUMP_FILE)
@dump_filename = "#{File.join(@tmp_directory, BackupRestore::DUMP_FILE)}.gz"
@meta_filename = File.join(@tmp_directory, BackupRestore::METADATA_FILE)
@archive_directory = File.join(Rails.root, "public", "backups", @current_db)
@archive_basename = File.join(@archive_directory, "#{SiteSetting.title.parameterize}-#{@timestamp}")
@ -192,6 +191,7 @@ module BackupRestore
"--no-owner", # do not output commands to set ownership of objects
"--no-privileges", # prevent dumping of access privileges
"--verbose", # specifies verbose mode
"--compress=4", # Compression level of 4
host_argument, # the hostname to connect to (if any)
port_argument, # the port to connect to (if any)
username_argument, # the username to connect as (if any)
@ -199,59 +199,32 @@ module BackupRestore
].join(" ")
end
def update_dump
log "Updating dump for more awesomeness..."
`#{sed_command}`
end
def sed_command
# in order to limit the downtime when restoring as much as possible
# we force the restoration to happen in the "restore" schema
# during the restoration, this make sure we
# - drop the "restore" schema if it exists
# - create the "restore" schema
# - prepend the "restore" schema into the search_path
regexp = "SET search_path = public, pg_catalog;"
replacement = [ "DROP SCHEMA IF EXISTS restore CASCADE;",
"CREATE SCHEMA restore;",
"SET search_path = restore, public, pg_catalog;",
].join(" ")
# we only want to replace the VERY first occurence of the search_path command
expression = "1,/^#{regexp}$/s/#{regexp}/#{replacement}/"
# I tried to use the --in-place argument but it was SLOOOWWWWwwwwww
# so I output the result into another file and rename it back afterwards
[ "sed -e '#{expression}' < #{@dump_filename} > #{@dump_filename}.tmp",
"&&",
"mv #{@dump_filename}.tmp #{@dump_filename}",
].join(" ")
end
def create_archive
log "Creating archive: #{File.basename(@archive_basename)}.tar.gz"
tar_filename = "#{@archive_basename}.tar"
log "Making sure archive does not already exist..."
`rm -f #{tar_filename}`
`rm -f #{tar_filename}.gz`
execute_command("rm -f #{tar_filename}")
execute_command("rm -f #{tar_filename}.gz")
log "Creating empty archive..."
`tar --create --file #{tar_filename} --files-from /dev/null`
execute_command("tar --create --file #{tar_filename} --files-from /dev/null")
log "Archiving data dump..."
FileUtils.cd(File.dirname(@dump_filename)) do
`tar --append --dereference --file #{tar_filename} #{File.basename(@dump_filename)}`
FileUtils.cd(File.dirname("#{@dump_filename}")) do
execute_command(
"tar --append --dereference --file #{tar_filename} #{File.basename(@dump_filename)}",
"Failed to archive data dump."
)
end
log "Archiving metadata..."
FileUtils.cd(File.dirname(@meta_filename)) do
`tar --append --dereference --file #{tar_filename} #{File.basename(@meta_filename)}`
execute_command(
"tar --append --dereference --file #{tar_filename} #{File.basename(@meta_filename)}",
"Failed to archive metadata."
)
end
if @with_uploads
@ -259,14 +232,17 @@ module BackupRestore
log "Archiving uploads..."
FileUtils.cd(File.join(Rails.root, "public")) do
`tar --append --dereference --file #{tar_filename} #{upload_directory}`
execute_command(
"tar --append --dereference --file #{tar_filename} #{upload_directory}",
"Failed to archive uploads."
)
end
end
remove_tmp_directory
log "Gzipping archive, this may take a while..."
`gzip -5 #{tar_filename}`
execute_command("gzip -5 #{tar_filename}", "Failed to gzip archive.")
end
def after_create_hook

63
lib/backup_restore/restorer.rb
View File

@ -4,6 +4,7 @@ module BackupRestore
class FilenameMissingError < RuntimeError; end
class Restorer
include BackupRestore::Utils
attr_reader :success
@ -67,8 +68,13 @@ module BackupRestore
else
@success = true
ensure
notify_user rescue nil
clean_up
begin
notify_user
clean_up
rescue => ex
Rails.logger.error("#{ex}\n" + ex.backtrace.join("\n"))
end
@success ? log("[SUCCESS]") : log("[FAILED]")
end
@ -157,17 +163,26 @@ module BackupRestore
def copy_archive_to_tmp_directory
log "Copying archive to tmp directory..."
source = File.join(Backup.base_directory, @filename)
`cp '#{source}' '#{@archive_filename}'`
execute_command("cp '#{source}' '#{@archive_filename}'", "Failed to copy archive to tmp directory.")
end
def unzip_archive
log "Unzipping archive, this may take a while..."
FileUtils.cd(@tmp_directory) { `gzip --decompress '#{@archive_filename}'` }
FileUtils.cd(@tmp_directory) do
execute_command("gzip --decompress '#{@archive_filename}'", "Failed to unzip archive.")
end
end
def extract_metadata
log "Extracting metadata file..."
FileUtils.cd(@tmp_directory) { `tar --extract --file '#{@tar_filename}' #{BackupRestore::METADATA_FILE}` }
FileUtils.cd(@tmp_directory) do
execute_command(
"tar --extract --file '#{@tar_filename}' #{BackupRestore::METADATA_FILE}",
"Failed to extract metadata file."
)
end
@metadata = Oj.load_file(@meta_filename)
end
@ -182,7 +197,13 @@ module BackupRestore
def extract_dump
log "Extracting dump file..."
FileUtils.cd(@tmp_directory) { `tar --extract --file '#{@tar_filename}' #{BackupRestore::DUMP_FILE}` }
FileUtils.cd(@tmp_directory) do
execute_command(
"tar --extract --file '#{@tar_filename}' #{BackupRestore::DUMP_FILE}.gz",
"Failed to extract dump file."
)
end
end
def restore_dump
@ -201,7 +222,7 @@ module BackupRestore
end
end
IO.popen("#{psql_command} 2>&1") do |pipe|
IO.popen("gzip -d < #{@dump_filename}.gz | #{sed_command} | #{psql_command} 2>&1") do |pipe|
begin
while line = pipe.readline
logs << line
@ -229,7 +250,6 @@ module BackupRestore
[ password_argument, # pass the password to psql (if any)
"psql", # the psql command
"--dbname='#{db_conf.database}'", # connect to database *dbname*
"--file='#{@dump_filename}'", # read the dump
"--single-transaction", # all or nothing (also runs COPY commands faster)
host_argument, # the hostname to connect to (if any)
port_argument, # the port to connect to (if any)
@ -237,6 +257,28 @@ module BackupRestore
].join(" ")
end
def sed_command
# in order to limit the downtime when restoring as much as possible
# we force the restoration to happen in the "restore" schema
# during the restoration, this make sure we
# - drop the "restore" schema if it exists
# - create the "restore" schema
# - prepend the "restore" schema into the search_path
regexp = "SET search_path = public, pg_catalog;"
replacement = [ "DROP SCHEMA IF EXISTS restore CASCADE;",
"CREATE SCHEMA restore;",
"SET search_path = restore, public, pg_catalog;",
].join(" ")
# we only want to replace the VERY first occurence of the search_path command
expression = "1,/^#{regexp}$/s/#{regexp}/#{replacement}/"
"sed -e '#{expression}'"
end
def switch_schema!
log "Switching schemas... try reloading the site in 5 minutes, if successful, then reboot and restore is complete."
@ -279,7 +321,10 @@ module BackupRestore
if `tar --list --file '#{@tar_filename}' | grep 'uploads/'`.present?
log "Extracting uploads..."
FileUtils.cd(File.join(Rails.root, "public")) do
`tar --extract --keep-newer-files --file '#{@tar_filename}' uploads/`
execute_command(
"tar --extract --keep-newer-files --file '#{@tar_filename}' uploads/",
"Failed to extract uploads."
)
end
end
end

14
lib/backup_restore/utils.rb Normal file
View File

@ -0,0 +1,14 @@
module BackupRestore
module Utils
def execute_command(command, failure_message = "")
output = `#{command} 2>&1`
if !$?.success?
failure_message = "#{failure_message}\n" if !failure_message.blank?
raise "#{failure_message}#{output}"
end
output
end
end
end