discourse/lib/jobs/exporter.rb

120 lines
3.7 KiB
Ruby

require_dependency 'export/json_encoder'
require_dependency 'export/export'
require_dependency 'import/import'
module Jobs
class Exporter < Jobs::Base
sidekiq_options :retry => false
def execute(args)
raise Import::ImportInProgressError if Import::is_import_running?
raise Export::ExportInProgressError if Export::is_export_running?
@format = args[:format] || :json
@output_base_filename = File.absolute_path( args[:filename] || File.join( Rails.root, 'tmp', "export-#{Time.now.strftime('%Y-%m-%d-%H%M%S')}" ) )
@output_base_filename = @output_base_filename[0...-3] if @output_base_filename[-3..-1] == '.gz'
@output_base_filename = @output_base_filename[0...-4] if @output_base_filename[-4..-1] == '.tar'
@user = args[:user_id] ? User.where(id: args[:user_id].to_i).first : nil
start_export
@encoder.write_schema_info( source: 'discourse', version: Export.current_schema_version )
ordered_models_for_export.each do |model|
log " #{model.table_name}"
column_info = model.columns
order_col = column_info.map(&:name).find {|x| x == 'id'} || order_columns_for(model)
@encoder.write_table(model.table_name, column_info) do |num_rows_written|
if order_col
model.connection.select_rows("select * from #{model.table_name} order by #{order_col} limit #{batch_size} offset #{num_rows_written}")
else
# Take the rows in the order the database returns them
log "WARNING: no order by clause is being used for #{model.name} (#{model.table_name}). Please update Jobs::Exporter order_columns_for for #{model.name}."
model.connection.select_rows("select * from #{model.table_name} limit #{batch_size} offset #{num_rows_written}")
end
end
end
"#{@output_base_filename}.tar.gz"
ensure
finish_export
end
def ordered_models_for_export
Export.models_included_in_export
end
def order_columns_for(model)
@order_columns_for_hash ||= {
'CategoryFeaturedTopic' => 'category_id, topic_id',
'PostOneboxRender' => 'post_id, onebox_render_id',
'PostReply' => 'post_id, reply_id',
'PostTiming' => 'topic_id, post_number, user_id',
'TopicUser' => 'topic_id, user_id',
'View' => 'parent_id, parent_type, ip, viewed_at'
}
@order_columns_for_hash[model.name]
end
def batch_size
1000
end
def start_export
if @format == :json
@encoder = Export::JsonEncoder.new
else
raise Export::FormatInvalidError
end
Export.set_export_started
Discourse.enable_maintenance_mode
end
def finish_export
if @encoder
@encoder.finish
create_tar_file
@encoder.cleanup_temp
end
ensure
Export.set_export_is_not_running
Discourse.disable_maintenance_mode
send_notification
end
def create_tar_file
filenames = @encoder.filenames
FileUtils.cd( File.dirname(filenames.first) ) do
`tar cvf #{@output_base_filename}.tar #{File.basename(filenames.first)}`
end
FileUtils.cd( File.join(Rails.root, 'public') ) do
Upload.find_each do |upload|
`tar rvf #{@output_base_filename}.tar #{upload.url[1..-1]}` unless upload.url[0,4] == 'http'
end
end
`gzip #{@output_base_filename}.tar`
true
end
def send_notification
SystemMessage.new(@user).create('export_succeeded') if @user
true
end
def log(*args)
puts args
args.each do |arg|
Rails.logger.info "#{Time.now.to_formatted_s(:db)}: [EXPORTER] #{arg}"
end
true
end
end
end