PERF: Speed up `migrate_to_s3` rake task.

* Prioritizes non-image uploads
* Does one remap per upload instead of 3 remaps previously
* Every 100 uploads migrated, do 2 remaps which fixes broken
  URLs
* Exclude email_logs table from remap
This commit is contained in:
Guo Xiang Tan 2018-11-08 16:37:19 +08:00
parent 57f92ac808
commit 7290145641
1 changed files with 49 additions and 39 deletions

View File

@ -232,50 +232,60 @@ def migrate_to_s3
search_logs
post_search_data
notifications
email_logs
}
# Migrate all uploads
Upload.where.not(sha1: nil)
.where("url NOT LIKE '#{s3.absolute_base_url}%'")
.find_each do |upload|
# remove invalid uploads
if upload.url.blank?
upload.destroy!
next
end
# store the old url
from = upload.url
# retrieve the path to the local file
path = local.path_for(upload)
# make sure the file exists locally
if !path || !File.exists?(path)
putc "X"
next
end
file_uploads = Upload.where.not(sha1: nil).where("url NOT LIKE '#{s3.absolute_base_url}%'")
image_uploads = file_uploads.where("lower(extension) NOT IN (?)", FileHelper.supported_images.to_a)
begin
file = File.open(path)
content_type = `file --mime-type -b #{path}`.strip
to = s3.store_upload(file, upload, content_type)
rescue
putc "X"
next
ensure
file&.close
[image_uploads, file_uploads].each do |uploads|
upload.find_in_batches(batch_size: 100) do |batch|
batch.each do |upload|
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
# remove invalid uploads
if upload.url.blank?
upload.destroy!
next
end
# store the old url
from = upload.url
# retrieve the path to the local file
path = local.path_for(upload)
# make sure the file exists locally
if !path || !File.exists?(path)
puts "#{from} does not exist locally"
next
end
begin
file = File.open(path)
content_type = `file --mime-type -b #{path}`.strip
to = s3.store_upload(file, upload, content_type)
rescue => e
puts "Encountered an error while migrating #{upload.url}: #{e.class}: #{e.message}"
next
ensure
file&.close
end
# remap the URL
DbHelper.remap(from, to, exclude_tables: exclude_tables)
upload.optimized_images.destroy_all
puts "Migrating #{from} --> #{to} took #{Process.clock_gettime(Process::CLOCK_MONOTONIC) - now} seconds"
end
[
Discourse.asset_host,
Discourse.base_url_no_prefix
].each do |from|
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
from = "#{from}#{SiteSetting.Upload.s3_base_url}"
to = SiteSetting.s3_cdn_url
DbHelper.remap(from, to, exclude_tables: exclude_tables)
puts "Remapping #{from} --> #{to} took #{Process.clock_gettime(Process::CLOCK_MONOTONIC) - now} seconds"
end
end
# remap the URL
[
[UrlHelper.absolute(from), Discourse.store.cdn_url(to)],
[UrlHelper.absolute_without_cdn(from), Discourse.store.cdn_url(to)],
[from, to],
].each do |from_url, to_url|
DbHelper.remap(from_url, to_url, exclude_tables: exclude_tables)
end
upload.optimized_images.destroy_all
putc "."
end
end