FIX: remove post upload record creation inside 'find_missing_uploads' method.

This commit is contained in:
Vinoth Kannan 2019-07-19 01:44:08 +05:30
parent 9cdc059a99
commit ad04ce9f43
4 changed files with 33 additions and 10 deletions

View File

@ -955,10 +955,7 @@ class Post < ActiveRecord::Base
upload_id = Upload.where(sha1: sha1).pluck(:id).first if sha1.present?
upload_id ||= yield(post, src, path, sha1)
if upload_id.present?
attributes = { post_id: post.id, upload_id: upload_id }
PostUpload.create!(attributes) unless PostUpload.exists?(attributes)
else
if upload_id.blank?
missing_uploads << src
missing_post_uploads[post.id] << src
end

View File

@ -82,7 +82,7 @@ class S3Inventory
def list_missing_post_uploads
log "Listing missing post uploads..."
missing = Post.find_missing_uploads(include_local_upload: false) do |_, _, _, sha1|
missing = Post.find_missing_uploads(include_local_upload: false) do |post, _, _, sha1|
next if sha1.blank?
upload_id = nil
@ -105,6 +105,7 @@ class S3Inventory
)
upload.save!(validate: false)
upload_id = upload.id
post.link_post_uploads
rescue Aws::S3::Errors::NotFound
next
end

View File

@ -82,8 +82,8 @@ describe "S3Inventory" do
it "should backfill etags to uploads table correctly" do
files = [
["#{Discourse.store.absolute_base_url}/original/0184537a4f419224404d013414e913a4f56018f2.jpg", "defcaac0b4aca535c284e95f30d608d0"],
["#{Discourse.store.absolute_base_url}/original/0789fbf5490babc68326b9cec90eeb0d6590db05.png", "25c02eaceef4cb779fc17030d33f7f06"]
["#{Discourse.store.absolute_base_url}/original/1X/0184537a4f419224404d013414e913a4f56018f2.jpg", "defcaac0b4aca535c284e95f30d608d0"],
["#{Discourse.store.absolute_base_url}/original/1X/0789fbf5490babc68326b9cec90eeb0d6590db05.png", "25c02eaceef4cb779fc17030d33f7f06"]
]
files.each { |file| Fabricate(:upload, url: file[0]) }
@ -97,4 +97,29 @@ describe "S3Inventory" do
expect(Upload.by_users.order(:url).pluck(:url, :etag)).to eq(files)
end
it "should recover missing uploads correctly" do
freeze_time
CSV.foreach(csv_filename, headers: false) do |row|
Fabricate(:upload, url: File.join(Discourse.store.absolute_base_url, row[S3Inventory::CSV_KEY_INDEX]), etag: row[S3Inventory::CSV_ETAG_INDEX], created_at: 2.days.ago)
end
upload = Upload.last
etag = upload.etag
post = Fabricate(:post, raw: "![](#{upload.url})")
post.link_post_uploads
upload.delete
inventory.expects(:download_inventory_files_to_tmp_directory)
inventory.expects(:decompress_inventory_files)
inventory.expects(:files).returns([{ key: "Key", filename: "#{csv_filename}.gz" }]).times(2)
output = capture_stdout do
inventory.backfill_etags_and_list_missing
end
expect(output).to eq("Listing missing post uploads...\n0 post uploads are missing.\n")
expect(post.uploads.first.etag).to eq(etag)
end
end

View File

@ -1,3 +1,3 @@
"abc","original/0184537a4f419224404d013414e913a4f56018f2.jpg","defcaac0b4aca535c284e95f30d608d0"
"abc","original/050afc0ab01debe8cf48fd2ce50fbbf5eb072815.jpg","0cdc623af39cde0adb382670a6dc702a"
"abc","original/0789fbf5490babc68326b9cec90eeb0d6590db05.png","25c02eaceef4cb779fc17030d33f7f06"
"abc","original/1X/0184537a4f419224404d013414e913a4f56018f2.jpg","defcaac0b4aca535c284e95f30d608d0"
"abc","original/1X/050afc0ab01debe8cf48fd2ce50fbbf5eb072815.jpg","0cdc623af39cde0adb382670a6dc702a"
"abc","original/1X/0789fbf5490babc68326b9cec90eeb0d6590db05.png","25c02eaceef4cb779fc17030d33f7f06"

1 abc original/0184537a4f419224404d013414e913a4f56018f2.jpg original/1X/0184537a4f419224404d013414e913a4f56018f2.jpg defcaac0b4aca535c284e95f30d608d0
2 abc original/050afc0ab01debe8cf48fd2ce50fbbf5eb072815.jpg original/1X/050afc0ab01debe8cf48fd2ce50fbbf5eb072815.jpg 0cdc623af39cde0adb382670a6dc702a
3 abc original/0789fbf5490babc68326b9cec90eeb0d6590db05.png original/1X/0789fbf5490babc68326b9cec90eeb0d6590db05.png 25c02eaceef4cb779fc17030d33f7f06