DEV: wrap find_missing_uploads method in distributed mutex
And skip posts with deleted topics.\ne8fafbc123170dd1f7d2a8adea4e7810585d3e76
This commit is contained in:
parent
a5a4f9dc1a
commit
56ada8374f
|
@ -919,53 +919,57 @@ class Post < ActiveRecord::Base
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.find_missing_uploads(include_local_upload: true)
|
def self.find_missing_uploads(include_local_upload: true)
|
||||||
PostCustomField.where(name: Post::MISSING_UPLOADS).delete_all
|
|
||||||
missing_uploads = []
|
missing_uploads = []
|
||||||
missing_post_uploads = {}
|
missing_post_uploads = {}
|
||||||
query = Post
|
count = 0
|
||||||
.have_uploads
|
|
||||||
.joins("LEFT JOIN post_custom_fields ON posts.id = post_custom_fields.post_id AND post_custom_fields.name = '#{Post::MISSING_UPLOADS_IGNORED}'")
|
|
||||||
.where("post_custom_fields.id IS NULL")
|
|
||||||
.select(:id, :cooked)
|
|
||||||
|
|
||||||
query.find_in_batches do |posts|
|
DistributedMutex.synchronize("find_missing_uploads") do
|
||||||
ids = posts.pluck(:id)
|
PostCustomField.where(name: Post::MISSING_UPLOADS).delete_all
|
||||||
sha1s = Upload.joins(:post_uploads).where("post_uploads.post_id >= ? AND post_uploads.post_id <= ?", ids.min, ids.max).pluck(:sha1)
|
query = Post
|
||||||
|
.have_uploads
|
||||||
|
.joins(:topic)
|
||||||
|
.joins("LEFT JOIN post_custom_fields ON posts.id = post_custom_fields.post_id AND post_custom_fields.name = '#{Post::MISSING_UPLOADS_IGNORED}'")
|
||||||
|
.where("post_custom_fields.id IS NULL")
|
||||||
|
.select(:id, :cooked)
|
||||||
|
|
||||||
posts.each do |post|
|
query.find_in_batches do |posts|
|
||||||
post.each_upload_url do |src, path, sha1|
|
ids = posts.pluck(:id)
|
||||||
next if sha1.present? && sha1s.include?(sha1)
|
sha1s = Upload.joins(:post_uploads).where("post_uploads.post_id >= ? AND post_uploads.post_id <= ?", ids.min, ids.max).pluck(:sha1)
|
||||||
|
|
||||||
missing_post_uploads[post.id] ||= []
|
posts.each do |post|
|
||||||
|
post.each_upload_url do |src, path, sha1|
|
||||||
|
next if sha1.present? && sha1s.include?(sha1)
|
||||||
|
|
||||||
if missing_uploads.include?(src)
|
missing_post_uploads[post.id] ||= []
|
||||||
missing_post_uploads[post.id] << src
|
|
||||||
next
|
|
||||||
end
|
|
||||||
|
|
||||||
upload_id = nil
|
if missing_uploads.include?(src)
|
||||||
upload_id = Upload.where(sha1: sha1).pluck(:id).first if sha1.present?
|
missing_post_uploads[post.id] << src
|
||||||
upload_id ||= yield(post, src, path, sha1)
|
next
|
||||||
|
end
|
||||||
|
|
||||||
if upload_id.present?
|
upload_id = nil
|
||||||
attributes = { post_id: post.id, upload_id: upload_id }
|
upload_id = Upload.where(sha1: sha1).pluck(:id).first if sha1.present?
|
||||||
PostUpload.create!(attributes) unless PostUpload.exists?(attributes)
|
upload_id ||= yield(post, src, path, sha1)
|
||||||
else
|
|
||||||
missing_uploads << src
|
if upload_id.present?
|
||||||
missing_post_uploads[post.id] << src
|
attributes = { post_id: post.id, upload_id: upload_id }
|
||||||
|
PostUpload.create!(attributes) unless PostUpload.exists?(attributes)
|
||||||
|
else
|
||||||
|
missing_uploads << src
|
||||||
|
missing_post_uploads[post.id] << src
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
|
||||||
|
|
||||||
count = 0
|
missing_post_uploads = missing_post_uploads.reject do |post_id, uploads|
|
||||||
missing_post_uploads = missing_post_uploads.reject do |post_id, uploads|
|
if uploads.present?
|
||||||
if uploads.present?
|
PostCustomField.create!(post_id: post_id, name: Post::MISSING_UPLOADS, value: uploads.to_json)
|
||||||
PostCustomField.create!(post_id: post_id, name: Post::MISSING_UPLOADS, value: uploads.to_json)
|
count += uploads.count
|
||||||
count += uploads.count
|
end
|
||||||
|
|
||||||
|
uploads.empty?
|
||||||
end
|
end
|
||||||
|
|
||||||
uploads.empty?
|
|
||||||
end
|
end
|
||||||
|
|
||||||
{ uploads: missing_uploads, post_uploads: missing_post_uploads, count: count }
|
{ uploads: missing_uploads, post_uploads: missing_post_uploads, count: count }
|
||||||
|
|
Loading…
Reference in New Issue