From 87cd4701b86b90bfa6fc01f54aa479e74e42a1dc Mon Sep 17 00:00:00 2001
From: Vinoth Kannan <vinothkannan@vinkas.com>
Date: Thu, 9 May 2019 05:11:15 +0530
Subject: [PATCH] FEATURE: option to skip posts with ignored missing uploads

---
 app/models/post.rb                                   |  9 ++++++++-
 ...ng_uploads_ignored_index_to_post_custom_fields.rb |  5 +++++
 lib/tasks/posts.rake                                 |  4 ++++
 spec/tasks/posts_spec.rb                             | 12 ++++++++++++
 4 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20190508193900_add_missing_uploads_ignored_index_to_post_custom_fields.rb

diff --git a/app/models/post.rb b/app/models/post.rb
index 65187aed0fa..d5d56b726a0 100644
--- a/app/models/post.rb
+++ b/app/models/post.rb
@@ -64,10 +64,12 @@ class Post < ActiveRecord::Base
   BROKEN_IMAGES     ||= "broken_images".freeze
   DOWNLOADED_IMAGES ||= "downloaded_images".freeze
   MISSING_UPLOADS ||= "missing uploads".freeze
+  MISSING_UPLOADS_IGNORED ||= "missing uploads ignored".freeze
 
   SHORT_POST_CHARS ||= 1200
 
   register_custom_field_type(MISSING_UPLOADS, :json)
+  register_custom_field_type(MISSING_UPLOADS_IGNORED, :boolean)
 
   scope :private_posts_for_user, ->(user) {
     where("posts.topic_id IN (SELECT topic_id
@@ -918,8 +920,13 @@ class Post < ActiveRecord::Base
     PostCustomField.where(name: Post::MISSING_UPLOADS).delete_all
     missing_uploads = []
     missing_post_uploads = {}
+    query = Post
+              .have_uploads
+              .joins("LEFT JOIN post_custom_fields ON posts.id = post_custom_fields.post_id AND post_custom_fields.name = '#{Post::MISSING_UPLOADS_IGNORED}'")
+              .where("post_custom_fields.id IS NULL")
+              .select(:id, :cooked)
 
-    Post.have_uploads.select(:id, :cooked).find_in_batches do |posts|
+    query.find_in_batches do |posts|
       ids = posts.pluck(:id)
       sha1s = Upload.joins(:post_uploads).where("post_uploads.post_id >= ? AND post_uploads.post_id <= ?", ids.min, ids.max).pluck(:sha1)
 
diff --git a/db/migrate/20190508193900_add_missing_uploads_ignored_index_to_post_custom_fields.rb b/db/migrate/20190508193900_add_missing_uploads_ignored_index_to_post_custom_fields.rb
new file mode 100644
index 00000000000..2672e171a3b
--- /dev/null
+++ b/db/migrate/20190508193900_add_missing_uploads_ignored_index_to_post_custom_fields.rb
@@ -0,0 +1,5 @@
+class AddMissingUploadsIgnoredIndexToPostCustomFields < ActiveRecord::Migration[5.2]
+  def change
+    add_index :post_custom_fields, :post_id, unique: true, where: "name = 'missing uploads ignored'", name: "index_post_id_where_missing_uploads_ignored"
+  end
+end
diff --git a/lib/tasks/posts.rake b/lib/tasks/posts.rake
index 241dbbff9cf..e4adc8012a2 100644
--- a/lib/tasks/posts.rake
+++ b/lib/tasks/posts.rake
@@ -447,3 +447,7 @@ task 'posts:missing_uploads' => :environment do
     puts "#{missing[:post_uploads].count} of #{Post.count} posts are affected.", ""
   end
 end
+
+desc 'Finds missing post upload records from cooked HTML content'
+task 'posts:missing_uploads' => :environment do
+end
diff --git a/spec/tasks/posts_spec.rb b/spec/tasks/posts_spec.rb
index 24f12fdf5a1..0188fae27ec 100644
--- a/spec/tasks/posts_spec.rb
+++ b/spec/tasks/posts_spec.rb
@@ -70,5 +70,17 @@ RSpec.describe "Post rake tasks" do
       post.reload
       expect(post.custom_fields[Post::MISSING_UPLOADS]).to eq([url])
     end
+
+    it 'should skip all the posts with "ignored" custom field' do
+      post = Fabricate(:post, raw: "A sample post <img src='#{url}'>")
+      post.custom_fields[Post::MISSING_UPLOADS_IGNORED] = true
+      post.save_custom_fields
+      upload.destroy!
+
+      Rake::Task['posts:missing_uploads'].invoke
+
+      post.reload
+      expect(post.custom_fields[Post::MISSING_UPLOADS]).to be_nil
+    end
   end
 end