From 85768cfb1cfca7ec2342e0d21b1aebd2fddfd758 Mon Sep 17 00:00:00 2001 From: Roman Rizzi Date: Fri, 24 Feb 2023 09:11:58 -0300 Subject: [PATCH] FEATURE: Classify posts looking for NSFW images --- lib/modules/nsfw/entry_point.rb | 2 +- .../jobs/regular/evaluate_post_uploads.rb | 16 +-- .../jobs/regular/toxicity_classify_post.rb | 2 +- lib/shared/flag_manager.rb | 9 +- spec/lib/modules/nsfw/entry_point_spec.rb | 33 ++++-- .../jobs/regular/evaluate_post_uploads.rb | 32 ------ .../regular/evaluate_post_uploads_spec.rb | 100 ++++++++++++++++++ spec/support/nsfw_inference_stubs.rb | 5 +- 8 files changed, 146 insertions(+), 53 deletions(-) delete mode 100644 spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb create mode 100644 spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads_spec.rb diff --git a/lib/modules/nsfw/entry_point.rb b/lib/modules/nsfw/entry_point.rb index 11cda25e..07dd7a44 100644 --- a/lib/modules/nsfw/entry_point.rb +++ b/lib/modules/nsfw/entry_point.rb @@ -11,7 +11,7 @@ module DiscourseAI def inject_into(plugin) nsfw_detection_cb = Proc.new do |post| - if SiteSetting.ai_nsfw_detection_enabled + if SiteSetting.ai_nsfw_detection_enabled && post.uploads.present? Jobs.enqueue(:evaluate_post_uploads, post_id: post.id) end end diff --git a/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb b/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb index ecdd0497..0077d662 100644 --- a/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb +++ b/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb @@ -3,15 +3,19 @@ module Jobs class EvaluatePostUploads < ::Jobs::Base def execute(args) - upload = Upload.find_by_id(args[:upload_id]) + return unless SiteSetting.ai_nsfw_detection_enabled + return if (post_id = args[:post_id]).blank? - return unless upload + post = Post.includes(:uploads).find_by_id(post_id) + return if post.nil? || post.uploads.empty? - result = DiscourseAI::NSFW::Evaluation.new.perform(upload) + nsfw_evaluation = DiscourseAI::NSFW::Evaluation.new - # FIXME(roman): This is a simplistic action just to create - # the basic flow. We'll introduce flagging capabilities in the future. - upload.destroy! if result[:verdict] + image_uploads = post.uploads.select { |upload| FileHelper.is_supported_image?(upload.url) } + + results = image_uploads.map { |upload| nsfw_evaluation.perform(upload) } + + DiscourseAI::FlagManager.new(post).flag! if results.any? { |r| r[:verdict] } end end end diff --git a/lib/modules/toxicity/jobs/regular/toxicity_classify_post.rb b/lib/modules/toxicity/jobs/regular/toxicity_classify_post.rb index 01f1d946..3db16428 100644 --- a/lib/modules/toxicity/jobs/regular/toxicity_classify_post.rb +++ b/lib/modules/toxicity/jobs/regular/toxicity_classify_post.rb @@ -8,7 +8,7 @@ module ::Jobs post_id = args[:post_id] return if post_id.blank? - post = Post.includes(:user).find_by(id: post_id, post_type: Post.types[:regular]) + post = Post.find_by(id: post_id, post_type: Post.types[:regular]) return if post&.raw.blank? ::DiscourseAI::Toxicity::PostClassifier.new.classify!(post) diff --git a/lib/shared/flag_manager.rb b/lib/shared/flag_manager.rb index 5f4be18f..405964f0 100644 --- a/lib/shared/flag_manager.rb +++ b/lib/shared/flag_manager.rb @@ -13,7 +13,14 @@ module ::DiscourseAI end def flag! - PostActionCreator.create(@flagger, @object, :inappropriate, reason: @reasons) + PostActionCreator.new( + @flagger, + @object, + PostActionType.types[:inappropriate], + reason: @reasons, + queue_for_review: true, + ).perform + @object.publish_change_to_clients! :acted end end diff --git a/spec/lib/modules/nsfw/entry_point_spec.rb b/spec/lib/modules/nsfw/entry_point_spec.rb index 8a454121..9d466931 100644 --- a/spec/lib/modules/nsfw/entry_point_spec.rb +++ b/spec/lib/modules/nsfw/entry_point_spec.rb @@ -6,18 +6,17 @@ describe DiscourseAI::NSFW::EntryPoint do fab!(:user) { Fabricate(:user) } describe "registering event callbacks" do + fab!(:image_upload) { Fabricate(:upload) } + let(:raw_with_upload) { "A public post with an image.\n![](#{image_upload.short_path})" } + + before { SiteSetting.ai_nsfw_detection_enabled = true } + context "when creating a post" do let(:creator) do - PostCreator.new( - user, - raw: "this is the new content for my topic", - title: "this is my new topic title", - ) + PostCreator.new(user, raw: raw_with_upload, title: "this is my new topic title") end it "queues a job on create if sentiment analysis is enabled" do - SiteSetting.ai_nsfw_detection_enabled = true - expect { creator.create }.to change(Jobs::EvaluatePostUploads.jobs, :size).by(1) end @@ -26,6 +25,13 @@ describe DiscourseAI::NSFW::EntryPoint do expect { creator.create }.not_to change(Jobs::EvaluatePostUploads.jobs, :size) end + + it "does nothing if the post has no uploads" do + creator_2 = + PostCreator.new(user, raw: "this is a test", title: "this is my new topic title") + + expect { creator_2.create }.not_to change(Jobs::EvaluatePostUploads.jobs, :size) + end end context "when editing a post" do @@ -33,9 +39,7 @@ describe DiscourseAI::NSFW::EntryPoint do let(:revisor) { PostRevisor.new(post) } it "queues a job on update if sentiment analysis is enabled" do - SiteSetting.ai_nsfw_detection_enabled = true - - expect { revisor.revise!(user, raw: "This is my new test") }.to change( + expect { revisor.revise!(user, raw: raw_with_upload) }.to change( Jobs::EvaluatePostUploads.jobs, :size, ).by(1) @@ -44,7 +48,14 @@ describe DiscourseAI::NSFW::EntryPoint do it "does nothing if sentiment analysis is disabled" do SiteSetting.ai_nsfw_detection_enabled = false - expect { revisor.revise!(user, raw: "This is my new test") }.not_to change( + expect { revisor.revise!(user, raw: raw_with_upload) }.not_to change( + Jobs::EvaluatePostUploads.jobs, + :size, + ) + end + + it "does nothing if the new raw has no uploads" do + expect { revisor.revise!(user, raw: "this is a test") }.not_to change( Jobs::EvaluatePostUploads.jobs, :size, ) diff --git a/spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb b/spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb deleted file mode 100644 index ec80c3ef..00000000 --- a/spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" -require_relative "../../../../../support/nsfw_inference_stubs" - -describe Jobs::EvaluatePostUploads do - fab!(:image) { Fabricate(:s3_image_upload) } - - describe "#execute" do - before { SiteSetting.ai_nsfw_inference_service_api_endpoint = "http://test.com" } - - context "when we conclude content is NSFW" do - before { NSFWInferenceStubs.positive(image) } - - it "deletes the upload" do - subject.execute(upload_id: image.id) - - expect { image.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when we conclude content is not NSFW" do - before { NSFWInferenceStubs.negative(image) } - - it "does nothing" do - subject.execute(upload_id: image.id) - - expect(image.reload).to be_present - end - end - end -end diff --git a/spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads_spec.rb b/spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads_spec.rb new file mode 100644 index 00000000..38b51859 --- /dev/null +++ b/spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require "rails_helper" +require_relative "../../../../../support/nsfw_inference_stubs" + +describe Jobs::EvaluatePostUploads do + describe "#execute" do + before do + SiteSetting.ai_nsfw_detection_enabled = true + SiteSetting.ai_nsfw_inference_service_api_endpoint = "http://test.com" + end + + fab!(:upload_1) { Fabricate(:s3_image_upload) } + fab!(:post) { Fabricate(:post, uploads: [upload_1]) } + + describe "scenarios where we return early without doing anything" do + before { NSFWInferenceStubs.positive(upload_1) } + + it "does nothing when ai_toxicity_enabled is disabled" do + SiteSetting.ai_nsfw_detection_enabled = false + + subject.execute({ post_id: post.id }) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + + it "does nothing if there's no arg called post_id" do + subject.execute({}) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + + it "does nothing if no post match the given id" do + subject.execute({ post_id: nil }) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + + it "does nothing if the post has no uploads" do + post_no_uploads = Fabricate(:post) + + subject.execute({ post_id: post_no_uploads.id }) + + expect(ReviewableFlaggedPost.where(target: post_no_uploads).count).to be_zero + end + + it "does nothing if the upload is not an image" do + SiteSetting.authorized_extensions = "pdf" + upload_1.update!(original_filename: "test.pdf", url: "test.pdf") + + subject.execute({ post_id: post.id }) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + end + + context "when the post has one upload" do + context "when we conclude content is NSFW" do + before { NSFWInferenceStubs.positive(upload_1) } + + it "flags and hides the post" do + subject.execute({ post_id: post.id }) + + expect(ReviewableFlaggedPost.where(target: post).count).to eq(1) + expect(post.reload.hidden?).to eq(true) + end + end + + context "when we conclude content is safe" do + before { NSFWInferenceStubs.negative(upload_1) } + + it "does nothing" do + subject.execute({ post_id: post.id }) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + end + end + + context "when the post has multiple uploads" do + fab!(:upload_2) { Fabricate(:upload) } + + before { post.uploads << upload_2 } + + context "when we conclude content is NSFW" do + before do + NSFWInferenceStubs.negative(upload_1) + NSFWInferenceStubs.positive(upload_2) + end + + it "flags and hides the post if at least one upload is considered NSFW" do + subject.execute({ post_id: post.id }) + + expect(ReviewableFlaggedPost.where(target: post).count).to eq(1) + expect(post.reload.hidden?).to eq(true) + end + end + end + end +end diff --git a/spec/support/nsfw_inference_stubs.rb b/spec/support/nsfw_inference_stubs.rb index 08730f57..49edbd82 100644 --- a/spec/support/nsfw_inference_stubs.rb +++ b/spec/support/nsfw_inference_stubs.rb @@ -7,7 +7,10 @@ class NSFWInferenceStubs end def upload_url(upload) - Discourse.store.cdn_url(upload.url) + upload_url = Discourse.store.cdn_url(upload.url) + upload_url = "#{Discourse.base_url_no_prefix}#{upload_url}" if upload_url.starts_with?("/") + + upload_url end def positive_result(model)