FEATURE: Classify posts looking for NSFW images

lib/modules/nsfw/entry_point.rb

@@ -11,7 +11,7 @@ module DiscourseAI
       def inject_into(plugin)
         nsfw_detection_cb =
           Proc.new do |post|
-            if SiteSetting.ai_nsfw_detection_enabled
+            if SiteSetting.ai_nsfw_detection_enabled && post.uploads.present?
               Jobs.enqueue(:evaluate_post_uploads, post_id: post.id)
             end
           end

lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb

@@ -3,15 +3,19 @@
 module Jobs
   class EvaluatePostUploads < ::Jobs::Base
     def execute(args)
-      upload = Upload.find_by_id(args[:upload_id])
+      return unless SiteSetting.ai_nsfw_detection_enabled
+      return if (post_id = args[:post_id]).blank?
 
-      return unless upload
+      post = Post.includes(:uploads).find_by_id(post_id)
+      return if post.nil? || post.uploads.empty?
 
-      result = DiscourseAI::NSFW::Evaluation.new.perform(upload)
+      nsfw_evaluation = DiscourseAI::NSFW::Evaluation.new
 
-      # FIXME(roman): This is a simplistic action just to create
-      # the basic flow. We'll introduce flagging capabilities in the future.
-      upload.destroy! if result[:verdict]
+      image_uploads = post.uploads.select { |upload| FileHelper.is_supported_image?(upload.url) }
+
+      results = image_uploads.map { |upload| nsfw_evaluation.perform(upload) }
+
+      DiscourseAI::FlagManager.new(post).flag! if results.any? { |r| r[:verdict] }
     end
   end
 end

lib/modules/toxicity/jobs/regular/toxicity_classify_post.rb

@@ -8,7 +8,7 @@ module ::Jobs
       post_id = args[:post_id]
       return if post_id.blank?
 
-      post = Post.includes(:user).find_by(id: post_id, post_type: Post.types[:regular])
+      post = Post.find_by(id: post_id, post_type: Post.types[:regular])
       return if post&.raw.blank?
 
       ::DiscourseAI::Toxicity::PostClassifier.new.classify!(post)

lib/shared/flag_manager.rb

@@ -13,7 +13,14 @@ module ::DiscourseAI
     end
 
     def flag!
-      PostActionCreator.create(@flagger, @object, :inappropriate, reason: @reasons)
+      PostActionCreator.new(
+        @flagger,
+        @object,
+        PostActionType.types[:inappropriate],
+        reason: @reasons,
+        queue_for_review: true,
+      ).perform
+
       @object.publish_change_to_clients! :acted
     end
   end

spec/lib/modules/nsfw/entry_point_spec.rb

@@ -6,18 +6,17 @@ describe DiscourseAI::NSFW::EntryPoint do
   fab!(:user) { Fabricate(:user) }
 
   describe "registering event callbacks" do
+    fab!(:image_upload) { Fabricate(:upload) }
+    let(:raw_with_upload) { "A public post with an image.\n![](#{image_upload.short_path})" }
+
+    before { SiteSetting.ai_nsfw_detection_enabled = true }
+
     context "when creating a post" do
       let(:creator) do
-        PostCreator.new(
-          user,
-          raw: "this is the new content for my topic",
-          title: "this is my new topic title",
-        )
+        PostCreator.new(user, raw: raw_with_upload, title: "this is my new topic title")
       end
 
       it "queues a job on create if sentiment analysis is enabled" do
-        SiteSetting.ai_nsfw_detection_enabled = true
-
         expect { creator.create }.to change(Jobs::EvaluatePostUploads.jobs, :size).by(1)
       end
 
@@ -26,6 +25,13 @@ describe DiscourseAI::NSFW::EntryPoint do
 
         expect { creator.create }.not_to change(Jobs::EvaluatePostUploads.jobs, :size)
       end
+
+      it "does nothing if the post has no uploads" do
+        creator_2 =
+          PostCreator.new(user, raw: "this is a test", title: "this is my new topic title")
+
+        expect { creator_2.create }.not_to change(Jobs::EvaluatePostUploads.jobs, :size)
+      end
     end
 
     context "when editing a post" do
@@ -33,9 +39,7 @@ describe DiscourseAI::NSFW::EntryPoint do
       let(:revisor) { PostRevisor.new(post) }
 
       it "queues a job on update if sentiment analysis is enabled" do
-        SiteSetting.ai_nsfw_detection_enabled = true
-
-        expect { revisor.revise!(user, raw: "This is my new test") }.to change(
+        expect { revisor.revise!(user, raw: raw_with_upload) }.to change(
           Jobs::EvaluatePostUploads.jobs,
           :size,
         ).by(1)
@@ -44,7 +48,14 @@ describe DiscourseAI::NSFW::EntryPoint do
       it "does nothing if sentiment analysis is disabled" do
         SiteSetting.ai_nsfw_detection_enabled = false
 
-        expect { revisor.revise!(user, raw: "This is my new test") }.not_to change(
+        expect { revisor.revise!(user, raw: raw_with_upload) }.not_to change(
+          Jobs::EvaluatePostUploads.jobs,
+          :size,
+        )
+      end
+
+      it "does nothing if the new raw has no uploads" do
+        expect { revisor.revise!(user, raw: "this is a test") }.not_to change(
           Jobs::EvaluatePostUploads.jobs,
           :size,
         )

spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb

@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-require "rails_helper"
-require_relative "../../../../../support/nsfw_inference_stubs"
-
-describe Jobs::EvaluatePostUploads do
-  fab!(:image) { Fabricate(:s3_image_upload) }
-
-  describe "#execute" do
-    before { SiteSetting.ai_nsfw_inference_service_api_endpoint = "http://test.com" }
-
-    context "when we conclude content is NSFW" do
-      before { NSFWInferenceStubs.positive(image) }
-
-      it "deletes the upload" do
-        subject.execute(upload_id: image.id)
-
-        expect { image.reload }.to raise_error(ActiveRecord::RecordNotFound)
-      end
-    end
-
-    context "when we conclude content is not NSFW" do
-      before { NSFWInferenceStubs.negative(image) }
-
-      it "does nothing" do
-        subject.execute(upload_id: image.id)
-
-        expect(image.reload).to be_present
-      end
-    end
-  end
-end

spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads_spec.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+require_relative "../../../../../support/nsfw_inference_stubs"
+
+describe Jobs::EvaluatePostUploads do
+  describe "#execute" do
+    before do
+      SiteSetting.ai_nsfw_detection_enabled = true
+      SiteSetting.ai_nsfw_inference_service_api_endpoint = "http://test.com"
+    end
+
+    fab!(:upload_1) { Fabricate(:s3_image_upload) }
+    fab!(:post) { Fabricate(:post, uploads: [upload_1]) }
+
+    describe "scenarios where we return early without doing anything" do
+      before { NSFWInferenceStubs.positive(upload_1) }
+
+      it "does nothing when ai_toxicity_enabled is disabled" do
+        SiteSetting.ai_nsfw_detection_enabled = false
+
+        subject.execute({ post_id: post.id })
+
+        expect(ReviewableFlaggedPost.where(target: post).count).to be_zero
+      end
+
+      it "does nothing if there's no arg called post_id" do
+        subject.execute({})
+
+        expect(ReviewableFlaggedPost.where(target: post).count).to be_zero
+      end
+
+      it "does nothing if no post match the given id" do
+        subject.execute({ post_id: nil })
+
+        expect(ReviewableFlaggedPost.where(target: post).count).to be_zero
+      end
+
+      it "does nothing if the post has no uploads" do
+        post_no_uploads = Fabricate(:post)
+
+        subject.execute({ post_id: post_no_uploads.id })
+
+        expect(ReviewableFlaggedPost.where(target: post_no_uploads).count).to be_zero
+      end
+
+      it "does nothing if the upload is not an image" do
+        SiteSetting.authorized_extensions = "pdf"
+        upload_1.update!(original_filename: "test.pdf", url: "test.pdf")
+
+        subject.execute({ post_id: post.id })
+
+        expect(ReviewableFlaggedPost.where(target: post).count).to be_zero
+      end
+    end
+
+    context "when the post has one upload" do
+      context "when we conclude content is NSFW" do
+        before { NSFWInferenceStubs.positive(upload_1) }
+
+        it "flags and hides the post" do
+          subject.execute({ post_id: post.id })
+
+          expect(ReviewableFlaggedPost.where(target: post).count).to eq(1)
+          expect(post.reload.hidden?).to eq(true)
+        end
+      end
+
+      context "when we conclude content is safe" do
+        before { NSFWInferenceStubs.negative(upload_1) }
+
+        it "does nothing" do
+          subject.execute({ post_id: post.id })
+
+          expect(ReviewableFlaggedPost.where(target: post).count).to be_zero
+        end
+      end
+    end
+
+    context "when the post has multiple uploads" do
+      fab!(:upload_2) { Fabricate(:upload) }
+
+      before { post.uploads << upload_2 }
+
+      context "when we conclude content is NSFW" do
+        before do
+          NSFWInferenceStubs.negative(upload_1)
+          NSFWInferenceStubs.positive(upload_2)
+        end
+
+        it "flags and hides the post if at least one upload is considered NSFW" do
+          subject.execute({ post_id: post.id })
+
+          expect(ReviewableFlaggedPost.where(target: post).count).to eq(1)
+          expect(post.reload.hidden?).to eq(true)
+        end
+      end
+    end
+  end
+end

spec/support/nsfw_inference_stubs.rb

@@ -7,7 +7,10 @@ class NSFWInferenceStubs
     end
 
     def upload_url(upload)
-      Discourse.store.cdn_url(upload.url)
+      upload_url = Discourse.store.cdn_url(upload.url)
+      upload_url = "#{Discourse.base_url_no_prefix}#{upload_url}" if upload_url.starts_with?("/")
+
+      upload_url
     end
 
     def positive_result(model)