diff --git a/config/settings.yml b/config/settings.yml index 61067bc4..dbcabbb8 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -65,7 +65,7 @@ plugins: - sentiment - emotion - ai_nsfw_live_detection_enabled: false + ai_nsfw_detection_enabled: false ai_nsfw_inference_service_api_endpoint: default: "" ai_nsfw_inference_service_api_key: diff --git a/lib/modules/nsfw/entry_point.rb b/lib/modules/nsfw/entry_point.rb index a2921feb..11cda25e 100644 --- a/lib/modules/nsfw/entry_point.rb +++ b/lib/modules/nsfw/entry_point.rb @@ -4,14 +4,20 @@ module DiscourseAI module NSFW class EntryPoint def load_files - require_relative "evaluation.rb" - require_relative "jobs/regular/evaluate_content.rb" + require_relative "evaluation" + require_relative "jobs/regular/evaluate_post_uploads" end def inject_into(plugin) - plugin.add_model_callback(Upload, :after_create) do - Jobs.enqueue(:evaluate_content, upload_id: self.id) - end + nsfw_detection_cb = + Proc.new do |post| + if SiteSetting.ai_nsfw_detection_enabled + Jobs.enqueue(:evaluate_post_uploads, post_id: post.id) + end + end + + plugin.on(:post_created, &nsfw_detection_cb) + plugin.on(:post_edited, &nsfw_detection_cb) end end end diff --git a/lib/modules/nsfw/jobs/regular/evaluate_content.rb b/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb similarity index 90% rename from lib/modules/nsfw/jobs/regular/evaluate_content.rb rename to lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb index 4976df83..ecdd0497 100644 --- a/lib/modules/nsfw/jobs/regular/evaluate_content.rb +++ b/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Jobs - class EvaluateContent < ::Jobs::Base + class EvaluatePostUploads < ::Jobs::Base def execute(args) upload = Upload.find_by_id(args[:upload_id]) diff --git a/lib/modules/sentiment/entry_point.rb b/lib/modules/sentiment/entry_point.rb index 2bba0f99..286b9dd6 100644 --- a/lib/modules/sentiment/entry_point.rb +++ b/lib/modules/sentiment/entry_point.rb @@ -3,8 +3,8 @@ module DiscourseAI module Sentiment class EntryPoint def load_files - require_relative "post_classifier.rb" - require_relative "jobs/regular/post_sentiment_analysis.rb" + require_relative "post_classifier" + require_relative "jobs/regular/post_sentiment_analysis" end def inject_into(plugin) diff --git a/lib/modules/sentiment/post_classifier.rb b/lib/modules/sentiment/post_classifier.rb index 72901592..301da2a2 100644 --- a/lib/modules/sentiment/post_classifier.rb +++ b/lib/modules/sentiment/post_classifier.rb @@ -3,10 +3,6 @@ module ::DiscourseAI module Sentiment class PostClassifier - SENTIMENT_LABELS = %w[anger disgust fear joy neutral sadness surprise] - - SENTIMENT_LABELS = %w[negative neutral positive] - def classify!(post) available_models.each do |model| classification = request_classification(post, model) diff --git a/lib/modules/toxicity/chat_message_classifier.rb b/lib/modules/toxicity/chat_message_classifier.rb index cb6b0898..f36f5b9f 100644 --- a/lib/modules/toxicity/chat_message_classifier.rb +++ b/lib/modules/toxicity/chat_message_classifier.rb @@ -3,25 +3,27 @@ module ::DiscourseAI module Toxicity class ChatMessageClassifier < Classifier - def content - @object.message + private + + def content(chat_message) + chat_message.message end - def store_classification + def store_classification(chat_message, classification) PluginStore.set( "toxicity", - "chat_message_#{@object.id}", + "chat_message_#{chat_message.id}", { - classification: @classification, + classification: classification, model: SiteSetting.ai_toxicity_inference_service_api_model, date: Time.now.utc, }, ) end - def flag! + def flag!(chat_message, _toxic_labels) Chat::ChatReviewQueue.new.flag_message( - @object, + chat_message, Guardian.new(flagger), ReviewableScore.types[:inappropriate], ) diff --git a/lib/modules/toxicity/classifier.rb b/lib/modules/toxicity/classifier.rb index ba8ca7e0..5a5e6fdf 100644 --- a/lib/modules/toxicity/classifier.rb +++ b/lib/modules/toxicity/classifier.rb @@ -13,47 +13,53 @@ module ::DiscourseAI sexual_explicit ] - def initialize(object) - @object = object + def classify!(target) + classification = request_classification(target) + + store_classification(target, classification) + + toxic_labels = filter_toxic_labels(classification) + + flag!(target, toxic_labels) if should_flag_based_on?(toxic_labels) end - def content + protected + + def flag!(_target, _toxic_labels) + raise NotImplemented end - def classify! - @classification = - ::DiscourseAI::InferenceManager.perform!( - "#{SiteSetting.ai_toxicity_inference_service_api_endpoint}/api/v1/classify", - SiteSetting.ai_toxicity_inference_service_api_model, - content, - SiteSetting.ai_toxicity_inference_service_api_key, - ) - store_classification - consider_flagging + def store_classification(_target, _classification) + raise NotImplemented end - def store_classification - end - - def automatic_flag_enabled? - SiteSetting.ai_toxicity_flag_automatically - end - - def consider_flagging - return unless automatic_flag_enabled? - @reasons = - CLASSIFICATION_LABELS.filter do |label| - @classification[label] >= SiteSetting.send("ai_toxicity_flag_threshold_#{label}") - end - - flag! unless @reasons.empty? + def content(_target) + raise NotImplemented end def flagger - User.find_by(id: -1) + Discourse.system_user end - def flag! + private + + def request_classification(target) + ::DiscourseAI::InferenceManager.perform!( + "#{SiteSetting.ai_toxicity_inference_service_api_endpoint}/api/v1/classify", + SiteSetting.ai_toxicity_inference_service_api_model, + content(target), + SiteSetting.ai_toxicity_inference_service_api_key, + ) + end + + def filter_toxic_labels(classification) + CLASSIFICATION_LABELS.filter do |label| + classification[label] >= SiteSetting.send("ai_toxicity_flag_threshold_#{label}") + end + end + + def should_flag_based_on?(toxic_labels) + SiteSetting.ai_toxicity_flag_automatically && toxic_labels.present? end end end diff --git a/lib/modules/toxicity/entry_point.rb b/lib/modules/toxicity/entry_point.rb index c207a362..b628ac2c 100644 --- a/lib/modules/toxicity/entry_point.rb +++ b/lib/modules/toxicity/entry_point.rb @@ -3,31 +3,26 @@ module DiscourseAI module Toxicity class EntryPoint def load_files - require_relative "event_handler.rb" - require_relative "classifier.rb" - require_relative "post_classifier.rb" - require_relative "chat_message_classifier.rb" + require_relative "scan_queue" + require_relative "classifier" + require_relative "post_classifier" + require_relative "chat_message_classifier" - require_relative "jobs/regular/toxicity_classify_post.rb" - require_relative "jobs/regular/toxicity_classify_chat_message.rb" + require_relative "jobs/regular/toxicity_classify_post" + require_relative "jobs/regular/toxicity_classify_chat_message" end def inject_into(plugin) - plugin.on(:post_created) do |post| - DiscourseAI::Toxicity::EventHandler.handle_post_async(post) - end + post_analysis_cb = Proc.new { |post| DiscourseAI::Toxicity::ScanQueue.enqueue_post(post) } - plugin.on(:post_edited) do |post| - DiscourseAI::Toxicity::EventHandler.handle_post_async(post) - end + plugin.on(:post_created, &post_analysis_cb) + plugin.on(:post_edited, &post_analysis_cb) - plugin.on(:chat_message_created) do |chat_message| - DiscourseAI::Toxicity::EventHandler.handle_chat_async(chat_message) - end + chat_message_analysis_cb = + Proc.new { |message| DiscourseAI::Toxicity::ScanQueue.enqueue_chat_message(message) } - plugin.on(:chat_message_edited) do |chat_message| - DiscourseAI::Toxicity::EventHandler.handle_chat_async(chat_message) - end + plugin.on(:chat_message_created, &chat_message_analysis_cb) + plugin.on(:chat_message_edited, &chat_message_analysis_cb) end end end diff --git a/lib/modules/toxicity/jobs/regular/toxicity_classify_chat_message.rb b/lib/modules/toxicity/jobs/regular/toxicity_classify_chat_message.rb index ecd6b11a..5c316456 100644 --- a/lib/modules/toxicity/jobs/regular/toxicity_classify_chat_message.rb +++ b/lib/modules/toxicity/jobs/regular/toxicity_classify_chat_message.rb @@ -1,17 +1,16 @@ # frozen_string_literal: true module ::Jobs - class ClassifyChatMessage < ::Jobs::Base + class ToxicityClassifyChatMessage < ::Jobs::Base def execute(args) return unless SiteSetting.ai_toxicity_enabled - chat_message_id = args[:chat_message_id] - return if chat_message_id.blank? + return if (chat_message_id = args[:chat_message_id]).blank? chat_message = ChatMessage.find_by(id: chat_message_id) return if chat_message&.message.blank? - ::DiscourseAI::Toxicity::ChatMessageClassifier.new(chat_message).classify! + ::DiscourseAI::Toxicity::ChatMessageClassifier.new.classify!(chat_message) end end end diff --git a/lib/modules/toxicity/jobs/regular/toxicity_classify_post.rb b/lib/modules/toxicity/jobs/regular/toxicity_classify_post.rb index bbb90447..01f1d946 100644 --- a/lib/modules/toxicity/jobs/regular/toxicity_classify_post.rb +++ b/lib/modules/toxicity/jobs/regular/toxicity_classify_post.rb @@ -8,10 +8,10 @@ module ::Jobs post_id = args[:post_id] return if post_id.blank? - post = Post.find_by(id: post_id, post_type: Post.types[:regular]) + post = Post.includes(:user).find_by(id: post_id, post_type: Post.types[:regular]) return if post&.raw.blank? - ::DiscourseAI::Toxicity::PostClassifier.new(post).classify! + ::DiscourseAI::Toxicity::PostClassifier.new.classify!(post) end end end diff --git a/lib/modules/toxicity/post_classifier.rb b/lib/modules/toxicity/post_classifier.rb index 02e1994d..65413a4c 100644 --- a/lib/modules/toxicity/post_classifier.rb +++ b/lib/modules/toxicity/post_classifier.rb @@ -3,23 +3,25 @@ module ::DiscourseAI module Toxicity class PostClassifier < Classifier - def content - object.post_number == 1 ? "#{object.topic.title}\n#{object.raw}" : object.raw + private + + def content(post) + post.post_number == 1 ? "#{post.topic.title}\n#{post.raw}" : post.raw end - def store_classification + def store_classification(post, classification) PostCustomField.create!( - post_id: @object.id, + post_id: post.id, name: "toxicity", value: { - classification: @classification, + classification: classification, model: SiteSetting.ai_toxicity_inference_service_api_model, }.to_json, ) end - def flag! - DiscourseAI::FlagManager.new(@object, reasons: @reasons).flag! + def flag!(target, toxic_labels) + ::DiscourseAI::FlagManager.new(target, reasons: toxic_labels).flag! end end end diff --git a/lib/modules/toxicity/event_handler.rb b/lib/modules/toxicity/scan_queue.rb similarity index 73% rename from lib/modules/toxicity/event_handler.rb rename to lib/modules/toxicity/scan_queue.rb index fd78c024..c890f3f6 100644 --- a/lib/modules/toxicity/event_handler.rb +++ b/lib/modules/toxicity/scan_queue.rb @@ -2,14 +2,14 @@ module ::DiscourseAI module Toxicity - class EventHandler + class ScanQueue class << self - def handle_post_async(post) + def enqueue_post(post) return if bypass?(post) Jobs.enqueue(:toxicity_classify_post, post_id: post.id) end - def handle_chat_async(chat_message) + def enqueue_chat_message(chat_message) return if bypass?(chat_message) Jobs.enqueue(:toxicity_classify_chat_message, chat_message_id: chat_message.id) end @@ -19,7 +19,7 @@ module ::DiscourseAI end def group_bypass?(user) - user.groups.pluck(:id).intersection(SiteSetting.disorder_groups_bypass_map).present? + user.groups.pluck(:id).intersection(SiteSetting.ai_toxicity_groups_bypass_map).present? end end end diff --git a/plugin.rb b/plugin.rb index 39b2f35c..0330d805 100644 --- a/plugin.rb +++ b/plugin.rb @@ -9,13 +9,18 @@ enabled_site_setting :discourse_ai_enabled -require_relative "lib/shared/inference_manager" - -require_relative "lib/modules/nsfw/entry_point" -require_relative "lib/modules/toxicity/entry_point" -require_relative "lib/modules/sentiment/entry_point" - after_initialize do + module ::DiscourseAI + PLUGIN_NAME = "discourse-ai" + end + + require_relative "lib/shared/inference_manager" + require_relative "lib/shared/flag_manager" + + require_relative "lib/modules/nsfw/entry_point" + require_relative "lib/modules/toxicity/entry_point" + require_relative "lib/modules/sentiment/entry_point" + modules = [ DiscourseAI::NSFW::EntryPoint.new, DiscourseAI::Toxicity::EntryPoint.new, @@ -26,8 +31,4 @@ after_initialize do a_module.load_files a_module.inject_into(self) end - - module ::DiscourseAI - PLUGIN_NAME = "discourse-ai" - end end diff --git a/spec/lib/modules/nsfw/entry_point_spec.rb b/spec/lib/modules/nsfw/entry_point_spec.rb new file mode 100644 index 00000000..8a454121 --- /dev/null +++ b/spec/lib/modules/nsfw/entry_point_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe DiscourseAI::NSFW::EntryPoint do + fab!(:user) { Fabricate(:user) } + + describe "registering event callbacks" do + context "when creating a post" do + let(:creator) do + PostCreator.new( + user, + raw: "this is the new content for my topic", + title: "this is my new topic title", + ) + end + + it "queues a job on create if sentiment analysis is enabled" do + SiteSetting.ai_nsfw_detection_enabled = true + + expect { creator.create }.to change(Jobs::EvaluatePostUploads.jobs, :size).by(1) + end + + it "does nothing if sentiment analysis is disabled" do + SiteSetting.ai_nsfw_detection_enabled = false + + expect { creator.create }.not_to change(Jobs::EvaluatePostUploads.jobs, :size) + end + end + + context "when editing a post" do + fab!(:post) { Fabricate(:post, user: user) } + let(:revisor) { PostRevisor.new(post) } + + it "queues a job on update if sentiment analysis is enabled" do + SiteSetting.ai_nsfw_detection_enabled = true + + expect { revisor.revise!(user, raw: "This is my new test") }.to change( + Jobs::EvaluatePostUploads.jobs, + :size, + ).by(1) + end + + it "does nothing if sentiment analysis is disabled" do + SiteSetting.ai_nsfw_detection_enabled = false + + expect { revisor.revise!(user, raw: "This is my new test") }.not_to change( + Jobs::EvaluatePostUploads.jobs, + :size, + ) + end + end + end +end diff --git a/spec/lib/modules/nsfw/evaluation_spec.rb b/spec/lib/modules/nsfw/evaluation_spec.rb index 320f932d..c5d36b9c 100644 --- a/spec/lib/modules/nsfw/evaluation_spec.rb +++ b/spec/lib/modules/nsfw/evaluation_spec.rb @@ -6,7 +6,7 @@ require_relative "../../../support/nsfw_inference_stubs" describe DiscourseAI::NSFW::Evaluation do before do SiteSetting.ai_nsfw_inference_service_api_endpoint = "http://test.com" - SiteSetting.ai_nsfw_live_detection_enabled = true + SiteSetting.ai_nsfw_detection_enabled = true end fab!(:image) { Fabricate(:s3_image_upload) } diff --git a/spec/lib/modules/nsfw/jobs/regular/evaluate_content_spec.rb b/spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb similarity index 95% rename from spec/lib/modules/nsfw/jobs/regular/evaluate_content_spec.rb rename to spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb index 3e7e1cf2..ec80c3ef 100644 --- a/spec/lib/modules/nsfw/jobs/regular/evaluate_content_spec.rb +++ b/spec/lib/modules/nsfw/jobs/regular/evaluate_post_uploads.rb @@ -3,7 +3,7 @@ require "rails_helper" require_relative "../../../../../support/nsfw_inference_stubs" -describe Jobs::EvaluateContent do +describe Jobs::EvaluatePostUploads do fab!(:image) { Fabricate(:s3_image_upload) } describe "#execute" do diff --git a/spec/lib/modules/toxicity/chat_message_classifier_spec.rb b/spec/lib/modules/toxicity/chat_message_classifier_spec.rb new file mode 100644 index 00000000..a40f9842 --- /dev/null +++ b/spec/lib/modules/toxicity/chat_message_classifier_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require "rails_helper" +require_relative "../../../support/toxicity_inference_stubs" + +describe DiscourseAI::Toxicity::ChatMessageClassifier do + before { SiteSetting.ai_toxicity_flag_automatically = true } + + fab!(:chat_message) { Fabricate(:chat_message) } + + describe "#classify!" do + it "creates a reviewable when the post is classified as toxic" do + ToxicityInferenceStubs.stub_chat_message_classification(chat_message, toxic: true) + + subject.classify!(chat_message) + + expect(ReviewableChatMessage.where(target: chat_message).count).to eq(1) + end + + it "doesn't create a reviewable if the post is not classified as toxic" do + ToxicityInferenceStubs.stub_chat_message_classification(chat_message, toxic: false) + + subject.classify!(chat_message) + + expect(ReviewableChatMessage.where(target: chat_message).count).to be_zero + end + + it "doesn't create a reviewable if flagging is disabled" do + SiteSetting.ai_toxicity_flag_automatically = false + ToxicityInferenceStubs.stub_chat_message_classification(chat_message, toxic: true) + + subject.classify!(chat_message) + + expect(ReviewableChatMessage.where(target: chat_message).count).to be_zero + end + + it "stores the classification in a custom field" do + ToxicityInferenceStubs.stub_chat_message_classification(chat_message, toxic: false) + + subject.classify!(chat_message) + store_row = PluginStore.get("toxicity", "chat_message_#{chat_message.id}").deep_symbolize_keys + + expect(store_row[:classification]).to eq(ToxicityInferenceStubs.civilized_response) + expect(store_row[:model]).to eq(SiteSetting.ai_toxicity_inference_service_api_model) + expect(store_row[:date]).to be_present + end + end +end diff --git a/spec/lib/modules/toxicity/entry_point_spec.rb b/spec/lib/modules/toxicity/entry_point_spec.rb new file mode 100644 index 00000000..1c7eecfb --- /dev/null +++ b/spec/lib/modules/toxicity/entry_point_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe DiscourseAI::Toxicity::EntryPoint do + fab!(:user) { Fabricate(:user) } + + describe "registering event callbacks" do + before { SiteSetting.ai_toxicity_enabled = true } + + context "when creating a post" do + let(:creator) do + PostCreator.new( + user, + raw: "this is the new content for my topic", + title: "this is my new topic title", + ) + end + + it "queues a job on post creation" do + SiteSetting.ai_toxicity_enabled = true + + expect { creator.create }.to change(Jobs::ToxicityClassifyPost.jobs, :size).by(1) + end + end + + context "when editing a post" do + fab!(:post) { Fabricate(:post, user: user) } + let(:revisor) { PostRevisor.new(post) } + + it "queues a job on post update" do + expect { revisor.revise!(user, raw: "This is my new test") }.to change( + Jobs::ToxicityClassifyPost.jobs, + :size, + ).by(1) + end + end + + context "when creating a chat message" do + let(:public_chat_channel) { Fabricate(:chat_channel) } + let(:creator) do + Chat::ChatMessageCreator.new( + chat_channel: public_chat_channel, + user: user, + content: "This is my new test", + ) + end + + it "queues a job when creating a chat message" do + expect { creator.create }.to change(Jobs::ToxicityClassifyChatMessage.jobs, :size).by(1) + end + end + + context "when editing a chat message" do + let(:chat_message) { Fabricate(:chat_message) } + let(:updater) do + Chat::ChatMessageUpdater.new( + guardian: Guardian.new(chat_message.user), + chat_message: chat_message, + new_content: "This is my updated message", + ) + end + + it "queues a job on chat message update" do + expect { updater.update }.to change(Jobs::ToxicityClassifyChatMessage.jobs, :size).by(1) + end + end + end +end diff --git a/spec/lib/modules/toxicity/jobs/regular/toxicity_classify_chat_message_spec.rb b/spec/lib/modules/toxicity/jobs/regular/toxicity_classify_chat_message_spec.rb new file mode 100644 index 00000000..11399b7f --- /dev/null +++ b/spec/lib/modules/toxicity/jobs/regular/toxicity_classify_chat_message_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require "rails_helper" +require_relative "../../../../../support/toxicity_inference_stubs" + +describe Jobs::ToxicityClassifyChatMessage do + describe "#execute" do + before do + SiteSetting.ai_toxicity_enabled = true + SiteSetting.ai_toxicity_flag_automatically = true + end + + fab!(:chat_message) { Fabricate(:chat_message) } + + describe "scenarios where we return early without doing anything" do + it "does nothing when ai_toxicity_enabled is disabled" do + SiteSetting.ai_toxicity_enabled = false + + subject.execute({ chat_message_id: chat_message.id }) + + expect(ReviewableChatMessage.where(target: chat_message).count).to be_zero + end + + it "does nothing if there's no arg called post_id" do + subject.execute({}) + + expect(ReviewableChatMessage.where(target: chat_message).count).to be_zero + end + + it "does nothing if no post match the given id" do + subject.execute({ chat_message_id: nil }) + + expect(ReviewableChatMessage.where(target: chat_message).count).to be_zero + end + + it "does nothing if the post content is blank" do + chat_message.update_columns(message: "") + + subject.execute({ chat_message_id: chat_message.id }) + + expect(ReviewableChatMessage.where(target: chat_message).count).to be_zero + end + end + + it "flags the message when classified as toxic" do + ToxicityInferenceStubs.stub_chat_message_classification(chat_message, toxic: true) + + subject.execute({ chat_message_id: chat_message.id }) + + expect(ReviewableChatMessage.where(target: chat_message).count).to eq(1) + end + end +end diff --git a/spec/lib/modules/toxicity/jobs/regular/toxicity_classify_post_spec.rb b/spec/lib/modules/toxicity/jobs/regular/toxicity_classify_post_spec.rb new file mode 100644 index 00000000..f5bdd160 --- /dev/null +++ b/spec/lib/modules/toxicity/jobs/regular/toxicity_classify_post_spec.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require "rails_helper" +require_relative "../../../../../support/toxicity_inference_stubs" + +describe Jobs::ToxicityClassifyPost do + describe "#execute" do + before do + SiteSetting.ai_toxicity_enabled = true + SiteSetting.ai_toxicity_flag_automatically = true + end + + fab!(:post) { Fabricate(:post) } + + describe "scenarios where we return early without doing anything" do + it "does nothing when ai_toxicity_enabled is disabled" do + SiteSetting.ai_toxicity_enabled = false + + subject.execute({ post_id: post.id }) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + + it "does nothing if there's no arg called post_id" do + subject.execute({}) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + + it "does nothing if no post match the given id" do + subject.execute({ post_id: nil }) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + + it "does nothing if the post content is blank" do + post.update_columns(raw: "") + + subject.execute({ post_id: post.id }) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + end + + it "flags the post when classified as toxic" do + ToxicityInferenceStubs.stub_post_classification(post, toxic: true) + + subject.execute({ post_id: post.id }) + + expect(ReviewableFlaggedPost.where(target: post).count).to eq(1) + end + end +end diff --git a/spec/lib/modules/toxicity/post_classifier_spec.rb b/spec/lib/modules/toxicity/post_classifier_spec.rb new file mode 100644 index 00000000..bc9832ce --- /dev/null +++ b/spec/lib/modules/toxicity/post_classifier_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require "rails_helper" +require_relative "../../../support/toxicity_inference_stubs" + +describe DiscourseAI::Toxicity::PostClassifier do + before { SiteSetting.ai_toxicity_flag_automatically = true } + + fab!(:post) { Fabricate(:post) } + + describe "#classify!" do + it "creates a reviewable when the post is classified as toxic" do + ToxicityInferenceStubs.stub_post_classification(post, toxic: true) + + subject.classify!(post) + + expect(ReviewableFlaggedPost.where(target: post).count).to eq(1) + end + + it "doesn't create a reviewable if the post is not classified as toxic" do + ToxicityInferenceStubs.stub_post_classification(post, toxic: false) + + subject.classify!(post) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + + it "doesn't create a reviewable if flagging is disabled" do + SiteSetting.ai_toxicity_flag_automatically = false + ToxicityInferenceStubs.stub_post_classification(post, toxic: true) + + subject.classify!(post) + + expect(ReviewableFlaggedPost.where(target: post).count).to be_zero + end + + it "stores the classification in a custom field" do + ToxicityInferenceStubs.stub_post_classification(post, toxic: false) + + subject.classify!(post) + custom_field = PostCustomField.find_by(post: post, name: "toxicity") + + expect(custom_field.value).to eq( + { + classification: ToxicityInferenceStubs.civilized_response, + model: SiteSetting.ai_toxicity_inference_service_api_model, + }.to_json, + ) + end + end +end diff --git a/spec/lib/modules/toxicity/scan_queue_spec.rb b/spec/lib/modules/toxicity/scan_queue_spec.rb new file mode 100644 index 00000000..7220b943 --- /dev/null +++ b/spec/lib/modules/toxicity/scan_queue_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe DiscourseAI::Toxicity::ScanQueue do + fab!(:group) { Fabricate(:group) } + + before do + SiteSetting.ai_toxicity_enabled = true + SiteSetting.ai_toxicity_groups_bypass = group.id.to_s + end + + describe "#enqueue_post" do + fab!(:post) { Fabricate(:post) } + + it "queues a job" do + expect { described_class.enqueue_post(post) }.to change( + Jobs::ToxicityClassifyPost.jobs, + :size, + ).by(1) + end + + it "does nothing if ai_toxicity_enabled is disabled" do + SiteSetting.ai_toxicity_enabled = false + + expect { described_class.enqueue_post(post) }.not_to change( + Jobs::ToxicityClassifyPost.jobs, + :size, + ) + end + + it "does nothing if the user group is allowlisted" do + group.add(post.user) + + expect { described_class.enqueue_post(post) }.not_to change( + Jobs::ToxicityClassifyPost.jobs, + :size, + ) + end + end + + describe "#enqueue_chat_message" do + fab!(:chat_message) { Fabricate(:chat_message) } + + it "queues a job" do + expect { described_class.enqueue_chat_message(chat_message) }.to change( + Jobs::ToxicityClassifyChatMessage.jobs, + :size, + ).by(1) + end + + it "does nothing if ai_toxicity_enabled is disabled" do + SiteSetting.ai_toxicity_enabled = false + + expect { described_class.enqueue_chat_message(chat_message) }.not_to change( + Jobs::ToxicityClassifyChatMessage.jobs, + :size, + ) + end + + it "does nothing if the user group is allowlisted" do + group.add(chat_message.user) + + expect { described_class.enqueue_chat_message(chat_message) }.not_to change( + Jobs::ToxicityClassifyChatMessage.jobs, + :size, + ) + end + end +end diff --git a/spec/support/toxicity_inference_stubs.rb b/spec/support/toxicity_inference_stubs.rb new file mode 100644 index 00000000..db67d7b7 --- /dev/null +++ b/spec/support/toxicity_inference_stubs.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +class ToxicityInferenceStubs + class << self + def endpoint + "#{SiteSetting.ai_toxicity_inference_service_api_endpoint}/api/v1/classify" + end + + def model + SiteSetting.ai_toxicity_inference_service_api_model + end + + def toxic_response + { + toxicity: 99, + severe_toxicity: 1, + obscene: 6, + identity_attack: 3, + insult: 4, + threat: 8, + sexual_explicit: 5, + } + end + + def civilized_response + { + toxicity: 2, + severe_toxicity: 1, + obscene: 6, + identity_attack: 3, + insult: 4, + threat: 8, + sexual_explicit: 5, + } + end + + def stub_post_classification(post, toxic: false) + content = post.post_number == 1 ? "#{post.topic.title}\n#{post.raw}" : post.raw + response = toxic ? toxic_response : civilized_response + + WebMock + .stub_request(:post, endpoint) + .with(body: JSON.dump(model: model, content: content)) + .to_return(status: 200, body: JSON.dump(response)) + end + + def stub_chat_message_classification(chat_message, toxic: false) + response = toxic ? toxic_response : civilized_response + + WebMock + .stub_request(:post, endpoint) + .with(body: JSON.dump(model: model, content: chat_message.message)) + .to_return(status: 200, body: JSON.dump(response)) + end + end +end