FIX: never block spam scanning user (#1437)

Previously staff and bots would get scanned if TL was low
Additionally if somehow spam scanner user was blocked
(deactivated, silenced, banned) it would stop the feature from working

This adds an override that ensures unconditionally the user is setup correctly prior to scanning
This commit is contained in:
Sam 2025-06-17 14:51:27 +10:00 committed by GitHub
parent bc8e57d7e8
commit 32dc45ba4f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 49 additions and 1 deletions

View File

@ -47,10 +47,20 @@ module DiscourseAi
user = nil
if SiteSetting.ai_spam_detection_user_id.present?
user = User.find_by(id: SiteSetting.ai_spam_detection_user_id)
ensure_safe_flagging_user!(user)
end
user || Discourse.system_user
end
def self.ensure_safe_flagging_user!(user)
# only do repair on bot users, if somehow it is set to a human skip repairs
return if !user.bot?
user.update!(silenced_till: nil) if user.silenced?
user.update!(trust_level: TrustLevel[4]) if user.trust_level != TrustLevel[4]
user.update!(suspended_till: nil, suspended_at: nil) if user.suspended?
user.update!(active: true) if !user.active?
end
def self.after_cooked_post(post)
return if !enabled?
return if !should_scan_post?(post)
@ -94,6 +104,9 @@ module DiscourseAi
return false if !post.present?
return false if post.user.trust_level > TrustLevel[1]
return false if post.topic.private_message?
return false if post.user.bot?
return false if post.user.staff?
if Post
.where(user_id: post.user_id)
.joins(:topic)

View File

@ -49,6 +49,16 @@ RSpec.describe DiscourseAi::AiModeration::SpamScanner do
expect(described_class.should_scan_post?(post)).to eq(false)
end
it "returns false for bots" do
post.user.id = -100
expect(described_class.should_scan_post?(post)).to eq(false)
end
it "returns false for staff" do
post.user.moderator = true
expect(described_class.should_scan_post?(post)).to eq(false)
end
it "returns false for users with many public posts" do
Fabricate(:post, user: user, topic: topic)
Fabricate(:post, user: user, topic: topic)
@ -207,6 +217,26 @@ RSpec.describe DiscourseAi::AiModeration::SpamScanner do
end
end
it "unsilences flagging user if erronuously silenced" do
described_class.flagging_user.update!(silenced_till: 1.day.from_now)
expect(described_class.flagging_user.silenced?).to eq(false)
end
it "ensures flagging user is tl4" do
described_class.flagging_user.update!(trust_level: 0)
expect(described_class.flagging_user.trust_level).to eq(4)
end
it "unsuspends user if it was erronuously suspended" do
described_class.flagging_user.update!(suspended_till: 1.day.from_now, suspended_at: 1.day.ago)
expect(described_class.flagging_user.suspended?).to eq(false)
end
it "makes sure account is active" do
described_class.flagging_user.update!(active: false)
expect(described_class.flagging_user.active).to eq(true)
end
describe "integration test" do
fab!(:llm_model)
let(:api_audit_log) { Fabricate(:api_audit_log) }
@ -243,8 +273,13 @@ RSpec.describe DiscourseAi::AiModeration::SpamScanner do
it "correctly handles spam scanning" do
expect(described_class.flagging_user.id).not_to eq(Discourse.system_user.id)
# flag post for scanning
post = post_with_uploaded_image
# this is surprising, core fabricator is not linking
# we need it linked so we scan uploads
post.link_post_uploads
expect(described_class.should_scan_post?(post)).to eq(true)
expect(post.upload_ids).to be_present
described_class.new_post(post)