2023-12-18 20:04:15 -05:00
|
|
|
# frozen_string_literal: true
|
|
|
|
describe DiscourseAi::Automation::LlmTriage do
|
2024-03-05 10:48:28 -05:00
|
|
|
fab!(:post)
|
2024-10-04 02:11:30 -04:00
|
|
|
fab!(:reply) { Fabricate(:post, topic: post.topic, user: Fabricate(:user)) }
|
2024-07-30 12:44:57 -04:00
|
|
|
fab!(:llm_model)
|
2023-12-18 20:04:15 -05:00
|
|
|
|
|
|
|
def triage(**args)
|
2024-08-27 14:45:55 -04:00
|
|
|
DiscourseAi::Automation::LlmTriage.handle(**args)
|
2023-12-18 20:04:15 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
it "does nothing if it does not pass triage" do
|
|
|
|
DiscourseAi::Completions::Llm.with_prepared_responses(["good"]) do
|
2024-08-27 14:45:55 -04:00
|
|
|
triage(
|
|
|
|
post: post,
|
|
|
|
model: "custom:#{llm_model.id}",
|
|
|
|
hide_topic: true,
|
|
|
|
system_prompt: "test %%POST%%",
|
|
|
|
search_for_text: "bad",
|
|
|
|
automation: nil,
|
|
|
|
)
|
2023-12-18 20:04:15 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
expect(post.topic.reload.visible).to eq(true)
|
|
|
|
end
|
|
|
|
|
2024-01-29 14:04:25 -05:00
|
|
|
it "can hide topics on triage" do
|
2023-12-18 20:04:15 -05:00
|
|
|
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
|
2024-08-27 14:45:55 -04:00
|
|
|
triage(
|
|
|
|
post: post,
|
|
|
|
model: "custom:#{llm_model.id}",
|
|
|
|
hide_topic: true,
|
|
|
|
system_prompt: "test %%POST%%",
|
|
|
|
search_for_text: "bad",
|
|
|
|
automation: nil,
|
|
|
|
)
|
2023-12-18 20:04:15 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
expect(post.topic.reload.visible).to eq(false)
|
|
|
|
end
|
|
|
|
|
|
|
|
it "can categorize topics on triage" do
|
|
|
|
category = Fabricate(:category)
|
|
|
|
|
|
|
|
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
|
2024-08-27 14:45:55 -04:00
|
|
|
triage(
|
|
|
|
post: post,
|
|
|
|
model: "custom:#{llm_model.id}",
|
|
|
|
category_id: category.id,
|
|
|
|
system_prompt: "test %%POST%%",
|
|
|
|
search_for_text: "bad",
|
|
|
|
automation: nil,
|
|
|
|
)
|
2023-12-18 20:04:15 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
expect(post.topic.reload.category_id).to eq(category.id)
|
|
|
|
end
|
|
|
|
|
|
|
|
it "can reply to topics on triage" do
|
|
|
|
user = Fabricate(:user)
|
|
|
|
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
|
|
|
|
triage(
|
|
|
|
post: post,
|
2024-08-27 14:45:55 -04:00
|
|
|
model: "custom:#{llm_model.id}",
|
|
|
|
system_prompt: "test %%POST%%",
|
2023-12-18 20:04:15 -05:00
|
|
|
search_for_text: "bad",
|
|
|
|
canned_reply: "test canned reply 123",
|
|
|
|
canned_reply_user: user.username,
|
2024-08-27 14:45:55 -04:00
|
|
|
automation: nil,
|
2023-12-18 20:04:15 -05:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
reply = post.topic.posts.order(:post_number).last
|
|
|
|
|
|
|
|
expect(reply.raw).to eq("test canned reply 123")
|
|
|
|
expect(reply.user.id).to eq(user.id)
|
|
|
|
end
|
2024-02-28 20:33:28 -05:00
|
|
|
|
|
|
|
it "can add posts to the review queue" do
|
|
|
|
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
|
2024-08-27 14:45:55 -04:00
|
|
|
triage(
|
|
|
|
post: post,
|
|
|
|
model: "custom:#{llm_model.id}",
|
|
|
|
system_prompt: "test %%POST%%",
|
|
|
|
search_for_text: "bad",
|
|
|
|
flag_post: true,
|
|
|
|
automation: nil,
|
|
|
|
)
|
2024-02-28 20:33:28 -05:00
|
|
|
end
|
|
|
|
|
|
|
|
reviewable = ReviewablePost.last
|
|
|
|
|
|
|
|
expect(reviewable.target).to eq(post)
|
2024-08-14 14:54:09 -04:00
|
|
|
expect(reviewable.reviewable_scores.first.reason).to include("bad")
|
2024-02-28 20:33:28 -05:00
|
|
|
end
|
2024-04-11 08:50:46 -04:00
|
|
|
|
2024-10-04 02:11:30 -04:00
|
|
|
it "can handle spam flags" do
|
|
|
|
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
|
|
|
|
triage(
|
|
|
|
post: post,
|
|
|
|
model: "custom:#{llm_model.id}",
|
|
|
|
system_prompt: "test %%POST%%",
|
|
|
|
search_for_text: "bad",
|
|
|
|
flag_post: true,
|
|
|
|
flag_type: :spam,
|
|
|
|
automation: nil,
|
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
expect(post.reload).to be_hidden
|
|
|
|
expect(post.topic.reload.visible).to eq(false)
|
|
|
|
end
|
|
|
|
|
2024-04-11 08:50:46 -04:00
|
|
|
it "can handle garbled output from LLM" do
|
|
|
|
DiscourseAi::Completions::Llm.with_prepared_responses(["Bad.\n\nYo"]) do
|
2024-08-27 14:45:55 -04:00
|
|
|
triage(
|
|
|
|
post: post,
|
|
|
|
model: "custom:#{llm_model.id}",
|
|
|
|
system_prompt: "test %%POST%%",
|
|
|
|
search_for_text: "bad",
|
|
|
|
flag_post: true,
|
|
|
|
automation: nil,
|
|
|
|
)
|
2024-04-11 08:50:46 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
reviewable = ReviewablePost.last
|
|
|
|
|
|
|
|
expect(reviewable&.target).to eq(post)
|
|
|
|
end
|
2024-08-22 17:32:42 -04:00
|
|
|
|
|
|
|
it "treats search_for_text as case-insensitive" do
|
|
|
|
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
|
2024-08-27 14:45:55 -04:00
|
|
|
triage(
|
|
|
|
post: post,
|
|
|
|
model: "custom:#{llm_model.id}",
|
|
|
|
system_prompt: "test %%POST%%",
|
|
|
|
search_for_text: "BAD",
|
|
|
|
flag_post: true,
|
|
|
|
automation: nil,
|
|
|
|
)
|
2024-08-22 17:32:42 -04:00
|
|
|
end
|
|
|
|
|
|
|
|
reviewable = ReviewablePost.last
|
|
|
|
|
|
|
|
expect(reviewable.target).to eq(post)
|
|
|
|
end
|
2023-12-18 20:04:15 -05:00
|
|
|
end
|