mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-10-31 14:38:37 +00:00
0cbf14e343
The new automation rule can be used to perform llm based classification and categorization of topics. You specify a system prompt (which has %%POST%% as an input), if it returns a particular piece of text then we will apply rules such as tagging, hiding, replying or categorizing. This can be used as a spam filter, a "oops you are in the wrong place" filter and so on. Co-authored-by: Joffrey JAFFEUX <j.jaffeux@gmail.com>
133 lines
3.6 KiB
Ruby
133 lines
3.6 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
return if !defined?(DiscourseAutomation)
|
|
|
|
describe DiscourseAutomation::LlmTriage do
|
|
fab!(:post) { Fabricate(:post) }
|
|
|
|
def triage(**args)
|
|
DiscourseAutomation::LlmTriage.handle(**args)
|
|
end
|
|
|
|
it "does nothing if it does not pass triage" do
|
|
stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(
|
|
status: 200,
|
|
body: { choices: [{ message: { content: "good" } }] }.to_json,
|
|
)
|
|
|
|
triage(
|
|
post: post,
|
|
model: "gpt-4",
|
|
hide_topic: true,
|
|
system_prompt: "test %%POST%%",
|
|
search_for_text: "bad",
|
|
)
|
|
|
|
expect(post.topic.reload.visible).to eq(true)
|
|
end
|
|
|
|
it "can hide topics on triage" do
|
|
stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(
|
|
status: 200,
|
|
body: { choices: [{ message: { content: "bad" } }] }.to_json,
|
|
)
|
|
|
|
triage(
|
|
post: post,
|
|
model: "gpt-4",
|
|
hide_topic: true,
|
|
system_prompt: "test %%POST%%",
|
|
search_for_text: "bad",
|
|
)
|
|
|
|
expect(post.topic.reload.visible).to eq(false)
|
|
end
|
|
|
|
it "can categorize topics on triage" do
|
|
category = Fabricate(:category)
|
|
|
|
stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(
|
|
status: 200,
|
|
body: { choices: [{ message: { content: "bad" } }] }.to_json,
|
|
)
|
|
|
|
triage(
|
|
post: post,
|
|
model: "gpt-4",
|
|
category_id: category.id,
|
|
system_prompt: "test %%POST%%",
|
|
search_for_text: "bad",
|
|
)
|
|
|
|
expect(post.topic.reload.category_id).to eq(category.id)
|
|
end
|
|
|
|
it "can reply to topics on triage" do
|
|
user = Fabricate(:user)
|
|
|
|
stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(
|
|
status: 200,
|
|
body: { choices: [{ message: { content: "bad" } }] }.to_json,
|
|
)
|
|
|
|
triage(
|
|
post: post,
|
|
model: "gpt-4",
|
|
system_prompt: "test %%POST%%",
|
|
search_for_text: "bad",
|
|
canned_reply: "test canned reply 123",
|
|
canned_reply_user: user.username,
|
|
)
|
|
|
|
reply = post.topic.posts.order(:post_number).last
|
|
|
|
expect(reply.raw).to eq("test canned reply 123")
|
|
expect(reply.user.id).to eq(user.id)
|
|
end
|
|
|
|
let(:automation) { Fabricate(:automation, script: "llm_triage", enabled: true) }
|
|
|
|
def add_automation_field(name, value, type: "text")
|
|
automation.fields.create!(
|
|
component: type,
|
|
name: name,
|
|
metadata: {
|
|
value: value,
|
|
},
|
|
target: "script",
|
|
)
|
|
end
|
|
|
|
it "can trigger via automation" do
|
|
SiteSetting.tagging_enabled = true
|
|
|
|
category = Fabricate(:category)
|
|
user = Fabricate(:user)
|
|
|
|
add_automation_field("system_prompt", "hello %%POST%%")
|
|
add_automation_field("search_for_text", "bad")
|
|
add_automation_field("model", "gpt-4")
|
|
add_automation_field("category", category.id, type: "category")
|
|
add_automation_field("tags", %w[aaa bbb], type: "tags")
|
|
add_automation_field("hide_topic", true, type: "boolean")
|
|
add_automation_field("canned_reply", "Yo this is a reply")
|
|
add_automation_field("canned_reply_user", user.username, type: "user")
|
|
|
|
stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(
|
|
status: 200,
|
|
body: { choices: [{ message: { content: "bad" } }] }.to_json,
|
|
)
|
|
|
|
automation.running_in_background!
|
|
automation.trigger!({ "post" => post })
|
|
|
|
topic = post.topic.reload
|
|
expect(topic.category_id).to eq(category.id)
|
|
expect(topic.tags.pluck(:name)).to contain_exactly("aaa", "bbb")
|
|
expect(topic.visible).to eq(false)
|
|
reply = topic.posts.order(:post_number).last
|
|
expect(reply.raw).to eq("Yo this is a reply")
|
|
expect(reply.user.id).to eq(user.id)
|
|
end
|
|
end
|