mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-03-06 17:30:20 +00:00
fbcc8e493a
Usually people do not want to scan personal messages. Sometimes they may. In that case they can enable triage on personal messages.
120 lines
3.8 KiB
Ruby
120 lines
3.8 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
return if !defined?(DiscourseAutomation)
|
|
|
|
describe DiscourseAi::Automation::LlmTriage do
|
|
fab!(:category)
|
|
fab!(:reply_user) { Fabricate(:user) }
|
|
fab!(:personal_message) { Fabricate(:private_message_topic) }
|
|
let(:canned_reply_text) { "Hello, this is a reply" }
|
|
|
|
let(:automation) { Fabricate(:automation, script: "llm_triage", enabled: true) }
|
|
|
|
fab!(:llm_model)
|
|
|
|
def add_automation_field(name, value, type: "text")
|
|
automation.fields.create!(
|
|
component: type,
|
|
name: name,
|
|
metadata: {
|
|
value: value,
|
|
},
|
|
target: "script",
|
|
)
|
|
end
|
|
|
|
before do
|
|
SiteSetting.tagging_enabled = true
|
|
add_automation_field("system_prompt", "hello %%POST%%")
|
|
add_automation_field("search_for_text", "bad")
|
|
add_automation_field("model", "custom:#{llm_model.id}")
|
|
add_automation_field("category", category.id, type: "category")
|
|
add_automation_field("tags", %w[aaa bbb], type: "tags")
|
|
add_automation_field("hide_topic", true, type: "boolean")
|
|
add_automation_field("flag_post", true, type: "boolean")
|
|
add_automation_field("canned_reply", canned_reply_text)
|
|
add_automation_field("canned_reply_user", reply_user.username, type: "user")
|
|
add_automation_field("max_post_tokens", 100)
|
|
end
|
|
|
|
it "can trigger via automation" do
|
|
post = Fabricate(:post, raw: "hello " * 5000)
|
|
|
|
body = {
|
|
model: "gpt-3.5-turbo-0301",
|
|
usage: {
|
|
prompt_tokens: 337,
|
|
completion_tokens: 162,
|
|
total_tokens: 499,
|
|
},
|
|
choices: [
|
|
{ message: { role: "assistant", content: "bad" }, finish_reason: "stop", index: 0 },
|
|
],
|
|
}.to_json
|
|
|
|
WebMock.stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(
|
|
status: 200,
|
|
body: body,
|
|
)
|
|
|
|
automation.running_in_background!
|
|
automation.trigger!({ "post" => post })
|
|
|
|
topic = post.topic.reload
|
|
expect(topic.category_id).to eq(category.id)
|
|
expect(topic.tags.pluck(:name)).to contain_exactly("aaa", "bbb")
|
|
expect(topic.visible).to eq(false)
|
|
reply = topic.posts.order(:post_number).last
|
|
expect(reply.raw).to eq(canned_reply_text)
|
|
expect(reply.user.id).to eq(reply_user.id)
|
|
|
|
ai_log = AiApiAuditLog.order("id desc").first
|
|
expect(ai_log.feature_name).to eq("llm_triage")
|
|
expect(ai_log.feature_context).to eq(
|
|
{ "automation_id" => automation.id, "automation_name" => automation.name },
|
|
)
|
|
|
|
count = ai_log.raw_request_payload.scan("hello").size
|
|
# we could use the exact count here but it can get fragile
|
|
# as we change tokenizers, this will give us reasonable confidence
|
|
expect(count).to be <= (100)
|
|
expect(count).to be > (50)
|
|
end
|
|
|
|
it "does not triage PMs by default" do
|
|
post = Fabricate(:post, topic: personal_message)
|
|
automation.running_in_background!
|
|
automation.trigger!({ "post" => post })
|
|
|
|
# nothing should happen, no classification, its a PM
|
|
end
|
|
|
|
it "will triage PMs if automation allows it" do
|
|
# needs to be admin or it will not be able to just step in to
|
|
# PM
|
|
reply_user.update!(admin: true)
|
|
add_automation_field("include_personal_messages", true, type: :boolean)
|
|
post = Fabricate(:post, topic: personal_message)
|
|
|
|
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
|
|
automation.running_in_background!
|
|
automation.trigger!({ "post" => post })
|
|
end
|
|
|
|
last_post = post.topic.reload.posts.order(:post_number).last
|
|
expect(last_post.raw).to eq(canned_reply_text)
|
|
end
|
|
|
|
it "does not reply to the canned_reply_user" do
|
|
post = Fabricate(:post, user: reply_user)
|
|
|
|
DiscourseAi::Completions::Llm.with_prepared_responses(["bad"]) do
|
|
automation.running_in_background!
|
|
automation.trigger!({ "post" => post })
|
|
end
|
|
|
|
last_post = post.topic.reload.posts.order(:post_number).last
|
|
expect(last_post.raw).to eq post.raw
|
|
end
|
|
end
|