Fix all specs
Also ensure triage is more consistent by reducing temp
This commit is contained in:
parent
db11fe391d
commit
c7781c57fc
|
@ -49,12 +49,12 @@ module DiscourseAi
|
|||
).dig(:completion)
|
||||
end
|
||||
|
||||
def submit_prompt(prompt, prefer_low_cost: false, &blk)
|
||||
def submit_prompt(prompt, max_tokens: nil, temperature: nil, prefer_low_cost: false, &blk)
|
||||
DiscourseAi::Inference::AnthropicCompletions.perform!(
|
||||
prompt,
|
||||
model_for,
|
||||
temperature: 0.4,
|
||||
max_tokens: 3000,
|
||||
temperature: temperature || 0.4,
|
||||
max_tokens: max_tokens || 3000,
|
||||
&blk
|
||||
)
|
||||
end
|
||||
|
|
|
@ -34,6 +34,7 @@ module DiscourseAi
|
|||
end
|
||||
|
||||
def debug(prompt)
|
||||
return if !Rails.env.development?
|
||||
if prompt.is_a?(Array)
|
||||
prompt.each { |p| p.keys.each { |k| puts "#{k}: #{p[k]}" } }
|
||||
else
|
||||
|
@ -105,6 +106,10 @@ module DiscourseAi
|
|||
PostCreator.create!(bot_user, topic_id: post.topic_id, raw: raw, skip_validations: false)
|
||||
end
|
||||
|
||||
def context_prompt(post, context, result_name)
|
||||
["Given the #{result_name} data:\n #{context}\nAnswer: #{post.raw}", post.user.username]
|
||||
end
|
||||
|
||||
def reply_to(post)
|
||||
command = triage_post(post)
|
||||
|
||||
|
@ -117,11 +122,7 @@ module DiscourseAi
|
|||
post.post_custom_prompt ||= post.build_post_custom_prompt(custom_prompt: [])
|
||||
prompt = post.post_custom_prompt.custom_prompt || []
|
||||
# TODO consider providing even more context
|
||||
# Given the last 10 posts of Sam are data:\n
|
||||
prompt << [
|
||||
"Given the #{command.result_name} data:\n #{context}\nAnswer: #{post.raw}",
|
||||
post.user.username,
|
||||
]
|
||||
prompt << context_prompt(post, context, command.result_name)
|
||||
post.post_custom_prompt.update!(custom_prompt: prompt)
|
||||
end
|
||||
end
|
||||
|
@ -149,6 +150,10 @@ module DiscourseAi
|
|||
Discourse.warn_exception(e, message: "ai-bot: Reply failed")
|
||||
end
|
||||
|
||||
def triage_params
|
||||
{ temperature: 0.1, max_tokens: 100 }
|
||||
end
|
||||
|
||||
def triage_post(post)
|
||||
prompt = bot_prompt_with_topic_context(post, triage: true)
|
||||
|
||||
|
@ -156,7 +161,9 @@ module DiscourseAi
|
|||
|
||||
reply = +""
|
||||
context = {}
|
||||
submit_prompt(prompt) { |partial, cancel| reply << get_delta(partial, context) }
|
||||
submit_prompt(prompt, **triage_params) do |partial, cancel|
|
||||
reply << get_delta(partial, context)
|
||||
end
|
||||
|
||||
debug(reply)
|
||||
|
||||
|
|
|
@ -30,21 +30,13 @@ module DiscourseAi
|
|||
1500
|
||||
end
|
||||
|
||||
{ temperature: 0.4, top_p: 0.9, max_tokens: max_tokens }
|
||||
{ temperature: 0.4, max_tokens: max_tokens }
|
||||
end
|
||||
|
||||
def submit_prompt(
|
||||
prompt,
|
||||
prefer_low_cost: false,
|
||||
temperature: nil,
|
||||
top_p: nil,
|
||||
max_tokens: nil,
|
||||
&blk
|
||||
)
|
||||
def submit_prompt(prompt, prefer_low_cost: false, temperature: nil, max_tokens: nil, &blk)
|
||||
params =
|
||||
reply_params.merge(
|
||||
temperature: temperature,
|
||||
top_p: top_p,
|
||||
max_tokens: max_tokens,
|
||||
) { |key, old_value, new_value| new_value.nil? ? old_value : new_value }
|
||||
|
||||
|
@ -56,8 +48,6 @@ module DiscourseAi
|
|||
DiscourseAi::Tokenizer::OpenAiTokenizer.tokenize(text)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def build_message(poster_username, content, system: false, last: false)
|
||||
is_bot = poster_username == bot_user.username
|
||||
|
||||
|
@ -70,6 +60,8 @@ module DiscourseAi
|
|||
{ role: role, content: is_bot ? content : "#{poster_username}: #{content}" }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def model_for
|
||||
return "gpt-4" if bot_user.id == DiscourseAi::AiBot::EntryPoint::GPT4_ID
|
||||
"gpt-3.5-turbo"
|
||||
|
|
|
@ -13,14 +13,20 @@ RSpec.describe DiscourseAi::AiBot::AnthropicBot do
|
|||
context = {}
|
||||
reply = +""
|
||||
|
||||
reply << subject.get_delta({ completion: "\n\nAssist" }, context)
|
||||
expect(reply).to eq("")
|
||||
full = +"test"
|
||||
|
||||
reply << subject.get_delta({ completion: "\n\nAssistant: test" }, context)
|
||||
expect(reply).to eq("test")
|
||||
reply << subject.get_delta({ completion: full }, context)
|
||||
expect(reply).to eq(full)
|
||||
|
||||
reply << subject.get_delta({ completion: "\n\nAssistant: test\nworld" }, context)
|
||||
expect(reply).to eq("test\nworld")
|
||||
full << "test2"
|
||||
|
||||
reply << subject.get_delta({ completion: full }, context)
|
||||
expect(reply).to eq(full)
|
||||
|
||||
full << "test3"
|
||||
|
||||
reply << subject.get_delta({ completion: full }, context)
|
||||
expect(reply).to eq(full)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -25,12 +25,10 @@ RSpec.describe DiscourseAi::AiBot::Bot do
|
|||
|
||||
describe "#system_prompt" do
|
||||
it "includes relevant context in system prompt" do
|
||||
bot.system_prompt_style!(:standard)
|
||||
|
||||
SiteSetting.title = "My Forum"
|
||||
SiteSetting.site_description = "My Forum Description"
|
||||
|
||||
system_prompt = bot.system_prompt(second_post)
|
||||
system_prompt = bot.system_prompt(second_post, triage: false)
|
||||
|
||||
expect(system_prompt).to include(SiteSetting.title)
|
||||
expect(system_prompt).to include(SiteSetting.site_description)
|
||||
|
@ -41,20 +39,23 @@ RSpec.describe DiscourseAi::AiBot::Bot do
|
|||
|
||||
describe "#reply_to" do
|
||||
it "can respond to !search" do
|
||||
bot.system_prompt_style!(:simple)
|
||||
expected_response = "!search test search"
|
||||
|
||||
expected_response = "ok, searching...\n!search test search"
|
||||
|
||||
prompt = bot.bot_prompt_with_topic_context(second_post)
|
||||
prompt = bot.bot_prompt_with_topic_context(second_post, triage: true)
|
||||
|
||||
OpenAiCompletionsInferenceStubs.stub_streamed_response(
|
||||
prompt,
|
||||
[{ content: expected_response }],
|
||||
req_opts: bot.reply_params.merge(stream: true),
|
||||
req_opts: bot.triage_params.merge(stream: true),
|
||||
)
|
||||
|
||||
prompt << { role: "assistant", content: "!search test search" }
|
||||
prompt << { role: "user", content: "results: No results found" }
|
||||
# second post will contain the search instruction
|
||||
prompt = bot.bot_prompt_with_topic_context(first_post)
|
||||
|
||||
command = DiscourseAi::AiBot::Commands::SearchCommand.new(nil, nil, SecureRandom.hex(10))
|
||||
content, username = bot.context_prompt(second_post, command.process, command.result_name)
|
||||
|
||||
prompt << bot.build_message(username, content)
|
||||
|
||||
OpenAiCompletionsInferenceStubs.stub_streamed_response(
|
||||
prompt,
|
||||
|
@ -69,7 +70,6 @@ RSpec.describe DiscourseAi::AiBot::Bot do
|
|||
expect(last.raw).to include("<details>")
|
||||
expect(last.raw).to include("<summary>Search</summary>")
|
||||
expect(last.raw).not_to include("translation missing")
|
||||
expect(last.raw).to include("ok, searching...")
|
||||
expect(last.raw).to include("We are done now")
|
||||
|
||||
expect(last.post_custom_prompt.custom_prompt.to_s).to include("We are done now")
|
||||
|
|
|
@ -7,7 +7,7 @@ RSpec.describe DiscourseAi::AiBot::Commands::CategoriesCommand do
|
|||
it "can generate correct info" do
|
||||
Fabricate(:category, name: "america", posts_year: 999)
|
||||
|
||||
info = DiscourseAi::AiBot::Commands::CategoriesCommand.new(nil, nil).process(nil)
|
||||
info = DiscourseAi::AiBot::Commands::CategoriesCommand.new(nil, nil, nil).process
|
||||
expect(info).to include("america")
|
||||
expect(info).to include("999")
|
||||
end
|
||||
|
|
|
@ -4,7 +4,7 @@ require_relative "../../../../support/openai_completions_inference_stubs"
|
|||
|
||||
RSpec.describe DiscourseAi::AiBot::Commands::Command do
|
||||
fab!(:bot_user) { User.find(DiscourseAi::AiBot::EntryPoint::GPT3_5_TURBO_ID) }
|
||||
let(:command) { DiscourseAi::AiBot::Commands::Command.new(bot_user, nil) }
|
||||
let(:command) { DiscourseAi::AiBot::Commands::Command.new(bot_user, nil, nil) }
|
||||
|
||||
describe "#format_results" do
|
||||
it "can generate efficient tables of data" do
|
||||
|
|
|
@ -32,8 +32,8 @@ RSpec.describe DiscourseAi::AiBot::Commands::GoogleCommand do
|
|||
"https://www.googleapis.com/customsearch/v1?cx=cx&key=abc&num=10&q=some%20search%20term",
|
||||
).to_return(status: 200, body: json_text, headers: {})
|
||||
|
||||
google = described_class.new(bot_user, post)
|
||||
info = google.process("some search term")
|
||||
google = described_class.new(bot_user, post, "some search term")
|
||||
info = google.process
|
||||
|
||||
expect(google.description_args[:count]).to eq(1)
|
||||
expect(info).to include("title1")
|
||||
|
|
|
@ -11,9 +11,9 @@ RSpec.describe DiscourseAi::AiBot::Commands::SearchCommand do
|
|||
describe "#process" do
|
||||
it "can handle no results" do
|
||||
post1 = Fabricate(:post)
|
||||
search = described_class.new(bot_user, post1)
|
||||
search = described_class.new(bot_user, post1, "order:fake ABDDCDCEDGDG")
|
||||
|
||||
results = search.process("order:fake ABDDCDCEDGDG")
|
||||
results = search.process
|
||||
expect(results).to eq("No results found")
|
||||
end
|
||||
|
||||
|
@ -23,15 +23,15 @@ RSpec.describe DiscourseAi::AiBot::Commands::SearchCommand do
|
|||
_post3 = Fabricate(:post, user: post1.user)
|
||||
|
||||
# search has no built in support for limit: so handle it from the outside
|
||||
search = described_class.new(bot_user, post1)
|
||||
search = described_class.new(bot_user, post1, "@#{post1.user.username} limit:2")
|
||||
|
||||
results = search.process("@#{post1.user.username} limit:2")
|
||||
results = search.process
|
||||
|
||||
# title + 2 rows
|
||||
expect(results.split("\n").length).to eq(3)
|
||||
|
||||
# just searching for everything
|
||||
results = search.process("order:latest_topic")
|
||||
search = described_class.new(bot_user, post1, "order:latest_topic")
|
||||
results = search.process
|
||||
expect(results.split("\n").length).to be > 1
|
||||
end
|
||||
end
|
||||
|
|
|
@ -14,8 +14,8 @@ RSpec.describe DiscourseAi::AiBot::Commands::SummarizeCommand do
|
|||
body: JSON.dump({ choices: [{ message: { content: "summary stuff" } }] }),
|
||||
)
|
||||
|
||||
summarizer = described_class.new(bot_user, post)
|
||||
info = summarizer.process("#{post.topic_id} why did it happen?")
|
||||
summarizer = described_class.new(bot_user, post, "#{post.topic_id} why did it happen?")
|
||||
info = summarizer.process
|
||||
|
||||
expect(info).to include("Topic summarized")
|
||||
expect(summarizer.custom_raw).to include("summary stuff")
|
||||
|
@ -30,8 +30,8 @@ RSpec.describe DiscourseAi::AiBot::Commands::SummarizeCommand do
|
|||
topic = Fabricate(:topic, category_id: category.id)
|
||||
post = Fabricate(:post, topic: topic)
|
||||
|
||||
summarizer = described_class.new(bot_user, post)
|
||||
info = summarizer.process("#{post.topic_id} why did it happen?")
|
||||
summarizer = described_class.new(bot_user, post, "#{post.topic_id} why did it happen?")
|
||||
info = summarizer.process
|
||||
|
||||
expect(info).not_to include(post.raw)
|
||||
|
||||
|
|
|
@ -10,7 +10,7 @@ RSpec.describe DiscourseAi::AiBot::Commands::TagsCommand do
|
|||
Fabricate(:tag, name: "america", public_topic_count: 100)
|
||||
Fabricate(:tag, name: "not_here", public_topic_count: 0)
|
||||
|
||||
info = DiscourseAi::AiBot::Commands::TagsCommand.new(nil, nil).process(nil)
|
||||
info = DiscourseAi::AiBot::Commands::TagsCommand.new(nil, nil, nil).process
|
||||
|
||||
expect(info).to include("america")
|
||||
expect(info).not_to include("not_here")
|
||||
|
|
|
@ -23,12 +23,24 @@ RSpec.describe Jobs::CreateAiReply do
|
|||
# time needs to be frozen so time in prompt does not drift
|
||||
freeze_time
|
||||
|
||||
OpenAiCompletionsInferenceStubs.stub_streamed_response(
|
||||
DiscourseAi::AiBot::OpenAiBot.new(bot_user).bot_prompt_with_topic_context(
|
||||
post,
|
||||
triage: true,
|
||||
),
|
||||
[{ content: "!noop" }],
|
||||
req_opts: {
|
||||
temperature: 0.1,
|
||||
max_tokens: 100,
|
||||
stream: true,
|
||||
},
|
||||
)
|
||||
|
||||
OpenAiCompletionsInferenceStubs.stub_streamed_response(
|
||||
DiscourseAi::AiBot::OpenAiBot.new(bot_user).bot_prompt_with_topic_context(post),
|
||||
deltas,
|
||||
req_opts: {
|
||||
temperature: 0.4,
|
||||
top_p: 0.9,
|
||||
max_tokens: 1500,
|
||||
stream: true,
|
||||
},
|
||||
|
@ -66,12 +78,25 @@ RSpec.describe Jobs::CreateAiReply do
|
|||
end
|
||||
|
||||
context "when chatting with Claude from Anthropic" do
|
||||
let(:claude_response) { "Assistant: #{expected_response}" }
|
||||
let(:claude_response) { "#{expected_response}" }
|
||||
let(:deltas) { claude_response.split(" ").map { |w| "#{w} " } }
|
||||
|
||||
before do
|
||||
bot_user = User.find(DiscourseAi::AiBot::EntryPoint::CLAUDE_V1_ID)
|
||||
|
||||
AnthropicCompletionStubs.stub_streamed_response(
|
||||
DiscourseAi::AiBot::AnthropicBot.new(bot_user).bot_prompt_with_topic_context(
|
||||
post,
|
||||
triage: true,
|
||||
),
|
||||
[{ content: "!noop" }],
|
||||
req_opts: {
|
||||
temperature: 0.1,
|
||||
max_tokens_to_sample: 100,
|
||||
stream: true,
|
||||
},
|
||||
)
|
||||
|
||||
AnthropicCompletionStubs.stub_streamed_response(
|
||||
DiscourseAi::AiBot::AnthropicBot.new(bot_user).bot_prompt_with_topic_context(post),
|
||||
deltas,
|
||||
|
|
|
@ -33,7 +33,7 @@ RSpec.describe DiscourseAi::AiBot::OpenAiBot do
|
|||
it "trims the prompt" do
|
||||
prompt_messages = subject.bot_prompt_with_topic_context(post_1)
|
||||
|
||||
expect(prompt_messages[-2][:role]).to eq("assistant")
|
||||
expect(prompt_messages[-2][:role]).to eq("system")
|
||||
expect(prompt_messages[-1][:role]).to eq("user")
|
||||
# trimming is tricky... it needs to account for system message as
|
||||
# well... just make sure we trim for now
|
||||
|
|
Loading…
Reference in New Issue