Fix all specs

Also ensure triage is more consistent by reducing temp
This commit is contained in:
Sam Saffron 2023-05-29 14:11:32 +10:00
parent db11fe391d
commit c7781c57fc
No known key found for this signature in database
GPG Key ID: B9606168D2FFD9F5
13 changed files with 86 additions and 56 deletions

View File

@ -49,12 +49,12 @@ module DiscourseAi
).dig(:completion)
end
def submit_prompt(prompt, prefer_low_cost: false, &blk)
def submit_prompt(prompt, max_tokens: nil, temperature: nil, prefer_low_cost: false, &blk)
DiscourseAi::Inference::AnthropicCompletions.perform!(
prompt,
model_for,
temperature: 0.4,
max_tokens: 3000,
temperature: temperature || 0.4,
max_tokens: max_tokens || 3000,
&blk
)
end

View File

@ -34,6 +34,7 @@ module DiscourseAi
end
def debug(prompt)
return if !Rails.env.development?
if prompt.is_a?(Array)
prompt.each { |p| p.keys.each { |k| puts "#{k}: #{p[k]}" } }
else
@ -105,6 +106,10 @@ module DiscourseAi
PostCreator.create!(bot_user, topic_id: post.topic_id, raw: raw, skip_validations: false)
end
def context_prompt(post, context, result_name)
["Given the #{result_name} data:\n #{context}\nAnswer: #{post.raw}", post.user.username]
end
def reply_to(post)
command = triage_post(post)
@ -117,11 +122,7 @@ module DiscourseAi
post.post_custom_prompt ||= post.build_post_custom_prompt(custom_prompt: [])
prompt = post.post_custom_prompt.custom_prompt || []
# TODO consider providing even more context
# Given the last 10 posts of Sam are data:\n
prompt << [
"Given the #{command.result_name} data:\n #{context}\nAnswer: #{post.raw}",
post.user.username,
]
prompt << context_prompt(post, context, command.result_name)
post.post_custom_prompt.update!(custom_prompt: prompt)
end
end
@ -149,6 +150,10 @@ module DiscourseAi
Discourse.warn_exception(e, message: "ai-bot: Reply failed")
end
def triage_params
{ temperature: 0.1, max_tokens: 100 }
end
def triage_post(post)
prompt = bot_prompt_with_topic_context(post, triage: true)
@ -156,7 +161,9 @@ module DiscourseAi
reply = +""
context = {}
submit_prompt(prompt) { |partial, cancel| reply << get_delta(partial, context) }
submit_prompt(prompt, **triage_params) do |partial, cancel|
reply << get_delta(partial, context)
end
debug(reply)

View File

@ -30,21 +30,13 @@ module DiscourseAi
1500
end
{ temperature: 0.4, top_p: 0.9, max_tokens: max_tokens }
{ temperature: 0.4, max_tokens: max_tokens }
end
def submit_prompt(
prompt,
prefer_low_cost: false,
temperature: nil,
top_p: nil,
max_tokens: nil,
&blk
)
def submit_prompt(prompt, prefer_low_cost: false, temperature: nil, max_tokens: nil, &blk)
params =
reply_params.merge(
temperature: temperature,
top_p: top_p,
max_tokens: max_tokens,
) { |key, old_value, new_value| new_value.nil? ? old_value : new_value }
@ -56,8 +48,6 @@ module DiscourseAi
DiscourseAi::Tokenizer::OpenAiTokenizer.tokenize(text)
end
private
def build_message(poster_username, content, system: false, last: false)
is_bot = poster_username == bot_user.username
@ -70,6 +60,8 @@ module DiscourseAi
{ role: role, content: is_bot ? content : "#{poster_username}: #{content}" }
end
private
def model_for
return "gpt-4" if bot_user.id == DiscourseAi::AiBot::EntryPoint::GPT4_ID
"gpt-3.5-turbo"

View File

@ -13,14 +13,20 @@ RSpec.describe DiscourseAi::AiBot::AnthropicBot do
context = {}
reply = +""
reply << subject.get_delta({ completion: "\n\nAssist" }, context)
expect(reply).to eq("")
full = +"test"
reply << subject.get_delta({ completion: "\n\nAssistant: test" }, context)
expect(reply).to eq("test")
reply << subject.get_delta({ completion: full }, context)
expect(reply).to eq(full)
reply << subject.get_delta({ completion: "\n\nAssistant: test\nworld" }, context)
expect(reply).to eq("test\nworld")
full << "test2"
reply << subject.get_delta({ completion: full }, context)
expect(reply).to eq(full)
full << "test3"
reply << subject.get_delta({ completion: full }, context)
expect(reply).to eq(full)
end
end
end

View File

@ -25,12 +25,10 @@ RSpec.describe DiscourseAi::AiBot::Bot do
describe "#system_prompt" do
it "includes relevant context in system prompt" do
bot.system_prompt_style!(:standard)
SiteSetting.title = "My Forum"
SiteSetting.site_description = "My Forum Description"
system_prompt = bot.system_prompt(second_post)
system_prompt = bot.system_prompt(second_post, triage: false)
expect(system_prompt).to include(SiteSetting.title)
expect(system_prompt).to include(SiteSetting.site_description)
@ -41,20 +39,23 @@ RSpec.describe DiscourseAi::AiBot::Bot do
describe "#reply_to" do
it "can respond to !search" do
bot.system_prompt_style!(:simple)
expected_response = "!search test search"
expected_response = "ok, searching...\n!search test search"
prompt = bot.bot_prompt_with_topic_context(second_post)
prompt = bot.bot_prompt_with_topic_context(second_post, triage: true)
OpenAiCompletionsInferenceStubs.stub_streamed_response(
prompt,
[{ content: expected_response }],
req_opts: bot.reply_params.merge(stream: true),
req_opts: bot.triage_params.merge(stream: true),
)
prompt << { role: "assistant", content: "!search test search" }
prompt << { role: "user", content: "results: No results found" }
# second post will contain the search instruction
prompt = bot.bot_prompt_with_topic_context(first_post)
command = DiscourseAi::AiBot::Commands::SearchCommand.new(nil, nil, SecureRandom.hex(10))
content, username = bot.context_prompt(second_post, command.process, command.result_name)
prompt << bot.build_message(username, content)
OpenAiCompletionsInferenceStubs.stub_streamed_response(
prompt,
@ -69,7 +70,6 @@ RSpec.describe DiscourseAi::AiBot::Bot do
expect(last.raw).to include("<details>")
expect(last.raw).to include("<summary>Search</summary>")
expect(last.raw).not_to include("translation missing")
expect(last.raw).to include("ok, searching...")
expect(last.raw).to include("We are done now")
expect(last.post_custom_prompt.custom_prompt.to_s).to include("We are done now")

View File

@ -7,7 +7,7 @@ RSpec.describe DiscourseAi::AiBot::Commands::CategoriesCommand do
it "can generate correct info" do
Fabricate(:category, name: "america", posts_year: 999)
info = DiscourseAi::AiBot::Commands::CategoriesCommand.new(nil, nil).process(nil)
info = DiscourseAi::AiBot::Commands::CategoriesCommand.new(nil, nil, nil).process
expect(info).to include("america")
expect(info).to include("999")
end

View File

@ -4,7 +4,7 @@ require_relative "../../../../support/openai_completions_inference_stubs"
RSpec.describe DiscourseAi::AiBot::Commands::Command do
fab!(:bot_user) { User.find(DiscourseAi::AiBot::EntryPoint::GPT3_5_TURBO_ID) }
let(:command) { DiscourseAi::AiBot::Commands::Command.new(bot_user, nil) }
let(:command) { DiscourseAi::AiBot::Commands::Command.new(bot_user, nil, nil) }
describe "#format_results" do
it "can generate efficient tables of data" do

View File

@ -32,8 +32,8 @@ RSpec.describe DiscourseAi::AiBot::Commands::GoogleCommand do
"https://www.googleapis.com/customsearch/v1?cx=cx&key=abc&num=10&q=some%20search%20term",
).to_return(status: 200, body: json_text, headers: {})
google = described_class.new(bot_user, post)
info = google.process("some search term")
google = described_class.new(bot_user, post, "some search term")
info = google.process
expect(google.description_args[:count]).to eq(1)
expect(info).to include("title1")

View File

@ -11,9 +11,9 @@ RSpec.describe DiscourseAi::AiBot::Commands::SearchCommand do
describe "#process" do
it "can handle no results" do
post1 = Fabricate(:post)
search = described_class.new(bot_user, post1)
search = described_class.new(bot_user, post1, "order:fake ABDDCDCEDGDG")
results = search.process("order:fake ABDDCDCEDGDG")
results = search.process
expect(results).to eq("No results found")
end
@ -23,15 +23,15 @@ RSpec.describe DiscourseAi::AiBot::Commands::SearchCommand do
_post3 = Fabricate(:post, user: post1.user)
# search has no built in support for limit: so handle it from the outside
search = described_class.new(bot_user, post1)
search = described_class.new(bot_user, post1, "@#{post1.user.username} limit:2")
results = search.process("@#{post1.user.username} limit:2")
results = search.process
# title + 2 rows
expect(results.split("\n").length).to eq(3)
# just searching for everything
results = search.process("order:latest_topic")
search = described_class.new(bot_user, post1, "order:latest_topic")
results = search.process
expect(results.split("\n").length).to be > 1
end
end

View File

@ -14,8 +14,8 @@ RSpec.describe DiscourseAi::AiBot::Commands::SummarizeCommand do
body: JSON.dump({ choices: [{ message: { content: "summary stuff" } }] }),
)
summarizer = described_class.new(bot_user, post)
info = summarizer.process("#{post.topic_id} why did it happen?")
summarizer = described_class.new(bot_user, post, "#{post.topic_id} why did it happen?")
info = summarizer.process
expect(info).to include("Topic summarized")
expect(summarizer.custom_raw).to include("summary stuff")
@ -30,8 +30,8 @@ RSpec.describe DiscourseAi::AiBot::Commands::SummarizeCommand do
topic = Fabricate(:topic, category_id: category.id)
post = Fabricate(:post, topic: topic)
summarizer = described_class.new(bot_user, post)
info = summarizer.process("#{post.topic_id} why did it happen?")
summarizer = described_class.new(bot_user, post, "#{post.topic_id} why did it happen?")
info = summarizer.process
expect(info).not_to include(post.raw)

View File

@ -10,7 +10,7 @@ RSpec.describe DiscourseAi::AiBot::Commands::TagsCommand do
Fabricate(:tag, name: "america", public_topic_count: 100)
Fabricate(:tag, name: "not_here", public_topic_count: 0)
info = DiscourseAi::AiBot::Commands::TagsCommand.new(nil, nil).process(nil)
info = DiscourseAi::AiBot::Commands::TagsCommand.new(nil, nil, nil).process
expect(info).to include("america")
expect(info).not_to include("not_here")

View File

@ -23,12 +23,24 @@ RSpec.describe Jobs::CreateAiReply do
# time needs to be frozen so time in prompt does not drift
freeze_time
OpenAiCompletionsInferenceStubs.stub_streamed_response(
DiscourseAi::AiBot::OpenAiBot.new(bot_user).bot_prompt_with_topic_context(
post,
triage: true,
),
[{ content: "!noop" }],
req_opts: {
temperature: 0.1,
max_tokens: 100,
stream: true,
},
)
OpenAiCompletionsInferenceStubs.stub_streamed_response(
DiscourseAi::AiBot::OpenAiBot.new(bot_user).bot_prompt_with_topic_context(post),
deltas,
req_opts: {
temperature: 0.4,
top_p: 0.9,
max_tokens: 1500,
stream: true,
},
@ -66,12 +78,25 @@ RSpec.describe Jobs::CreateAiReply do
end
context "when chatting with Claude from Anthropic" do
let(:claude_response) { "Assistant: #{expected_response}" }
let(:claude_response) { "#{expected_response}" }
let(:deltas) { claude_response.split(" ").map { |w| "#{w} " } }
before do
bot_user = User.find(DiscourseAi::AiBot::EntryPoint::CLAUDE_V1_ID)
AnthropicCompletionStubs.stub_streamed_response(
DiscourseAi::AiBot::AnthropicBot.new(bot_user).bot_prompt_with_topic_context(
post,
triage: true,
),
[{ content: "!noop" }],
req_opts: {
temperature: 0.1,
max_tokens_to_sample: 100,
stream: true,
},
)
AnthropicCompletionStubs.stub_streamed_response(
DiscourseAi::AiBot::AnthropicBot.new(bot_user).bot_prompt_with_topic_context(post),
deltas,

View File

@ -33,7 +33,7 @@ RSpec.describe DiscourseAi::AiBot::OpenAiBot do
it "trims the prompt" do
prompt_messages = subject.bot_prompt_with_topic_context(post_1)
expect(prompt_messages[-2][:role]).to eq("assistant")
expect(prompt_messages[-2][:role]).to eq("system")
expect(prompt_messages[-1][:role]).to eq("user")
# trimming is tricky... it needs to account for system message as
# well... just make sure we trim for now