Fix all specs

Also ensure triage is more consistent by reducing temp

lib/modules/ai_bot/anthropic_bot.rb

@@ -49,12 +49,12 @@ module DiscourseAi
         ).dig(:completion)
       end
 
-      def submit_prompt(prompt, prefer_low_cost: false, &blk)
+      def submit_prompt(prompt, max_tokens: nil, temperature: nil, prefer_low_cost: false, &blk)
         DiscourseAi::Inference::AnthropicCompletions.perform!(
           prompt,
           model_for,
-          temperature: 0.4,
+          temperature: temperature || 0.4,
-          max_tokens: 3000,
+          max_tokens: max_tokens || 3000,
           &blk
         )
       end

lib/modules/ai_bot/bot.rb

@@ -34,6 +34,7 @@ module DiscourseAi
       end
 
       def debug(prompt)
+        return if !Rails.env.development?
         if prompt.is_a?(Array)
           prompt.each { |p| p.keys.each { |k| puts "#{k}: #{p[k]}" } }
         else
@@ -105,6 +106,10 @@ module DiscourseAi
         PostCreator.create!(bot_user, topic_id: post.topic_id, raw: raw, skip_validations: false)
       end
 
+      def context_prompt(post, context, result_name)
+        ["Given the #{result_name} data:\n #{context}\nAnswer: #{post.raw}", post.user.username]
+      end
+
       def reply_to(post)
         command = triage_post(post)
 
@@ -117,11 +122,7 @@ module DiscourseAi
             post.post_custom_prompt ||= post.build_post_custom_prompt(custom_prompt: [])
             prompt = post.post_custom_prompt.custom_prompt || []
             # TODO consider providing even more context
-            # Given the last 10 posts of Sam are data:\n
+            prompt << context_prompt(post, context, command.result_name)
-            prompt << [
-              "Given the #{command.result_name} data:\n #{context}\nAnswer: #{post.raw}",
-              post.user.username,
-            ]
             post.post_custom_prompt.update!(custom_prompt: prompt)
           end
         end
@@ -149,6 +150,10 @@ module DiscourseAi
         Discourse.warn_exception(e, message: "ai-bot: Reply failed")
       end
 
+      def triage_params
+        { temperature: 0.1, max_tokens: 100 }
+      end
+
       def triage_post(post)
         prompt = bot_prompt_with_topic_context(post, triage: true)
 
@@ -156,7 +161,9 @@ module DiscourseAi
 
         reply = +""
         context = {}
-        submit_prompt(prompt) { |partial, cancel| reply << get_delta(partial, context) }
+        submit_prompt(prompt, **triage_params) do |partial, cancel|
+          reply << get_delta(partial, context)
+        end
 
         debug(reply)
 

lib/modules/ai_bot/open_ai_bot.rb

@@ -30,21 +30,13 @@ module DiscourseAi
             1500
           end
 
-        { temperature: 0.4, top_p: 0.9, max_tokens: max_tokens }
+        { temperature: 0.4, max_tokens: max_tokens }
       end
 
-      def submit_prompt(
+      def submit_prompt(prompt, prefer_low_cost: false, temperature: nil, max_tokens: nil, &blk)
-        prompt,
-        prefer_low_cost: false,
-        temperature: nil,
-        top_p: nil,
-        max_tokens: nil,
-        &blk
-      )
         params =
           reply_params.merge(
             temperature: temperature,
-            top_p: top_p,
             max_tokens: max_tokens,
           ) { |key, old_value, new_value| new_value.nil? ? old_value : new_value }
 
@@ -56,8 +48,6 @@ module DiscourseAi
         DiscourseAi::Tokenizer::OpenAiTokenizer.tokenize(text)
       end
 
-      private
-
       def build_message(poster_username, content, system: false, last: false)
         is_bot = poster_username == bot_user.username
 
@@ -70,6 +60,8 @@ module DiscourseAi
         { role: role, content: is_bot ? content : "#{poster_username}: #{content}" }
       end
 
+      private
+
       def model_for
         return "gpt-4" if bot_user.id == DiscourseAi::AiBot::EntryPoint::GPT4_ID
         "gpt-3.5-turbo"

spec/lib/modules/ai_bot/anthropic_bot_spec.rb

@@ -13,14 +13,20 @@ RSpec.describe DiscourseAi::AiBot::AnthropicBot do
         context = {}
         reply = +""
 
-        reply << subject.get_delta({ completion: "\n\nAssist" }, context)
+        full = +"test"
-        expect(reply).to eq("")
 
-        reply << subject.get_delta({ completion: "\n\nAssistant: test" }, context)
+        reply << subject.get_delta({ completion: full }, context)
-        expect(reply).to eq("test")
+        expect(reply).to eq(full)
 
-        reply << subject.get_delta({ completion: "\n\nAssistant: test\nworld" }, context)
+        full << "test2"
-        expect(reply).to eq("test\nworld")
+
+        reply << subject.get_delta({ completion: full }, context)
+        expect(reply).to eq(full)
+
+        full << "test3"
+
+        reply << subject.get_delta({ completion: full }, context)
+        expect(reply).to eq(full)
       end
     end
   end

spec/lib/modules/ai_bot/bot_spec.rb

@@ -25,12 +25,10 @@ RSpec.describe DiscourseAi::AiBot::Bot do
 
   describe "#system_prompt" do
     it "includes relevant context in system prompt" do
-      bot.system_prompt_style!(:standard)
-
       SiteSetting.title = "My Forum"
       SiteSetting.site_description = "My Forum Description"
 
-      system_prompt = bot.system_prompt(second_post)
+      system_prompt = bot.system_prompt(second_post, triage: false)
 
       expect(system_prompt).to include(SiteSetting.title)
       expect(system_prompt).to include(SiteSetting.site_description)
@@ -41,20 +39,23 @@ RSpec.describe DiscourseAi::AiBot::Bot do
 
   describe "#reply_to" do
     it "can respond to !search" do
-      bot.system_prompt_style!(:simple)
+      expected_response = "!search test search"
 
-      expected_response = "ok, searching...\n!search test search"
+      prompt = bot.bot_prompt_with_topic_context(second_post, triage: true)
-
-      prompt = bot.bot_prompt_with_topic_context(second_post)
 
       OpenAiCompletionsInferenceStubs.stub_streamed_response(
         prompt,
         [{ content: expected_response }],
-        req_opts: bot.reply_params.merge(stream: true),
+        req_opts: bot.triage_params.merge(stream: true),
       )
 
-      prompt << { role: "assistant", content: "!search test search" }
+      # second post will contain the search instruction
-      prompt << { role: "user", content: "results: No results found" }
+      prompt = bot.bot_prompt_with_topic_context(first_post)
+
+      command = DiscourseAi::AiBot::Commands::SearchCommand.new(nil, nil, SecureRandom.hex(10))
+      content, username = bot.context_prompt(second_post, command.process, command.result_name)
+
+      prompt << bot.build_message(username, content)
 
       OpenAiCompletionsInferenceStubs.stub_streamed_response(
         prompt,
@@ -69,7 +70,6 @@ RSpec.describe DiscourseAi::AiBot::Bot do
       expect(last.raw).to include("<details>")
       expect(last.raw).to include("<summary>Search</summary>")
       expect(last.raw).not_to include("translation missing")
-      expect(last.raw).to include("ok, searching...")
       expect(last.raw).to include("We are done now")
 
       expect(last.post_custom_prompt.custom_prompt.to_s).to include("We are done now")

spec/lib/modules/ai_bot/commands/categories_command_spec.rb

@@ -7,7 +7,7 @@ RSpec.describe DiscourseAi::AiBot::Commands::CategoriesCommand do
     it "can generate correct info" do
       Fabricate(:category, name: "america", posts_year: 999)
 
-      info = DiscourseAi::AiBot::Commands::CategoriesCommand.new(nil, nil).process(nil)
+      info = DiscourseAi::AiBot::Commands::CategoriesCommand.new(nil, nil, nil).process
       expect(info).to include("america")
       expect(info).to include("999")
     end

spec/lib/modules/ai_bot/commands/command_spec.rb

@@ -4,7 +4,7 @@ require_relative "../../../../support/openai_completions_inference_stubs"
 
 RSpec.describe DiscourseAi::AiBot::Commands::Command do
   fab!(:bot_user) { User.find(DiscourseAi::AiBot::EntryPoint::GPT3_5_TURBO_ID) }
-  let(:command) { DiscourseAi::AiBot::Commands::Command.new(bot_user, nil) }
+  let(:command) { DiscourseAi::AiBot::Commands::Command.new(bot_user, nil, nil) }
 
   describe "#format_results" do
     it "can generate efficient tables of data" do

spec/lib/modules/ai_bot/commands/google_command_spec.rb

@@ -32,8 +32,8 @@ RSpec.describe DiscourseAi::AiBot::Commands::GoogleCommand do
         "https://www.googleapis.com/customsearch/v1?cx=cx&key=abc&num=10&q=some%20search%20term",
       ).to_return(status: 200, body: json_text, headers: {})
 
-      google = described_class.new(bot_user, post)
+      google = described_class.new(bot_user, post, "some search term")
-      info = google.process("some search term")
+      info = google.process
 
       expect(google.description_args[:count]).to eq(1)
       expect(info).to include("title1")

spec/lib/modules/ai_bot/commands/search_command_spec.rb

@@ -11,9 +11,9 @@ RSpec.describe DiscourseAi::AiBot::Commands::SearchCommand do
   describe "#process" do
     it "can handle no results" do
       post1 = Fabricate(:post)
-      search = described_class.new(bot_user, post1)
+      search = described_class.new(bot_user, post1, "order:fake ABDDCDCEDGDG")
 
-      results = search.process("order:fake ABDDCDCEDGDG")
+      results = search.process
       expect(results).to eq("No results found")
     end
 
@@ -23,15 +23,15 @@ RSpec.describe DiscourseAi::AiBot::Commands::SearchCommand do
       _post3 = Fabricate(:post, user: post1.user)
 
       # search has no built in support for limit: so handle it from the outside
-      search = described_class.new(bot_user, post1)
+      search = described_class.new(bot_user, post1, "@#{post1.user.username} limit:2")
 
-      results = search.process("@#{post1.user.username} limit:2")
+      results = search.process
 
       # title + 2 rows
       expect(results.split("\n").length).to eq(3)
 
-      # just searching for everything
+      search = described_class.new(bot_user, post1, "order:latest_topic")
-      results = search.process("order:latest_topic")
+      results = search.process
       expect(results.split("\n").length).to be > 1
     end
   end

spec/lib/modules/ai_bot/commands/summarize_command_spec.rb

@@ -14,8 +14,8 @@ RSpec.describe DiscourseAi::AiBot::Commands::SummarizeCommand do
         body: JSON.dump({ choices: [{ message: { content: "summary stuff" } }] }),
       )
 
-      summarizer = described_class.new(bot_user, post)
+      summarizer = described_class.new(bot_user, post, "#{post.topic_id} why did it happen?")
-      info = summarizer.process("#{post.topic_id} why did it happen?")
+      info = summarizer.process
 
       expect(info).to include("Topic summarized")
       expect(summarizer.custom_raw).to include("summary stuff")
@@ -30,8 +30,8 @@ RSpec.describe DiscourseAi::AiBot::Commands::SummarizeCommand do
       topic = Fabricate(:topic, category_id: category.id)
       post = Fabricate(:post, topic: topic)
 
-      summarizer = described_class.new(bot_user, post)
+      summarizer = described_class.new(bot_user, post, "#{post.topic_id} why did it happen?")
-      info = summarizer.process("#{post.topic_id} why did it happen?")
+      info = summarizer.process
 
       expect(info).not_to include(post.raw)
 

spec/lib/modules/ai_bot/commands/tags_command_spec.rb

@@ -10,7 +10,7 @@ RSpec.describe DiscourseAi::AiBot::Commands::TagsCommand do
       Fabricate(:tag, name: "america", public_topic_count: 100)
       Fabricate(:tag, name: "not_here", public_topic_count: 0)
 
-      info = DiscourseAi::AiBot::Commands::TagsCommand.new(nil, nil).process(nil)
+      info = DiscourseAi::AiBot::Commands::TagsCommand.new(nil, nil, nil).process
 
       expect(info).to include("america")
       expect(info).not_to include("not_here")

spec/lib/modules/ai_bot/jobs/regular/create_ai_reply_spec.rb

@@ -23,12 +23,24 @@ RSpec.describe Jobs::CreateAiReply do
         # time needs to be frozen so time in prompt does not drift
         freeze_time
 
+        OpenAiCompletionsInferenceStubs.stub_streamed_response(
+          DiscourseAi::AiBot::OpenAiBot.new(bot_user).bot_prompt_with_topic_context(
+            post,
+            triage: true,
+          ),
+          [{ content: "!noop" }],
+          req_opts: {
+            temperature: 0.1,
+            max_tokens: 100,
+            stream: true,
+          },
+        )
+
         OpenAiCompletionsInferenceStubs.stub_streamed_response(
           DiscourseAi::AiBot::OpenAiBot.new(bot_user).bot_prompt_with_topic_context(post),
           deltas,
           req_opts: {
             temperature: 0.4,
-            top_p: 0.9,
             max_tokens: 1500,
             stream: true,
           },
@@ -66,12 +78,25 @@ RSpec.describe Jobs::CreateAiReply do
     end
 
     context "when chatting with Claude from Anthropic" do
-      let(:claude_response) { "Assistant: #{expected_response}" }
+      let(:claude_response) { "#{expected_response}" }
       let(:deltas) { claude_response.split(" ").map { |w| "#{w} " } }
 
       before do
         bot_user = User.find(DiscourseAi::AiBot::EntryPoint::CLAUDE_V1_ID)
 
+        AnthropicCompletionStubs.stub_streamed_response(
+          DiscourseAi::AiBot::AnthropicBot.new(bot_user).bot_prompt_with_topic_context(
+            post,
+            triage: true,
+          ),
+          [{ content: "!noop" }],
+          req_opts: {
+            temperature: 0.1,
+            max_tokens_to_sample: 100,
+            stream: true,
+          },
+        )
+
         AnthropicCompletionStubs.stub_streamed_response(
           DiscourseAi::AiBot::AnthropicBot.new(bot_user).bot_prompt_with_topic_context(post),
           deltas,

spec/lib/modules/ai_bot/open_ai_bot_spec.rb

@@ -33,7 +33,7 @@ RSpec.describe DiscourseAi::AiBot::OpenAiBot do
       it "trims the prompt" do
         prompt_messages = subject.bot_prompt_with_topic_context(post_1)
 
-        expect(prompt_messages[-2][:role]).to eq("assistant")
+        expect(prompt_messages[-2][:role]).to eq("system")
         expect(prompt_messages[-1][:role]).to eq("user")
         # trimming is tricky... it needs to account for system message as
         # well... just make sure we trim for now