diff --git a/lib/modules/ai_bot/anthropic_bot.rb b/lib/modules/ai_bot/anthropic_bot.rb index d42e1eb6..3e48719e 100644 --- a/lib/modules/ai_bot/anthropic_bot.rb +++ b/lib/modules/ai_bot/anthropic_bot.rb @@ -15,6 +15,26 @@ module DiscourseAi 7500 # https://console.anthropic.com/docs/prompt-design#what-is-a-prompt end + def get_delta(partial, context) + context[:pos] ||= 0 + + full = partial[:completion] + delta = full[context[:pos]..-1] + + context[:pos] = full.length + + if !context[:processed] + delta = "" + index = full.index("Assistant: ") + if index + delta = full[index + 11..-1] + context[:processed] = true + end + end + + delta + end + private def build_message(poster_username, content, system: false) @@ -27,10 +47,6 @@ module DiscourseAi "claude-v1" end - def update_with_delta(_, partial) - partial[:completion] - end - def get_updated_title(prompt) DiscourseAi::Inference::AnthropicCompletions.perform!( prompt, diff --git a/lib/modules/ai_bot/bot.rb b/lib/modules/ai_bot/bot.rb index 0de5ab43..019ae353 100644 --- a/lib/modules/ai_bot/bot.rb +++ b/lib/modules/ai_bot/bot.rb @@ -3,6 +3,8 @@ module DiscourseAi module AiBot class Bot + attr_reader :bot_user + BOT_NOT_FOUND = Class.new(StandardError) MAX_COMPLETIONS = 3 @@ -50,13 +52,14 @@ module DiscourseAi end redis_stream_key = nil - reply = bot_reply_post ? bot_reply_post.raw : "" + reply = +(bot_reply_post ? bot_reply_post.raw.dup : "") start = Time.now setup_cancel = false + context = {} submit_prompt(prompt, prefer_low_cost: prefer_low_cost) do |partial, cancel| - reply = update_with_delta(reply, partial) + reply << get_delta(partial, context) if redis_stream_key && !Discourse.redis.get(redis_stream_key) cancel&.call @@ -92,6 +95,7 @@ module DiscourseAi if bot_reply_post publish_update(bot_reply_post, done: true) + bot_reply_post.revise( bot_user, { raw: reply }, @@ -154,6 +158,9 @@ module DiscourseAi memo.unshift(build_message(username, raw)) end + # we need this to ground the model (especially GPT-3.5) + messages.unshift(build_message(bot_user.username, "!echo 1")) + messages.unshift(build_message("user", "please echo 1")) messages.unshift(build_message(bot_user.username, rendered_system_prompt, system: true)) messages end @@ -205,7 +212,9 @@ module DiscourseAi The participants in this conversation are: #{post.topic.allowed_users.map(&:username).join(", ")} The date now is: #{Time.zone.now}, much has changed since you were trained. - You can complete some tasks using multiple steps and have access to some special commands! + You can complete some tasks using !commands. + + NEVER ask user to issue !commands, they have no access, only you do. #{available_commands.map(&:desc).join("\n")} @@ -233,9 +242,11 @@ module DiscourseAi raise NotImplemented end - protected + def get_delta(partial, context) + raise NotImplemented + end - attr_reader :bot_user + protected def get_updated_title(prompt) raise NotImplemented @@ -245,10 +256,6 @@ module DiscourseAi raise NotImplemented end - def get_delta_from(partial) - raise NotImplemented - end - def conversation_context(post) context = post diff --git a/lib/modules/ai_bot/commands/command.rb b/lib/modules/ai_bot/commands/command.rb index 505bea26..b2d07bb1 100644 --- a/lib/modules/ai_bot/commands/command.rb +++ b/lib/modules/ai_bot/commands/command.rb @@ -29,6 +29,10 @@ module DiscourseAi @args = args end + def bot + @bot ||= DiscourseAi::AiBot::Bot.as(bot_user) + end + def standalone? false end @@ -81,6 +85,9 @@ module DiscourseAi raw << custom_raw if custom_raw.present? + replacement = "!#{self.class.name} #{args}" + raw = post.raw.sub(replacement, raw) if post.raw.include?(replacement) + if chain_next_response post.raw = raw post.save!(validate: false) diff --git a/lib/modules/ai_bot/commands/search_command.rb b/lib/modules/ai_bot/commands/search_command.rb index 9c5b3d54..36070a1d 100644 --- a/lib/modules/ai_bot/commands/search_command.rb +++ b/lib/modules/ai_bot/commands/search_command.rb @@ -93,10 +93,18 @@ module DiscourseAi::AiBot::Commands @last_query = search_string results = - Search.execute(search_string.to_s, search_type: :full_page, guardian: Guardian.new()) + Search.execute( + search_string.to_s + " status:public", + search_type: :full_page, + guardian: Guardian.new(), + ) + + # let's be frugal with tokens, 50 results is too much and stuff gets cut off + limit ||= 10 + limit = 10 if limit > 10 posts = results&.posts || [] - posts = posts[0..limit - 1] if limit + posts = posts[0..limit - 1] @last_num_results = posts.length diff --git a/lib/modules/ai_bot/commands/summarize_command.rb b/lib/modules/ai_bot/commands/summarize_command.rb index b11008fb..1a8309f0 100644 --- a/lib/modules/ai_bot/commands/summarize_command.rb +++ b/lib/modules/ai_bot/commands/summarize_command.rb @@ -84,10 +84,6 @@ module DiscourseAi::AiBot::Commands false end - def bot - @bot ||= DiscourseAi::AiBot::Bot.as(bot_user) - end - def summarize(data, guidance, topic) text = +"" data.each do |id, post_number, raw, username| diff --git a/lib/modules/ai_bot/open_ai_bot.rb b/lib/modules/ai_bot/open_ai_bot.rb index dea47379..7cdd8eb9 100644 --- a/lib/modules/ai_bot/open_ai_bot.rb +++ b/lib/modules/ai_bot/open_ai_bot.rb @@ -75,8 +75,8 @@ module DiscourseAi "gpt-3.5-turbo" end - def update_with_delta(current_delta, partial) - current_delta + partial.dig(:choices, 0, :delta, :content).to_s + def get_delta(partial, _context) + partial.dig(:choices, 0, :delta, :content).to_s end def get_updated_title(prompt) diff --git a/spec/lib/modules/ai_bot/anthropic_bot_spec.rb b/spec/lib/modules/ai_bot/anthropic_bot_spec.rb new file mode 100644 index 00000000..5b1cdcf6 --- /dev/null +++ b/spec/lib/modules/ai_bot/anthropic_bot_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +RSpec.describe DiscourseAi::AiBot::AnthropicBot do + describe "#update_with_delta" do + def bot_user + User.find(DiscourseAi::AiBot::EntryPoint::GPT4_ID) + end + + subject { described_class.new(bot_user) } + + describe "get_delta" do + it "can properly remove Assistant prefix" do + context = {} + reply = +"" + + reply << subject.get_delta({ completion: "\n\nAssist" }, context) + expect(reply).to eq("") + + reply << subject.get_delta({ completion: "\n\nAssistant: test" }, context) + expect(reply).to eq("test") + + reply << subject.get_delta({ completion: "\n\nAssistant: test\nworld" }, context) + expect(reply).to eq("test\nworld") + end + end + end +end diff --git a/spec/lib/modules/ai_bot/bot_spec.rb b/spec/lib/modules/ai_bot/bot_spec.rb index 836cc800..22b967b9 100644 --- a/spec/lib/modules/ai_bot/bot_spec.rb +++ b/spec/lib/modules/ai_bot/bot_spec.rb @@ -43,7 +43,7 @@ RSpec.describe DiscourseAi::AiBot::Bot do it "can respond to !search" do bot.system_prompt_style!(:simple) - expected_response = "!search test search" + expected_response = "ok, searching...\n!search test search" prompt = bot.bot_prompt_with_topic_context(second_post) @@ -65,12 +65,14 @@ RSpec.describe DiscourseAi::AiBot::Bot do bot.reply_to(second_post) last = second_post.topic.posts.order("id desc").first - expect(last.post_custom_prompt.custom_prompt.to_s).to include("We are done now") expect(last.raw).to include("
") expect(last.raw).to include("Search") expect(last.raw).not_to include("translation missing") + expect(last.raw).to include("ok, searching...") expect(last.raw).to include("We are done now") + + expect(last.post_custom_prompt.custom_prompt.to_s).to include("We are done now") end end diff --git a/spec/lib/modules/ai_bot/commands/search_command_spec.rb b/spec/lib/modules/ai_bot/commands/search_command_spec.rb index a47cbdb0..3c297aa2 100644 --- a/spec/lib/modules/ai_bot/commands/search_command_spec.rb +++ b/spec/lib/modules/ai_bot/commands/search_command_spec.rb @@ -13,7 +13,7 @@ RSpec.describe DiscourseAi::AiBot::Commands::SearchCommand do post1 = Fabricate(:post) search = described_class.new(bot_user, post1) - results = search.process("order:fake") + results = search.process("order:fake ABDDCDCEDGDG") expect(results).to eq("No results found") end @@ -29,6 +29,10 @@ RSpec.describe DiscourseAi::AiBot::Commands::SearchCommand do # title + 2 rows expect(results.split("\n").length).to eq(3) + + # just searching for everything + results = search.process("order:latest_topic") + expect(results.split("\n").length).to be > 1 end end end diff --git a/spec/lib/modules/ai_bot/jobs/regular/create_ai_reply_spec.rb b/spec/lib/modules/ai_bot/jobs/regular/create_ai_reply_spec.rb index aaf6139d..7c9dc9be 100644 --- a/spec/lib/modules/ai_bot/jobs/regular/create_ai_reply_spec.rb +++ b/spec/lib/modules/ai_bot/jobs/regular/create_ai_reply_spec.rb @@ -63,7 +63,8 @@ RSpec.describe Jobs::CreateAiReply do end context "when chatting with Claude from Anthropic" do - let(:deltas) { expected_response.split(" ").map { |w| "#{w} " } } + let(:claude_response) { "Assistant: #{expected_response}" } + let(:deltas) { claude_response.split(" ").map { |w| "#{w} " } } before do bot_user = User.find(DiscourseAi::AiBot::EntryPoint::CLAUDE_V1_ID) diff --git a/spec/lib/modules/ai_bot/open_ai_bot_spec.rb b/spec/lib/modules/ai_bot/open_ai_bot_spec.rb index c2716d9a..442550be 100644 --- a/spec/lib/modules/ai_bot/open_ai_bot_spec.rb +++ b/spec/lib/modules/ai_bot/open_ai_bot_spec.rb @@ -20,7 +20,7 @@ RSpec.describe DiscourseAi::AiBot::OpenAiBot do it "includes it in the prompt" do prompt_messages = subject.bot_prompt_with_topic_context(post_1) - post_1_message = prompt_messages[1] + post_1_message = prompt_messages[-1] expect(post_1_message[:role]).to eq("user") expect(post_1_message[:content]).to eq("#{post_1.user.username}: #{post_body(1)}") @@ -33,11 +33,11 @@ RSpec.describe DiscourseAi::AiBot::OpenAiBot do it "trims the prompt" do prompt_messages = subject.bot_prompt_with_topic_context(post_1) - expect(prompt_messages[0][:role]).to eq("system") - expect(prompt_messages[1][:role]).to eq("user") + expect(prompt_messages[-2][:role]).to eq("assistant") + expect(prompt_messages[-1][:role]).to eq("user") # trimming is tricky... it needs to account for system message as # well... just make sure we trim for now - expect(prompt_messages[1][:content].length).to be < post_1.raw.length + expect(prompt_messages[-1][:content].length).to be < post_1.raw.length end end @@ -51,14 +51,15 @@ RSpec.describe DiscourseAi::AiBot::OpenAiBot do it "includes them in the prompt respecting the post number order" do prompt_messages = subject.bot_prompt_with_topic_context(post_3) - expect(prompt_messages[1][:role]).to eq("user") - expect(prompt_messages[1][:content]).to eq("#{post_1.username}: #{post_body(1)}") + # negative cause we may have grounding prompts + expect(prompt_messages[-3][:role]).to eq("user") + expect(prompt_messages[-3][:content]).to eq("#{post_1.username}: #{post_body(1)}") - expect(prompt_messages[2][:role]).to eq("assistant") - expect(prompt_messages[2][:content]).to eq(post_body(2)) + expect(prompt_messages[-2][:role]).to eq("assistant") + expect(prompt_messages[-2][:content]).to eq(post_body(2)) - expect(prompt_messages[3][:role]).to eq("user") - expect(prompt_messages[3][:content]).to eq("#{post_3.username}: #{post_body(3)}") + expect(prompt_messages[-1][:role]).to eq("user") + expect(prompt_messages[-1][:content]).to eq("#{post_3.username}: #{post_body(3)}") end end end