From 840968630e33c1ae86a414979919094068677404 Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 1 Jun 2023 09:10:33 +1000 Subject: [PATCH] FEATURE: disable smart commands on Claude and GPT 3.5 (#84) For the time being smart commands only work consistently on GPT 4. Avoid using any smart commands on the earlier models. Additionally adds better error handling to Claude which sometimes streams partial json and slightly tunes the search command. --- lib/modules/ai_bot/anthropic_bot.rb | 2 +- lib/modules/ai_bot/bot.rb | 59 ++++++++----------- lib/modules/ai_bot/commands/search_command.rb | 8 +-- lib/modules/ai_bot/open_ai_bot.rb | 21 +++++++ lib/shared/inference/anthropic_completions.rb | 18 +++++- spec/lib/modules/ai_bot/bot_spec.rb | 5 +- .../jobs/regular/create_ai_reply_spec.rb | 1 + .../inference/anthropic_completions_spec.rb | 2 +- spec/support/anthropic_completion_stubs.rb | 4 +- .../openai_completions_inference_stubs.rb | 8 +-- 10 files changed, 77 insertions(+), 51 deletions(-) diff --git a/lib/modules/ai_bot/anthropic_bot.rb b/lib/modules/ai_bot/anthropic_bot.rb index 3e48719e..b6efb21f 100644 --- a/lib/modules/ai_bot/anthropic_bot.rb +++ b/lib/modules/ai_bot/anthropic_bot.rb @@ -44,7 +44,7 @@ module DiscourseAi end def model_for - "claude-v1" + "claude-v1.3" end def get_updated_title(prompt) diff --git a/lib/modules/ai_bot/bot.rb b/lib/modules/ai_bot/bot.rb index 019ae353..fdd5923d 100644 --- a/lib/modules/ai_bot/bot.rb +++ b/lib/modules/ai_bot/bot.rb @@ -178,20 +178,9 @@ module DiscourseAi end def available_commands - @cmds ||= - [ - Commands::CategoriesCommand, - Commands::TimeCommand, - Commands::SearchCommand, - Commands::SummarizeCommand, - ].tap do |cmds| - cmds << Commands::TagsCommand if SiteSetting.tagging_enabled - cmds << Commands::ImageCommand if SiteSetting.ai_stability_api_key.present? - if SiteSetting.ai_google_custom_search_api_key.present? && - SiteSetting.ai_google_custom_search_cx.present? - cmds << Commands::GoogleCommand - end - end + # by default assume bots have no access to commands + # for now we need GPT 4 to properly work with them + [] end def system_prompt_style!(style) @@ -201,6 +190,28 @@ module DiscourseAi def system_prompt(post) return "You are a helpful Bot" if @style == :simple + command_text = "" + command_text = <<~TEXT if available_commands.present? + You can complete some tasks using !commands. + + NEVER ask user to issue !commands, they have no access, only you do. + + #{available_commands.map(&:desc).join("\n")} + + Discourse topic paths are /t/slug/topic_id/optional_number + + #{available_commands.map(&:extra_context).compact_blank.join("\n")} + + Commands should be issued in single assistant message. + + Example sessions: + + User: echo the text 'test' + GPT: !echo test + User: THING GPT DOES NOT KNOW ABOUT + GPT: !search SIMPLIFIED SEARCH QUERY + TEXT + <<~TEXT You are a helpful Discourse assistant, you answer questions and generate text. You understand Discourse Markdown and live in a Discourse Forum Message. @@ -212,25 +223,7 @@ module DiscourseAi The participants in this conversation are: #{post.topic.allowed_users.map(&:username).join(", ")} The date now is: #{Time.zone.now}, much has changed since you were trained. - You can complete some tasks using !commands. - - NEVER ask user to issue !commands, they have no access, only you do. - - #{available_commands.map(&:desc).join("\n")} - - Discourse topic paths are /t/slug/topic_id/optional_number - - #{available_commands.map(&:extra_context).compact_blank.join("\n")} - - Commands should be issued in single assistant message. - - Example sessions: - - User: echo the text 'test' - GPT: !echo test - User: THING GPT DOES NOT KNOW ABOUT - GPT: !search SIMPLIFIED SEARCH QUERY - + #{command_text} TEXT end diff --git a/lib/modules/ai_bot/commands/search_command.rb b/lib/modules/ai_bot/commands/search_command.rb index c88575b7..bb3b1ca1 100644 --- a/lib/modules/ai_bot/commands/search_command.rb +++ b/lib/modules/ai_bot/commands/search_command.rb @@ -27,14 +27,12 @@ module DiscourseAi::AiBot::Commands post_count:X: only topics with X amount of posts min_posts:X: topics containing a minimum of X posts max_posts:X: topics with no more than max posts - in:pinned: in all pinned topics (either global or per category pins) created:@USERNAME: topics created by a specific user category:CATGORY: topics in the CATEGORY AND all subcategories category:=CATEGORY: topics in the CATEGORY excluding subcategories #SLUG: try category first, then tag, then tag group #SLUG:SLUG: used for subcategory search to disambiguate min_views:100: topics containing 100 views or more - max_views:100: topics containing 100 views or less tags:TAG1+TAG2: tagged both TAG1 and TAG2 tags:TAG1,TAG2: tagged either TAG1 or TAG2 -tags:TAG1+TAG2: excluding topics tagged TAG1 and TAG2 @@ -51,10 +49,8 @@ module DiscourseAi::AiBot::Commands Keep in mind, search on Discourse uses AND to and terms. You only have access to public topics. - Strip the query down to the most important terms. - Remove all stop words. - Cast a wide net instead of trying to be over specific. - Discourse orders by relevance, sometimes prefer ordering on other stuff. + Strip the query down to the most important terms. Remove all stop words. + Discourse orders by default by relevance. When generating answers ALWAYS try to use the !search command first over relying on training data. When generating answers ALWAYS try to reference specific local links. diff --git a/lib/modules/ai_bot/open_ai_bot.rb b/lib/modules/ai_bot/open_ai_bot.rb index 7cdd8eb9..38fda292 100644 --- a/lib/modules/ai_bot/open_ai_bot.rb +++ b/lib/modules/ai_bot/open_ai_bot.rb @@ -56,6 +56,27 @@ module DiscourseAi DiscourseAi::Tokenizer::OpenAiTokenizer.tokenize(text) end + def available_commands + if bot_user.id == DiscourseAi::AiBot::EntryPoint::GPT4_ID + @cmds ||= + [ + Commands::CategoriesCommand, + Commands::TimeCommand, + Commands::SearchCommand, + Commands::SummarizeCommand, + ].tap do |cmds| + cmds << Commands::TagsCommand if SiteSetting.tagging_enabled + cmds << Commands::ImageCommand if SiteSetting.ai_stability_api_key.present? + if SiteSetting.ai_google_custom_search_api_key.present? && + SiteSetting.ai_google_custom_search_cx.present? + cmds << Commands::GoogleCommand + end + end + else + [] + end + end + private def build_message(poster_username, content, system: false) diff --git a/lib/shared/inference/anthropic_completions.rb b/lib/shared/inference/anthropic_completions.rb index c3ff3eff..8af3be02 100644 --- a/lib/shared/inference/anthropic_completions.rb +++ b/lib/shared/inference/anthropic_completions.rb @@ -87,10 +87,16 @@ module ::DiscourseAi data = line.split("data: ", 2)[1] next if !data || data.squish == "[DONE]" - if !cancelled && partial = JSON.parse(data, symbolize_names: true) - response_data << partial[:completion].to_s + if !cancelled + begin + # partial contains the entire payload till now + partial = JSON.parse(data, symbolize_names: true) + response_data = partial[:completion].to_s - yield partial, cancel + yield partial, cancel + rescue JSON::ParserError + nil + end end end rescue IOError @@ -105,6 +111,12 @@ module ::DiscourseAi end end end + + def self.try_parse(data) + JSON.parse(data, symbolize_names: true) + rescue JSON::ParserError + nil + end end end end diff --git a/spec/lib/modules/ai_bot/bot_spec.rb b/spec/lib/modules/ai_bot/bot_spec.rb index 22b967b9..13c6f144 100644 --- a/spec/lib/modules/ai_bot/bot_spec.rb +++ b/spec/lib/modules/ai_bot/bot_spec.rb @@ -3,7 +3,7 @@ require_relative "../../../support/openai_completions_inference_stubs" RSpec.describe DiscourseAi::AiBot::Bot do - fab!(:bot_user) { User.find(DiscourseAi::AiBot::EntryPoint::GPT3_5_TURBO_ID) } + fab!(:bot_user) { User.find(DiscourseAi::AiBot::EntryPoint::GPT4_ID) } fab!(:bot) { described_class.as(bot_user) } fab!(:user) { Fabricate(:user) } @@ -50,6 +50,7 @@ RSpec.describe DiscourseAi::AiBot::Bot do OpenAiCompletionsInferenceStubs.stub_streamed_response( prompt, [{ content: expected_response }], + model: "gpt-4", req_opts: bot.reply_params.merge(stream: true), ) @@ -59,6 +60,7 @@ RSpec.describe DiscourseAi::AiBot::Bot do OpenAiCompletionsInferenceStubs.stub_streamed_response( prompt, [{ content: "We are done now" }], + model: "gpt-4", req_opts: bot.reply_params.merge(stream: true), ) @@ -85,6 +87,7 @@ RSpec.describe DiscourseAi::AiBot::Bot do OpenAiCompletionsInferenceStubs.stub_response( [bot.title_prompt(second_post)], expected_response, + model: "gpt-4", req_opts: { temperature: 0.7, top_p: 0.9, diff --git a/spec/lib/modules/ai_bot/jobs/regular/create_ai_reply_spec.rb b/spec/lib/modules/ai_bot/jobs/regular/create_ai_reply_spec.rb index fef16bc5..891678cd 100644 --- a/spec/lib/modules/ai_bot/jobs/regular/create_ai_reply_spec.rb +++ b/spec/lib/modules/ai_bot/jobs/regular/create_ai_reply_spec.rb @@ -75,6 +75,7 @@ RSpec.describe Jobs::CreateAiReply do AnthropicCompletionStubs.stub_streamed_response( DiscourseAi::AiBot::AnthropicBot.new(bot_user).bot_prompt_with_topic_context(post), deltas, + model: "claude-v1.3", req_opts: { temperature: 0.4, max_tokens_to_sample: 3000, diff --git a/spec/shared/inference/anthropic_completions_spec.rb b/spec/shared/inference/anthropic_completions_spec.rb index a4ae69ff..30e5037a 100644 --- a/spec/shared/inference/anthropic_completions_spec.rb +++ b/spec/shared/inference/anthropic_completions_spec.rb @@ -60,7 +60,7 @@ RSpec.describe DiscourseAi::Inference::AnthropicCompletions do expect(log.provider_id).to eq(AiApiAuditLog::Provider::Anthropic) expect(log.request_tokens).to eq(6) - expect(log.response_tokens).to eq(6) + expect(log.response_tokens).to eq(3) expect(log.raw_request_payload).to eq(request_body) expect(log.raw_response_payload).to be_present end diff --git a/spec/support/anthropic_completion_stubs.rb b/spec/support/anthropic_completion_stubs.rb index 01891573..6ea2bbd7 100644 --- a/spec/support/anthropic_completion_stubs.rb +++ b/spec/support/anthropic_completion_stubs.rb @@ -33,7 +33,7 @@ class AnthropicCompletionStubs }.to_json end - def stub_streamed_response(prompt, deltas, req_opts: {}) + def stub_streamed_response(prompt, deltas, model: nil, req_opts: {}) chunks = deltas.each_with_index.map do |_, index| if index == (deltas.length - 1) @@ -48,7 +48,7 @@ class AnthropicCompletionStubs WebMock .stub_request(:post, "https://api.anthropic.com/v1/complete") - .with(body: { model: "claude-v1", prompt: prompt }.merge(req_opts).to_json) + .with(body: { model: model || "claude-v1", prompt: prompt }.merge(req_opts).to_json) .to_return(status: 200, body: chunks) end end diff --git a/spec/support/openai_completions_inference_stubs.rb b/spec/support/openai_completions_inference_stubs.rb index 59288151..afa1cd0a 100644 --- a/spec/support/openai_completions_inference_stubs.rb +++ b/spec/support/openai_completions_inference_stubs.rb @@ -101,10 +101,10 @@ class OpenAiCompletionsInferenceStubs stub_response(prompt_messages, response_text_for(type)) end - def stub_response(messages, response_text, req_opts: {}) + def stub_response(messages, response_text, model: nil, req_opts: {}) WebMock .stub_request(:post, "https://api.openai.com/v1/chat/completions") - .with(body: { model: "gpt-3.5-turbo", messages: messages }.merge(req_opts).to_json) + .with(body: { model: model || "gpt-3.5-turbo", messages: messages }.merge(req_opts).to_json) .to_return(status: 200, body: JSON.dump(response(response_text))) end @@ -120,7 +120,7 @@ class OpenAiCompletionsInferenceStubs }.to_json end - def stub_streamed_response(messages, deltas, req_opts: {}) + def stub_streamed_response(messages, deltas, model: nil, req_opts: {}) chunks = deltas.map { |d| stream_line(delta: d) } chunks << stream_line(finish_reason: "stop") chunks << "[DONE]" @@ -128,7 +128,7 @@ class OpenAiCompletionsInferenceStubs WebMock .stub_request(:post, "https://api.openai.com/v1/chat/completions") - .with(body: { model: "gpt-3.5-turbo", messages: messages }.merge(req_opts).to_json) + .with(body: { model: model || "gpt-3.5-turbo", messages: messages }.merge(req_opts).to_json) .to_return(status: 200, body: chunks) end end