From 181113159bf5b3c8c7dfcdc491f52addad953993 Mon Sep 17 00:00:00 2001 From: Sam Date: Fri, 1 Sep 2023 11:48:51 +1000 Subject: [PATCH] FIX: setting explorer was exceeding token budget This refactor changes it so we only include minimal data in the system prompt which leaves us lots of tokens for specific searches The new search command allows us to pull in settings on demand Descriptions are include in short search results, and names only in longer results Also: * In dev it is important to tell when calls are made to open ai this adds a console log to increase awareness around token usage * PERF: stop counting tokens so often This changes it so we only count tokens once per response Previously each time we heard back from open ai we would count tokens, leading to uneeded delays * bug fix, commands may reach in for tokenizer * add logging to console for anthropic calls as well * Update lib/shared/inference/openai_completions.rb Co-authored-by: Martin Brennan --- config/locales/server.en.yml | 4 + lib/modules/ai_bot/anthropic_bot.rb | 8 +- .../commands/search_settings_command.rb | 85 +++++++++++++++++++ lib/modules/ai_bot/entry_point.rb | 1 + .../ai_bot/personas/settings_explorer.rb | 13 ++- lib/shared/inference/anthropic_completions.rb | 24 ++++-- lib/shared/inference/openai_completions.rb | 27 +++--- .../commands/search_settings_command_spec.rb | 29 +++++++ .../ai_bot/personas/settings_explorer_spec.rb | 12 ++- 9 files changed, 170 insertions(+), 33 deletions(-) create mode 100644 lib/modules/ai_bot/commands/search_settings_command.rb create mode 100644 spec/lib/modules/ai_bot/commands/search_settings_command_spec.rb diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index b70b072c..3c2104f4 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -120,6 +120,7 @@ en: read: "Read topic" setting_context: "Look up site setting context" schema: "Look up database schema" + search_settings: "Searching site settings" command_description: read: "Reading: %{title}" time: "Time in %{timezone} is %{time}" @@ -139,6 +140,9 @@ en: other: "Found %{count} results for '%{query}'" setting_context: "Reading context for: %{setting_name}" schema: "%{tables}" + search_settings: + one: "Found %{count} result for '%{query}'" + other: "Found %{count} results for '%{query}'" summarization: configuration_hint: diff --git a/lib/modules/ai_bot/anthropic_bot.rb b/lib/modules/ai_bot/anthropic_bot.rb index c71dd95c..364e9df8 100644 --- a/lib/modules/ai_bot/anthropic_bot.rb +++ b/lib/modules/ai_bot/anthropic_bot.rb @@ -28,6 +28,10 @@ module DiscourseAi completion end + def tokenizer + DiscourseAi::Tokenizer::AnthropicTokenizer + end + private def build_message(poster_username, content, system: false, function: nil) @@ -58,10 +62,6 @@ module DiscourseAi &blk ) end - - def tokenizer - DiscourseAi::Tokenizer::AnthropicTokenizer - end end end end diff --git a/lib/modules/ai_bot/commands/search_settings_command.rb b/lib/modules/ai_bot/commands/search_settings_command.rb new file mode 100644 index 00000000..b1c86337 --- /dev/null +++ b/lib/modules/ai_bot/commands/search_settings_command.rb @@ -0,0 +1,85 @@ +#frozen_string_literal: true + +module DiscourseAi::AiBot::Commands + class SearchSettingsCommand < Command + class << self + def name + "search_settings" + end + + def desc + "Will search through site settings and return top 20 results" + end + + def parameters + [ + Parameter.new( + name: "query", + description: + "comma delimited list of settings to search for (e.g. 'setting_1,setting_2')", + type: "string", + required: true, + ), + ] + end + end + + def result_name + "results" + end + + def description_args + { count: @last_num_results || 0, query: @last_query || "" } + end + + INCLUDE_DESCRIPTIONS_MAX_LENGTH = 10 + MAX_RESULTS = 200 + + def process(query:) + @last_query = query + @last_num_results = 0 + + terms = query.split(",").map(&:strip).map(&:downcase).reject(&:blank?) + + found = + SiteSetting.all_settings.filter do |setting| + name = setting[:setting].to_s.downcase + description = setting[:description].to_s.downcase + plugin = setting[:plugin].to_s.downcase + + search_string = "#{name} #{description} #{plugin}" + + terms.any? { |term| search_string.include?(term) } + end + + if found.blank? + { + args: { + query: query, + }, + rows: [], + instruction: "no settings matched #{query}, expand your search", + } + else + include_descriptions = false + + if found.length > MAX_RESULTS + found = found[0..MAX_RESULTS] + elsif found.length < INCLUDE_DESCRIPTIONS_MAX_LENGTH + include_descriptions = true + end + + @last_num_results = found.length + + format_results(found, args: { query: query }) do |setting| + result = { name: setting[:setting] } + if include_descriptions + result[:description] = setting[:description] + result[:plugin] = setting[:plugin] + end + result + end + end + end + end +end diff --git a/lib/modules/ai_bot/entry_point.rb b/lib/modules/ai_bot/entry_point.rb index da31adb4..711efb27 100644 --- a/lib/modules/ai_bot/entry_point.rb +++ b/lib/modules/ai_bot/entry_point.rb @@ -39,6 +39,7 @@ module DiscourseAi require_relative "commands/google_command" require_relative "commands/read_command" require_relative "commands/setting_context_command" + require_relative "commands/search_settings_command" require_relative "commands/db_schema_command" require_relative "personas/persona" require_relative "personas/artist" diff --git a/lib/modules/ai_bot/personas/settings_explorer.rb b/lib/modules/ai_bot/personas/settings_explorer.rb index 5dff3b57..676dab3d 100644 --- a/lib/modules/ai_bot/personas/settings_explorer.rb +++ b/lib/modules/ai_bot/personas/settings_explorer.rb @@ -9,24 +9,23 @@ module DiscourseAi end def all_available_commands - [DiscourseAi::AiBot::Commands::SettingContextCommand] + [ + DiscourseAi::AiBot::Commands::SettingContextCommand, + DiscourseAi::AiBot::Commands::SearchSettingsCommand, + ] end def system_prompt <<~PROMPT You are Discourse Site settings bot. - - You know the full list of all the site settings. + - You are able to find information about all the site settings. - You are able to request context for a specific setting. - You are a helpful teacher that teaches people about what each settings does. + - Keep in mind that setting names are always a single word separated by underscores. eg. 'site_description' Current time is: {time} - Full list of all the site settings: - {{ - #{SiteSetting.all_settings.map { |setting| setting[:setting].to_s }.join("\n")} - }} - {commands} PROMPT diff --git a/lib/shared/inference/anthropic_completions.rb b/lib/shared/inference/anthropic_completions.rb index db9f6496..f6f2b920 100644 --- a/lib/shared/inference/anthropic_completions.rb +++ b/lib/shared/inference/anthropic_completions.rb @@ -14,6 +14,10 @@ module ::DiscourseAi max_tokens: nil, user_id: nil ) + log = nil + response_data = +"" + response_raw = +"" + url = URI("https://api.anthropic.com/v1/complete") headers = { "anthropic-version" => "2023-06-01", @@ -68,12 +72,9 @@ module ::DiscourseAi return parsed_response end - response_data = +"" - begin cancelled = false cancel = lambda { cancelled = true } - response_raw = +"" response.read_body do |chunk| if cancelled @@ -104,17 +105,22 @@ module ::DiscourseAi end rescue IOError raise if !cancelled - ensure - log.update!( - raw_response_payload: response_raw, - request_tokens: DiscourseAi::Tokenizer::AnthropicTokenizer.size(prompt), - response_tokens: DiscourseAi::Tokenizer::AnthropicTokenizer.size(response_data), - ) end end return response_data end + ensure + if block_given? + log.update!( + raw_response_payload: response_raw, + request_tokens: DiscourseAi::Tokenizer::AnthropicTokenizer.size(prompt), + response_tokens: DiscourseAi::Tokenizer::AnthropicTokenizer.size(response_data), + ) + end + if Rails.env.development? && log + puts "AnthropicCompletions: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}" + end end def self.try_parse(data) diff --git a/lib/shared/inference/openai_completions.rb b/lib/shared/inference/openai_completions.rb index 5a445c30..2a0d5828 100644 --- a/lib/shared/inference/openai_completions.rb +++ b/lib/shared/inference/openai_completions.rb @@ -15,6 +15,10 @@ module ::DiscourseAi functions: nil, user_id: nil ) + log = nil + response_data = +"" + response_raw = +"" + url = if model.include?("gpt-4") if model.include?("32k") @@ -84,12 +88,9 @@ module ::DiscourseAi return parsed_response end - response_data = +"" - begin cancelled = false cancel = lambda { cancelled = true } - response_raw = +"" leftover = "" @@ -125,19 +126,25 @@ module ::DiscourseAi end rescue IOError raise if !cancelled - ensure - log.update!( - raw_response_payload: response_raw, - request_tokens: - DiscourseAi::Tokenizer::OpenAiTokenizer.size(extract_prompt(messages)), - response_tokens: DiscourseAi::Tokenizer::OpenAiTokenizer.size(response_data), - ) end end return response_data end end + ensure + if log && block_given? + request_tokens = DiscourseAi::Tokenizer::OpenAiTokenizer.size(extract_prompt(messages)) + response_tokens = DiscourseAi::Tokenizer::OpenAiTokenizer.size(response_data) + log.update!( + raw_response_payload: response_raw, + request_tokens: request_tokens, + response_tokens: response_tokens, + ) + end + if log && Rails.env.development? + puts "OpenAiCompletions: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}" + end end def self.extract_prompt(messages) diff --git a/spec/lib/modules/ai_bot/commands/search_settings_command_spec.rb b/spec/lib/modules/ai_bot/commands/search_settings_command_spec.rb new file mode 100644 index 00000000..e9bb61bd --- /dev/null +++ b/spec/lib/modules/ai_bot/commands/search_settings_command_spec.rb @@ -0,0 +1,29 @@ +#frozen_string_literal: true + +RSpec.describe DiscourseAi::AiBot::Commands::SearchSettingsCommand do + let(:search) { described_class.new(bot_user: nil, args: nil) } + + describe "#process" do + it "can handle no results" do + results = search.process(query: "this will not exist frogs") + expect(results[:args]).to eq({ query: "this will not exist frogs" }) + expect(results[:rows]).to eq([]) + end + + it "can return more many settings with no descriptions if there are lots of hits" do + results = search.process(query: "a") + + expect(results[:rows].length).to be > 30 + expect(results[:rows][0].length).to eq(1) + end + + it "can return descriptions if there are few matches" do + results = + search.process(query: "this will not be found!@,default_locale,ai_bot_enabled_personas") + + expect(results[:rows].length).to eq(2) + + expect(results[:rows][0][1]).not_to eq(nil) + end + end +end diff --git a/spec/lib/modules/ai_bot/personas/settings_explorer_spec.rb b/spec/lib/modules/ai_bot/personas/settings_explorer_spec.rb index 68a7beae..39ab3d12 100644 --- a/spec/lib/modules/ai_bot/personas/settings_explorer_spec.rb +++ b/spec/lib/modules/ai_bot/personas/settings_explorer_spec.rb @@ -7,11 +7,17 @@ RSpec.describe DiscourseAi::AiBot::Personas::SettingsExplorer do it "renders schema" do prompt = settings_explorer.render_system_prompt - # check we render settings - expect(prompt).to include("ai_bot_enabled_personas") + + # check we do not render plugin settings + expect(prompt).not_to include("ai_bot_enabled_personas") + + expect(prompt).to include("site_description") expect(settings_explorer.available_commands).to eq( - [DiscourseAi::AiBot::Commands::SettingContextCommand], + [ + DiscourseAi::AiBot::Commands::SettingContextCommand, + DiscourseAi::AiBot::Commands::SearchSettingsCommand, + ], ) end end