FIX: setting explorer was exceeding token budget

This refactor changes it so we only include minimal data in the
system prompt which leaves us lots of tokens for specific searches

The new search command allows us to pull in settings on demand

Descriptions are include in short search results, and names only
in longer results

Also: 

* In dev it is important to tell when calls are made to open ai
this adds a console log to increase awareness around token usage

* PERF: stop counting tokens so often

This changes it so we only count tokens once per response

Previously each time we heard back from open ai we would count
tokens, leading to uneeded delays

* bug fix, commands may reach in for tokenizer

* add logging to console for anthropic calls as well

* Update lib/shared/inference/openai_completions.rb

Co-authored-by: Martin Brennan <mjrbrennan@gmail.com>

config/locales/server.en.yml

@@ -120,6 +120,7 @@ en:
         read: "Read topic"
         setting_context: "Look up site setting context"
         schema: "Look up database schema"
+        search_settings: "Searching site settings"
       command_description:
         read: "Reading: <a href='%{url}'>%{title}</a>"
         time: "Time in %{timezone} is %{time}"
@@ -139,6 +140,9 @@ en:
           other: "Found %{count} <a href='%{url}'>results</a> for '%{query}'"
         setting_context: "Reading context for: %{setting_name}"
         schema: "%{tables}"
+        search_settings:
+          one: "Found %{count} result for '%{query}'"
+          other: "Found %{count} results for '%{query}'"
 
     summarization:
       configuration_hint:

lib/modules/ai_bot/anthropic_bot.rb

@@ -28,6 +28,10 @@ module DiscourseAi
         completion
       end
 
+      def tokenizer
+        DiscourseAi::Tokenizer::AnthropicTokenizer
+      end
+
       private
 
       def build_message(poster_username, content, system: false, function: nil)
@@ -58,10 +62,6 @@ module DiscourseAi
           &blk
         )
       end
-
-      def tokenizer
-        DiscourseAi::Tokenizer::AnthropicTokenizer
-      end
     end
   end
 end

lib/modules/ai_bot/commands/search_settings_command.rb

@@ -0,0 +1,85 @@
+#frozen_string_literal: true
+
+module DiscourseAi::AiBot::Commands
+  class SearchSettingsCommand < Command
+    class << self
+      def name
+        "search_settings"
+      end
+
+      def desc
+        "Will search through site settings and return top 20 results"
+      end
+
+      def parameters
+        [
+          Parameter.new(
+            name: "query",
+            description:
+              "comma delimited list of settings to search for (e.g. 'setting_1,setting_2')",
+            type: "string",
+            required: true,
+          ),
+        ]
+      end
+    end
+
+    def result_name
+      "results"
+    end
+
+    def description_args
+      { count: @last_num_results || 0, query: @last_query || "" }
+    end
+
+    INCLUDE_DESCRIPTIONS_MAX_LENGTH = 10
+    MAX_RESULTS = 200
+
+    def process(query:)
+      @last_query = query
+      @last_num_results = 0
+
+      terms = query.split(",").map(&:strip).map(&:downcase).reject(&:blank?)
+
+      found =
+        SiteSetting.all_settings.filter do |setting|
+          name = setting[:setting].to_s.downcase
+          description = setting[:description].to_s.downcase
+          plugin = setting[:plugin].to_s.downcase
+
+          search_string = "#{name} #{description} #{plugin}"
+
+          terms.any? { |term| search_string.include?(term) }
+        end
+
+      if found.blank?
+        {
+          args: {
+            query: query,
+          },
+          rows: [],
+          instruction: "no settings matched #{query}, expand your search",
+        }
+      else
+        include_descriptions = false
+
+        if found.length > MAX_RESULTS
+          found = found[0..MAX_RESULTS]
+        elsif found.length < INCLUDE_DESCRIPTIONS_MAX_LENGTH
+          include_descriptions = true
+        end
+
+        @last_num_results = found.length
+
+        format_results(found, args: { query: query }) do |setting|
+          result = { name: setting[:setting] }
+          if include_descriptions
+            result[:description] = setting[:description]
+            result[:plugin] = setting[:plugin]
+          end
+          result
+        end
+      end
+    end
+  end
+end

lib/modules/ai_bot/entry_point.rb

@@ -39,6 +39,7 @@ module DiscourseAi
         require_relative "commands/google_command"
         require_relative "commands/read_command"
         require_relative "commands/setting_context_command"
+        require_relative "commands/search_settings_command"
         require_relative "commands/db_schema_command"
         require_relative "personas/persona"
         require_relative "personas/artist"

lib/modules/ai_bot/personas/settings_explorer.rb

@@ -9,24 +9,23 @@ module DiscourseAi
         end
 
         def all_available_commands
-          [DiscourseAi::AiBot::Commands::SettingContextCommand]
+          [
+            DiscourseAi::AiBot::Commands::SettingContextCommand,
+            DiscourseAi::AiBot::Commands::SearchSettingsCommand,
+          ]
         end
 
         def system_prompt
           <<~PROMPT
             You are Discourse Site settings bot.
 
-            - You know the full list of all the site settings.
+            - You are able to find information about all the site settings.
             - You are able to request context for a specific setting.
             - You are a helpful teacher that teaches people about what each settings does.
+            - Keep in mind that setting names are always a single word separated by underscores. eg. 'site_description'
 
             Current time is: {time}
 
-            Full list of all the site settings:
-            {{
-            #{SiteSetting.all_settings.map { |setting| setting[:setting].to_s }.join("\n")}
-            }}
-
             {commands}
 
           PROMPT

lib/shared/inference/anthropic_completions.rb

@@ -14,6 +14,10 @@ module ::DiscourseAi
         max_tokens: nil,
         user_id: nil
       )
+        log = nil
+        response_data = +""
+        response_raw = +""
+
         url = URI("https://api.anthropic.com/v1/complete")
         headers = {
           "anthropic-version" => "2023-06-01",
@@ -68,12 +72,9 @@ module ::DiscourseAi
               return parsed_response
             end
 
-            response_data = +""
-
             begin
               cancelled = false
               cancel = lambda { cancelled = true }
-              response_raw = +""
 
               response.read_body do |chunk|
                 if cancelled
@@ -104,16 +105,21 @@ module ::DiscourseAi
                   end
               rescue IOError
                 raise if !cancelled
+              end
+            end
+
+            return response_data
+          end
         ensure
+          if block_given?
             log.update!(
               raw_response_payload: response_raw,
               request_tokens: DiscourseAi::Tokenizer::AnthropicTokenizer.size(prompt),
               response_tokens: DiscourseAi::Tokenizer::AnthropicTokenizer.size(response_data),
             )
           end
-            end
-
-            return response_data
+          if Rails.env.development? && log
+            puts "AnthropicCompletions: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}"
           end
         end
 

lib/shared/inference/openai_completions.rb

@@ -15,6 +15,10 @@ module ::DiscourseAi
         functions: nil,
         user_id: nil
       )
+        log = nil
+        response_data = +""
+        response_raw = +""
+
         url =
           if model.include?("gpt-4")
             if model.include?("32k")
@@ -84,12 +88,9 @@ module ::DiscourseAi
               return parsed_response
             end
 
-            response_data = +""
-
             begin
               cancelled = false
               cancel = lambda { cancelled = true }
-              response_raw = +""
 
               leftover = ""
 
@@ -125,19 +126,25 @@ module ::DiscourseAi
                   end
               rescue IOError
                 raise if !cancelled
-              ensure
-                log.update!(
-                  raw_response_payload: response_raw,
-                  request_tokens:
-                    DiscourseAi::Tokenizer::OpenAiTokenizer.size(extract_prompt(messages)),
-                  response_tokens: DiscourseAi::Tokenizer::OpenAiTokenizer.size(response_data),
-                )
               end
             end
 
             return response_data
           end
         end
+      ensure
+        if log && block_given?
+          request_tokens = DiscourseAi::Tokenizer::OpenAiTokenizer.size(extract_prompt(messages))
+          response_tokens = DiscourseAi::Tokenizer::OpenAiTokenizer.size(response_data)
+          log.update!(
+            raw_response_payload: response_raw,
+            request_tokens: request_tokens,
+            response_tokens: response_tokens,
+          )
+        end
+        if log && Rails.env.development?
+          puts "OpenAiCompletions: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}"
+        end
       end
 
       def self.extract_prompt(messages)

spec/lib/modules/ai_bot/commands/search_settings_command_spec.rb

@@ -0,0 +1,29 @@
+#frozen_string_literal: true
+
+RSpec.describe DiscourseAi::AiBot::Commands::SearchSettingsCommand do
+  let(:search) { described_class.new(bot_user: nil, args: nil) }
+
+  describe "#process" do
+    it "can handle no results" do
+      results = search.process(query: "this will not exist frogs")
+      expect(results[:args]).to eq({ query: "this will not exist frogs" })
+      expect(results[:rows]).to eq([])
+    end
+
+    it "can return more many settings with no descriptions if there are lots of hits" do
+      results = search.process(query: "a")
+
+      expect(results[:rows].length).to be > 30
+      expect(results[:rows][0].length).to eq(1)
+    end
+
+    it "can return descriptions if there are few matches" do
+      results =
+        search.process(query: "this will not be found!@,default_locale,ai_bot_enabled_personas")
+
+      expect(results[:rows].length).to eq(2)
+
+      expect(results[:rows][0][1]).not_to eq(nil)
+    end
+  end
+end

spec/lib/modules/ai_bot/personas/settings_explorer_spec.rb

@@ -7,11 +7,17 @@ RSpec.describe DiscourseAi::AiBot::Personas::SettingsExplorer do
 
   it "renders schema" do
     prompt = settings_explorer.render_system_prompt
-    # check we render settings
-    expect(prompt).to include("ai_bot_enabled_personas")
+
+    # check we do not render plugin settings
+    expect(prompt).not_to include("ai_bot_enabled_personas")
+
+    expect(prompt).to include("site_description")
 
     expect(settings_explorer.available_commands).to eq(
-      [DiscourseAi::AiBot::Commands::SettingContextCommand],
+      [
+        DiscourseAi::AiBot::Commands::SettingContextCommand,
+        DiscourseAi::AiBot::Commands::SearchSettingsCommand,
+      ],
     )
   end
 end