FIX: setting explorer was exceeding token budget

This refactor changes it so we only include minimal data in the
system prompt which leaves us lots of tokens for specific searches

The new search command allows us to pull in settings on demand

Descriptions are include in short search results, and names only
in longer results

Also: 

* In dev it is important to tell when calls are made to open ai
this adds a console log to increase awareness around token usage

* PERF: stop counting tokens so often

This changes it so we only count tokens once per response

Previously each time we heard back from open ai we would count
tokens, leading to uneeded delays

* bug fix, commands may reach in for tokenizer

* add logging to console for anthropic calls as well

* Update lib/shared/inference/openai_completions.rb

Co-authored-by: Martin Brennan <mjrbrennan@gmail.com>
This commit is contained in:
Sam 2023-09-01 11:48:51 +10:00 committed by GitHub
parent 65091690eb
commit 181113159b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 170 additions and 33 deletions

View File

@ -120,6 +120,7 @@ en:
read: "Read topic"
setting_context: "Look up site setting context"
schema: "Look up database schema"
search_settings: "Searching site settings"
command_description:
read: "Reading: <a href='%{url}'>%{title}</a>"
time: "Time in %{timezone} is %{time}"
@ -139,6 +140,9 @@ en:
other: "Found %{count} <a href='%{url}'>results</a> for '%{query}'"
setting_context: "Reading context for: %{setting_name}"
schema: "%{tables}"
search_settings:
one: "Found %{count} result for '%{query}'"
other: "Found %{count} results for '%{query}'"
summarization:
configuration_hint:

View File

@ -28,6 +28,10 @@ module DiscourseAi
completion
end
def tokenizer
DiscourseAi::Tokenizer::AnthropicTokenizer
end
private
def build_message(poster_username, content, system: false, function: nil)
@ -58,10 +62,6 @@ module DiscourseAi
&blk
)
end
def tokenizer
DiscourseAi::Tokenizer::AnthropicTokenizer
end
end
end
end

View File

@ -0,0 +1,85 @@
#frozen_string_literal: true
module DiscourseAi::AiBot::Commands
class SearchSettingsCommand < Command
class << self
def name
"search_settings"
end
def desc
"Will search through site settings and return top 20 results"
end
def parameters
[
Parameter.new(
name: "query",
description:
"comma delimited list of settings to search for (e.g. 'setting_1,setting_2')",
type: "string",
required: true,
),
]
end
end
def result_name
"results"
end
def description_args
{ count: @last_num_results || 0, query: @last_query || "" }
end
INCLUDE_DESCRIPTIONS_MAX_LENGTH = 10
MAX_RESULTS = 200
def process(query:)
@last_query = query
@last_num_results = 0
terms = query.split(",").map(&:strip).map(&:downcase).reject(&:blank?)
found =
SiteSetting.all_settings.filter do |setting|
name = setting[:setting].to_s.downcase
description = setting[:description].to_s.downcase
plugin = setting[:plugin].to_s.downcase
search_string = "#{name} #{description} #{plugin}"
terms.any? { |term| search_string.include?(term) }
end
if found.blank?
{
args: {
query: query,
},
rows: [],
instruction: "no settings matched #{query}, expand your search",
}
else
include_descriptions = false
if found.length > MAX_RESULTS
found = found[0..MAX_RESULTS]
elsif found.length < INCLUDE_DESCRIPTIONS_MAX_LENGTH
include_descriptions = true
end
@last_num_results = found.length
format_results(found, args: { query: query }) do |setting|
result = { name: setting[:setting] }
if include_descriptions
result[:description] = setting[:description]
result[:plugin] = setting[:plugin]
end
result
end
end
end
end
end

View File

@ -39,6 +39,7 @@ module DiscourseAi
require_relative "commands/google_command"
require_relative "commands/read_command"
require_relative "commands/setting_context_command"
require_relative "commands/search_settings_command"
require_relative "commands/db_schema_command"
require_relative "personas/persona"
require_relative "personas/artist"

View File

@ -9,24 +9,23 @@ module DiscourseAi
end
def all_available_commands
[DiscourseAi::AiBot::Commands::SettingContextCommand]
[
DiscourseAi::AiBot::Commands::SettingContextCommand,
DiscourseAi::AiBot::Commands::SearchSettingsCommand,
]
end
def system_prompt
<<~PROMPT
You are Discourse Site settings bot.
- You know the full list of all the site settings.
- You are able to find information about all the site settings.
- You are able to request context for a specific setting.
- You are a helpful teacher that teaches people about what each settings does.
- Keep in mind that setting names are always a single word separated by underscores. eg. 'site_description'
Current time is: {time}
Full list of all the site settings:
{{
#{SiteSetting.all_settings.map { |setting| setting[:setting].to_s }.join("\n")}
}}
{commands}
PROMPT

View File

@ -14,6 +14,10 @@ module ::DiscourseAi
max_tokens: nil,
user_id: nil
)
log = nil
response_data = +""
response_raw = +""
url = URI("https://api.anthropic.com/v1/complete")
headers = {
"anthropic-version" => "2023-06-01",
@ -68,12 +72,9 @@ module ::DiscourseAi
return parsed_response
end
response_data = +""
begin
cancelled = false
cancel = lambda { cancelled = true }
response_raw = +""
response.read_body do |chunk|
if cancelled
@ -104,17 +105,22 @@ module ::DiscourseAi
end
rescue IOError
raise if !cancelled
ensure
log.update!(
raw_response_payload: response_raw,
request_tokens: DiscourseAi::Tokenizer::AnthropicTokenizer.size(prompt),
response_tokens: DiscourseAi::Tokenizer::AnthropicTokenizer.size(response_data),
)
end
end
return response_data
end
ensure
if block_given?
log.update!(
raw_response_payload: response_raw,
request_tokens: DiscourseAi::Tokenizer::AnthropicTokenizer.size(prompt),
response_tokens: DiscourseAi::Tokenizer::AnthropicTokenizer.size(response_data),
)
end
if Rails.env.development? && log
puts "AnthropicCompletions: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}"
end
end
def self.try_parse(data)

View File

@ -15,6 +15,10 @@ module ::DiscourseAi
functions: nil,
user_id: nil
)
log = nil
response_data = +""
response_raw = +""
url =
if model.include?("gpt-4")
if model.include?("32k")
@ -84,12 +88,9 @@ module ::DiscourseAi
return parsed_response
end
response_data = +""
begin
cancelled = false
cancel = lambda { cancelled = true }
response_raw = +""
leftover = ""
@ -125,19 +126,25 @@ module ::DiscourseAi
end
rescue IOError
raise if !cancelled
ensure
log.update!(
raw_response_payload: response_raw,
request_tokens:
DiscourseAi::Tokenizer::OpenAiTokenizer.size(extract_prompt(messages)),
response_tokens: DiscourseAi::Tokenizer::OpenAiTokenizer.size(response_data),
)
end
end
return response_data
end
end
ensure
if log && block_given?
request_tokens = DiscourseAi::Tokenizer::OpenAiTokenizer.size(extract_prompt(messages))
response_tokens = DiscourseAi::Tokenizer::OpenAiTokenizer.size(response_data)
log.update!(
raw_response_payload: response_raw,
request_tokens: request_tokens,
response_tokens: response_tokens,
)
end
if log && Rails.env.development?
puts "OpenAiCompletions: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}"
end
end
def self.extract_prompt(messages)

View File

@ -0,0 +1,29 @@
#frozen_string_literal: true
RSpec.describe DiscourseAi::AiBot::Commands::SearchSettingsCommand do
let(:search) { described_class.new(bot_user: nil, args: nil) }
describe "#process" do
it "can handle no results" do
results = search.process(query: "this will not exist frogs")
expect(results[:args]).to eq({ query: "this will not exist frogs" })
expect(results[:rows]).to eq([])
end
it "can return more many settings with no descriptions if there are lots of hits" do
results = search.process(query: "a")
expect(results[:rows].length).to be > 30
expect(results[:rows][0].length).to eq(1)
end
it "can return descriptions if there are few matches" do
results =
search.process(query: "this will not be found!@,default_locale,ai_bot_enabled_personas")
expect(results[:rows].length).to eq(2)
expect(results[:rows][0][1]).not_to eq(nil)
end
end
end

View File

@ -7,11 +7,17 @@ RSpec.describe DiscourseAi::AiBot::Personas::SettingsExplorer do
it "renders schema" do
prompt = settings_explorer.render_system_prompt
# check we render settings
expect(prompt).to include("ai_bot_enabled_personas")
# check we do not render plugin settings
expect(prompt).not_to include("ai_bot_enabled_personas")
expect(prompt).to include("site_description")
expect(settings_explorer.available_commands).to eq(
[DiscourseAi::AiBot::Commands::SettingContextCommand],
[
DiscourseAi::AiBot::Commands::SettingContextCommand,
DiscourseAi::AiBot::Commands::SearchSettingsCommand,
],
)
end
end