mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-02-26 13:36:54 +00:00
181113159b
This refactor changes it so we only include minimal data in the system prompt which leaves us lots of tokens for specific searches The new search command allows us to pull in settings on demand Descriptions are include in short search results, and names only in longer results Also: * In dev it is important to tell when calls are made to open ai this adds a console log to increase awareness around token usage * PERF: stop counting tokens so often This changes it so we only count tokens once per response Previously each time we heard back from open ai we would count tokens, leading to uneeded delays * bug fix, commands may reach in for tokenizer * add logging to console for anthropic calls as well * Update lib/shared/inference/openai_completions.rb Co-authored-by: Martin Brennan <mjrbrennan@gmail.com>
68 lines
1.6 KiB
Ruby
68 lines
1.6 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module DiscourseAi
|
|
module AiBot
|
|
class AnthropicBot < Bot
|
|
def self.can_reply_as?(bot_user)
|
|
bot_user.id == DiscourseAi::AiBot::EntryPoint::CLAUDE_V2_ID
|
|
end
|
|
|
|
def bot_prompt_with_topic_context(post)
|
|
super(post).join("\n\n") + "\n\nAssistant:"
|
|
end
|
|
|
|
def prompt_limit
|
|
50_000 # https://console.anthropic.com/docs/prompt-design#what-is-a-prompt
|
|
end
|
|
|
|
def title_prompt(post)
|
|
super(post).join("\n\n") + "\n\nAssistant:"
|
|
end
|
|
|
|
def get_delta(partial, context)
|
|
completion = partial[:completion]
|
|
if completion&.start_with?(" ") && !context[:processed_first]
|
|
completion = completion[1..-1]
|
|
context[:processed_first] = true
|
|
end
|
|
completion
|
|
end
|
|
|
|
def tokenizer
|
|
DiscourseAi::Tokenizer::AnthropicTokenizer
|
|
end
|
|
|
|
private
|
|
|
|
def build_message(poster_username, content, system: false, function: nil)
|
|
role = poster_username == bot_user.username ? "Assistant" : "Human"
|
|
|
|
"#{role}: #{content}"
|
|
end
|
|
|
|
def model_for
|
|
"claude-2"
|
|
end
|
|
|
|
def get_updated_title(prompt)
|
|
DiscourseAi::Inference::AnthropicCompletions.perform!(
|
|
prompt,
|
|
model_for,
|
|
temperature: 0.4,
|
|
max_tokens: 40,
|
|
).dig(:completion)
|
|
end
|
|
|
|
def submit_prompt(prompt, prefer_low_cost: false, &blk)
|
|
DiscourseAi::Inference::AnthropicCompletions.perform!(
|
|
prompt,
|
|
model_for,
|
|
temperature: 0.4,
|
|
max_tokens: 3000,
|
|
&blk
|
|
)
|
|
end
|
|
end
|
|
end
|
|
end
|