mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-03-03 07:49:30 +00:00
e817b7dc11
This re-implements tool support in DiscourseAi::Completions::Llm #generate
Previously tool support was always returned via XML and it would be the responsibility of the caller to parse XML
New implementation has the endpoints return ToolCall objects.
Additionally this simplifies the Llm endpoint interface and gives it more clarity. Llms must implement
decode, decode_chunk (for streaming)
It is the implementers responsibility to figure out how to decode chunks, base no longer implements. To make this easy we ship a flexible json decoder which is easy to wire up.
Also (new)
Better debugging for PMs, we now have a next / previous button to see all the Llm messages associated with a PM
Token accounting is fixed for vllm (we were not correctly counting tokens)
129 lines
3.6 KiB
Ruby
129 lines
3.6 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module DiscourseAi
|
|
module Completions
|
|
module Endpoints
|
|
class Anthropic < Base
|
|
def self.can_contact?(model_provider)
|
|
model_provider == "anthropic"
|
|
end
|
|
|
|
def normalize_model_params(model_params)
|
|
# max_tokens, temperature, stop_sequences are already supported
|
|
model_params
|
|
end
|
|
|
|
def default_options(dialect)
|
|
mapped_model =
|
|
case llm_model.name
|
|
when "claude-2"
|
|
"claude-2.1"
|
|
when "claude-instant-1"
|
|
"claude-instant-1.2"
|
|
when "claude-3-haiku"
|
|
"claude-3-haiku-20240307"
|
|
when "claude-3-sonnet"
|
|
"claude-3-sonnet-20240229"
|
|
when "claude-3-opus"
|
|
"claude-3-opus-20240229"
|
|
when "claude-3-5-sonnet"
|
|
"claude-3-5-sonnet-latest"
|
|
else
|
|
llm_model.name
|
|
end
|
|
|
|
options = { model: mapped_model, max_tokens: 3_000 }
|
|
|
|
options[:stop_sequences] = ["</function_calls>"] if !dialect.native_tool_support? &&
|
|
dialect.prompt.has_tools?
|
|
|
|
options
|
|
end
|
|
|
|
def provider_id
|
|
AiApiAuditLog::Provider::Anthropic
|
|
end
|
|
|
|
private
|
|
|
|
def xml_tags_to_strip(dialect)
|
|
if dialect.prompt.has_tools?
|
|
%w[thinking search_quality_reflection search_quality_score]
|
|
else
|
|
[]
|
|
end
|
|
end
|
|
|
|
# this is an approximation, we will update it later if request goes through
|
|
def prompt_size(prompt)
|
|
tokenizer.size(prompt.system_prompt.to_s + " " + prompt.messages.to_s)
|
|
end
|
|
|
|
def model_uri
|
|
URI(llm_model.url)
|
|
end
|
|
|
|
def xml_tools_enabled?
|
|
!@native_tool_support
|
|
end
|
|
|
|
def prepare_payload(prompt, model_params, dialect)
|
|
@native_tool_support = dialect.native_tool_support?
|
|
|
|
payload = default_options(dialect).merge(model_params).merge(messages: prompt.messages)
|
|
|
|
payload[:system] = prompt.system_prompt if prompt.system_prompt.present?
|
|
payload[:stream] = true if @streaming_mode
|
|
if prompt.has_tools?
|
|
payload[:tools] = prompt.tools
|
|
if dialect.tool_choice.present?
|
|
payload[:tool_choice] = { type: "tool", name: dialect.tool_choice }
|
|
end
|
|
end
|
|
|
|
payload
|
|
end
|
|
|
|
def prepare_request(payload)
|
|
headers = {
|
|
"anthropic-version" => "2023-06-01",
|
|
"x-api-key" => llm_model.api_key,
|
|
"content-type" => "application/json",
|
|
}
|
|
|
|
Net::HTTP::Post.new(model_uri, headers).tap { |r| r.body = payload }
|
|
end
|
|
|
|
def decode_chunk(partial_data)
|
|
@decoder ||= JsonStreamDecoder.new
|
|
(@decoder << partial_data)
|
|
.map { |parsed_json| processor.process_streamed_message(parsed_json) }
|
|
.compact
|
|
end
|
|
|
|
def decode(response_data)
|
|
processor.process_message(response_data)
|
|
end
|
|
|
|
def processor
|
|
@processor ||=
|
|
DiscourseAi::Completions::AnthropicMessageProcessor.new(streaming_mode: @streaming_mode)
|
|
end
|
|
|
|
def has_tool?(_response_data)
|
|
processor.tool_calls.present?
|
|
end
|
|
|
|
def tool_calls
|
|
processor.to_tool_calls
|
|
end
|
|
|
|
def final_log_update(log)
|
|
log.request_tokens = processor.input_tokens if processor.input_tokens
|
|
log.response_tokens = processor.output_tokens if processor.output_tokens
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|