mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-07-01 12:02:16 +00:00
This re-implements tool support in DiscourseAi::Completions::Llm #generate Previously tool support was always returned via XML and it would be the responsibility of the caller to parse XML New implementation has the endpoints return ToolCall objects. Additionally this simplifies the Llm endpoint interface and gives it more clarity. Llms must implement decode, decode_chunk (for streaming) It is the implementers responsibility to figure out how to decode chunks, base no longer implements. To make this easy we ship a flexible json decoder which is easy to wire up. Also (new) Better debugging for PMs, we now have a next / previous button to see all the Llm messages associated with a PM Token accounting is fixed for vllm (we were not correctly counting tokens)
95 lines
2.7 KiB
Ruby
95 lines
2.7 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module DiscourseAi
|
|
module Completions
|
|
module Endpoints
|
|
class Ollama < Base
|
|
def self.can_contact?(model_provider)
|
|
model_provider == "ollama"
|
|
end
|
|
|
|
def normalize_model_params(model_params)
|
|
model_params = model_params.dup
|
|
|
|
# max_tokens, temperature are already supported
|
|
if model_params[:stop_sequences]
|
|
model_params[:stop] = model_params.delete(:stop_sequences)
|
|
end
|
|
|
|
model_params
|
|
end
|
|
|
|
def default_options
|
|
{ max_tokens: 2000, model: llm_model.name }
|
|
end
|
|
|
|
def provider_id
|
|
AiApiAuditLog::Provider::Ollama
|
|
end
|
|
|
|
def use_ssl?
|
|
false
|
|
end
|
|
|
|
private
|
|
|
|
def model_uri
|
|
URI(llm_model.url)
|
|
end
|
|
|
|
def xml_tools_enabled?
|
|
!@native_tool_support
|
|
end
|
|
|
|
def prepare_payload(prompt, model_params, dialect)
|
|
@native_tool_support = dialect.native_tool_support?
|
|
|
|
# https://github.com/ollama/ollama/blob/main/docs/api.md#parameters-1
|
|
# Due to ollama enforce a 'stream: false' for tool calls, instead of complicating the code,
|
|
# we will just disable streaming for all ollama calls if native tool support is enabled
|
|
|
|
default_options
|
|
.merge(model_params)
|
|
.merge(messages: prompt)
|
|
.tap { |payload| payload[:stream] = false if @native_tool_support || !@streaming_mode }
|
|
.tap do |payload|
|
|
payload[:tools] = dialect.tools if @native_tool_support && dialect.tools.present?
|
|
end
|
|
end
|
|
|
|
def prepare_request(payload)
|
|
headers = { "Content-Type" => "application/json" }
|
|
|
|
Net::HTTP::Post.new(model_uri, headers).tap { |r| r.body = payload }
|
|
end
|
|
|
|
def decode_chunk(chunk)
|
|
# Native tool calls are not working right in streaming mode, use XML
|
|
@json_decoder ||= JsonStreamDecoder.new(line_regex: /^\s*({.*})$/)
|
|
(@json_decoder << chunk).map { |parsed| parsed.dig(:message, :content) }.compact
|
|
end
|
|
|
|
def decode(response_raw)
|
|
rval = []
|
|
parsed = JSON.parse(response_raw, symbolize_names: true)
|
|
content = parsed.dig(:message, :content)
|
|
rval << content if !content.to_s.empty?
|
|
|
|
idx = -1
|
|
parsed
|
|
.dig(:message, :tool_calls)
|
|
&.each do |tool_call|
|
|
idx += 1
|
|
id = "tool_#{idx}"
|
|
name = tool_call.dig(:function, :name)
|
|
args = tool_call.dig(:function, :arguments)
|
|
rval << ToolCall.new(id: id, name: name, parameters: args)
|
|
end
|
|
|
|
rval
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|