mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-03-09 11:48:47 +00:00
03fc94684b
* FIX: AI helper not working correctly with mixtral This PR introduces a new function on the generic llm called #generate This will replace the implementation of completion! #generate introduces a new way to pass temperature, max_tokens and stop_sequences Then LLM implementers need to implement #normalize_model_params to ensure the generic names match the LLM specific endpoint This also adds temperature and stop_sequences to completion_prompts this allows for much more robust completion prompts * port everything over to #generate * Fix translation - On anthropic this no longer throws random "This is your translation:" - On mixtral this actually works * fix markdown table generation as well
111 lines
3.2 KiB
Ruby
111 lines
3.2 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require "aws-sigv4"
|
|
|
|
module DiscourseAi
|
|
module Completions
|
|
module Endpoints
|
|
class AwsBedrock < Base
|
|
def self.can_contact?(model_name)
|
|
%w[claude-instant-1 claude-2].include?(model_name) &&
|
|
SiteSetting.ai_bedrock_access_key_id.present? &&
|
|
SiteSetting.ai_bedrock_secret_access_key.present? &&
|
|
SiteSetting.ai_bedrock_region.present?
|
|
end
|
|
|
|
def normalize_model_params(model_params)
|
|
model_params = model_params.dup
|
|
|
|
# temperature, stop_sequences are already supported
|
|
#
|
|
if model_params[:max_tokens]
|
|
model_params[:max_tokens_to_sample] = model_params.delete(:max_tokens)
|
|
end
|
|
|
|
model_params
|
|
end
|
|
|
|
def default_options
|
|
{
|
|
model: model,
|
|
max_tokens_to_sample: 2_000,
|
|
stop_sequences: ["\n\nHuman:", "</function_calls>"],
|
|
}
|
|
end
|
|
|
|
def provider_id
|
|
AiApiAuditLog::Provider::Anthropic
|
|
end
|
|
|
|
private
|
|
|
|
def model_uri
|
|
# Bedrock uses slightly different names
|
|
bedrock_model_id = model.split("-")
|
|
bedrock_model_id[-1] = "v#{bedrock_model_id.last}"
|
|
bedrock_model_id = bedrock_model_id.join("-")
|
|
|
|
api_url =
|
|
"https://bedrock-runtime.#{SiteSetting.ai_bedrock_region}.amazonaws.com/model/anthropic.#{bedrock_model_id}/invoke"
|
|
|
|
api_url = @streaming_mode ? (api_url + "-with-response-stream") : api_url
|
|
|
|
URI(api_url)
|
|
end
|
|
|
|
def prepare_payload(prompt, model_params, _dialect)
|
|
default_options.merge(prompt: prompt).merge(model_params)
|
|
end
|
|
|
|
def prepare_request(payload)
|
|
headers = { "content-type" => "application/json", "Accept" => "*/*" }
|
|
|
|
signer =
|
|
Aws::Sigv4::Signer.new(
|
|
access_key_id: SiteSetting.ai_bedrock_access_key_id,
|
|
region: SiteSetting.ai_bedrock_region,
|
|
secret_access_key: SiteSetting.ai_bedrock_secret_access_key,
|
|
service: "bedrock",
|
|
)
|
|
|
|
Net::HTTP::Post
|
|
.new(model_uri)
|
|
.tap do |r|
|
|
r.body = payload
|
|
|
|
signed_request =
|
|
signer.sign_request(req: r, http_method: r.method, url: model_uri, body: r.body)
|
|
|
|
r.initialize_http_header(headers.merge(signed_request.headers))
|
|
end
|
|
end
|
|
|
|
def decode(chunk)
|
|
Aws::EventStream::Decoder
|
|
.new
|
|
.decode_chunk(chunk)
|
|
.first
|
|
.payload
|
|
.string
|
|
.then { JSON.parse(_1) }
|
|
.dig("bytes")
|
|
.then { Base64.decode64(_1) }
|
|
rescue JSON::ParserError,
|
|
Aws::EventStream::Errors::MessageChecksumError,
|
|
Aws::EventStream::Errors::PreludeChecksumError => e
|
|
Rails.logger.error("#{self.class.name}: #{e.message}")
|
|
nil
|
|
end
|
|
|
|
def extract_completion_from(response_raw)
|
|
JSON.parse(response_raw, symbolize_names: true)[:completion].to_s
|
|
end
|
|
|
|
def partials_from(decoded_chunk)
|
|
[decoded_chunk]
|
|
end
|
|
end
|
|
end
|
|
end
|
|
end
|