2024-01-12 12:36:44 -05:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
|
|
module DiscourseAi
|
|
|
|
module Completions
|
|
|
|
class Prompt
|
|
|
|
INVALID_TURN = Class.new(StandardError)
|
|
|
|
|
2024-01-15 02:51:14 -05:00
|
|
|
attr_reader :messages
|
FEATURE: Add vision support to AI personas (Claude 3) (#546)
This commit adds the ability to enable vision for AI personas, allowing them to understand images that are posted in the conversation.
For personas with vision enabled, any images the user has posted will be resized to be within the configured max_pixels limit, base64 encoded and included in the prompt sent to the AI provider.
The persona editor allows enabling/disabling vision and has a dropdown to select the max supported image size (low, medium, high). Vision is disabled by default.
This initial vision support has been tested and implemented with Anthropic's claude-3 models which accept images in a special format as part of the prompt.
Other integrations will need to be updated to support images.
Several specs were added to test the new functionality at the persona, prompt building and API layers.
- Gemini is omitted, pending API support for Gemini 1.5. Current Gemini bot is not performing well, adding images is unlikely to make it perform any better.
- Open AI is omitted, vision support on GPT-4 it limited in that the API has no tool support when images are enabled so we would need to full back to a different prompting technique, something that would add lots of complexity
---------
Co-authored-by: Martin Brennan <martin@discourse.org>
2024-03-26 23:30:11 -04:00
|
|
|
attr_accessor :tools, :topic_id, :post_id, :max_pixels
|
2024-03-01 15:53:21 -05:00
|
|
|
|
|
|
|
def initialize(
|
|
|
|
system_message_text = nil,
|
|
|
|
messages: [],
|
|
|
|
tools: [],
|
|
|
|
skip_validations: false,
|
|
|
|
topic_id: nil,
|
FEATURE: Add vision support to AI personas (Claude 3) (#546)
This commit adds the ability to enable vision for AI personas, allowing them to understand images that are posted in the conversation.
For personas with vision enabled, any images the user has posted will be resized to be within the configured max_pixels limit, base64 encoded and included in the prompt sent to the AI provider.
The persona editor allows enabling/disabling vision and has a dropdown to select the max supported image size (low, medium, high). Vision is disabled by default.
This initial vision support has been tested and implemented with Anthropic's claude-3 models which accept images in a special format as part of the prompt.
Other integrations will need to be updated to support images.
Several specs were added to test the new functionality at the persona, prompt building and API layers.
- Gemini is omitted, pending API support for Gemini 1.5. Current Gemini bot is not performing well, adding images is unlikely to make it perform any better.
- Open AI is omitted, vision support on GPT-4 it limited in that the API has no tool support when images are enabled so we would need to full back to a different prompting technique, something that would add lots of complexity
---------
Co-authored-by: Martin Brennan <martin@discourse.org>
2024-03-26 23:30:11 -04:00
|
|
|
post_id: nil,
|
|
|
|
max_pixels: nil
|
2024-03-01 15:53:21 -05:00
|
|
|
)
|
2024-01-12 12:36:44 -05:00
|
|
|
raise ArgumentError, "messages must be an array" if !messages.is_a?(Array)
|
|
|
|
raise ArgumentError, "tools must be an array" if !tools.is_a?(Array)
|
|
|
|
|
FEATURE: Add vision support to AI personas (Claude 3) (#546)
This commit adds the ability to enable vision for AI personas, allowing them to understand images that are posted in the conversation.
For personas with vision enabled, any images the user has posted will be resized to be within the configured max_pixels limit, base64 encoded and included in the prompt sent to the AI provider.
The persona editor allows enabling/disabling vision and has a dropdown to select the max supported image size (low, medium, high). Vision is disabled by default.
This initial vision support has been tested and implemented with Anthropic's claude-3 models which accept images in a special format as part of the prompt.
Other integrations will need to be updated to support images.
Several specs were added to test the new functionality at the persona, prompt building and API layers.
- Gemini is omitted, pending API support for Gemini 1.5. Current Gemini bot is not performing well, adding images is unlikely to make it perform any better.
- Open AI is omitted, vision support on GPT-4 it limited in that the API has no tool support when images are enabled so we would need to full back to a different prompting technique, something that would add lots of complexity
---------
Co-authored-by: Martin Brennan <martin@discourse.org>
2024-03-26 23:30:11 -04:00
|
|
|
@max_pixels = max_pixels || 1_048_576
|
|
|
|
|
2024-03-01 15:53:21 -05:00
|
|
|
@topic_id = topic_id
|
|
|
|
@post_id = post_id
|
|
|
|
|
2024-01-15 02:51:14 -05:00
|
|
|
@messages = []
|
2024-02-19 12:56:28 -05:00
|
|
|
@skip_validations = skip_validations
|
2024-01-15 02:51:14 -05:00
|
|
|
|
|
|
|
if system_message_text
|
|
|
|
system_message = { type: :system, content: system_message_text }
|
|
|
|
@messages << system_message
|
|
|
|
end
|
|
|
|
|
|
|
|
@messages.concat(messages)
|
2024-01-12 12:36:44 -05:00
|
|
|
|
|
|
|
@messages.each { |message| validate_message(message) }
|
|
|
|
@messages.each_cons(2) { |last_turn, new_turn| validate_turn(last_turn, new_turn) }
|
|
|
|
|
|
|
|
@tools = tools
|
|
|
|
end
|
|
|
|
|
FEATURE: Add vision support to AI personas (Claude 3) (#546)
This commit adds the ability to enable vision for AI personas, allowing them to understand images that are posted in the conversation.
For personas with vision enabled, any images the user has posted will be resized to be within the configured max_pixels limit, base64 encoded and included in the prompt sent to the AI provider.
The persona editor allows enabling/disabling vision and has a dropdown to select the max supported image size (low, medium, high). Vision is disabled by default.
This initial vision support has been tested and implemented with Anthropic's claude-3 models which accept images in a special format as part of the prompt.
Other integrations will need to be updated to support images.
Several specs were added to test the new functionality at the persona, prompt building and API layers.
- Gemini is omitted, pending API support for Gemini 1.5. Current Gemini bot is not performing well, adding images is unlikely to make it perform any better.
- Open AI is omitted, vision support on GPT-4 it limited in that the API has no tool support when images are enabled so we would need to full back to a different prompting technique, something that would add lots of complexity
---------
Co-authored-by: Martin Brennan <martin@discourse.org>
2024-03-26 23:30:11 -04:00
|
|
|
def push(type:, content:, id: nil, name: nil, upload_ids: nil)
|
2024-01-12 12:36:44 -05:00
|
|
|
return if type == :system
|
|
|
|
new_message = { type: type, content: content }
|
2024-03-08 16:46:40 -05:00
|
|
|
new_message[:name] = name.to_s if name
|
2024-01-15 21:48:00 -05:00
|
|
|
new_message[:id] = id.to_s if id
|
FEATURE: Add vision support to AI personas (Claude 3) (#546)
This commit adds the ability to enable vision for AI personas, allowing them to understand images that are posted in the conversation.
For personas with vision enabled, any images the user has posted will be resized to be within the configured max_pixels limit, base64 encoded and included in the prompt sent to the AI provider.
The persona editor allows enabling/disabling vision and has a dropdown to select the max supported image size (low, medium, high). Vision is disabled by default.
This initial vision support has been tested and implemented with Anthropic's claude-3 models which accept images in a special format as part of the prompt.
Other integrations will need to be updated to support images.
Several specs were added to test the new functionality at the persona, prompt building and API layers.
- Gemini is omitted, pending API support for Gemini 1.5. Current Gemini bot is not performing well, adding images is unlikely to make it perform any better.
- Open AI is omitted, vision support on GPT-4 it limited in that the API has no tool support when images are enabled so we would need to full back to a different prompting technique, something that would add lots of complexity
---------
Co-authored-by: Martin Brennan <martin@discourse.org>
2024-03-26 23:30:11 -04:00
|
|
|
new_message[:upload_ids] = upload_ids if upload_ids
|
2024-01-12 12:36:44 -05:00
|
|
|
|
|
|
|
validate_message(new_message)
|
|
|
|
validate_turn(messages.last, new_message)
|
|
|
|
|
|
|
|
messages << new_message
|
|
|
|
end
|
|
|
|
|
2024-03-07 14:37:23 -05:00
|
|
|
def has_tools?
|
|
|
|
tools.present?
|
|
|
|
end
|
|
|
|
|
FEATURE: Add vision support to AI personas (Claude 3) (#546)
This commit adds the ability to enable vision for AI personas, allowing them to understand images that are posted in the conversation.
For personas with vision enabled, any images the user has posted will be resized to be within the configured max_pixels limit, base64 encoded and included in the prompt sent to the AI provider.
The persona editor allows enabling/disabling vision and has a dropdown to select the max supported image size (low, medium, high). Vision is disabled by default.
This initial vision support has been tested and implemented with Anthropic's claude-3 models which accept images in a special format as part of the prompt.
Other integrations will need to be updated to support images.
Several specs were added to test the new functionality at the persona, prompt building and API layers.
- Gemini is omitted, pending API support for Gemini 1.5. Current Gemini bot is not performing well, adding images is unlikely to make it perform any better.
- Open AI is omitted, vision support on GPT-4 it limited in that the API has no tool support when images are enabled so we would need to full back to a different prompting technique, something that would add lots of complexity
---------
Co-authored-by: Martin Brennan <martin@discourse.org>
2024-03-26 23:30:11 -04:00
|
|
|
# helper method to get base64 encoded uploads
|
|
|
|
# at the correct dimentions
|
|
|
|
def encoded_uploads(message)
|
|
|
|
return [] if message[:upload_ids].blank?
|
|
|
|
UploadEncoder.encode(upload_ids: message[:upload_ids], max_pixels: max_pixels)
|
|
|
|
end
|
|
|
|
|
2024-01-12 12:36:44 -05:00
|
|
|
private
|
|
|
|
|
|
|
|
def validate_message(message)
|
2024-02-19 12:56:28 -05:00
|
|
|
return if @skip_validations
|
2024-01-12 12:36:44 -05:00
|
|
|
valid_types = %i[system user model tool tool_call]
|
|
|
|
if !valid_types.include?(message[:type])
|
|
|
|
raise ArgumentError, "message type must be one of #{valid_types}"
|
|
|
|
end
|
|
|
|
|
FEATURE: Add vision support to AI personas (Claude 3) (#546)
This commit adds the ability to enable vision for AI personas, allowing them to understand images that are posted in the conversation.
For personas with vision enabled, any images the user has posted will be resized to be within the configured max_pixels limit, base64 encoded and included in the prompt sent to the AI provider.
The persona editor allows enabling/disabling vision and has a dropdown to select the max supported image size (low, medium, high). Vision is disabled by default.
This initial vision support has been tested and implemented with Anthropic's claude-3 models which accept images in a special format as part of the prompt.
Other integrations will need to be updated to support images.
Several specs were added to test the new functionality at the persona, prompt building and API layers.
- Gemini is omitted, pending API support for Gemini 1.5. Current Gemini bot is not performing well, adding images is unlikely to make it perform any better.
- Open AI is omitted, vision support on GPT-4 it limited in that the API has no tool support when images are enabled so we would need to full back to a different prompting technique, something that would add lots of complexity
---------
Co-authored-by: Martin Brennan <martin@discourse.org>
2024-03-26 23:30:11 -04:00
|
|
|
valid_keys = %i[type content id name upload_ids]
|
2024-01-12 12:36:44 -05:00
|
|
|
if (invalid_keys = message.keys - valid_keys).any?
|
|
|
|
raise ArgumentError, "message contains invalid keys: #{invalid_keys}"
|
|
|
|
end
|
|
|
|
|
FEATURE: Add vision support to AI personas (Claude 3) (#546)
This commit adds the ability to enable vision for AI personas, allowing them to understand images that are posted in the conversation.
For personas with vision enabled, any images the user has posted will be resized to be within the configured max_pixels limit, base64 encoded and included in the prompt sent to the AI provider.
The persona editor allows enabling/disabling vision and has a dropdown to select the max supported image size (low, medium, high). Vision is disabled by default.
This initial vision support has been tested and implemented with Anthropic's claude-3 models which accept images in a special format as part of the prompt.
Other integrations will need to be updated to support images.
Several specs were added to test the new functionality at the persona, prompt building and API layers.
- Gemini is omitted, pending API support for Gemini 1.5. Current Gemini bot is not performing well, adding images is unlikely to make it perform any better.
- Open AI is omitted, vision support on GPT-4 it limited in that the API has no tool support when images are enabled so we would need to full back to a different prompting technique, something that would add lots of complexity
---------
Co-authored-by: Martin Brennan <martin@discourse.org>
2024-03-26 23:30:11 -04:00
|
|
|
if message[:type] == :upload_ids && !message[:upload_ids].is_a?(Array)
|
|
|
|
raise ArgumentError, "upload_ids must be an array of ids"
|
|
|
|
end
|
|
|
|
|
|
|
|
if message[:upload_ids].present? && message[:type] != :user
|
|
|
|
raise ArgumentError, "upload_ids are only supported for users"
|
|
|
|
end
|
|
|
|
|
2024-01-12 12:36:44 -05:00
|
|
|
raise ArgumentError, "message content must be a string" if !message[:content].is_a?(String)
|
|
|
|
end
|
|
|
|
|
|
|
|
def validate_turn(last_turn, new_turn)
|
2024-02-19 12:56:28 -05:00
|
|
|
return if @skip_validations
|
2024-01-12 12:36:44 -05:00
|
|
|
valid_types = %i[tool tool_call model user]
|
|
|
|
raise INVALID_TURN if !valid_types.include?(new_turn[:type])
|
|
|
|
|
|
|
|
if last_turn[:type] == :system && %i[tool tool_call model].include?(new_turn[:type])
|
|
|
|
raise INVALID_TURN
|
|
|
|
end
|
|
|
|
|
|
|
|
raise INVALID_TURN if new_turn[:type] == :tool && last_turn[:type] != :tool_call
|
|
|
|
raise INVALID_TURN if new_turn[:type] == :model && last_turn[:type] == :model
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|