diff --git a/app/controllers/discourse_ai/admin/ai_personas_controller.rb b/app/controllers/discourse_ai/admin/ai_personas_controller.rb index 4082692e..ee653d5b 100644 --- a/app/controllers/discourse_ai/admin/ai_personas_controller.rb +++ b/app/controllers/discourse_ai/admin/ai_personas_controller.rb @@ -5,8 +5,7 @@ module DiscourseAi class AiPersonasController < ::Admin::AdminController requires_plugin ::DiscourseAi::PLUGIN_NAME - before_action :find_ai_persona, - only: %i[show update destroy create_user indexing_status_check] + before_action :find_ai_persona, only: %i[show update destroy create_user] def index ai_personas = @@ -75,37 +74,6 @@ module DiscourseAi end end - def upload_file - file = params[:file] || params[:files].first - - if !SiteSetting.ai_embeddings_enabled? - raise Discourse::InvalidAccess.new("Embeddings not enabled") - end - - validate_extension!(file.original_filename) - validate_file_size!(file.tempfile.size) - - hijack do - upload = - UploadCreator.new( - file.tempfile, - file.original_filename, - type: "discourse_ai_rag_upload", - skip_validations: true, - ).create_for(current_user.id) - - if upload.persisted? - render json: UploadSerializer.new(upload) - else - render json: failed_json.merge(errors: upload.errors.full_messages), status: 422 - end - end - end - - def indexing_status_check - render json: RagDocumentFragment.indexing_status(@ai_persona, @ai_persona.uploads) - end - private def find_ai_persona @@ -163,31 +131,6 @@ module DiscourseAi end end end - - def validate_extension!(filename) - extension = File.extname(filename)[1..-1] || "" - authorized_extensions = %w[txt md] - if !authorized_extensions.include?(extension) - raise Discourse::InvalidParameters.new( - I18n.t( - "upload.unauthorized", - authorized_extensions: authorized_extensions.join(" "), - ), - ) - end - end - - def validate_file_size!(filesize) - max_size_bytes = 20.megabytes - if filesize > max_size_bytes - raise Discourse::InvalidParameters.new( - I18n.t( - "upload.attachments.too_large_humanized", - max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes), - ), - ) - end - end end end end diff --git a/app/controllers/discourse_ai/admin/ai_tools_controller.rb b/app/controllers/discourse_ai/admin/ai_tools_controller.rb index 4eb53578..4f231870 100644 --- a/app/controllers/discourse_ai/admin/ai_tools_controller.rb +++ b/app/controllers/discourse_ai/admin/ai_tools_controller.rb @@ -17,10 +17,11 @@ module DiscourseAi end def create - ai_tool = AiTool.new(ai_tool_params) + ai_tool = AiTool.new(ai_tool_params.except(:rag_uploads)) ai_tool.created_by_id = current_user.id if ai_tool.save + RagDocumentFragment.link_target_and_uploads(ai_tool, attached_upload_ids) render_serialized(ai_tool, AiCustomToolSerializer, status: :created) else render_json_error ai_tool @@ -28,7 +29,8 @@ module DiscourseAi end def update - if @ai_tool.update(ai_tool_params) + if @ai_tool.update(ai_tool_params.except(:rag_uploads)) + RagDocumentFragment.update_target_uploads(@ai_tool, attached_upload_ids) render_serialized(@ai_tool, AiCustomToolSerializer) else render_json_error @ai_tool @@ -71,6 +73,10 @@ module DiscourseAi private + def attached_upload_ids + ai_tool_params[:rag_uploads].to_a.map { |h| h[:id] } + end + def find_ai_tool @ai_tool = AiTool.find(params[:id]) end @@ -81,6 +87,9 @@ module DiscourseAi :description, :script, :summary, + :rag_chunk_tokens, + :rag_chunk_overlap_tokens, + rag_uploads: [:id], parameters: [:name, :type, :description, :required, enum: []], ) end diff --git a/app/controllers/discourse_ai/admin/rag_document_fragments_controller.rb b/app/controllers/discourse_ai/admin/rag_document_fragments_controller.rb new file mode 100644 index 00000000..fecc2c01 --- /dev/null +++ b/app/controllers/discourse_ai/admin/rag_document_fragments_controller.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module DiscourseAi + module Admin + class RagDocumentFragmentsController < ::Admin::AdminController + requires_plugin ::DiscourseAi::PLUGIN_NAME + + def indexing_status_check + if params[:target_type] == "AiPersona" + @target = AiPersona.find(params[:target_id]) + elsif params[:target_type] == "AiTool" + @target = AiTool.find(params[:target_id]) + else + raise Discourse::InvalidParameters.new("Invalid target type") + end + + render json: RagDocumentFragment.indexing_status(@target, @target.uploads) + end + + def upload_file + file = params[:file] || params[:files].first + + if !SiteSetting.ai_embeddings_enabled? + raise Discourse::InvalidAccess.new("Embeddings not enabled") + end + + validate_extension!(file.original_filename) + validate_file_size!(file.tempfile.size) + + hijack do + upload = + UploadCreator.new( + file.tempfile, + file.original_filename, + type: "discourse_ai_rag_upload", + skip_validations: true, + ).create_for(current_user.id) + + if upload.persisted? + render json: UploadSerializer.new(upload) + else + render json: failed_json.merge(errors: upload.errors.full_messages), status: 422 + end + end + end + + private + + def validate_extension!(filename) + extension = File.extname(filename)[1..-1] || "" + authorized_extensions = %w[txt md] + if !authorized_extensions.include?(extension) + raise Discourse::InvalidParameters.new( + I18n.t( + "upload.unauthorized", + authorized_extensions: authorized_extensions.join(" "), + ), + ) + end + end + + def validate_file_size!(filesize) + max_size_bytes = 20.megabytes + if filesize > max_size_bytes + raise Discourse::InvalidParameters.new( + I18n.t( + "upload.attachments.too_large_humanized", + max_size: ActiveSupport::NumberHelper.number_to_human_size(max_size_bytes), + ), + ) + end + end + end + end +end diff --git a/app/jobs/regular/digest_rag_upload.rb b/app/jobs/regular/digest_rag_upload.rb index b24bba1e..9ea10c06 100644 --- a/app/jobs/regular/digest_rag_upload.rb +++ b/app/jobs/regular/digest_rag_upload.rb @@ -126,7 +126,9 @@ module ::Jobs while overlap_token_ids.present? begin - overlap = tokenizer.decode(overlap_token_ids) + split_char + padding = split_char + padding = " " if padding.empty? + overlap = tokenizer.decode(overlap_token_ids) + padding break if overlap.encoding == Encoding::UTF_8 rescue StandardError # it is possible that we truncated mid char @@ -135,7 +137,7 @@ module ::Jobs end # remove first word it is probably truncated - overlap = overlap.split(" ", 2).last + overlap = overlap.split(/\s/, 2).last.to_s.lstrip end end diff --git a/app/models/ai_tool.rb b/app/models/ai_tool.rb index 8b286c38..b3638223 100644 --- a/app/models/ai_tool.rb +++ b/app/models/ai_tool.rb @@ -7,6 +7,10 @@ class AiTool < ActiveRecord::Base validates :script, presence: true, length: { maximum: 100_000 } validates :created_by_id, presence: true belongs_to :created_by, class_name: "User" + has_many :rag_document_fragments, dependent: :destroy, as: :target + has_many :upload_references, as: :target, dependent: :destroy + has_many :uploads, through: :upload_references + before_update :regenerate_rag_fragments def signature { name: name, description: description, parameters: parameters.map(&:symbolize_keys) } @@ -28,6 +32,82 @@ class AiTool < ActiveRecord::Base AiPersona.persona_cache.flush! end + def regenerate_rag_fragments + if rag_chunk_tokens_changed? || rag_chunk_overlap_tokens_changed? + RagDocumentFragment.where(target: self).delete_all + end + end + + def self.preamble + <<~JS + /** + * Tool API Quick Reference + * + * Entry Functions + * + * invoke(parameters): Main function. Receives parameters (Object). Must return a JSON-serializable value. + * Example: + * function invoke(parameters) { return "result"; } + * + * details(): Optional. Returns a string describing the tool. + * Example: + * function details() { return "Tool description."; } + * + * Provided Objects + * + * 1. http + * http.get(url, options?): Performs an HTTP GET request. + * Parameters: + * url (string): The request URL. + * options (Object, optional): + * headers (Object): Request headers. + * Returns: + * { status: number, body: string } + * + * http.post(url, options?): Performs an HTTP POST request. + * Parameters: + * url (string): The request URL. + * options (Object, optional): + * headers (Object): Request headers. + * body (string): Request body. + * Returns: + * { status: number, body: string } + * + * Note: Max 20 HTTP requests per execution. + * + * 2. llm + * llm.truncate(text, length): Truncates text to a specified token length. + * Parameters: + * text (string): Text to truncate. + * length (number): Max tokens. + * Returns: + * Truncated string. + * + * 3. index + * index.search(query, options?): Searches indexed documents. + * Parameters: + * query (string): Search query. + * options (Object, optional): + * filenames (Array): Limit search to specific files. + * limit (number): Max fragments (up to 200). + * Returns: + * Array of { fragment: string, metadata: string } + * + * Constraints + * + * Execution Time: ≤ 2000ms + * Memory: ≤ 10MB + * HTTP Requests: ≤ 20 per execution + * Exceeding limits will result in errors or termination. + * + * Security + * + * Sandboxed Environment: No access to system or global objects. + * No File System Access: Cannot read or write files. + */ + JS + end + def self.presets [ { @@ -38,6 +118,7 @@ class AiTool < ActiveRecord::Base { name: "url", type: "string", required: true, description: "The URL to browse" }, ], script: <<~SCRIPT, + #{preamble} let url; function invoke(p) { url = p.url; @@ -70,6 +151,7 @@ class AiTool < ActiveRecord::Base { name: "amount", type: "number", description: "Amount to convert eg: 123.45" }, ], script: <<~SCRIPT, + #{preamble} // note: this script uses the open.er-api.com service, it is only updated // once every 24 hours, for more up to date rates see: https://www.exchangerate-api.com function invoke(params) { @@ -118,6 +200,7 @@ class AiTool < ActiveRecord::Base }, ], script: <<~SCRIPT, + #{preamble} function invoke(params) { const apiKey = 'YOUR_ALPHAVANTAGE_API_KEY'; // Replace with your actual API key const url = `https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol=${params.symbol}&apikey=${apiKey}`; @@ -154,6 +237,7 @@ class AiTool < ActiveRecord::Base summary: "Get real-time stock quotes using AlphaVantage API", }, { preset_id: "empty_tool", script: <<~SCRIPT }, + #{preamble} function invoke(params) { // logic here return params; @@ -173,14 +257,16 @@ end # # Table name: ai_tools # -# id :bigint not null, primary key -# name :string not null -# description :string not null -# summary :string not null -# parameters :jsonb not null -# script :text not null -# created_by_id :integer not null -# enabled :boolean default(TRUE), not null -# created_at :datetime not null -# updated_at :datetime not null +# id :bigint not null, primary key +# name :string not null +# description :string not null +# summary :string not null +# parameters :jsonb not null +# script :text not null +# created_by_id :integer not null +# enabled :boolean default(TRUE), not null +# created_at :datetime not null +# updated_at :datetime not null +# rag_chunk_tokens :integer default(374), not null +# rag_chunk_overlap_tokens :integer default(10), not null # diff --git a/app/models/rag_document_fragment.rb b/app/models/rag_document_fragment.rb index 344506be..744568d6 100644 --- a/app/models/rag_document_fragment.rb +++ b/app/models/rag_document_fragment.rb @@ -72,11 +72,7 @@ class RagDocumentFragment < ActiveRecord::Base end def publish_status(upload, status) - MessageBus.publish( - "/discourse-ai/ai-persona-rag/#{upload.id}", - status, - user_ids: [upload.user_id], - ) + MessageBus.publish("/discourse-ai/rag/#{upload.id}", status, user_ids: [upload.user_id]) end end end diff --git a/app/serializers/ai_custom_tool_serializer.rb b/app/serializers/ai_custom_tool_serializer.rb index c95482ad..8fe1fd41 100644 --- a/app/serializers/ai_custom_tool_serializer.rb +++ b/app/serializers/ai_custom_tool_serializer.rb @@ -7,9 +7,17 @@ class AiCustomToolSerializer < ApplicationSerializer :summary, :parameters, :script, + :rag_chunk_tokens, + :rag_chunk_overlap_tokens, :created_by_id, :created_at, :updated_at self.root = "ai_tool" + + has_many :rag_uploads, serializer: UploadSerializer, embed: :object + + def rag_uploads + object.uploads + end end diff --git a/assets/javascripts/discourse/admin/models/ai-tool.js b/assets/javascripts/discourse/admin/models/ai-tool.js index 8cf94f47..bfeea21b 100644 --- a/assets/javascripts/discourse/admin/models/ai-tool.js +++ b/assets/javascripts/discourse/admin/models/ai-tool.js @@ -8,6 +8,9 @@ const CREATE_ATTRIBUTES = [ "parameters", "script", "summary", + "rag_uploads", + "rag_chunk_tokens", + "rag_chunk_overlap_tokens", "enabled", ]; diff --git a/assets/javascripts/discourse/components/ai-persona-editor.gjs b/assets/javascripts/discourse/components/ai-persona-editor.gjs index a4ba5580..139768c1 100644 --- a/assets/javascripts/discourse/components/ai-persona-editor.gjs +++ b/assets/javascripts/discourse/components/ai-persona-editor.gjs @@ -23,7 +23,8 @@ import DTooltip from "float-kit/components/d-tooltip"; import AiLlmSelector from "./ai-llm-selector"; import AiPersonaToolOptions from "./ai-persona-tool-options"; import AiToolSelector from "./ai-tool-selector"; -import PersonaRagUploader from "./persona-rag-uploader"; +import RagOptions from "./rag-options"; +import RagUploader from "./rag-uploader"; export default class PersonaEditor extends Component { @service router; @@ -38,7 +39,6 @@ export default class PersonaEditor extends Component { @tracked showDelete = false; @tracked maxPixelsValue = null; @tracked ragIndexingStatuses = null; - @tracked showIndexingOptions = false; get chatPluginEnabled() { return this.siteSettings.chat_enabled; @@ -53,13 +53,6 @@ export default class PersonaEditor extends Component { ); } - @action - toggleIndexingOptions(event) { - this.showIndexingOptions = !this.showIndexingOptions; - event.preventDefault(); - event.stopPropagation(); - } - findClosestPixelValue(pixels) { let value = "high"; this.maxPixelValues.forEach((info) => { @@ -81,12 +74,6 @@ export default class PersonaEditor extends Component { ]; } - get indexingOptionsText() { - return this.showIndexingOptions - ? I18n.t("discourse_ai.ai_persona.hide_indexing_options") - : I18n.t("discourse_ai.ai_persona.show_indexing_options"); - } - @action async updateAllGroups() { this.allGroups = await Group.findAll(); @@ -487,54 +474,13 @@ export default class PersonaEditor extends Component { {{/if}} {{#if this.siteSettings.ai_embeddings_enabled}}
- - {{#if this.editingModel.rag_uploads}} - {{this.indexingOptionsText}} - {{/if}}
- {{#if this.showIndexingOptions}} -
- - - -
-
- - - -
+
-
- {{/if}} + {{/if}}
+ {{#if this.siteSettings.ai_embeddings_enabled}} +
+ +
+ + {{/if}} +
+ {{#if @model.rag_uploads}} + {{this.indexingOptionsText}} + {{/if}} + + {{#if this.showIndexingOptions}} +
+ + + +
+
+ + + +
+ {{yield}} + {{/if}} + +} diff --git a/assets/javascripts/discourse/components/rag-upload-progress.gjs b/assets/javascripts/discourse/components/rag-upload-progress.gjs index 3f43904b..fa47672a 100644 --- a/assets/javascripts/discourse/components/rag-upload-progress.gjs +++ b/assets/javascripts/discourse/components/rag-upload-progress.gjs @@ -9,20 +9,17 @@ import I18n from "discourse-i18n"; export default class RagUploadProgress extends Component { @service messageBus; - @tracked updatedProgress = null; willDestroy() { super.willDestroy(...arguments); - this.messageBus.unsubscribe( - `/discourse-ai/ai-persona-rag/${this.args.upload.id}` - ); + this.messageBus.unsubscribe(`/discourse-ai/rag/${this.args.upload.id}`); } @action trackProgress() { this.messageBus.subscribe( - `/discourse-ai/ai-persona-rag/${this.args.upload.id}`, + `/discourse-ai/rag/${this.args.upload.id}`, this.onIndexingUpdate ); } @@ -32,8 +29,9 @@ export default class RagUploadProgress extends Component { // Order not guaranteed. Discard old updates. if ( !this.updatedProgress || - data.total === 0 || - this.updatedProgress.left > data.left + this.updatedProgress.left === 0 || + this.updatedProgress.left > data.left || + data.total === data.indexed ) { this.updatedProgress = data; } @@ -64,26 +62,23 @@ export default class RagUploadProgress extends Component { }