FEATURE: AI Bot RAG support. (#537)
This PR lets you associate uploads to an AI persona, which we'll split and generate embeddings from. When building the system prompt to get a bot reply, we'll do a similarity search followed by a re-ranking (if available). This will let us find the most relevant fragments from the body of knowledge you associated with the persona, resulting in better, more informed responses.
For now, we'll only allow plain-text files, but this will change in the future.
Commits:
* FEATURE: RAG embeddings for the AI Bot
This first commit introduces a UI where admins can upload text files, which we'll store, split into fragments,
and generate embeddings of. In a next commit, we'll use those to give the bot additional information during
conversations.
* Basic asymmetric similarity search to provide guidance in system prompt
* Fix tests and lint
* Apply reranker to fragments
* Uploads filter, css adjustments and file validations
* Add placeholder for rag fragments
* Update annotations
2024-04-01 12:43:34 -04:00
|
|
|
import { tracked } from "@glimmer/tracking";
|
|
|
|
import Component, { Input } from "@ember/component";
|
|
|
|
import { fn } from "@ember/helper";
|
|
|
|
import { on } from "@ember/modifier";
|
|
|
|
import { action } from "@ember/object";
|
|
|
|
import { inject as service } from "@ember/service";
|
|
|
|
import DButton from "discourse/components/d-button";
|
2024-04-09 10:03:07 -04:00
|
|
|
import { ajax } from "discourse/lib/ajax";
|
FEATURE: AI Bot RAG support. (#537)
This PR lets you associate uploads to an AI persona, which we'll split and generate embeddings from. When building the system prompt to get a bot reply, we'll do a similarity search followed by a re-ranking (if available). This will let us find the most relevant fragments from the body of knowledge you associated with the persona, resulting in better, more informed responses.
For now, we'll only allow plain-text files, but this will change in the future.
Commits:
* FEATURE: RAG embeddings for the AI Bot
This first commit introduces a UI where admins can upload text files, which we'll store, split into fragments,
and generate embeddings of. In a next commit, we'll use those to give the bot additional information during
conversations.
* Basic asymmetric similarity search to provide guidance in system prompt
* Fix tests and lint
* Apply reranker to fragments
* Uploads filter, css adjustments and file validations
* Add placeholder for rag fragments
* Update annotations
2024-04-01 12:43:34 -04:00
|
|
|
import UppyUploadMixin from "discourse/mixins/uppy-upload";
|
|
|
|
import icon from "discourse-common/helpers/d-icon";
|
|
|
|
import discourseDebounce from "discourse-common/lib/debounce";
|
|
|
|
import I18n from "discourse-i18n";
|
2024-04-09 10:03:07 -04:00
|
|
|
import RagUploadProgress from "./rag-upload-progress";
|
FEATURE: AI Bot RAG support. (#537)
This PR lets you associate uploads to an AI persona, which we'll split and generate embeddings from. When building the system prompt to get a bot reply, we'll do a similarity search followed by a re-ranking (if available). This will let us find the most relevant fragments from the body of knowledge you associated with the persona, resulting in better, more informed responses.
For now, we'll only allow plain-text files, but this will change in the future.
Commits:
* FEATURE: RAG embeddings for the AI Bot
This first commit introduces a UI where admins can upload text files, which we'll store, split into fragments,
and generate embeddings of. In a next commit, we'll use those to give the bot additional information during
conversations.
* Basic asymmetric similarity search to provide guidance in system prompt
* Fix tests and lint
* Apply reranker to fragments
* Uploads filter, css adjustments and file validations
* Add placeholder for rag fragments
* Update annotations
2024-04-01 12:43:34 -04:00
|
|
|
|
|
|
|
export default class PersonaRagUploader extends Component.extend(
|
|
|
|
UppyUploadMixin
|
|
|
|
) {
|
|
|
|
@service appEvents;
|
|
|
|
|
|
|
|
@tracked term = null;
|
|
|
|
@tracked filteredUploads = null;
|
2024-04-09 10:03:07 -04:00
|
|
|
@tracked ragIndexingStatuses = null;
|
FEATURE: AI Bot RAG support. (#537)
This PR lets you associate uploads to an AI persona, which we'll split and generate embeddings from. When building the system prompt to get a bot reply, we'll do a similarity search followed by a re-ranking (if available). This will let us find the most relevant fragments from the body of knowledge you associated with the persona, resulting in better, more informed responses.
For now, we'll only allow plain-text files, but this will change in the future.
Commits:
* FEATURE: RAG embeddings for the AI Bot
This first commit introduces a UI where admins can upload text files, which we'll store, split into fragments,
and generate embeddings of. In a next commit, we'll use those to give the bot additional information during
conversations.
* Basic asymmetric similarity search to provide guidance in system prompt
* Fix tests and lint
* Apply reranker to fragments
* Uploads filter, css adjustments and file validations
* Add placeholder for rag fragments
* Update annotations
2024-04-01 12:43:34 -04:00
|
|
|
id = "discourse-ai-persona-rag-uploader";
|
|
|
|
maxFiles = 20;
|
|
|
|
uploadUrl = "/admin/plugins/discourse-ai/ai-personas/files/upload";
|
|
|
|
preventDirectS3Uploads = true;
|
|
|
|
|
|
|
|
didReceiveAttrs() {
|
|
|
|
super.didReceiveAttrs(...arguments);
|
|
|
|
|
|
|
|
if (this.inProgressUploads?.length > 0) {
|
|
|
|
this._uppyInstance?.cancelAll();
|
|
|
|
}
|
|
|
|
|
|
|
|
this.filteredUploads = this.ragUploads || [];
|
2024-04-09 10:03:07 -04:00
|
|
|
|
2024-04-15 09:22:06 -04:00
|
|
|
if (this.ragUploads?.length && this.persona?.id) {
|
2024-04-09 10:03:07 -04:00
|
|
|
ajax(
|
|
|
|
`/admin/plugins/discourse-ai/ai-personas/${this.persona.id}/files/status.json`
|
|
|
|
).then((statuses) => {
|
|
|
|
this.set("ragIndexingStatuses", statuses);
|
|
|
|
});
|
|
|
|
}
|
FEATURE: AI Bot RAG support. (#537)
This PR lets you associate uploads to an AI persona, which we'll split and generate embeddings from. When building the system prompt to get a bot reply, we'll do a similarity search followed by a re-ranking (if available). This will let us find the most relevant fragments from the body of knowledge you associated with the persona, resulting in better, more informed responses.
For now, we'll only allow plain-text files, but this will change in the future.
Commits:
* FEATURE: RAG embeddings for the AI Bot
This first commit introduces a UI where admins can upload text files, which we'll store, split into fragments,
and generate embeddings of. In a next commit, we'll use those to give the bot additional information during
conversations.
* Basic asymmetric similarity search to provide guidance in system prompt
* Fix tests and lint
* Apply reranker to fragments
* Uploads filter, css adjustments and file validations
* Add placeholder for rag fragments
* Update annotations
2024-04-01 12:43:34 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
uploadDone(uploadedFile) {
|
|
|
|
this.onAdd(uploadedFile.upload);
|
|
|
|
this.debouncedSearch();
|
|
|
|
}
|
|
|
|
|
|
|
|
@action
|
|
|
|
submitFiles() {
|
|
|
|
this.fileInputEl.click();
|
|
|
|
}
|
|
|
|
|
|
|
|
@action
|
|
|
|
cancelUploading(upload) {
|
|
|
|
this.appEvents.trigger(`upload-mixin:${this.id}:cancel-upload`, {
|
|
|
|
fileId: upload.id,
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
@action
|
|
|
|
search() {
|
|
|
|
if (this.term) {
|
|
|
|
this.filteredUploads = this.ragUploads.filter((u) => {
|
|
|
|
return (
|
|
|
|
u.original_filename.toUpperCase().indexOf(this.term.toUpperCase()) >
|
|
|
|
-1
|
|
|
|
);
|
|
|
|
});
|
|
|
|
} else {
|
|
|
|
this.filteredUploads = this.ragUploads;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
@action
|
|
|
|
debouncedSearch() {
|
|
|
|
discourseDebounce(this, this.search, 100);
|
|
|
|
}
|
|
|
|
|
|
|
|
<template>
|
|
|
|
<div class="persona-rag-uploader">
|
|
|
|
<h3>{{I18n.t "discourse_ai.ai_persona.uploads.title"}}</h3>
|
|
|
|
<p>{{I18n.t "discourse_ai.ai_persona.uploads.description"}}</p>
|
|
|
|
<p>{{I18n.t "discourse_ai.ai_persona.uploads.hint"}}</p>
|
|
|
|
|
|
|
|
<div class="persona-rag-uploader__search-input-container">
|
|
|
|
<div class="persona-rag-uploader__search-input">
|
|
|
|
{{icon
|
|
|
|
"search"
|
|
|
|
class="persona-rag-uploader__search-input__search-icon"
|
|
|
|
}}
|
|
|
|
<Input
|
|
|
|
class="persona-rag-uploader__search-input__input"
|
|
|
|
placeholder={{I18n.t "discourse_ai.ai_persona.uploads.filter"}}
|
|
|
|
@value={{this.term}}
|
|
|
|
{{on "keyup" this.debouncedSearch}}
|
|
|
|
/>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
<table class="persona-rag-uploader__uploads-list">
|
|
|
|
<tbody>
|
|
|
|
{{#each this.filteredUploads as |upload|}}
|
|
|
|
<tr>
|
|
|
|
<td>
|
|
|
|
<span class="persona-rag-uploader__rag-file-icon">{{icon
|
|
|
|
"file"
|
|
|
|
}}</span>
|
2024-04-09 10:03:07 -04:00
|
|
|
{{upload.original_filename}}
|
|
|
|
</td>
|
|
|
|
<RagUploadProgress
|
|
|
|
@upload={{upload}}
|
|
|
|
@ragIndexingStatuses={{this.ragIndexingStatuses}}
|
|
|
|
/>
|
FEATURE: AI Bot RAG support. (#537)
This PR lets you associate uploads to an AI persona, which we'll split and generate embeddings from. When building the system prompt to get a bot reply, we'll do a similarity search followed by a re-ranking (if available). This will let us find the most relevant fragments from the body of knowledge you associated with the persona, resulting in better, more informed responses.
For now, we'll only allow plain-text files, but this will change in the future.
Commits:
* FEATURE: RAG embeddings for the AI Bot
This first commit introduces a UI where admins can upload text files, which we'll store, split into fragments,
and generate embeddings of. In a next commit, we'll use those to give the bot additional information during
conversations.
* Basic asymmetric similarity search to provide guidance in system prompt
* Fix tests and lint
* Apply reranker to fragments
* Uploads filter, css adjustments and file validations
* Add placeholder for rag fragments
* Update annotations
2024-04-01 12:43:34 -04:00
|
|
|
<td class="persona-rag-uploader__remove-file">
|
|
|
|
<DButton
|
|
|
|
@icon="times"
|
|
|
|
@title="discourse_ai.ai_persona.uploads.remove"
|
|
|
|
@action={{fn @onRemove upload}}
|
|
|
|
@class="btn-flat"
|
|
|
|
/>
|
|
|
|
</td>
|
|
|
|
</tr>
|
|
|
|
{{/each}}
|
|
|
|
{{#each this.inProgressUploads as |upload|}}
|
|
|
|
<tr>
|
|
|
|
<td><span class="persona-rag-uploader__rag-file-icon">{{icon
|
|
|
|
"file"
|
|
|
|
}}</span>
|
|
|
|
{{upload.original_filename}}</td>
|
|
|
|
<td class="persona-rag-uploader__upload-status">
|
|
|
|
<div class="spinner small"></div>
|
|
|
|
<span>{{I18n.t "discourse_ai.ai_persona.uploads.uploading"}}
|
|
|
|
{{upload.uploadProgress}}%</span>
|
|
|
|
</td>
|
|
|
|
<td class="persona-rag-uploader__remove-file">
|
|
|
|
<DButton
|
|
|
|
@icon="times"
|
|
|
|
@title="discourse_ai.ai_persona.uploads.remove"
|
|
|
|
@action={{fn this.cancelUploading upload}}
|
|
|
|
@class="btn-flat"
|
|
|
|
/>
|
|
|
|
</td>
|
|
|
|
</tr>
|
|
|
|
{{/each}}
|
|
|
|
</tbody>
|
|
|
|
</table>
|
|
|
|
|
|
|
|
<input
|
|
|
|
class="hidden-upload-field"
|
|
|
|
disabled={{this.uploading}}
|
|
|
|
type="file"
|
|
|
|
multiple="multiple"
|
|
|
|
accept=".txt"
|
|
|
|
/>
|
|
|
|
<DButton
|
|
|
|
@label="discourse_ai.ai_persona.uploads.button"
|
|
|
|
@icon="plus"
|
|
|
|
@title="discourse_ai.ai_persona.uploads.button"
|
|
|
|
@action={{this.submitFiles}}
|
|
|
|
class="btn-default"
|
|
|
|
/>
|
|
|
|
</div>
|
|
|
|
</template>
|
|
|
|
}
|