FIX: RAG uploader must support multi-file indexing. (#592)

Updating the editing model's rag_uploads in the editor component broke multi-file uploading. Instead, we'll keep the uploads in the uploader and update the model when we finish.

This PR also fast-tracks the initial update so we can show feedback to the user quickly, and allows uploading MD files.

Bug reported on https://meta.discourse.org/t/discourse-ai-persona-upload-support/304049/11
This commit is contained in:
Roman Rizzi 2024-04-25 10:48:55 -03:00 committed by GitHub
parent 0c4069ab3f
commit 283445cf81
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 56 additions and 22 deletions

View File

@ -152,10 +152,13 @@ module DiscourseAi
def validate_extension!(filename) def validate_extension!(filename)
extension = File.extname(filename)[1..-1] || "" extension = File.extname(filename)[1..-1] || ""
authorized_extension = "txt" authorized_extensions = %w[txt md]
if extension != authorized_extension if !authorized_extensions.include?(extension)
raise Discourse::InvalidParameters.new( raise Discourse::InvalidParameters.new(
I18n.t("upload.unauthorized", authorized_extensions: authorized_extension), I18n.t(
"upload.unauthorized",
authorized_extensions: authorized_extensions.join(" "),
),
) )
end end
end end

View File

@ -26,6 +26,8 @@ module ::Jobs
document = get_uploaded_file(upload) document = get_uploaded_file(upload)
return if document.nil? return if document.nil?
RagDocumentFragment.publish_status(upload, { total: 0, indexed: 0, left: 0 })
fragment_ids = [] fragment_ids = []
idx = 0 idx = 0
@ -54,11 +56,6 @@ module ::Jobs
end end
end end
RagDocumentFragment.publish_status(
upload,
{ total: fragment_ids.size, indexed: 0, left: fragment_ids.size },
)
fragment_ids.each_slice(50) do |slice| fragment_ids.each_slice(50) do |slice|
Jobs.enqueue(:generate_rag_embeddings, fragment_ids: slice) Jobs.enqueue(:generate_rag_embeddings, fragment_ids: slice)
end end

View File

@ -207,11 +207,8 @@ export default class PersonaEditor extends Component {
} }
@action @action
addUpload(upload) { updateUploads(uploads) {
const newUpload = upload; this.editingModel.rag_uploads = uploads;
newUpload.status = "uploaded";
newUpload.statusText = I18n.t("discourse_ai.ai_persona.uploads.uploaded");
this.editingModel.rag_uploads.addObject(newUpload);
} }
@action @action
@ -460,8 +457,7 @@ export default class PersonaEditor extends Component {
<div class="control-group"> <div class="control-group">
<PersonaRagUploader <PersonaRagUploader
@persona={{this.editingModel}} @persona={{this.editingModel}}
@ragUploads={{this.editingModel.rag_uploads}} @updateUploads={{this.updateUploads}}
@onAdd={{this.addUpload}}
@onRemove={{this.removeUpload}} @onRemove={{this.removeUpload}}
/> />
<a <a

View File

@ -3,6 +3,7 @@ import Component, { Input } from "@ember/component";
import { fn } from "@ember/helper"; import { fn } from "@ember/helper";
import { on } from "@ember/modifier"; import { on } from "@ember/modifier";
import { action } from "@ember/object"; import { action } from "@ember/object";
import willDestroy from "@ember/render-modifiers/modifiers/will-destroy";
import { inject as service } from "@ember/service"; import { inject as service } from "@ember/service";
import DButton from "discourse/components/d-button"; import DButton from "discourse/components/d-button";
import { ajax } from "discourse/lib/ajax"; import { ajax } from "discourse/lib/ajax";
@ -20,6 +21,7 @@ export default class PersonaRagUploader extends Component.extend(
@tracked term = null; @tracked term = null;
@tracked filteredUploads = null; @tracked filteredUploads = null;
@tracked ragIndexingStatuses = null; @tracked ragIndexingStatuses = null;
@tracked ragUploads = null;
id = "discourse-ai-persona-rag-uploader"; id = "discourse-ai-persona-rag-uploader";
maxFiles = 20; maxFiles = 20;
uploadUrl = "/admin/plugins/discourse-ai/ai-personas/files/upload"; uploadUrl = "/admin/plugins/discourse-ai/ai-personas/files/upload";
@ -32,7 +34,8 @@ export default class PersonaRagUploader extends Component.extend(
this._uppyInstance?.cancelAll(); this._uppyInstance?.cancelAll();
} }
this.filteredUploads = this.ragUploads || []; this.ragUploads = this.persona?.rag_uploads || [];
this.filteredUploads = this.ragUploads;
if (this.ragUploads?.length && this.persona?.id) { if (this.ragUploads?.length && this.persona?.id) {
ajax( ajax(
@ -41,10 +44,27 @@ export default class PersonaRagUploader extends Component.extend(
this.set("ragIndexingStatuses", statuses); this.set("ragIndexingStatuses", statuses);
}); });
} }
this.appEvents.on(
`upload-mixin:${this.id}:all-uploads-complete`,
this,
"_updatePersonaWithUploads"
);
}
removeListener() {
this.appEvents.off(`upload-mixin:${this.id}:all-uploads-complete`);
}
_updatePersonaWithUploads() {
this.updateUploads(this.ragUploads);
} }
uploadDone(uploadedFile) { uploadDone(uploadedFile) {
this.onAdd(uploadedFile.upload); const newUpload = uploadedFile.upload;
newUpload.status = "uploaded";
newUpload.statusText = I18n.t("discourse_ai.ai_persona.uploads.uploaded");
this.ragUploads.pushObject(newUpload);
this.debouncedSearch(); this.debouncedSearch();
} }
@ -79,8 +99,16 @@ export default class PersonaRagUploader extends Component.extend(
discourseDebounce(this, this.search, 100); discourseDebounce(this, this.search, 100);
} }
@action
removeUpload(upload) {
this.ragUploads.removeObject(upload);
this.onRemove(upload);
this.debouncedSearch();
}
<template> <template>
<div class="persona-rag-uploader"> <div class="persona-rag-uploader" {{willDestroy this.removeListener}}>
<h3>{{I18n.t "discourse_ai.ai_persona.uploads.title"}}</h3> <h3>{{I18n.t "discourse_ai.ai_persona.uploads.title"}}</h3>
<p>{{I18n.t "discourse_ai.ai_persona.uploads.description"}}</p> <p>{{I18n.t "discourse_ai.ai_persona.uploads.description"}}</p>
<p>{{I18n.t "discourse_ai.ai_persona.uploads.hint"}}</p> <p>{{I18n.t "discourse_ai.ai_persona.uploads.hint"}}</p>
@ -118,7 +146,7 @@ export default class PersonaRagUploader extends Component.extend(
<DButton <DButton
@icon="times" @icon="times"
@title="discourse_ai.ai_persona.uploads.remove" @title="discourse_ai.ai_persona.uploads.remove"
@action={{fn @onRemove upload}} @action={{fn this.removeUpload upload}}
@class="btn-flat" @class="btn-flat"
/> />
</td> </td>
@ -153,7 +181,7 @@ export default class PersonaRagUploader extends Component.extend(
disabled={{this.uploading}} disabled={{this.uploading}}
type="file" type="file"
multiple="multiple" multiple="multiple"
accept=".txt" accept=".txt,.md"
/> />
<DButton <DButton
@label="discourse_ai.ai_persona.uploads.button" @label="discourse_ai.ai_persona.uploads.button"

View File

@ -30,17 +30,27 @@ export default class RagUploadProgress extends Component {
@bind @bind
onIndexingUpdate(data) { onIndexingUpdate(data) {
// Order not guaranteed. Discard old updates. // Order not guaranteed. Discard old updates.
if (!this.updatedProgress || this.updatedProgress.left > data.left) { if (
!this.updatedProgress ||
data.total === 0 ||
this.updatedProgress.left > data.left
) {
this.updatedProgress = data; this.updatedProgress = data;
} }
} }
get calculateProgress() { get calculateProgress() {
if (this.progress.total === 0) {
return 0;
}
return Math.ceil((this.progress.indexed * 100) / this.progress.total); return Math.ceil((this.progress.indexed * 100) / this.progress.total);
} }
get fullyIndexed() { get fullyIndexed() {
return this.progress && this.progress.left === 0; return (
this.progress && this.progress.total !== 0 && this.progress.left === 0
);
} }
get progress() { get progress() {