mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-02-07 12:08:13 +00:00
03eccbe392
Polymorphic RAG means that we will be able to access RAG fragments both from AiPersona and AiCustomTool In turn this gives us support for richer RAG implementations.
28 lines
1.0 KiB
Ruby
28 lines
1.0 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
module ::Jobs
|
|
class GenerateRagEmbeddings < ::Jobs::Base
|
|
sidekiq_options queue: "ultra_low"
|
|
# we could also restrict concurrency but this takes so long if it is not concurrent
|
|
|
|
def execute(args)
|
|
return if (fragments = RagDocumentFragment.where(id: args[:fragment_ids].to_a)).empty?
|
|
|
|
truncation = DiscourseAi::Embeddings::Strategies::Truncation.new
|
|
vector_rep =
|
|
DiscourseAi::Embeddings::VectorRepresentations::Base.current_representation(truncation)
|
|
|
|
# generate_representation_from checks compares the digest value to make sure
|
|
# the embedding is only generated once per fragment unless something changes.
|
|
fragments.map { |fragment| vector_rep.generate_representation_from(fragment) }
|
|
|
|
last_fragment = fragments.last
|
|
target = last_fragment.target
|
|
upload = last_fragment.upload
|
|
|
|
indexing_status = RagDocumentFragment.indexing_status(target, [upload])[upload.id]
|
|
RagDocumentFragment.publish_status(upload, indexing_status)
|
|
end
|
|
end
|
|
end
|