mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-02-06 11:38:13 +00:00
eb93b21769
BAAI/bge-m3 is an interesting model, that is multilingual and with a context size of 8192. Even with a 16x larger context, it's only 4x slower to compute it's embeddings on the worst case scenario. Also includes a minor refactor of the rake task, including setting model and concurrency levels when running the backfill task.
39 lines
1.2 KiB
Ruby
39 lines
1.2 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
class AddEmbeddingsTablesforBgeM3 < ActiveRecord::Migration[7.0]
|
|
def change
|
|
create_table :ai_topic_embeddings_8_1, id: false do |t|
|
|
t.integer :topic_id, null: false
|
|
t.integer :model_version, null: false
|
|
t.integer :strategy_version, null: false
|
|
t.text :digest, null: false
|
|
t.column :embeddings, "vector(1024)", null: false
|
|
t.timestamps
|
|
|
|
t.index :topic_id, unique: true
|
|
end
|
|
create_table :ai_post_embeddings_8_1, id: false do |t|
|
|
t.integer :post_id, null: false
|
|
t.integer :model_version, null: false
|
|
t.integer :strategy_version, null: false
|
|
t.text :digest, null: false
|
|
t.column :embeddings, "vector(1024)", null: false
|
|
t.timestamps
|
|
|
|
t.index :post_id, unique: true
|
|
end
|
|
create_table :ai_document_fragment_embeddings_8_1, id: false do |t|
|
|
t.integer :rag_document_fragment_id, null: false
|
|
t.integer :model_version, null: false
|
|
t.integer :strategy_version, null: false
|
|
t.text :digest, null: false
|
|
t.column :embeddings, "vector(1024)", null: false
|
|
t.timestamps
|
|
|
|
t.index :rag_document_fragment_id,
|
|
unique: true,
|
|
name: "rag_document_fragment_id_embeddings_8_1"
|
|
end
|
|
end
|
|
end
|