From 7357280e8892a2faf5c91737e76d7fb7168f393e Mon Sep 17 00:00:00 2001 From: Rafael dos Santos Silva Date: Mon, 7 Jul 2025 15:07:27 -0300 Subject: [PATCH] FEATURE: Add old OpenAI tokenizer to embeddings (#1487) --- app/models/embedding_definition.rb | 1 + plugin.rb | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/embedding_definition.rb b/app/models/embedding_definition.rb index 8a144f91..23b37ec4 100644 --- a/app/models/embedding_definition.rb +++ b/app/models/embedding_definition.rb @@ -23,6 +23,7 @@ class EmbeddingDefinition < ActiveRecord::Base DiscourseAi::Tokenizer::GeminiTokenizer, DiscourseAi::Tokenizer::MultilingualE5LargeTokenizer, DiscourseAi::Tokenizer::OpenAiTokenizer, + DiscourseAi::Tokenizer::OpenAiCl100kTokenizer, DiscourseAi::Tokenizer::MistralTokenizer, DiscourseAi::Tokenizer::QwenTokenizer, ].map(&:name) diff --git a/plugin.rb b/plugin.rb index daf9c10e..6758238f 100644 --- a/plugin.rb +++ b/plugin.rb @@ -10,7 +10,7 @@ gem "tokenizers", "0.5.4" gem "tiktoken_ruby", "0.0.11.1" -gem "discourse_ai-tokenizers", "0.2.0", require_name: "discourse_ai/tokenizers" +gem "discourse_ai-tokenizers", "0.3.1", require_name: "discourse_ai/tokenizers" gem "ed25519", "1.2.4" #TODO remove this as existing ssl gem should handle this # we probably want to move all dependencies directly in to the Discourse Gemfile, this