discourse-ai/db/migrate/20250702073222_rename_mixtral_tokenizer_to_mistral_tokenizer.rb
Rafael dos Santos Silva d792919ddf
DEV: Move tokenizers to a gem (#1481)
Also renames the Mixtral tokenizer to Mistral.

See gem at github.com/discourse/discourse_ai-tokenizers


Co-authored-by: Roman Rizzi <roman@discourse.org>
2025-07-02 14:43:03 -03:00

44 lines
950 B
Ruby

# frozen_string_literal: true
class RenameMixtralTokenizerToMistralTokenizer < ActiveRecord::Migration[7.2]
def up
execute <<~SQL
UPDATE
llm_models
SET
tokenizer = 'DiscourseAi::Tokenizer::Mistral'
WHERE
tokenizer = 'DiscourseAi::Tokenizer::Mixtral'
SQL
execute <<~SQL
UPDATE
embedding_definitions
SET
tokenizer_class = 'DiscourseAi::Tokenizer::Mistral'
WHERE
tokenizer_class = 'DiscourseAi::Tokenizer::Mixtral'
SQL
end
def down
execute <<~SQL
UPDATE
llm_models
SET
tokenizer = 'DiscourseAi::Tokenizer::Mixtral'
WHERE
tokenizer = 'DiscourseAi::Tokenizer::Mistral'
SQL
execute <<~SQL
UPDATE
embedding_definitions
SET
tokenizer_class = 'DiscourseAi::Tokenizer::Mixtral'
WHERE
tokenizer_class = 'DiscourseAi::Tokenizer::Mistral'
SQL
end
end