discourse-ai/lib/tokenizer/mixtral_tokenizer.rb

12 lines
261 B
Ruby
Raw Normal View History

# frozen_string_literal: true
module DiscourseAi
module Tokenizer
class MixtralTokenizer < BasicTokenizer
def self.tokenizer
@@tokenizer ||= Tokenizers.from_file("./plugins/discourse-ai/tokenizers/mixtral.json")
end
end
end
end