mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-06-30 11:32:18 +00:00
12 lines
259 B
Ruby
12 lines
259 B
Ruby
|
# frozen_string_literal: true
|
||
|
|
|
||
|
|
module DiscourseAi
|
||
|
|
module Tokenizer
|
||
|
|
class GeminiTokenizer < BasicTokenizer
|
||
|
|
def self.tokenizer
|
||
|
|
@@tokenizer ||= Tokenizers.from_file("./plugins/discourse-ai/tokenizers/gemma2.json")
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|