FEATURE: add support for an open LLM implementation

This is very much a work in progress, testing at the moment on:

TheBloke/Llama-2-7B-chat-fp16

lib/modules/ai_bot/bot.rb

@@ -73,7 +73,11 @@ module DiscourseAi
       MAX_COMPLETIONS = 5
 
       def self.as(bot_user)
-        available_bots = [DiscourseAi::AiBot::OpenAiBot, DiscourseAi::AiBot::AnthropicBot]
+        available_bots = [
+          DiscourseAi::AiBot::OpenAiBot,
+          DiscourseAi::AiBot::AnthropicBot,
+          DiscourseAi::AiBot::OpenLlmBot,
+        ]
 
         bot =
           available_bots.detect(-> { raise BOT_NOT_FOUND }) do |bot_klass|

lib/modules/ai_bot/entry_point.rb

@@ -8,7 +8,14 @@ module DiscourseAi
       GPT4_ID = -110
       GPT3_5_TURBO_ID = -111
       CLAUDE_V2_ID = -112
-      BOTS = [[GPT4_ID, "gpt4_bot"], [GPT3_5_TURBO_ID, "gpt3.5_bot"], [CLAUDE_V2_ID, "claude_bot"]]
+      OPEN_LLM_ID = -113
+
+      BOTS = [
+        [GPT4_ID, "gpt4_bot"],
+        [GPT3_5_TURBO_ID, "gpt3.5_bot"],
+        [CLAUDE_V2_ID, "claude_bot"],
+        [OPEN_LLM_ID, "open_llm_bot"],
+      ]
 
       def self.map_bot_model_to_user_id(model_name)
         case model_name
@@ -18,6 +25,8 @@ module DiscourseAi
           GPT4_ID
         in "claude-2"
           CLAUDE_V2_ID
+        in "open-llm"
+          OPEN_LLM_ID
         else
           nil
         end
@@ -29,6 +38,7 @@ module DiscourseAi
         require_relative "bot"
         require_relative "anthropic_bot"
         require_relative "open_ai_bot"
+        require_relative "open_llm_bot"
         require_relative "commands/command"
         require_relative "commands/search_command"
         require_relative "commands/categories_command"

lib/modules/ai_bot/open_llm_bot.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module AiBot
+    class OpenLlmBot < Bot
+      # format of thebloke chat models is:
+      # [INST] <<SYS>> sys message <</SYS>> {prompt} [/INST] {model_reply} [INST] {prompt} [/INST]
+
+      def self.can_reply_as?(bot_user)
+        bot_user.id == DiscourseAi::AiBot::EntryPoint::OPEN_LLM_ID
+      end
+
+      def bot_prompt_with_topic_context(post)
+        messages = super(post)
+
+        # start with system
+        result = +""
+        result << "[INST] <<SYS>>\n #{messages.shift[:content]} <</SYS>>\n\n #{messages.shift[:content]} [/INST]"
+
+        messages.each do |message|
+          result << "\n\n[INST]#{message[:bot] ? "" : message[:username] + ":"} #{message[:content]} [/INST]"
+        end
+
+        result
+      end
+
+      def prompt_limit
+        2000
+      end
+
+      def title_prompt(post)
+        super(post).join("\n\n") + "\n\nAssistant:"
+      end
+
+      def get_delta(partial, context)
+        partial.dig(:token, :text) || ""
+      end
+
+      private
+
+      def populate_functions(partial, function)
+        # nothing to do here, no proper function support quite yet
+      end
+
+      def build_message(poster_username, content, system: false, function: nil)
+        { bot: poster_username == bot_user.username, username: poster_username, content: content }
+      end
+
+      def model_for
+        # we only support single model hosting for huggingface api for now
+        "random-string-for-now"
+      end
+
+      def get_updated_title(prompt)
+        DiscourseAi::Inference::HuggingFaceTextGeneration.perform!(
+          prompt,
+          model_for,
+          temperature: 0.7,
+          max_tokens: 40,
+        ).dig(:completion)
+      end
+
+      def submit_prompt(prompt, prefer_low_cost: false, &blk)
+        DiscourseAi::Inference::HuggingFaceTextGeneration.perform!(
+          prompt,
+          model_for,
+          temperature: 0.4,
+          max_tokens: 200,
+          &blk
+        )
+      end
+
+      def tokenize(text)
+        DiscourseAi::Tokenizer::AnthropicTokenizer.tokenize(text)
+      end
+    end
+  end
+end