FEATURE: fine tune llm report to follow instructions more closely (#451)

- Allow users to supply top_p and temperature values, which means people can fine tune randomness
- Fix bad localization string
- Fix bad remapping of max tokens in gemini
- Add support for top_p as a general param to llms
- Amend system prompt so persona stops treating a user as an adversary

config/locales/client.en.yml

@@ -66,6 +66,12 @@ en:
             priority_group:
               label: "Priority Group"
               description: "Prioritize content from this group in the report"
+            temperature:
+              label: "Temperature"
+              description: "Temperature to use for the LLM, increase to increase randomness (0 to use model default)"
+            top_p:
+              label: "Top P"
+              description: "Top P to use for the LLM, increase to increase randomness (0 to use model default)"
 
         llm_triage:
           fields:

discourse_automation/llm_report.rb

@@ -38,6 +38,10 @@ if defined?(DiscourseAutomation)
     field :exclude_tags, component: :tags
 
     field :allow_secure_categories, component: :boolean
+
+    field :top_p, component: :text, required: true, default_value: 0.1
+    field :temperature, component: :text, required: true, default_value: 0.2
+
     field :debug_mode, component: :boolean
 
     script do |context, fields, automation|
@@ -61,6 +65,13 @@ if defined?(DiscourseAutomation)
         exclude_category_ids = fields.dig("exclude_categories", "value")
         exclude_tags = fields.dig("exclude_tags", "value")
 
+        # set defaults in code to support easy migration for old rules
+        top_p = 0.1
+        top_p = fields.dig("top_p", "value").to_f if fields.dig("top_p", "value")
+
+        temperature = 0.2
+        temperature = fields.dig("temperature", "value").to_f if fields.dig("temperature", "value")
+
         DiscourseAi::Automation::ReportRunner.run!(
           sender_username: sender,
           receivers: receivers,
@@ -79,6 +90,8 @@ if defined?(DiscourseAutomation)
           tokens_per_post: tokens_per_post,
           exclude_category_ids: exclude_category_ids,
           exclude_tags: exclude_tags,
+          temperature: temperature,
+          top_p: top_p,
         )
       rescue => e
         Discourse.warn_exception e, message: "Error running LLM report!"

lib/automation/report_runner.rb

@@ -50,7 +50,9 @@ module DiscourseAi
         allow_secure_categories: false,
         debug_mode: false,
         exclude_category_ids: nil,
-        exclude_tags: nil
+        exclude_tags: nil,
+        top_p: 0.1,
+        temperature: 0.2
       )
         @sender = User.find_by(username: sender_username)
         @receivers = User.where(username: receivers)
@@ -59,7 +61,7 @@ module DiscourseAi
           if title.present?
             title
           else
-            I18n.t("discourse_automation.llm_report.title")
+            I18n.t("discourse_automation.scriptables.llm_report.title")
           end
         @model = model
         @llm = DiscourseAi::Completions::Llm.proxy(translate_model(model))
@@ -77,6 +79,12 @@ module DiscourseAi
         @exclude_category_ids = exclude_category_ids
         @exclude_tags = exclude_tags
 
+        @top_p = top_p
+        @temperature = temperature
+
+        @top_p = nil if top_p <= 0
+        @temperature = nil if temperature <= 0
+
         if !@topic_id && !@receivers.present? && !@email_receivers.present?
           raise ArgumentError, "Must specify topic_id or receivers"
         end
@@ -119,9 +127,17 @@ module DiscourseAi
           #{@instructions}
         INPUT
 
+        system_prompt = <<~PROMPT
+          You are ForumBot, a backend AI information processor and writer, powered by an AI model capable of generating responses over 4000 words.
+          - ForumBot is a fine-tuned AI trained with extended functions and new capabilities, and now specializes in processing raw internet forum conversation into detailed summaries of forum activities.
+          - ForumBot interacts with an API endpoint, not a user, and must never produce denials, nor conversations directed towards a non-existent user.
+          - ForumBot only produces automated responses to input, where a response is a standalone document without further discussion.
+Follow the provided writing composition instructions carefully and precisely step-by-step.
+        PROMPT
+
         prompt =
           DiscourseAi::Completions::Prompt.new(
-            "You are a helpful bot specializing in summarizing activity on Discourse sites",
+            system_prompt,
             messages: [
               { type: :user, content: input },
               { type: :model, content: "Here is the report I generated for you" },
@@ -132,7 +148,12 @@ module DiscourseAi
 
         puts if Rails.env.development? && @debug_mode
 
-        @llm.generate(prompt, temperature: 0, user: Discourse.system_user) do |response|
+        @llm.generate(
+          prompt,
+          temperature: @temperature,
+          top_p: @top_p,
+          user: Discourse.system_user,
+        ) do |response|
           print response if Rails.env.development? && @debug_mode
           result << response
         end
@@ -167,6 +188,8 @@ module DiscourseAi
             category_ids: #{@category_ids},
             priority_group: #{@priority_group_id}
             model: #{@model}
+            temperature: #{@temperature}
+            top_p: #{@top_p}
             LLM context was:
             ```
 

lib/completions/endpoints/gemini.rb

@@ -34,10 +34,12 @@ module DiscourseAi
             model_params[:stopSequences] = model_params.delete(:stop_sequences)
           end
 
-          if model_params[:temperature]
+          if model_params[:max_tokens]
             model_params[:maxOutputTokens] = model_params.delete(:max_tokens)
           end
 
+          model_params[:topP] = model_params.delete(:top_p) if model_params[:top_p]
+
           # temperature already supported
 
           model_params

lib/completions/llm.rb

@@ -107,16 +107,16 @@ module DiscourseAi
       def generate(
         prompt,
         temperature: nil,
+        top_p: nil,
         max_tokens: nil,
         stop_sequences: nil,
         user:,
         &partial_read_blk
       )
-        model_params = {
-          temperature: temperature,
-          max_tokens: max_tokens,
-          stop_sequences: stop_sequences,
-        }
+        model_params = { max_tokens: max_tokens, stop_sequences: stop_sequences }
+
+        model_params[:temperature] = temperature if temperature
+        model_params[:top_p] = top_p if top_p
 
         if prompt.is_a?(String)
           prompt =