diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 2d2afc6e..2450795c 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -66,6 +66,12 @@ en: priority_group: label: "Priority Group" description: "Prioritize content from this group in the report" + temperature: + label: "Temperature" + description: "Temperature to use for the LLM, increase to increase randomness (0 to use model default)" + top_p: + label: "Top P" + description: "Top P to use for the LLM, increase to increase randomness (0 to use model default)" llm_triage: fields: diff --git a/discourse_automation/llm_report.rb b/discourse_automation/llm_report.rb index 215a3558..7c6332ba 100644 --- a/discourse_automation/llm_report.rb +++ b/discourse_automation/llm_report.rb @@ -38,6 +38,10 @@ if defined?(DiscourseAutomation) field :exclude_tags, component: :tags field :allow_secure_categories, component: :boolean + + field :top_p, component: :text, required: true, default_value: 0.1 + field :temperature, component: :text, required: true, default_value: 0.2 + field :debug_mode, component: :boolean script do |context, fields, automation| @@ -61,6 +65,13 @@ if defined?(DiscourseAutomation) exclude_category_ids = fields.dig("exclude_categories", "value") exclude_tags = fields.dig("exclude_tags", "value") + # set defaults in code to support easy migration for old rules + top_p = 0.1 + top_p = fields.dig("top_p", "value").to_f if fields.dig("top_p", "value") + + temperature = 0.2 + temperature = fields.dig("temperature", "value").to_f if fields.dig("temperature", "value") + DiscourseAi::Automation::ReportRunner.run!( sender_username: sender, receivers: receivers, @@ -79,6 +90,8 @@ if defined?(DiscourseAutomation) tokens_per_post: tokens_per_post, exclude_category_ids: exclude_category_ids, exclude_tags: exclude_tags, + temperature: temperature, + top_p: top_p, ) rescue => e Discourse.warn_exception e, message: "Error running LLM report!" diff --git a/lib/automation/report_runner.rb b/lib/automation/report_runner.rb index 7b4d856c..7713c5c4 100644 --- a/lib/automation/report_runner.rb +++ b/lib/automation/report_runner.rb @@ -50,7 +50,9 @@ module DiscourseAi allow_secure_categories: false, debug_mode: false, exclude_category_ids: nil, - exclude_tags: nil + exclude_tags: nil, + top_p: 0.1, + temperature: 0.2 ) @sender = User.find_by(username: sender_username) @receivers = User.where(username: receivers) @@ -59,7 +61,7 @@ module DiscourseAi if title.present? title else - I18n.t("discourse_automation.llm_report.title") + I18n.t("discourse_automation.scriptables.llm_report.title") end @model = model @llm = DiscourseAi::Completions::Llm.proxy(translate_model(model)) @@ -77,6 +79,12 @@ module DiscourseAi @exclude_category_ids = exclude_category_ids @exclude_tags = exclude_tags + @top_p = top_p + @temperature = temperature + + @top_p = nil if top_p <= 0 + @temperature = nil if temperature <= 0 + if !@topic_id && !@receivers.present? && !@email_receivers.present? raise ArgumentError, "Must specify topic_id or receivers" end @@ -119,9 +127,17 @@ module DiscourseAi #{@instructions} INPUT + system_prompt = <<~PROMPT + You are ForumBot, a backend AI information processor and writer, powered by an AI model capable of generating responses over 4000 words. + - ForumBot is a fine-tuned AI trained with extended functions and new capabilities, and now specializes in processing raw internet forum conversation into detailed summaries of forum activities. + - ForumBot interacts with an API endpoint, not a user, and must never produce denials, nor conversations directed towards a non-existent user. + - ForumBot only produces automated responses to input, where a response is a standalone document without further discussion. +Follow the provided writing composition instructions carefully and precisely step-by-step. + PROMPT + prompt = DiscourseAi::Completions::Prompt.new( - "You are a helpful bot specializing in summarizing activity on Discourse sites", + system_prompt, messages: [ { type: :user, content: input }, { type: :model, content: "Here is the report I generated for you" }, @@ -132,7 +148,12 @@ module DiscourseAi puts if Rails.env.development? && @debug_mode - @llm.generate(prompt, temperature: 0, user: Discourse.system_user) do |response| + @llm.generate( + prompt, + temperature: @temperature, + top_p: @top_p, + user: Discourse.system_user, + ) do |response| print response if Rails.env.development? && @debug_mode result << response end @@ -167,6 +188,8 @@ module DiscourseAi category_ids: #{@category_ids}, priority_group: #{@priority_group_id} model: #{@model} + temperature: #{@temperature} + top_p: #{@top_p} LLM context was: ``` diff --git a/lib/completions/endpoints/gemini.rb b/lib/completions/endpoints/gemini.rb index 2bf2dc14..8668c658 100644 --- a/lib/completions/endpoints/gemini.rb +++ b/lib/completions/endpoints/gemini.rb @@ -34,10 +34,12 @@ module DiscourseAi model_params[:stopSequences] = model_params.delete(:stop_sequences) end - if model_params[:temperature] + if model_params[:max_tokens] model_params[:maxOutputTokens] = model_params.delete(:max_tokens) end + model_params[:topP] = model_params.delete(:top_p) if model_params[:top_p] + # temperature already supported model_params diff --git a/lib/completions/llm.rb b/lib/completions/llm.rb index 758071a9..a7932fe9 100644 --- a/lib/completions/llm.rb +++ b/lib/completions/llm.rb @@ -107,16 +107,16 @@ module DiscourseAi def generate( prompt, temperature: nil, + top_p: nil, max_tokens: nil, stop_sequences: nil, user:, &partial_read_blk ) - model_params = { - temperature: temperature, - max_tokens: max_tokens, - stop_sequences: stop_sequences, - } + model_params = { max_tokens: max_tokens, stop_sequences: stop_sequences } + + model_params[:temperature] = temperature if temperature + model_params[:top_p] = top_p if top_p if prompt.is_a?(String) prompt =