diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb index 084091e8..366fdef3 100644 --- a/app/models/llm_model.rb +++ b/app/models/llm_model.rb @@ -26,17 +26,23 @@ class LlmModel < ActiveRecord::Base access_key_id: :text, region: :text, disable_native_tools: :checkbox, + disable_temperature: :checkbox, + disable_top_p: :checkbox, enable_reasoning: :checkbox, reasoning_tokens: :number, }, anthropic: { disable_native_tools: :checkbox, + disable_temperature: :checkbox, + disable_top_p: :checkbox, enable_reasoning: :checkbox, reasoning_tokens: :number, }, open_ai: { organization: :text, disable_native_tools: :checkbox, + disable_temperature: :checkbox, + disable_top_p: :checkbox, disable_streaming: :checkbox, reasoning_effort: { type: :enum, @@ -69,6 +75,8 @@ class LlmModel < ActiveRecord::Base provider_order: :text, provider_quantizations: :text, disable_streaming: :checkbox, + disable_temperature: :checkbox, + disable_top_p: :checkbox, }, } end diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index a6272d0c..15792461 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -488,6 +488,8 @@ en: reasoning_effort: "Reasoning effort (only applicable to reasoning models)" enable_reasoning: "Enable reasoning (only applicable to Sonnet 3.7)" reasoning_tokens: "Number of tokens used for reasoning" + disable_temperature: "Disable temperature (some thinking models don't support temperature)" + disable_top_p: "Disable top P (some thinking models don't support top P)" related_topics: title: "Related topics" diff --git a/lib/completions/endpoints/anthropic.rb b/lib/completions/endpoints/anthropic.rb index 2fb0b28d..75d1e460 100644 --- a/lib/completions/endpoints/anthropic.rb +++ b/lib/completions/endpoints/anthropic.rb @@ -10,6 +10,9 @@ module DiscourseAi def normalize_model_params(model_params) # max_tokens, temperature, stop_sequences are already supported + model_params = model_params.dup + model_params.delete(:top_p) if llm_model.lookup_custom_param("disable_top_p") + model_params.delete(:temperature) if llm_model.lookup_custom_param("disable_temperature") model_params end diff --git a/lib/completions/endpoints/aws_bedrock.rb b/lib/completions/endpoints/aws_bedrock.rb index 604d0073..d5b56b70 100644 --- a/lib/completions/endpoints/aws_bedrock.rb +++ b/lib/completions/endpoints/aws_bedrock.rb @@ -16,6 +16,9 @@ module DiscourseAi model_params = model_params.dup # max_tokens, temperature, stop_sequences, top_p are already supported + # + model_params.delete(:top_p) if llm_model.lookup_custom_param("disable_top_p") + model_params.delete(:temperature) if llm_model.lookup_custom_param("disable_temperature") model_params end diff --git a/lib/completions/endpoints/open_ai.rb b/lib/completions/endpoints/open_ai.rb index 35afd778..24cd1285 100644 --- a/lib/completions/endpoints/open_ai.rb +++ b/lib/completions/endpoints/open_ai.rb @@ -24,6 +24,9 @@ module DiscourseAi model_params[:stop] = model_params.delete(:stop_sequences) end + model_params.delete(:top_p) if llm_model.lookup_custom_param("disable_top_p") + model_params.delete(:temperature) if llm_model.lookup_custom_param("disable_temperature") + model_params end diff --git a/lib/completions/endpoints/open_router.rb b/lib/completions/endpoints/open_router.rb index bac646d9..08122799 100644 --- a/lib/completions/endpoints/open_router.rb +++ b/lib/completions/endpoints/open_router.rb @@ -16,6 +16,9 @@ module DiscourseAi model_params[:stop] = model_params.delete(:stop_sequences) end + model_params.delete(:top_p) if llm_model.lookup_custom_param("disable_top_p") + model_params.delete(:temperature) if llm_model.lookup_custom_param("disable_temperature") + model_params end diff --git a/spec/lib/completions/endpoints/anthropic_spec.rb b/spec/lib/completions/endpoints/anthropic_spec.rb index 579af695..625d2184 100644 --- a/spec/lib/completions/endpoints/anthropic_spec.rb +++ b/spec/lib/completions/endpoints/anthropic_spec.rb @@ -664,4 +664,54 @@ data: {"type":"content_block_start","index":0,"content_block":{"type":"redacted_ expect(log.feature_name).to eq("testing") expect(log.response_tokens).to eq(30) end + + describe "parameter disabling" do + it "excludes disabled parameters from the request" do + model.update!(provider_params: { disable_top_p: true, disable_temperature: true }) + + parsed_body = nil + stub_request(:post, url).with( + body: + proc do |req_body| + parsed_body = JSON.parse(req_body, symbolize_names: true) + true + end, + headers: { + "Content-Type" => "application/json", + "X-Api-Key" => "123", + "Anthropic-Version" => "2023-06-01", + }, + ).to_return( + status: 200, + body: { + id: "msg_123", + type: "message", + role: "assistant", + content: [{ type: "text", text: "test response" }], + model: "claude-3-opus-20240229", + usage: { + input_tokens: 10, + output_tokens: 5, + }, + }.to_json, + ) + + # Request with parameters that should be ignored + llm.generate( + prompt, + user: Discourse.system_user, + top_p: 0.9, + temperature: 0.8, + max_tokens: 500, + ) + + # Verify disabled parameters aren't included + expect(parsed_body).not_to have_key(:top_p) + expect(parsed_body).not_to have_key(:temperature) + + # Verify other parameters still work + expect(parsed_body).to have_key(:max_tokens) + expect(parsed_body[:max_tokens]).to eq(500) + end + end end diff --git a/spec/lib/completions/endpoints/aws_bedrock_spec.rb b/spec/lib/completions/endpoints/aws_bedrock_spec.rb index ebe8094b..373ba4c9 100644 --- a/spec/lib/completions/endpoints/aws_bedrock_spec.rb +++ b/spec/lib/completions/endpoints/aws_bedrock_spec.rb @@ -436,4 +436,52 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrock do end end end + + describe "parameter disabling" do + it "excludes disabled parameters from the request" do + model.update!( + provider_params: { + access_key_id: "123", + region: "us-east-1", + disable_top_p: true, + disable_temperature: true, + }, + ) + + proxy = DiscourseAi::Completions::Llm.proxy("custom:#{model.id}") + request = nil + + content = { + content: [text: "test response"], + usage: { + input_tokens: 10, + output_tokens: 5, + }, + }.to_json + + stub_request( + :post, + "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet-20240229-v1:0/invoke", + ) + .with do |inner_request| + request = inner_request + true + end + .to_return(status: 200, body: content) + + # Request with parameters that should be ignored + proxy.generate("test prompt", user: user, top_p: 0.9, temperature: 0.8, max_tokens: 500) + + # Parse the request body + request_body = JSON.parse(request.body) + + # Verify disabled parameters aren't included + expect(request_body).not_to have_key("top_p") + expect(request_body).not_to have_key("temperature") + + # Verify other parameters still work + expect(request_body).to have_key("max_tokens") + expect(request_body["max_tokens"]).to eq(500) + end + end end diff --git a/spec/lib/completions/endpoints/open_ai_spec.rb b/spec/lib/completions/endpoints/open_ai_spec.rb index 67aabd55..fb9f07f3 100644 --- a/spec/lib/completions/endpoints/open_ai_spec.rb +++ b/spec/lib/completions/endpoints/open_ai_spec.rb @@ -395,6 +395,37 @@ RSpec.describe DiscourseAi::Completions::Endpoints::OpenAi do end end + describe "parameter disabling" do + it "excludes disabled parameters from the request" do + model.update!(provider_params: { disable_top_p: true, disable_temperature: true }) + + parsed_body = nil + stub_request(:post, "https://api.openai.com/v1/chat/completions").with( + body: + proc do |req_body| + parsed_body = JSON.parse(req_body, symbolize_names: true) + true + end, + ).to_return( + status: 200, + body: { choices: [{ message: { content: "test response" } }] }.to_json, + ) + + dialect = compliance.dialect(prompt: compliance.generic_prompt) + + # Request with parameters that should be ignored + endpoint.perform_completion!(dialect, user, { top_p: 0.9, temperature: 0.8, max_tokens: 100 }) + + # Verify disabled parameters aren't included + expect(parsed_body).not_to have_key(:top_p) + expect(parsed_body).not_to have_key(:temperature) + + # Verify other parameters still work + expect(parsed_body).to have_key(:max_tokens) + expect(parsed_body[:max_tokens]).to eq(100) + end + end + describe "image support" do it "can handle images" do model = Fabricate(:llm_model, vision_enabled: true) diff --git a/spec/lib/completions/endpoints/open_router_spec.rb b/spec/lib/completions/endpoints/open_router_spec.rb index e21dd695..8beb48ac 100644 --- a/spec/lib/completions/endpoints/open_router_spec.rb +++ b/spec/lib/completions/endpoints/open_router_spec.rb @@ -44,4 +44,35 @@ RSpec.describe DiscourseAi::Completions::Endpoints::OpenRouter do expect(parsed_body).to eq(expected) end + + it "excludes disabled parameters from the request" do + open_router_model.update!(provider_params: { disable_top_p: true, disable_temperature: true }) + + parsed_body = nil + stub_request(:post, open_router_model.url).with( + body: proc { |body| parsed_body = JSON.parse(body, symbolize_names: true) }, + headers: { + "Content-Type" => "application/json", + "X-Title" => "Discourse AI", + "HTTP-Referer" => "https://www.discourse.org/ai", + "Authorization" => "Bearer 123", + }, + ).to_return( + status: 200, + body: { "choices" => [message: { role: "assistant", content: "test response" }] }.to_json, + ) + + proxy = DiscourseAi::Completions::Llm.proxy("custom:#{open_router_model.id}") + + # Request with parameters that should be ignored + proxy.generate("test", user: user, top_p: 0.9, temperature: 0.8, max_tokens: 500) + + # Verify disabled parameters aren't included + expect(parsed_body).not_to have_key(:top_p) + expect(parsed_body).not_to have_key(:temperature) + + # Verify other parameters still work + expect(parsed_body).to have_key(:max_tokens) + expect(parsed_body[:max_tokens]).to eq(500) + end end