discourse-ai/spec/requests/ai_helper/assistant_controller_spec.rb
Roman Rizzi 0634b85a81
UX: Validations to LLM-backed features (except AI Bot) (#436)
* UX: Validations to Llm-backed features (except AI Bot)

This change is part of an ongoing effort to prevent enabling a broken feature due to lack of configuration. We also want to explicit which provider we are going to use. For example, Claude models are available through AWS Bedrock and Anthropic, but the configuration differs.

Validations are:

* You must choose a model before enabling the feature.
* You must turn off the feature before setting the model to blank.
* You must configure each model settings before being able to select it.

* Add provider name to summarization options

* vLLM can technically support same models as HF

* Check we can talk to the selected model

* Check for Bedrock instead of anthropic as a site could have both creds setup
2024-01-29 16:04:25 -03:00

158 lines
5.3 KiB
Ruby

# frozen_string_literal: true
RSpec.describe DiscourseAi::AiHelper::AssistantController do
before { SiteSetting.ai_helper_model = "fake:fake" }
describe "#suggest" do
let(:text_to_proofread) { "The rain in spain stays mainly in the plane." }
let(:proofread_text) { "The rain in Spain, stays mainly in the Plane." }
let(:mode) { CompletionPrompt::PROOFREAD }
context "when not logged in" do
it "returns a 403 response" do
post "/discourse-ai/ai-helper/suggest", params: { text: text_to_proofread, mode: mode }
expect(response.status).to eq(403)
end
end
context "when logged in as an user without enough privileges" do
fab!(:user) { Fabricate(:newuser) }
before do
sign_in(user)
SiteSetting.ai_helper_allowed_groups = Group::AUTO_GROUPS[:staff]
end
it "returns a 403 response" do
post "/discourse-ai/ai-helper/suggest", params: { text: text_to_proofread, mode: mode }
expect(response.status).to eq(403)
end
end
context "when logged in as an allowed user" do
fab!(:user) { Fabricate(:user) }
before do
sign_in(user)
user.group_ids = [Group::AUTO_GROUPS[:trust_level_1]]
SiteSetting.ai_helper_allowed_groups = Group::AUTO_GROUPS[:trust_level_1]
end
it "returns a 400 if the helper mode is invalid" do
invalid_mode = "asd"
post "/discourse-ai/ai-helper/suggest",
params: {
text: text_to_proofread,
mode: invalid_mode,
}
expect(response.status).to eq(400)
end
it "returns a 400 if the text is blank" do
post "/discourse-ai/ai-helper/suggest", params: { mode: mode }
expect(response.status).to eq(400)
end
it "returns a generic error when the completion call fails" do
DiscourseAi::Completions::Llm
.any_instance
.expects(:generate)
.raises(DiscourseAi::Completions::Endpoints::Base::CompletionFailed)
post "/discourse-ai/ai-helper/suggest", params: { mode: mode, text: text_to_proofread }
expect(response.status).to eq(502)
end
it "returns a suggestion" do
expected_diff =
"<div class=\"inline-diff\"><p>The rain in <ins>Spain</ins><ins>,</ins><ins> </ins><del>spain </del>stays mainly in the <ins>Plane</ins><del>plane</del>.</p></div>"
DiscourseAi::Completions::Llm.with_prepared_responses([proofread_text]) do
post "/discourse-ai/ai-helper/suggest", params: { mode: mode, text: text_to_proofread }
expect(response.status).to eq(200)
expect(response.parsed_body["suggestions"].first).to eq(proofread_text)
expect(response.parsed_body["diff"]).to eq(expected_diff)
end
end
it "uses custom instruction when using custom_prompt mode" do
translated_text = "Un usuario escribio esto"
expected_diff =
"<div class=\"inline-diff\"><p><ins>Un </ins><ins>usuario </ins><ins>escribio </ins><ins>esto</ins><del>A </del><del>user </del><del>wrote </del><del>this</del></p></div>"
expected_input = <<~TEXT.strip
<input>Translate to Spanish:
A user wrote this</input>
TEXT
DiscourseAi::Completions::Llm.with_prepared_responses([translated_text]) do
post "/discourse-ai/ai-helper/suggest",
params: {
mode: CompletionPrompt::CUSTOM_PROMPT,
text: "A user wrote this",
custom_prompt: "Translate to Spanish",
}
expect(response.status).to eq(200)
expect(response.parsed_body["suggestions"].first).to eq(translated_text)
expect(response.parsed_body["diff"]).to eq(expected_diff)
end
end
end
end
describe "#prompts" do
context "when not logged in" do
it "returns a 403 response" do
get "/discourse-ai/ai-helper/prompts"
expect(response.status).to eq(403)
end
end
context "when logged in as a user without enough privileges" do
fab!(:user) { Fabricate(:newuser) }
before do
sign_in(user)
SiteSetting.ai_helper_allowed_groups = Group::AUTO_GROUPS[:staff]
end
it "returns a 403 response" do
get "/discourse-ai/ai-helper/prompts"
expect(response.status).to eq(403)
end
end
context "when logged in as an allowed user" do
fab!(:user) { Fabricate(:user) }
before do
sign_in(user)
user.group_ids = [Group::AUTO_GROUPS[:trust_level_1]]
SiteSetting.ai_helper_allowed_groups = Group::AUTO_GROUPS[:trust_level_1]
SiteSetting.ai_helper_illustrate_post_model = "stable_diffusion_xl"
end
it "returns a list of prompts when no name_filter is provided" do
get "/discourse-ai/ai-helper/prompts"
expect(response.status).to eq(200)
expect(response.parsed_body.length).to eq(7)
end
it "returns a list with with filtered prompts when name_filter is provided" do
get "/discourse-ai/ai-helper/prompts", params: { name_filter: "proofread" }
expect(response.status).to eq(200)
expect(response.parsed_body.length).to eq(1)
expect(response.parsed_body.first["name"]).to eq("proofread")
end
end
end
end