mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-07-07 15:02:25 +00:00
ab5edae121
* FIX: make AI helper more robust - If JSON is broken for structured output then lean on a more forgiving parser - Gemini 2.5 flash does not support temp, support opting out - Evals for assistant were broken, fix interface - Add some missing LLMs - Translator was not mapped correctly to the feature - fix that - Don't mix XML in prompt for translator * lint * correct logic * simplify code * implement best effort json parsing direct in the structured output object
158 lines
4.5 KiB
Ruby
158 lines
4.5 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
RSpec.describe DiscourseAi::Completions::StructuredOutput do
|
|
subject(:structured_output) do
|
|
described_class.new(
|
|
{
|
|
message: {
|
|
type: "string",
|
|
},
|
|
bool: {
|
|
type: "boolean",
|
|
},
|
|
number: {
|
|
type: "integer",
|
|
},
|
|
status: {
|
|
type: "string",
|
|
},
|
|
list: {
|
|
type: "array",
|
|
items: {
|
|
type: "string",
|
|
},
|
|
},
|
|
},
|
|
)
|
|
end
|
|
|
|
describe "Parsing structured output on the fly" do
|
|
it "acts as a buffer for an streamed JSON" do
|
|
chunks = [
|
|
+"{\"message\": \"Line 1\\n",
|
|
+"Line 2\\n",
|
|
+"Line 3\", ",
|
|
+"\"bool\": true,",
|
|
+"\"number\": 4",
|
|
+"2,",
|
|
+"\"status\": \"o",
|
|
+"\\\"k\\\"\"}",
|
|
]
|
|
|
|
structured_output << chunks[0]
|
|
expect(structured_output.read_buffered_property(:message)).to eq("Line 1\n")
|
|
|
|
structured_output << chunks[1]
|
|
expect(structured_output.read_buffered_property(:message)).to eq("Line 2\n")
|
|
|
|
structured_output << chunks[2]
|
|
expect(structured_output.read_buffered_property(:message)).to eq("Line 3")
|
|
|
|
structured_output << chunks[3]
|
|
expect(structured_output.read_buffered_property(:bool)).to eq(true)
|
|
|
|
# Waiting for number to be fully buffered.
|
|
structured_output << chunks[4]
|
|
expect(structured_output.read_buffered_property(:bool)).to eq(true)
|
|
expect(structured_output.read_buffered_property(:number)).to be_nil
|
|
|
|
structured_output << chunks[5]
|
|
expect(structured_output.read_buffered_property(:number)).to eq(42)
|
|
|
|
structured_output << chunks[6]
|
|
expect(structured_output.read_buffered_property(:number)).to eq(42)
|
|
expect(structured_output.read_buffered_property(:bool)).to eq(true)
|
|
expect(structured_output.read_buffered_property(:status)).to eq("o")
|
|
|
|
structured_output << chunks[7]
|
|
expect(structured_output.read_buffered_property(:status)).to eq("\"k\"")
|
|
|
|
# No partial string left to read.
|
|
expect(structured_output.read_buffered_property(:status)).to eq("")
|
|
end
|
|
|
|
it "supports array types" do
|
|
chunks = [
|
|
+"{ \"",
|
|
+"list",
|
|
+"\":",
|
|
+" [\"",
|
|
+"Hello!",
|
|
+" I am",
|
|
+" a ",
|
|
+"chunk\",",
|
|
+"\"There\"",
|
|
+"]}",
|
|
]
|
|
|
|
structured_output << chunks[0]
|
|
structured_output << chunks[1]
|
|
structured_output << chunks[2]
|
|
expect(structured_output.read_buffered_property(:list)).to eq(nil)
|
|
|
|
structured_output << chunks[3]
|
|
expect(structured_output.read_buffered_property(:list)).to eq([""])
|
|
|
|
structured_output << chunks[4]
|
|
expect(structured_output.read_buffered_property(:list)).to eq(["Hello!"])
|
|
|
|
structured_output << chunks[5]
|
|
structured_output << chunks[6]
|
|
structured_output << chunks[7]
|
|
|
|
expect(structured_output.read_buffered_property(:list)).to eq(["Hello! I am a chunk"])
|
|
|
|
structured_output << chunks[8]
|
|
expect(structured_output.read_buffered_property(:list)).to eq(
|
|
["Hello! I am a chunk", "There"],
|
|
)
|
|
|
|
structured_output << chunks[9]
|
|
expect(structured_output.read_buffered_property(:list)).to eq(
|
|
["Hello! I am a chunk", "There"],
|
|
)
|
|
end
|
|
|
|
it "handles empty newline chunks" do
|
|
chunks = [+"{\"", +"message", +"\":\"", +"Hello!", +"\n", +"\"", +"}"]
|
|
|
|
chunks.each { |c| structured_output << c }
|
|
|
|
expect(structured_output.read_buffered_property(:message)).to eq("Hello!\n")
|
|
end
|
|
end
|
|
|
|
describe "dealing with non-JSON responses" do
|
|
it "treat it as plain text once we determined it's invalid JSON" do
|
|
chunks = [+"I'm not", +"a", +"JSON :)"]
|
|
|
|
structured_output << chunks[0]
|
|
expect(structured_output.read_buffered_property(:bob)).to eq(nil)
|
|
|
|
structured_output << chunks[1]
|
|
expect(structured_output.read_buffered_property(:bob)).to eq(nil)
|
|
|
|
structured_output << chunks[2]
|
|
|
|
structured_output.finish
|
|
expect(structured_output.read_buffered_property(:bob)).to eq(nil)
|
|
end
|
|
|
|
it "can handle broken JSON" do
|
|
broken_json = <<~JSON
|
|
```json
|
|
{
|
|
"message": "This is a broken JSON",
|
|
bool: true
|
|
}
|
|
JSON
|
|
|
|
structured_output << broken_json
|
|
structured_output.finish
|
|
|
|
expect(structured_output.read_buffered_property(:message)).to eq("This is a broken JSON")
|
|
expect(structured_output.read_buffered_property(:bool)).to eq(true)
|
|
end
|
|
end
|
|
end
|