mirror of
https://github.com/discourse/discourse-ai.git
synced 2025-03-09 11:48:47 +00:00
9a79afcdbf
* DEV: Better strategies for summarization The strategy responsibility needs to be "Given a collection of texts, I know how to summarize them most efficiently, using the minimum amount of requests and maximizing token usage". There are different token limits for each model, so it all boils down to two different strategies: Fold all these texts into a single one, doing the summarization in chunks, and then build a summary from those. Build it by combining texts in a single prompt, and truncate it according to your token limits. While the latter is less than ideal, we need it for "bart-large-cnn-samsum" and "flan-t5-base-samsum", both with low limits. The rest will rely on folding. * Expose summarized chunks to users
117 lines
3.6 KiB
Ruby
117 lines
3.6 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require_relative "../../../../support/openai_completions_inference_stubs"
|
|
|
|
RSpec.describe DiscourseAi::Summarization::Models::OpenAi do
|
|
let(:model) { "gpt-3.5-turbo" }
|
|
let(:max_tokens) { 720 }
|
|
|
|
subject { described_class.new(model, max_tokens: max_tokens) }
|
|
|
|
let(:content) do
|
|
{
|
|
resource_path: "/t/1/POST_NUMBER",
|
|
content_title: "This is a title",
|
|
contents: [{ poster: "asd", id: 1, text: "This is a text" }],
|
|
}
|
|
end
|
|
|
|
def expected_messages(contents, opts)
|
|
base_prompt = <<~TEXT
|
|
You are a summarization bot.
|
|
You effectively summarise any text and reply ONLY with ONLY the summarized text.
|
|
You condense it into a shorter version.
|
|
You understand and generate Discourse forum Markdown.
|
|
Try generating links as well the format is #{opts[:resource_path]}. eg: [ref](#{opts[:resource_path]}/77)
|
|
The discussion title is: #{opts[:content_title]}.
|
|
TEXT
|
|
|
|
messages = [{ role: "system", content: base_prompt }]
|
|
|
|
text =
|
|
contents.reduce("") do |memo, item|
|
|
memo += "(#{item[:id]} #{item[:poster]} said: #{item[:text]} "
|
|
end
|
|
|
|
messages << { role: "user", content: "Summarize the following in 400 words:\n#{text}" }
|
|
end
|
|
|
|
describe "#summarize_in_chunks" do
|
|
context "when the content fits in a single chunk" do
|
|
it "performs a request to summarize" do
|
|
opts = content.except(:contents)
|
|
|
|
OpenAiCompletionsInferenceStubs.stub_response(
|
|
expected_messages(content[:contents], opts),
|
|
"This is summary 1",
|
|
)
|
|
|
|
summarized_chunks =
|
|
subject.summarize_in_chunks(content[:contents], opts).map { |c| c[:summary] }
|
|
|
|
expect(summarized_chunks).to contain_exactly("This is summary 1")
|
|
end
|
|
end
|
|
|
|
context "when the content fits in multiple chunks" do
|
|
it "performs a request for each one to summarize" do
|
|
content[:contents] << {
|
|
poster: "asd2",
|
|
id: 2,
|
|
text: "This is a different text to summarize",
|
|
}
|
|
opts = content.except(:contents)
|
|
|
|
content[:contents].each_with_index do |item, idx|
|
|
OpenAiCompletionsInferenceStubs.stub_response(
|
|
expected_messages([item], opts),
|
|
"This is summary #{idx + 1}",
|
|
)
|
|
end
|
|
|
|
summarized_chunks =
|
|
subject.summarize_in_chunks(content[:contents], opts).map { |c| c[:summary] }
|
|
|
|
expect(summarized_chunks).to contain_exactly("This is summary 1", "This is summary 2")
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "#concatenate_summaries" do
|
|
it "combines all the different summaries into a single one" do
|
|
messages = [
|
|
{ role: "system", content: "You are a helpful bot" },
|
|
{
|
|
role: "user",
|
|
content:
|
|
"Concatenate these disjoint summaries, creating a cohesive narrative:\nsummary 1\nsummary 2",
|
|
},
|
|
]
|
|
|
|
OpenAiCompletionsInferenceStubs.stub_response(messages, "concatenated summary")
|
|
|
|
expect(subject.concatenate_summaries(["summary 1", "summary 2"])).to eq(
|
|
"concatenated summary",
|
|
)
|
|
end
|
|
end
|
|
|
|
describe "#summarize_with_truncation" do
|
|
let(:max_tokens) { 709 }
|
|
|
|
it "truncates the context to meet the token limit" do
|
|
opts = content.except(:contents)
|
|
|
|
truncated_version = expected_messages(content[:contents], opts)
|
|
|
|
truncated_version.last[
|
|
:content
|
|
] = "Summarize the following in 400 words:\n(1 asd said: This is a"
|
|
|
|
OpenAiCompletionsInferenceStubs.stub_response(truncated_version, "truncated summary")
|
|
|
|
expect(subject.summarize_with_truncation(content[:contents], opts)).to eq("truncated summary")
|
|
end
|
|
end
|
|
end
|