From 1f9f330ce21b2a4d189a6da608c54a4d1bb6641d Mon Sep 17 00:00:00 2001
From: Keegan George <kgeorge13@gmail.com>
Date: Thu, 20 Feb 2025 09:07:23 -0800
Subject: [PATCH] DEV: Add summarization type to eval (#1138)

Adds `type: summarization` for topic summarization eval: https://github.com/discourse/discourse-ai-evals/pull/4
---
 evals/lib/eval.rb | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/evals/lib/eval.rb b/evals/lib/eval.rb
index 280ab871..4a62e332 100644
--- a/evals/lib/eval.rb
+++ b/evals/lib/eval.rb
@@ -60,6 +60,8 @@ class DiscourseAi::Evals::Eval
         prompt_call(llm, **args)
       when "edit_artifact"
         edit_artifact(llm, **args)
+      when "summarization"
+        summarization(llm, **args)
       end
 
     if expected_output
@@ -312,4 +314,24 @@ class DiscourseAi::Evals::Eval
   rescue StandardError
     false
   end
+
+  def summarization(llm, input:)
+    topic =
+      Topic.new(
+        category: Category.last,
+        title: "Eval topic for topic summarization",
+        id: -99,
+        user_id: Discourse.system_user.id,
+      )
+    Post.new(topic: topic, id: -99, user_id: Discourse.system_user.id, raw: input)
+
+    strategy =
+      DiscourseAi::Summarization::FoldContent.new(
+        llm.llm_proxy,
+        DiscourseAi::Summarization::Strategies::TopicSummary.new(topic),
+      )
+
+    summary = DiscourseAi::TopicSummarization.new(strategy, Discourse.system_user).summarize
+    summary.summarized_text
+  end
 end