From f26adf2cf67cde7a302de3252fad890576894b95 Mon Sep 17 00:00:00 2001 From: Roman Rizzi Date: Tue, 28 Nov 2023 12:52:22 -0300 Subject: [PATCH] FIX: Use XML tags in generate_titles prompt. (#322) We must ensure we can isolate titles, and the models sometimes ignore the example we give them. Additionally, anons can generate HyDE posts, so we need to check if user is nil when attempting to log requests. --- app/models/completion_prompt.rb | 2 +- .../ai_helper/603_completion_prompts.rb | 45 ++++++++++--------- ...8151234_recreate_generate_titles_prompt.rb | 11 +++++ lib/completions/endpoints/base.rb | 2 +- lib/modules/ai_helper/assistant.rb | 2 +- spec/lib/modules/ai_helper/assistant_spec.rb | 10 ++++- .../ai_helper/chat_thread_titler_spec.rb | 5 +-- .../ai_helper/ai_composer_helper_spec.rb | 2 +- 8 files changed, 48 insertions(+), 31 deletions(-) create mode 100644 db/post_migrate/20231128151234_recreate_generate_titles_prompt.rb diff --git a/app/models/completion_prompt.rb b/app/models/completion_prompt.rb index 36288782..0bcde5ff 100644 --- a/app/models/completion_prompt.rb +++ b/app/models/completion_prompt.rb @@ -5,7 +5,7 @@ class CompletionPrompt < ActiveRecord::Base self.ignored_columns = ["provider"] TRANSLATE = -301 - GENERATE_TITLES = -302 + GENERATE_TITLES = -307 PROOFREAD = -303 MARKDOWN_TABLE = -304 CUSTOM_PROMPT = -305 diff --git a/db/fixtures/ai_helper/603_completion_prompts.rb b/db/fixtures/ai_helper/603_completion_prompts.rb index 27b5f61b..4b8df0f7 100644 --- a/db/fixtures/ai_helper/603_completion_prompts.rb +++ b/db/fixtures/ai_helper/603_completion_prompts.rb @@ -16,28 +16,6 @@ CompletionPrompt.seed do |cp| TEXT end -CompletionPrompt.seed do |cp| - cp.id = -302 - cp.name = "generate_titles" - cp.prompt_type = CompletionPrompt.prompt_types[:list] - cp.messages = { - insts: <<~TEXT, - I want you to act as a title generator for written pieces. I will provide you with a text, - and you will generate five attention-grabbing titles. Please keep the title concise and under 20 words, - and ensure that the meaning is maintained. Replies will utilize the language type of the topic. - I want you to only reply the list of options and nothing else, do not write explanations. - Each title you generate must be separated by *. - You will find the text between XML tags. - TEXT - examples: [ - [ - "In the labyrinth of time, a solitary horse, etched in gold by the setting sun, embarked on an infinite journey.", - "The solitary horse*The horse etched in gold*A horse's infinite journey*A horse lost in time*A horse's last ride", - ], - ], - } -end - CompletionPrompt.seed do |cp| cp.id = -303 cp.name = "proofread" @@ -172,3 +150,26 @@ CompletionPrompt.seed do |cp| nothing more. TEXT end + +CompletionPrompt.seed do |cp| + cp.id = -307 + cp.name = "generate_titles" + cp.prompt_type = CompletionPrompt.prompt_types[:list] + cp.messages = { + insts: <<~TEXT, + I want you to act as a title generator for written pieces. I will provide you with a text, + and you will generate five attention-grabbing titles. Please keep the title concise and under 20 words, + and ensure that the meaning is maintained. Replies will utilize the language type of the topic. + I want you to only reply the list of options and nothing else, do not write explanations. + Each title you generate must be separated by *. + You will find the text between XML tags. + TEXT + examples: [ + [ + "In the labyrinth of time, a solitary horse, etched in gold by the setting sun, embarked on an infinite journey.", + "The solitary horseThe horse etched in goldA horse's infinite journeyA horse lost in timeA horse's last ride", + ], + ], + post_insts: "Wrap each title between XML tags.", + } +end diff --git a/db/post_migrate/20231128151234_recreate_generate_titles_prompt.rb b/db/post_migrate/20231128151234_recreate_generate_titles_prompt.rb new file mode 100644 index 00000000..7153295a --- /dev/null +++ b/db/post_migrate/20231128151234_recreate_generate_titles_prompt.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class RecreateGenerateTitlesPrompt < ActiveRecord::Migration[7.0] + def up + DB.exec("DELETE FROM completion_prompts WHERE id = -302") + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/lib/completions/endpoints/base.rb b/lib/completions/endpoints/base.rb index 5b86f1b1..624be145 100644 --- a/lib/completions/endpoints/base.rb +++ b/lib/completions/endpoints/base.rb @@ -57,7 +57,7 @@ module DiscourseAi log = AiApiAuditLog.new( provider_id: provider_id, - user_id: user.id, + user_id: user&.id, raw_request_payload: request_body, request_tokens: prompt_size(prompt), ) diff --git a/lib/modules/ai_helper/assistant.rb b/lib/modules/ai_helper/assistant.rb index 660350bc..04f8fb76 100644 --- a/lib/modules/ai_helper/assistant.rb +++ b/lib/modules/ai_helper/assistant.rb @@ -103,7 +103,7 @@ module DiscourseAi end def parse_list(list) - list.split("*") + Nokogiri::HTML5.fragment(list).css("item").map(&:text) end end end diff --git a/spec/lib/modules/ai_helper/assistant_spec.rb b/spec/lib/modules/ai_helper/assistant_spec.rb index bf5bd886..913bbfec 100644 --- a/spec/lib/modules/ai_helper/assistant_spec.rb +++ b/spec/lib/modules/ai_helper/assistant_spec.rb @@ -34,11 +34,17 @@ RSpec.describe DiscourseAi::AiHelper::Assistant do let(:mode) { CompletionPrompt::GENERATE_TITLES } let(:titles) do - "The solitary horse*The horse etched in gold*A horse's infinite journey*A horse lost in time*A horse's last ride" + "The solitary horseThe horse etched in goldA horse's infinite journeyA horse lost in timeA horse's last ride" end it "returns an array with each title" do - expected = titles.split("*") + expected = [ + "The solitary horse", + "The horse etched in gold", + "A horse's infinite journey", + "A horse lost in time", + "A horse's last ride", + ] response = DiscourseAi::Completions::LLM.with_prepared_responses([titles]) do diff --git a/spec/lib/modules/ai_helper/chat_thread_titler_spec.rb b/spec/lib/modules/ai_helper/chat_thread_titler_spec.rb index e2714fb6..b07f7940 100644 --- a/spec/lib/modules/ai_helper/chat_thread_titler_spec.rb +++ b/spec/lib/modules/ai_helper/chat_thread_titler_spec.rb @@ -9,9 +9,8 @@ RSpec.describe DiscourseAi::AiHelper::ChatThreadTitler do describe "#suggested_title" do it "suggest the first option from the generate_titles prompt" do titles = - "The solitary horse*The horse etched in gold*A horse's infinite journey*A horse lost in time*A horse's last ride" - expected_title = titles.split("*").first - + "The solitary horseThe horse etched in goldA horse's infinite journeyA horse lost in timeA horse's last ride" + expected_title = "The solitary horse" result = DiscourseAi::Completions::LLM.with_prepared_responses([titles]) { titler.suggested_title } diff --git a/spec/system/ai_helper/ai_composer_helper_spec.rb b/spec/system/ai_helper/ai_composer_helper_spec.rb index db286741..f157f2c0 100644 --- a/spec/system/ai_helper/ai_composer_helper_spec.rb +++ b/spec/system/ai_helper/ai_composer_helper_spec.rb @@ -265,7 +265,7 @@ RSpec.describe "AI Composer helper", type: :system, js: true do let(:mode) { CompletionPrompt::GENERATE_TITLES } let(:titles) do - "Rainy Spain*Plane-Bound Delights*Mysterious Spain*Plane-Rain Chronicles*Unveiling Spain" + "Rainy SpainPlane-Bound DelightsMysterious SpainPlane-Rain ChroniclesUnveiling Spain" end it "opens a menu with title suggestions" do