From a48acc894a2bcf18f3eb4d6b4a29e9936166c35f Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 3 Sep 2024 15:52:20 +1000 Subject: [PATCH] FEATURE: more accurate and faster titles (#791) Previously we waited 1 minute before automatically titling PMs The new change introduces adding a title immediately after the the llm replies Prompt was also modified to include the LLM reply in title suggestion. This helps situation like: user: tell me a joke llm: a very funy joke about horses Then the title would be "A Funny Horse Joke" Specs already covered some auto title logic, amended to also catch the new message bus message we have been sending. --- app/jobs/regular/update_ai_bot_pm_title.rb | 24 ------- lib/ai_bot/bot.rb | 36 +++++++--- lib/ai_bot/entry_point.rb | 1 - lib/ai_bot/playground.rb | 34 ++++------ .../regular/update_ai_bot_pm_title_spec.rb | 65 ------------------- spec/lib/modules/ai_bot/playground_spec.rb | 29 +++++---- 6 files changed, 56 insertions(+), 133 deletions(-) delete mode 100644 app/jobs/regular/update_ai_bot_pm_title.rb delete mode 100644 spec/lib/modules/ai_bot/jobs/regular/update_ai_bot_pm_title_spec.rb diff --git a/app/jobs/regular/update_ai_bot_pm_title.rb b/app/jobs/regular/update_ai_bot_pm_title.rb deleted file mode 100644 index 0e4be014..00000000 --- a/app/jobs/regular/update_ai_bot_pm_title.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module ::Jobs - class UpdateAiBotPmTitle < ::Jobs::Base - sidekiq_options retry: false - - def execute(args) - return unless bot_user = User.find_by(id: args[:bot_user_id]) - return unless bot = DiscourseAi::AiBot::Bot.as(bot_user, model: args[:model]) - return unless post = Post.includes(:topic).find_by(id: args[:post_id]) - - return unless post.topic.custom_fields[DiscourseAi::AiBot::EntryPoint::REQUIRE_TITLE_UPDATE] - - DiscourseAi::AiBot::Playground.new(bot).title_playground(post) - - publish_update(post.topic, { title: post.topic.title }) - end - - def publish_update(topic, payload) - allowed_users = topic.topic_allowed_users.pluck(:user_id) - MessageBus.publish("/discourse-ai/ai-bot/topic/#{topic.id}", payload, user_ids: allowed_users) - end - end -end diff --git a/lib/ai_bot/bot.rb b/lib/ai_bot/bot.rb index 36fbe098..e3941e0e 100644 --- a/lib/ai_bot/bot.rb +++ b/lib/ai_bot/bot.rb @@ -24,23 +24,41 @@ module DiscourseAi def get_updated_title(conversation_context, post) system_insts = <<~TEXT.strip - You are titlebot. Given a topic, you will figure out a title. - You will never respond with anything but 7 word topic title. + You are titlebot. Given a conversation, you will suggest a title. + + - You will never respond with anything but the suggested title. + - You will always match the conversation language in your title suggestion. + - Title will capture the essence of the conversation. + TEXT + + # conversation context may contain tool calls, and confusing user names + # clean it up + conversation = +"" + conversation_context.each do |context| + if context[:type] == :user + conversation << "User said:\n#{context[:content]}\n\n" + elsif context[:type] == :model + conversation << "Model said:\n#{context[:content]}\n\n" + end + end + + instruction = <<~TEXT.strip + Given the following conversation: + + {{{ + #{conversation} + }}} + + Reply only with a title that is 7 words or less. TEXT title_prompt = DiscourseAi::Completions::Prompt.new( system_insts, - messages: conversation_context, + messages: [type: :user, content: instruction], topic_id: post.topic_id, ) - title_prompt.push( - type: :user, - content: - "Based on our previous conversation, suggest a 7 word title without quoting any of it.", - ) - DiscourseAi::Completions::Llm .proxy(model) .generate(title_prompt, user: post.user, feature_name: "bot_title") diff --git a/lib/ai_bot/entry_point.rb b/lib/ai_bot/entry_point.rb index 8c214631..5ff8ea36 100644 --- a/lib/ai_bot/entry_point.rb +++ b/lib/ai_bot/entry_point.rb @@ -5,7 +5,6 @@ module DiscourseAi USER_AGENT = "Discourse AI Bot 1.0 (https://www.discourse.org)" class EntryPoint - REQUIRE_TITLE_UPDATE = "discourse-ai-title-update" Bot = Struct.new(:id, :name, :llm) def self.all_bot_ids diff --git a/lib/ai_bot/playground.rb b/lib/ai_bot/playground.rb index a294ae76..fb41b4f5 100644 --- a/lib/ai_bot/playground.rb +++ b/lib/ai_bot/playground.rb @@ -9,8 +9,6 @@ module DiscourseAi # The bot will take care of completions while this class updates the topic title # and stream replies. - REQUIRE_TITLE_UPDATE = "discourse-ai-title-update" - def self.find_chat_persona(message, channel, user) if channel.direct_message_channel? AiPersona.allowed_chat.find do |p| @@ -126,10 +124,7 @@ module DiscourseAi end def update_playground_with(post) - if can_attach?(post) - schedule_playground_titling(post) - schedule_bot_reply(post) - end + schedule_bot_reply(post) if can_attach?(post) end def conversation_context(post) @@ -217,9 +212,14 @@ module DiscourseAi bot.bot_user, title: new_title.sub(/\A"/, "").sub(/"\Z/, ""), ) - post.topic.custom_fields.delete(DiscourseAi::AiBot::EntryPoint::REQUIRE_TITLE_UPDATE) - post.topic.save_custom_fields end + + allowed_users = post.topic.topic_allowed_users.pluck(:user_id) + MessageBus.publish( + "/discourse-ai/ai-bot/topic/#{post.topic.id}", + { title: post.topic.title }, + user_ids: allowed_users, + ) end def chat_context(message, channel, persona_user, context_post_ids) @@ -487,6 +487,9 @@ module DiscourseAi reply_post ensure publish_final_update(reply_post) if stream_reply + if reply_post && post.post_number == 1 && post.topic.private_message? + title_playground(reply_post) + end end def available_bot_usernames @@ -526,21 +529,6 @@ module DiscourseAi true end - def schedule_playground_titling(post) - if post.post_number == 1 && post.topic.private_message? - post.topic.custom_fields[REQUIRE_TITLE_UPDATE] = true - post.topic.save_custom_fields - - ::Jobs.enqueue_in( - 1.minute, - :update_ai_bot_pm_title, - post_id: post.id, - bot_user_id: bot.bot_user.id, - model: bot.model, - ) - end - end - def schedule_bot_reply(post) persona_id = DiscourseAi::AiBot::Personas::Persona.system_personas[bot.persona.class] || diff --git a/spec/lib/modules/ai_bot/jobs/regular/update_ai_bot_pm_title_spec.rb b/spec/lib/modules/ai_bot/jobs/regular/update_ai_bot_pm_title_spec.rb deleted file mode 100644 index 788d75a8..00000000 --- a/spec/lib/modules/ai_bot/jobs/regular/update_ai_bot_pm_title_spec.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Jobs::UpdateAiBotPmTitle do - let(:user) { Fabricate(:admin) } - - fab!(:claude_2) { Fabricate(:llm_model, name: "claude-2") } - let(:bot_user) { DiscourseAi::AiBot::EntryPoint.find_user_from_model("claude-2") } - - before do - toggle_enabled_bots(bots: [claude_2]) - SiteSetting.ai_bot_enabled = true - end - - it "will properly update title on bot PMs" do - SiteSetting.ai_bot_allowed_groups = Group::AUTO_GROUPS[:staff] - - post = - create_post( - user: user, - raw: "Hello there", - title: "does not matter should be updated", - archetype: Archetype.private_message, - target_usernames: bot_user.username, - ) - - title_result = "A great title would be:\n\nMy amazing title\n\n" - - DiscourseAi::Completions::Llm.with_prepared_responses([title_result]) do - subject.execute(bot_user_id: bot_user.id, post_id: post.id) - - expect(post.reload.topic.title).to eq("My amazing title") - end - - another_title = "I'm a different title" - - DiscourseAi::Completions::Llm.with_prepared_responses([another_title]) do - subject.execute(bot_user_id: bot_user.id, post_id: post.id) - - expect(post.reload.topic.title).to eq("My amazing title") - end - end - - it "will post an update with the new title to the message bus channel" do - SiteSetting.ai_bot_allowed_groups = Group::AUTO_GROUPS[:staff] - post = - create_post( - user: user, - raw: "Hello there", - title: "does not matter should be updated", - archetype: Archetype.private_message, - target_usernames: bot_user.username, - ) - title_result = "A great title would be:\n\nMy amazing title\n\n" - - DiscourseAi::Completions::Llm.with_prepared_responses([title_result]) do - messages = - MessageBus.track_publish("/discourse-ai/ai-bot/topic/#{post.topic.id}") do - subject.execute(bot_user_id: bot_user.id, post_id: post.id) - end - - final_update = messages.last.data - expect(final_update[:title]).to eq("My amazing title") - end - end -end diff --git a/spec/lib/modules/ai_bot/playground_spec.rb b/spec/lib/modules/ai_bot/playground_spec.rb index e25d4721..f087414b 100644 --- a/spec/lib/modules/ai_bot/playground_spec.rb +++ b/spec/lib/modules/ai_bot/playground_spec.rb @@ -517,7 +517,7 @@ RSpec.describe DiscourseAi::AiBot::Playground do post = nil DiscourseAi::Completions::Llm.with_prepared_responses( - ["Magic title", "Yes I can"], + ["Yes I can", "Magic Title"], llm: "custom:#{claude_2.id}", ) do post = @@ -549,22 +549,29 @@ RSpec.describe DiscourseAi::AiBot::Playground do post = nil gpt3_5_bot_user = gpt_35_turbo.reload.user + messages = nil - # title is queued first, ensures it uses the llm targeted via target_usernames not claude DiscourseAi::Completions::Llm.with_prepared_responses( - ["Magic title", "Yes I can"], + ["Yes I can", "Magic Title"], llm: "custom:#{gpt_35_turbo.id}", ) do - post = - create_post( - title: "I just made a PM", - raw: "Hey @#{persona.user.username}, can you help me?", - target_usernames: "#{user.username},#{gpt3_5_bot_user.username}", - archetype: Archetype.private_message, - user: admin, - ) + messages = + MessageBus.track_publish do + post = + create_post( + title: "I just made a PM", + raw: "Hey @#{persona.user.username}, can you help me?", + target_usernames: "#{user.username},#{gpt3_5_bot_user.username}", + archetype: Archetype.private_message, + user: admin, + ) + end end + title_update_message = + messages.find { |m| m.channel == "/discourse-ai/ai-bot/topic/#{post.topic.id}" } + + expect(title_update_message.data).to eq({ title: "Magic Title" }) last_post = post.topic.posts.order(:post_number).last expect(last_post.raw).to eq("Yes I can") expect(last_post.user_id).to eq(persona.user_id)