FEATURE: more accurate and faster titles (#791)

Previously we waited 1 minute before automatically titling PMs

The new change introduces adding a title immediately after the the
llm replies

Prompt was also modified to include the LLM reply in title suggestion.

This helps situation like:

user: tell me a joke
llm: a very funy joke about horses

Then the title would be "A Funny Horse Joke"

Specs already covered some auto title logic, amended to also
catch the new message bus message we have been sending.
This commit is contained in:
Sam 2024-09-03 15:52:20 +10:00 committed by GitHub
parent b0ae2138af
commit a48acc894a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 56 additions and 133 deletions

View File

@ -1,24 +0,0 @@
# frozen_string_literal: true
module ::Jobs
class UpdateAiBotPmTitle < ::Jobs::Base
sidekiq_options retry: false
def execute(args)
return unless bot_user = User.find_by(id: args[:bot_user_id])
return unless bot = DiscourseAi::AiBot::Bot.as(bot_user, model: args[:model])
return unless post = Post.includes(:topic).find_by(id: args[:post_id])
return unless post.topic.custom_fields[DiscourseAi::AiBot::EntryPoint::REQUIRE_TITLE_UPDATE]
DiscourseAi::AiBot::Playground.new(bot).title_playground(post)
publish_update(post.topic, { title: post.topic.title })
end
def publish_update(topic, payload)
allowed_users = topic.topic_allowed_users.pluck(:user_id)
MessageBus.publish("/discourse-ai/ai-bot/topic/#{topic.id}", payload, user_ids: allowed_users)
end
end
end

View File

@ -24,23 +24,41 @@ module DiscourseAi
def get_updated_title(conversation_context, post)
system_insts = <<~TEXT.strip
You are titlebot. Given a topic, you will figure out a title.
You will never respond with anything but 7 word topic title.
You are titlebot. Given a conversation, you will suggest a title.
- You will never respond with anything but the suggested title.
- You will always match the conversation language in your title suggestion.
- Title will capture the essence of the conversation.
TEXT
# conversation context may contain tool calls, and confusing user names
# clean it up
conversation = +""
conversation_context.each do |context|
if context[:type] == :user
conversation << "User said:\n#{context[:content]}\n\n"
elsif context[:type] == :model
conversation << "Model said:\n#{context[:content]}\n\n"
end
end
instruction = <<~TEXT.strip
Given the following conversation:
{{{
#{conversation}
}}}
Reply only with a title that is 7 words or less.
TEXT
title_prompt =
DiscourseAi::Completions::Prompt.new(
system_insts,
messages: conversation_context,
messages: [type: :user, content: instruction],
topic_id: post.topic_id,
)
title_prompt.push(
type: :user,
content:
"Based on our previous conversation, suggest a 7 word title without quoting any of it.",
)
DiscourseAi::Completions::Llm
.proxy(model)
.generate(title_prompt, user: post.user, feature_name: "bot_title")

View File

@ -5,7 +5,6 @@ module DiscourseAi
USER_AGENT = "Discourse AI Bot 1.0 (https://www.discourse.org)"
class EntryPoint
REQUIRE_TITLE_UPDATE = "discourse-ai-title-update"
Bot = Struct.new(:id, :name, :llm)
def self.all_bot_ids

View File

@ -9,8 +9,6 @@ module DiscourseAi
# The bot will take care of completions while this class updates the topic title
# and stream replies.
REQUIRE_TITLE_UPDATE = "discourse-ai-title-update"
def self.find_chat_persona(message, channel, user)
if channel.direct_message_channel?
AiPersona.allowed_chat.find do |p|
@ -126,10 +124,7 @@ module DiscourseAi
end
def update_playground_with(post)
if can_attach?(post)
schedule_playground_titling(post)
schedule_bot_reply(post)
end
schedule_bot_reply(post) if can_attach?(post)
end
def conversation_context(post)
@ -217,9 +212,14 @@ module DiscourseAi
bot.bot_user,
title: new_title.sub(/\A"/, "").sub(/"\Z/, ""),
)
post.topic.custom_fields.delete(DiscourseAi::AiBot::EntryPoint::REQUIRE_TITLE_UPDATE)
post.topic.save_custom_fields
end
allowed_users = post.topic.topic_allowed_users.pluck(:user_id)
MessageBus.publish(
"/discourse-ai/ai-bot/topic/#{post.topic.id}",
{ title: post.topic.title },
user_ids: allowed_users,
)
end
def chat_context(message, channel, persona_user, context_post_ids)
@ -487,6 +487,9 @@ module DiscourseAi
reply_post
ensure
publish_final_update(reply_post) if stream_reply
if reply_post && post.post_number == 1 && post.topic.private_message?
title_playground(reply_post)
end
end
def available_bot_usernames
@ -526,21 +529,6 @@ module DiscourseAi
true
end
def schedule_playground_titling(post)
if post.post_number == 1 && post.topic.private_message?
post.topic.custom_fields[REQUIRE_TITLE_UPDATE] = true
post.topic.save_custom_fields
::Jobs.enqueue_in(
1.minute,
:update_ai_bot_pm_title,
post_id: post.id,
bot_user_id: bot.bot_user.id,
model: bot.model,
)
end
end
def schedule_bot_reply(post)
persona_id =
DiscourseAi::AiBot::Personas::Persona.system_personas[bot.persona.class] ||

View File

@ -1,65 +0,0 @@
# frozen_string_literal: true
RSpec.describe Jobs::UpdateAiBotPmTitle do
let(:user) { Fabricate(:admin) }
fab!(:claude_2) { Fabricate(:llm_model, name: "claude-2") }
let(:bot_user) { DiscourseAi::AiBot::EntryPoint.find_user_from_model("claude-2") }
before do
toggle_enabled_bots(bots: [claude_2])
SiteSetting.ai_bot_enabled = true
end
it "will properly update title on bot PMs" do
SiteSetting.ai_bot_allowed_groups = Group::AUTO_GROUPS[:staff]
post =
create_post(
user: user,
raw: "Hello there",
title: "does not matter should be updated",
archetype: Archetype.private_message,
target_usernames: bot_user.username,
)
title_result = "A great title would be:\n\nMy amazing title\n\n"
DiscourseAi::Completions::Llm.with_prepared_responses([title_result]) do
subject.execute(bot_user_id: bot_user.id, post_id: post.id)
expect(post.reload.topic.title).to eq("My amazing title")
end
another_title = "I'm a different title"
DiscourseAi::Completions::Llm.with_prepared_responses([another_title]) do
subject.execute(bot_user_id: bot_user.id, post_id: post.id)
expect(post.reload.topic.title).to eq("My amazing title")
end
end
it "will post an update with the new title to the message bus channel" do
SiteSetting.ai_bot_allowed_groups = Group::AUTO_GROUPS[:staff]
post =
create_post(
user: user,
raw: "Hello there",
title: "does not matter should be updated",
archetype: Archetype.private_message,
target_usernames: bot_user.username,
)
title_result = "A great title would be:\n\nMy amazing title\n\n"
DiscourseAi::Completions::Llm.with_prepared_responses([title_result]) do
messages =
MessageBus.track_publish("/discourse-ai/ai-bot/topic/#{post.topic.id}") do
subject.execute(bot_user_id: bot_user.id, post_id: post.id)
end
final_update = messages.last.data
expect(final_update[:title]).to eq("My amazing title")
end
end
end

View File

@ -517,7 +517,7 @@ RSpec.describe DiscourseAi::AiBot::Playground do
post = nil
DiscourseAi::Completions::Llm.with_prepared_responses(
["Magic title", "Yes I can"],
["Yes I can", "Magic Title"],
llm: "custom:#{claude_2.id}",
) do
post =
@ -549,12 +549,14 @@ RSpec.describe DiscourseAi::AiBot::Playground do
post = nil
gpt3_5_bot_user = gpt_35_turbo.reload.user
messages = nil
# title is queued first, ensures it uses the llm targeted via target_usernames not claude
DiscourseAi::Completions::Llm.with_prepared_responses(
["Magic title", "Yes I can"],
["Yes I can", "Magic Title"],
llm: "custom:#{gpt_35_turbo.id}",
) do
messages =
MessageBus.track_publish do
post =
create_post(
title: "I just made a PM",
@ -564,7 +566,12 @@ RSpec.describe DiscourseAi::AiBot::Playground do
user: admin,
)
end
end
title_update_message =
messages.find { |m| m.channel == "/discourse-ai/ai-bot/topic/#{post.topic.id}" }
expect(title_update_message.data).to eq({ title: "Magic Title" })
last_post = post.topic.posts.order(:post_number).last
expect(last_post.raw).to eq("Yes I can")
expect(last_post.user_id).to eq(persona.user_id)