diff --git a/app/jobs/regular/localize_categories.rb b/app/jobs/regular/localize_categories.rb index 1b9c3fd3..9ede9f21 100644 --- a/app/jobs/regular/localize_categories.rb +++ b/app/jobs/regular/localize_categories.rb @@ -5,51 +5,51 @@ module Jobs cluster_concurrency 1 sidekiq_options retry: false - BATCH_SIZE = 50 - def execute(args) + limit = args[:limit] + raise Discourse::InvalidParameters.new(:limit) if limit.nil? + return if limit <= 0 + return if !SiteSetting.discourse_ai_enabled return if !SiteSetting.ai_translation_enabled - locales = SiteSetting.content_localization_supported_locales.split("|") return if locales.blank? - cat_id = args[:from_category_id] || Category.order(:id).first&.id - last_id = nil + categories = Category.where("locale IS NOT NULL") - categories = - Category.where("id >= ? AND locale IS NOT NULL", cat_id).order(:id).limit(BATCH_SIZE) - return if categories.empty? - - categories.each do |category| - if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted? - last_id = category.id - next - end - - locales.each do |locale| - localization = category.category_localizations.find_by(locale:) - - if locale == category.locale && localization - localization.destroy - else - next if locale == category.locale - begin - DiscourseAi::Translation::CategoryLocalizer.localize(category, locale) - rescue FinalDestination::SSRFDetector::LookupFailedError - # do nothing, there are too many sporadic lookup failures - rescue => e - DiscourseAi::Translation::VerboseLogger.log( - "Failed to translate category #{category.id} to #{locale}: #{e.message}", - ) - end - end - end - last_id = category.id + if SiteSetting.ai_translation_backfill_limit_to_public_content + categories = categories.where(read_restricted: false) end - if categories.size == BATCH_SIZE - Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1) + categories = categories.order(:id).limit(limit) + return if categories.empty? + + remaining_limit = limit + + categories.each do |category| + break if remaining_limit <= 0 + + existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale) + missing_locales = locales - existing_locales - [category.locale] + missing_locales.each do |locale| + break if remaining_limit <= 0 + + begin + DiscourseAi::Translation::CategoryLocalizer.localize(category, locale) + rescue FinalDestination::SSRFDetector::LookupFailedError + # do nothing, there are too many sporadic lookup failures + rescue => e + DiscourseAi::Translation::VerboseLogger.log( + "Failed to translate category #{category.id} to #{locale}: #{e.message}", + ) + ensure + remaining_limit -= 1 + end + end + + if existing_locales.include?(category.locale) + CategoryLocalization.find_by(category_id: category.id, locale: category.locale).destroy + end end end end diff --git a/app/jobs/regular/localize_posts.rb b/app/jobs/regular/localize_posts.rb index 34a80185..3dad28e0 100644 --- a/app/jobs/regular/localize_posts.rb +++ b/app/jobs/regular/localize_posts.rb @@ -5,17 +5,16 @@ module Jobs cluster_concurrency 1 sidekiq_options retry: false - BATCH_SIZE = 50 - def execute(args) + limit = args[:limit] + raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0 + return if !SiteSetting.discourse_ai_enabled return if !SiteSetting.ai_translation_enabled locales = SiteSetting.content_localization_supported_locales.split("|") return if locales.blank? - limit = args[:limit] || BATCH_SIZE - locales.each do |locale| posts = Post diff --git a/app/jobs/regular/localize_topics.rb b/app/jobs/regular/localize_topics.rb index 06598058..dceedf98 100644 --- a/app/jobs/regular/localize_topics.rb +++ b/app/jobs/regular/localize_topics.rb @@ -5,17 +5,16 @@ module Jobs cluster_concurrency 1 sidekiq_options retry: false - BATCH_SIZE = 50 - def execute(args) + limit = args[:limit] + raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0 + return if !SiteSetting.discourse_ai_enabled return if !SiteSetting.ai_translation_enabled locales = SiteSetting.content_localization_supported_locales.split("|") return if locales.blank? - limit = args[:limit] || BATCH_SIZE - locales.each do |locale| topics = Topic diff --git a/app/jobs/scheduled/categories_locale_detection_backfill.rb b/app/jobs/scheduled/categories_locale_detection_backfill.rb index 4ea30c00..3d9d6e25 100644 --- a/app/jobs/scheduled/categories_locale_detection_backfill.rb +++ b/app/jobs/scheduled/categories_locale_detection_backfill.rb @@ -9,7 +9,8 @@ module Jobs def execute(args) return if !SiteSetting.discourse_ai_enabled return if !SiteSetting.ai_translation_enabled - return if SiteSetting.ai_translation_backfill_rate == 0 + limit = SiteSetting.ai_translation_backfill_hourly_rate + return if limit == 0 categories = Category.where(locale: nil) @@ -17,7 +18,7 @@ module Jobs categories = categories.where(read_restricted: false) end - categories = categories.limit(SiteSetting.ai_translation_backfill_rate) + categories = categories.limit(limit) return if categories.empty? categories.each do |category| diff --git a/app/jobs/scheduled/category_localization_backfill.rb b/app/jobs/scheduled/category_localization_backfill.rb index 0ebb41fe..cffe79a6 100644 --- a/app/jobs/scheduled/category_localization_backfill.rb +++ b/app/jobs/scheduled/category_localization_backfill.rb @@ -2,15 +2,17 @@ module Jobs class CategoryLocalizationBackfill < ::Jobs::Scheduled - every 12.hours + every 1.hour cluster_concurrency 1 def execute(args) return if !SiteSetting.discourse_ai_enabled return if !SiteSetting.ai_translation_enabled return if SiteSetting.content_localization_supported_locales.blank? + limit = SiteSetting.ai_translation_backfill_hourly_rate + return if limit == 0 - Jobs.enqueue(:localize_categories) + Jobs.enqueue(:localize_categories, limit:) end end end diff --git a/app/jobs/scheduled/post_localization_backfill.rb b/app/jobs/scheduled/post_localization_backfill.rb index e81305b4..d79c6f0d 100644 --- a/app/jobs/scheduled/post_localization_backfill.rb +++ b/app/jobs/scheduled/post_localization_backfill.rb @@ -10,9 +10,10 @@ module Jobs return if !SiteSetting.ai_translation_enabled return if SiteSetting.content_localization_supported_locales.blank? - return if SiteSetting.ai_translation_backfill_rate == 0 + limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals + return if limit == 0 - Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate) + Jobs.enqueue(:localize_posts, limit:) end end end diff --git a/app/jobs/scheduled/posts_locale_detection_backfill.rb b/app/jobs/scheduled/posts_locale_detection_backfill.rb index 2372b77c..71091efd 100644 --- a/app/jobs/scheduled/posts_locale_detection_backfill.rb +++ b/app/jobs/scheduled/posts_locale_detection_backfill.rb @@ -9,7 +9,8 @@ module Jobs def execute(args) return if !SiteSetting.discourse_ai_enabled return if !SiteSetting.ai_translation_enabled - return if SiteSetting.ai_translation_backfill_rate == 0 + limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals + return if limit == 0 posts = Post @@ -35,7 +36,7 @@ module Jobs ) end - posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate) + posts = posts.order(updated_at: :desc).limit(limit) return if posts.empty? posts.each do |post| diff --git a/app/jobs/scheduled/topic_localization_backfill.rb b/app/jobs/scheduled/topic_localization_backfill.rb index 069646ca..c1bd64c0 100644 --- a/app/jobs/scheduled/topic_localization_backfill.rb +++ b/app/jobs/scheduled/topic_localization_backfill.rb @@ -10,9 +10,10 @@ module Jobs return if !SiteSetting.ai_translation_enabled return if SiteSetting.content_localization_supported_locales.blank? - return if SiteSetting.ai_translation_backfill_rate == 0 + limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals + return if limit == 0 - Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate) + Jobs.enqueue(:localize_topics, limit:) end end end diff --git a/app/jobs/scheduled/topics_locale_detection_backfill.rb b/app/jobs/scheduled/topics_locale_detection_backfill.rb index 075c36c1..bed5f4e4 100644 --- a/app/jobs/scheduled/topics_locale_detection_backfill.rb +++ b/app/jobs/scheduled/topics_locale_detection_backfill.rb @@ -9,8 +9,7 @@ module Jobs def execute(args) return if !SiteSetting.discourse_ai_enabled return if !SiteSetting.ai_translation_enabled - limit = SiteSetting.ai_translation_backfill_rate - + limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals return if limit == 0 topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0") diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 5af0e2fa..3cfe6068 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -118,6 +118,12 @@ en: ai_discord_allowed_guilds: "Discord guilds (servers) where the bot is allowed to search" ai_bot_enable_dedicated_ux: "Allow for full screen bot interface, instead of a PM" + ai_translation_enabled: "Enables the AI translation feature" + ai_translation_model: "The model to use for translation. This model must support translation. Personas can override this setting." + ai_translation_backfill_limit_to_public_content: "When enabled, only content in public categories will be translated. When disabled, content in group PMs and private categories will also be sent for translation." + ai_translation_max_post_length: "The maximum length of a post to be translated. Posts longer than this will not be translated." + ai_translation_backfill_max_age_days: "The maximum age of a post and topic to be translated. Posts and topics older than this will not be translated." + reviewables: reasons: flagged_by_toxicity: The AI plugin flagged this after classifying it as toxic. diff --git a/config/settings.yml b/config/settings.yml index a54eea2a..6230d201 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -480,7 +480,7 @@ discourse_ai: type: enum enum: "DiscourseAi::Configuration::PersonaEnumerator" area: "ai-features/translation" - ai_translation_backfill_rate: + ai_translation_backfill_hourly_rate: default: 0 min: 0 max: 1000 @@ -491,6 +491,10 @@ discourse_ai: default: true client: false area: "ai-features/translation" + ai_translation_max_post_length: + default: 10000 + client: false + area: "ai-features/translation" ai_translation_backfill_max_age_days: default: 5 client: false diff --git a/db/migrate/20250620073222_specify_rate_frequency_in_backfill_setting.rb b/db/migrate/20250620073222_specify_rate_frequency_in_backfill_setting.rb new file mode 100644 index 00000000..6f98377c --- /dev/null +++ b/db/migrate/20250620073222_specify_rate_frequency_in_backfill_setting.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class SpecifyRateFrequencyInBackfillSetting < ActiveRecord::Migration[7.2] + def up + execute "UPDATE site_settings SET name = 'ai_translation_backfill_hourly_rate' WHERE name = 'ai_translation_backfill_rate'" + end + + def down + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/lib/translation/post_localizer.rb b/lib/translation/post_localizer.rb index 7c2f6c69..6f75ea1c 100644 --- a/lib/translation/post_localizer.rb +++ b/lib/translation/post_localizer.rb @@ -4,7 +4,11 @@ module DiscourseAi module Translation class PostLocalizer def self.localize(post, target_locale = I18n.locale) - return if post.blank? || target_locale.blank? || post.locale == target_locale.to_s + if post.blank? || target_locale.blank? || post.locale == target_locale.to_s || + post.raw.blank? + return + end + return if post.raw.length > SiteSetting.ai_translation_max_post_length target_locale = target_locale.to_s.sub("-", "_") translated_raw = diff --git a/spec/jobs/regular/localize_categories_spec.rb b/spec/jobs/regular/localize_categories_spec.rb index 7e35ac01..96de1faf 100644 --- a/spec/jobs/regular/localize_categories_spec.rb +++ b/spec/jobs/regular/localize_categories_spec.rb @@ -25,7 +25,7 @@ describe Jobs::LocalizeCategories do DiscourseAi::Translation::CategoryLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when ai_translation_enabled is disabled" do @@ -33,7 +33,7 @@ describe Jobs::LocalizeCategories do DiscourseAi::Translation::CategoryLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when no target languages are configured" do @@ -41,7 +41,7 @@ describe Jobs::LocalizeCategories do DiscourseAi::Translation::CategoryLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when no categories exist" do @@ -49,7 +49,14 @@ describe Jobs::LocalizeCategories do DiscourseAi::Translation::CategoryLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) + end + + it "does nothing when the limit is zero" do + DiscourseAi::Translation::CategoryLocalizer.expects(:localize).never + + expect { job.execute({}) }.to raise_error(Discourse::InvalidParameters, /limit/) + job.execute({ limit: 0 }) end it "translates categories to the configured locales" do @@ -65,7 +72,21 @@ describe Jobs::LocalizeCategories do .with(is_a(Category), "zh_CN") .times(number_of_categories) - job.execute({}) + job.execute({ limit: 10 }) + end + + it "limits the number of localizations" do + SiteSetting.content_localization_supported_locales = "pt" + + 6.times { Fabricate(:category) } + Category.update_all(locale: "en") + + DiscourseAi::Translation::CategoryLocalizer + .expects(:localize) + .with(is_a(Category), "pt") + .times(5) + + job.execute({ limit: 5 }) end it "skips categories that already have localizations" do @@ -77,24 +98,7 @@ describe Jobs::LocalizeCategories do .with(is_a(Category), "zh_CN") .never - job.execute({}) - end - - it "continues from a specified category ID" do - category1 = Fabricate(:category, name: "First", description: "First description", locale: "en") - category2 = - Fabricate(:category, name: "Second", description: "Second description", locale: "en") - - DiscourseAi::Translation::CategoryLocalizer - .expects(:localize) - .with(category1, any_parameters) - .never - DiscourseAi::Translation::CategoryLocalizer - .expects(:localize) - .with(category2, any_parameters) - .twice - - job.execute(from_category_id: category2.id) + job.execute({ limit: 10 }) end it "handles translation errors gracefully" do @@ -107,31 +111,7 @@ describe Jobs::LocalizeCategories do .raises(StandardError.new("API error")) DiscourseAi::Translation::CategoryLocalizer.expects(:localize).with(category1, "zh_CN").once - expect { job.execute({}) }.not_to raise_error - end - - it "enqueues the next batch when there are more categories" do - Category.update_all(locale: "en") - - Jobs.run_later! - freeze_time - Jobs::LocalizeCategories.const_set(:BATCH_SIZE, 1) - - job.execute({}) - - Category.all.each do |category| - puts category.id - expect_job_enqueued( - job: :localize_categories, - args: { - from_category_id: category.id + 1, - }, - at: 10.seconds.from_now, - ) - end - - Jobs::LocalizeCategories.send(:remove_const, :BATCH_SIZE) - Jobs::LocalizeCategories.const_set(:BATCH_SIZE, 50) + expect { job.execute({ limit: 10 }) }.not_to raise_error end it "skips read-restricted categories when configured" do @@ -149,7 +129,7 @@ describe Jobs::LocalizeCategories do .with(category2, any_parameters) .never - job.execute({}) + job.execute({ limit: 10 }) end it "skips creating localizations in the same language as the category's locale" do @@ -161,7 +141,7 @@ describe Jobs::LocalizeCategories do .with(is_a(Category), "zh_CN") .times(Category.count) - job.execute({}) + job.execute({ limit: 10 }) end it "deletes existing localizations that match the category's locale" do @@ -170,9 +150,9 @@ describe Jobs::LocalizeCategories do localize_all_categories("pt", "zh_CN") - expect { job.execute({}) }.to change { CategoryLocalization.exists?(locale: "pt") }.from( - true, - ).to(false) + expect { job.execute({ limit: 10 }) }.to change { + CategoryLocalization.exists?(locale: "pt") + }.from(true).to(false) end it "doesn't process categories with nil locale" do @@ -185,6 +165,6 @@ describe Jobs::LocalizeCategories do .with(nil_locale_category, any_parameters) .never - job.execute({}) + job.execute({ limit: 10 }) end end diff --git a/spec/jobs/regular/localize_posts_spec.rb b/spec/jobs/regular/localize_posts_spec.rb index 25421ddc..6381f982 100644 --- a/spec/jobs/regular/localize_posts_spec.rb +++ b/spec/jobs/regular/localize_posts_spec.rb @@ -19,28 +19,28 @@ describe Jobs::LocalizePosts do SiteSetting.discourse_ai_enabled = false DiscourseAi::Translation::PostLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when ai_translation_enabled is disabled" do SiteSetting.ai_translation_enabled = false DiscourseAi::Translation::PostLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when no target languages are configured" do SiteSetting.content_localization_supported_locales = "" DiscourseAi::Translation::PostLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when there are no posts to translate" do Post.destroy_all DiscourseAi::Translation::PostLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "skips posts that already have localizations" do @@ -50,7 +50,7 @@ describe Jobs::LocalizePosts do end DiscourseAi::Translation::PostLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "skips bot posts" do @@ -58,7 +58,7 @@ describe Jobs::LocalizePosts do DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "en").never DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "ja").never - job.execute({}) + job.execute({ limit: 10 }) end it "handles translation errors gracefully" do @@ -70,7 +70,7 @@ describe Jobs::LocalizePosts do DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "ja").once DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "de").once - expect { job.execute({}) }.not_to raise_error + expect { job.execute({ limit: 10 }) }.not_to raise_error end it "logs a summary after translation" do @@ -80,14 +80,14 @@ describe Jobs::LocalizePosts do DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Translated 1 posts to ja")) DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Translated 1 posts to de")) - job.execute({}) + job.execute({ limit: 10 }) end context "for translation scenarios" do it "scenario 1: skips post when locale is not set" do DiscourseAi::Translation::PostLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "scenario 2: returns post with locale 'es' if localizations for en/ja/de do not exist" do @@ -97,7 +97,7 @@ describe Jobs::LocalizePosts do DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "ja").once DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "de").once - job.execute({}) + job.execute({ limit: 10 }) end it "scenario 3: returns post with locale 'en' if ja/de localization does not exist" do @@ -107,7 +107,7 @@ describe Jobs::LocalizePosts do DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "de").once DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "en").never - job.execute({}) + job.execute({ limit: 10 }) end it "scenario 4: skips post with locale 'en' if 'ja' localization already exists" do @@ -118,7 +118,7 @@ describe Jobs::LocalizePosts do DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "ja").never DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "de").once - job.execute({}) + job.execute({ limit: 10 }) end end @@ -158,7 +158,7 @@ describe Jobs::LocalizePosts do .with(group_pm_post, any_parameters) .never - job.execute({}) + job.execute({ limit: 10 }) end end @@ -176,7 +176,7 @@ describe Jobs::LocalizePosts do .with(personal_pm_post, any_parameters) .never - job.execute({}) + job.execute({ limit: 10 }) end end end @@ -198,7 +198,7 @@ describe Jobs::LocalizePosts do .with(old_post, any_parameters) .never - job.execute({}) + job.execute({ limit: 10 }) end it "processes all posts when setting is disabled" do @@ -208,7 +208,7 @@ describe Jobs::LocalizePosts do DiscourseAi::Translation::PostLocalizer.expects(:localize).with(old_post, "ja").once - job.execute({}) + job.execute({ limit: 10 }) end end end diff --git a/spec/jobs/regular/localize_topics_spec.rb b/spec/jobs/regular/localize_topics_spec.rb index 9e435ffe..ae263e08 100644 --- a/spec/jobs/regular/localize_topics_spec.rb +++ b/spec/jobs/regular/localize_topics_spec.rb @@ -19,28 +19,28 @@ describe Jobs::LocalizeTopics do SiteSetting.discourse_ai_enabled = false DiscourseAi::Translation::TopicLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when ai_translation_enabled is disabled" do SiteSetting.ai_translation_enabled = false DiscourseAi::Translation::TopicLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when no target languages are configured" do SiteSetting.content_localization_supported_locales = "" DiscourseAi::Translation::TopicLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when there are no topics to translate" do Topic.destroy_all DiscourseAi::Translation::TopicLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "skips topics that already have localizations" do @@ -50,7 +50,7 @@ describe Jobs::LocalizeTopics do end DiscourseAi::Translation::TopicLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "skips bot topics" do @@ -58,7 +58,7 @@ describe Jobs::LocalizeTopics do DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "en").never DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "ja").never - job.execute({}) + job.execute({ limit: 10 }) end it "handles translation errors gracefully" do @@ -70,7 +70,7 @@ describe Jobs::LocalizeTopics do DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "ja").once DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "de").once - expect { job.execute({}) }.not_to raise_error + expect { job.execute({ limit: 10 }) }.not_to raise_error end it "logs a summary after translation" do @@ -86,14 +86,14 @@ describe Jobs::LocalizeTopics do includes("Translated 1 topics to de"), ) - job.execute({}) + job.execute({ limit: 10 }) end context "for translation scenarios" do it "scenario 1: skips topic when locale is not set" do DiscourseAi::Translation::TopicLocalizer.expects(:localize).never - job.execute({}) + job.execute({ limit: 10 }) end it "scenario 2: returns topic with locale 'es' if localizations for en/ja/de do not exist" do @@ -103,7 +103,7 @@ describe Jobs::LocalizeTopics do DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "ja").once DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "de").once - job.execute({}) + job.execute({ limit: 10 }) end it "scenario 3: returns topic with locale 'en' if ja/de localization does not exist" do @@ -113,7 +113,7 @@ describe Jobs::LocalizeTopics do DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "de").once DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "en").never - job.execute({}) + job.execute({ limit: 10 }) end it "scenario 4: skips topic with locale 'en' if 'ja' localization already exists" do @@ -124,7 +124,7 @@ describe Jobs::LocalizeTopics do DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "ja").never DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "de").once - job.execute({}) + job.execute({ limit: 10 }) end end @@ -162,7 +162,7 @@ describe Jobs::LocalizeTopics do .with(group_pm_topic, any_parameters) .never - job.execute({}) + job.execute({ limit: 10 }) end end @@ -181,7 +181,7 @@ describe Jobs::LocalizeTopics do .with(personal_pm_topic, any_parameters) .never - job.execute({}) + job.execute({ limit: 10 }) end end end @@ -202,7 +202,7 @@ describe Jobs::LocalizeTopics do .with(old_topic, any_parameters) .never - job.execute({}) + job.execute({ limit: 10 }) end it "processes all topics when setting is disabled" do @@ -216,7 +216,7 @@ describe Jobs::LocalizeTopics do DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(old_topic, "ja").once DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(old_topic, "de").once - job.execute({}) + job.execute({ limit: 10 }) end end end diff --git a/spec/jobs/scheduled/categories_locale_detection_backfill_spec.rb b/spec/jobs/scheduled/categories_locale_detection_backfill_spec.rb index 5eff85fd..2ce7b066 100644 --- a/spec/jobs/scheduled/categories_locale_detection_backfill_spec.rb +++ b/spec/jobs/scheduled/categories_locale_detection_backfill_spec.rb @@ -10,7 +10,7 @@ describe Jobs::CategoriesLocaleDetectionBackfill do SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}") end SiteSetting.ai_translation_enabled = true - SiteSetting.ai_translation_backfill_rate = 100 + SiteSetting.ai_translation_backfill_hourly_rate = 100 end it "does nothing when AI is disabled" do @@ -28,7 +28,7 @@ describe Jobs::CategoriesLocaleDetectionBackfill do end it "does nothing when backfill rate is 0" do - SiteSetting.ai_translation_backfill_rate = 0 + SiteSetting.ai_translation_backfill_hourly_rate = 0 DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).never job.execute({}) @@ -42,13 +42,19 @@ describe Jobs::CategoriesLocaleDetectionBackfill do end it "detects locale for categories with nil locale" do - DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(is_a(Category)).times(Category.count) + DiscourseAi::Translation::CategoryLocaleDetector + .expects(:detect_locale) + .with(is_a(Category)) + .times(Category.count) job.execute({}) end it "handles detection errors gracefully" do - DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(is_a(Category)).at_least_once + DiscourseAi::Translation::CategoryLocaleDetector + .expects(:detect_locale) + .with(is_a(Category)) + .at_least_once DiscourseAi::Translation::CategoryLocaleDetector .expects(:detect_locale) .with(category) @@ -60,7 +66,9 @@ describe Jobs::CategoriesLocaleDetectionBackfill do it "logs a summary after running" do DiscourseAi::Translation::CategoryLocaleDetector.stubs(:detect_locale) - DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Detected #{Category.count} category locales")) + DiscourseAi::Translation::VerboseLogger.expects(:log).with( + includes("Detected #{Category.count} category locales"), + ) job.execute({}) end @@ -70,14 +78,20 @@ describe Jobs::CategoriesLocaleDetectionBackfill do before do # catch-all for other categories - DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(is_a(Category)).at_least_once + DiscourseAi::Translation::CategoryLocaleDetector + .expects(:detect_locale) + .with(is_a(Category)) + .at_least_once SiteSetting.ai_translation_backfill_limit_to_public_content = true end it "only processes public categories" do DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(category).once - DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(private_category).never + DiscourseAi::Translation::CategoryLocaleDetector + .expects(:detect_locale) + .with(private_category) + .never job.execute({}) end @@ -86,14 +100,17 @@ describe Jobs::CategoriesLocaleDetectionBackfill do SiteSetting.ai_translation_backfill_limit_to_public_content = false DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(category).once - DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(private_category).once + DiscourseAi::Translation::CategoryLocaleDetector + .expects(:detect_locale) + .with(private_category) + .once job.execute({}) end end it "limits processing to the backfill rate" do - SiteSetting.ai_translation_backfill_rate = 1 + SiteSetting.ai_translation_backfill_hourly_rate = 1 Fabricate(:category, locale: nil) DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).once diff --git a/spec/jobs/scheduled/post_localization_backfill_spec.rb b/spec/jobs/scheduled/post_localization_backfill_spec.rb index d52de4ef..f43d890d 100644 --- a/spec/jobs/scheduled/post_localization_backfill_spec.rb +++ b/spec/jobs/scheduled/post_localization_backfill_spec.rb @@ -2,11 +2,13 @@ describe Jobs::PostLocalizationBackfill do before do - SiteSetting.ai_translation_backfill_rate = 100 + SiteSetting.ai_translation_backfill_hourly_rate = 100 SiteSetting.content_localization_supported_locales = "en" Fabricate(:fake_model).tap do |fake_llm| SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}") end + SiteSetting.ai_translation_enabled = true + SiteSetting.discourse_ai_enabled = true end it "does not enqueue post translation when translator disabled" do @@ -18,7 +20,6 @@ describe Jobs::PostLocalizationBackfill do end it "does not enqueue post translation when experimental translation disabled" do - SiteSetting.discourse_ai_enabled = true SiteSetting.ai_translation_enabled = false described_class.new.execute({}) @@ -26,9 +27,7 @@ describe Jobs::PostLocalizationBackfill do expect_not_enqueued_with(job: :localize_posts) end - it "does not enqueue psot translation if backfill languages are not set" do - SiteSetting.discourse_ai_enabled = true - SiteSetting.ai_translation_enabled = true + it "does not enqueue post translation if backfill languages are not set" do SiteSetting.content_localization_supported_locales = "" described_class.new.execute({}) @@ -39,7 +38,7 @@ describe Jobs::PostLocalizationBackfill do it "does not enqueue post translation if backfill limit is set to 0" do SiteSetting.discourse_ai_enabled = true SiteSetting.ai_translation_enabled = true - SiteSetting.ai_translation_backfill_rate = 0 + SiteSetting.ai_translation_backfill_hourly_rate = 0 described_class.new.execute({}) @@ -49,10 +48,10 @@ describe Jobs::PostLocalizationBackfill do it "enqueues post translation with correct limit" do SiteSetting.discourse_ai_enabled = true SiteSetting.ai_translation_enabled = true - SiteSetting.ai_translation_backfill_rate = 10 + SiteSetting.ai_translation_backfill_hourly_rate = 100 described_class.new.execute({}) - expect_job_enqueued(job: :localize_posts, args: { limit: 10 }) + expect_job_enqueued(job: :localize_posts, args: { limit: 8 }) end end diff --git a/spec/jobs/scheduled/posts_locale_detection_backfill_spec.rb b/spec/jobs/scheduled/posts_locale_detection_backfill_spec.rb index b586e3ce..00693f9a 100644 --- a/spec/jobs/scheduled/posts_locale_detection_backfill_spec.rb +++ b/spec/jobs/scheduled/posts_locale_detection_backfill_spec.rb @@ -10,7 +10,7 @@ describe Jobs::PostsLocaleDetectionBackfill do SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}") end SiteSetting.ai_translation_enabled = true - SiteSetting.ai_translation_backfill_rate = 100 + SiteSetting.ai_translation_backfill_hourly_rate = 100 end it "does nothing when translator is disabled" do @@ -47,7 +47,7 @@ describe Jobs::PostsLocaleDetectionBackfill do post_2.update!(updated_at: 2.day.ago) post_3.update!(updated_at: 4.day.ago) - SiteSetting.ai_translation_backfill_rate = 1 + SiteSetting.ai_translation_backfill_hourly_rate = 12 DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(post_2).once DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(post).never diff --git a/spec/jobs/scheduled/topics_locale_detection_backfill_spec.rb b/spec/jobs/scheduled/topics_locale_detection_backfill_spec.rb index dbf1b425..3fbcb84f 100644 --- a/spec/jobs/scheduled/topics_locale_detection_backfill_spec.rb +++ b/spec/jobs/scheduled/topics_locale_detection_backfill_spec.rb @@ -10,33 +10,33 @@ describe Jobs::TopicsLocaleDetectionBackfill do SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}") end SiteSetting.ai_translation_enabled = true - SiteSetting.ai_translation_backfill_rate = 100 + SiteSetting.ai_translation_backfill_hourly_rate = 100 end it "does nothing when translator is disabled" do SiteSetting.discourse_ai_enabled = false DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when content translation is disabled" do SiteSetting.ai_translation_enabled = false DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).never - job.execute({}) + job.execute({ limit: 10 }) end it "does nothing when there are no topics to detect" do Topic.update_all(locale: "en") DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).never - job.execute({}) + job.execute({ limit: 10 }) end it "detects locale for topics with nil locale" do DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(topic).once - job.execute({}) + job.execute({ limit: 10 }) end it "detects most recently updated topics first" do @@ -47,20 +47,20 @@ describe Jobs::TopicsLocaleDetectionBackfill do topic_2.update!(updated_at: 2.day.ago) topic_3.update!(updated_at: 4.day.ago) - SiteSetting.ai_translation_backfill_rate = 1 + SiteSetting.ai_translation_backfill_hourly_rate = 12 DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(topic_2).once DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(topic).never DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(topic_3).never - job.execute({}) + job.execute({ limit: 10 }) end it "skips bot topics" do topic.update!(user: Discourse.system_user) DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(topic).never - job.execute({}) + job.execute({ limit: 10 }) end it "handles detection errors gracefully" do @@ -70,14 +70,14 @@ describe Jobs::TopicsLocaleDetectionBackfill do .raises(StandardError.new("jiboomz")) .once - expect { job.execute({}) }.not_to raise_error + expect { job.execute({ limit: 10 }) }.not_to raise_error end it "logs a summary after running" do DiscourseAi::Translation::TopicLocaleDetector.stubs(:detect_locale) DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Detected 1 topic locales")) - job.execute({}) + job.execute({ limit: 10 }) end describe "with public content limitation" do @@ -98,7 +98,7 @@ describe Jobs::TopicsLocaleDetectionBackfill do .with(private_topic) .never - job.execute({}) + job.execute({ limit: 10 }) end it "processes all topics when setting is disabled" do @@ -107,7 +107,7 @@ describe Jobs::TopicsLocaleDetectionBackfill do DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(private_topic).once - job.execute({}) + job.execute({ limit: 10 }) end end @@ -125,7 +125,7 @@ describe Jobs::TopicsLocaleDetectionBackfill do DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(old_topic).never - job.execute({}) + job.execute({ limit: 10 }) end it "processes all topics when setting is disabled" do @@ -134,7 +134,7 @@ describe Jobs::TopicsLocaleDetectionBackfill do DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(old_topic).once - job.execute({}) + job.execute({ limit: 10 }) end end end diff --git a/spec/lib/translation/post_localizer_spec.rb b/spec/lib/translation/post_localizer_spec.rb index f498e62b..f75040d8 100644 --- a/spec/lib/translation/post_localizer_spec.rb +++ b/spec/lib/translation/post_localizer_spec.rb @@ -18,7 +18,7 @@ describe DiscourseAi::Translation::PostLocalizer do allow(mock).to receive(:translate).and_return(opts[:translated]) end - it "returns nil if post is blank" do + it "returns nil if post does not exist" do expect(described_class.localize(nil, "ja")).to eq(nil) end @@ -33,6 +33,19 @@ describe DiscourseAi::Translation::PostLocalizer do expect(described_class.localize(post, "en")).to eq(nil) end + it "returns nil if post raw is blank" do + post.raw = "" + + expect(described_class.localize(post, "ja")).to eq(nil) + end + + it "returns nil if post raw is too long" do + SiteSetting.ai_translation_max_post_length = 10 + post.raw = "This is a very long post that exceeds the limit." + + expect(described_class.localize(post, "ja")).to eq(nil) + end + it "translates with post and locale" do post_raw_translator_stub({ text: post.raw, target_locale: "ja", translated: translated_raw })