DEV: Indicate backfill rate for translations is hourly (#1451)

* DEV: Indicate backfill rate for translations is hourly

* add ai_translation_max_post_length

* default value update

app/jobs/regular/localize_categories.rb

@@ -5,51 +5,51 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.nil?
+      return if limit <= 0
+
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
-      cat_id = args[:from_category_id] || Category.order(:id).first&.id
+      categories = Category.where("locale IS NOT NULL")
-      last_id = nil
 
-      categories =
+      if SiteSetting.ai_translation_backfill_limit_to_public_content
-        Category.where("id >= ? AND locale IS NOT NULL", cat_id).order(:id).limit(BATCH_SIZE)
+        categories = categories.where(read_restricted: false)
-      return if categories.empty?
-
-      categories.each do |category|
-        if SiteSetting.ai_translation_backfill_limit_to_public_content && category.read_restricted?
-          last_id = category.id
-          next
-        end
-
-        locales.each do |locale|
-          localization = category.category_localizations.find_by(locale:)
-
-          if locale == category.locale && localization
-            localization.destroy
-          else
-            next if locale == category.locale
-            begin
-              DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
-            rescue FinalDestination::SSRFDetector::LookupFailedError
-              # do nothing, there are too many sporadic lookup failures
-            rescue => e
-              DiscourseAi::Translation::VerboseLogger.log(
-                "Failed to translate category #{category.id} to #{locale}: #{e.message}",
-              )
-            end
-          end
-        end
-        last_id = category.id
       end
 
-      if categories.size == BATCH_SIZE
+      categories = categories.order(:id).limit(limit)
-        Jobs.enqueue_in(10.seconds, :localize_categories, from_category_id: last_id + 1)
+      return if categories.empty?
+
+      remaining_limit = limit
+
+      categories.each do |category|
+        break if remaining_limit <= 0
+
+        existing_locales = CategoryLocalization.where(category_id: category.id).pluck(:locale)
+        missing_locales = locales - existing_locales - [category.locale]
+        missing_locales.each do |locale|
+          break if remaining_limit <= 0
+
+          begin
+            DiscourseAi::Translation::CategoryLocalizer.localize(category, locale)
+          rescue FinalDestination::SSRFDetector::LookupFailedError
+            # do nothing, there are too many sporadic lookup failures
+          rescue => e
+            DiscourseAi::Translation::VerboseLogger.log(
+              "Failed to translate category #{category.id} to #{locale}: #{e.message}",
+            )
+          ensure
+            remaining_limit -= 1
+          end
+        end
+
+        if existing_locales.include?(category.locale)
+          CategoryLocalization.find_by(category_id: category.id, locale: category.locale).destroy
+        end
       end
     end
   end

app/jobs/regular/localize_posts.rb

@@ -5,17 +5,16 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
+
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
 
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
-      limit = args[:limit] || BATCH_SIZE
-
       locales.each do |locale|
         posts =
           Post

app/jobs/regular/localize_topics.rb

@@ -5,17 +5,16 @@ module Jobs
     cluster_concurrency 1
     sidekiq_options retry: false
 
-    BATCH_SIZE = 50
-
     def execute(args)
+      limit = args[:limit]
+      raise Discourse::InvalidParameters.new(:limit) if limit.blank? || limit <= 0
+
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
 
       locales = SiteSetting.content_localization_supported_locales.split("|")
       return if locales.blank?
 
-      limit = args[:limit] || BATCH_SIZE
-
       locales.each do |locale|
         topics =
           Topic

app/jobs/scheduled/categories_locale_detection_backfill.rb

@@ -9,7 +9,8 @@ module Jobs
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate
+      return if limit == 0
 
       categories = Category.where(locale: nil)
 
@@ -17,7 +18,7 @@ module Jobs
         categories = categories.where(read_restricted: false)
       end
 
-      categories = categories.limit(SiteSetting.ai_translation_backfill_rate)
+      categories = categories.limit(limit)
       return if categories.empty?
 
       categories.each do |category|

app/jobs/scheduled/category_localization_backfill.rb

@@ -2,15 +2,17 @@
 
 module Jobs
   class CategoryLocalizationBackfill < ::Jobs::Scheduled
-    every 12.hours
+    every 1.hour
     cluster_concurrency 1
 
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
       return if SiteSetting.content_localization_supported_locales.blank?
+      limit = SiteSetting.ai_translation_backfill_hourly_rate
+      return if limit == 0
 
-      Jobs.enqueue(:localize_categories)
+      Jobs.enqueue(:localize_categories, limit:)
     end
   end
 end

app/jobs/scheduled/post_localization_backfill.rb

@@ -10,9 +10,10 @@ module Jobs
       return if !SiteSetting.ai_translation_enabled
 
       return if SiteSetting.content_localization_supported_locales.blank?
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
-      Jobs.enqueue(:localize_posts, limit: SiteSetting.ai_translation_backfill_rate)
+      Jobs.enqueue(:localize_posts, limit:)
     end
   end
 end

app/jobs/scheduled/posts_locale_detection_backfill.rb

@@ -9,7 +9,8 @@ module Jobs
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
       posts =
         Post
@@ -35,7 +36,7 @@ module Jobs
           )
       end
 
-      posts = posts.order(updated_at: :desc).limit(SiteSetting.ai_translation_backfill_rate)
+      posts = posts.order(updated_at: :desc).limit(limit)
       return if posts.empty?
 
       posts.each do |post|

app/jobs/scheduled/topic_localization_backfill.rb

@@ -10,9 +10,10 @@ module Jobs
       return if !SiteSetting.ai_translation_enabled
 
       return if SiteSetting.content_localization_supported_locales.blank?
-      return if SiteSetting.ai_translation_backfill_rate == 0
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
+      return if limit == 0
 
-      Jobs.enqueue(:localize_topics, limit: SiteSetting.ai_translation_backfill_rate)
+      Jobs.enqueue(:localize_topics, limit:)
     end
   end
 end

app/jobs/scheduled/topics_locale_detection_backfill.rb

@@ -9,8 +9,7 @@ module Jobs
     def execute(args)
       return if !SiteSetting.discourse_ai_enabled
       return if !SiteSetting.ai_translation_enabled
-      limit = SiteSetting.ai_translation_backfill_rate
+      limit = SiteSetting.ai_translation_backfill_hourly_rate / (60 / 5) # this job runs in 5-minute intervals
-
       return if limit == 0
 
       topics = Topic.where(locale: nil, deleted_at: nil).where("topics.user_id > 0")

config/locales/server.en.yml

@@ -118,6 +118,12 @@ en:
     ai_discord_allowed_guilds: "Discord guilds (servers) where the bot is allowed to search"
     ai_bot_enable_dedicated_ux: "Allow for full screen bot interface, instead of a PM"
 
+    ai_translation_enabled: "Enables the AI translation feature"
+    ai_translation_model: "The model to use for translation. This model must support translation. Personas can override this setting."
+    ai_translation_backfill_limit_to_public_content: "When enabled, only content in public categories will be translated. When disabled, content in group PMs and private categories will also be sent for translation."
+    ai_translation_max_post_length: "The maximum length of a post to be translated. Posts longer than this will not be translated."
+    ai_translation_backfill_max_age_days: "The maximum age of a post and topic to be translated. Posts and topics older than this will not be translated."
+
   reviewables:
     reasons:
       flagged_by_toxicity: The AI plugin flagged this after classifying it as toxic.

config/settings.yml

@@ -480,7 +480,7 @@ discourse_ai:
     type: enum
     enum: "DiscourseAi::Configuration::PersonaEnumerator"
     area: "ai-features/translation"
-  ai_translation_backfill_rate:
+  ai_translation_backfill_hourly_rate:
     default: 0
     min: 0
     max: 1000
@@ -491,6 +491,10 @@ discourse_ai:
     default: true
     client: false
     area: "ai-features/translation"
+  ai_translation_max_post_length:
+    default: 10000
+    client: false
+    area: "ai-features/translation"
   ai_translation_backfill_max_age_days:
     default: 5
     client: false

db/migrate/20250620073222_specify_rate_frequency_in_backfill_setting.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class SpecifyRateFrequencyInBackfillSetting < ActiveRecord::Migration[7.2]
+  def up
+    execute "UPDATE site_settings SET name = 'ai_translation_backfill_hourly_rate' WHERE name = 'ai_translation_backfill_rate'"
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end

lib/translation/post_localizer.rb

@@ -4,7 +4,11 @@ module DiscourseAi
   module Translation
     class PostLocalizer
       def self.localize(post, target_locale = I18n.locale)
-        return if post.blank? || target_locale.blank? || post.locale == target_locale.to_s
+        if post.blank? || target_locale.blank? || post.locale == target_locale.to_s ||
+             post.raw.blank?
+          return
+        end
+        return if post.raw.length > SiteSetting.ai_translation_max_post_length
         target_locale = target_locale.to_s.sub("-", "_")
 
         translated_raw =

spec/jobs/regular/localize_categories_spec.rb

@@ -25,7 +25,7 @@ describe Jobs::LocalizeCategories do
 
     DiscourseAi::Translation::CategoryLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when ai_translation_enabled is disabled" do
@@ -33,7 +33,7 @@ describe Jobs::LocalizeCategories do
 
     DiscourseAi::Translation::CategoryLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when no target languages are configured" do
@@ -41,7 +41,7 @@ describe Jobs::LocalizeCategories do
 
     DiscourseAi::Translation::CategoryLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when no categories exist" do
@@ -49,7 +49,14 @@ describe Jobs::LocalizeCategories do
 
     DiscourseAi::Translation::CategoryLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
+  end
+
+  it "does nothing when the limit is zero" do
+    DiscourseAi::Translation::CategoryLocalizer.expects(:localize).never
+
+    expect { job.execute({}) }.to raise_error(Discourse::InvalidParameters, /limit/)
+    job.execute({ limit: 0 })
   end
 
   it "translates categories to the configured locales" do
@@ -65,7 +72,21 @@ describe Jobs::LocalizeCategories do
       .with(is_a(Category), "zh_CN")
       .times(number_of_categories)
 
-    job.execute({})
+    job.execute({ limit: 10 })
+  end
+
+  it "limits the number of localizations" do
+    SiteSetting.content_localization_supported_locales = "pt"
+
+    6.times { Fabricate(:category) }
+    Category.update_all(locale: "en")
+
+    DiscourseAi::Translation::CategoryLocalizer
+      .expects(:localize)
+      .with(is_a(Category), "pt")
+      .times(5)
+
+    job.execute({ limit: 5 })
   end
 
   it "skips categories that already have localizations" do
@@ -77,24 +98,7 @@ describe Jobs::LocalizeCategories do
       .with(is_a(Category), "zh_CN")
       .never
 
-    job.execute({})
+    job.execute({ limit: 10 })
-  end
-
-  it "continues from a specified category ID" do
-    category1 = Fabricate(:category, name: "First", description: "First description", locale: "en")
-    category2 =
-      Fabricate(:category, name: "Second", description: "Second description", locale: "en")
-
-    DiscourseAi::Translation::CategoryLocalizer
-      .expects(:localize)
-      .with(category1, any_parameters)
-      .never
-    DiscourseAi::Translation::CategoryLocalizer
-      .expects(:localize)
-      .with(category2, any_parameters)
-      .twice
-
-    job.execute(from_category_id: category2.id)
   end
 
   it "handles translation errors gracefully" do
@@ -107,31 +111,7 @@ describe Jobs::LocalizeCategories do
       .raises(StandardError.new("API error"))
     DiscourseAi::Translation::CategoryLocalizer.expects(:localize).with(category1, "zh_CN").once
 
-    expect { job.execute({}) }.not_to raise_error
+    expect { job.execute({ limit: 10 }) }.not_to raise_error
-  end
-
-  it "enqueues the next batch when there are more categories" do
-    Category.update_all(locale: "en")
-
-    Jobs.run_later!
-    freeze_time
-    Jobs::LocalizeCategories.const_set(:BATCH_SIZE, 1)
-
-    job.execute({})
-
-    Category.all.each do |category|
-      puts category.id
-      expect_job_enqueued(
-        job: :localize_categories,
-        args: {
-          from_category_id: category.id + 1,
-        },
-        at: 10.seconds.from_now,
-      )
-    end
-
-    Jobs::LocalizeCategories.send(:remove_const, :BATCH_SIZE)
-    Jobs::LocalizeCategories.const_set(:BATCH_SIZE, 50)
   end
 
   it "skips read-restricted categories when configured" do
@@ -149,7 +129,7 @@ describe Jobs::LocalizeCategories do
       .with(category2, any_parameters)
       .never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "skips creating localizations in the same language as the category's locale" do
@@ -161,7 +141,7 @@ describe Jobs::LocalizeCategories do
       .with(is_a(Category), "zh_CN")
       .times(Category.count)
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "deletes existing localizations that match the category's locale" do
@@ -170,9 +150,9 @@ describe Jobs::LocalizeCategories do
 
     localize_all_categories("pt", "zh_CN")
 
-    expect { job.execute({}) }.to change { CategoryLocalization.exists?(locale: "pt") }.from(
+    expect { job.execute({ limit: 10 }) }.to change {
-      true,
+      CategoryLocalization.exists?(locale: "pt")
-    ).to(false)
+    }.from(true).to(false)
   end
 
   it "doesn't process categories with nil locale" do
@@ -185,6 +165,6 @@ describe Jobs::LocalizeCategories do
       .with(nil_locale_category, any_parameters)
       .never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 end

spec/jobs/regular/localize_posts_spec.rb

@@ -19,28 +19,28 @@ describe Jobs::LocalizePosts do
     SiteSetting.discourse_ai_enabled = false
     DiscourseAi::Translation::PostLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when ai_translation_enabled is disabled" do
     SiteSetting.ai_translation_enabled = false
     DiscourseAi::Translation::PostLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when no target languages are configured" do
     SiteSetting.content_localization_supported_locales = ""
     DiscourseAi::Translation::PostLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when there are no posts to translate" do
     Post.destroy_all
     DiscourseAi::Translation::PostLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "skips posts that already have localizations" do
@@ -50,7 +50,7 @@ describe Jobs::LocalizePosts do
     end
     DiscourseAi::Translation::PostLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "skips bot posts" do
@@ -58,7 +58,7 @@ describe Jobs::LocalizePosts do
     DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "en").never
     DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "ja").never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "handles translation errors gracefully" do
@@ -70,7 +70,7 @@ describe Jobs::LocalizePosts do
     DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "ja").once
     DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "de").once
 
-    expect { job.execute({}) }.not_to raise_error
+    expect { job.execute({ limit: 10 }) }.not_to raise_error
   end
 
   it "logs a summary after translation" do
@@ -80,14 +80,14 @@ describe Jobs::LocalizePosts do
     DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Translated 1 posts to ja"))
     DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Translated 1 posts to de"))
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   context "for translation scenarios" do
     it "scenario 1: skips post when locale is not set" do
       DiscourseAi::Translation::PostLocalizer.expects(:localize).never
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
 
     it "scenario 2: returns post with locale 'es' if localizations for en/ja/de do not exist" do
@@ -97,7 +97,7 @@ describe Jobs::LocalizePosts do
       DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "ja").once
       DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "de").once
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
 
     it "scenario 3: returns post with locale 'en' if ja/de localization does not exist" do
@@ -107,7 +107,7 @@ describe Jobs::LocalizePosts do
       DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "de").once
       DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "en").never
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
 
     it "scenario 4: skips post with locale 'en' if 'ja' localization already exists" do
@@ -118,7 +118,7 @@ describe Jobs::LocalizePosts do
       DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "ja").never
       DiscourseAi::Translation::PostLocalizer.expects(:localize).with(post, "de").once
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
   end
 
@@ -158,7 +158,7 @@ describe Jobs::LocalizePosts do
           .with(group_pm_post, any_parameters)
           .never
 
-        job.execute({})
+        job.execute({ limit: 10 })
       end
     end
 
@@ -176,7 +176,7 @@ describe Jobs::LocalizePosts do
           .with(personal_pm_post, any_parameters)
           .never
 
-        job.execute({})
+        job.execute({ limit: 10 })
       end
     end
   end
@@ -198,7 +198,7 @@ describe Jobs::LocalizePosts do
         .with(old_post, any_parameters)
         .never
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
 
     it "processes all posts when setting is disabled" do
@@ -208,7 +208,7 @@ describe Jobs::LocalizePosts do
 
       DiscourseAi::Translation::PostLocalizer.expects(:localize).with(old_post, "ja").once
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
   end
 end

spec/jobs/regular/localize_topics_spec.rb

@@ -19,28 +19,28 @@ describe Jobs::LocalizeTopics do
     SiteSetting.discourse_ai_enabled = false
     DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when ai_translation_enabled is disabled" do
     SiteSetting.ai_translation_enabled = false
     DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when no target languages are configured" do
     SiteSetting.content_localization_supported_locales = ""
     DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when there are no topics to translate" do
     Topic.destroy_all
     DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "skips topics that already have localizations" do
@@ -50,7 +50,7 @@ describe Jobs::LocalizeTopics do
     end
     DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "skips bot topics" do
@@ -58,7 +58,7 @@ describe Jobs::LocalizeTopics do
     DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "en").never
     DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "ja").never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "handles translation errors gracefully" do
@@ -70,7 +70,7 @@ describe Jobs::LocalizeTopics do
     DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "ja").once
     DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "de").once
 
-    expect { job.execute({}) }.not_to raise_error
+    expect { job.execute({ limit: 10 }) }.not_to raise_error
   end
 
   it "logs a summary after translation" do
@@ -86,14 +86,14 @@ describe Jobs::LocalizeTopics do
       includes("Translated 1 topics to de"),
     )
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   context "for translation scenarios" do
     it "scenario 1: skips topic when locale is not set" do
       DiscourseAi::Translation::TopicLocalizer.expects(:localize).never
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
 
     it "scenario 2: returns topic with locale 'es' if localizations for en/ja/de do not exist" do
@@ -103,7 +103,7 @@ describe Jobs::LocalizeTopics do
       DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "ja").once
       DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "de").once
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
 
     it "scenario 3: returns topic with locale 'en' if ja/de localization does not exist" do
@@ -113,7 +113,7 @@ describe Jobs::LocalizeTopics do
       DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "de").once
       DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "en").never
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
 
     it "scenario 4: skips topic with locale 'en' if 'ja' localization already exists" do
@@ -124,7 +124,7 @@ describe Jobs::LocalizeTopics do
       DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "ja").never
       DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(topic, "de").once
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
   end
 
@@ -162,7 +162,7 @@ describe Jobs::LocalizeTopics do
           .with(group_pm_topic, any_parameters)
           .never
 
-        job.execute({})
+        job.execute({ limit: 10 })
       end
     end
 
@@ -181,7 +181,7 @@ describe Jobs::LocalizeTopics do
           .with(personal_pm_topic, any_parameters)
           .never
 
-        job.execute({})
+        job.execute({ limit: 10 })
       end
     end
   end
@@ -202,7 +202,7 @@ describe Jobs::LocalizeTopics do
         .with(old_topic, any_parameters)
         .never
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
 
     it "processes all topics when setting is disabled" do
@@ -216,7 +216,7 @@ describe Jobs::LocalizeTopics do
       DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(old_topic, "ja").once
       DiscourseAi::Translation::TopicLocalizer.expects(:localize).with(old_topic, "de").once
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
   end
 end

spec/jobs/scheduled/categories_locale_detection_backfill_spec.rb

@@ -10,7 +10,7 @@ describe Jobs::CategoriesLocaleDetectionBackfill do
       SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}")
     end
     SiteSetting.ai_translation_enabled = true
-    SiteSetting.ai_translation_backfill_rate = 100
+    SiteSetting.ai_translation_backfill_hourly_rate = 100
   end
 
   it "does nothing when AI is disabled" do
@@ -28,7 +28,7 @@ describe Jobs::CategoriesLocaleDetectionBackfill do
   end
 
   it "does nothing when backfill rate is 0" do
-    SiteSetting.ai_translation_backfill_rate = 0
+    SiteSetting.ai_translation_backfill_hourly_rate = 0
     DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).never
 
     job.execute({})
@@ -42,13 +42,19 @@ describe Jobs::CategoriesLocaleDetectionBackfill do
   end
 
   it "detects locale for categories with nil locale" do
-    DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(is_a(Category)).times(Category.count)
+    DiscourseAi::Translation::CategoryLocaleDetector
+      .expects(:detect_locale)
+      .with(is_a(Category))
+      .times(Category.count)
 
     job.execute({})
   end
 
   it "handles detection errors gracefully" do
-    DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(is_a(Category)).at_least_once
+    DiscourseAi::Translation::CategoryLocaleDetector
+      .expects(:detect_locale)
+      .with(is_a(Category))
+      .at_least_once
     DiscourseAi::Translation::CategoryLocaleDetector
       .expects(:detect_locale)
       .with(category)
@@ -60,7 +66,9 @@ describe Jobs::CategoriesLocaleDetectionBackfill do
 
   it "logs a summary after running" do
     DiscourseAi::Translation::CategoryLocaleDetector.stubs(:detect_locale)
-    DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Detected #{Category.count} category locales"))
+    DiscourseAi::Translation::VerboseLogger.expects(:log).with(
+      includes("Detected #{Category.count} category locales"),
+    )
 
     job.execute({})
   end
@@ -70,14 +78,20 @@ describe Jobs::CategoriesLocaleDetectionBackfill do
 
     before do
       # catch-all for other categories
-      DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(is_a(Category)).at_least_once
+      DiscourseAi::Translation::CategoryLocaleDetector
+        .expects(:detect_locale)
+        .with(is_a(Category))
+        .at_least_once
 
       SiteSetting.ai_translation_backfill_limit_to_public_content = true
     end
 
     it "only processes public categories" do
       DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(category).once
-      DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(private_category).never
+      DiscourseAi::Translation::CategoryLocaleDetector
+        .expects(:detect_locale)
+        .with(private_category)
+        .never
 
       job.execute({})
     end
@@ -86,14 +100,17 @@ describe Jobs::CategoriesLocaleDetectionBackfill do
       SiteSetting.ai_translation_backfill_limit_to_public_content = false
 
       DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(category).once
-      DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).with(private_category).once
+      DiscourseAi::Translation::CategoryLocaleDetector
+        .expects(:detect_locale)
+        .with(private_category)
+        .once
 
       job.execute({})
     end
   end
 
   it "limits processing to the backfill rate" do
-    SiteSetting.ai_translation_backfill_rate = 1
+    SiteSetting.ai_translation_backfill_hourly_rate = 1
     Fabricate(:category, locale: nil)
 
     DiscourseAi::Translation::CategoryLocaleDetector.expects(:detect_locale).once

spec/jobs/scheduled/post_localization_backfill_spec.rb

@@ -2,11 +2,13 @@
 
 describe Jobs::PostLocalizationBackfill do
   before do
-    SiteSetting.ai_translation_backfill_rate = 100
+    SiteSetting.ai_translation_backfill_hourly_rate = 100
     SiteSetting.content_localization_supported_locales = "en"
     Fabricate(:fake_model).tap do |fake_llm|
       SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}")
     end
+    SiteSetting.ai_translation_enabled = true
+    SiteSetting.discourse_ai_enabled = true
   end
 
   it "does not enqueue post translation when translator disabled" do
@@ -18,7 +20,6 @@ describe Jobs::PostLocalizationBackfill do
   end
 
   it "does not enqueue post translation when experimental translation disabled" do
-    SiteSetting.discourse_ai_enabled = true
     SiteSetting.ai_translation_enabled = false
 
     described_class.new.execute({})
@@ -26,9 +27,7 @@ describe Jobs::PostLocalizationBackfill do
     expect_not_enqueued_with(job: :localize_posts)
   end
 
-  it "does not enqueue psot translation if backfill languages are not set" do
+  it "does not enqueue post translation if backfill languages are not set" do
-    SiteSetting.discourse_ai_enabled = true
-    SiteSetting.ai_translation_enabled = true
     SiteSetting.content_localization_supported_locales = ""
 
     described_class.new.execute({})
@@ -39,7 +38,7 @@ describe Jobs::PostLocalizationBackfill do
   it "does not enqueue post translation if backfill limit is set to 0" do
     SiteSetting.discourse_ai_enabled = true
     SiteSetting.ai_translation_enabled = true
-    SiteSetting.ai_translation_backfill_rate = 0
+    SiteSetting.ai_translation_backfill_hourly_rate = 0
 
     described_class.new.execute({})
 
@@ -49,10 +48,10 @@ describe Jobs::PostLocalizationBackfill do
   it "enqueues post translation with correct limit" do
     SiteSetting.discourse_ai_enabled = true
     SiteSetting.ai_translation_enabled = true
-    SiteSetting.ai_translation_backfill_rate = 10
+    SiteSetting.ai_translation_backfill_hourly_rate = 100
 
     described_class.new.execute({})
 
-    expect_job_enqueued(job: :localize_posts, args: { limit: 10 })
+    expect_job_enqueued(job: :localize_posts, args: { limit: 8 })
   end
 end

spec/jobs/scheduled/posts_locale_detection_backfill_spec.rb

@@ -10,7 +10,7 @@ describe Jobs::PostsLocaleDetectionBackfill do
       SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}")
     end
     SiteSetting.ai_translation_enabled = true
-    SiteSetting.ai_translation_backfill_rate = 100
+    SiteSetting.ai_translation_backfill_hourly_rate = 100
   end
 
   it "does nothing when translator is disabled" do
@@ -47,7 +47,7 @@ describe Jobs::PostsLocaleDetectionBackfill do
     post_2.update!(updated_at: 2.day.ago)
     post_3.update!(updated_at: 4.day.ago)
 
-    SiteSetting.ai_translation_backfill_rate = 1
+    SiteSetting.ai_translation_backfill_hourly_rate = 12
 
     DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(post_2).once
     DiscourseAi::Translation::PostLocaleDetector.expects(:detect_locale).with(post).never

spec/jobs/scheduled/topics_locale_detection_backfill_spec.rb

@@ -10,33 +10,33 @@ describe Jobs::TopicsLocaleDetectionBackfill do
       SiteSetting.public_send("ai_translation_model=", "custom:#{fake_llm.id}")
     end
     SiteSetting.ai_translation_enabled = true
-    SiteSetting.ai_translation_backfill_rate = 100
+    SiteSetting.ai_translation_backfill_hourly_rate = 100
   end
 
   it "does nothing when translator is disabled" do
     SiteSetting.discourse_ai_enabled = false
     DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when content translation is disabled" do
     SiteSetting.ai_translation_enabled = false
     DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "does nothing when there are no topics to detect" do
     Topic.update_all(locale: "en")
     DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "detects locale for topics with nil locale" do
     DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(topic).once
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "detects most recently updated topics first" do
@@ -47,20 +47,20 @@ describe Jobs::TopicsLocaleDetectionBackfill do
     topic_2.update!(updated_at: 2.day.ago)
     topic_3.update!(updated_at: 4.day.ago)
 
-    SiteSetting.ai_translation_backfill_rate = 1
+    SiteSetting.ai_translation_backfill_hourly_rate = 12
 
     DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(topic_2).once
     DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(topic).never
     DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(topic_3).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "skips bot topics" do
     topic.update!(user: Discourse.system_user)
     DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(topic).never
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   it "handles detection errors gracefully" do
@@ -70,14 +70,14 @@ describe Jobs::TopicsLocaleDetectionBackfill do
       .raises(StandardError.new("jiboomz"))
       .once
 
-    expect { job.execute({}) }.not_to raise_error
+    expect { job.execute({ limit: 10 }) }.not_to raise_error
   end
 
   it "logs a summary after running" do
     DiscourseAi::Translation::TopicLocaleDetector.stubs(:detect_locale)
     DiscourseAi::Translation::VerboseLogger.expects(:log).with(includes("Detected 1 topic locales"))
 
-    job.execute({})
+    job.execute({ limit: 10 })
   end
 
   describe "with public content limitation" do
@@ -98,7 +98,7 @@ describe Jobs::TopicsLocaleDetectionBackfill do
         .with(private_topic)
         .never
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
 
     it "processes all topics when setting is disabled" do
@@ -107,7 +107,7 @@ describe Jobs::TopicsLocaleDetectionBackfill do
       DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(public_topic).once
       DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(private_topic).once
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
   end
 
@@ -125,7 +125,7 @@ describe Jobs::TopicsLocaleDetectionBackfill do
       DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once
       DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(old_topic).never
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
 
     it "processes all topics when setting is disabled" do
@@ -134,7 +134,7 @@ describe Jobs::TopicsLocaleDetectionBackfill do
       DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(new_topic).once
       DiscourseAi::Translation::TopicLocaleDetector.expects(:detect_locale).with(old_topic).once
 
-      job.execute({})
+      job.execute({ limit: 10 })
     end
   end
 end

spec/lib/translation/post_localizer_spec.rb

@@ -18,7 +18,7 @@ describe DiscourseAi::Translation::PostLocalizer do
       allow(mock).to receive(:translate).and_return(opts[:translated])
     end
 
-    it "returns nil if post is blank" do
+    it "returns nil if post does not exist" do
       expect(described_class.localize(nil, "ja")).to eq(nil)
     end
 
@@ -33,6 +33,19 @@ describe DiscourseAi::Translation::PostLocalizer do
       expect(described_class.localize(post, "en")).to eq(nil)
     end
 
+    it "returns nil if post raw is blank" do
+      post.raw = ""
+
+      expect(described_class.localize(post, "ja")).to eq(nil)
+    end
+
+    it "returns nil if post raw is too long" do
+      SiteSetting.ai_translation_max_post_length = 10
+      post.raw = "This is a very long post that exceeds the limit."
+
+      expect(described_class.localize(post, "ja")).to eq(nil)
+    end
+
     it "translates with post and locale" do
       post_raw_translator_stub({ text: post.raw, target_locale: "ja", translated: translated_raw })