PERF: add option to limit how many categories are processed in one call to CategoryFeaturedTopic.feature_topics (#5446)

app/jobs/scheduled/periodical_updates.rb

@@ -17,7 +17,7 @@ module Jobs
 
     def execute(args)
       # Feature topics in categories
-      CategoryFeaturedTopic.feature_topics
+      CategoryFeaturedTopic.feature_topics(batched: true)
 
       # Update the scores of posts
       args = { min_topic_age: 1.day.ago }

app/models/category_featured_topic.rb

@@ -2,13 +2,33 @@ class CategoryFeaturedTopic < ActiveRecord::Base
   belongs_to :category
   belongs_to :topic
 
-  # Populates the category featured topics
+  NEXT_CATEGORY_ID_KEY = 'category-featured-topic:next-category-id'.freeze
-  def self.feature_topics
+  BATCH_SIZE = 100
+
+  # Populates the category featured topics.
+  def self.feature_topics(batched: false)
     current = {}
     CategoryFeaturedTopic.select(:topic_id, :category_id).order(:rank).each do |f|
       (current[f.category_id] ||= []) << f.topic_id
     end
-    Category.select(:id, :topic_id, :num_featured_topics).find_each do |c|
+
+    next_category_id = batched ? ($redis.get(NEXT_CATEGORY_ID_KEY) || 0) : 0
+
+    categories = Category.select(:id, :topic_id, :num_featured_topics)
+      .where('id >= ?', next_category_id)
+      .order('id ASC')
+      .limit(BATCH_SIZE)
+
+    if batched
+      if categories.count == BATCH_SIZE
+        next_id = Category.where('id > ?', categories.last.id).order('id asc').limit(1).pluck(:id)[0]
+        next_id ? $redis.setex(NEXT_CATEGORY_ID_KEY, 1.day, next_id) : $redis.del(NEXT_CATEGORY_ID_KEY)
+      else
+        $redis.del(NEXT_CATEGORY_ID_KEY)
+      end
+    end
+
+    categories.find_each do |c|
       CategoryFeaturedTopic.feature_topics_for(c, current[c.id] || [])
     end
   end