FEATURE: introduce ultra_low priority queue

This commit introduces an ultra low priority queue for post rebakes. This
way rebakes can never interfere with regular sidekiq processing for cases
where we perform a large scale rebake.

Additionally it allows Post.rebake_old to be run with rate_limiter: false
to avoid triggering the limiter when rebaking. This is handy for cases
where you want to just force the full rebake and not wait for it to trickle

app/jobs/scheduled/periodical_updates.rb

@@ -29,7 +29,7 @@ module Jobs
 
       # Forces rebake of old posts where needed, as long as no system avatars need updating
       if !SiteSetting.automatically_download_gravatars || !UserAvatar.where("last_gravatar_download_attempt IS NULL").limit(1).first
-        problems = Post.rebake_old(SiteSetting.rebake_old_posts_count, priority: :low)
+        problems = Post.rebake_old(SiteSetting.rebake_old_posts_count, priority: :ultra_low)
         problems.each do |hash|
           post_id = hash[:post].id
           Discourse.handle_job_exception(hash[:ex], error_context(args, "Rebaking post id #{post_id}", post_id: post_id))

app/models/post.rb

@@ -515,7 +515,7 @@ class Post < ActiveRecord::Base
     PostRevisor.new(self).revise!(updated_by, changes, opts)
   end
 
-  def self.rebake_old(limit, priority: :normal)
+  def self.rebake_old(limit, priority: :normal, rate_limiter: true)
 
     limiter = RateLimiter.new(
       nil,
@@ -537,7 +537,7 @@ class Post < ActiveRecord::Base
         post.rebake!(priority: priority)
 
         begin
-          limiter.performed!
+          limiter.performed! if rate_limiter
         rescue RateLimiter::LimitExceeded
           break
         end
@@ -700,8 +700,8 @@ class Post < ActiveRecord::Base
     args[:invalidate_oneboxes] = true if invalidate_oneboxes.present?
     args[:cooking_options] = self.cooking_options
 
-    if priority == :low
-      args[:queue] = 'low'
+    if priority && priority != :normal
+      args[:queue] = priority.to_s
     end
 
     Jobs.enqueue(:process_post, args)

bin/docker/sidekiq

@@ -1,5 +1,5 @@
 #!/bin/bash
 
 PARAMS="$@"
-CMD="cd /src && USER=discourse RAILS_ENV=${RAILS_ENV:=development} bundle exec sidekiq -q critical -q low -q default"
+CMD="cd /src && USER=discourse RAILS_ENV=${RAILS_ENV:=development} bundle exec sidekiq -q critical -q low -q default -q ultra_low"
 docker exec -it -u discourse:discourse discourse_dev /bin/bash -c "$CMD"

config/sidekiq.yml

@@ -2,6 +2,7 @@
 development:
   :concurrency: 5
   :queues:
-    - [critical,4]
-    - [default, 2]
-    - [low]
+    - [critical, 8]
+    - [default, 4]
+    - [low, 2]
+    - [ultra_low]

lib/demon/sidekiq.rb

@@ -34,7 +34,7 @@ class Demon::Sidekiq < Demon::Base
 
     options = ["-c", GlobalSetting.sidekiq_workers.to_s]
 
-    [['critical', 4], ['default', 2], ['low', 1]].each do |queue_name, weight|
+    [['critical', 8], ['default', 4], ['low', 2], ['ultra_low', 1]].each do |queue_name, weight|
       custom_queue_hostname = ENV["UNICORN_SIDEKIQ_#{queue_name.upcase}_QUEUE_HOSTNAME"]
 
       if !custom_queue_hostname || custom_queue_hostname.split(',').include?(`hostname`.strip)

spec/jobs/periodical_updates_spec.rb

@@ -22,7 +22,7 @@ describe Jobs::PeriodicalUpdates do
       jobs = Jobs::ProcessPost.jobs
       expect(jobs.length).to eq(1)
 
-      expect(jobs[0]["queue"]).to eq("low")
+      expect(jobs[0]["queue"]).to eq("ultra_low")
     end
 
     post.reload