From 384135845b386d4014dd6daf82e61722304e6d32 Mon Sep 17 00:00:00 2001
From: Sam <sam.saffron@gmail.com>
Date: Thu, 17 Jan 2019 14:53:09 +1100
Subject: [PATCH] FEATURE: introduce ultra_low priority queue

This commit introduces an ultra low priority queue for post rebakes. This
way rebakes can never interfere with regular sidekiq processing for cases
where we perform a large scale rebake.

Additionally it allows Post.rebake_old to be run with rate_limiter: false
to avoid triggering the limiter when rebaking. This is handy for cases
where you want to just force the full rebake and not wait for it to trickle
---
 app/jobs/scheduled/periodical_updates.rb | 2 +-
 app/models/post.rb                       | 8 ++++----
 bin/docker/sidekiq                       | 2 +-
 config/sidekiq.yml                       | 7 ++++---
 lib/demon/sidekiq.rb                     | 2 +-
 spec/jobs/periodical_updates_spec.rb     | 2 +-
 6 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/app/jobs/scheduled/periodical_updates.rb b/app/jobs/scheduled/periodical_updates.rb
index 5701e6d6fd3..d607b96cd92 100644
--- a/app/jobs/scheduled/periodical_updates.rb
+++ b/app/jobs/scheduled/periodical_updates.rb
@@ -29,7 +29,7 @@ module Jobs
 
       # Forces rebake of old posts where needed, as long as no system avatars need updating
       if !SiteSetting.automatically_download_gravatars || !UserAvatar.where("last_gravatar_download_attempt IS NULL").limit(1).first
-        problems = Post.rebake_old(SiteSetting.rebake_old_posts_count, priority: :low)
+        problems = Post.rebake_old(SiteSetting.rebake_old_posts_count, priority: :ultra_low)
         problems.each do |hash|
           post_id = hash[:post].id
           Discourse.handle_job_exception(hash[:ex], error_context(args, "Rebaking post id #{post_id}", post_id: post_id))
diff --git a/app/models/post.rb b/app/models/post.rb
index 329ed664edc..cf8cdf514ad 100644
--- a/app/models/post.rb
+++ b/app/models/post.rb
@@ -515,7 +515,7 @@ class Post < ActiveRecord::Base
     PostRevisor.new(self).revise!(updated_by, changes, opts)
   end
 
-  def self.rebake_old(limit, priority: :normal)
+  def self.rebake_old(limit, priority: :normal, rate_limiter: true)
 
     limiter = RateLimiter.new(
       nil,
@@ -537,7 +537,7 @@ class Post < ActiveRecord::Base
         post.rebake!(priority: priority)
 
         begin
-          limiter.performed!
+          limiter.performed! if rate_limiter
         rescue RateLimiter::LimitExceeded
           break
         end
@@ -700,8 +700,8 @@ class Post < ActiveRecord::Base
     args[:invalidate_oneboxes] = true if invalidate_oneboxes.present?
     args[:cooking_options] = self.cooking_options
 
-    if priority == :low
-      args[:queue] = 'low'
+    if priority && priority != :normal
+      args[:queue] = priority.to_s
     end
 
     Jobs.enqueue(:process_post, args)
diff --git a/bin/docker/sidekiq b/bin/docker/sidekiq
index cfb36311aa2..0919990713f 100755
--- a/bin/docker/sidekiq
+++ b/bin/docker/sidekiq
@@ -1,5 +1,5 @@
 #!/bin/bash
 
 PARAMS="$@"
-CMD="cd /src && USER=discourse RAILS_ENV=${RAILS_ENV:=development} bundle exec sidekiq -q critical -q low -q default"
+CMD="cd /src && USER=discourse RAILS_ENV=${RAILS_ENV:=development} bundle exec sidekiq -q critical -q low -q default -q ultra_low"
 docker exec -it -u discourse:discourse discourse_dev /bin/bash -c "$CMD"
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 3f94e6a21b8..29935e0c158 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -2,6 +2,7 @@
 development:
   :concurrency: 5
   :queues:
-    - [critical,4]
-    - [default, 2]
-    - [low]
+    - [critical, 8]
+    - [default, 4]
+    - [low, 2]
+    - [ultra_low]
diff --git a/lib/demon/sidekiq.rb b/lib/demon/sidekiq.rb
index bfb25f2bf5e..affcd3b9f4a 100644
--- a/lib/demon/sidekiq.rb
+++ b/lib/demon/sidekiq.rb
@@ -34,7 +34,7 @@ class Demon::Sidekiq < Demon::Base
 
     options = ["-c", GlobalSetting.sidekiq_workers.to_s]
 
-    [['critical', 4], ['default', 2], ['low', 1]].each do |queue_name, weight|
+    [['critical', 8], ['default', 4], ['low', 2], ['ultra_low', 1]].each do |queue_name, weight|
       custom_queue_hostname = ENV["UNICORN_SIDEKIQ_#{queue_name.upcase}_QUEUE_HOSTNAME"]
 
       if !custom_queue_hostname || custom_queue_hostname.split(',').include?(`hostname`.strip)
diff --git a/spec/jobs/periodical_updates_spec.rb b/spec/jobs/periodical_updates_spec.rb
index 8b0e88bfc68..47f4282215f 100644
--- a/spec/jobs/periodical_updates_spec.rb
+++ b/spec/jobs/periodical_updates_spec.rb
@@ -22,7 +22,7 @@ describe Jobs::PeriodicalUpdates do
       jobs = Jobs::ProcessPost.jobs
       expect(jobs.length).to eq(1)
 
-      expect(jobs[0]["queue"]).to eq("low")
+      expect(jobs[0]["queue"]).to eq("ultra_low")
     end
 
     post.reload