From 8ec7fd84fd4d5318926ec5aa3944ca5a3032c49b Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 7 Apr 2016 12:56:43 +1000 Subject: [PATCH] FEATURE: prioritize sidekiq jobs This commit introduces 3 queues for sidekiq "critical" for urgent jobs (weighted at 4x weight) "default" for standard jobs(weighted at 2x weight) "low" for less important jobs "critical jobs" Reset Password emails has been seperated to its own job Heartbeat which is required to keep sidekiq running Test email which needs to return real quick "low priority jobs" Notify mailing list Pull hotlinked images Update gravatar "default" All the rest Note: for people running sidekiq from command line use bin/sidekiq -q critical,4 -q default,2 -q low --- app/controllers/session_controller.rb | 2 +- app/jobs/base.rb | 1 - app/jobs/regular/forgot_password.rb | 14 ++++++++++++++ .../regular/notify_mailing_list_subscribers.rb | 2 ++ app/jobs/regular/pull_hotlinked_images.rb | 3 +++ app/jobs/regular/run_heartbeat.rb | 2 ++ app/jobs/regular/test_email.rb | 2 ++ app/jobs/regular/update_gravatar.rb | 2 ++ lib/demon/sidekiq.rb | 2 +- spec/controllers/session_controller_spec.rb | 2 +- 10 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 app/jobs/regular/forgot_password.rb diff --git a/app/controllers/session_controller.rb b/app/controllers/session_controller.rb index 6a09a8060e5..0e7da392277 100644 --- a/app/controllers/session_controller.rb +++ b/app/controllers/session_controller.rb @@ -204,7 +204,7 @@ class SessionController < ApplicationController user_presence = user.present? && user.id != Discourse::SYSTEM_USER_ID && !user.staged if user_presence email_token = user.email_tokens.create(email: user.email) - Jobs.enqueue(:user_email, type: :forgot_password, user_id: user.id, email_token: email_token.token) + Jobs.enqueue(:forgot_password, user_id: user.id, email_token: email_token.token) end json = { result: "ok" } diff --git a/app/jobs/base.rb b/app/jobs/base.rb index 427a720079d..2f0c65e32d9 100644 --- a/app/jobs/base.rb +++ b/app/jobs/base.rb @@ -262,6 +262,5 @@ module Jobs end end -# Require all jobs Dir["#{Rails.root}/app/jobs/regular/*.rb"].each {|file| require_dependency file } Dir["#{Rails.root}/app/jobs/scheduled/*.rb"].each {|file| require_dependency file } diff --git a/app/jobs/regular/forgot_password.rb b/app/jobs/regular/forgot_password.rb new file mode 100644 index 00000000000..6f335d0f30f --- /dev/null +++ b/app/jobs/regular/forgot_password.rb @@ -0,0 +1,14 @@ +# base.rb uses this style of require, so maintain usage of it here +require_dependency "#{Rails.root}/app/jobs/regular/user_email.rb" + +module Jobs + class ForgotPassword < UserEmail + + sidekiq_options queue: 'critical' + + def execute(args) + args[:type] = :forgot_password + super(args) + end + end +end diff --git a/app/jobs/regular/notify_mailing_list_subscribers.rb b/app/jobs/regular/notify_mailing_list_subscribers.rb index c273ce0e47b..e24aa6ae6ef 100644 --- a/app/jobs/regular/notify_mailing_list_subscribers.rb +++ b/app/jobs/regular/notify_mailing_list_subscribers.rb @@ -2,6 +2,8 @@ module Jobs class NotifyMailingListSubscribers < Jobs::Base + sidekiq_options queue: 'low' + def execute(args) return if SiteSetting.disable_mailing_list_mode diff --git a/app/jobs/regular/pull_hotlinked_images.rb b/app/jobs/regular/pull_hotlinked_images.rb index 379ad102ec2..c3f4623f197 100644 --- a/app/jobs/regular/pull_hotlinked_images.rb +++ b/app/jobs/regular/pull_hotlinked_images.rb @@ -4,6 +4,9 @@ require_dependency 'file_helper' module Jobs class PullHotlinkedImages < Jobs::Base + + sidekiq_options queue: 'low' + def initialize # maximum size of the file in bytes @max_size = SiteSetting.max_image_size_kb.kilobytes diff --git a/app/jobs/regular/run_heartbeat.rb b/app/jobs/regular/run_heartbeat.rb index 5cf0072588b..226ac1934d5 100644 --- a/app/jobs/regular/run_heartbeat.rb +++ b/app/jobs/regular/run_heartbeat.rb @@ -1,6 +1,8 @@ module Jobs class RunHeartbeat < Jobs::Base + sidekiq_options queue: 'critical' + def self.heartbeat_key 'heartbeat_last_run' end diff --git a/app/jobs/regular/test_email.rb b/app/jobs/regular/test_email.rb index 7bf00f9a1f6..b0397260536 100644 --- a/app/jobs/regular/test_email.rb +++ b/app/jobs/regular/test_email.rb @@ -5,6 +5,8 @@ module Jobs # Asynchronously send an email class TestEmail < Jobs::Base + sidekiq_options queue: 'critical' + def execute(args) raise Discourse::InvalidParameters.new(:to_address) unless args[:to_address].present? diff --git a/app/jobs/regular/update_gravatar.rb b/app/jobs/regular/update_gravatar.rb index bd8ef140254..bbff6a6b226 100644 --- a/app/jobs/regular/update_gravatar.rb +++ b/app/jobs/regular/update_gravatar.rb @@ -2,6 +2,8 @@ module Jobs class UpdateGravatar < Jobs::Base + sidekiq_options queue: 'low' + def execute(args) user = User.find_by(id: args[:user_id]) avatar = UserAvatar.find_by(id: args[:avatar_id]) diff --git a/lib/demon/sidekiq.rb b/lib/demon/sidekiq.rb index d8182312de6..4e984bc7816 100644 --- a/lib/demon/sidekiq.rb +++ b/lib/demon/sidekiq.rb @@ -25,7 +25,7 @@ class Demon::Sidekiq < Demon::Base # parent process is in charge of the file anyway. Sidekiq::Logging.logger = nil cli = Sidekiq::CLI.instance - cli.parse(["-c", GlobalSetting.sidekiq_workers.to_s]) + cli.parse(["-c", GlobalSetting.sidekiq_workers.to_s, "-q", "critical,4", "-q", "default,2", "-q", "low"]) load Rails.root + "config/initializers/100-sidekiq.rb" cli.run diff --git a/spec/controllers/session_controller_spec.rb b/spec/controllers/session_controller_spec.rb index 492093b438e..0c128bfc275 100644 --- a/spec/controllers/session_controller_spec.rb +++ b/spec/controllers/session_controller_spec.rb @@ -632,7 +632,7 @@ describe SessionController do end it "enqueues an email" do - Jobs.expects(:enqueue).with(:user_email, has_entries(type: :forgot_password, user_id: user.id)) + Jobs.expects(:enqueue).with(:forgot_password, has_entries(user_id: user.id)) xhr :post, :forgot_password, login: user.username end end