From 44dadc43107447605cc8f5bbe3bc86e663cf8311 Mon Sep 17 00:00:00 2001 From: Alexandr Yakubenko Date: Tue, 16 Sep 2014 16:26:44 +0300 Subject: [PATCH 1/5] implemented ability to split sidekiq_roles by count of sidekiq-processes --- lib/capistrano/tasks/capistrano2.rb | 61 ++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/lib/capistrano/tasks/capistrano2.rb b/lib/capistrano/tasks/capistrano2.rb index 015cf2a..676d355 100644 --- a/lib/capistrano/tasks/capistrano2.rb +++ b/lib/capistrano/tasks/capistrano2.rb @@ -17,6 +17,7 @@ _cset(:sidekiq_timeout) { 10 } _cset(:sidekiq_role) { :app } + _cset(:sidekiq_processes) { 1 } _cset(:sidekiq_options_per_process) { nil } @@ -28,9 +29,10 @@ end namespace :sidekiq do - def for_each_process(&block) - fetch(:sidekiq_processes).times do |idx| - if idx.zero? && fetch(:sidekiq_processes) <= 1 + def for_each_process(sidekiq_role, &block) + sidekiq_processes = fetch(:"#{ sidekiq_role }_processes") rescue 1 + sidekiq_processes.times do |idx| + if idx.zero? && sidekiq_processes <= 1 pid_file = fetch(:sidekiq_pid) else pid_file = fetch(:sidekiq_pid).gsub(/\.pid$/, "-#{idx}.pid") @@ -39,15 +41,30 @@ def for_each_process(&block) end end - def quiet_process(pid_file, idx) - run "if [ -d #{current_path} ] && [ -f #{pid_file} ] && kill -0 `cat #{pid_file}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} quiet #{pid_file} ; else echo 'Sidekiq is not running'; fi" + def for_each_role + sidekiq_roles = fetch(:sidekiq_role) + + sidekiq_roles = if sidekiq_roles.respond_to?(:to_ary) + sidekiq_roles.to_ary + else + [sidekiq_roles] + end + + sidekiq_roles.to_ary.each do |sidekiq_role| + puts "executing on ##{ sidekiq_role }" if sidekiq_roles.many? + yield(sidekiq_role) + end + end + + def quiet_process(pid_file, idx, sidekiq_role) + run "if [ -d #{current_path} ] && [ -f #{pid_file} ] && kill -0 `cat #{pid_file}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} quiet #{pid_file} ; else echo 'Sidekiq is not running'; fi", roles: sidekiq_role end - def stop_process(pid_file, idx) - run "if [ -d #{current_path} ] && [ -f #{pid_file} ] && kill -0 `cat #{pid_file}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} stop #{pid_file} #{fetch :sidekiq_timeout} ; else echo 'Sidekiq is not running' && if [ -f #{pid_file} ] ; then rm #{pid_file} ; fi ; fi" + def stop_process(pid_file, idx, sidekiq_role) + run "if [ -d #{current_path} ] && [ -f #{pid_file} ] && kill -0 `cat #{pid_file}`> /dev/null 2>&1; then cd #{current_path} && #{fetch(:sidekiqctl_cmd)} stop #{pid_file} #{fetch :sidekiq_timeout} ; else echo 'Sidekiq is not running'; fi", roles: sidekiq_role end - def start_process(pid_file, idx) + def start_process(pid_file, idx, sidekiq_role) args = [] args.push "--index #{idx}" args.push "--pidfile #{pid_file}" @@ -73,35 +90,43 @@ def start_process(pid_file, idx) args.push '--daemon' end - run "if [ -d #{current_path} ] && [ ! -f #{pid_file} ] || ! kill -0 `cat #{pid_file}` > /dev/null 2>&1; then cd #{current_path} ; #{fetch(:sidekiq_cmd)} #{args.compact.join(' ')} ; else echo 'Sidekiq is already running'; fi", pty: false + run "if [ -d #{current_path} ] && [ ! -f #{pid_file} ] || ! kill -0 `cat #{pid_file}` > /dev/null 2>&1; then cd #{current_path} ; #{fetch(:sidekiq_cmd)} #{args.compact.join(' ')} ; else echo 'Sidekiq is already running'; fi", pty: false, roles: sidekiq_role end desc 'Quiet sidekiq (stop accepting new work)' task :quiet, roles: lambda { fetch(:sidekiq_role) }, on_no_matching_servers: :continue do - for_each_process do |pid_file, idx| - quiet_process(pid_file, idx) + for_each_role do |sidekiq_role| + for_each_process(sidekiq_role) do |pid_file, idx| + quiet_process(pid_file, idx, sidekiq_role) + end end end desc 'Stop sidekiq' task :stop, roles: lambda { fetch(:sidekiq_role) }, on_no_matching_servers: :continue do - for_each_process do |pid_file, idx| - stop_process(pid_file, idx) + for_each_role do |sidekiq_role| + for_each_process(sidekiq_role) do |pid_file, idx| + stop_process(pid_file, idx, sidekiq_role) + end end end desc 'Start sidekiq' task :start, roles: lambda { fetch(:sidekiq_role) }, on_no_matching_servers: :continue do - for_each_process do |pid_file, idx| - start_process(pid_file, idx) + for_each_role do |sidekiq_role| + for_each_process(sidekiq_role) do |pid_file, idx| + start_process(pid_file, idx, sidekiq_role) + end end end desc 'Rolling-restart sidekiq' task :rolling_restart, roles: lambda { fetch(:sidekiq_role) }, on_no_matching_servers: :continue do - for_each_process do |pid_file, idx| - stop_process(pid_file, idx) - start_process(pid_file, idx) + for_each_role do |sidekiq_role| + for_each_process(sidekiq_role) do |pid_file, idx| + stop_process(pid_file, idx, sidekiq_role) + start_process(pid_file, idx, sidekiq_role) + end end end From c8b0f2ef4ffe73539dfaea21d7c317248f340818 Mon Sep 17 00:00:00 2001 From: Alexandr Yakubenko Date: Tue, 16 Sep 2014 17:10:15 +0300 Subject: [PATCH 2/5] fixed Array#many? missing --- lib/capistrano/tasks/capistrano2.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/capistrano/tasks/capistrano2.rb b/lib/capistrano/tasks/capistrano2.rb index 676d355..595f728 100644 --- a/lib/capistrano/tasks/capistrano2.rb +++ b/lib/capistrano/tasks/capistrano2.rb @@ -51,7 +51,7 @@ def for_each_role end sidekiq_roles.to_ary.each do |sidekiq_role| - puts "executing on ##{ sidekiq_role }" if sidekiq_roles.many? + puts "executing on ##{ sidekiq_role }" if sidekiq_roles.size > 1 yield(sidekiq_role) end end From 5628d3ba2e31ad0414d5980a2df74ae9b8733235 Mon Sep 17 00:00:00 2001 From: Alexandr Yakubenko Date: Tue, 31 Mar 2015 12:39:59 +0300 Subject: [PATCH 3/5] implemented ability to split sidekiq_roles by count of sidekiq-processes for cap3 --- lib/capistrano/tasks/sidekiq.cap | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/capistrano/tasks/sidekiq.cap b/lib/capistrano/tasks/sidekiq.cap index 37a84c6..bf5147f 100644 --- a/lib/capistrano/tasks/sidekiq.cap +++ b/lib/capistrano/tasks/sidekiq.cap @@ -40,12 +40,17 @@ namespace :sidekiq do def processes_pids pids = [] - fetch(:sidekiq_processes).times do |idx| - pids.push (idx.zero? && fetch(:sidekiq_processes) <= 1) ? - fetch(:sidekiq_pid) : - fetch(:sidekiq_pid).gsub(/\.pid$/, "-#{idx}.pid") - + sidekiq_roles = Array(fetch(:sidekiq_role)) + sidekiq_roles.each do |role| + next unless host.roles.include?(role) + processes = fetch(:"#{ role }_processes") || fetch(:sidekiq_processes) + processes.times do |idx| + pids.push (idx.zero? && processes <= 1) ? + fetch(:sidekiq_pid) : + fetch(:sidekiq_pid).gsub(/\.pid$/, "-#{idx}.pid") + end end + pids end From 806cdb41ab99d703bf12b81c95655fb52d7a4187 Mon Sep 17 00:00:00 2001 From: Alexandr Yakubenko Date: Tue, 31 Mar 2015 13:11:07 +0300 Subject: [PATCH 4/5] updated README for count-of-processes-per-host --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 98eb3ef..f491c00 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,18 @@ and `low`: set :sidekiq_options_per_process, ["--queue high", "--queue default --queue low"] ``` +## Different number of processes per host + +You can configure how many processes you want to run on each host next way: + +```ruby +set :sidekiq_role, [:sidekiq_small, :sidekiq_big] +set :sidekiq_small_processes, 1 +set :sidekiq_big_processes, 4 +server 'example-small.com', roles: [:sidekiq_small] +server 'example-big.com', roles: [:sidekiq_big] +``` + ## Customizing the monit sidekiq templates If you need change some config in redactor, you can From 5b2e9f928cf441b23ebf5a78aad90b74d3512420 Mon Sep 17 00:00:00 2001 From: Alexandr Yakubenko Date: Tue, 31 Mar 2015 22:53:22 +0300 Subject: [PATCH 5/5] Version bump, added entry to changelog about improvements of multiprocessing --- README.md | 3 +++ lib/capistrano/sidekiq/version.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f491c00..c417604 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ If you need change some config in redactor, you can ``` ## Changelog +- 0.5.3: Custom count of processes per each host - 0.5.0: Multiple processes @mrsimo - 0.3.9: Restore daemon flag from Monit template - 0.3.8: @@ -120,6 +121,8 @@ If you need change some config in redactor, you can - [Fabien Penso] (https://github.com/penso) - [Alex Dunae] (https://github.com/alexdunae) - [andreygerasimchuk] (https://github.com/andreygerasimchuk) +- [Saicheg] (https://github.com/Saicheg) +- [Alex Yakubenko] (https://github.com/alexyakubenko) ## Contributing diff --git a/lib/capistrano/sidekiq/version.rb b/lib/capistrano/sidekiq/version.rb index 5a1d109..3064c77 100644 --- a/lib/capistrano/sidekiq/version.rb +++ b/lib/capistrano/sidekiq/version.rb @@ -1,5 +1,5 @@ module Capistrano module Sidekiq - VERSION = '0.5.2' + VERSION = '0.5.3' end end