Refactor demonizer in prep for unicorn forking

Upgrade sidekiq

Gemfile

@@ -140,7 +140,10 @@ gem 'rinku'
 gem 'sanitize'
 gem 'sass'
 gem 'sidekiq'
-gem 'sidekiq-failures'
+
+# https://github.com/mhfs/sidekiq-failures/issues/72
+gem 'sidekiq-failures-discourse', require: 'sidekiq-failures'
+
 gem 'sinatra', require: nil
 gem 'slim'  # required for sidekiq-web
 

Gemfile.lock

@@ -62,7 +62,7 @@ GEM
       timers (~> 1.1.0)
     certified (0.1.1)
     coderay (1.1.0)
-    connection_pool (1.2.0)
+    connection_pool (2.0.0)
     daemons (1.1.9)
     debug_inspector (0.0.2)
     diff-lcs (1.2.5)
@@ -324,7 +324,7 @@ GEM
       json
       redis (>= 3.0.6)
       redis-namespace (>= 1.3.1)
-    sidekiq-failures (0.3.0)
+    sidekiq-failures-discourse (0.3.0)
       sidekiq (>= 2.14.0)
     simple-rss (1.3.1)
     simplecov (0.8.2)
@@ -464,7 +464,7 @@ DEPENDENCIES
   seed-fu-discourse
   shoulda
   sidekiq
-  sidekiq-failures
+  sidekiq-failures-discourse
   simple-rss
   simplecov
   sinatra

lib/demon/base.rb

@@ -3,7 +3,7 @@ module Demon; end
 # intelligent fork based demonizer
 class Demon::Base
 
-  def self.start(count)
+  def self.start(count=1)
     @demons ||= {}
     count.times do |i|
       (@demons["#{prefix}_#{i}"] ||= new(i)).start
@@ -17,11 +17,17 @@ class Demon::Base
     end
   end
 
+  def self.ensure_running
+    @demons.values.each do |demon|
+      demon.ensure_running
+    end
+  end
+
   def initialize(index)
     @index = index
     @pid = nil
     @parent_pid = Process.pid
-    @monitor = nil
+    @started = false
   end
 
   def pid_file
@@ -29,29 +35,47 @@ class Demon::Base
   end
 
   def stop
-    if @monitor
+    @started = false
-      @monitor.kill
-      @monitor.join
-      @monitor = nil
-    end
-
     if @pid
       Process.kill("HUP",@pid)
       @pid = nil
     end
   end
 
+  def ensure_running
+    return unless @started
+
+    if !@pid
+      @started = false
+      start
+      return
+    end
+
+    dead = Process.waitpid(@pid, Process::WNOHANG) rescue -1
+    if dead
+      STDERR.puts "Detected dead worker #{@pid}, restarting..."
+      @pid = nil
+      @started = false
+      start
+    end
+  end
+
   def start
+    return if @pid || @started
+
     if existing = already_running?
       # should not happen ... so kill violently
+      STDERR.puts "Attempting to kill pid #{existing}"
       Process.kill("TERM",existing)
     end
 
-    return if @pid
+    @started = true
+    run
+  end
 
+  def run
     if @pid = fork
       write_pid_file
-      monitor_child
       return
     end
 
@@ -73,19 +97,6 @@ class Demon::Base
 
   private
 
-  def monitor_child
-    @monitor ||= Thread.new do
-      while true
-        sleep 5
-        unless alive?(@pid)
-          STDERR.puts "#{@pid} died, restarting the process"
-          @pid = nil
-          start
-        end
-      end
-    end
-  end
-
   def write_pid_file
     FileUtils.mkdir_p(Rails.root + "tmp/pids")
     File.open(pid_file,'w') do |f|
@@ -110,13 +121,21 @@ class Demon::Base
 
   def alive?(pid)
     begin
-      Process.getpgid(pid)
+      Process.kill(0, pid)
       true
-    rescue Errno::ESRCH
+    rescue
       false
     end
   end
 
+  def suppress_stdout
+    true
+  end
+
+  def suppress_stderr
+    true
+  end
+
   def establish_app
     Discourse.after_fork
 
@@ -129,8 +148,8 @@ class Demon::Base
     end
 
     # keep stuff simple for now
-    $stdout.reopen("/dev/null", "w")
+    $stdout.reopen("/dev/null", "w") if suppress_stdout
-    $stderr.reopen("/dev/null", "w")
+    $stderr.reopen("/dev/null", "w") if suppress_stderr
   end
 
   def after_fork

lib/demon/sidekiq.rb

@@ -8,12 +8,21 @@ class Demon::Sidekiq < Demon::Base
 
   private
 
+  def suppress_stdout
+    false
+  end
+
+  def suppress_stderr
+    false
+  end
+
   def after_fork
+    STDERR.puts "Loading Sidekiq in process id #{Process.pid}"
     require 'sidekiq/cli'
-    # Reload initializer cause it needs to run after sidekiq/cli was required
-    load Rails.root + "config/initializers/sidekiq.rb"
     cli = Sidekiq::CLI.instance
     cli.parse([])
+
+    load Rails.root + "config/initializers/sidekiq.rb"
     cli.run
   rescue => e
     STDERR.puts e.message

lib/discourse.rb

@@ -4,6 +4,7 @@ require_dependency 'auth/default_current_user_provider'
 
 module Discourse
 
+  require 'sidekiq/exception_handler'
   class SidekiqExceptionHandler
     extend Sidekiq::ExceptionHandler
   end

lib/scheduler/manager.rb

@@ -229,9 +229,17 @@ module Scheduler
 
 
     def self.discover_schedules
+      # hack for developemnt reloader is crazytown
+      # multiple classes with same name can be in
+      # object space
+      unique = Set.new
       schedules = []
       ObjectSpace.each_object(Scheduler::Schedule) do |schedule|
-        schedules << schedule if schedule.scheduled?
+        if schedule.scheduled?
+          next if unique.include?(schedule.to_s)
+          schedules << schedule
+          unique << schedule.to_s
+        end
       end
       schedules
     end

lib/scheduler/schedule_info.rb

@@ -10,7 +10,6 @@ module Scheduler
       @klass = klass
       @manager = manager
 
-      key = Manager.schedule_key(klass)
       data = nil
 
       if data = $redis.get(key)
@@ -85,7 +84,6 @@ module Scheduler
     end
 
     def write!
-      key = Manager.schedule_key(@klass)
       clear!
       redis.set key, {
         next_run: @next_run,
@@ -94,6 +92,7 @@ module Scheduler
         prev_result: @prev_result,
         current_owner: @current_owner
       }.to_json
+
       redis.zadd Manager.queue_key, @next_run , @klass
     end
 
@@ -112,9 +111,8 @@ module Scheduler
 
     private
     def clear!
-      key = Manager.schedule_key(@klass)
       redis.del key
-      redis.zrem Manager.queue_key, key
+      redis.zrem Manager.queue_key, @klass
     end
 
   end