[deploy] Bug Fix:Increase look back time for Cron jobs to 120 seconds (#9797)

config/initializers/sidekiq.rb

@@ -1,3 +1,21 @@
+module Sidekiq
+  module Cron
+    class Job
+      def not_past_scheduled_time?(current_time)
+        last_cron_time = parsed_cron.previous_time(current_time).utc
+        # @mstruve/@sre: method monkey patched to increase time we look back for
+        # unrun scheduled jobs from 60 to 120. Heroku takes 60 seconds to restart
+        # sidekiq workers, we need to ensure any job scheduled during that down time
+        # is run once Sidekiq boots back up
+        # https://github.com/ondrejbartas/sidekiq-cron/blob/074a87546f16122c1f508bb2805b9951588f2510/lib/sidekiq/cron/job.rb#L593
+        return false if (current_time.to_i - last_cron_time.to_i) > (ENV["CRON_LOOKBACK_TIME"] || 60).to_i
+
+        true
+      end
+    end
+  end
+end
+
 Rails.application.config.to_prepare do
   Dir.glob(Rails.root.join("lib/sidekiq/*.rb")).sort.each do |filename|
     require_dependency filename
@@ -6,9 +24,15 @@ end
 
 Sidekiq.configure_server do |config|
   schedule_file = "config/schedule.yml"
+  # @mstruve/@sre: sidekiq-cron still uses the removed poll_interval
+  # to determine how often to poll for jobs so we should manually set it
+  # https://github.com/ondrejbartas/sidekiq-cron/issues/254
+  # Sidekiq default is 5, we don't need it quite that often but would like it more than
+  # every 30 seconds which the gem defaults to
+  Sidekiq.options[:poll_interval] = 10
 
   if File.exist?(schedule_file)
-    Sidekiq::Cron::Job.load_from_hash(YAML.load_file(schedule_file))
+    Sidekiq::Cron::Job.load_from_hash!(YAML.load_file(schedule_file))
   end
 
   sidekiq_url = ApplicationConfig["REDIS_SIDEKIQ_URL"] || ApplicationConfig["REDIS_URL"]