Merge pull request #4963 from tchak/schedule-cron-jobs

Schedule cron jobs

README.md

@@ -65,22 +65,9 @@ L'application tourne à l'adresse `http://localhost:3000`.
 
 En local, un utilisateur de test est créé automatiquement, avec les identifiants `test@exemple.fr`/`this is a very complicated password !`. (voir [db/seeds.rb](https://github.com/betagouv/demarches-simplifiees.fr/blob/dev/db/seeds.rb))
 
-### Programmation des jobs
+### Programmation des tâches récurrentes
 
-    AutoArchiveProcedureJob.set(cron: "* * * * *").perform_later
+    rails jobs:schedule
-    WeeklyOverviewJob.set(cron: "0 7 * * MON").perform_later
-    DeclarativeProceduresJob.set(cron: "* * * * *").perform_later
-    UpdateAdministrateurUsageStatisticsJob.set(cron: "0 10 * * *").perform_later
-    FindDubiousProceduresJob.set(cron: "0 0 * * *").perform_later
-    Administrateurs::ActivateBeforeExpirationJob.set(cron: "0 8 * * *").perform_later
-    WarnExpiringDossiersJob.set(cron: "0 0 1 * *").perform_later
-    InstructeurEmailNotificationJob.set(cron: "0 10 * * MON-FRI").perform_later
-    PurgeUnattachedBlobsJob.set(cron: "0 0 * * *").perform_later
-    OperationsSignatureJob.set(cron: "0 6 * * *").perform_later
-    ExpiredDossiersDeletionJob.set(cron: "0 7 * * *").perform_later
-    PurgeStaleExportsJob.set(cron: "*/5 * * * *").perform_later
-    NotifyDraftNotSubmittedJob.set(cron: "0 7 * * *").perform_later
-    DiscardedDossiersDeletionJob.set(cron: "0 7 * * *").perform_later
 
 ### Voir les emails envoyés en local
 

app/jobs/administrateur_activate_before_expiration_job.rb

@@ -1,5 +1,5 @@
-class Administrateurs::ActivateBeforeExpirationJob < ApplicationJob
+class AdministrateurActivateBeforeExpirationJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 8 * * *"
 
   def perform(*args)
     Administrateur

app/jobs/auto_archive_procedure_job.rb

@@ -1,5 +1,5 @@
-class AutoArchiveProcedureJob < ApplicationJob
+class AutoArchiveProcedureJob < CronJob
-  queue_as :cron
+  self.cron_expression = "* * * * *"
 
   def perform(*args)
     Procedure.publiees.where("auto_archive_on <= ?", Time.zone.today).each do |procedure|

app/jobs/auto_receive_dossiers_for_procedure_job.rb

@@ -1,22 +0,0 @@
-class AutoReceiveDossiersForProcedureJob < ApplicationJob
-  queue_as :cron
-
-  def perform(procedure_id, state)
-    procedure = Procedure.find(procedure_id)
-
-    case state
-    when Dossier.states.fetch(:en_instruction)
-      procedure
-        .dossiers
-        .state_en_construction
-        .find_each(&:passer_automatiquement_en_instruction!)
-    when Dossier.states.fetch(:accepte)
-      procedure
-        .dossiers
-        .state_en_construction
-        .find_each(&:accepter_automatiquement!)
-    else
-      raise "Receiving Procedure##{procedure_id} in invalid state \"#{state}\""
-    end
-  end
-end

app/jobs/cron_job.rb

@@ -0,0 +1,29 @@
+class CronJob < ApplicationJob
+  queue_as :cron
+  class_attribute :cron_expression
+
+  class << self
+    def schedule
+      remove if cron_expression_changed?
+      set(cron: cron_expression).perform_later if !scheduled?
+    end
+
+    def remove
+      delayed_job.destroy if scheduled?
+    end
+
+    def scheduled?
+      delayed_job.present?
+    end
+
+    def cron_expression_changed?
+      scheduled? && delayed_job.cron != cron_expression
+    end
+
+    def delayed_job
+      Delayed::Job
+        .where('handler LIKE ?', "%job_class: #{name}%")
+        .first
+    end
+  end
+end

app/jobs/declarative_procedures_job.rb

@@ -1,5 +1,5 @@
-class DeclarativeProceduresJob < ApplicationJob
+class DeclarativeProceduresJob < CronJob
-  queue_as :cron
+  self.cron_expression = "* * * * *"
 
   def perform(*args)
     Procedure.declarative.find_each(&:process_dossiers!)

app/jobs/discarded_dossiers_deletion_job.rb

@@ -1,5 +1,5 @@
-class DiscardedDossiersDeletionJob < ApplicationJob
+class DiscardedDossiersDeletionJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 7 * * *"
 
   def perform(*args)
     Dossier.discarded_brouillon_expired.destroy_all

app/jobs/etablissement_update_job.rb

@@ -1,6 +1,4 @@
 class EtablissementUpdateJob < ApplicationJob
-  queue_as :default
-
   def perform(dossier, siret)
     begin
       etablissement_attributes = ApiEntrepriseService.get_etablissement_params_for_siret(siret, dossier.procedure_id)

app/jobs/expired_dossiers_deletion_job.rb

@@ -1,5 +1,5 @@
-class ExpiredDossiersDeletionJob < ApplicationJob
+class ExpiredDossiersDeletionJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 7 * * *"
 
   def perform(*args)
     ExpiredDossiersDeletionService.process_expired_dossiers_brouillon

app/jobs/find_dubious_procedures_job.rb

@@ -1,5 +1,5 @@
-class FindDubiousProceduresJob < ApplicationJob
+class FindDubiousProceduresJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 0 * * *"
 
   FORBIDDEN_KEYWORDS = [
     'NIR', 'NIRPP', 'race', 'religion',

app/jobs/instructeur_email_notification_job.rb

@@ -1,5 +1,5 @@
-class InstructeurEmailNotificationJob < ApplicationJob
+class InstructeurEmailNotificationJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 10 * * MON-FRI"
 
   def perform(*args)
     NotificationService.send_instructeur_email_notification

app/jobs/notify_draft_not_submitted_job.rb

@@ -1,5 +1,5 @@
-class NotifyDraftNotSubmittedJob < ApplicationJob
+class NotifyDraftNotSubmittedJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 7 * * *"
 
   def perform(*args)
     Dossier.notify_draft_not_submitted

app/jobs/operations_signature_job.rb

@@ -1,5 +1,5 @@
-class OperationsSignatureJob < ApplicationJob
+class OperationsSignatureJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 6 * * *"
 
   def perform(*args)
     last_midnight = Time.zone.today.beginning_of_day

app/jobs/purge_stale_exports_job.rb

@@ -1,5 +1,5 @@
-class PurgeStaleExportsJob < ApplicationJob
+class PurgeStaleExportsJob < CronJob
-  queue_as :cron
+  self.cron_expression = "*/5 * * * *"
 
   def perform
     Export.stale.destroy_all

app/jobs/purge_unattached_blobs_job.rb

@@ -1,5 +1,5 @@
-class PurgeUnattachedBlobsJob < ApplicationJob
+class PurgeUnattachedBlobsJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 0 * * *"
 
   def perform(*args)
     ActiveStorage::Blob.unattached

app/jobs/update_administrateur_usage_statistics_job.rb

@@ -1,5 +1,5 @@
-class UpdateAdministrateurUsageStatisticsJob < ApplicationJob
+class UpdateAdministrateurUsageStatisticsJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 10 * * *"
 
   def perform
     AdministrateurUsageStatisticsService.new.update_administrateurs

app/jobs/warn_expiring_dossiers_job.rb

@@ -1,5 +1,5 @@
-class WarnExpiringDossiersJob < ApplicationJob
+class WarnExpiringDossiersJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 0 1 * *"
 
   def perform(*args)
     expiring, expired = Dossier

app/jobs/web_hook_job.rb

@@ -1,6 +1,4 @@
 class WebHookJob < ApplicationJob
-  queue_as :default
-
   TIMEOUT = 10
 
   def perform(procedure, dossier)

app/jobs/weekly_overview_job.rb

@@ -1,5 +1,5 @@
-class WeeklyOverviewJob < ApplicationJob
+class WeeklyOverviewJob < CronJob
-  queue_as :cron
+  self.cron_expression = "0 7 * * MON"
 
   def perform(*args)
     # Feature flipped to avoid mails in staging due to unprocessed dossier

config/deploy.rb

@@ -64,6 +64,16 @@ namespace :after_party do
   end
 end
 
+namespace :jobs_schedule do
+  desc "Run jobs_schedule tasks."
+  task :run do
+    command %{
+      echo "-----> Running jobs_schedule"
+      #{echo_cmd %[bundle exec rake jobs:schedule]}
+    }
+  end
+end
+
 namespace :service do
   desc "Restart puma"
   task :restart_puma do
@@ -123,4 +133,5 @@ task :post_deploy do
   command 'cd /home/ds/current'
 
   invoke :'after_party:run'
+  invoke :'jobs_schedule:run'
 end

lib/tasks/jobs.rake

@@ -0,0 +1,8 @@
+namespace :jobs do
+  desc 'Schedule all cron jobs'
+  task schedule: :environment do
+    glob = Rails.root.join('app', 'jobs', '**', '*_job.rb')
+    Dir.glob(glob).each { |f| require f }
+    CronJob.subclasses.each(&:schedule)
+  end
+end

spec/jobs/administrateur_activate_before_expiration_job_spec.rb

@@ -1,12 +1,12 @@
 require 'rails_helper'
 
-RSpec.describe Administrateurs::ActivateBeforeExpirationJob, type: :job do
+RSpec.describe AdministrateurActivateBeforeExpirationJob, type: :job do
   describe 'perform' do
     let(:administrateur) { create(:administrateur) }
     let(:user) { administrateur.user }
     let(:mailer_double) { double('mailer', deliver_later: true) }
 
-    subject { Administrateurs::ActivateBeforeExpirationJob.perform_now }
+    subject { AdministrateurActivateBeforeExpirationJob.perform_now }
 
     before do
       Timecop.freeze(Time.zone.local(2018, 03, 20))

spec/jobs/auto_receive_dossiers_for_procedure_job_spec.rb

@@ -1,81 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe AutoReceiveDossiersForProcedureJob, type: :job do
-  describe "perform" do
-    let(:date) { Time.utc(2017, 9, 1, 10, 5, 0) }
-    let(:instruction_date) { date + 120 }
-
-    let(:procedure) { create(:procedure, :published, :with_instructeur) }
-    let(:nouveau_dossier1) { create(:dossier, :en_construction, procedure: procedure) }
-    let(:nouveau_dossier2) { create(:dossier, :en_construction, procedure: procedure) }
-    let(:dossier_recu) { create(:dossier, :en_instruction, procedure: procedure) }
-    let(:dossier_brouillon) { create(:dossier, procedure: procedure) }
-
-    before do
-      Timecop.freeze(date)
-      dossiers = [
-        nouveau_dossier1,
-        nouveau_dossier2,
-        dossier_recu,
-        dossier_brouillon
-      ]
-
-      create(:attestation_template, procedure: procedure)
-      AutoReceiveDossiersForProcedureJob.new.perform(procedure.id, state)
-
-      dossiers.each(&:reload)
-    end
-
-    after { Timecop.return }
-
-    context "with some dossiers" do
-      context "en_construction" do
-        let(:state) { Dossier.states.fetch(:en_instruction) }
-        let(:last_operation) { nouveau_dossier1.dossier_operation_logs.last }
-
-        it {
-          expect(nouveau_dossier1.en_instruction?).to be true
-          expect(nouveau_dossier1.en_instruction_at).to eq(date)
-          expect(last_operation.operation).to eq('passer_en_instruction')
-          expect(last_operation.automatic_operation?).to be_truthy
-
-          expect(nouveau_dossier2.en_instruction?).to be true
-          expect(nouveau_dossier2.en_instruction_at).to eq(date)
-
-          expect(dossier_recu.en_instruction?).to be true
-          expect(dossier_recu.en_instruction_at).to eq(instruction_date)
-
-          expect(dossier_brouillon.brouillon?).to be true
-          expect(dossier_brouillon.en_instruction_at).to eq(nil)
-        }
-      end
-
-      context "accepte" do
-        let(:state) { Dossier.states.fetch(:accepte) }
-        let(:last_operation) { nouveau_dossier1.dossier_operation_logs.last }
-
-        it {
-          expect(nouveau_dossier1.accepte?).to be true
-          expect(nouveau_dossier1.en_instruction_at).to eq(date)
-          expect(nouveau_dossier1.processed_at).to eq(date)
-          expect(nouveau_dossier1.attestation).to be_present
-          expect(last_operation.operation).to eq('accepter')
-          expect(last_operation.automatic_operation?).to be_truthy
-
-          expect(nouveau_dossier2.accepte?).to be true
-          expect(nouveau_dossier2.en_instruction_at).to eq(date)
-          expect(nouveau_dossier2.processed_at).to eq(date)
-          expect(nouveau_dossier2.attestation).to be_present
-
-          expect(dossier_recu.en_instruction?).to be true
-          expect(dossier_recu.en_instruction_at).to eq(instruction_date)
-          expect(dossier_recu.processed_at).to eq(nil)
-
-          expect(dossier_brouillon.brouillon?).to be true
-          expect(dossier_brouillon.en_instruction_at).to eq(nil)
-          expect(dossier_brouillon.processed_at).to eq(nil)
-        }
-      end
-    end
-  end
-end