From c6ae4139a21ceb4c010d0a20a75d5b9a80b09df9 Mon Sep 17 00:00:00 2001 From: simon lehericey Date: Wed, 11 Oct 2023 17:28:58 +0200 Subject: [PATCH 1/2] add task to remove dol orphan --- ..._remove_orphan_dossier_operation_logs.rake | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 lib/tasks/deployment/20231011144554_remove_orphan_dossier_operation_logs.rake diff --git a/lib/tasks/deployment/20231011144554_remove_orphan_dossier_operation_logs.rake b/lib/tasks/deployment/20231011144554_remove_orphan_dossier_operation_logs.rake new file mode 100644 index 000000000..0b94628ac --- /dev/null +++ b/lib/tasks/deployment/20231011144554_remove_orphan_dossier_operation_logs.rake @@ -0,0 +1,40 @@ +namespace :after_party do + desc 'Deployment task: remove_orphan_dossier_operation_logs' + task remove_orphan_dossier_operation_logs: :environment do + puts "Running deploy task 'remove_orphan_dossier_operation_logs'" + + job_limit = 200_000 + + not_deletion_orphans = DossierOperationLog + .not_deletion + .where.missing(:dossier) + .limit(job_limit) + + batch_size = 1_000 + + rake_puts "Suppression des dols avec operation != supprimer" + + progress = ProgressReport.new(not_deletion_orphans.count) + + not_deletion_orphans.in_batches(of: batch_size) do |b| + b.destroy_all + progress.inc(batch_size) + end + + rake_puts "Supression des serialized des dols avec operation == supprimer" + + deletion_orphans = DossierOperationLog + .supprimer + .limit(job_limit) + + progress = ProgressReport.new(deletion_orphans.count) + + deletion_orphans.find_each(batch_size:) do |dossier_operation_log| + dossier_operation_log.serialized.purge_later + progress.inc + end + + AfterParty::TaskRecord + .create version: AfterParty::TaskRecorder.new(__FILE__).timestamp + end +end From e00054a6eab3174ede4b081f9b9f07b2757ebb7e Mon Sep 17 00:00:00 2001 From: simon lehericey Date: Mon, 13 Nov 2023 10:52:45 +0100 Subject: [PATCH 2/2] first test for sidekiq --- config/initializers/transition_to_sidekiq.rb | 9 +++++++++ ...54_remove_orphan_dossier_operation_logs.rake | 17 +++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 config/initializers/transition_to_sidekiq.rb diff --git a/config/initializers/transition_to_sidekiq.rb b/config/initializers/transition_to_sidekiq.rb new file mode 100644 index 000000000..e64c9aa4f --- /dev/null +++ b/config/initializers/transition_to_sidekiq.rb @@ -0,0 +1,9 @@ +sidekiq_enabled = ENV.has_key?('REDIS_SIDEKIQ_SENTINELS') || ENV.has_key?('REDIS_URL') + +if Rails.env.production? && sidekiq_enabled + ActiveSupport.on_load(:after_initialize) do + class ActiveStorage::PurgeJob < ActiveStorage::BaseJob + self.queue_adapter = :sidekiq + end + end +end diff --git a/lib/tasks/deployment/20231011144554_remove_orphan_dossier_operation_logs.rake b/lib/tasks/deployment/20231011144554_remove_orphan_dossier_operation_logs.rake index 0b94628ac..c1d7b983e 100644 --- a/lib/tasks/deployment/20231011144554_remove_orphan_dossier_operation_logs.rake +++ b/lib/tasks/deployment/20231011144554_remove_orphan_dossier_operation_logs.rake @@ -1,14 +1,17 @@ namespace :after_party do - desc 'Deployment task: remove_orphan_dossier_operation_logs' + desc 'Deployment task: remove_orphan_dossier_operation_logs. You can add JOB_LIMIT env var to limit the load and manually relaunch the process if needed.' task remove_orphan_dossier_operation_logs: :environment do puts "Running deploy task 'remove_orphan_dossier_operation_logs'" - job_limit = 200_000 + job_limit = ENV['JOB_LIMIT'] not_deletion_orphans = DossierOperationLog .not_deletion .where.missing(:dossier) - .limit(job_limit) + + if job_limit.present? + not_deletion_orphans = not_deletion_orphans.limit(job_limit) + end batch_size = 1_000 @@ -23,9 +26,11 @@ namespace :after_party do rake_puts "Supression des serialized des dols avec operation == supprimer" - deletion_orphans = DossierOperationLog - .supprimer - .limit(job_limit) + deletion_orphans = DossierOperationLog.supprimer + + if job_limit.present? + deletion_orphans = deletion_orphans.limit(job_limit) + end progress = ProgressReport.new(deletion_orphans.count)