From bd788ec6bf499240f6fe7069b65da3d108308c2b Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Tue, 26 Jul 2022 19:04:25 +0200 Subject: [PATCH 1/3] fix(manager/archives): status => job_status Closes #7614 --- app/dashboards/archive_dashboard.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/dashboards/archive_dashboard.rb b/app/dashboards/archive_dashboard.rb index a18e8790b..51afb65d5 100644 --- a/app/dashboards/archive_dashboard.rb +++ b/app/dashboards/archive_dashboard.rb @@ -11,7 +11,7 @@ class ArchiveDashboard < Administrate::BaseDashboard id: Field::Number, created_at: Field::DateTime, updated_at: Field::DateTime, - status: Field::String, + job_status: Field::String, file: AttachmentField }.freeze @@ -24,7 +24,7 @@ class ArchiveDashboard < Administrate::BaseDashboard :id, :created_at, :updated_at, - :status, + :job_status, :file ].freeze @@ -34,6 +34,6 @@ class ArchiveDashboard < Administrate::BaseDashboard :id, :created_at, :updated_at, - :status + :job_status ].freeze end From f1957e51f817aa64eefe7f0e0e92aa3c72ef417b Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Wed, 27 Jul 2022 10:23:21 +0200 Subject: [PATCH 2/3] fix(exports): purge stuck (pending) exports, not generated or failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avec l'autre PR, ça reproduit le comportement d'avant, c'est à dire que quoiqu'il arrive un export est purgé : - soit 16h après sa génération (on a 16h pour le télécharger) - soit 12h après sa création, et qu'il est bloqué Auparavant, tous les exports étaient purgés au bout de 3h quelle que soit le statut. --- app/jobs/cron/purge_stale_exports_job.rb | 1 + .../transient_models_with_purgeable_job_concern.rb | 5 +++++ app/models/export.rb | 1 + spec/models/export_spec.rb | 10 ++++++++++ 4 files changed, 17 insertions(+) diff --git a/app/jobs/cron/purge_stale_exports_job.rb b/app/jobs/cron/purge_stale_exports_job.rb index 2945e26a6..20462ce0e 100644 --- a/app/jobs/cron/purge_stale_exports_job.rb +++ b/app/jobs/cron/purge_stale_exports_job.rb @@ -3,5 +3,6 @@ class Cron::PurgeStaleExportsJob < Cron::CronJob def perform Export.stale(Export::MAX_DUREE_CONSERVATION_EXPORT).destroy_all + Export.stuck(Export::MAX_DUREE_GENERATION).destroy_all end end diff --git a/app/models/concerns/transient_models_with_purgeable_job_concern.rb b/app/models/concerns/transient_models_with_purgeable_job_concern.rb index 8edc63a80..2a95803cd 100644 --- a/app/models/concerns/transient_models_with_purgeable_job_concern.rb +++ b/app/models/concerns/transient_models_with_purgeable_job_concern.rb @@ -35,6 +35,11 @@ module TransientModelsWithPurgeableJobConcern .where('updated_at < ?', (Time.zone.now - duration)) } + scope :stuck, lambda { |duration| + where(job_status: [job_statuses.fetch(:pending)]) + .where('updated_at < ?', (Time.zone.now - duration)) + } + def available? generated? end diff --git a/app/models/export.rb b/app/models/export.rb index 990fa2f42..a4ad0e832 100644 --- a/app/models/export.rb +++ b/app/models/export.rb @@ -17,6 +17,7 @@ class Export < ApplicationRecord include TransientModelsWithPurgeableJobConcern MAX_DUREE_CONSERVATION_EXPORT = 16.hours + MAX_DUREE_GENERATION = 12.hours enum format: { csv: 'csv', diff --git a/spec/models/export_spec.rb b/spec/models/export_spec.rb index c64c51f4f..d06371042 100644 --- a/spec/models/export_spec.rb +++ b/spec/models/export_spec.rb @@ -31,6 +31,16 @@ RSpec.describe Export, type: :model do it { expect(Export.stale(Export::MAX_DUREE_CONSERVATION_EXPORT)).to match_array([stale_export_generated, stale_export_failed]) } end + describe '.stuck' do + let!(:export) { create(:export) } + let(:stuck_date) { Time.zone.now() - (Export::MAX_DUREE_GENERATION + 1.minute) } + let!(:stale_export_generated) { create(:export, :generated, updated_at: stuck_date) } + let!(:stale_export_failed) { create(:export, :failed, updated_at: stuck_date) } + let!(:stale_export_pending) { create(:export, :pending, updated_at: stuck_date) } + + it { expect(Export.stuck(Export::MAX_DUREE_GENERATION)).to match_array([stale_export_pending]) } + end + describe '.destroy' do let!(:groupe_instructeur) { create(:groupe_instructeur) } let!(:export) { create(:export, groupe_instructeurs: [groupe_instructeur]) } From 197fa54ea46a66d2cccba8d25223b4e872be9a04 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Wed, 27 Jul 2022 10:32:54 +0200 Subject: [PATCH 3/3] fix(archives): purge stuck pending archives Cf 0cb096b3d6440a4da05ca2902ed6fa72a93e6579 --- app/jobs/cron/purge_stale_archives_job.rb | 1 + app/models/archive.rb | 1 + spec/models/archive_spec.rb | 28 +++++++++++++++-------- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/app/jobs/cron/purge_stale_archives_job.rb b/app/jobs/cron/purge_stale_archives_job.rb index 124c7935e..196002d69 100644 --- a/app/jobs/cron/purge_stale_archives_job.rb +++ b/app/jobs/cron/purge_stale_archives_job.rb @@ -3,5 +3,6 @@ class Cron::PurgeStaleArchivesJob < Cron::CronJob def perform Archive.stale(Archive::RETENTION_DURATION).destroy_all + Archive.stuck(Archive::MAX_DUREE_GENERATION).destroy_all end end diff --git a/app/models/archive.rb b/app/models/archive.rb index d4081d8f4..e6230b8b7 100644 --- a/app/models/archive.rb +++ b/app/models/archive.rb @@ -14,6 +14,7 @@ class Archive < ApplicationRecord include TransientModelsWithPurgeableJobConcern RETENTION_DURATION = 4.days + MAX_DUREE_GENERATION = 24.hours MAX_SIZE = 100.gigabytes has_and_belongs_to_many :groupe_instructeurs diff --git a/spec/models/archive_spec.rb b/spec/models/archive_spec.rb index c69ecdc5b..ee5cc6dd7 100644 --- a/spec/models/archive_spec.rb +++ b/spec/models/archive_spec.rb @@ -4,26 +4,34 @@ describe Archive do before { Timecop.freeze(Time.zone.now) } after { Timecop.return } - let(:archive) { create(:archive, job_status: :pending) } + let!(:archive) { create(:archive, job_status: :pending) } describe 'scopes' do describe 'staled' do - let(:recent_archive) { create(:archive, job_status: :pending) } - let(:staled_archive_still_pending) { create(:archive, job_status: :pending, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) } - let(:staled_archive_still_failed) { create(:archive, job_status: :failed, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) } - let(:staled_archive_still_generated) { create(:archive, job_status: :generated, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) } + let!(:recent_archive) { create(:archive, job_status: :pending) } + let!(:staled_archive_still_pending) { create(:archive, job_status: :pending, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) } + let!(:staled_archive_still_failed) { create(:archive, job_status: :failed, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) } + let!(:staled_archive_still_generated) { create(:archive, job_status: :generated, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) } subject do - archive - recent_archive - staled_archive_still_pending - staled_archive_still_failed - staled_archive_still_generated Archive.stale(Archive::RETENTION_DURATION) end it { is_expected.to match_array([staled_archive_still_failed, staled_archive_still_generated]) } end + + describe 'stuck' do + let!(:recent_archive) { create(:archive, job_status: :pending) } + let!(:staled_archive_still_pending) { create(:archive, job_status: :pending, updated_at: (Archive::MAX_DUREE_GENERATION + 2).days.ago) } + let!(:staled_archive_still_failed) { create(:archive, job_status: :failed, updated_at: (Archive::MAX_DUREE_GENERATION + 2).days.ago) } + let!(:staled_archive_still_generated) { create(:archive, job_status: :generated, updated_at: (Archive::MAX_DUREE_GENERATION + 2).days.ago) } + + subject do + Archive.stuck(Archive::MAX_DUREE_GENERATION) + end + + it { is_expected.to match_array([staled_archive_still_pending]) } + end end describe '.job_status' do