Merge pull request #7629 from betagouv/fix-exports
Fix: purge stuck exports & archives
This commit is contained in:
commit
40a7b4a025
8 changed files with 40 additions and 13 deletions
|
@ -11,7 +11,7 @@ class ArchiveDashboard < Administrate::BaseDashboard
|
|||
id: Field::Number,
|
||||
created_at: Field::DateTime,
|
||||
updated_at: Field::DateTime,
|
||||
status: Field::String,
|
||||
job_status: Field::String,
|
||||
file: AttachmentField
|
||||
}.freeze
|
||||
|
||||
|
@ -24,7 +24,7 @@ class ArchiveDashboard < Administrate::BaseDashboard
|
|||
:id,
|
||||
:created_at,
|
||||
:updated_at,
|
||||
:status,
|
||||
:job_status,
|
||||
:file
|
||||
].freeze
|
||||
|
||||
|
@ -34,6 +34,6 @@ class ArchiveDashboard < Administrate::BaseDashboard
|
|||
:id,
|
||||
:created_at,
|
||||
:updated_at,
|
||||
:status
|
||||
:job_status
|
||||
].freeze
|
||||
end
|
||||
|
|
|
@ -3,5 +3,6 @@ class Cron::PurgeStaleArchivesJob < Cron::CronJob
|
|||
|
||||
def perform
|
||||
Archive.stale(Archive::RETENTION_DURATION).destroy_all
|
||||
Archive.stuck(Archive::MAX_DUREE_GENERATION).destroy_all
|
||||
end
|
||||
end
|
||||
|
|
|
@ -3,5 +3,6 @@ class Cron::PurgeStaleExportsJob < Cron::CronJob
|
|||
|
||||
def perform
|
||||
Export.stale(Export::MAX_DUREE_CONSERVATION_EXPORT).destroy_all
|
||||
Export.stuck(Export::MAX_DUREE_GENERATION).destroy_all
|
||||
end
|
||||
end
|
||||
|
|
|
@ -14,6 +14,7 @@ class Archive < ApplicationRecord
|
|||
include TransientModelsWithPurgeableJobConcern
|
||||
|
||||
RETENTION_DURATION = 4.days
|
||||
MAX_DUREE_GENERATION = 24.hours
|
||||
MAX_SIZE = 100.gigabytes
|
||||
|
||||
has_and_belongs_to_many :groupe_instructeurs
|
||||
|
|
|
@ -35,6 +35,11 @@ module TransientModelsWithPurgeableJobConcern
|
|||
.where('updated_at < ?', (Time.zone.now - duration))
|
||||
}
|
||||
|
||||
scope :stuck, lambda { |duration|
|
||||
where(job_status: [job_statuses.fetch(:pending)])
|
||||
.where('updated_at < ?', (Time.zone.now - duration))
|
||||
}
|
||||
|
||||
def available?
|
||||
generated?
|
||||
end
|
||||
|
|
|
@ -17,6 +17,7 @@ class Export < ApplicationRecord
|
|||
include TransientModelsWithPurgeableJobConcern
|
||||
|
||||
MAX_DUREE_CONSERVATION_EXPORT = 16.hours
|
||||
MAX_DUREE_GENERATION = 12.hours
|
||||
|
||||
enum format: {
|
||||
csv: 'csv',
|
||||
|
|
|
@ -4,26 +4,34 @@ describe Archive do
|
|||
before { Timecop.freeze(Time.zone.now) }
|
||||
after { Timecop.return }
|
||||
|
||||
let(:archive) { create(:archive, job_status: :pending) }
|
||||
let!(:archive) { create(:archive, job_status: :pending) }
|
||||
|
||||
describe 'scopes' do
|
||||
describe 'staled' do
|
||||
let(:recent_archive) { create(:archive, job_status: :pending) }
|
||||
let(:staled_archive_still_pending) { create(:archive, job_status: :pending, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
|
||||
let(:staled_archive_still_failed) { create(:archive, job_status: :failed, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
|
||||
let(:staled_archive_still_generated) { create(:archive, job_status: :generated, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
|
||||
let!(:recent_archive) { create(:archive, job_status: :pending) }
|
||||
let!(:staled_archive_still_pending) { create(:archive, job_status: :pending, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
|
||||
let!(:staled_archive_still_failed) { create(:archive, job_status: :failed, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
|
||||
let!(:staled_archive_still_generated) { create(:archive, job_status: :generated, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
|
||||
|
||||
subject do
|
||||
archive
|
||||
recent_archive
|
||||
staled_archive_still_pending
|
||||
staled_archive_still_failed
|
||||
staled_archive_still_generated
|
||||
Archive.stale(Archive::RETENTION_DURATION)
|
||||
end
|
||||
|
||||
it { is_expected.to match_array([staled_archive_still_failed, staled_archive_still_generated]) }
|
||||
end
|
||||
|
||||
describe 'stuck' do
|
||||
let!(:recent_archive) { create(:archive, job_status: :pending) }
|
||||
let!(:staled_archive_still_pending) { create(:archive, job_status: :pending, updated_at: (Archive::MAX_DUREE_GENERATION + 2).days.ago) }
|
||||
let!(:staled_archive_still_failed) { create(:archive, job_status: :failed, updated_at: (Archive::MAX_DUREE_GENERATION + 2).days.ago) }
|
||||
let!(:staled_archive_still_generated) { create(:archive, job_status: :generated, updated_at: (Archive::MAX_DUREE_GENERATION + 2).days.ago) }
|
||||
|
||||
subject do
|
||||
Archive.stuck(Archive::MAX_DUREE_GENERATION)
|
||||
end
|
||||
|
||||
it { is_expected.to match_array([staled_archive_still_pending]) }
|
||||
end
|
||||
end
|
||||
|
||||
describe '.job_status' do
|
||||
|
|
|
@ -31,6 +31,16 @@ RSpec.describe Export, type: :model do
|
|||
it { expect(Export.stale(Export::MAX_DUREE_CONSERVATION_EXPORT)).to match_array([stale_export_generated, stale_export_failed]) }
|
||||
end
|
||||
|
||||
describe '.stuck' do
|
||||
let!(:export) { create(:export) }
|
||||
let(:stuck_date) { Time.zone.now() - (Export::MAX_DUREE_GENERATION + 1.minute) }
|
||||
let!(:stale_export_generated) { create(:export, :generated, updated_at: stuck_date) }
|
||||
let!(:stale_export_failed) { create(:export, :failed, updated_at: stuck_date) }
|
||||
let!(:stale_export_pending) { create(:export, :pending, updated_at: stuck_date) }
|
||||
|
||||
it { expect(Export.stuck(Export::MAX_DUREE_GENERATION)).to match_array([stale_export_pending]) }
|
||||
end
|
||||
|
||||
describe '.destroy' do
|
||||
let!(:groupe_instructeur) { create(:groupe_instructeur) }
|
||||
let!(:export) { create(:export, groupe_instructeurs: [groupe_instructeur]) }
|
||||
|
|
Loading…
Add table
Reference in a new issue