DGNum/demarches-normaliennes
13
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Merge pull request #7629 from betagouv/fix-exports

Browse source 
Fix: purge stuck exports & archives
This commit is contained in:
Colin Darie 2022-07-27 15:37:33 +02:00 • committed by GitHub
commit 40a7b4a025
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 40 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/dashboards/archive_dashboard.rb
View file

@ -11,7 +11,7 @@ class ArchiveDashboard < Administrate::BaseDashboard
id: Field::Number,
created_at: Field::DateTime,
updated_at: Field::DateTime,
status: Field::String,
job_status: Field::String,
file: AttachmentField
}.freeze
@ -24,7 +24,7 @@ class ArchiveDashboard < Administrate::BaseDashboard
:id,
:created_at,
:updated_at,
:status,
:job_status,
:file
].freeze
@ -34,6 +34,6 @@ class ArchiveDashboard < Administrate::BaseDashboard
:id,
:created_at,
:updated_at,
:status
:job_status
].freeze
end

1
app/jobs/cron/purge_stale_archives_job.rb
View file

@ -3,5 +3,6 @@ class Cron::PurgeStaleArchivesJob < Cron::CronJob
def perform
Archive.stale(Archive::RETENTION_DURATION).destroy_all
Archive.stuck(Archive::MAX_DUREE_GENERATION).destroy_all
end
end

1
app/jobs/cron/purge_stale_exports_job.rb
View file

@ -3,5 +3,6 @@ class Cron::PurgeStaleExportsJob < Cron::CronJob
def perform
Export.stale(Export::MAX_DUREE_CONSERVATION_EXPORT).destroy_all
Export.stuck(Export::MAX_DUREE_GENERATION).destroy_all
end
end

1
app/models/archive.rb
View file

@ -14,6 +14,7 @@ class Archive < ApplicationRecord
include TransientModelsWithPurgeableJobConcern
RETENTION_DURATION = 4.days
MAX_DUREE_GENERATION = 24.hours
MAX_SIZE = 100.gigabytes
has_and_belongs_to_many :groupe_instructeurs

5
app/models/concerns/transient_models_with_purgeable_job_concern.rb
View file

@ -35,6 +35,11 @@ module TransientModelsWithPurgeableJobConcern
.where('updated_at < ?', (Time.zone.now - duration))
}
scope :stuck, lambda { |duration|
where(job_status: [job_statuses.fetch(:pending)])
.where('updated_at < ?', (Time.zone.now - duration))
}
def available?
generated?
end

1
app/models/export.rb
View file

@ -17,6 +17,7 @@ class Export < ApplicationRecord
include TransientModelsWithPurgeableJobConcern
MAX_DUREE_CONSERVATION_EXPORT = 16.hours
MAX_DUREE_GENERATION = 12.hours
enum format: {
csv: 'csv',

28
spec/models/archive_spec.rb
View file

@ -4,26 +4,34 @@ describe Archive do
before { Timecop.freeze(Time.zone.now) }
after { Timecop.return }
let(:archive) { create(:archive, job_status: :pending) }
let!(:archive) { create(:archive, job_status: :pending) }
describe 'scopes' do
describe 'staled' do
let(:recent_archive) { create(:archive, job_status: :pending) }
let(:staled_archive_still_pending) { create(:archive, job_status: :pending, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
let(:staled_archive_still_failed) { create(:archive, job_status: :failed, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
let(:staled_archive_still_generated) { create(:archive, job_status: :generated, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
let!(:recent_archive) { create(:archive, job_status: :pending) }
let!(:staled_archive_still_pending) { create(:archive, job_status: :pending, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
let!(:staled_archive_still_failed) { create(:archive, job_status: :failed, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
let!(:staled_archive_still_generated) { create(:archive, job_status: :generated, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
subject do
archive
recent_archive
staled_archive_still_pending
staled_archive_still_failed
staled_archive_still_generated
Archive.stale(Archive::RETENTION_DURATION)
end
it { is_expected.to match_array([staled_archive_still_failed, staled_archive_still_generated]) }
end
describe 'stuck' do
let!(:recent_archive) { create(:archive, job_status: :pending) }
let!(:staled_archive_still_pending) { create(:archive, job_status: :pending, updated_at: (Archive::MAX_DUREE_GENERATION + 2).days.ago) }
let!(:staled_archive_still_failed) { create(:archive, job_status: :failed, updated_at: (Archive::MAX_DUREE_GENERATION + 2).days.ago) }
let!(:staled_archive_still_generated) { create(:archive, job_status: :generated, updated_at: (Archive::MAX_DUREE_GENERATION + 2).days.ago) }
subject do
Archive.stuck(Archive::MAX_DUREE_GENERATION)
end
it { is_expected.to match_array([staled_archive_still_pending]) }
end
end
describe '.job_status' do

10
spec/models/export_spec.rb
View file

@ -31,6 +31,16 @@ RSpec.describe Export, type: :model do
it { expect(Export.stale(Export::MAX_DUREE_CONSERVATION_EXPORT)).to match_array([stale_export_generated, stale_export_failed]) }
end
describe '.stuck' do
let!(:export) { create(:export) }
let(:stuck_date) { Time.zone.now() - (Export::MAX_DUREE_GENERATION + 1.minute) }
let!(:stale_export_generated) { create(:export, :generated, updated_at: stuck_date) }
let!(:stale_export_failed) { create(:export, :failed, updated_at: stuck_date) }
let!(:stale_export_pending) { create(:export, :pending, updated_at: stuck_date) }
it { expect(Export.stuck(Export::MAX_DUREE_GENERATION)).to match_array([stale_export_pending]) }
end
describe '.destroy' do
let!(:groupe_instructeur) { create(:groupe_instructeur) }
let!(:export) { create(:export, groupe_instructeurs: [groupe_instructeur]) }