fix(export): when it takes more than 3 hours, exports are purge before being generated. make it possible to have an export that takes more than 3 hours and share this behaviour with kind of same class archive

app/components/dossiers/export_component/export_component.html.haml

@@ -11,7 +11,7 @@
         %li
           - if export.nil?
             = link_to t(".everything_#{item[:format]}_html"), download_export_path(export_format: item[:format]), data: { turbo_method: :post }
-          - elsif export.ready?
+          - elsif export.available?
             = link_to ready_link_label(export), export.file.service_url, target: "_blank", rel: "noopener"
             - if export.old?
               = button_to download_export_path(export_format: export.format, force_export: true), **refresh_button_options(export) do

app/controllers/administrateurs/exports_controller.rb

@@ -5,12 +5,12 @@ module Administrateurs
     def download
       export = Export.find_or_create_export(export_format, all_groupe_instructeurs, **export_options)
 
-      if export.ready? && export.old? && force_export?
+      if export.available? && export.old? && force_export?
         export.destroy
         export = Export.find_or_create_export(export_format, all_groupe_instructeurs, **export_options)
       end
 
-      if export.ready?
+      if export.available?
         respond_to do |format|
           format.turbo_stream do
             @dossiers_count = export.count

app/controllers/instructeurs/procedures_controller.rb

@@ -149,12 +149,12 @@ module Instructeurs
 
       export = Export.find_or_create_export(export_format, groupe_instructeurs, **export_options)
 
-      if export.ready? && export.old? && force_export?
+      if export.available? && export.old? && force_export?
         export.destroy
         export = Export.find_or_create_export(export_format, groupe_instructeurs, **export_options)
       end
 
-      if export.ready?
+      if export.available?
         respond_to do |format|
           format.turbo_stream do
             @procedure = procedure

app/jobs/archive_creation_job.rb

@@ -2,14 +2,11 @@ class ArchiveCreationJob < ApplicationJob
   queue_as :archives
 
   def perform(procedure, archive, administrateur_or_instructeur)
-    archive.restart! if archive.failed? # restart for AASM
+    archive.compute_with_safe_stale_for_purge do
       ProcedureArchiveService
         .new(procedure)
         .make_and_upload_archive(archive)
-    archive.make_available!
       UserMailer.send_archive(administrateur_or_instructeur, procedure, archive).deliver_later
-  rescue => e
+    end
-    archive.fail! # fail for observability
-    raise e       # re-raise for retryable behaviour
   end
 end

app/jobs/cron/purge_stale_archives_job.rb

@@ -2,6 +2,6 @@ class Cron::PurgeStaleArchivesJob < Cron::CronJob
   self.schedule_expression = "every 5 minutes"
 
   def perform
-    Archive.stale.destroy_all
+    Archive.stale(Archive::RETENTION_DURATION).destroy_all
   end
 end

app/jobs/cron/purge_stale_exports_job.rb

@@ -2,6 +2,6 @@ class Cron::PurgeStaleExportsJob < Cron::CronJob
   self.schedule_expression = "every 5 minutes"
 
   def perform
-    Export.stale.destroy_all
+    Export.stale(Export::MAX_DUREE_CONSERVATION_EXPORT).destroy_all
   end
 end

app/jobs/export_job.rb

@@ -4,6 +4,8 @@ class ExportJob < ApplicationJob
   discard_on ActiveRecord::RecordNotFound
 
   def perform(export)
+    export.compute_with_safe_stale_for_purge do
       export.compute
     end
+  end
 end

app/models/archive.rb

@@ -3,15 +3,15 @@
 # Table name: archives
 #
 #  id             :bigint           not null, primary key
+#  job_status     :string           not null
 #  key            :text             not null
 #  month          :date
-#  status         :string           not null
 #  time_span_type :string           not null
 #  created_at     :datetime         not null
 #  updated_at     :datetime         not null
 #
 class Archive < ApplicationRecord
-  include AASM
+  include TransientModelsWithPurgeableJobConcern
 
   RETENTION_DURATION = 4.days
   MAX_SIZE = 100.gigabytes
@@ -20,7 +20,6 @@ class Archive < ApplicationRecord
 
   has_one_attached :file
 
-  scope :stale, -> { where('updated_at < ?', (Time.zone.now - RETENTION_DURATION)) }
   scope :for_groupe_instructeur, -> (groupe_instructeur) {
     joins(:archives_groupe_instructeurs)
       .where(
@@ -33,32 +32,6 @@ class Archive < ApplicationRecord
     monthly:    'monthly'
   }
 
-  enum status: {
-    pending: 'pending',
-    generated: 'generated',
-    failed: 'failed'
-  }
-
-  aasm whiny_persistence: true, column: :status, enum: true do
-    state :pending, initial: true
-    state :generated
-    state :failed
-
-    event :make_available do
-      transitions from: :pending, to: :generated
-    end
-    event :restart do
-      transitions from: :failed, to: :pending
-    end
-    event :fail do
-      transitions from: :pending, to: :failed
-    end
-  end
-
-  def available?
-    status == 'generated' && file.attached?
-  end
-
   def filename(procedure)
     if time_span_type == 'everything'
       "procedure-#{procedure.id}.zip"

app/models/concerns/transient_models_with_purgeable_job_concern.rb

@@ -0,0 +1,51 @@
+# Archive and Export models are generated in background
+#   those models being are destroy after an expiration period
+#   but, it might take more time to process than the expiration period
+#   this module expose the shared behaviour to compute a job and purge instances
+#   based on a state machine
+module TransientModelsWithPurgeableJobConcern
+  extend ActiveSupport::Concern
+  included do
+    include AASM
+
+    enum job_status: {
+      pending: 'pending',
+      generated: 'generated',
+      failed: 'failed'
+    }
+
+    aasm whiny_persistence: true, column: :job_status, enum: true do
+      state :pending, initial: true
+      state :generated
+      state :failed
+
+      event :make_available do
+        transitions from: :pending, to: :generated
+      end
+      event :restart do
+        transitions from: :failed, to: :pending
+      end
+      event :fail do
+        transitions from: :pending, to: :failed
+      end
+    end
+
+    scope :stale, lambda { |duration|
+      where(job_status: [job_statuses.fetch(:generated), job_statuses.fetch(:failed)])
+        .where('updated_at < ?', (Time.zone.now - duration))
+    }
+
+    def available?
+      generated?
+    end
+
+    def compute_with_safe_stale_for_purge(&block)
+      restart! if failed? # restart for AASM
+      yield
+      make_available!
+    rescue => e
+      fail!         # fail for observability
+      raise e       # re-raise for retryable behaviour
+    end
+  end
+end

app/models/export.rb

@@ -4,6 +4,7 @@
 #
 #  id                              :bigint           not null, primary key
 #  format                          :string           not null
+#  job_status                      :string
 #  key                             :text             not null
 #  procedure_presentation_snapshot :jsonb
 #  statut                          :string           default("tous")
@@ -13,7 +14,9 @@
 #  procedure_presentation_id       :bigint
 #
 class Export < ApplicationRecord
-  MAX_DUREE_CONSERVATION_EXPORT = 3.hours
+  include TransientModelsWithPurgeableJobConcern
+
+  MAX_DUREE_CONSERVATION_EXPORT = 16.hours
 
   enum format: {
     csv: 'csv',
@@ -69,10 +72,6 @@ class Export < ApplicationRecord
     time_span_type == Export.time_span_types.fetch(:monthly) ? 30.days.ago : nil
   end
 
-  def ready?
-    file.attached?
-  end
-
   def old?
     updated_at < 20.minutes.ago || filters_changed?
   end

app/views/shared/archives/_table.html.haml

@@ -22,7 +22,7 @@
         = number_to_human_size(weight)
       %td.center
         - if matching_archive.present?
-          - if matching_archive.status == 'generated' && matching_archive.file.attached?
+          - if matching_archive.available?
             = link_to url_for(matching_archive.file), class: 'button primary' do
               %span.icon.download-white
               = t(:archive_ready_html, scope: [:instructeurs, :procedure], generated_period: time_ago_in_words(matching_archive.updated_at))

db/migrate/20220707152632_cleanup_export_and_archive.rb

@@ -0,0 +1,7 @@
+class CleanupExportAndArchive < ActiveRecord::Migration[6.1]
+  def change
+    safety_assured do
+      rename_column :archives, :status, :job_status
+    end
+  end
+end

db/schema.rb

@@ -88,9 +88,9 @@ ActiveRecord::Schema.define(version: 2022_07_08_152039) do
 
   create_table "archives", force: :cascade do |t|
     t.datetime "created_at", precision: 6, null: false
+    t.string "job_status", null: false
     t.text "key", null: false
     t.date "month"
-    t.string "status", null: false
     t.string "time_span_type", null: false
     t.datetime "updated_at", precision: 6, null: false
     t.index ["key", "time_span_type", "month"], name: "index_archives_on_key_and_time_span_type_and_month", unique: true
@@ -430,6 +430,7 @@ ActiveRecord::Schema.define(version: 2022_07_08_152039) do
   create_table "exports", force: :cascade do |t|
     t.datetime "created_at", null: false
     t.string "format", null: false
+    t.string "job_status"
     t.text "key", null: false
     t.bigint "procedure_presentation_id"
     t.jsonb "procedure_presentation_snapshot"

spec/controllers/administrateurs/exports_controller_spec.rb

@@ -38,7 +38,7 @@ describe Administrateurs::ExportsController, type: :controller do
       end
 
       context 'when the export is ready' do
-        let(:export) { create(:export, groupe_instructeurs: procedure.groupe_instructeurs) }
+        let(:export) { create(:export, job_status: :generated, groupe_instructeurs: procedure.groupe_instructeurs) }
 
         before do
           export.file.attach(io: StringIO.new('export'), filename: 'file.csv')

spec/controllers/instructeurs/procedures_controller_spec.rb

@@ -498,7 +498,7 @@ describe Instructeurs::ProceduresController, type: :controller do
     end
 
     context 'when the export is ready' do
-      let(:export) { create(:export, groupe_instructeurs: [gi_1]) }
+      let(:export) { create(:export, groupe_instructeurs: [gi_1], job_status: 'generated') }
 
       before do
         export.file.attach(io: StringIO.new('export'), filename: 'file.csv')

spec/factories/archive.rb

@@ -5,11 +5,11 @@ FactoryBot.define do
     key { 'unique-key' }
 
     trait :pending do
-      status { 'pending' }
+      job_status { 'pending' }
     end
 
     trait :generated do
-      status { 'generated' }
+      job_status { 'generated' }
     end
   end
 end

spec/jobs/archive_creation_job_spec.rb

@@ -1,6 +1,6 @@
 describe ArchiveCreationJob, type: :job do
   describe 'perform' do
-    let(:archive) { create(:archive, status: status, groupe_instructeurs: [procedure.groupe_instructeurs.first]) }
+    let(:archive) { create(:archive, job_status: status, groupe_instructeurs: [procedure.groupe_instructeurs.first]) }
     let(:instructeur) { create(:instructeur) }
     let(:procedure) { create(:procedure, instructeurs: [instructeur]) }
     let(:job) { ArchiveCreationJob.new(procedure, archive, instructeur) }

spec/models/archive_spec.rb

@@ -1,32 +1,38 @@
-describe Dossier do
+describe Archive do
   include ActiveJob::TestHelper
 
   before { Timecop.freeze(Time.zone.now) }
   after { Timecop.return }
 
-  let(:archive) { create(:archive) }
+  let(:archive) { create(:archive, job_status: :pending) }
 
   describe 'scopes' do
     describe 'staled' do
-      let(:recent_archive) { create(:archive) }
+      let(:recent_archive) { create(:archive, job_status: :pending) }
-      let(:staled_archive) { create(:archive, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
+      let(:staled_archive_still_pending) { create(:archive, job_status: :pending, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
+      let(:staled_archive_still_failed) { create(:archive, job_status: :failed, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
+      let(:staled_archive_still_generated) { create(:archive, job_status: :generated, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) }
 
       subject do
-        archive; recent_archive; staled_archive
+        archive
-        Archive.stale
+        recent_archive
+        staled_archive_still_pending
+        staled_archive_still_failed
+        staled_archive_still_generated
+        Archive.stale(Archive::RETENTION_DURATION)
       end
 
-      it { is_expected.to match_array([staled_archive]) }
+      it { is_expected.to match_array([staled_archive_still_failed, staled_archive_still_generated]) }
     end
   end
 
-  describe '.status' do
+  describe '.job_status' do
-    it { expect(archive.status).to eq('pending') }
+    it { expect(archive.job_status).to eq('pending') }
   end
 
   describe '#make_available!' do
     before { archive.make_available! }
-    it { expect(archive.status).to eq('generated') }
+    it { expect(archive.job_status).to eq('generated') }
   end
 
   describe '#available?' do

spec/services/procedure_archive_service_spec.rb

@@ -18,7 +18,7 @@ describe ProcedureArchiveService do
     after { Timecop.return }
 
     context 'for a specific month' do
-      let(:archive) { create(:archive, time_span_type: 'monthly', status: 'pending', month: date_month, groupe_instructeurs: groupe_instructeurs) }
+      let(:archive) { create(:archive, time_span_type: 'monthly', job_status: 'pending', month: date_month, groupe_instructeurs: groupe_instructeurs) }
       let(:year) { 2021 }
 
       it 'collects files with success' do
@@ -120,7 +120,7 @@ describe ProcedureArchiveService do
     end
 
     context 'for all months' do
-      let(:archive) { create(:archive, time_span_type: 'everything', status: 'pending', groupe_instructeurs: groupe_instructeurs) }
+      let(:archive) { create(:archive, time_span_type: 'everything', job_status: 'pending', groupe_instructeurs: groupe_instructeurs) }
 
       it 'collect files' do
         allow_any_instance_of(ActiveStorage::Attachment).to receive(:url).and_return("https://opengraph.githubassets.com/5e61989aecb78e369c93674f877d7bf4ecde378850114a9563cdf8b6a2472536/typhoeus/typhoeus/issues/110")