Merge pull request #9055 from demarches-simplifiees/US/restore-dossier-kc

tech(recuperation-de-données): en cas de bug critique les techs DS aimeraient faciliter le re-import de données depuis un backup

app/lib/recovery/exporter.rb

@@ -0,0 +1,30 @@
+module Recovery
+  class Exporter
+    FILE_PATH = Rails.root.join('lib', 'data', 'export.dump')
+
+    attr_reader :dossiers, :file_path
+    def initialize(dossier_ids:, file_path: FILE_PATH)
+      dossier_with_data = Dossier.where(id: dossier_ids)
+        .preload(:user,
+                 :individual,
+                 :invites,
+                 :traitements,
+                 :transfer_logs,
+                 commentaires: { piece_jointe_attachment: :blob },
+                 avis: { introduction_file_attachment: :blob, piece_justificative_file_attachment: :blob },
+                 dossier_operation_logs: { serialized_attachment: :blob },
+                 attestation: { pdf_attachment: :blob },
+                 justificatif_motivation_attachment: :blob,
+                 etablissement: :exercices,
+                 revision: :procedure)
+      @dossiers = DossierPreloader.new(dossier_with_data,
+                                       includes_for_dossier: [:geo_areas, etablissement: :exercices],
+                                       includes_for_etablissement: [:exercices]).all
+      @file_path = file_path
+    end
+
+    def dump
+      @file_path.binwrite(Marshal.dump(@dossiers))
+    end
+  end
+end

app/lib/recovery/importer.rb

@@ -0,0 +1,110 @@
+module Recovery
+  class Importer
+    attr_reader :dossiers
+
+    def initialize(file_path: Recovery::Exporter::FILE_PATH)
+      # rubocop:disable Security/MarshalLoad
+      @dossiers = Marshal.load(File.read(file_path))
+      # rubocop:enable Security/MarshalLoad
+    end
+
+    def load
+      @dossiers.map do |dossier|
+        dossier.instance_variable_set :@new_record, true
+
+        Dossier.insert(dossier.attributes)
+
+        if dossier.etablissement.present?
+          Etablissement.insert(dossier.etablissement.attributes)
+          if dossier.etablissement.present?
+            APIEntreprise::EntrepriseJob.perform_later(dossier.etablissement.id, dossier.procedure.id)
+          end
+
+          dossier.etablissement.exercices.each do |exercice|
+            Exercice.insert(exercice.attributes)
+          end
+        end
+        if dossier.individual.present?
+          Individual.insert(dossier.individual.attributes)
+        end
+
+        dossier.invites.each do |invite|
+          Invite.insert(invite.attributes)
+        end
+
+        dossier.traitements.each do |traitement|
+          Traitement.insert(traitement.attributes)
+        end
+
+        dossier.transfer_logs.each do |transfer|
+          DossierTransferLog.insert(transfer.attributes)
+        end
+
+        dossier.commentaires.each do |commentaire|
+          Commentaire.insert(commentaire.attributes)
+          if commentaire.piece_jointe.attached?
+            import(commentaire.piece_jointe)
+          end
+        end
+
+        dossier.avis.each do |avis|
+          Avis.insert(avis.attributes)
+
+          if avis.introduction_file.attached?
+            import(avis.introduction_file)
+          end
+
+          if avis.piece_justificative_file.attached?
+            import(avis.piece_justificative_file)
+          end
+        end
+        dossier.dossier_operation_logs.each do |dol|
+          if dol.operation.nil?
+            puts "dol nil: #{dol.id}"
+            next
+          end
+          DossierOperationLog.insert(dol.attributes)
+
+          if dol.serialized.attached?
+            import(dol.serialized)
+          end
+        end
+
+        if dossier.attestation.present?
+          Attestation.insert(dossier.attestation.attributes)
+          import(dossier.attestation.pdf)
+        end
+
+        if dossier.justificatif_motivation.attached?
+          import(dossier.justificatif_motivation)
+        end
+
+        dossier.champs.each do |champ|
+          champ.piece_justificative_file.each { |pj| import(pj) }
+
+          if champ.etablissement.present?
+            APIEntreprise::EntrepriseJob.perform_later(champ.etablissement.id, dossier.procedure.id)
+
+            champ.etablissement.exercices.each do |exercice|
+              Exercice.insert(exercice.attributes)
+            end
+
+            Etablissement.insert(champ.etablissement.attributes)
+          end
+
+          Champ.insert(champ.attributes)
+
+          if champ.geo_areas.present?
+            champ.geo_areas.each { GeoArea.insert(_1.attributes) }
+          end
+        end
+        puts "imported dossier: #{dossier.id}"
+      end
+    end
+
+    def import(pj)
+      ActiveStorage::Blob.insert(pj.blob.attributes)
+      ActiveStorage::Attachment.insert(pj.attributes)
+    end
+  end
+end

app/models/dossier_preloader.rb

@@ -1,8 +1,10 @@
 class DossierPreloader
   DEFAULT_BATCH_SIZE = 2000
 
-  def initialize(dossiers)
+  def initialize(dossiers, includes_for_dossier: [], includes_for_etablissement: [])
     @dossiers = dossiers
+    @includes_for_etablissement = includes_for_etablissement
+    @includes_for_dossier = includes_for_dossier
   end
 
   def in_batches(size = DEFAULT_BATCH_SIZE)
@@ -35,7 +37,8 @@ class DossierPreloader
   end
 
   def load_dossiers(dossiers, pj_template: false)
-    to_include = [piece_justificative_file_attachments: :blob]
+    to_include = @includes_for_dossier.dup
+    to_include << [piece_justificative_file_attachments: :blob]
 
     if pj_template
       to_include << { type_de_champ: { piece_justificative_template_attachment: :blob } }
@@ -64,8 +67,9 @@ class DossierPreloader
   end
 
   def load_etablissements(champs)
+    to_include = @includes_for_etablissement.dup
     champs_siret = champs.filter(&:siret?)
-    etablissements_by_id = Etablissement.where(id: champs_siret.map(&:etablissement_id).compact).index_by(&:id)
+    etablissements_by_id = Etablissement.includes(to_include).where(id: champs_siret.map(&:etablissement_id).compact).index_by(&:id)
     champs_siret.each do |champ|
       etablissement = etablissements_by_id[champ.etablissement_id]
       champ.association(:etablissement).target = etablissement

lib/tasks/recovery.rake

@@ -0,0 +1,38 @@
+namespace :recovery do
+  desc <<~USAGE
+    given a file path, read it as json data, preload dossier data and export to marshal.dump.
+    the given file should be a json formatted as follow
+      {
+        procedure_id_1: [
+          dossier_id_1,
+          dossier_id_2,
+          ...
+        ],
+        procedure_id_2: [
+          ...
+        ],
+        ...
+      }
+    ex: rails recovery:export[missing_dossier_ids_per_procedure.json]
+  USAGE
+  task :export, [:file_path] => :environment do |_t, args|
+    dossier_ids = JSON.parse(File.read(args[:file_path])).values.flatten
+    rake_puts "Expecting to generate a dump with #{dossier_ids.size} dossiers"
+    exporter = Recovery::Exporter.new(dossier_ids:)
+    rake_puts "Found on db #{exporter.dossiers.size} dossiers"
+    exporter.dump
+    rake_puts "Export done, see: #{exporter.file_path}"
+  end
+
+  desc <<~USAGE
+    given a file path, read it as marshal data
+    the given file should be the result of recover:export
+    ex: rails recovery:import[/absolute/path/to/lib/data/export.dump]
+  USAGE
+  task :import, [:file_path] => :environment do |_t, args|
+    importer = Recovery::Importer.new(file_path: args[:file_path])
+    rake_puts "Expecting to load  #{importer.dossiers.size} dossiers"
+    importer.load
+    rake_puts "Mise à jour terminée"
+  end
+end

spec/factories/avis.rb

@@ -29,5 +29,13 @@ FactoryBot.define do
     trait :with_answer do
       answer { "Mon avis se décompose en deux points :\n- La demande semble pertinente\n- Le demandeur remplit les conditions." }
     end
+
+    trait :with_introduction do
+      introduction_file { Rack::Test::UploadedFile.new('spec/fixtures/files/white.png', 'image/png') }
+    end
+
+    trait :with_piece_justificative do
+      piece_justificative_file { Rack::Test::UploadedFile.new('spec/fixtures/files/white.png', 'image/png') }
+    end
   end
 end

spec/factories/dossier_operation_log.rb

@@ -1,5 +1,9 @@
 FactoryBot.define do
   factory :dossier_operation_log do
     operation { :passer_en_instruction }
+
+    trait :with_serialized do
+      serialized { Rack::Test::UploadedFile.new('spec/fixtures/files/white.png', 'image/png') }
+    end
   end
 end

spec/lib/recovery/exporter_spec.rb

@@ -0,0 +1,23 @@
+describe Recovery::Exporter do
+  let(:dossier_ids) { [create(:dossier, :with_individual).id, create(:dossier, :with_individual).id] }
+  let(:fp) { Rails.root.join('spec', 'fixtures', 'export.dump') }
+  subject { Recovery::Exporter.new(dossier_ids:, file_path: fp).dump }
+
+  def cleanup_export_file
+    if File.exist?(fp)
+      FileUtils.rm(fp)
+    end
+  end
+
+  before { cleanup_export_file }
+  after { cleanup_export_file }
+
+  it 'exports dossiers to .dump' do
+    expect { subject }.not_to raise_error
+  end
+
+  it 'exports dossiers local file .dump' do
+    expect { subject }.to change { File.exist?(fp) }
+      .from(false).to(true)
+  end
+end

spec/lib/recovery/life_cycle_spec.rb

@@ -0,0 +1,119 @@
+describe 'Recovery::LifeCycle' do
+  describe '.load_export_destroy_and_import' do
+    let(:procedure) do
+      create(:procedure,
+             types_de_champ_public: [
+               { type: :repetition, children: [{ type: :piece_justificative }] },
+               { type: :carte },
+               { type: :siret }
+             ])
+    end
+
+    let(:some_file) { Rack::Test::UploadedFile.new('spec/fixtures/files/white.png', 'image/png') }
+    let(:geo_area) { build(:geo_area, :selection_utilisateur, :polygon) }
+    let(:fp) { Rails.root.join('spec', 'fixtures', 'export.dump') }
+    let(:dossier) do
+      d = create(:dossier, procedure:)
+
+      repetition(d).add_row(d.revision)
+      pj_champ(d).piece_justificative_file.attach(some_file)
+      carte(d).update(geo_areas: [geo_area])
+      d.etablissement = create(:etablissement, :with_exercices)
+      d.etablissement.entreprise_attestation_sociale.attach(some_file)
+      d.etablissement.entreprise_attestation_fiscale.attach(some_file)
+
+      siret(d).update(etablissement: create(:etablissement, :with_exercices))
+      siret(d).etablissement.entreprise_attestation_sociale.attach(some_file)
+      siret(d).etablissement.entreprise_attestation_fiscale.attach(some_file)
+
+      d.individual = build(:individual)
+
+      d.attestation = build(:attestation, :with_pdf)
+      d.justificatif_motivation.attach(some_file)
+
+      d.commentaires << build(:commentaire, :with_file)
+
+      d.invites << build(:invite, :with_user)
+
+      d.avis << build(:avis, :with_introduction, :with_piece_justificative)
+
+      d.traitements.accepter(motivation: 'oui', processed_at: Time.zone.now)
+      d.save
+
+      d.dossier_operation_logs << build(:dossier_operation_log, :with_serialized)
+
+      d.transfer_logs.create(from: create(:user), to: create(:user))
+
+      d
+    end
+
+    def repetition(d) = d.champs.find_by(type: "Champs::RepetitionChamp")
+    def pj_champ(d) = d.champs.find_by(type: "Champs::PieceJustificativeChamp")
+    def carte(d) = d.champs.find_by(type: "Champs::CarteChamp")
+    def siret(d) = d.champs.find_by(type: "Champs::SiretChamp")
+
+    def cleanup_export_file
+      if File.exist?(fp)
+        FileUtils.rm(fp)
+      end
+    end
+    let(:instructeur) { create(:instructeur) }
+
+    before do
+      instructeur.followed_dossiers << dossier
+      cleanup_export_file
+    end
+
+    after { cleanup_export_file }
+    it 'reloads the full grappe' do
+      expect(Dossier.count).to eq(1)
+      expect(Dossier.first.champs.count).not_to be(0)
+
+      @dossier_ids = Dossier.ids
+
+      Recovery::Exporter.new(dossier_ids: @dossier_ids, file_path: fp).dump
+      Dossier.where(id: @dossier_ids).destroy_all
+      Recovery::Importer.new(file_path: fp).load
+
+      expect(Dossier.count).to eq(1)
+
+      reloaded_dossier = Dossier.first
+
+      expect(reloaded_dossier.champs.count).not_to be(0)
+
+      expect(repetition(reloaded_dossier).champs.map(&:type)).to match_array(["Champs::PieceJustificativeChamp"])
+      expect(pj_champ(reloaded_dossier).piece_justificative_file).to be_attached
+      expect(carte(reloaded_dossier).geo_areas).to be_present
+
+      expect(reloaded_dossier.etablissement.exercices).to be_present
+
+      # launch a job
+      # expect(reloaded_dossier.etablissement.entreprise_attestation_sociale).to be_attached
+      # expect(reloaded_dossier.etablissement.entreprise_attestation_fiscale).to be_attached
+
+      expect(siret(reloaded_dossier).etablissement.exercices).to be_present
+
+      # launch a job
+      # expect(siret(reloaded_dossier).etablissement.entreprise_attestation_sociale).to be_attached
+      # expect(siret(reloaded_dossier).etablissement.entreprise_attestation_fiscale).to be_attached
+
+      expect(reloaded_dossier.individual).to be_present
+      expect(reloaded_dossier.attestation.pdf).to be_attached
+      expect(reloaded_dossier.justificatif_motivation).to be_attached
+
+      expect(reloaded_dossier.commentaires.first.piece_jointe).to be_attached
+
+      expect(reloaded_dossier.invites.first.user).to be_present
+      expect(reloaded_dossier.followers_instructeurs).to match_array([instructeur])
+
+      expect(reloaded_dossier.avis.first.introduction_file).to be_attached
+      expect(reloaded_dossier.avis.first.piece_justificative_file).to be_attached
+
+      expect(reloaded_dossier.traitements).to be_present
+
+      expect(reloaded_dossier.dossier_operation_logs.first.serialized).to be_attached
+
+      expect(reloaded_dossier.transfer_logs).to be_present
+    end
+  end
+end