tech(tache.recovery): ajoute une tache pour re-importer des dossiers venant d'un backup

Update app/lib/recovery/exporter.rb

Co-authored-by: Colin Darie <colin@darie.eu>

app/lib/recovery/exporter.rb

@@ -2,7 +2,7 @@ module Recovery
   class Exporter
     FILE_PATH = Rails.root.join('lib', 'data', 'export.dump')
 
-    attr_reader :dossiers
+    attr_reader :dossiers, :file_path
     def initialize(dossier_ids:, file_path: FILE_PATH)
       dossier_with_data = Dossier.where(id: dossier_ids)
         .preload(:user,
@@ -17,12 +17,14 @@ module Recovery
                  justificatif_motivation_attachment: :blob,
                  etablissement: :exercices,
                  revision: :procedure)
-      @dossiers = DossierPreloader.new(dossier_with_data).all
+      @dossiers = DossierPreloader.new(dossier_with_data,
+                                       includes_for_dossier: [:geo_areas, etablissement: :exercices],
+                                       includes_for_etablissement: [:exercices]).all
       @file_path = file_path
     end
 
     def dump
-      File.open(@file_path, 'wb') { _1.write(Marshal.dump(@dossiers)) }
+      @file_path.binwrite(Marshal.dump(@dossiers))
     end
   end
 end

app/lib/recovery/importer.rb

@@ -3,7 +3,9 @@ module Recovery
     attr_reader :dossiers
 
     def initialize(file_path: Recovery::Exporter::FILE_PATH)
+      # rubocop:disable Security/MarshalLoad
       @dossiers = Marshal.load(File.read(file_path))
+      # rubocop:enable Security/MarshalLoad
     end
 
     def load
@@ -12,12 +14,19 @@ module Recovery
 
         Dossier.insert(dossier.attributes)
 
-        Etablissement.insert(dossier.etablissement.attributes)
         if dossier.etablissement.present?
-          APIEntreprise::EntrepriseJob.perform_later(dossier.etablissement.id, dossier.procedure.id)
+          Etablissement.insert(dossier.etablissement.attributes)
-        end
+          if dossier.etablissement.present?
+            APIEntreprise::EntrepriseJob.perform_later(dossier.etablissement.id, dossier.procedure.id)
+          end
 
-        Individual.insert(dossier.individual.attributes)
+          dossier.etablissement.exercices.each do |exercice|
+            Exercice.insert(exercice.attributes)
+          end
+        end
+        if dossier.individual.present?
+          Individual.insert(dossier.individual.attributes)
+        end
 
         dossier.invites.each do |invite|
           Invite.insert(invite.attributes)
@@ -31,10 +40,6 @@ module Recovery
           DossierTransferLog.insert(transfer.attributes)
         end
 
-        dossier.etablissement.exercices.each do |exercice|
-          Exercice.insert(exercice.attributes)
-        end
-
         dossier.commentaires.each do |commentaire|
           Commentaire.insert(commentaire.attributes)
           if commentaire.piece_jointe.attached?
@@ -53,8 +58,11 @@ module Recovery
             import(avis.piece_justificative_file)
           end
         end
-
         dossier.dossier_operation_logs.each do |dol|
+          if dol.operation.nil?
+            puts "dol nil: #{dol.id}"
+            next
+          end
           DossierOperationLog.insert(dol.attributes)
 
           if dol.serialized.attached?
@@ -90,6 +98,7 @@ module Recovery
             champ.geo_areas.each { GeoArea.insert(_1.attributes) }
           end
         end
+        puts "imported dossier: #{dossier.id}"
       end
     end
 

app/models/dossier_preloader.rb

@@ -1,8 +1,10 @@
 class DossierPreloader
   DEFAULT_BATCH_SIZE = 2000
 
-  def initialize(dossiers)
+  def initialize(dossiers, includes_for_dossier: [], includes_for_etablissement: [])
     @dossiers = dossiers
+    @includes_for_etablissement = includes_for_etablissement
+    @includes_for_dossier = includes_for_dossier
   end
 
   def in_batches(size = DEFAULT_BATCH_SIZE)
@@ -35,7 +37,8 @@ class DossierPreloader
   end
 
   def load_dossiers(dossiers, pj_template: false)
-    to_include = [:geo_areas, piece_justificative_file_attachments: :blob, etablissement: :exercices]
+    to_include = @includes_for_dossier.dup
+    to_include << [piece_justificative_file_attachments: :blob]
 
     if pj_template
       to_include << { type_de_champ: { piece_justificative_template_attachment: :blob } }
@@ -64,8 +67,9 @@ class DossierPreloader
   end
 
   def load_etablissements(champs)
+    to_include = @includes_for_etablissement.dup
     champs_siret = champs.filter(&:siret?)
-    etablissements_by_id = Etablissement.includes(:exercices).where(id: champs_siret.map(&:etablissement_id).compact).index_by(&:id)
+    etablissements_by_id = Etablissement.includes(to_include).where(id: champs_siret.map(&:etablissement_id).compact).index_by(&:id)
     champs_siret.each do |champ|
       etablissement = etablissements_by_id[champ.etablissement_id]
       champ.association(:etablissement).target = etablissement

lib/tasks/recovery.rake

@@ -0,0 +1,38 @@
+namespace :recovery do
+  desc <<~USAGE
+    given a file path, read it as json data, preload dossier data and export to marshal.dump.
+    the given file should be a json formatted as follow
+      {
+        procedure_id_1: [
+          dossier_id_1,
+          dossier_id_2,
+          ...
+        ],
+        procedure_id_2: [
+          ...
+        ],
+        ...
+      }
+    ex: rails recovery:export[missing_dossier_ids_per_procedure.json]
+  USAGE
+  task :export, [:file_path] => :environment do |_t, args|
+    dossier_ids = JSON.parse(File.read(args[:file_path])).values.flatten
+    rake_puts "Expecting to generate a dump with #{dossier_ids.size} dossiers"
+    exporter = Recovery::Exporter.new(dossier_ids:)
+    rake_puts "Found on db #{exporter.dossiers.size} dossiers"
+    exporter.dump
+    rake_puts "Export done, see: #{exporter.file_path}"
+  end
+
+  desc <<~USAGE
+    given a file path, read it as marshal data
+    the given file should be the result of recover:export
+    ex: rails recovery:import[/absolute/path/to/lib/data/export.dump]
+  USAGE
+  task :import, [:file_path] => :environment do |_t, args|
+    importer = Recovery::Importer.new(file_path: args[:file_path])
+    rake_puts "Expecting to load  #{importer.dossiers.size} dossiers"
+    importer.load
+    rake_puts "Mise à jour terminée"
+  end
+end

spec/lib/recovery/exporter_spec.rb

@@ -1,12 +1,12 @@
 describe Recovery::Exporter do
   let(:dossier_ids) { [create(:dossier, :with_individual).id, create(:dossier, :with_individual).id] }
-  let(:fp) { Rails.root.join('spec', 'fixtures', 'recovery', 'export.dump') }
+  let(:fp) { Rails.root.join('spec', 'fixtures', 'export.dump') }
   subject { Recovery::Exporter.new(dossier_ids:, file_path: fp).dump }
 
   def cleanup_export_file
-    # if File.exist?(fp)
+    if File.exist?(fp)
-    # FileUtils.rm(fp)
+      FileUtils.rm(fp)
-    # end
+    end
   end
 
   before { cleanup_export_file }
@@ -20,29 +20,4 @@ describe Recovery::Exporter do
     expect { subject }.to change { File.exist?(fp) }
       .from(false).to(true)
   end
-
-  context 'exported' do
-    before { subject }
-    let(:exported_dossiers) { Marshal.load(File.read(fp)) }
-
-    it 'contains as much as dossiers as input' do
-      expect(exported_dossiers.size).to eq(dossier_ids.size)
-    end
-
-    it 'contains input dossier ids' do
-      expect(exported_dossiers.map(&:id)).to match_array(dossier_ids)
-    end
-
-    it 'contains procedure dossier ids' do
-      expect(exported_dossiers.first.procedure).to be_an_instance_of(Procedure)
-    end
-
-    it 'contains dossier.revision ids' do
-      expect(exported_dossiers.first.revision).to be_an_instance_of(ProcedureRevision)
-    end
-
-    it 'contains dossier.user' do
-      expect(exported_dossiers.first.user).to be_an_instance_of(User)
-    end
-  end
 end

spec/lib/recovery/importer_spec.rb

@@ -1,16 +0,0 @@
-describe Recovery::Importer do
-  let(:file_path) { Rails.root.join('spec', 'fixtures', 'recovery', 'export.dump') }
-  let(:importer) { Recovery::Importer.new(file_path:) }
-  subject { importer.load }
-  context 'loaded_data' do
-    let(:loaded_dossiers) { importer.dossiers }
-
-    it 'contains user' do
-      expect(loaded_dossiers.first.user).to be_an_instance_of(User)
-    end
-  end
-
-  it 're-import dossiers from .dump' do
-    expect { subject }.to change { Dossier.count }.by(importer.dossiers.size)
-  end
-end

spec/lib/recovery/life_cycle_spec.rb

@@ -11,7 +11,7 @@ describe 'Recovery::LifeCycle' do
 
     let(:some_file) { Rack::Test::UploadedFile.new('spec/fixtures/files/white.png', 'image/png') }
     let(:geo_area) { build(:geo_area, :selection_utilisateur, :polygon) }
-
+    let(:fp) { Rails.root.join('spec', 'fixtures', 'export.dump') }
     let(:dossier) do
       d = create(:dossier, procedure:)
 
@@ -52,21 +52,28 @@ describe 'Recovery::LifeCycle' do
     def carte(d) = d.champs.find_by(type: "Champs::CarteChamp")
     def siret(d) = d.champs.find_by(type: "Champs::SiretChamp")
 
+    def cleanup_export_file
+      if File.exist?(fp)
+        FileUtils.rm(fp)
+      end
+    end
     let(:instructeur) { create(:instructeur) }
 
     before do
       instructeur.followed_dossiers << dossier
+      cleanup_export_file
     end
 
+    after { cleanup_export_file }
     it 'reloads the full grappe' do
       expect(Dossier.count).to eq(1)
       expect(Dossier.first.champs.count).not_to be(0)
 
       @dossier_ids = Dossier.ids
 
-      Recovery::Exporter.new(dossier_ids: @dossier_ids).dump
+      Recovery::Exporter.new(dossier_ids: @dossier_ids, file_path: fp).dump
       Dossier.where(id: @dossier_ids).destroy_all
-      Recovery::Importer.new().load
+      Recovery::Importer.new(file_path: fp).load
 
       expect(Dossier.count).to eq(1)