diff --git a/app/lib/recovery/exporter.rb b/app/lib/recovery/exporter.rb new file mode 100644 index 000000000..4781896f1 --- /dev/null +++ b/app/lib/recovery/exporter.rb @@ -0,0 +1,30 @@ +module Recovery + class Exporter + FILE_PATH = Rails.root.join('lib', 'data', 'export.dump') + + attr_reader :dossiers, :file_path + def initialize(dossier_ids:, file_path: FILE_PATH) + dossier_with_data = Dossier.where(id: dossier_ids) + .preload(:user, + :individual, + :invites, + :traitements, + :transfer_logs, + commentaires: { piece_jointe_attachment: :blob }, + avis: { introduction_file_attachment: :blob, piece_justificative_file_attachment: :blob }, + dossier_operation_logs: { serialized_attachment: :blob }, + attestation: { pdf_attachment: :blob }, + justificatif_motivation_attachment: :blob, + etablissement: :exercices, + revision: :procedure) + @dossiers = DossierPreloader.new(dossier_with_data, + includes_for_dossier: [:geo_areas, etablissement: :exercices], + includes_for_etablissement: [:exercices]).all + @file_path = file_path + end + + def dump + @file_path.binwrite(Marshal.dump(@dossiers)) + end + end +end diff --git a/app/lib/recovery/importer.rb b/app/lib/recovery/importer.rb new file mode 100644 index 000000000..cb8f6500b --- /dev/null +++ b/app/lib/recovery/importer.rb @@ -0,0 +1,110 @@ +module Recovery + class Importer + attr_reader :dossiers + + def initialize(file_path: Recovery::Exporter::FILE_PATH) + # rubocop:disable Security/MarshalLoad + @dossiers = Marshal.load(File.read(file_path)) + # rubocop:enable Security/MarshalLoad + end + + def load + @dossiers.map do |dossier| + dossier.instance_variable_set :@new_record, true + + Dossier.insert(dossier.attributes) + + if dossier.etablissement.present? + Etablissement.insert(dossier.etablissement.attributes) + if dossier.etablissement.present? + APIEntreprise::EntrepriseJob.perform_later(dossier.etablissement.id, dossier.procedure.id) + end + + dossier.etablissement.exercices.each do |exercice| + Exercice.insert(exercice.attributes) + end + end + if dossier.individual.present? + Individual.insert(dossier.individual.attributes) + end + + dossier.invites.each do |invite| + Invite.insert(invite.attributes) + end + + dossier.traitements.each do |traitement| + Traitement.insert(traitement.attributes) + end + + dossier.transfer_logs.each do |transfer| + DossierTransferLog.insert(transfer.attributes) + end + + dossier.commentaires.each do |commentaire| + Commentaire.insert(commentaire.attributes) + if commentaire.piece_jointe.attached? + import(commentaire.piece_jointe) + end + end + + dossier.avis.each do |avis| + Avis.insert(avis.attributes) + + if avis.introduction_file.attached? + import(avis.introduction_file) + end + + if avis.piece_justificative_file.attached? + import(avis.piece_justificative_file) + end + end + dossier.dossier_operation_logs.each do |dol| + if dol.operation.nil? + puts "dol nil: #{dol.id}" + next + end + DossierOperationLog.insert(dol.attributes) + + if dol.serialized.attached? + import(dol.serialized) + end + end + + if dossier.attestation.present? + Attestation.insert(dossier.attestation.attributes) + import(dossier.attestation.pdf) + end + + if dossier.justificatif_motivation.attached? + import(dossier.justificatif_motivation) + end + + dossier.champs.each do |champ| + champ.piece_justificative_file.each { |pj| import(pj) } + + if champ.etablissement.present? + APIEntreprise::EntrepriseJob.perform_later(champ.etablissement.id, dossier.procedure.id) + + champ.etablissement.exercices.each do |exercice| + Exercice.insert(exercice.attributes) + end + + Etablissement.insert(champ.etablissement.attributes) + end + + Champ.insert(champ.attributes) + + if champ.geo_areas.present? + champ.geo_areas.each { GeoArea.insert(_1.attributes) } + end + end + puts "imported dossier: #{dossier.id}" + end + end + + def import(pj) + ActiveStorage::Blob.insert(pj.blob.attributes) + ActiveStorage::Attachment.insert(pj.attributes) + end + end +end diff --git a/app/models/dossier_preloader.rb b/app/models/dossier_preloader.rb index 61e147492..d17d67a49 100644 --- a/app/models/dossier_preloader.rb +++ b/app/models/dossier_preloader.rb @@ -1,8 +1,10 @@ class DossierPreloader DEFAULT_BATCH_SIZE = 2000 - def initialize(dossiers) + def initialize(dossiers, includes_for_dossier: [], includes_for_etablissement: []) @dossiers = dossiers + @includes_for_etablissement = includes_for_etablissement + @includes_for_dossier = includes_for_dossier end def in_batches(size = DEFAULT_BATCH_SIZE) @@ -35,7 +37,8 @@ class DossierPreloader end def load_dossiers(dossiers, pj_template: false) - to_include = [piece_justificative_file_attachments: :blob] + to_include = @includes_for_dossier.dup + to_include << [piece_justificative_file_attachments: :blob] if pj_template to_include << { type_de_champ: { piece_justificative_template_attachment: :blob } } @@ -64,8 +67,9 @@ class DossierPreloader end def load_etablissements(champs) + to_include = @includes_for_etablissement.dup champs_siret = champs.filter(&:siret?) - etablissements_by_id = Etablissement.where(id: champs_siret.map(&:etablissement_id).compact).index_by(&:id) + etablissements_by_id = Etablissement.includes(to_include).where(id: champs_siret.map(&:etablissement_id).compact).index_by(&:id) champs_siret.each do |champ| etablissement = etablissements_by_id[champ.etablissement_id] champ.association(:etablissement).target = etablissement diff --git a/lib/tasks/recovery.rake b/lib/tasks/recovery.rake new file mode 100644 index 000000000..b5696d83c --- /dev/null +++ b/lib/tasks/recovery.rake @@ -0,0 +1,38 @@ +namespace :recovery do + desc <<~USAGE + given a file path, read it as json data, preload dossier data and export to marshal.dump. + the given file should be a json formatted as follow + { + procedure_id_1: [ + dossier_id_1, + dossier_id_2, + ... + ], + procedure_id_2: [ + ... + ], + ... + } + ex: rails recovery:export[missing_dossier_ids_per_procedure.json] + USAGE + task :export, [:file_path] => :environment do |_t, args| + dossier_ids = JSON.parse(File.read(args[:file_path])).values.flatten + rake_puts "Expecting to generate a dump with #{dossier_ids.size} dossiers" + exporter = Recovery::Exporter.new(dossier_ids:) + rake_puts "Found on db #{exporter.dossiers.size} dossiers" + exporter.dump + rake_puts "Export done, see: #{exporter.file_path}" + end + + desc <<~USAGE + given a file path, read it as marshal data + the given file should be the result of recover:export + ex: rails recovery:import[/absolute/path/to/lib/data/export.dump] + USAGE + task :import, [:file_path] => :environment do |_t, args| + importer = Recovery::Importer.new(file_path: args[:file_path]) + rake_puts "Expecting to load #{importer.dossiers.size} dossiers" + importer.load + rake_puts "Mise à jour terminée" + end +end diff --git a/spec/factories/avis.rb b/spec/factories/avis.rb index 0a16c7528..dde9a3b22 100644 --- a/spec/factories/avis.rb +++ b/spec/factories/avis.rb @@ -29,5 +29,13 @@ FactoryBot.define do trait :with_answer do answer { "Mon avis se décompose en deux points :\n- La demande semble pertinente\n- Le demandeur remplit les conditions." } end + + trait :with_introduction do + introduction_file { Rack::Test::UploadedFile.new('spec/fixtures/files/white.png', 'image/png') } + end + + trait :with_piece_justificative do + piece_justificative_file { Rack::Test::UploadedFile.new('spec/fixtures/files/white.png', 'image/png') } + end end end diff --git a/spec/factories/dossier_operation_log.rb b/spec/factories/dossier_operation_log.rb index ff6a80a53..191da8c27 100644 --- a/spec/factories/dossier_operation_log.rb +++ b/spec/factories/dossier_operation_log.rb @@ -1,5 +1,9 @@ FactoryBot.define do factory :dossier_operation_log do operation { :passer_en_instruction } + + trait :with_serialized do + serialized { Rack::Test::UploadedFile.new('spec/fixtures/files/white.png', 'image/png') } + end end end diff --git a/spec/lib/recovery/exporter_spec.rb b/spec/lib/recovery/exporter_spec.rb new file mode 100644 index 000000000..231e3d174 --- /dev/null +++ b/spec/lib/recovery/exporter_spec.rb @@ -0,0 +1,23 @@ +describe Recovery::Exporter do + let(:dossier_ids) { [create(:dossier, :with_individual).id, create(:dossier, :with_individual).id] } + let(:fp) { Rails.root.join('spec', 'fixtures', 'export.dump') } + subject { Recovery::Exporter.new(dossier_ids:, file_path: fp).dump } + + def cleanup_export_file + if File.exist?(fp) + FileUtils.rm(fp) + end + end + + before { cleanup_export_file } + after { cleanup_export_file } + + it 'exports dossiers to .dump' do + expect { subject }.not_to raise_error + end + + it 'exports dossiers local file .dump' do + expect { subject }.to change { File.exist?(fp) } + .from(false).to(true) + end +end diff --git a/spec/lib/recovery/life_cycle_spec.rb b/spec/lib/recovery/life_cycle_spec.rb new file mode 100644 index 000000000..8209ada1f --- /dev/null +++ b/spec/lib/recovery/life_cycle_spec.rb @@ -0,0 +1,119 @@ +describe 'Recovery::LifeCycle' do + describe '.load_export_destroy_and_import' do + let(:procedure) do + create(:procedure, + types_de_champ_public: [ + { type: :repetition, children: [{ type: :piece_justificative }] }, + { type: :carte }, + { type: :siret } + ]) + end + + let(:some_file) { Rack::Test::UploadedFile.new('spec/fixtures/files/white.png', 'image/png') } + let(:geo_area) { build(:geo_area, :selection_utilisateur, :polygon) } + let(:fp) { Rails.root.join('spec', 'fixtures', 'export.dump') } + let(:dossier) do + d = create(:dossier, procedure:) + + repetition(d).add_row(d.revision) + pj_champ(d).piece_justificative_file.attach(some_file) + carte(d).update(geo_areas: [geo_area]) + d.etablissement = create(:etablissement, :with_exercices) + d.etablissement.entreprise_attestation_sociale.attach(some_file) + d.etablissement.entreprise_attestation_fiscale.attach(some_file) + + siret(d).update(etablissement: create(:etablissement, :with_exercices)) + siret(d).etablissement.entreprise_attestation_sociale.attach(some_file) + siret(d).etablissement.entreprise_attestation_fiscale.attach(some_file) + + d.individual = build(:individual) + + d.attestation = build(:attestation, :with_pdf) + d.justificatif_motivation.attach(some_file) + + d.commentaires << build(:commentaire, :with_file) + + d.invites << build(:invite, :with_user) + + d.avis << build(:avis, :with_introduction, :with_piece_justificative) + + d.traitements.accepter(motivation: 'oui', processed_at: Time.zone.now) + d.save + + d.dossier_operation_logs << build(:dossier_operation_log, :with_serialized) + + d.transfer_logs.create(from: create(:user), to: create(:user)) + + d + end + + def repetition(d) = d.champs.find_by(type: "Champs::RepetitionChamp") + def pj_champ(d) = d.champs.find_by(type: "Champs::PieceJustificativeChamp") + def carte(d) = d.champs.find_by(type: "Champs::CarteChamp") + def siret(d) = d.champs.find_by(type: "Champs::SiretChamp") + + def cleanup_export_file + if File.exist?(fp) + FileUtils.rm(fp) + end + end + let(:instructeur) { create(:instructeur) } + + before do + instructeur.followed_dossiers << dossier + cleanup_export_file + end + + after { cleanup_export_file } + it 'reloads the full grappe' do + expect(Dossier.count).to eq(1) + expect(Dossier.first.champs.count).not_to be(0) + + @dossier_ids = Dossier.ids + + Recovery::Exporter.new(dossier_ids: @dossier_ids, file_path: fp).dump + Dossier.where(id: @dossier_ids).destroy_all + Recovery::Importer.new(file_path: fp).load + + expect(Dossier.count).to eq(1) + + reloaded_dossier = Dossier.first + + expect(reloaded_dossier.champs.count).not_to be(0) + + expect(repetition(reloaded_dossier).champs.map(&:type)).to match_array(["Champs::PieceJustificativeChamp"]) + expect(pj_champ(reloaded_dossier).piece_justificative_file).to be_attached + expect(carte(reloaded_dossier).geo_areas).to be_present + + expect(reloaded_dossier.etablissement.exercices).to be_present + + # launch a job + # expect(reloaded_dossier.etablissement.entreprise_attestation_sociale).to be_attached + # expect(reloaded_dossier.etablissement.entreprise_attestation_fiscale).to be_attached + + expect(siret(reloaded_dossier).etablissement.exercices).to be_present + + # launch a job + # expect(siret(reloaded_dossier).etablissement.entreprise_attestation_sociale).to be_attached + # expect(siret(reloaded_dossier).etablissement.entreprise_attestation_fiscale).to be_attached + + expect(reloaded_dossier.individual).to be_present + expect(reloaded_dossier.attestation.pdf).to be_attached + expect(reloaded_dossier.justificatif_motivation).to be_attached + + expect(reloaded_dossier.commentaires.first.piece_jointe).to be_attached + + expect(reloaded_dossier.invites.first.user).to be_present + expect(reloaded_dossier.followers_instructeurs).to match_array([instructeur]) + + expect(reloaded_dossier.avis.first.introduction_file).to be_attached + expect(reloaded_dossier.avis.first.piece_justificative_file).to be_attached + + expect(reloaded_dossier.traitements).to be_present + + expect(reloaded_dossier.dossier_operation_logs.first.serialized).to be_attached + + expect(reloaded_dossier.transfer_logs).to be_present + end + end +end