Merge pull request #3880 from betagouv/batch-migrate-pjs

Ajout d'une tâche pour migrer les pièces justificatives de plusieurs démarches

app/services/piece_justificative_to_champ_piece_jointe_migration_service.rb

@@ -9,9 +9,18 @@ class PieceJustificativeToChampPieceJointeMigrationService
     storage_service.ensure_openstack_copy_possible!(PieceJustificativeUploader)
   end
 
-  def convert_procedure_pjs_to_champ_pjs(procedure)
+  def procedures_with_pjs_in_range(ids_range)
+    procedures_with_pj = Procedure.unscope(where: :hidden_at).joins(:types_de_piece_justificative).distinct
+    procedures_with_pj.where(id: ids_range)
+  end
+
+  def number_of_champs_to_migrate(procedure)
+    (procedure.types_de_piece_justificative.count + 1) * procedure.dossiers.unscope(where: :hidden_at).count
+  end
+
+  def convert_procedure_pjs_to_champ_pjs(procedure, &progress)
     types_de_champ_pj = PiecesJustificativesService.types_pj_as_types_de_champ(procedure)
-    populate_champs_pjs!(procedure, types_de_champ_pj)
+    populate_champs_pjs!(procedure, types_de_champ_pj, &progress)
 
     # Only destroy the old types PJ once everything has been safely migrated to
     # champs PJs. Destroying the types PJ will cascade and destroy the PJs,
@@ -24,7 +33,7 @@ class PieceJustificativeToChampPieceJointeMigrationService
     @storage_service ||= CarrierwaveActiveStorageMigrationService.new
   end
 
-  def populate_champs_pjs!(procedure, types_de_champ_pj)
+  def populate_champs_pjs!(procedure, types_de_champ_pj, &progress)
     procedure.types_de_champ += types_de_champ_pj
 
     # Unscope to make sure all dossiers are migrated, even the soft-deleted ones
@@ -49,6 +58,8 @@ class PieceJustificativeToChampPieceJointeMigrationService
             created_at: dossier.created_at
           )
         end
+
+        yield if block_given?
       end
     end
   rescue

lib/tasks/2019_03_13_migrate_pjs_to_champs.rake

@@ -1,8 +0,0 @@
-namespace :'2019_03_13_migrate_pjs_to_champs' do
-  task run: :environment do
-    procedure_id = ENV['PROCEDURE_ID']
-    service = PieceJustificativeToChampPieceJointeMigrationService.new
-    service.ensure_correct_storage_configuration!
-    service.convert_procedure_pjs_to_champ_pjs(Procedure.find(procedure_id))
-  end
-end

lib/tasks/pieces_justificatives.rake

@@ -0,0 +1,46 @@
+require Rails.root.join("lib", "tasks", "task_helper")
+
+namespace :pieces_justificatives do
+  task migrate_procedure_to_champs: :environment do
+    procedure_id = ENV['PROCEDURE_ID']
+    procedure = Procedure.find(procedure_id)
+
+    service = PieceJustificativeToChampPieceJointeMigrationService.new
+    service.ensure_correct_storage_configuration!
+
+    progress = ProgressReport.new(service.number_of_champs_to_migrate(procedure))
+
+    service.convert_procedure_pjs_to_champ_pjs(procedure) do
+      progress.inc
+    end
+
+    progress.finish
+  end
+
+  task migrate_procedures_range_to_champs: :environment do
+    if ENV['RANGE_START'].nil? || ENV['RANGE_END'].nil?
+      fail "RANGE_START and RANGE_END must be specified"
+    end
+    procedures_range = ENV['RANGE_START']..ENV['RANGE_END']
+
+    service = PieceJustificativeToChampPieceJointeMigrationService.new
+    service.ensure_correct_storage_configuration!
+    procedures_to_migrate = service.procedures_with_pjs_in_range(procedures_range)
+
+    total_number_of_champs_to_migrate = procedures_to_migrate
+      .map { |p| service.number_of_champs_to_migrate(p) }
+      .sum
+    progress = ProgressReport.new(total_number_of_champs_to_migrate)
+
+    procedures_to_migrate.find_each do |procedure|
+      rake_puts ''
+      rake_puts "Migrating procedure #{procedure.id}…"
+
+      service.convert_procedure_pjs_to_champ_pjs(procedure) do
+        progress.inc
+      end
+    end
+
+    progress.finish
+  end
+end

spec/lib/tasks/pieces_justificatives_spec.rb

@@ -0,0 +1,51 @@
+describe 'pieces_justificatives' do
+  describe 'migrate_procedure_to_champs' do
+    let(:rake_task) { Rake::Task['pieces_justificatives:migrate_procedure_to_champs'] }
+    let(:procedure) { create(:procedure, :with_two_type_de_piece_justificative) }
+
+    before do
+      ENV['PROCEDURE_ID'] = procedure.id.to_s
+
+      allow_any_instance_of(PieceJustificativeToChampPieceJointeMigrationService).to receive(:ensure_correct_storage_configuration!)
+
+      rake_task.invoke
+    end
+
+    after { rake_task.reenable }
+
+    it 'migrates the procedure' do
+      expect(procedure.reload.types_de_piece_justificative).to be_empty
+    end
+  end
+
+  describe 'migrate_procedures_range_to_champs' do
+    let(:rake_task) { Rake::Task['pieces_justificatives:migrate_procedures_range_to_champs'] }
+    let(:procedure_in_range_1) { create(:procedure, :with_two_type_de_piece_justificative) }
+    let(:procedure_in_range_2) { create(:procedure, :with_two_type_de_piece_justificative) }
+    let(:procedure_out_of_range) { create(:procedure, :with_two_type_de_piece_justificative) }
+
+    before do
+      procedure_in_range_1
+      procedure_in_range_2
+      procedure_out_of_range
+
+      ENV['RANGE_START'] = procedure_in_range_1.id.to_s
+      ENV['RANGE_END'] = procedure_in_range_2.id.to_s
+
+      allow_any_instance_of(PieceJustificativeToChampPieceJointeMigrationService).to receive(:ensure_correct_storage_configuration!)
+
+      rake_task.invoke
+    end
+
+    after { rake_task.reenable }
+
+    it 'migrates procedures in the ids range' do
+      expect(procedure_in_range_1.reload.types_de_piece_justificative).to be_empty
+      expect(procedure_in_range_2.reload.types_de_piece_justificative).to be_empty
+    end
+
+    it 'doesn’t migrate procedures not in the range' do
+      expect(procedure_out_of_range.reload.types_de_piece_justificative).to be_present
+    end
+  end
+end

spec/services/piece_justificative_to_champ_piece_jointe_migration_service_spec.rb

@@ -9,16 +9,17 @@ describe PieceJustificativeToChampPieceJointeMigrationService do
   let(:procedure) { create(:procedure, types_de_piece_justificative: types_pj) }
   let(:types_pj) { [create(:type_de_piece_justificative)] }
 
-  let!(:dossier) do
+  let!(:dossier) { make_dossier }
-    create(
-      :dossier,
-      procedure: procedure,
-      pieces_justificatives: pjs
-    )
-  end
 
   let(:pjs) { [] }
 
+  def make_dossier(hidden: false)
+    create(:dossier,
+      procedure: procedure,
+      pieces_justificatives: pjs,
+      hidden_at: hidden ? Time.zone.now : nil)
+  end
+
   def make_pjs
     types_pj.map do |tpj|
       create(:piece_justificative, :contrat, type_de_piece_justificative: tpj)
@@ -31,6 +32,22 @@ describe PieceJustificativeToChampPieceJointeMigrationService do
     expect(storage_service).to receive(:make_attachment)
   end
 
+  describe '.number_of_champs_to_migrate' do
+    let!(:other_dossier) { make_dossier }
+
+    it 'reports the numbers of champs to be migrated' do
+      expect(service.number_of_champs_to_migrate(procedure)).to eq(4)
+    end
+
+    context 'when the procedure has hidden dossiers' do
+      let!(:hidden_dossier) { make_dossier(hidden: true) }
+
+      it 'reports the numbers of champs including those of hidden dossiers' do
+        expect(service.number_of_champs_to_migrate(procedure)).to eq(6)
+      end
+    end
+  end
+
   context 'when conversion succeeds' do
     context 'for the procedure' do
       it 'types de champ are created for the "pièces jointes" header and for each PJ' do
@@ -114,19 +131,26 @@ describe PieceJustificativeToChampPieceJointeMigrationService do
           .to change { dossier.pieces_justificatives.count }
           .to(0)
       end
+
+      context 'when the procedure has several dossiers' do
+        let!(:other_dossier) { make_dossier }
+
+        it 'sends progress callback for each migrated champ' do
+          number_of_champs_to_migrate = service.number_of_champs_to_migrate(procedure)
+
+          progress_count = 0
+          service.convert_procedure_pjs_to_champ_pjs(procedure) do
+            progress_count += 1
+          end
+
+          expect(progress_count).to eq(number_of_champs_to_migrate)
+        end
+      end
     end
 
     context 'when the dossier is soft-deleted it still gets converted' do
       let(:pjs) { make_pjs }
-
+      let!(:dossier) { make_dossier(hidden: true) }
-      let!(:dossier) do
-        create(
-          :dossier,
-          procedure: procedure,
-          pieces_justificatives: pjs,
-          hidden_at: Time.zone.now
-        )
-      end
 
       before { expect_storage_service_to_convert_object }