From 21ac60ad04fdb0abca6e6ab5ea73615b4a385980 Mon Sep 17 00:00:00 2001 From: Pierre de La Morinerie Date: Thu, 16 May 2019 15:26:58 +0200 Subject: [PATCH 1/2] tasks: add a task to migrate pjs of procedures in batches --- ...to_champ_piece_jointe_migration_service.rb | 5 +++ .../2019_03_13_migrate_pjs_to_champs.rake | 8 ----- lib/tasks/pieces_justificatives.rake | 28 ++++++++++++++++ spec/lib/tasks/pieces_justificatives_spec.rb | 32 +++++++++++++++++++ 4 files changed, 65 insertions(+), 8 deletions(-) delete mode 100644 lib/tasks/2019_03_13_migrate_pjs_to_champs.rake create mode 100644 lib/tasks/pieces_justificatives.rake create mode 100644 spec/lib/tasks/pieces_justificatives_spec.rb diff --git a/app/services/piece_justificative_to_champ_piece_jointe_migration_service.rb b/app/services/piece_justificative_to_champ_piece_jointe_migration_service.rb index 0778454f3..52314b697 100644 --- a/app/services/piece_justificative_to_champ_piece_jointe_migration_service.rb +++ b/app/services/piece_justificative_to_champ_piece_jointe_migration_service.rb @@ -9,6 +9,11 @@ class PieceJustificativeToChampPieceJointeMigrationService storage_service.ensure_openstack_copy_possible!(PieceJustificativeUploader) end + def procedures_with_pjs_in_range(ids_range) + procedures_with_pj = Procedure.unscope(where: :hidden_at).joins(:types_de_piece_justificative).distinct + procedures_with_pj.where(id: ids_range) + end + def convert_procedure_pjs_to_champ_pjs(procedure) types_de_champ_pj = PiecesJustificativesService.types_pj_as_types_de_champ(procedure) populate_champs_pjs!(procedure, types_de_champ_pj) diff --git a/lib/tasks/2019_03_13_migrate_pjs_to_champs.rake b/lib/tasks/2019_03_13_migrate_pjs_to_champs.rake deleted file mode 100644 index ad8194116..000000000 --- a/lib/tasks/2019_03_13_migrate_pjs_to_champs.rake +++ /dev/null @@ -1,8 +0,0 @@ -namespace :'2019_03_13_migrate_pjs_to_champs' do - task run: :environment do - procedure_id = ENV['PROCEDURE_ID'] - service = PieceJustificativeToChampPieceJointeMigrationService.new - service.ensure_correct_storage_configuration! - service.convert_procedure_pjs_to_champ_pjs(Procedure.find(procedure_id)) - end -end diff --git a/lib/tasks/pieces_justificatives.rake b/lib/tasks/pieces_justificatives.rake new file mode 100644 index 000000000..a30855090 --- /dev/null +++ b/lib/tasks/pieces_justificatives.rake @@ -0,0 +1,28 @@ +require Rails.root.join("lib", "tasks", "task_helper") + +namespace :pieces_justificatives do + task migrate_procedure_to_champs: :environment do + procedure_id = ENV['PROCEDURE_ID'] + service = PieceJustificativeToChampPieceJointeMigrationService.new + service.ensure_correct_storage_configuration! + service.convert_procedure_pjs_to_champ_pjs(Procedure.find(procedure_id)) + end + + task migrate_procedures_range_to_champs: :environment do + if ENV['RANGE_START'].nil? || ENV['RANGE_END'].nil? + fail "RANGE_START and RANGE_END must be specified" + end + procedures_range = ENV['RANGE_START']..ENV['RANGE_END'] + + service = PieceJustificativeToChampPieceJointeMigrationService.new + service.ensure_correct_storage_configuration! + procedures_to_migrate = service.procedures_with_pjs_in_range(procedures_range) + + procedures_to_migrate.find_each do |procedure| + rake_puts '' + rake_puts "Migrating procedure #{procedure.id}…" + + service.convert_procedure_pjs_to_champ_pjs(procedure) + end + end +end diff --git a/spec/lib/tasks/pieces_justificatives_spec.rb b/spec/lib/tasks/pieces_justificatives_spec.rb new file mode 100644 index 000000000..5031c772d --- /dev/null +++ b/spec/lib/tasks/pieces_justificatives_spec.rb @@ -0,0 +1,32 @@ +describe 'pieces_justificatives' do + describe 'migrate_procedures_range_to_champs' do + let(:rake_task) { Rake::Task['pieces_justificatives:migrate_procedures_range_to_champs'] } + let(:procedure_in_range_1) { create(:procedure, :with_two_type_de_piece_justificative) } + let(:procedure_in_range_2) { create(:procedure, :with_two_type_de_piece_justificative) } + let(:procedure_out_of_range) { create(:procedure, :with_two_type_de_piece_justificative) } + + before do + procedure_in_range_1 + procedure_in_range_2 + procedure_out_of_range + + ENV['RANGE_START'] = procedure_in_range_1.id.to_s + ENV['RANGE_END'] = procedure_in_range_2.id.to_s + + allow_any_instance_of(PieceJustificativeToChampPieceJointeMigrationService).to receive(:ensure_correct_storage_configuration!) + + rake_task.invoke + end + + after { rake_task.reenable } + + it 'migrates procedures in the ids range' do + expect(procedure_in_range_1.reload.types_de_piece_justificative).to be_empty + expect(procedure_in_range_2.reload.types_de_piece_justificative).to be_empty + end + + it 'doesn’t migrate procedures not in the range' do + expect(procedure_out_of_range.reload.types_de_piece_justificative).to be_present + end + end +end From 4cf54e0d28acec96bb8bb17df6ebc8c8e2902891 Mon Sep 17 00:00:00 2001 From: Pierre de La Morinerie Date: Thu, 16 May 2019 13:37:33 +0000 Subject: [PATCH 2/2] tasks: add progress report to the pjs migration task Progress is indicated per migrated champ. --- ...to_champ_piece_jointe_migration_service.rb | 12 +++- lib/tasks/pieces_justificatives.rake | 22 +++++++- spec/lib/tasks/pieces_justificatives_spec.rb | 19 +++++++ ...amp_piece_jointe_migration_service_spec.rb | 56 +++++++++++++------ 4 files changed, 88 insertions(+), 21 deletions(-) diff --git a/app/services/piece_justificative_to_champ_piece_jointe_migration_service.rb b/app/services/piece_justificative_to_champ_piece_jointe_migration_service.rb index 52314b697..30b025201 100644 --- a/app/services/piece_justificative_to_champ_piece_jointe_migration_service.rb +++ b/app/services/piece_justificative_to_champ_piece_jointe_migration_service.rb @@ -14,9 +14,13 @@ class PieceJustificativeToChampPieceJointeMigrationService procedures_with_pj.where(id: ids_range) end - def convert_procedure_pjs_to_champ_pjs(procedure) + def number_of_champs_to_migrate(procedure) + (procedure.types_de_piece_justificative.count + 1) * procedure.dossiers.unscope(where: :hidden_at).count + end + + def convert_procedure_pjs_to_champ_pjs(procedure, &progress) types_de_champ_pj = PiecesJustificativesService.types_pj_as_types_de_champ(procedure) - populate_champs_pjs!(procedure, types_de_champ_pj) + populate_champs_pjs!(procedure, types_de_champ_pj, &progress) # Only destroy the old types PJ once everything has been safely migrated to # champs PJs. Destroying the types PJ will cascade and destroy the PJs, @@ -29,7 +33,7 @@ class PieceJustificativeToChampPieceJointeMigrationService @storage_service ||= CarrierwaveActiveStorageMigrationService.new end - def populate_champs_pjs!(procedure, types_de_champ_pj) + def populate_champs_pjs!(procedure, types_de_champ_pj, &progress) procedure.types_de_champ += types_de_champ_pj # Unscope to make sure all dossiers are migrated, even the soft-deleted ones @@ -54,6 +58,8 @@ class PieceJustificativeToChampPieceJointeMigrationService created_at: dossier.created_at ) end + + yield if block_given? end end rescue diff --git a/lib/tasks/pieces_justificatives.rake b/lib/tasks/pieces_justificatives.rake index a30855090..49fe00193 100644 --- a/lib/tasks/pieces_justificatives.rake +++ b/lib/tasks/pieces_justificatives.rake @@ -3,9 +3,18 @@ require Rails.root.join("lib", "tasks", "task_helper") namespace :pieces_justificatives do task migrate_procedure_to_champs: :environment do procedure_id = ENV['PROCEDURE_ID'] + procedure = Procedure.find(procedure_id) + service = PieceJustificativeToChampPieceJointeMigrationService.new service.ensure_correct_storage_configuration! - service.convert_procedure_pjs_to_champ_pjs(Procedure.find(procedure_id)) + + progress = ProgressReport.new(service.number_of_champs_to_migrate(procedure)) + + service.convert_procedure_pjs_to_champ_pjs(procedure) do + progress.inc + end + + progress.finish end task migrate_procedures_range_to_champs: :environment do @@ -18,11 +27,20 @@ namespace :pieces_justificatives do service.ensure_correct_storage_configuration! procedures_to_migrate = service.procedures_with_pjs_in_range(procedures_range) + total_number_of_champs_to_migrate = procedures_to_migrate + .map { |p| service.number_of_champs_to_migrate(p) } + .sum + progress = ProgressReport.new(total_number_of_champs_to_migrate) + procedures_to_migrate.find_each do |procedure| rake_puts '' rake_puts "Migrating procedure #{procedure.id}…" - service.convert_procedure_pjs_to_champ_pjs(procedure) + service.convert_procedure_pjs_to_champ_pjs(procedure) do + progress.inc + end end + + progress.finish end end diff --git a/spec/lib/tasks/pieces_justificatives_spec.rb b/spec/lib/tasks/pieces_justificatives_spec.rb index 5031c772d..fc929d51d 100644 --- a/spec/lib/tasks/pieces_justificatives_spec.rb +++ b/spec/lib/tasks/pieces_justificatives_spec.rb @@ -1,4 +1,23 @@ describe 'pieces_justificatives' do + describe 'migrate_procedure_to_champs' do + let(:rake_task) { Rake::Task['pieces_justificatives:migrate_procedure_to_champs'] } + let(:procedure) { create(:procedure, :with_two_type_de_piece_justificative) } + + before do + ENV['PROCEDURE_ID'] = procedure.id.to_s + + allow_any_instance_of(PieceJustificativeToChampPieceJointeMigrationService).to receive(:ensure_correct_storage_configuration!) + + rake_task.invoke + end + + after { rake_task.reenable } + + it 'migrates the procedure' do + expect(procedure.reload.types_de_piece_justificative).to be_empty + end + end + describe 'migrate_procedures_range_to_champs' do let(:rake_task) { Rake::Task['pieces_justificatives:migrate_procedures_range_to_champs'] } let(:procedure_in_range_1) { create(:procedure, :with_two_type_de_piece_justificative) } diff --git a/spec/services/piece_justificative_to_champ_piece_jointe_migration_service_spec.rb b/spec/services/piece_justificative_to_champ_piece_jointe_migration_service_spec.rb index 9f7f83e34..bbd94d8ef 100644 --- a/spec/services/piece_justificative_to_champ_piece_jointe_migration_service_spec.rb +++ b/spec/services/piece_justificative_to_champ_piece_jointe_migration_service_spec.rb @@ -9,16 +9,17 @@ describe PieceJustificativeToChampPieceJointeMigrationService do let(:procedure) { create(:procedure, types_de_piece_justificative: types_pj) } let(:types_pj) { [create(:type_de_piece_justificative)] } - let!(:dossier) do - create( - :dossier, - procedure: procedure, - pieces_justificatives: pjs - ) - end + let!(:dossier) { make_dossier } let(:pjs) { [] } + def make_dossier(hidden: false) + create(:dossier, + procedure: procedure, + pieces_justificatives: pjs, + hidden_at: hidden ? Time.zone.now : nil) + end + def make_pjs types_pj.map do |tpj| create(:piece_justificative, :contrat, type_de_piece_justificative: tpj) @@ -31,6 +32,22 @@ describe PieceJustificativeToChampPieceJointeMigrationService do expect(storage_service).to receive(:make_attachment) end + describe '.number_of_champs_to_migrate' do + let!(:other_dossier) { make_dossier } + + it 'reports the numbers of champs to be migrated' do + expect(service.number_of_champs_to_migrate(procedure)).to eq(4) + end + + context 'when the procedure has hidden dossiers' do + let!(:hidden_dossier) { make_dossier(hidden: true) } + + it 'reports the numbers of champs including those of hidden dossiers' do + expect(service.number_of_champs_to_migrate(procedure)).to eq(6) + end + end + end + context 'when conversion succeeds' do context 'for the procedure' do it 'types de champ are created for the "pièces jointes" header and for each PJ' do @@ -114,19 +131,26 @@ describe PieceJustificativeToChampPieceJointeMigrationService do .to change { dossier.pieces_justificatives.count } .to(0) end + + context 'when the procedure has several dossiers' do + let!(:other_dossier) { make_dossier } + + it 'sends progress callback for each migrated champ' do + number_of_champs_to_migrate = service.number_of_champs_to_migrate(procedure) + + progress_count = 0 + service.convert_procedure_pjs_to_champ_pjs(procedure) do + progress_count += 1 + end + + expect(progress_count).to eq(number_of_champs_to_migrate) + end + end end context 'when the dossier is soft-deleted it still gets converted' do let(:pjs) { make_pjs } - - let!(:dossier) do - create( - :dossier, - procedure: procedure, - pieces_justificatives: pjs, - hidden_at: Time.zone.now - ) - end + let!(:dossier) { make_dossier(hidden: true) } before { expect_storage_service_to_convert_object }