From 55dfba68a3312b443b90c59ef50792a9558b0834 Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Tue, 9 Oct 2018 18:16:21 +0200 Subject: [PATCH 01/14] [#2772] Extract method --- lib/tasks/dossier_procedure_migrator.rb | 36 ++++++++++++++----------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/tasks/dossier_procedure_migrator.rb b/lib/tasks/dossier_procedure_migrator.rb index ae4d77c12..ff006f8c0 100644 --- a/lib/tasks/dossier_procedure_migrator.rb +++ b/lib/tasks/dossier_procedure_migrator.rb @@ -21,6 +21,25 @@ module Tasks @source_to_destination_mapping[champ.type_de_champ.order_place] end + def migrate_champs(dossier) + # Since we’re going to iterate and change the champs at the same time, + # we use to_a to make the list static and avoid nasty surprises + original_champs = dossier.champs.to_a + + compute_new_champs(dossier) + + original_champs.each do |c| + tdc_to = destination_type_de_champ(c) + if tdc_to.present? + c.update(type_de_champ: tdc_to) + elsif discard_champ?(c) + dossier.champs.destroy(c) + else + fail "Unhandled source type de champ #{c.type_de_champ.order_place}" + end + end + end + def discard_champ?(champ) @source_champs_to_discard.member?(champ.type_de_champ.order_place) end @@ -124,22 +143,7 @@ module Tasks def migrate_dossiers @source_procedure.dossiers.find_each(batch_size: 100) do |d| - # Since we’re going to iterate and change the champs at the same time, - # we use to_a to make the list static and avoid nasty surprises - original_champs = d.champs.to_a - - @champ_mapping.compute_new_champs(d) - - original_champs.each do |c| - tdc_to = @champ_mapping.destination_type_de_champ(c) - if tdc_to.present? - c.update(type_de_champ: tdc_to) - elsif @champ_mapping.discard_champ?(c) - d.champs.destroy(c) - else - fail "Unhandled source type de champ #{c.type_de_champ.order_place}" - end - end + @champ_mapping.migrate_champs(d) # Use update_columns to avoid triggering build_default_champs d.update_columns(procedure_id: @destination_procedure.id) From 11bd34289792c687bbd6e2ac679bf3ca46cd1950 Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Wed, 10 Oct 2018 13:56:35 +0200 Subject: [PATCH 02/14] [#2772] Helper methods can now be private --- lib/tasks/dossier_procedure_migrator.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/tasks/dossier_procedure_migrator.rb b/lib/tasks/dossier_procedure_migrator.rb index ff006f8c0..4fbd5b715 100644 --- a/lib/tasks/dossier_procedure_migrator.rb +++ b/lib/tasks/dossier_procedure_migrator.rb @@ -17,10 +17,6 @@ module Tasks @destination_champ_computations = [] end - def destination_type_de_champ(champ) - @source_to_destination_mapping[champ.type_de_champ.order_place] - end - def migrate_champs(dossier) # Since we’re going to iterate and change the champs at the same time, # we use to_a to make the list static and avoid nasty surprises @@ -40,9 +36,7 @@ module Tasks end end - def discard_champ?(champ) - @source_champs_to_discard.member?(champ.type_de_champ.order_place) - end + private def compute_new_champs(dossier) @destination_champ_computations.each do |tdc, block| @@ -50,7 +44,13 @@ module Tasks end end - private + def destination_type_de_champ(champ) + @source_to_destination_mapping[champ.type_de_champ.order_place] + end + + def discard_champ?(champ) + @source_champs_to_discard.member?(champ.type_de_champ.order_place) + end def map_source_to_destination_champ(source_order_place, destination_order_place, source_overrides: {}, destination_overrides: {}) destination_type_de_champ = @destination_procedure.types_de_champ.find_by(order_place: destination_order_place) From 968ccce7f05c6b000d936248ee4f47ce1c0d7a9e Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Tue, 9 Oct 2018 18:58:26 +0200 Subject: [PATCH 03/14] [#2772] Move method to champ class --- lib/tasks/dossier_procedure_migrator.rb | 69 ++++++++++++------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/lib/tasks/dossier_procedure_migrator.rb b/lib/tasks/dossier_procedure_migrator.rb index 4fbd5b715..380a8f016 100644 --- a/lib/tasks/dossier_procedure_migrator.rb +++ b/lib/tasks/dossier_procedure_migrator.rb @@ -3,9 +3,6 @@ module Tasks # Migrates dossiers from an old source procedure to a revised destination procedure. class ChampMapping - attr_reader :expected_source_types_de_champ - attr_reader :expected_destination_types_de_champ - def initialize(source_procedure, destination_procedure) @source_procedure = source_procedure @destination_procedure = destination_procedure @@ -17,6 +14,11 @@ module Tasks @destination_champ_computations = [] end + def check_source_destination_champs_consistency + check_champs_consistency('source', @expected_source_types_de_champ, @source_procedure.types_de_champ) + check_champs_consistency('destination', @expected_destination_types_de_champ, @destination_procedure.types_de_champ) + end + def migrate_champs(dossier) # Since we’re going to iterate and change the champs at the same time, # we use to_a to make the list static and avoid nasty surprises @@ -52,6 +54,33 @@ module Tasks @source_champs_to_discard.member?(champ.type_de_champ.order_place) end + def check_champs_consistency(label, expected_tdcs, actual_tdcs) + if actual_tdcs.size != expected_tdcs.size + raise "Incorrect #{label} size #{actual_tdcs.size} (expected #{expected_tdcs.size})" + end + actual_tdcs.each { |tdc| check_champ_consistency(label, expected_tdcs[tdc.order_place], tdc) } + end + + def check_champ_consistency(label, expected_tdc, actual_tdc) + errors = [] + if actual_tdc.libelle != expected_tdc['libelle'] + errors.append("incorrect libelle #{actual_tdc.libelle} (expected #{expected_tdc['libelle']})") + end + if actual_tdc.type_champ != expected_tdc['type_champ'] + errors.append("incorrect type champ #{actual_tdc.type_champ} (expected #{expected_tdc['type_champ']})") + end + if (!actual_tdc.mandatory) && expected_tdc['mandatory'] + errors.append("champ should be mandatory") + end + drop_down = actual_tdc.drop_down_list.presence&.options&.presence + if drop_down != expected_tdc['drop_down'] + errors.append("incorrect drop down list #{drop_down} (expected #{expected_tdc['drop_down']})") + end + if errors.present? + fail "On #{label} type de champ #{actual_tdc.order_place} (#{actual_tdc.libelle}) " + errors.join(', ') + end + end + def map_source_to_destination_champ(source_order_place, destination_order_place, source_overrides: {}, destination_overrides: {}) destination_type_de_champ = @destination_procedure.types_de_champ.find_by(order_place: destination_order_place) @expected_source_types_de_champ[source_order_place] = @@ -100,7 +129,7 @@ module Tasks def check_consistency check_same_administrateur - check_source_destination_champs_consistency + @champ_mapping.check_source_destination_champs_consistency end def check_same_administrateur @@ -109,38 +138,6 @@ module Tasks end end - def check_source_destination_champs_consistency - check_champs_consistency('source', @champ_mapping.expected_source_types_de_champ, @source_procedure.types_de_champ) - check_champs_consistency('destination', @champ_mapping.expected_destination_types_de_champ, @destination_procedure.types_de_champ) - end - - def check_champs_consistency(label, expected_tdcs, actual_tdcs) - if actual_tdcs.size != expected_tdcs.size - raise "Incorrect #{label} size #{actual_tdcs.size} (expected #{expected_tdcs.size})" - end - actual_tdcs.each { |tdc| check_champ_consistency(label, expected_tdcs[tdc.order_place], tdc) } - end - - def check_champ_consistency(label, expected_tdc, actual_tdc) - errors = [] - if actual_tdc.libelle != expected_tdc['libelle'] - errors.append("incorrect libelle #{actual_tdc.libelle} (expected #{expected_tdc['libelle']})") - end - if actual_tdc.type_champ != expected_tdc['type_champ'] - errors.append("incorrect type champ #{actual_tdc.type_champ} (expected #{expected_tdc['type_champ']})") - end - if (!actual_tdc.mandatory) && expected_tdc['mandatory'] - errors.append("champ should be mandatory") - end - drop_down = actual_tdc.drop_down_list.presence&.options&.presence - if drop_down != expected_tdc['drop_down'] - errors.append("incorrect drop down list #{drop_down} (expected #{expected_tdc['drop_down']})") - end - if errors.present? - fail "On #{label} type de champ #{actual_tdc.order_place} (#{actual_tdc.libelle}) " + errors.join(', ') - end - end - def migrate_dossiers @source_procedure.dossiers.find_each(batch_size: 100) do |d| @champ_mapping.migrate_champs(d) From 6a3f7438d5a513ce3d2564e2427f8d3fba619b03 Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Tue, 9 Oct 2018 20:06:11 +0200 Subject: [PATCH 04/14] [#2772] Let migrator instantiate mapping from provided class --- lib/tasks/2018_07_31_nutriscore.rake | 2 +- lib/tasks/dossier_procedure_migrator.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/2018_07_31_nutriscore.rake b/lib/tasks/2018_07_31_nutriscore.rake index 0f4b86793..f69c1b57a 100644 --- a/lib/tasks/2018_07_31_nutriscore.rake +++ b/lib/tasks/2018_07_31_nutriscore.rake @@ -100,7 +100,7 @@ namespace :'2018_07_31_nutriscore' do target_tdc.champ.create(dossier: d, value: JSON.unparse(['FRANCE'])) end end - end.new(source_procedure, destination_procedure) + end Tasks::DossierProcedureMigrator.new(source_procedure, destination_procedure, mapping).migrate_procedure AutoReceiveDossiersForProcedureJob.set(cron: "* * * * *").perform_later(destination_procedure_id, 'accepte') diff --git a/lib/tasks/dossier_procedure_migrator.rb b/lib/tasks/dossier_procedure_migrator.rb index 380a8f016..6bbe22cee 100644 --- a/lib/tasks/dossier_procedure_migrator.rb +++ b/lib/tasks/dossier_procedure_migrator.rb @@ -117,7 +117,7 @@ module Tasks def initialize(source_procedure, destination_procedure, champ_mapping) @source_procedure = source_procedure @destination_procedure = destination_procedure - @champ_mapping = champ_mapping + @champ_mapping = champ_mapping.new(source_procedure, destination_procedure) end def migrate_procedure From f7fcf46f0d2de6f724b8a2e771a3859088b28f8d Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Tue, 9 Oct 2018 20:09:13 +0200 Subject: [PATCH 05/14] [#2772] Avoid overriding initialize in subclasses --- lib/tasks/2018_07_31_nutriscore.rake | 7 +------ lib/tasks/dossier_procedure_migrator.rb | 2 ++ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/tasks/2018_07_31_nutriscore.rake b/lib/tasks/2018_07_31_nutriscore.rake index f69c1b57a..76cb7dc47 100644 --- a/lib/tasks/2018_07_31_nutriscore.rake +++ b/lib/tasks/2018_07_31_nutriscore.rake @@ -9,12 +9,7 @@ namespace :'2018_07_31_nutriscore' do destination_procedure = Procedure.find(destination_procedure_id) mapping = Class.new(Tasks::DossierProcedureMigrator::ChampMapping) do - def initialize(source_procedure, destination_procedure) - super - setup_champ_mapping - end - - def setup_champ_mapping + def setup_mapping siret_order_place = 2 fonction_order_place = 9 zone_geographique_header_order_place = 18 diff --git a/lib/tasks/dossier_procedure_migrator.rb b/lib/tasks/dossier_procedure_migrator.rb index 6bbe22cee..c2164455a 100644 --- a/lib/tasks/dossier_procedure_migrator.rb +++ b/lib/tasks/dossier_procedure_migrator.rb @@ -12,6 +12,8 @@ module Tasks @source_to_destination_mapping = {} @source_champs_to_discard = Set[] @destination_champ_computations = [] + + setup_mapping end def check_source_destination_champs_consistency From dfe7f2cc456b21cd53d524acb95c234b62537dd5 Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Tue, 9 Oct 2018 21:23:27 +0200 Subject: [PATCH 06/14] [#2772] Also migrate private champs --- lib/tasks/dossier_procedure_migrator.rb | 43 ++++++++++++++++++------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/lib/tasks/dossier_procedure_migrator.rb b/lib/tasks/dossier_procedure_migrator.rb index c2164455a..0b1273728 100644 --- a/lib/tasks/dossier_procedure_migrator.rb +++ b/lib/tasks/dossier_procedure_migrator.rb @@ -3,9 +3,10 @@ module Tasks # Migrates dossiers from an old source procedure to a revised destination procedure. class ChampMapping - def initialize(source_procedure, destination_procedure) + def initialize(source_procedure, destination_procedure, is_private) @source_procedure = source_procedure @destination_procedure = destination_procedure + @is_private = is_private @expected_source_types_de_champ = {} @expected_destination_types_de_champ = {} @@ -17,14 +18,14 @@ module Tasks end def check_source_destination_champs_consistency - check_champs_consistency('source', @expected_source_types_de_champ, @source_procedure.types_de_champ) - check_champs_consistency('destination', @expected_destination_types_de_champ, @destination_procedure.types_de_champ) + check_champs_consistency("#{privacy_label}source", @expected_source_types_de_champ, types_de_champ(@source_procedure)) + check_champs_consistency("#{privacy_label}destination", @expected_destination_types_de_champ, types_de_champ(@destination_procedure)) end def migrate_champs(dossier) # Since we’re going to iterate and change the champs at the same time, # we use to_a to make the list static and avoid nasty surprises - original_champs = dossier.champs.to_a + original_champs = champs(dossier).to_a compute_new_champs(dossier) @@ -33,9 +34,9 @@ module Tasks if tdc_to.present? c.update(type_de_champ: tdc_to) elsif discard_champ?(c) - dossier.champs.destroy(c) + champs(dossier).destroy(c) else - fail "Unhandled source type de champ #{c.type_de_champ.order_place}" + fail "Unhandled source #{privacy_label}type de champ #{c.type_de_champ.order_place}" end end end @@ -44,7 +45,7 @@ module Tasks def compute_new_champs(dossier) @destination_champ_computations.each do |tdc, block| - dossier.champs << block.call(dossier, tdc) + champs(dossier) << block.call(dossier, tdc) end end @@ -56,6 +57,21 @@ module Tasks @source_champs_to_discard.member?(champ.type_de_champ.order_place) end + def setup_mapping + end + + def champs(dossier) + @is_private ? dossier.champs_private : dossier.champs + end + + def types_de_champ(procedure) + @is_private ? procedure.types_de_champ_private : procedure.types_de_champ + end + + def privacy_label + @is_private ? 'private ' : '' + end + def check_champs_consistency(label, expected_tdcs, actual_tdcs) if actual_tdcs.size != expected_tdcs.size raise "Incorrect #{label} size #{actual_tdcs.size} (expected #{expected_tdcs.size})" @@ -84,12 +100,12 @@ module Tasks end def map_source_to_destination_champ(source_order_place, destination_order_place, source_overrides: {}, destination_overrides: {}) - destination_type_de_champ = @destination_procedure.types_de_champ.find_by(order_place: destination_order_place) + destination_type_de_champ = types_de_champ(@destination_procedure).find_by(order_place: destination_order_place) @expected_source_types_de_champ[source_order_place] = type_de_champ_to_expectation(destination_type_de_champ) .merge!(source_overrides) @expected_destination_types_de_champ[destination_order_place] = - type_de_champ_to_expectation(@source_procedure.types_de_champ.find_by(order_place: source_order_place)) + type_de_champ_to_expectation(types_de_champ(@source_procedure).find_by(order_place: source_order_place)) .merge!({ "mandatory" => false }) # Even if the source was mandatory, it’s ok for the destination to be optional .merge!(destination_overrides) @source_to_destination_mapping[source_order_place] = destination_type_de_champ @@ -102,7 +118,7 @@ module Tasks def compute_destination_champ(destination_type_de_champ, &block) @expected_destination_types_de_champ[destination_type_de_champ.order_place] = type_de_champ_to_expectation(destination_type_de_champ) - @destination_champ_computations << [@destination_procedure.types_de_champ.find_by(order_place: destination_type_de_champ.order_place), block] + @destination_champ_computations << [types_de_champ(@destination_procedure).find_by(order_place: destination_type_de_champ.order_place), block] end def type_de_champ_to_expectation(tdc) @@ -116,10 +132,11 @@ module Tasks end end - def initialize(source_procedure, destination_procedure, champ_mapping) + def initialize(source_procedure, destination_procedure, champ_mapping, private_champ_mapping = ChampMapping) @source_procedure = source_procedure @destination_procedure = destination_procedure - @champ_mapping = champ_mapping.new(source_procedure, destination_procedure) + @champ_mapping = champ_mapping.new(source_procedure, destination_procedure, false) + @private_champ_mapping = private_champ_mapping.new(source_procedure, destination_procedure, true) end def migrate_procedure @@ -132,6 +149,7 @@ module Tasks def check_consistency check_same_administrateur @champ_mapping.check_source_destination_champs_consistency + @private_champ_mapping.check_source_destination_champs_consistency end def check_same_administrateur @@ -143,6 +161,7 @@ module Tasks def migrate_dossiers @source_procedure.dossiers.find_each(batch_size: 100) do |d| @champ_mapping.migrate_champs(d) + @private_champ_mapping.migrate_champs(d) # Use update_columns to avoid triggering build_default_champs d.update_columns(procedure_id: @destination_procedure.id) From 840b049cdb42691500dd69acb19b7186c826ee89 Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Wed, 10 Oct 2018 13:07:12 +0200 Subject: [PATCH 07/14] [#2772] Allow untranslatable dossiers to be skipped --- lib/tasks/dossier_procedure_migrator.rb | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/tasks/dossier_procedure_migrator.rb b/lib/tasks/dossier_procedure_migrator.rb index 0b1273728..ba0cafbcb 100644 --- a/lib/tasks/dossier_procedure_migrator.rb +++ b/lib/tasks/dossier_procedure_migrator.rb @@ -22,6 +22,10 @@ module Tasks check_champs_consistency("#{privacy_label}destination", @expected_destination_types_de_champ, types_de_champ(@destination_procedure)) end + def can_migrate?(dossier) + true + end + def migrate_champs(dossier) # Since we’re going to iterate and change the champs at the same time, # we use to_a to make the list static and avoid nasty surprises @@ -160,11 +164,13 @@ module Tasks def migrate_dossiers @source_procedure.dossiers.find_each(batch_size: 100) do |d| - @champ_mapping.migrate_champs(d) - @private_champ_mapping.migrate_champs(d) + if @champ_mapping.can_migrate?(d) && @private_champ_mapping.can_migrate?(d) + @champ_mapping.migrate_champs(d) + @private_champ_mapping.migrate_champs(d) - # Use update_columns to avoid triggering build_default_champs - d.update_columns(procedure_id: @destination_procedure.id) + # Use update_columns to avoid triggering build_default_champs + d.update_columns(procedure_id: @destination_procedure.id) + end end end From 1d0ce21555e93a811ea0fd3590f8790689114ca3 Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Wed, 10 Oct 2018 15:00:59 +0200 Subject: [PATCH 08/14] [#2772] Migrate pieces justificatives --- lib/tasks/dossier_procedure_migrator.rb | 116 ++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 8 deletions(-) diff --git a/lib/tasks/dossier_procedure_migrator.rb b/lib/tasks/dossier_procedure_migrator.rb index ba0cafbcb..ed9391762 100644 --- a/lib/tasks/dossier_procedure_migrator.rb +++ b/lib/tasks/dossier_procedure_migrator.rb @@ -17,7 +17,7 @@ module Tasks setup_mapping end - def check_source_destination_champs_consistency + def check_source_destination_consistency check_champs_consistency("#{privacy_label}source", @expected_source_types_de_champ, types_de_champ(@source_procedure)) check_champs_consistency("#{privacy_label}destination", @expected_destination_types_de_champ, types_de_champ(@destination_procedure)) end @@ -26,7 +26,7 @@ module Tasks true end - def migrate_champs(dossier) + def migrate(dossier) # Since we’re going to iterate and change the champs at the same time, # we use to_a to make the list static and avoid nasty surprises original_champs = champs(dossier).to_a @@ -136,11 +136,109 @@ module Tasks end end - def initialize(source_procedure, destination_procedure, champ_mapping, private_champ_mapping = ChampMapping) + class PieceJustificativeMapping + def initialize(source_procedure, destination_procedure) + @source_procedure = source_procedure + @destination_procedure = destination_procedure + + @expected_source_pj = {} + @expected_destination_pj = {} + @source_to_destination_mapping = {} + + setup_mapping + end + + def check_source_destination_consistency + check_pjs_consistency('source', @expected_source_pj, @source_procedure.types_de_piece_justificative) + check_pjs_consistency('destination', @expected_destination_pj, @destination_procedure.types_de_piece_justificative) + end + + def can_migrate?(dossier) + true + end + + def migrate(dossier) + # Since we’re going to iterate and change the pjs at the same time, + # we use to_a to make the list static and avoid nasty surprises + original_pjs = dossier.pieces_justificatives.to_a + + original_pjs.each do |pj| + pj_to = destination_pj(pj) + if pj_to.present? + pj.update_columns(type_de_piece_justificative_id: pj_to.id) + elsif discard_pj?(pj) + dossier.pieces_justificatives.destroy(pj) + else + fail "Unhandled source pièce justificative #{c.type_de_piece_justificative.order_place}" + end + end + end + + private + + def destination_pj(pj) + @source_to_destination_mapping[pj.order_place] + end + + def discard_pj?(champ) + @source_pjs_to_discard.member?(pj.order_place) + end + + def setup_mapping + end + + def map_source_to_destination_pj(source_order_place, destination_order_place, source_overrides: {}, destination_overrides: {}) + destination_pj = @destination_procedure.types_de_piece_justificative.find_by(order_place: destination_order_place) + @expected_source_pj[source_order_place] = + pj_to_expectation(destination_pj) + .merge!(source_overrides) + @expected_destination_pj[destination_order_place] = + pj_to_expectation(@source_procedure.types_de_piece_justificative.find_by(order_place: source_order_place)) + .merge!({ "mandatory" => false }) # Even if the source was mandatory, it’s ok for the destination to be optional + .merge!(destination_overrides) + @source_to_destination_mapping[source_order_place] = destination_pj + end + + def discard_source_pj(source_pj) + @expected_source_pj[source_pj.order_place] = pj_to_expectation(source_pj) + @source_pjs_to_discard << source_pj.order_place + end + + def leave_destination_pj_blank(destination_pj) + @expected_destination_pj[destination_pj.order_place] = pj_to_expectation(destination_pj) + end + + def pj_to_expectation(pj) + pj&.as_json(only: [:libelle, :mandatory]) || {} + end + + def check_pjs_consistency(label, expected_pjs, actual_pjs) + if actual_pjs.size != expected_pjs.size + raise "Incorrect #{label} pièce justificative count #{actual_pjs.size} (expected #{expected_pjs.size})" + end + actual_pjs.each { |pj| check_pj_consistency(label, expected_pjs[pj.order_place], pj) } + end + + def check_pj_consistency(label, expected_pj, actual_pj) + errors = [] + if actual_pj.libelle != expected_pj['libelle'] + errors.append("incorrect libelle #{actual_pj.libelle} (expected #{expected_pj['libelle']})") + end + if (!actual_pj.mandatory) && expected_pj['mandatory'] + errors.append("pj should be mandatory") + end + if errors.present? + fail "On #{label} type de pièce justificative #{actual_pj.order_place} (#{actual_pj.libelle}) " + errors.join(', ') + end + end + end + + def initialize(source_procedure, destination_procedure, champ_mapping, private_champ_mapping = ChampMapping, piece_justificative_mapping = PieceJustificativeMapping) @source_procedure = source_procedure @destination_procedure = destination_procedure @champ_mapping = champ_mapping.new(source_procedure, destination_procedure, false) @private_champ_mapping = private_champ_mapping.new(source_procedure, destination_procedure, true) + @piece_justificative_mapping = piece_justificative_mapping.new(source_procedure, destination_procedure) end def migrate_procedure @@ -152,8 +250,9 @@ module Tasks def check_consistency check_same_administrateur - @champ_mapping.check_source_destination_champs_consistency - @private_champ_mapping.check_source_destination_champs_consistency + @champ_mapping.check_source_destination_consistency + @private_champ_mapping.check_source_destination_consistency + @piece_justificative_mapping.check_source_destination_consistency end def check_same_administrateur @@ -164,9 +263,10 @@ module Tasks def migrate_dossiers @source_procedure.dossiers.find_each(batch_size: 100) do |d| - if @champ_mapping.can_migrate?(d) && @private_champ_mapping.can_migrate?(d) - @champ_mapping.migrate_champs(d) - @private_champ_mapping.migrate_champs(d) + if @champ_mapping.can_migrate?(d) && @private_champ_mapping.can_migrate?(d) && @piece_justificative_mapping.can_migrate?(d) + @champ_mapping.migrate(d) + @private_champ_mapping.migrate(d) + @piece_justificative_mapping.migrate(d) # Use update_columns to avoid triggering build_default_champs d.update_columns(procedure_id: @destination_procedure.id) From ecfacf8417b6f41e5efd218241f73dd675edb34e Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Wed, 10 Oct 2018 17:13:14 +0200 Subject: [PATCH 09/14] [#2772] Avoid touching champ update date --- lib/tasks/dossier_procedure_migrator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/dossier_procedure_migrator.rb b/lib/tasks/dossier_procedure_migrator.rb index ed9391762..fbc004319 100644 --- a/lib/tasks/dossier_procedure_migrator.rb +++ b/lib/tasks/dossier_procedure_migrator.rb @@ -36,7 +36,7 @@ module Tasks original_champs.each do |c| tdc_to = destination_type_de_champ(c) if tdc_to.present? - c.update(type_de_champ: tdc_to) + c.update_columns(type_de_champ_id: tdc_to.id) elsif discard_champ?(c) champs(dossier).destroy(c) else From f236e31c29c20b22d764f08930b01b8126365a85 Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Tue, 9 Oct 2018 22:07:31 +0200 Subject: [PATCH 10/14] [#2772] Restore deleted dossiers for candidats libres procedure --- ...181009130216_restore_deleted_dossiers.rake | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake diff --git a/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake b/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake new file mode 100644 index 000000000..35426ebeb --- /dev/null +++ b/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake @@ -0,0 +1,102 @@ +require Rails.root.join("lib", "tasks", "task_helper") + +namespace :after_party do + desc 'Deployment task: restore_deleted_dossiers' + task restore_deleted_dossiers: :environment do + Class.new do + def run + rake_puts "Running deploy task 'restore_deleted_dossiers'" + restore_candidats_libres_deleted_dossiers + AfterParty::TaskRecord.create version: '20181009130216' + end + + def restore_candidats_libres_deleted_dossiers + mapping = Class.new(Tasks::DossierProcedureMigrator::ChampMapping) do + def setup_mapping + champ_opts = { + 16 => { + source_overrides: { 'libelle' => 'Adresse postale du candidat' }, + destination_overrides: { 'libelle' => 'Adresse postale complète du candidat' } + } + } + (0..23).each do |i| + map_source_to_destination_champ(i, i, **(champ_opts[i] || {})) + end + end + end + + private_mapping = Class.new(Tasks::DossierProcedureMigrator::ChampMapping) do + def setup_mapping + compute_destination_champ( + TypeDeChamp.new( + type_champ: 'datetime', + order_place: 0, + libelle: 'Date et heure de convocation', + mandatory: false + ) + ) do |d, target_tdc| + target_tdc.champ.create(dossier: d) + end + compute_destination_champ( + TypeDeChamp.new( + type_champ: 'text', + order_place: 1, + libelle: 'Lieu de convocation', + mandatory: false + ) + ) do |d, target_tdc| + target_tdc.champ.create(dossier: d) + end + compute_destination_champ( + TypeDeChamp.new( + type_champ: 'address', + order_place: 2, + libelle: 'Adresse centre examen', + mandatory: false + ) + ) do |d, target_tdc| + target_tdc.champ.create(dossier: d) + end + end + end + + pj_mapping = Class.new(Tasks::DossierProcedureMigrator::PieceJustificativeMapping) do + def setup_mapping + (0..3).each do |i| + map_source_to_destination_pj(i, i + 2) + end + leave_destination_pj_blank( + TypeDePieceJustificative.new( + order_place: 0, + libelle: "Télécharger la Charte de l'accompagnateur" + ) + ) + leave_destination_pj_blank( + TypeDePieceJustificative.new( + order_place: 1, + libelle: "Télécharger l'attestation d'assurance" + ) + ) + end + end + + restore_deleted_dossiers(4860, 8603, mapping, private_mapping, pj_mapping) + end + + def restore_deleted_dossiers(deleted_procedure_id, new_procedure_id, champ_mapping, champ_private_mapping, pj_mapping) + Dossier.unscoped + .joins('JOIN procedures ON procedures.id = dossiers.procedure_id') + .where(procedure_id: deleted_procedure_id) + .where('dossiers.hidden_at >= procedures.hidden_at') + .update_all(hidden_at: nil) + + source_procedure = Procedure.unscoped.find(deleted_procedure_id) + destination_procedure = Procedure.find(new_procedure_id) + + migrator = Tasks::DossierProcedureMigrator.new(source_procedure, destination_procedure, champ_mapping, champ_private_mapping, pj_mapping) + migrator.check_consistency + migrator.migrate_dossiers + end + end.new.run + end +end From bc22c975017b05f640b230ca2c58e5078d2c478e Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Wed, 10 Oct 2018 15:36:23 +0200 Subject: [PATCH 11/14] [#2772] Restore deleted dossiers for neph procedure --- ...181009130216_restore_deleted_dossiers.rake | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake b/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake index 35426ebeb..bb0a61c86 100644 --- a/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake +++ b/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake @@ -7,6 +7,7 @@ namespace :after_party do def run rake_puts "Running deploy task 'restore_deleted_dossiers'" restore_candidats_libres_deleted_dossiers + restore_neph_deleted_dossiers AfterParty::TaskRecord.create version: '20181009130216' end @@ -83,6 +84,93 @@ namespace :after_party do restore_deleted_dossiers(4860, 8603, mapping, private_mapping, pj_mapping) end + def restore_neph_deleted_dossiers + mapping = Class.new(Tasks::DossierProcedureMigrator::ChampMapping) do + def can_migrate?(dossier) + !(dossier.termine? || + dossier.champs.joins(:type_de_champ).find_by(types_de_champ: { order_place: 3 }).value&.include?('"Demande de duplicata de dossier d\'inscription (suite perte)"')) + end + + def setup_mapping + champ_opts = { + 3 => { + source_overrides: { 'drop_down' => ["", "Demande de réactualisation du numéro NEPH", "Demande de communication du numéro NEPH", "Demande de duplicata de dossier d'inscription (suite perte)", "Demande de correction sur le Fichier National des Permis de conduire"] }, + destination_overrides: { 'drop_down' => ["", "Demande de réactualisation du numéro NEPH", "Demande de communication du numéro NEPH", "Demande de correction sur le Fichier National des Permis de conduire"] } + } + } + (0..14).each do |i| + map_source_to_destination_champ(i, i, **(champ_opts[i] || {})) + end + (16..22).each do |i| + map_source_to_destination_champ(i, i + 2, **(champ_opts[i] || {})) + end + + discard_source_champ( + TypeDeChamp.new( + type_champ: 'address', + order_place: 15, + libelle: 'Adresse du candidat' + ) + ) + + compute_destination_champ( + TypeDeChamp.new( + type_champ: 'address', + order_place: 15, + libelle: 'Adresse du candidat', + mandatory: true + ) + ) do |d, target_tdc| + value = d.champs.joins(:type_de_champ).find_by(types_de_champ: { order_place: 3 }).value + if !d.brouillon? + value ||= 'non renseigné' + end + target_tdc.champ.create(dossier: d, value: value) + end + + compute_destination_champ( + TypeDeChamp.new( + type_champ: 'address', + order_place: 16, + libelle: 'Code postal', + mandatory: true + ) + ) do |d, target_tdc| + target_tdc.champ.create(dossier: d, value: d.brouillon? ? nil : 'non renseigné') + end + + compute_destination_champ( + TypeDeChamp.new( + type_champ: 'address', + order_place: 17, + libelle: 'Ville', + mandatory: true + ) + ) do |d, target_tdc| + target_tdc.champ.create(dossier: d, value: d.brouillon? ? nil : 'non renseigné') + end + end + end + + private_mapping = Class.new(Tasks::DossierProcedureMigrator::ChampMapping) do + def setup_mapping + (0..2).each do |i| + map_source_to_destination_champ(i, i) + end + end + end + + pj_mapping = Class.new(Tasks::DossierProcedureMigrator::PieceJustificativeMapping) do + def setup_mapping + (0..3).each do |i| + map_source_to_destination_pj(i, i) + end + end + end + + restore_deleted_dossiers(6388, 8770, mapping, private_mapping, pj_mapping) + end + def restore_deleted_dossiers(deleted_procedure_id, new_procedure_id, champ_mapping, champ_private_mapping, pj_mapping) Dossier.unscoped .joins('JOIN procedures ON procedures.id = dossiers.procedure_id') From 3283cb1ae723e7ec9d0489e8869bf04e75faf873 Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Wed, 10 Oct 2018 17:19:20 +0200 Subject: [PATCH 12/14] [#2772] Allow per-migrated-dossier callback --- lib/tasks/dossier_procedure_migrator.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/tasks/dossier_procedure_migrator.rb b/lib/tasks/dossier_procedure_migrator.rb index fbc004319..0d5a57073 100644 --- a/lib/tasks/dossier_procedure_migrator.rb +++ b/lib/tasks/dossier_procedure_migrator.rb @@ -233,12 +233,16 @@ module Tasks end end - def initialize(source_procedure, destination_procedure, champ_mapping, private_champ_mapping = ChampMapping, piece_justificative_mapping = PieceJustificativeMapping) + def initialize(source_procedure, destination_procedure, champ_mapping, private_champ_mapping = ChampMapping, piece_justificative_mapping = PieceJustificativeMapping, &block) @source_procedure = source_procedure @destination_procedure = destination_procedure @champ_mapping = champ_mapping.new(source_procedure, destination_procedure, false) @private_champ_mapping = private_champ_mapping.new(source_procedure, destination_procedure, true) @piece_justificative_mapping = piece_justificative_mapping.new(source_procedure, destination_procedure) + + if block_given? + @on_dossier_migration = block + end end def migrate_procedure @@ -270,6 +274,7 @@ module Tasks # Use update_columns to avoid triggering build_default_champs d.update_columns(procedure_id: @destination_procedure.id) + @on_dossier_migration&.call(d) end end end From 1e4ccda14ca251d23ee0530657c4e8db1c501e37 Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Wed, 10 Oct 2018 17:19:59 +0200 Subject: [PATCH 13/14] [#2772] Notify users that their dossier was restored --- app/mailers/dossier_mailer.rb | 8 ++++++++ .../notify_undelete_to_user.html.haml | 15 +++++++++++++++ .../20181009130216_restore_deleted_dossiers.rake | 4 +++- 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 app/views/dossier_mailer/notify_undelete_to_user.html.haml diff --git a/app/mailers/dossier_mailer.rb b/app/mailers/dossier_mailer.rb index 03147e8b9..eb8f08629 100644 --- a/app/mailers/dossier_mailer.rb +++ b/app/mailers/dossier_mailer.rb @@ -21,4 +21,12 @@ class DossierMailer < ApplicationMailer mail(to: dossier.user.email, subject: subject) end + + def notify_undelete_to_user(dossier) + @dossier = dossier + @dossier_kind = dossier.brouillon? ? 'brouillon' : 'dossier' + @subject = "Votre #{@dossier_kind} n° #{@dossier.id} est à nouveau accessible" + + mail(to: dossier.user.email, subject: @subject) + end end diff --git a/app/views/dossier_mailer/notify_undelete_to_user.html.haml b/app/views/dossier_mailer/notify_undelete_to_user.html.haml new file mode 100644 index 000000000..e61846dda --- /dev/null +++ b/app/views/dossier_mailer/notify_undelete_to_user.html.haml @@ -0,0 +1,15 @@ +- content_for(:title, @subject) + +%h1 Bonjour, + +%p + En raison d’un incident, votre + = link_to("#{@dossier_kind} n° #{@dossier.id}", dossier_url(@dossier)) + sur la procédure « #{@dossier.procedure.libelle} » a été inaccessible pendant quelques jours. + + L’accès est à présent à nouveau possible. Nous vous présentons nos excuses pour toute gène occasionnée. +%p + Bonne journée, + +%p + L'équipe demarches-simplifiees.fr diff --git a/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake b/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake index bb0a61c86..28c434c24 100644 --- a/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake +++ b/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake @@ -181,7 +181,9 @@ namespace :after_party do source_procedure = Procedure.unscoped.find(deleted_procedure_id) destination_procedure = Procedure.find(new_procedure_id) - migrator = Tasks::DossierProcedureMigrator.new(source_procedure, destination_procedure, champ_mapping, champ_private_mapping, pj_mapping) + migrator = Tasks::DossierProcedureMigrator.new(source_procedure, destination_procedure, champ_mapping, champ_private_mapping, pj_mapping) do |dossier| + DossierMailer.notify_undelete_to_user(dossier).deliver_later + end migrator.check_consistency migrator.migrate_dossiers end From bbdb446dfd35752c9c6fcadac5ae570007a9efed Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Thu, 11 Oct 2018 19:27:58 +0200 Subject: [PATCH 14/14] [#2772] For unmigrated dossiers, make read-only access possible and notify usager --- app/mailers/dossier_mailer.rb | 9 ++++++ .../notify_unmigrated_to_user.html.haml | 26 +++++++++++++++++ ...181009130216_restore_deleted_dossiers.rake | 29 +++++++++++++++---- 3 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 app/views/dossier_mailer/notify_unmigrated_to_user.html.haml diff --git a/app/mailers/dossier_mailer.rb b/app/mailers/dossier_mailer.rb index eb8f08629..65d7e68e6 100644 --- a/app/mailers/dossier_mailer.rb +++ b/app/mailers/dossier_mailer.rb @@ -29,4 +29,13 @@ class DossierMailer < ApplicationMailer mail(to: dossier.user.email, subject: @subject) end + + def notify_unmigrated_to_user(dossier, new_procedure) + @dossier = dossier + @dossier_kind = dossier.brouillon? ? 'brouillon' : 'dossier' + @subject = "Changement de procédure pour votre #{@dossier_kind} n° #{@dossier.id}" + @new_procedure = new_procedure + + mail(to: dossier.user.email, subject: @subject) + end end diff --git a/app/views/dossier_mailer/notify_unmigrated_to_user.html.haml b/app/views/dossier_mailer/notify_unmigrated_to_user.html.haml new file mode 100644 index 000000000..0bf06540c --- /dev/null +++ b/app/views/dossier_mailer/notify_unmigrated_to_user.html.haml @@ -0,0 +1,26 @@ +- content_for(:title, @subject) + +%h1 Bonjour, + +%p + Vous avez commencé un #{@dossier_kind}, + = link_to("n° #{@dossier.id}", dossier_url(@dossier)) + sur la procédure « #{@dossier.procedure.libelle} ». + En raison d’un changement dans la procédure, votre #{@dossier_kind} a été inaccessible pendant quelques jours. + L’accès est à présent à nouveau possible. + +%p + Malheureusement, en raison des changements dans le procédure, vous ne pourrez pas mener à terme le #{@dossier_kind} commencé. + Si votre démarche est toujours d’actualité, nous vous invitons à la recommencer sur + = link_to("la nouvelle procédure", commencer_url(@new_procedure.path)) + \. + +%p + Nous avons pris des mesures pour nous assurer qu’un tel désagrément ne se reproduise pas, + et vous présentons nos excuses pour la gène occasionnée. + +%p + Bonne journée, + +%p + L'équipe demarches-simplifiees.fr diff --git a/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake b/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake index 28c434c24..387d5cf99 100644 --- a/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake +++ b/lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake @@ -172,20 +172,37 @@ namespace :after_party do end def restore_deleted_dossiers(deleted_procedure_id, new_procedure_id, champ_mapping, champ_private_mapping, pj_mapping) - Dossier.unscoped - .joins('JOIN procedures ON procedures.id = dossiers.procedure_id') - .where(procedure_id: deleted_procedure_id) - .where('dossiers.hidden_at >= procedures.hidden_at') - .update_all(hidden_at: nil) - source_procedure = Procedure.unscoped.find(deleted_procedure_id) destination_procedure = Procedure.find(new_procedure_id) + deleted_dossiers = Dossier.unscoped + .where(procedure_id: deleted_procedure_id) + .where('dossiers.hidden_at >= ?', source_procedure.hidden_at) + + deleted_dossier_ids = deleted_dossiers.pluck(:id).to_a + deleted_dossiers.update_all(hidden_at: nil) + + source_procedure + .update_columns( + hidden_at: nil, + archived_at: source_procedure.hidden_at, + aasm_state: :archivee + ) + migrator = Tasks::DossierProcedureMigrator.new(source_procedure, destination_procedure, champ_mapping, champ_private_mapping, pj_mapping) do |dossier| DossierMailer.notify_undelete_to_user(dossier).deliver_later end migrator.check_consistency migrator.migrate_dossiers + + source_procedure.dossiers.where(id: deleted_dossier_ids).find_each do |dossier| + if dossier.termine? + DossierMailer.notify_undelete_to_user(dossier).deliver_later + else + rake_puts "Dossier #{dossier.id} non migré\n" + DossierMailer.notify_unmigrated_to_user(dossier, destination_procedure).deliver_later + end + end end end.new.run end