Merge pull request #2802 from betagouv/frederic/fix_2772-migrate_dossiers

Restore deleted dossiers

app/mailers/dossier_mailer.rb

@@ -21,4 +21,21 @@ class DossierMailer < ApplicationMailer
 
     mail(to: dossier.user.email, subject: subject)
   end
+
+  def notify_undelete_to_user(dossier)
+    @dossier = dossier
+    @dossier_kind = dossier.brouillon? ? 'brouillon' : 'dossier'
+    @subject = "Votre #{@dossier_kind} n° #{@dossier.id} est à nouveau accessible"
+
+    mail(to: dossier.user.email, subject: @subject)
+  end
+
+  def notify_unmigrated_to_user(dossier, new_procedure)
+    @dossier = dossier
+    @dossier_kind = dossier.brouillon? ? 'brouillon' : 'dossier'
+    @subject = "Changement de procédure pour votre #{@dossier_kind} n° #{@dossier.id}"
+    @new_procedure = new_procedure
+
+    mail(to: dossier.user.email, subject: @subject)
+  end
 end

app/views/dossier_mailer/notify_undelete_to_user.html.haml

@@ -0,0 +1,15 @@
+- content_for(:title, @subject)
+
+%h1 Bonjour,
+
+%p
+  En raison d’un incident, votre
+  = link_to("#{@dossier_kind} n° #{@dossier.id}", dossier_url(@dossier))
+  sur la procédure « #{@dossier.procedure.libelle} » a été inaccessible pendant quelques jours.
+
+  L’accès est à présent à nouveau possible. Nous vous présentons nos excuses pour toute gène occasionnée.
+%p
+  Bonne journée,
+
+%p
+  L'équipe demarches-simplifiees.fr

app/views/dossier_mailer/notify_unmigrated_to_user.html.haml

@@ -0,0 +1,26 @@
+- content_for(:title, @subject)
+
+%h1 Bonjour,
+
+%p
+  Vous avez commencé un #{@dossier_kind},
+  = link_to("n° #{@dossier.id}", dossier_url(@dossier))
+  sur la procédure « #{@dossier.procedure.libelle} ».
+  En raison d’un changement dans la procédure, votre #{@dossier_kind} a été inaccessible pendant quelques jours.
+  L’accès est à présent à nouveau possible.
+
+%p
+  Malheureusement, en raison des changements dans le procédure, vous ne pourrez pas mener à terme le #{@dossier_kind} commencé.
+  Si votre démarche est toujours d’actualité, nous vous invitons à la recommencer sur
+  = link_to("la nouvelle procédure", commencer_url(@new_procedure.path))
+  \.
+
+%p
+  Nous avons pris des mesures pour nous assurer qu’un tel désagrément ne se reproduise pas,
+  et vous présentons nos excuses pour la gène occasionnée.
+
+%p
+  Bonne journée,
+
+%p
+  L'équipe demarches-simplifiees.fr

lib/tasks/2018_07_31_nutriscore.rake

@@ -9,12 +9,7 @@ namespace :'2018_07_31_nutriscore' do
     destination_procedure = Procedure.find(destination_procedure_id)
 
     mapping = Class.new(Tasks::DossierProcedureMigrator::ChampMapping) do
-      def initialize(source_procedure, destination_procedure)
-        super
-        setup_champ_mapping
-      end
-
-      def setup_champ_mapping
+      def setup_mapping
         siret_order_place = 2
         fonction_order_place = 9
         zone_geographique_header_order_place = 18
@@ -100,7 +95,7 @@ namespace :'2018_07_31_nutriscore' do
           target_tdc.champ.create(dossier: d, value: JSON.unparse(['FRANCE']))
         end
       end
-    end.new(source_procedure, destination_procedure)
+    end
 
     Tasks::DossierProcedureMigrator.new(source_procedure, destination_procedure, mapping).migrate_procedure
     AutoReceiveDossiersForProcedureJob.set(cron: "* * * * *").perform_later(destination_procedure_id, 'accepte')

lib/tasks/deployment/20181009130216_restore_deleted_dossiers.rake

@@ -0,0 +1,209 @@
+require Rails.root.join("lib", "tasks", "task_helper")
+
+namespace :after_party do
+  desc 'Deployment task: restore_deleted_dossiers'
+  task restore_deleted_dossiers: :environment do
+    Class.new do
+      def run
+        rake_puts "Running deploy task 'restore_deleted_dossiers'"
+        restore_candidats_libres_deleted_dossiers
+        restore_neph_deleted_dossiers
+        AfterParty::TaskRecord.create version: '20181009130216'
+      end
+
+      def restore_candidats_libres_deleted_dossiers
+        mapping = Class.new(Tasks::DossierProcedureMigrator::ChampMapping) do
+          def setup_mapping
+            champ_opts = {
+              16 => {
+                source_overrides: { 'libelle' => 'Adresse postale du candidat' },
+                destination_overrides: { 'libelle' => 'Adresse postale complète du candidat' }
+              }
+            }
+            (0..23).each do |i|
+              map_source_to_destination_champ(i, i, **(champ_opts[i] || {}))
+            end
+          end
+        end
+
+        private_mapping = Class.new(Tasks::DossierProcedureMigrator::ChampMapping) do
+          def setup_mapping
+            compute_destination_champ(
+              TypeDeChamp.new(
+                type_champ: 'datetime',
+                order_place: 0,
+                libelle: 'Date et heure de convocation',
+                mandatory: false
+              )
+            ) do |d, target_tdc|
+              target_tdc.champ.create(dossier: d)
+            end
+            compute_destination_champ(
+              TypeDeChamp.new(
+                type_champ: 'text',
+                order_place: 1,
+                libelle: 'Lieu de convocation',
+                mandatory: false
+              )
+            ) do |d, target_tdc|
+              target_tdc.champ.create(dossier: d)
+            end
+            compute_destination_champ(
+              TypeDeChamp.new(
+                type_champ: 'address',
+                order_place: 2,
+                libelle: 'Adresse centre examen',
+                mandatory: false
+              )
+            ) do |d, target_tdc|
+              target_tdc.champ.create(dossier: d)
+            end
+          end
+        end
+
+        pj_mapping = Class.new(Tasks::DossierProcedureMigrator::PieceJustificativeMapping) do
+          def setup_mapping
+            (0..3).each do |i|
+              map_source_to_destination_pj(i, i + 2)
+            end
+            leave_destination_pj_blank(
+              TypeDePieceJustificative.new(
+                order_place: 0,
+                libelle: "Télécharger la Charte de l'accompagnateur"
+              )
+            )
+            leave_destination_pj_blank(
+              TypeDePieceJustificative.new(
+                order_place: 1,
+                libelle: "Télécharger l'attestation d'assurance"
+              )
+            )
+          end
+        end
+
+        restore_deleted_dossiers(4860, 8603, mapping, private_mapping, pj_mapping)
+      end
+
+      def restore_neph_deleted_dossiers
+        mapping = Class.new(Tasks::DossierProcedureMigrator::ChampMapping) do
+          def can_migrate?(dossier)
+            !(dossier.termine? ||
+              dossier.champs.joins(:type_de_champ).find_by(types_de_champ: { order_place: 3 }).value&.include?('"Demande de duplicata de dossier d\'inscription (suite perte)"'))
+          end
+
+          def setup_mapping
+            champ_opts = {
+              3 => {
+                source_overrides: { 'drop_down' => ["", "Demande de réactualisation du numéro NEPH", "Demande de communication du numéro NEPH", "Demande de duplicata de dossier d'inscription (suite perte)", "Demande de correction sur le Fichier National des Permis de conduire"] },
+                destination_overrides: { 'drop_down' => ["", "Demande de réactualisation du numéro NEPH", "Demande de communication du numéro NEPH", "Demande de correction sur le Fichier National des Permis de conduire"] }
+              }
+            }
+            (0..14).each do |i|
+              map_source_to_destination_champ(i, i, **(champ_opts[i] || {}))
+            end
+            (16..22).each do |i|
+              map_source_to_destination_champ(i, i + 2, **(champ_opts[i] || {}))
+            end
+
+            discard_source_champ(
+              TypeDeChamp.new(
+                type_champ: 'address',
+                order_place: 15,
+                libelle: 'Adresse du candidat'
+              )
+            )
+
+            compute_destination_champ(
+              TypeDeChamp.new(
+                type_champ: 'address',
+                order_place: 15,
+                libelle: 'Adresse du candidat',
+                mandatory: true
+              )
+            ) do |d, target_tdc|
+              value = d.champs.joins(:type_de_champ).find_by(types_de_champ: { order_place: 3 }).value
+              if !d.brouillon?
+                value ||= 'non renseigné'
+              end
+              target_tdc.champ.create(dossier: d, value: value)
+            end
+
+            compute_destination_champ(
+              TypeDeChamp.new(
+                type_champ: 'address',
+                order_place: 16,
+                libelle: 'Code postal',
+                mandatory: true
+              )
+            ) do |d, target_tdc|
+              target_tdc.champ.create(dossier: d, value: d.brouillon? ? nil : 'non renseigné')
+            end
+
+            compute_destination_champ(
+              TypeDeChamp.new(
+                type_champ: 'address',
+                order_place: 17,
+                libelle: 'Ville',
+                mandatory: true
+              )
+            ) do |d, target_tdc|
+              target_tdc.champ.create(dossier: d, value: d.brouillon? ? nil : 'non renseigné')
+            end
+          end
+        end
+
+        private_mapping = Class.new(Tasks::DossierProcedureMigrator::ChampMapping) do
+          def setup_mapping
+            (0..2).each do |i|
+              map_source_to_destination_champ(i, i)
+            end
+          end
+        end
+
+        pj_mapping = Class.new(Tasks::DossierProcedureMigrator::PieceJustificativeMapping) do
+          def setup_mapping
+            (0..3).each do |i|
+              map_source_to_destination_pj(i, i)
+            end
+          end
+        end
+
+        restore_deleted_dossiers(6388, 8770, mapping, private_mapping, pj_mapping)
+      end
+
+      def restore_deleted_dossiers(deleted_procedure_id, new_procedure_id, champ_mapping, champ_private_mapping, pj_mapping)
+        source_procedure = Procedure.unscoped.find(deleted_procedure_id)
+        destination_procedure = Procedure.find(new_procedure_id)
+
+        deleted_dossiers = Dossier.unscoped
+          .where(procedure_id: deleted_procedure_id)
+          .where('dossiers.hidden_at >= ?', source_procedure.hidden_at)
+
+        deleted_dossier_ids = deleted_dossiers.pluck(:id).to_a
+        deleted_dossiers.update_all(hidden_at: nil)
+
+        source_procedure
+          .update_columns(
+            hidden_at: nil,
+            archived_at: source_procedure.hidden_at,
+            aasm_state: :archivee
+          )
+
+        migrator = Tasks::DossierProcedureMigrator.new(source_procedure, destination_procedure, champ_mapping, champ_private_mapping, pj_mapping) do |dossier|
+          DossierMailer.notify_undelete_to_user(dossier).deliver_later
+        end
+        migrator.check_consistency
+        migrator.migrate_dossiers
+
+        source_procedure.dossiers.where(id: deleted_dossier_ids).find_each do |dossier|
+          if dossier.termine?
+            DossierMailer.notify_undelete_to_user(dossier).deliver_later
+          else
+            rake_puts "Dossier #{dossier.id} non migré\n"
+            DossierMailer.notify_unmigrated_to_user(dossier, destination_procedure).deliver_later
+          end
+        end
+      end
+    end.new.run
+  end
+end

lib/tasks/dossier_procedure_migrator.rb

@@ -3,18 +3,54 @@ module Tasks
     # Migrates dossiers from an old source procedure to a revised destination procedure.
 
     class ChampMapping
-      attr_reader :expected_source_types_de_champ
-      attr_reader :expected_destination_types_de_champ
-
-      def initialize(source_procedure, destination_procedure)
+      def initialize(source_procedure, destination_procedure, is_private)
         @source_procedure = source_procedure
         @destination_procedure = destination_procedure
+        @is_private = is_private
 
         @expected_source_types_de_champ = {}
         @expected_destination_types_de_champ = {}
         @source_to_destination_mapping = {}
         @source_champs_to_discard = Set[]
         @destination_champ_computations = []
+
+        setup_mapping
+      end
+
+      def check_source_destination_consistency
+        check_champs_consistency("#{privacy_label}source", @expected_source_types_de_champ, types_de_champ(@source_procedure))
+        check_champs_consistency("#{privacy_label}destination", @expected_destination_types_de_champ, types_de_champ(@destination_procedure))
+      end
+
+      def can_migrate?(dossier)
+        true
+      end
+
+      def migrate(dossier)
+        # Since we’re going to iterate and change the champs at the same time,
+        # we use to_a to make the list static and avoid nasty surprises
+        original_champs = champs(dossier).to_a
+
+        compute_new_champs(dossier)
+
+        original_champs.each do |c|
+          tdc_to = destination_type_de_champ(c)
+          if tdc_to.present?
+            c.update_columns(type_de_champ_id: tdc_to.id)
+          elsif discard_champ?(c)
+            champs(dossier).destroy(c)
+          else
+            fail "Unhandled source #{privacy_label}type de champ #{c.type_de_champ.order_place}"
+          end
+        end
+      end
+
+      private
+
+      def compute_new_champs(dossier)
+        @destination_champ_computations.each do |tdc, block|
+          champs(dossier) << block.call(dossier, tdc)
+        end
       end
 
       def destination_type_de_champ(champ)
@@ -25,21 +61,55 @@ module Tasks
         @source_champs_to_discard.member?(champ.type_de_champ.order_place)
       end
 
-      def compute_new_champs(dossier)
-        @destination_champ_computations.each do |tdc, block|
-          dossier.champs << block.call(dossier, tdc)
+      def setup_mapping
+      end
+
+      def champs(dossier)
+        @is_private ? dossier.champs_private : dossier.champs
+      end
+
+      def types_de_champ(procedure)
+        @is_private ? procedure.types_de_champ_private : procedure.types_de_champ
+      end
+
+      def privacy_label
+        @is_private ? 'private ' : ''
+      end
+
+      def check_champs_consistency(label, expected_tdcs, actual_tdcs)
+        if actual_tdcs.size != expected_tdcs.size
+          raise "Incorrect #{label} size #{actual_tdcs.size} (expected #{expected_tdcs.size})"
+        end
+        actual_tdcs.each { |tdc| check_champ_consistency(label, expected_tdcs[tdc.order_place], tdc) }
+      end
+
+      def check_champ_consistency(label, expected_tdc, actual_tdc)
+        errors = []
+        if actual_tdc.libelle != expected_tdc['libelle']
+          errors.append("incorrect libelle #{actual_tdc.libelle} (expected #{expected_tdc['libelle']})")
+        end
+        if actual_tdc.type_champ != expected_tdc['type_champ']
+          errors.append("incorrect type champ #{actual_tdc.type_champ} (expected #{expected_tdc['type_champ']})")
+        end
+        if (!actual_tdc.mandatory) && expected_tdc['mandatory']
+          errors.append("champ should be mandatory")
+        end
+        drop_down = actual_tdc.drop_down_list.presence&.options&.presence
+        if drop_down != expected_tdc['drop_down']
+          errors.append("incorrect drop down list #{drop_down} (expected #{expected_tdc['drop_down']})")
+        end
+        if errors.present?
+          fail "On #{label} type de champ #{actual_tdc.order_place} (#{actual_tdc.libelle}) " + errors.join(', ')
         end
       end
 
-      private
-
       def map_source_to_destination_champ(source_order_place, destination_order_place, source_overrides: {}, destination_overrides: {})
-        destination_type_de_champ = @destination_procedure.types_de_champ.find_by(order_place: destination_order_place)
+        destination_type_de_champ = types_de_champ(@destination_procedure).find_by(order_place: destination_order_place)
         @expected_source_types_de_champ[source_order_place] =
           type_de_champ_to_expectation(destination_type_de_champ)
           .merge!(source_overrides)
         @expected_destination_types_de_champ[destination_order_place] =
-          type_de_champ_to_expectation(@source_procedure.types_de_champ.find_by(order_place: source_order_place))
+          type_de_champ_to_expectation(types_de_champ(@source_procedure).find_by(order_place: source_order_place))
           .merge!({ "mandatory" => false }) # Even if the source was mandatory, it’s ok for the destination to be optional
           .merge!(destination_overrides)
         @source_to_destination_mapping[source_order_place] = destination_type_de_champ
@@ -52,7 +122,7 @@ module Tasks
 
       def compute_destination_champ(destination_type_de_champ, &block)
         @expected_destination_types_de_champ[destination_type_de_champ.order_place] = type_de_champ_to_expectation(destination_type_de_champ)
-        @destination_champ_computations << [@destination_procedure.types_de_champ.find_by(order_place: destination_type_de_champ.order_place), block]
+        @destination_champ_computations << [types_de_champ(@destination_procedure).find_by(order_place: destination_type_de_champ.order_place), block]
       end
 
       def type_de_champ_to_expectation(tdc)
@@ -66,10 +136,113 @@ module Tasks
       end
     end
 
-    def initialize(source_procedure, destination_procedure, champ_mapping)
+    class PieceJustificativeMapping
+      def initialize(source_procedure, destination_procedure)
+        @source_procedure = source_procedure
+        @destination_procedure = destination_procedure
+
+        @expected_source_pj = {}
+        @expected_destination_pj = {}
+        @source_to_destination_mapping = {}
+
+        setup_mapping
+      end
+
+      def check_source_destination_consistency
+        check_pjs_consistency('source', @expected_source_pj, @source_procedure.types_de_piece_justificative)
+        check_pjs_consistency('destination', @expected_destination_pj, @destination_procedure.types_de_piece_justificative)
+      end
+
+      def can_migrate?(dossier)
+        true
+      end
+
+      def migrate(dossier)
+        # Since we’re going to iterate and change the pjs at the same time,
+        # we use to_a to make the list static and avoid nasty surprises
+        original_pjs = dossier.pieces_justificatives.to_a
+
+        original_pjs.each do |pj|
+          pj_to = destination_pj(pj)
+          if pj_to.present?
+            pj.update_columns(type_de_piece_justificative_id: pj_to.id)
+          elsif discard_pj?(pj)
+            dossier.pieces_justificatives.destroy(pj)
+          else
+            fail "Unhandled source pièce justificative #{c.type_de_piece_justificative.order_place}"
+          end
+        end
+      end
+
+      private
+
+      def destination_pj(pj)
+        @source_to_destination_mapping[pj.order_place]
+      end
+
+      def discard_pj?(champ)
+        @source_pjs_to_discard.member?(pj.order_place)
+      end
+
+      def setup_mapping
+      end
+
+      def map_source_to_destination_pj(source_order_place, destination_order_place, source_overrides: {}, destination_overrides: {})
+        destination_pj = @destination_procedure.types_de_piece_justificative.find_by(order_place: destination_order_place)
+        @expected_source_pj[source_order_place] =
+          pj_to_expectation(destination_pj)
+          .merge!(source_overrides)
+        @expected_destination_pj[destination_order_place] =
+          pj_to_expectation(@source_procedure.types_de_piece_justificative.find_by(order_place: source_order_place))
+          .merge!({ "mandatory" => false }) # Even if the source was mandatory, it’s ok for the destination to be optional
+          .merge!(destination_overrides)
+        @source_to_destination_mapping[source_order_place] = destination_pj
+      end
+
+      def discard_source_pj(source_pj)
+        @expected_source_pj[source_pj.order_place] = pj_to_expectation(source_pj)
+        @source_pjs_to_discard << source_pj.order_place
+      end
+
+      def leave_destination_pj_blank(destination_pj)
+        @expected_destination_pj[destination_pj.order_place] = pj_to_expectation(destination_pj)
+      end
+
+      def pj_to_expectation(pj)
+        pj&.as_json(only: [:libelle, :mandatory]) || {}
+      end
+
+      def check_pjs_consistency(label, expected_pjs, actual_pjs)
+        if actual_pjs.size != expected_pjs.size
+          raise "Incorrect #{label} pièce justificative count #{actual_pjs.size} (expected #{expected_pjs.size})"
+        end
+        actual_pjs.each { |pj| check_pj_consistency(label, expected_pjs[pj.order_place], pj) }
+      end
+
+      def check_pj_consistency(label, expected_pj, actual_pj)
+        errors = []
+        if actual_pj.libelle != expected_pj['libelle']
+          errors.append("incorrect libelle #{actual_pj.libelle} (expected #{expected_pj['libelle']})")
+        end
+        if (!actual_pj.mandatory) && expected_pj['mandatory']
+          errors.append("pj should be mandatory")
+        end
+        if errors.present?
+          fail "On #{label} type de pièce justificative #{actual_pj.order_place} (#{actual_pj.libelle}) " + errors.join(', ')
+        end
+      end
+    end
+
+    def initialize(source_procedure, destination_procedure, champ_mapping, private_champ_mapping = ChampMapping, piece_justificative_mapping = PieceJustificativeMapping, &block)
       @source_procedure = source_procedure
       @destination_procedure = destination_procedure
-      @champ_mapping = champ_mapping
+      @champ_mapping = champ_mapping.new(source_procedure, destination_procedure, false)
+      @private_champ_mapping = private_champ_mapping.new(source_procedure, destination_procedure, true)
+      @piece_justificative_mapping = piece_justificative_mapping.new(source_procedure, destination_procedure)
+
+      if block_given?
+        @on_dossier_migration = block
+      end
     end
 
     def migrate_procedure
@@ -81,7 +254,9 @@ module Tasks
 
     def check_consistency
       check_same_administrateur
-      check_source_destination_champs_consistency
+      @champ_mapping.check_source_destination_consistency
+      @private_champ_mapping.check_source_destination_consistency
+      @piece_justificative_mapping.check_source_destination_consistency
     end
 
     def check_same_administrateur
@@ -90,59 +265,17 @@ module Tasks
       end
     end
 
-    def check_source_destination_champs_consistency
-      check_champs_consistency('source', @champ_mapping.expected_source_types_de_champ, @source_procedure.types_de_champ)
-      check_champs_consistency('destination', @champ_mapping.expected_destination_types_de_champ, @destination_procedure.types_de_champ)
-    end
-
-    def check_champs_consistency(label, expected_tdcs, actual_tdcs)
-      if actual_tdcs.size != expected_tdcs.size
-        raise "Incorrect #{label} size #{actual_tdcs.size} (expected #{expected_tdcs.size})"
-      end
-      actual_tdcs.each { |tdc| check_champ_consistency(label, expected_tdcs[tdc.order_place], tdc) }
-    end
-
-    def check_champ_consistency(label, expected_tdc, actual_tdc)
-      errors = []
-      if actual_tdc.libelle != expected_tdc['libelle']
-        errors.append("incorrect libelle #{actual_tdc.libelle} (expected #{expected_tdc['libelle']})")
-      end
-      if actual_tdc.type_champ != expected_tdc['type_champ']
-        errors.append("incorrect type champ #{actual_tdc.type_champ} (expected #{expected_tdc['type_champ']})")
-      end
-      if (!actual_tdc.mandatory) && expected_tdc['mandatory']
-        errors.append("champ should be mandatory")
-      end
-      drop_down = actual_tdc.drop_down_list.presence&.options&.presence
-      if drop_down != expected_tdc['drop_down']
-        errors.append("incorrect drop down list #{drop_down} (expected #{expected_tdc['drop_down']})")
-      end
-      if errors.present?
-        fail "On #{label} type de champ #{actual_tdc.order_place} (#{actual_tdc.libelle}) " + errors.join(', ')
-      end
-    end
-
     def migrate_dossiers
       @source_procedure.dossiers.find_each(batch_size: 100) do |d|
-        # Since we’re going to iterate and change the champs at the same time,
-        # we use to_a to make the list static and avoid nasty surprises
-        original_champs = d.champs.to_a
+        if @champ_mapping.can_migrate?(d) && @private_champ_mapping.can_migrate?(d) && @piece_justificative_mapping.can_migrate?(d)
+          @champ_mapping.migrate(d)
+          @private_champ_mapping.migrate(d)
+          @piece_justificative_mapping.migrate(d)
 
-        @champ_mapping.compute_new_champs(d)
-
-        original_champs.each do |c|
-          tdc_to = @champ_mapping.destination_type_de_champ(c)
-          if tdc_to.present?
-            c.update(type_de_champ: tdc_to)
-          elsif @champ_mapping.discard_champ?(c)
-            d.champs.destroy(c)
-          else
-            fail "Unhandled source type de champ #{c.type_de_champ.order_place}"
-          end
+          # Use update_columns to avoid triggering build_default_champs
+          d.update_columns(procedure_id: @destination_procedure.id)
+          @on_dossier_migration&.call(d)
         end
-
-        # Use update_columns to avoid triggering build_default_champs
-        d.update_columns(procedure_id: @destination_procedure.id)
       end
     end