Merge pull request #6612 from betagouv/main

2021-11-04-01

Gemfile.lock

@@ -18,7 +18,7 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    aasm (5.1.1)
+    aasm (5.2.0)
       concurrent-ruby (~> 1.0)
     acsv (0.0.1)
     actioncable (6.1.4.1)

app/controllers/api/v2/graphql_controller.rb

@@ -18,6 +18,47 @@ class API::V2::GraphqlController < API::V2::BaseController
 
   private
 
+  def append_info_to_payload(payload)
+    super
+
+    payload.merge!({
+      graphql_operation: operation_log(params[:query], params[:operationName], params[:variables])
+    })
+  end
+
+  def operation_log(query, operation_name, variables)
+    return "NoQuery" if query.nil?
+
+    operation = GraphQL.parse(query).children.find do |node|
+      if node.is_a?(GraphQL::Language::Nodes::OperationDefinition)
+        node.name == operation_name
+      end
+    end
+
+    return "InvalidQuery" if operation.nil?
+    return "IntrospectionQuery" if operation.name == "IntrospectionQuery"
+
+    message = operation.operation_type
+    if operation.name
+      message += ": #{operation.name} { "
+    end
+    message += operation.selections.map(&:name).join(', ')
+    message += " }"
+    if variables.present?
+      message += " "
+      message += variables.to_unsafe_h.flat_map do |(name, value)|
+        if name == "input"
+          value.map do |(name, value)|
+            "#{name}: \"#{value.to_s.truncate(10)}\""
+          end
+        else
+          "#{name}: \"#{value.to_s.truncate(10)}\""
+        end
+      end.join(', ')
+    end
+    message
+  end
+
   def process_action(*args)
     super
   rescue ActionDispatch::Http::Parameters::ParseError => exception

app/controllers/new_administrateur/groupe_instructeurs_controller.rb

@@ -63,7 +63,7 @@ module NewAdministrateur
     end
 
     def destroy
-      if !groupe_instructeur.dossiers.empty?
+      if !groupe_instructeur.dossiers.with_discarded.empty?
         flash[:alert] = "Impossible de supprimer un groupe avec des dossiers. Il faut le réaffecter avant"
       elsif procedure.groupe_instructeurs.one?
         flash[:alert] = "Suppression impossible : il doit y avoir au moins un groupe instructeur sur chaque procédure"

app/models/avis.rb

@@ -50,6 +50,7 @@ class Avis < ApplicationRecord
   scope :by_latest, -> { order(updated_at: :desc) }
   scope :updated_since?, -> (date) { where('avis.updated_at > ?', date) }
   scope :discarded_termine_expired, -> { unscope(:joins).where(dossier: Dossier.discarded_termine_expired) }
+  scope :discarded_en_construction_expired, -> { unscope(:joins).where(dossier: Dossier.discarded_en_construction_expired) }
 
   # The form allows subtmitting avis requests to several emails at once,
   # hence this virtual attribute.

app/models/champs/repetition_champ.rb

@@ -21,6 +21,7 @@
 #
 class Champs::RepetitionChamp < Champ
   accepts_nested_attributes_for :champs, allow_destroy: true
+  delegate :libelle_for_export, to: :type_de_champ
 
   def rows
     champs.group_by(&:row).values
@@ -57,13 +58,6 @@ class Champs::RepetitionChamp < Champ
     end
   end
 
-  # We have to truncate the label here as spreadsheets have a (30 char) limit on length.
-  def libelle_for_export
-    str = "(#{stable_id}) #{libelle}"
-    # /\*?[] are invalid Excel worksheet characters
-    ActiveStorage::Filename.new(str.delete('[]*?')).sanitized
-  end
-
   class Row < Hashie::Dash
     property :index
     property :dossier_id
@@ -73,19 +67,11 @@ class Champs::RepetitionChamp < Champ
       self[attribute]
     end
 
-    def spreadsheet_columns
+    def spreadsheet_columns(types_de_champ)
       [
         ['Dossier ID', :dossier_id],
         ['Ligne', :index]
-      ] + exported_champs
+      ] + Dossier.champs_for_export(champs, types_de_champ)
-    end
-
-    private
-
-    def exported_champs
-      champs.reject(&:exclude_from_export?).map do |champ|
-        [champ.libelle, champ.for_export]
-      end
     end
   end
 end

app/models/deleted_dossier.rb

@@ -17,8 +17,6 @@
 class DeletedDossier < ApplicationRecord
   belongs_to :procedure, -> { with_discarded }, inverse_of: :deleted_dossiers, optional: false
 
-  validates :dossier_id, uniqueness: true
-
   scope :order_by_updated_at, -> (order = :desc) { order(created_at: order) }
   scope :deleted_since,       -> (since) { where('deleted_dossiers.deleted_at >= ?', since) }
 
@@ -32,16 +30,17 @@ class DeletedDossier < ApplicationRecord
   }
 
   def self.create_from_dossier(dossier, reason)
-    create!(
+    # We have some bad data because of partially deleted dossiers in the past.
+    # For now use find_or_create_by! to avoid errors.
+    create_with(
       reason: reasons.fetch(reason),
-      dossier_id: dossier.id,
       groupe_instructeur_id: dossier.groupe_instructeur_id,
       revision_id: dossier.revision_id,
       user_id: dossier.user_id,
       procedure: dossier.procedure,
       state: dossier.state,
       deleted_at: Time.zone.now
-    )
+    ).create_or_find_by!(dossier_id: dossier.id)
   end
 
   def procedure_removed?

app/models/dossier.rb

@@ -606,7 +606,7 @@ class Dossier < ApplicationRecord
   end
 
   def keep_track_on_deletion?
-    !procedure.brouillon?
+    !procedure.brouillon? && !brouillon?
   end
 
   def expose_legacy_carto_api?
@@ -645,56 +645,68 @@ class Dossier < ApplicationRecord
     end
   end
 
-  def expired_keep_track!
+  def expired_keep_track_and_destroy!
+    transaction do
       if keep_track_on_deletion?
         DeletedDossier.create_from_dossier(self, :expired)
         log_automatic_dossier_operation(:supprimer, self)
       end
+      destroy!
+    end
+    true
+  rescue
+    false
   end
 
   def discard_and_keep_track!(author, reason)
+    user_email = user_deleted? ? nil : user_email_for(:notification)
+    deleted_dossier = nil
+
+    transaction do
       if keep_track_on_deletion?
-      if en_construction?
-        deleted_dossier = DeletedDossier.create_from_dossier(self, reason)
-
-        administration_emails = followers_instructeurs.present? ? followers_instructeurs.map(&:email) : procedure.administrateurs.map(&:email)
-        administration_emails.each do |email|
-          DossierMailer.notify_deletion_to_administration(deleted_dossier, email).deliver_later
-        end
-
-        if !user_deleted?
-          DossierMailer.notify_deletion_to_user(deleted_dossier, user_email_for(:notification)).deliver_later
-        end
-
         log_dossier_operation(author, :supprimer, self)
-      elsif termine?
         deleted_dossier = DeletedDossier.create_from_dossier(self, reason)
-
-        if !user_deleted?
-          DossierMailer.notify_instructeur_deletion_to_user(deleted_dossier, user_email_for(:notification)).deliver_later
-        end
-
-        log_dossier_operation(author, :supprimer, self)
-      end
       end
 
       update!(dossier_transfer_id: nil)
       discard!
     end
 
-  def restore(author, only_discarded_with_procedure = false)
+    if deleted_dossier.present?
-    if discarded?
+      if en_construction?
-      deleted_dossier = DeletedDossier.find_by(dossier_id: id)
+        administration_emails = followers_instructeurs.present? ? followers_instructeurs.map(&:email) : procedure.administrateurs.map(&:email)
+        administration_emails.each do |email|
+          DossierMailer.notify_deletion_to_administration(deleted_dossier, email).deliver_later
+        end
+      end
 
-      if !only_discarded_with_procedure || deleted_dossier&.procedure_removed?
+      if user_email.present?
-        if undiscard && keep_track_on_deletion? && en_construction?
+        if reason == :user_request
-          deleted_dossier&.destroy
+          DossierMailer.notify_deletion_to_user(deleted_dossier, user_email).deliver_later
+        else
+          DossierMailer.notify_instructeur_deletion_to_user(deleted_dossier, user_email).deliver_later
+        end
+      end
+    end
+  end
+
+  def restore(author)
+    if discarded?
+      transaction do
+        if undiscard && keep_track_on_deletion?
+          deleted_dossier&.destroy!
           log_dossier_operation(author, :restaurer, self)
         end
       end
     end
   end
 
+  def restore_if_discarded_with_procedure(author)
+    if deleted_dossier&.procedure_removed?
+      restore(author)
+    end
+  end
+
   def after_passer_en_construction
     update!(conservation_extension: 0.days)
     update!(en_construction_at: Time.zone.now) if self.en_construction_at.nil?
@@ -906,12 +918,12 @@ class Dossier < ApplicationRecord
       columns << ['Groupe instructeur', groupe_instructeur.label]
     end
 
-    columns + champs_for_export(types_de_champ)
+    columns + self.class.champs_for_export(champs + champs_private, types_de_champ)
   end
 
-  def champs_for_export(types_de_champ)
+  def self.champs_for_export(champs, types_de_champ)
     # Index values by stable_id
-    values = (champs + champs_private).reject(&:exclude_from_export?)
+    values = champs.reject(&:exclude_from_export?)
       .index_by(&:stable_id)
       .transform_values(&:for_export)
 
@@ -965,6 +977,7 @@ class Dossier < ApplicationRecord
 
     transaction do
       DossierOperationLog.discarded_en_construction_expired.destroy_all
+      Avis.discarded_en_construction_expired.destroy_all
       discarded_en_construction_expired.destroy_all
     end
 
@@ -977,6 +990,10 @@ class Dossier < ApplicationRecord
 
   private
 
+  def deleted_dossier
+    @deleted_dossier ||= DeletedDossier.find_by(dossier_id: id)
+  end
+
   def defaut_groupe_instructeur?
     groupe_instructeur == procedure.defaut_groupe_instructeur
   end

app/models/procedure.rb

@@ -99,20 +99,16 @@ class Procedure < ApplicationRecord
   end
 
   def types_de_champ_for_procedure_presentation
-    explanatory_types_de_champ = [:header_section, :explication, :repetition].map { |k| TypeDeChamp.type_champs.fetch(k) }
-
     if brouillon?
-      TypeDeChamp
+      TypeDeChamp.fillable
-        .joins(:revisions)
+        .joins(:revision_types_de_champ)
-        .where.not(type_champ: explanatory_types_de_champ)
+        .where(revision_types_de_champ: { revision: draft_revision })
-        .where(procedure_revisions: { id: draft_revision_id })
         .order(:private, :position)
     else
       # fetch all type_de_champ.stable_id for all the revisions expect draft
       # and for each stable_id take the bigger (more recent) type_de_champ.id
-      recent_ids = TypeDeChamp
+      recent_ids = TypeDeChamp.fillable
         .joins(:revisions)
-        .where.not(type_champ: explanatory_types_de_champ)
         .where(procedure_revisions: { procedure_id: id })
         .where.not(procedure_revisions: { id: draft_revision_id })
         .group(:stable_id)
@@ -139,6 +135,7 @@ class Procedure < ApplicationRecord
     else
       TypeDeChamp.root
         .public_only
+        .fillable
         .where(revision: revisions - [draft_revision])
         .order(:created_at)
         .uniq
@@ -151,6 +148,7 @@ class Procedure < ApplicationRecord
     else
       TypeDeChamp.root
         .private_only
+        .fillable
         .where(revision: revisions - [draft_revision])
         .order(:created_at)
         .uniq
@@ -672,7 +670,7 @@ class Procedure < ApplicationRecord
   def restore(author)
     if discarded? && undiscard
       dossiers.with_discarded.discarded.find_each do |dossier|
-        dossier.restore(author, true)
+        dossier.restore_if_discarded_with_procedure(author)
       end
     end
   end

app/models/type_de_champ.rb

@@ -86,6 +86,8 @@ class TypeDeChamp < ApplicationRecord
   scope :ordered, -> { order(order_place: :asc) }
   scope :root, -> { where(parent_id: nil) }
   scope :repetition, -> { where(type_champ: type_champs.fetch(:repetition)) }
+  scope :not_repetition, -> { where.not(type_champ: type_champs.fetch(:repetition)) }
+  scope :fillable, -> { where.not(type_champ: [type_champs.fetch(:header_section), type_champs.fetch(:explication)]) }
 
   has_many :champ, inverse_of: :type_de_champ, dependent: :destroy do
     def build(params = {})
@@ -287,6 +289,24 @@ class TypeDeChamp < ApplicationRecord
     options.slice(*TypesDeChamp::CarteTypeDeChamp::LAYERS)
   end
 
+  def types_de_champ_for_revision(revision)
+    if revision.draft?
+      # if we are asking for children on a draft revision, just use current child types_de_champ
+      types_de_champ.fillable
+    else
+      # otherwise return all types_de_champ in their latest state
+      types_de_champ = TypeDeChamp
+        .fillable
+        .joins(:parent)
+        .where(parent: { stable_id: stable_id })
+
+      TypeDeChamp
+        .where(id: types_de_champ.group(:stable_id).select('MAX(types_de_champ.id)'))
+        .joins(parent: :revision_types_de_champ)
+        .order(:order_place, 'procedure_revision_types_de_champ.revision_id': :desc)
+    end
+  end
+
   FEATURE_FLAGS = {}
 
   def self.type_de_champ_types_for(procedure, user)

app/models/types_de_champ/repetition_type_de_champ.rb

@@ -4,4 +4,11 @@ class TypesDeChamp::RepetitionTypeDeChamp < TypesDeChamp::TypeDeChampBase
     champ.add_row
     champ
   end
+
+  # We have to truncate the label here as spreadsheets have a (30 char) limit on length.
+  def libelle_for_export(index = 0)
+    str = "(#{stable_id}) #{libelle}"
+    # /\*?[] are invalid Excel worksheet characters
+    ActiveStorage::Filename.new(str.delete('[]*?')).sanitized
+  end
 end

app/models/types_de_champ/type_de_champ_base.rb

@@ -1,14 +1,14 @@
 class TypesDeChamp::TypeDeChampBase
   include ActiveModel::Validations
 
-  delegate :description, :libelle, to: :@type_de_champ
+  delegate :description, :libelle, :stable_id, to: :@type_de_champ
 
   def initialize(type_de_champ)
     @type_de_champ = type_de_champ
   end
 
   def tags_for_template
-    stable_id = @type_de_champ.stable_id
+    stable_id = self.stable_id
     [
       {
         libelle: libelle,
@@ -21,7 +21,7 @@ class TypesDeChamp::TypeDeChampBase
     ]
   end
 
-  def libelle_for_export(index)
+  def libelle_for_export(index = 0)
     libelle
   end
 

app/services/expired_dossiers_deletion_service.rb

@@ -19,22 +19,16 @@ class ExpiredDossiersDeletionService
       .brouillon_close_to_expiration
       .without_brouillon_expiration_notice_sent
 
-    dossiers_close_to_expiration
+    user_notifications = group_by_user_email(dossiers_close_to_expiration)
-      .with_notifiable_procedure
+
-      .includes(:user, :procedure)
+    dossiers_close_to_expiration.update_all(brouillon_close_to_expiration_notice_sent_at: Time.zone.now)
-      .group_by(&:user)
+
-      .each do |(user, dossiers)|
+    user_notifications.each do |(email, dossiers)|
       DossierMailer.notify_brouillon_near_deletion(
         dossiers,
-          user.email
+        email
       ).deliver_later
-
-        # mark as sent dossiers from current notification
-        Dossier.where(id: dossiers).update_all(brouillon_close_to_expiration_notice_sent_at: Time.zone.now)
     end
-
-    # mark as sent dossiers without notification
-    dossiers_close_to_expiration.update_all(brouillon_close_to_expiration_notice_sent_at: Time.zone.now)
   end
 
   def self.send_en_construction_expiration_notices
@@ -52,24 +46,17 @@ class ExpiredDossiersDeletionService
   end
 
   def self.delete_expired_brouillons_and_notify
-    dossiers_to_remove = Dossier.brouillon_expired
+    user_notifications = group_by_user_email(Dossier.brouillon_expired)
+      .map { |(email, dossiers)| [email, dossiers.map(&:hash_for_deletion_mail)] }
 
-    dossiers_to_remove
+    Dossier.brouillon_expired.destroy_all
-      .with_notifiable_procedure
+
-      .includes(:user, :procedure)
+    user_notifications.each do |(email, dossiers_hash)|
-      .group_by(&:user)
-      .each do |(user, dossiers)|
       DossierMailer.notify_brouillon_deletion(
-          dossiers.map(&:hash_for_deletion_mail),
+        dossiers_hash,
-          user.email
+        email
       ).deliver_later
-
-        # destroy dossiers from current notification
-        Dossier.where(id: dossiers).destroy_all
     end
-
-    # destroy dossiers without notification
-    dossiers_to_remove.destroy_all
   end
 
   def self.delete_expired_en_construction_and_notify
@@ -83,57 +70,58 @@ class ExpiredDossiersDeletionService
   private
 
   def self.send_expiration_notices(dossiers_close_to_expiration, close_to_expiration_flag)
-    dossiers_close_to_expiration
+    user_notifications = group_by_user_email(dossiers_close_to_expiration)
-      .with_notifiable_procedure
+    administration_notifications = group_by_fonctionnaire_email(dossiers_close_to_expiration)
-      .includes(:user)
-      .group_by(&:user)
-      .each do |(user, dossiers)|
-        DossierMailer.notify_near_deletion_to_user(
-          dossiers,
-          user.email
-        ).deliver_later
-      end
 
-    group_by_fonctionnaire_email(dossiers_close_to_expiration).each do |(email, dossiers)|
-      DossierMailer.notify_near_deletion_to_administration(
-        dossiers.to_a,
-        email
-      ).deliver_later
-
-      # mark as sent dossiers from current notification
-      Dossier.where(id: dossiers.to_a).update_all(close_to_expiration_flag => Time.zone.now)
-    end
-
-    # mark as sent dossiers without notification
     dossiers_close_to_expiration.update_all(close_to_expiration_flag => Time.zone.now)
+
+    user_notifications.each do |(email, dossiers)|
+      DossierMailer.notify_near_deletion_to_user(dossiers, email).deliver_later
+    end
+    administration_notifications.each do |(email, dossiers)|
+      DossierMailer.notify_near_deletion_to_administration(dossiers, email).deliver_later
+    end
   end
 
   def self.delete_expired_and_notify(dossiers_to_remove, notify_on_closed_procedures_to_user: false)
-    dossiers_to_remove.each(&:expired_keep_track!)
+    user_notifications = group_by_user_email(dossiers_to_remove, notify_on_closed_procedures_to_user: notify_on_closed_procedures_to_user)
+      .map { |(email, dossiers)| [email, dossiers.map(&:id)] }
+    administration_notifications = group_by_fonctionnaire_email(dossiers_to_remove)
+      .map { |(email, dossiers)| [email, dossiers.map(&:id)] }
 
-    dossiers_to_remove
+    deleted_dossier_ids = []
-      .with_notifiable_procedure(notify_on_closed: notify_on_closed_procedures_to_user)
+    dossiers_to_remove.find_each do |dossier|
-      .includes(:user)
+      if dossier.expired_keep_track_and_destroy!
-      .group_by(&:user)
+        deleted_dossier_ids << dossier.id
-      .each do |(user, dossiers)|
+      end
-        DossierMailer.notify_automatic_deletion_to_user(
-          DeletedDossier.where(dossier_id: dossiers.map(&:id)).to_a,
-          user.email
-        ).deliver_later
     end
 
-    self.group_by_fonctionnaire_email(dossiers_to_remove).each do |(email, dossiers)|
+    user_notifications.each do |(email, dossier_ids)|
-      DossierMailer.notify_automatic_deletion_to_administration(
+      dossier_ids = dossier_ids.intersection(deleted_dossier_ids)
-        DeletedDossier.where(dossier_id: dossiers.map(&:id)).to_a,
+      if dossier_ids.present?
+        DossierMailer.notify_automatic_deletion_to_user(
+          DeletedDossier.where(dossier_id: dossier_ids).to_a,
           email
         ).deliver_later
-
+      end
-      # destroy dossiers from current notification
+    end
-      Dossier.where(id: dossiers.to_a).destroy_all
+    administration_notifications.each do |(email, dossier_ids)|
+      dossier_ids = dossier_ids.intersection(deleted_dossier_ids)
+      if dossier_ids.present?
+        DossierMailer.notify_automatic_deletion_to_administration(
+          DeletedDossier.where(dossier_id: dossier_ids).to_a,
+          email
+        ).deliver_later
+      end
+    end
   end
 
-    # destroy dossiers without notification
+  def self.group_by_user_email(dossiers, notify_on_closed_procedures_to_user: false)
-    dossiers_to_remove.destroy_all
+    dossiers
+      .with_notifiable_procedure(notify_on_closed: notify_on_closed_procedures_to_user)
+      .includes(:user, :procedure)
+      .group_by(&:user)
+      .map { |(user, dossiers)| [user.email, dossiers] }
   end
 
   def self.group_by_fonctionnaire_email(dossiers)
@@ -143,5 +131,6 @@ class ExpiredDossiersDeletionService
       .each_with_object(Hash.new { |h, k| h[k] = Set.new }) do |dossier, h|
         (dossier.followers_instructeurs + dossier.procedure.administrateurs).each { |destinataire| h[destinataire.email] << dossier }
       end
+      .map { |(email, dossiers)| [email, dossiers.to_a] }
   end
 end

app/services/procedure_export_service.rb

@@ -39,20 +39,21 @@ class ProcedureExportService
     @avis ||= dossiers.flat_map(&:avis)
   end
 
-  def champs_repetables
-    @champs_repetables ||= dossiers.flat_map do |dossier|
-      [dossier.champs, dossier.champs_private]
-        .flatten
-        .filter { |champ| champ.is_a?(Champs::RepetitionChamp) }
-    end.group_by(&:libelle_for_export)
-  end
-
   def champs_repetables_options
-    champs_repetables.map do |libelle, champs|
+    revision = @procedure.active_revision
-      [
+    champs_by_stable_id = dossiers
-        libelle,
+      .flat_map { |dossier| (dossier.champs + dossier.champs_private).filter(&:repetition?) }
-        champs.flat_map(&:rows_for_export)
+      .group_by(&:stable_id)
-      ]
+
+    @procedure.types_de_champ_for_procedure_presentation.repetition
+      .map { |type_de_champ_repetition| [type_de_champ_repetition, type_de_champ_repetition.types_de_champ_for_revision(revision).to_a] }
+      .filter { |(_, types_de_champ)| types_de_champ.present? }
+      .map do |(type_de_champ_repetition, types_de_champ)|
+        {
+          sheet_name: type_de_champ_repetition.libelle_for_export,
+          instances: champs_by_stable_id.fetch(type_de_champ_repetition.stable_id, []).flat_map(&:rows_for_export),
+          spreadsheet_columns: Proc.new { |instance| instance.spreadsheet_columns(types_de_champ) }
+        }
       end
   end
 
@@ -69,8 +70,8 @@ class ProcedureExportService
       { instances: etablissements.to_a, sheet_name: 'Etablissements' }
     when :avis
       { instances: avis.to_a, sheet_name: 'Avis' }
-    when Array
+    when Hash
-      { instances: table.last, sheet_name: table.first }
+      table
     end.merge(DEFAULT_STYLES)
 
     # transliterate: convert to ASCII characters
@@ -84,7 +85,7 @@ class ProcedureExportService
   end
 
   def spreadsheet_columns(format)
-    types_de_champ = @procedure.types_de_champ_for_procedure_presentation.to_a
+    types_de_champ = @procedure.types_de_champ_for_procedure_presentation.not_repetition.to_a
 
     Proc.new do |instance|
       instance.send(:"spreadsheet_columns_#{format}", types_de_champ: types_de_champ)

config/environments/test.rb

@@ -76,4 +76,10 @@ Rails.application.configure do
     debounce_delay: 500,
     status_visible_duration: 500
   }
+
+  # BCrypt is slow by design - but during tests we want to make it faster
+  # to compute hashes of passwords.
+  silence_warnings do
+    BCrypt::Engine::DEFAULT_COST = BCrypt::Engine::MIN_COST
+  end
 end

lib/tasks/ds/destroy_expired_dossiers_mae.rake

@@ -0,0 +1,16 @@
+namespace :ds do
+  desc 'DS task: destroy_expired_dossiers_mae'
+  task destroy_expired_dossiers_mae: :environment do
+    dossiers = Dossier.state_termine
+      .where("termine_close_to_expiration_notice_sent_at + INTERVAL :expires_in < :now", { now: Time.zone.now, expires_in: '30 days' })
+      .joins(:groupe_instructeur)
+      .where(groupe_instructeur: { procedure_id: [47787, 47844, 47478, 47865] })
+    progress = ProgressReport.new(dossiers.count)
+
+    dossiers.find_each do |dossier|
+      dossier.expired_keep_track_and_destroy!
+      progress.inc
+    end
+    progress.finish
+  end
+end

spec/controllers/api/v1/dossiers_controller_spec.rb

@@ -256,7 +256,7 @@ describe API::V1::DossiersController do
 
           describe 'repetition' do
             let(:procedure) { create(:procedure, :with_repetition, administrateur: admin) }
-            let(:dossier) { create(:dossier, :en_construction, :with_all_champs, procedure: procedure) }
+            let(:dossier) { create(:dossier, :en_construction, :with_populated_champs, procedure: procedure) }
 
             subject { super().first[:rows] }
 

spec/controllers/api/v2/graphql_controller_spec.rb

@@ -1,21 +1,11 @@
 describe API::V2::GraphqlController do
   let(:admin) { create(:administrateur) }
   let(:token) { admin.renew_api_token }
-  let(:procedure) { create(:procedure, :published, :for_individual, :with_service, :with_all_champs, :with_all_annotations, administrateurs: [admin]) }
+  let(:procedure) { create(:procedure, :published, :for_individual, :with_service, administrateurs: [admin]) }
-  let(:dossier) do
+  let(:dossier)  { create(:dossier, :en_construction, :with_individual, procedure: procedure) }
-    dossier = create(:dossier,
-      :en_construction,
-      :with_all_champs,
-      :with_all_annotations,
-      :with_individual,
-      procedure: procedure)
-    create(:commentaire, :with_file, dossier: dossier, email: 'test@test.com')
-    dossier
-  end
   let(:dossier1) { create(:dossier, :en_construction, :with_individual, procedure: procedure, en_construction_at: 1.day.ago) }
   let(:dossier2) { create(:dossier, :en_construction, :with_individual, :archived, procedure: procedure, en_construction_at: 3.days.ago) }
-  let(:dossier_brouillon) { create(:dossier, :with_individual, procedure: procedure) }
+  let(:dossiers) { [dossier] }
-  let(:dossiers) { [dossier2, dossier1, dossier] }
   let(:instructeur) { create(:instructeur, followed_dossiers: dossiers) }
 
   def compute_checksum_in_chunks(io)
@@ -116,8 +106,11 @@ describe API::V2::GraphqlController do
       request.env['HTTP_AUTHORIZATION'] = authorization_header
     end
 
-    context "demarche" do
+    describe "demarche" do
-      it "should be returned" do
+      describe "query a demarche" do
+        let(:procedure) { create(:procedure, :published, :for_individual, :with_service, :with_all_champs, :with_all_annotations, administrateurs: [admin]) }
+
+        it "returns the demarche" do
           expect(gql_errors).to eq(nil)
           expect(gql_data).to eq(demarche: {
             id: procedure.to_typed_id,
@@ -165,8 +158,11 @@ describe API::V2::GraphqlController do
             }
           })
         end
+      end
+
+      describe "filter dossiers" do
+        let(:dossiers) { [dossier, dossier1, dossier2] }
 
-      context "filter dossiers" do
         let(:query) do
           "{
             demarche(number: #{procedure.id}) {
@@ -193,7 +189,8 @@ describe API::V2::GraphqlController do
         end
       end
 
-      context "filter archived dossiers" do
+      describe "filter archived dossiers" do
+        let(:dossiers) { [dossier, dossier1, dossier2] }
         let(:query) do
           "{
             demarche(number: #{procedure.id}) {
@@ -210,7 +207,8 @@ describe API::V2::GraphqlController do
 
         context 'with archived=true' do
           let(:archived_filter) { 'true' }
-          it "only archived dossiers should be returned" do
+
+          it 'returns only archived dossiers' do
             expect(gql_errors).to eq(nil)
             expect(gql_data).to eq(demarche: {
               id: procedure.to_typed_id,
@@ -224,7 +222,8 @@ describe API::V2::GraphqlController do
 
         context 'with archived=false' do
           let(:archived_filter) { 'false' }
-          it "only not archived dossiers should be returned" do
+
+          it 'returns only non-archived dossiers' do
             expect(gql_errors).to eq(nil)
             expect(gql_data).to eq(demarche: {
               id: procedure.to_typed_id,
@@ -237,7 +236,7 @@ describe API::V2::GraphqlController do
         end
       end
 
-      context "filter by minRevision" do
+      describe "filter by minRevision" do
         let(:query) do
           "{
             demarche(number: #{procedure.id}) {
@@ -266,7 +265,7 @@ describe API::V2::GraphqlController do
         end
       end
 
-      context "filter by maxRevision" do
+      describe "filter by maxRevision" do
         let(:query) do
           "{
             demarche(number: #{procedure.id}) {
@@ -296,8 +295,20 @@ describe API::V2::GraphqlController do
       end
     end
 
-    context "dossier" do
+    describe "dossier" do
-      context "with individual" do
+      let(:dossier) do
+        dossier = create(:dossier,
+                         :en_construction,
+                         :with_populated_champs,
+                         :with_populated_annotations,
+                         :with_individual,
+                         procedure: procedure)
+        create(:commentaire, :with_file, dossier: dossier, email: 'test@test.com')
+        dossier
+      end
+
+      context "for individual" do
+        let(:procedure) { create(:procedure, :published, :for_individual, :with_service, :with_all_champs, :with_all_annotations, administrateurs: [admin]) }
         let(:query) do
           "{
             dossier(number: #{dossier.id}) {
@@ -449,7 +460,7 @@ describe API::V2::GraphqlController do
         end
       end
 
-      context "with entreprise" do
+      context "for entreprise" do
         let(:procedure_for_entreprise) { create(:procedure, :published, administrateurs: [admin]) }
         let(:dossier) { create(:dossier, :en_construction, :with_entreprise, procedure: procedure_for_entreprise) }
 
@@ -661,7 +672,7 @@ describe API::V2::GraphqlController do
       end
     end
 
-    context "deletedDossiers" do
+    describe "deletedDossiers" do
       let(:query) do
         "{
           demarche(number: #{procedure.id}) {
@@ -699,7 +710,7 @@ describe API::V2::GraphqlController do
       end
     end
 
-    context "champ" do
+    describe "champ" do
       let(:champ) { create(:champ_piece_justificative, dossier: dossier) }
       let(:byte_size) { 2712286911 }
 
@@ -722,7 +733,7 @@ describe API::V2::GraphqlController do
         }
       end
 
-      context "when file is really big" do
+      context "when the file is really big" do
         before do
           champ.piece_justificative_file.blob.update(byte_size: byte_size)
         end
@@ -766,7 +777,7 @@ describe API::V2::GraphqlController do
       end
     end
 
-    context "groupeInstructeur" do
+    describe "groupeInstructeur" do
       let(:groupe_instructeur) { procedure.groupe_instructeurs.first }
       let(:query) do
         "{
@@ -796,7 +807,7 @@ describe API::V2::GraphqlController do
       end
     end
 
-    context "mutations" do
+    describe "mutations" do
       describe 'dossierEnvoyerMessage' do
         context 'success' do
           let(:query) do
@@ -902,6 +913,7 @@ describe API::V2::GraphqlController do
       end
 
       describe 'dossierPasserEnInstruction' do
+        let(:dossiers) { [dossier2, dossier1, dossier] }
         let(:dossier) { create(:dossier, :en_construction, :with_individual, procedure: procedure) }
         let(:instructeur_id) { instructeur.to_typed_id }
         let(:disable_notification) { false }
@@ -1322,6 +1334,8 @@ describe API::V2::GraphqlController do
       end
 
       describe 'dossierModifierAnnotation' do
+        let(:procedure) { create(:procedure, :published, :for_individual, :with_service, :with_all_annotations, administrateurs: [admin]) }
+
         describe 'text' do
           let(:query) do
             "mutation {
@@ -1563,7 +1577,7 @@ describe API::V2::GraphqlController do
       expect(gql_errors).not_to eq(nil)
     end
 
-    context "dossier" do
+    describe "dossier" do
       let(:query) { "{ dossier(number: #{dossier.id}) { id number usager { email } } }" }
 
       it "should return error" do
@@ -1572,7 +1586,7 @@ describe API::V2::GraphqlController do
       end
     end
 
-    context "mutation" do
+    describe "mutation" do
       let(:query) do
         "mutation {
           dossierEnvoyerMessage(input: {

spec/controllers/instructeurs/dossiers_controller_spec.rb

@@ -560,8 +560,8 @@ describe Instructeurs::DossiersController, type: :controller do
       let(:dossier) do
         create(:dossier,
           :accepte,
-          :with_all_champs,
+          :with_populated_champs,
-          :with_all_annotations,
+          :with_populated_annotations,
           :with_motivation,
           :with_entreprise,
           :with_commentaires, procedure: procedure)
@@ -593,7 +593,7 @@ describe Instructeurs::DossiersController, type: :controller do
         build(:type_de_champ_repetition, :with_types_de_champ, position: 3)
       ], instructeurs: instructeurs)
     end
-    let(:dossier) { create(:dossier, :en_construction, :with_all_annotations, procedure: procedure) }
+    let(:dossier) { create(:dossier, :en_construction, :with_populated_annotations, procedure: procedure) }
     let(:another_instructeur) { create(:instructeur) }
     let(:now) { Time.zone.parse('01/01/2100') }
 

spec/controllers/recherche_controller_spec.rb

@@ -1,10 +1,10 @@
 describe RechercheController, type: :controller do
-  let(:dossier) { create(:dossier, :en_construction, :with_all_annotations) }
+  let(:dossier) { create(:dossier, :en_construction, :with_populated_annotations) }
   let(:dossier2) { create(:dossier, :en_construction, procedure: dossier.procedure) }
   let(:instructeur) { create(:instructeur) }
 
   let(:dossier_with_expert) { avis.dossier }
-  let(:avis) { create(:avis, dossier: create(:dossier, :en_construction, :with_all_annotations)) }
+  let(:avis) { create(:avis, dossier: create(:dossier, :en_construction, :with_populated_annotations)) }
 
   let(:user) { instructeur.user }
 

spec/controllers/users/dossiers_controller_spec.rb

@@ -879,7 +879,7 @@ describe Users::DossiersController, type: :controller do
       let(:dossier) do
         create(:dossier,
           :accepte,
-          :with_all_champs,
+          :with_populated_champs,
           :with_motivation,
           :with_commentaires,
           procedure: procedure,

spec/factories/dossier.rb

@@ -212,7 +212,7 @@ FactoryBot.define do
       end
     end
 
-    trait :with_all_champs do
+    trait :with_populated_champs do
       after(:create) do |dossier, _evaluator|
         dossier.champs = dossier.types_de_champ.map do |type_de_champ|
           build(:"champ_#{type_de_champ.type_champ}", dossier: dossier, type_de_champ: type_de_champ)
@@ -221,7 +221,7 @@ FactoryBot.define do
       end
     end
 
-    trait :with_all_annotations do
+    trait :with_populated_annotations do
       after(:create) do |dossier, _evaluator|
         dossier.champs_private = dossier.types_de_champ_private.map do |type_de_champ|
           build(:"champ_#{type_de_champ.type_champ}", private: true, dossier: dossier, type_de_champ: type_de_champ)

spec/models/dossier_spec.rb

@@ -1341,14 +1341,20 @@ describe Dossier do
   end
 
   describe "champs_for_export" do
-    let(:procedure) { create(:procedure, :with_type_de_champ, :with_datetime, :with_yes_no, :with_explication, :with_commune) }
+    let(:procedure) { create(:procedure, :with_type_de_champ, :with_datetime, :with_yes_no, :with_explication, :with_commune, :with_repetition) }
     let(:text_type_de_champ) { procedure.types_de_champ.find { |type_de_champ| type_de_champ.type_champ == TypeDeChamp.type_champs.fetch(:text) } }
     let(:yes_no_type_de_champ) { procedure.types_de_champ.find { |type_de_champ| type_de_champ.type_champ == TypeDeChamp.type_champs.fetch(:yes_no) } }
     let(:datetime_type_de_champ) { procedure.types_de_champ.find { |type_de_champ| type_de_champ.type_champ == TypeDeChamp.type_champs.fetch(:datetime) } }
     let(:explication_type_de_champ) { procedure.types_de_champ.find { |type_de_champ| type_de_champ.type_champ == TypeDeChamp.type_champs.fetch(:explication) } }
     let(:commune_type_de_champ) { procedure.types_de_champ.find { |type_de_champ| type_de_champ.type_champ == TypeDeChamp.type_champs.fetch(:communes) } }
+    let(:repetition_type_de_champ) { procedure.types_de_champ.find { |type_de_champ| type_de_champ.type_champ == TypeDeChamp.type_champs.fetch(:repetition) } }
+    let(:repetition_champ) { dossier.champs.find(&:repetition?) }
+    let(:repetition_second_revision_champ) { dossier_second_revision.champs.find(&:repetition?) }
     let(:dossier) { create(:dossier, procedure: procedure) }
     let(:dossier_second_revision) { create(:dossier, procedure: procedure) }
+    let(:dossier_champs_for_export) { Dossier.champs_for_export(dossier.champs, procedure.types_de_champ_for_procedure_presentation.not_repetition) }
+    let(:dossier_second_revision_champs_for_export) { Dossier.champs_for_export(dossier_second_revision.champs, procedure.types_de_champ_for_procedure_presentation.not_repetition) }
+    let(:repetition_second_revision_champs_for_export) { Dossier.champs_for_export(repetition_second_revision_champ.champs, procedure.types_de_champ_for_procedure_presentation.repetition) }
 
     context "when procedure published" do
       before do
@@ -1358,16 +1364,19 @@ describe Dossier do
         procedure.draft_revision.add_type_de_champ(type_champ: TypeDeChamp.type_champs.fetch(:text), libelle: 'New text field')
         procedure.draft_revision.find_or_clone_type_de_champ(yes_no_type_de_champ.stable_id).update(libelle: 'Updated yes/no')
         procedure.draft_revision.find_or_clone_type_de_champ(commune_type_de_champ.stable_id).update(libelle: 'Commune de naissance')
+        procedure.draft_revision.find_or_clone_type_de_champ(repetition_type_de_champ.stable_id).update(libelle: 'Repetition')
         procedure.update(published_revision: procedure.draft_revision, draft_revision: procedure.create_new_revision)
         dossier.reload
         procedure.reload
       end
 
       it "should have champs from all revisions" do
-        expect(dossier.types_de_champ.map(&:libelle)).to eq([text_type_de_champ.libelle, datetime_type_de_champ.libelle, "Yes/no", explication_type_de_champ.libelle, commune_type_de_champ.libelle])
+        expect(dossier.types_de_champ.map(&:libelle)).to eq([text_type_de_champ.libelle, datetime_type_de_champ.libelle, "Yes/no", explication_type_de_champ.libelle, commune_type_de_champ.libelle, repetition_type_de_champ.libelle])
-        expect(dossier_second_revision.types_de_champ.map(&:libelle)).to eq([datetime_type_de_champ.libelle, "Updated yes/no", explication_type_de_champ.libelle, 'Commune de naissance', "New text field"])
+        expect(dossier_second_revision.types_de_champ.map(&:libelle)).to eq([datetime_type_de_champ.libelle, "Updated yes/no", explication_type_de_champ.libelle, 'Commune de naissance', "Repetition", "New text field"])
-        expect(dossier.champs_for_export(dossier.procedure.types_de_champ_for_procedure_presentation).map { |(libelle)| libelle }).to eq([text_type_de_champ.libelle, datetime_type_de_champ.libelle, "Updated yes/no", "Commune de naissance", "Commune de naissance (Code insee)", "New text field"])
+        expect(dossier_champs_for_export.map { |(libelle)| libelle }).to eq([text_type_de_champ.libelle, datetime_type_de_champ.libelle, "Updated yes/no", "Commune de naissance", "Commune de naissance (Code insee)", "New text field"])
-        expect(dossier.champs_for_export(dossier.procedure.types_de_champ_for_procedure_presentation)).to eq(dossier_second_revision.champs_for_export(dossier_second_revision.procedure.types_de_champ_for_procedure_presentation))
+        expect(dossier_champs_for_export).to eq(dossier_second_revision_champs_for_export)
+        expect(repetition_second_revision_champs_for_export.map { |(libelle)| libelle }).to eq(procedure.types_de_champ_for_procedure_presentation.repetition.map(&:libelle_for_export))
+        expect(repetition_second_revision_champs_for_export.first.size).to eq(2)
       end
     end
 
@@ -1375,7 +1384,7 @@ describe Dossier do
       let(:procedure) { create(:procedure, :with_type_de_champ, :with_explication) }
 
       it "should not contain non-exportable types de champ" do
-        expect(dossier.champs_for_export(dossier.procedure.types_de_champ_for_procedure_presentation).map { |(libelle)| libelle }).to eq([text_type_de_champ.libelle])
+        expect(dossier_champs_for_export.map { |(libelle)| libelle }).to eq([text_type_de_champ.libelle])
       end
     end
   end
@@ -1452,6 +1461,42 @@ describe Dossier do
       expect(dossier.destroy).to be_truthy
       expect(transfer.reload).not_to be_nil
     end
+
+    context 'discarded' do
+      context 'en_construction' do
+        let(:dossier) { create(:dossier, :en_construction) }
+
+        before do
+          create(:avis, dossier: dossier)
+          Timecop.travel(2.weeks.ago) do
+            dossier.discard!
+          end
+          dossier.reload
+        end
+
+        it "can destroy dossier with avis" do
+          Avis.discarded_en_construction_expired.destroy_all
+          expect(dossier.destroy).to be_truthy
+        end
+      end
+
+      context 'termine' do
+        let(:dossier) { create(:dossier, :accepte) }
+
+        before do
+          create(:avis, dossier: dossier)
+          Timecop.travel(2.weeks.ago) do
+            dossier.discard!
+          end
+          dossier.reload
+        end
+
+        it "can destroy dossier with avis" do
+          Avis.discarded_termine_expired.destroy_all
+          expect(dossier.destroy).to be_truthy
+        end
+      end
+    end
   end
 
   describe "#spreadsheet_columns" do

spec/models/procedure_presentation_and_revisions_spec.rb

@@ -1,6 +1,6 @@
 describe ProcedurePresentation do
   describe "#types_de_champ_for_procedure_presentation" do
-    subject { procedure.types_de_champ_for_procedure_presentation.pluck(:libelle) }
+    subject { procedure.types_de_champ_for_procedure_presentation.not_repetition.pluck(:libelle) }
 
     context 'for a draft procedure' do
       let(:procedure) { create(:procedure) }

spec/models/procedure_revision_spec.rb

@@ -56,7 +56,7 @@ describe ProcedureRevision do
       revision.reload
       expect(revision.types_de_champ.index(type_de_champ)).to eq(2)
       expect(revision.procedure.types_de_champ.index(type_de_champ)).to eq(2)
-      expect(revision.procedure.types_de_champ_for_procedure_presentation.index(type_de_champ)).to eq(2)
+      expect(revision.procedure.types_de_champ_for_procedure_presentation.not_repetition.index(type_de_champ)).to eq(2)
     end
 
     it 'move up' do
@@ -66,7 +66,7 @@ describe ProcedureRevision do
       revision.reload
       expect(revision.types_de_champ.index(last_type_de_champ)).to eq(0)
       expect(revision.procedure.types_de_champ.index(last_type_de_champ)).to eq(0)
-      expect(revision.procedure.types_de_champ_for_procedure_presentation.index(last_type_de_champ)).to eq(0)
+      expect(revision.procedure.types_de_champ_for_procedure_presentation.not_repetition.index(last_type_de_champ)).to eq(0)
     end
 
     context 'repetition' do

spec/services/procedure_export_service_spec.rb

@@ -32,7 +32,7 @@ describe ProcedureExportService do
     end
 
     describe 'Dossiers sheet' do
-      let!(:dossier) { create(:dossier, :en_instruction, :with_all_champs, :with_individual, procedure: procedure) }
+      let!(:dossier) { create(:dossier, :en_instruction, :with_populated_champs, :with_individual, procedure: procedure) }
 
       let(:nominal_headers) do
         [
@@ -119,7 +119,7 @@ describe ProcedureExportService do
 
     describe 'Etablissement sheet' do
       let(:procedure) { create(:procedure, :published, :with_all_champs) }
-      let!(:dossier) { create(:dossier, :en_instruction, :with_all_champs, :with_entreprise, procedure: procedure) }
+      let!(:dossier) { create(:dossier, :en_instruction, :with_populated_champs, :with_entreprise, procedure: procedure) }
 
       let(:dossier_etablissement) { etablissements_sheet.data[1] }
       let(:champ_etablissement) { etablissements_sheet.data[0] }
@@ -309,7 +309,7 @@ describe ProcedureExportService do
     end
 
     describe 'Avis sheet' do
-      let!(:dossier) { create(:dossier, :en_instruction, :with_all_champs, :with_individual, procedure: procedure) }
+      let!(:dossier) { create(:dossier, :en_instruction, :with_populated_champs, :with_individual, procedure: procedure) }
       let!(:avis) { create(:avis, :with_answer, dossier: dossier) }
 
       it 'should have headers' do
@@ -332,8 +332,8 @@ describe ProcedureExportService do
     describe 'Repetitions sheet' do
       let!(:dossiers) do
         [
-          create(:dossier, :en_instruction, :with_all_champs, :with_individual, procedure: procedure),
+          create(:dossier, :en_instruction, :with_populated_champs, :with_individual, procedure: procedure),
-          create(:dossier, :en_instruction, :with_all_champs, :with_individual, procedure: procedure)
+          create(:dossier, :en_instruction, :with_populated_champs, :with_individual, procedure: procedure)
         ]
       end
       let(:champ_repetition) { dossiers.first.champs.find { |champ| champ.type_champ == 'repetition' } }
@@ -381,7 +381,7 @@ describe ProcedureExportService do
       end
 
       context 'with non unique labels' do
-        let(:dossier) { create(:dossier, :en_instruction, :with_all_champs, :with_individual, procedure: procedure) }
+        let(:dossier) { create(:dossier, :en_instruction, :with_populated_champs, :with_individual, procedure: procedure) }
         let(:champ_repetition) { dossier.champs.find { |champ| champ.type_champ == 'repetition' } }
         let(:type_de_champ_repetition) { create(:type_de_champ_repetition, procedure: procedure, libelle: champ_repetition.libelle) }
         let!(:another_champ_repetition) { create(:champ_repetition, type_de_champ: type_de_champ_repetition, dossier: dossier) }

spec/system/users/list_dossiers_spec.rb

@@ -1,7 +1,7 @@
 describe 'user access to the list of their dossiers' do
   let(:user) { create(:user) }
   let!(:dossier_brouillon)       { create(:dossier, user: user) }
-  let!(:dossier_en_construction) { create(:dossier, :with_all_champs, :en_construction, user: user) }
+  let!(:dossier_en_construction) { create(:dossier, :with_populated_champs, :en_construction, user: user) }
   let!(:dossier_en_instruction)  { create(:dossier, :en_instruction, user: user) }
   let!(:dossier_archived)        { create(:dossier, :en_instruction, :archived, user: user) }
   let(:dossiers_per_page) { 25 }
@@ -121,7 +121,7 @@ describe 'user access to the list of their dossiers' do
     end
 
     context "when user search for something inside the dossier" do
-      let(:dossier_en_construction2) { create(:dossier, :with_all_champs, :en_construction, user: user) }
+      let(:dossier_en_construction2) { create(:dossier, :with_populated_champs, :en_construction, user: user) }
       before do
         page.find_by_id('q').set(dossier_en_construction.champs.first.value)
       end