Merge pull request #6571 from tchak/fix-improuve-export

Améliorer les perfs des exports

app/models/dossier.rb

@@ -230,25 +230,7 @@ class Dossier < ApplicationRecord
           :published_types_de_champ_private
         ],
         avis: [:claimant, :expert],
-        etablissement: :champ,
-        champs: {
-          type_de_champ: [],
-          etablissement: :champ,
-          piece_justificative_file_attachment: :blob,
-          champs: [
-            type_de_champ: [],
-            piece_justificative_file_attachment: :blob
-          ]
-        },
-        champs_private: {
-          type_de_champ: [],
-          etablissement: :champ,
-          piece_justificative_file_attachment: :blob,
-          champs: [
-            type_de_champ: [],
-            piece_justificative_file_attachment: :blob
-          ]
-        }
+        etablissement: :champ
       ).order(en_construction_at: 'asc')
   }
   scope :en_cours,                    -> { not_archived.state_en_construction_ou_instruction }
@@ -387,6 +369,41 @@ class Dossier < ApplicationRecord
   validates :individual, presence: true, if: -> { revision.procedure.for_individual? }
   validates :groupe_instructeur, presence: true, if: -> { !brouillon? }
 
+  EXPORT_BATCH_SIZE = 5000
+
+  def self.downloadable_sorted_batch
+    dossiers = downloadable_sorted.to_a
+    (dossiers.size.to_f / EXPORT_BATCH_SIZE).ceil.times do |i|
+      start_index = i * EXPORT_BATCH_SIZE
+      end_index = start_index + EXPORT_BATCH_SIZE - 1
+      load_champs(dossiers[start_index..end_index])
+    end
+    dossiers
+  end
+
+  def self.load_champs(dossiers)
+    ::ActiveRecord::Associations::Preloader.new.preload(dossiers, {
+      champs: {
+        type_de_champ: [],
+        etablissement: :champ,
+        piece_justificative_file_attachment: :blob,
+        champs: [
+          type_de_champ: [],
+          piece_justificative_file_attachment: :blob
+        ]
+      },
+      champs_private: {
+        type_de_champ: [],
+        etablissement: :champ,
+        piece_justificative_file_attachment: :blob,
+        champs: [
+          type_de_champ: [],
+          piece_justificative_file_attachment: :blob
+        ]
+      }
+    })
+  end
+
   def user_deleted?
     persisted? && user_id.nil?
   end

app/services/procedure_export_service.rb

@@ -3,7 +3,7 @@ class ProcedureExportService
 
   def initialize(procedure, dossiers)
     @procedure = procedure
-    @dossiers = dossiers.downloadable_sorted
+    @dossiers = dossiers.downloadable_sorted_batch
     @tables = [:dossiers, :etablissements, :avis] + champs_repetables_options
   end
 

db/migrate/20211020104237_add_champs_etablissement_id_index.rb

@@ -0,0 +1,7 @@
+class AddChampsEtablissementIdIndex < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :champs, :etablissement_id, algorithm: :concurrently
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_10_12_100819) do
+ActiveRecord::Schema.define(version: 2021_10_20_104237) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -191,6 +191,7 @@ ActiveRecord::Schema.define(version: 2021_10_12_100819) do
     t.string "fetch_external_data_exceptions", array: true
     t.jsonb "value_json"
     t.index ["dossier_id"], name: "index_champs_on_dossier_id"
+    t.index ["etablissement_id"], name: "index_champs_on_etablissement_id"
     t.index ["parent_id"], name: "index_champs_on_parent_id"
     t.index ["private"], name: "index_champs_on_private"
     t.index ["row"], name: "index_champs_on_row"