From 7e5d388ef8dad2e104a4b34328f170c92653f5e5 Mon Sep 17 00:00:00 2001
From: Paul Chavard <github@paul.chavard.net>
Date: Wed, 20 Oct 2021 16:50:18 +0200
Subject: [PATCH 1/2] fix(export): add index on champs.etablissement_id

---
 .../20211020104237_add_champs_etablissement_id_index.rb    | 7 +++++++
 db/schema.rb                                               | 3 ++-
 2 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20211020104237_add_champs_etablissement_id_index.rb

diff --git a/db/migrate/20211020104237_add_champs_etablissement_id_index.rb b/db/migrate/20211020104237_add_champs_etablissement_id_index.rb
new file mode 100644
index 000000000..444fdb405
--- /dev/null
+++ b/db/migrate/20211020104237_add_champs_etablissement_id_index.rb
@@ -0,0 +1,7 @@
+class AddChampsEtablissementIdIndex < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :champs, :etablissement_id, algorithm: :concurrently
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 219a263a8..b8c245f1c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_10_12_100819) do
+ActiveRecord::Schema.define(version: 2021_10_20_104237) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -191,6 +191,7 @@ ActiveRecord::Schema.define(version: 2021_10_12_100819) do
     t.string "fetch_external_data_exceptions", array: true
     t.jsonb "value_json"
     t.index ["dossier_id"], name: "index_champs_on_dossier_id"
+    t.index ["etablissement_id"], name: "index_champs_on_etablissement_id"
     t.index ["parent_id"], name: "index_champs_on_parent_id"
     t.index ["private"], name: "index_champs_on_private"
     t.index ["row"], name: "index_champs_on_row"

From 1ca8192864dd979a98d6c472adc97d8d32fa49b2 Mon Sep 17 00:00:00 2001
From: Paul Chavard <github@paul.chavard.net>
Date: Wed, 20 Oct 2021 16:52:38 +0200
Subject: [PATCH 2/2] perf(export): load dossiers.champs in batches

---
 app/models/dossier.rb                    | 55 ++++++++++++++++--------
 app/services/procedure_export_service.rb |  2 +-
 2 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/app/models/dossier.rb b/app/models/dossier.rb
index 0e8aee571..6a35c42d9 100644
--- a/app/models/dossier.rb
+++ b/app/models/dossier.rb
@@ -230,25 +230,7 @@ class Dossier < ApplicationRecord
           :published_types_de_champ_private
         ],
         avis: [:claimant, :expert],
-        etablissement: :champ,
-        champs: {
-          type_de_champ: [],
-          etablissement: :champ,
-          piece_justificative_file_attachment: :blob,
-          champs: [
-            type_de_champ: [],
-            piece_justificative_file_attachment: :blob
-          ]
-        },
-        champs_private: {
-          type_de_champ: [],
-          etablissement: :champ,
-          piece_justificative_file_attachment: :blob,
-          champs: [
-            type_de_champ: [],
-            piece_justificative_file_attachment: :blob
-          ]
-        }
+        etablissement: :champ
       ).order(en_construction_at: 'asc')
   }
   scope :en_cours,                    -> { not_archived.state_en_construction_ou_instruction }
@@ -387,6 +369,41 @@ class Dossier < ApplicationRecord
   validates :individual, presence: true, if: -> { revision.procedure.for_individual? }
   validates :groupe_instructeur, presence: true, if: -> { !brouillon? }
 
+  EXPORT_BATCH_SIZE = 5000
+
+  def self.downloadable_sorted_batch
+    dossiers = downloadable_sorted.to_a
+    (dossiers.size.to_f / EXPORT_BATCH_SIZE).ceil.times do |i|
+      start_index = i * EXPORT_BATCH_SIZE
+      end_index = start_index + EXPORT_BATCH_SIZE - 1
+      load_champs(dossiers[start_index..end_index])
+    end
+    dossiers
+  end
+
+  def self.load_champs(dossiers)
+    ::ActiveRecord::Associations::Preloader.new.preload(dossiers, {
+      champs: {
+        type_de_champ: [],
+        etablissement: :champ,
+        piece_justificative_file_attachment: :blob,
+        champs: [
+          type_de_champ: [],
+          piece_justificative_file_attachment: :blob
+        ]
+      },
+      champs_private: {
+        type_de_champ: [],
+        etablissement: :champ,
+        piece_justificative_file_attachment: :blob,
+        champs: [
+          type_de_champ: [],
+          piece_justificative_file_attachment: :blob
+        ]
+      }
+    })
+  end
+
   def user_deleted?
     persisted? && user_id.nil?
   end
diff --git a/app/services/procedure_export_service.rb b/app/services/procedure_export_service.rb
index d6c8ed69d..2de0cf91c 100644
--- a/app/services/procedure_export_service.rb
+++ b/app/services/procedure_export_service.rb
@@ -3,7 +3,7 @@ class ProcedureExportService
 
   def initialize(procedure, dossiers)
     @procedure = procedure
-    @dossiers = dossiers.downloadable_sorted
+    @dossiers = dossiers.downloadable_sorted_batch
     @tables = [:dossiers, :etablissements, :avis] + champs_repetables_options
   end