From 1eebc2f1e1f20303ed825f6b11c3fa9fef849a0e Mon Sep 17 00:00:00 2001
From: Pierre de La Morinerie <kemenaran@gmail.com>
Date: Tue, 15 Mar 2022 16:08:13 +0100
Subject: [PATCH 1/4] specs: make test migrations safer

This will avoid strong_migrations to flag them as dangerous.
---
 spec/lib/database/migration_helpers_spec.rb | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/spec/lib/database/migration_helpers_spec.rb b/spec/lib/database/migration_helpers_spec.rb
index 1ca896cfb..033b4b41a 100644
--- a/spec/lib/database/migration_helpers_spec.rb
+++ b/spec/lib/database/migration_helpers_spec.rb
@@ -5,11 +5,7 @@ describe Database::MigrationHelpers do
 
     before(:all) do
       ActiveRecord::Migration.suppress_messages do
-        ActiveRecord::Migration.create_table "test_labels", force: true do |t|
-          t.string :label
-          t.integer :user_id
-        end
-        ActiveRecord::Migration.create_table "test_labels", force: true do |t|
+        ActiveRecord::Migration.create_table "test_labels" do |t|
           t.string :label
           t.integer :user_id
         end
@@ -103,13 +99,13 @@ describe Database::MigrationHelpers do
 
     before(:all) do
       ActiveRecord::Migration.suppress_messages do
-        ActiveRecord::Migration.create_table "test_physicians", force: true do |t|
+        ActiveRecord::Migration.create_table "test_physicians" do |t|
           t.string :name
         end
-        ActiveRecord::Migration.create_table "test_patients", force: true do |t|
+        ActiveRecord::Migration.create_table "test_patients" do |t|
           t.string :name
         end
-        ActiveRecord::Migration.create_table "test_appointments", id: false, force: true do |t|
+        ActiveRecord::Migration.create_table "test_appointments", id: false do |t|
           t.integer  :test_physician_id
           t.integer  :test_patient_id
           t.datetime :datetime

From 2e04435117783af60e00611f07b28697ac8396d2 Mon Sep 17 00:00:00 2001
From: Pierre de La Morinerie <kemenaran@gmail.com>
Date: Tue, 15 Mar 2022 13:59:59 +0100
Subject: [PATCH 2/4] gems: add strong_migrations

---
 Gemfile                                  |  1 +
 Gemfile.lock                             |  3 +++
 config/initializers/strong_migrations.rb | 26 ++++++++++++++++++++++++
 3 files changed, 30 insertions(+)
 create mode 100644 config/initializers/strong_migrations.rb

diff --git a/Gemfile b/Gemfile
index 0a5ce4a84..e210be466 100644
--- a/Gemfile
+++ b/Gemfile
@@ -80,6 +80,7 @@ gem 'sentry-ruby'
 gem 'sib-api-v3-sdk'
 gem 'skylight'
 gem 'spreadsheet_architect'
+gem 'strong_migrations' # lint database migrations
 gem 'typhoeus'
 gem 'warden'
 gem 'webpacker'
diff --git a/Gemfile.lock b/Gemfile.lock
index 1a9ba160c..f2fbfeb94 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -691,6 +691,8 @@ GEM
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
+    strong_migrations (0.8.0)
+      activerecord (>= 5.2)
     swd (1.3.0)
       activesupport (>= 3)
       attr_required (>= 0.0.5)
@@ -875,6 +877,7 @@ DEPENDENCIES
   spreadsheet_architect
   spring
   spring-commands-rspec
+  strong_migrations
   timecop
   typhoeus
   vcr
diff --git a/config/initializers/strong_migrations.rb b/config/initializers/strong_migrations.rb
new file mode 100644
index 000000000..132263236
--- /dev/null
+++ b/config/initializers/strong_migrations.rb
@@ -0,0 +1,26 @@
+# Mark existing migrations as safe
+StrongMigrations.start_after = 20220315125851
+
+# Set timeouts for migrations
+# If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user
+StrongMigrations.lock_timeout = 10.seconds
+StrongMigrations.statement_timeout = 1.hour
+
+# Analyze tables after indexes are added
+# Outdated statistics can sometimes hurt performance
+StrongMigrations.auto_analyze = true
+
+# Set the version of the production database
+# so the right checks are run in development
+# StrongMigrations.target_version = 10
+
+# Add custom checks
+# StrongMigrations.add_check do |method, args|
+#   if method == :add_index && args[0].to_s == "users"
+#     stop! "No more indexes on the users table"
+#   end
+# end
+
+# Make some operations safe by default
+# See https://github.com/ankane/strong_migrations#safe-by-default
+# StrongMigrations.safe_by_default = true

From 21bf4d38cf2fb0703fec53fbe5b51e10121f2f01 Mon Sep 17 00:00:00 2001
From: Pierre de La Morinerie <kemenaran@gmail.com>
Date: Tue, 15 Mar 2022 14:24:20 +0100
Subject: [PATCH 3/4] db: alphabetize schema.rb

This avoids the column order being de-synchronized between different
machines.
---
 Rakefile     |   3 +
 db/schema.rb | 655 +++++++++++++++++++++++++--------------------------
 2 files changed, 329 insertions(+), 329 deletions(-)

diff --git a/Rakefile b/Rakefile
index f7a26ddaf..9e233babc 100644
--- a/Rakefile
+++ b/Rakefile
@@ -4,3 +4,6 @@
 require File.expand_path('config/application', __dir__)
 
 Rails.application.load_tasks
+
+# Alphabetize schema.rb
+task 'db:schema:dump': 'strong_migrations:alphabetize_columns'
diff --git a/db/schema.rb b/db/schema.rb
index 870305058..c5dac3a04 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -17,35 +17,35 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
   enable_extension "unaccent"
 
   create_table "action_text_rich_texts", force: :cascade do |t|
-    t.string "name", null: false
     t.text "body"
-    t.string "record_type", null: false
-    t.bigint "record_id", null: false
     t.datetime "created_at", null: false
+    t.string "name", null: false
+    t.bigint "record_id", null: false
+    t.string "record_type", null: false
     t.datetime "updated_at", null: false
     t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true
   end
 
   create_table "active_storage_attachments", force: :cascade do |t|
-    t.string "name", null: false
-    t.string "record_type", null: false
-    t.bigint "record_id", null: false
     t.bigint "blob_id", null: false
     t.datetime "created_at", null: false
+    t.string "name", null: false
+    t.bigint "record_id", null: false
+    t.string "record_type", null: false
     t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id"
     t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true
   end
 
   create_table "active_storage_blobs", force: :cascade do |t|
-    t.string "key", null: false
-    t.string "filename", null: false
-    t.string "content_type"
-    t.text "metadata"
     t.bigint "byte_size", null: false
     t.string "checksum", null: false
+    t.string "content_type"
     t.datetime "created_at", null: false
-    t.string "service_name", null: false
+    t.string "filename", null: false
+    t.string "key", null: false
     t.integer "lock_version"
+    t.text "metadata"
+    t.string "service_name", null: false
     t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true
   end
 
@@ -56,17 +56,17 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
   end
 
   create_table "administrateurs", id: :serial, force: :cascade do |t|
-    t.datetime "created_at"
-    t.datetime "updated_at"
     t.boolean "active", default: false
+    t.datetime "created_at"
     t.string "encrypted_token"
+    t.datetime "updated_at"
     t.bigint "user_id"
   end
 
   create_table "administrateurs_instructeurs", id: false, force: :cascade do |t|
     t.integer "administrateur_id"
-    t.integer "instructeur_id"
     t.datetime "created_at"
+    t.integer "instructeur_id"
     t.datetime "updated_at"
     t.index ["administrateur_id"], name: "index_administrateurs_instructeurs_on_administrateur_id"
     t.index ["instructeur_id", "administrateur_id"], name: "unique_couple_administrateur_instructeur", unique: true
@@ -75,97 +75,97 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
 
   create_table "administrateurs_procedures", id: false, force: :cascade do |t|
     t.bigint "administrateur_id", null: false
-    t.bigint "procedure_id", null: false
     t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
     t.boolean "manager"
+    t.bigint "procedure_id", null: false
+    t.datetime "updated_at", null: false
     t.index ["administrateur_id", "procedure_id"], name: "index_unique_admin_proc_couple", unique: true
     t.index ["administrateur_id"], name: "index_administrateurs_procedures_on_administrateur_id"
     t.index ["procedure_id"], name: "index_administrateurs_procedures_on_procedure_id"
   end
 
   create_table "archives", force: :cascade do |t|
-    t.string "status", null: false
-    t.date "month"
-    t.string "time_span_type", null: false
     t.datetime "created_at", precision: 6, null: false
-    t.datetime "updated_at", precision: 6, null: false
     t.text "key", null: false
+    t.date "month"
+    t.string "status", null: false
+    t.string "time_span_type", null: false
+    t.datetime "updated_at", precision: 6, null: false
     t.index ["key", "time_span_type", "month"], name: "index_archives_on_key_and_time_span_type_and_month", unique: true
   end
 
   create_table "archives_groupe_instructeurs", force: :cascade do |t|
     t.bigint "archive_id", null: false
-    t.bigint "groupe_instructeur_id", null: false
     t.datetime "created_at", precision: 6, null: false
+    t.bigint "groupe_instructeur_id", null: false
     t.datetime "updated_at", precision: 6, null: false
     t.index ["archive_id"], name: "index_archives_groupe_instructeurs_on_archive_id"
     t.index ["groupe_instructeur_id"], name: "index_archives_groupe_instructeurs_on_groupe_instructeur_id"
   end
 
   create_table "assign_tos", id: :serial, force: :cascade do |t|
-    t.integer "instructeur_id"
     t.datetime "created_at"
-    t.datetime "updated_at"
-    t.bigint "groupe_instructeur_id"
-    t.boolean "weekly_email_notifications_enabled", default: true, null: false
     t.boolean "daily_email_notifications_enabled", default: false, null: false
-    t.boolean "instant_email_message_notifications_enabled", default: false, null: false
+    t.bigint "groupe_instructeur_id"
     t.boolean "instant_email_dossier_notifications_enabled", default: false, null: false
+    t.boolean "instant_email_message_notifications_enabled", default: false, null: false
+    t.integer "instructeur_id"
+    t.datetime "updated_at"
+    t.boolean "weekly_email_notifications_enabled", default: true, null: false
     t.index ["groupe_instructeur_id", "instructeur_id"], name: "unique_couple_groupe_instructeur_instructeur", unique: true
     t.index ["groupe_instructeur_id"], name: "index_assign_tos_on_groupe_instructeur_id"
     t.index ["instructeur_id"], name: "index_assign_tos_on_instructeur_id"
   end
 
   create_table "attestation_templates", id: :serial, force: :cascade do |t|
-    t.text "title"
-    t.text "body"
-    t.text "footer"
     t.boolean "activated"
+    t.text "body"
     t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
+    t.text "footer"
     t.integer "procedure_id"
+    t.text "title"
+    t.datetime "updated_at", null: false
     t.index ["procedure_id"], name: "index_attestation_templates_on_procedure_id", unique: true
   end
 
   create_table "attestations", id: :serial, force: :cascade do |t|
-    t.string "title"
-    t.integer "dossier_id", null: false
     t.datetime "created_at", null: false
+    t.integer "dossier_id", null: false
+    t.string "title"
     t.datetime "updated_at", null: false
     t.index ["dossier_id"], name: "index_attestations_on_dossier_id"
   end
 
   create_table "avis", id: :serial, force: :cascade do |t|
-    t.string "email"
-    t.text "introduction"
     t.text "answer"
-    t.integer "dossier_id"
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
     t.integer "claimant_id", null: false
-    t.boolean "confidentiel", default: false, null: false
-    t.datetime "revoked_at"
-    t.bigint "experts_procedure_id"
     t.string "claimant_type"
+    t.boolean "confidentiel", default: false, null: false
+    t.datetime "created_at", null: false
+    t.integer "dossier_id"
+    t.string "email"
+    t.bigint "experts_procedure_id"
+    t.text "introduction"
+    t.datetime "revoked_at"
+    t.datetime "updated_at", null: false
     t.index ["claimant_id"], name: "index_avis_on_claimant_id"
     t.index ["dossier_id"], name: "index_avis_on_dossier_id"
     t.index ["experts_procedure_id"], name: "index_avis_on_experts_procedure_id"
   end
 
   create_table "bill_signatures", force: :cascade do |t|
-    t.string "digest"
     t.datetime "created_at", null: false
+    t.string "digest"
     t.datetime "updated_at", null: false
   end
 
   create_table "bulk_messages", force: :cascade do |t|
     t.text "body", null: false
+    t.datetime "created_at", precision: 6, null: false
     t.integer "dossier_count"
     t.string "dossier_state"
-    t.datetime "sent_at", null: false
     t.bigint "instructeur_id", null: false
-    t.datetime "created_at", precision: 6, null: false
+    t.datetime "sent_at", null: false
     t.datetime "updated_at", precision: 6, null: false
   end
 
@@ -178,21 +178,21 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
   end
 
   create_table "champs", id: :serial, force: :cascade do |t|
-    t.string "value"
-    t.integer "type_de_champ_id"
-    t.integer "dossier_id"
-    t.string "type"
     t.datetime "created_at"
-    t.datetime "updated_at"
-    t.boolean "private", default: false, null: false
-    t.integer "etablissement_id"
-    t.bigint "parent_id"
-    t.integer "row"
     t.jsonb "data"
+    t.integer "dossier_id"
+    t.integer "etablissement_id"
     t.string "external_id"
     t.string "fetch_external_data_exceptions", array: true
-    t.jsonb "value_json"
+    t.bigint "parent_id"
+    t.boolean "private", default: false, null: false
     t.datetime "rebased_at"
+    t.integer "row"
+    t.string "type"
+    t.integer "type_de_champ_id"
+    t.datetime "updated_at"
+    t.string "value"
+    t.jsonb "value_json"
     t.index ["dossier_id"], name: "index_champs_on_dossier_id"
     t.index ["etablissement_id"], name: "index_champs_on_etablissement_id"
     t.index ["parent_id"], name: "index_champs_on_parent_id"
@@ -204,23 +204,23 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
 
   create_table "closed_mails", id: :serial, force: :cascade do |t|
     t.text "body"
-    t.string "subject"
-    t.integer "procedure_id"
     t.datetime "created_at", null: false
+    t.integer "procedure_id"
+    t.string "subject"
     t.datetime "updated_at", null: false
     t.index ["procedure_id"], name: "index_closed_mails_on_procedure_id"
   end
 
   create_table "commentaires", id: :serial, force: :cascade do |t|
-    t.string "email"
-    t.datetime "created_at", null: false
     t.string "body"
+    t.datetime "created_at", null: false
+    t.datetime "discarded_at"
     t.integer "dossier_id"
+    t.string "email"
+    t.bigint "expert_id"
+    t.bigint "instructeur_id"
     t.datetime "updated_at", null: false
     t.bigint "user_id"
-    t.bigint "instructeur_id"
-    t.bigint "expert_id"
-    t.datetime "discarded_at"
     t.index ["dossier_id"], name: "index_commentaires_on_dossier_id"
     t.index ["expert_id"], name: "index_commentaires_on_expert_id"
     t.index ["instructeur_id"], name: "index_commentaires_on_instructeur_id"
@@ -228,48 +228,48 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
   end
 
   create_table "delayed_jobs", id: :serial, force: :cascade do |t|
-    t.integer "priority", default: 0, null: false
     t.integer "attempts", default: 0, null: false
+    t.datetime "created_at"
+    t.string "cron"
+    t.datetime "failed_at"
     t.text "handler", null: false
     t.text "last_error"
-    t.datetime "run_at"
     t.datetime "locked_at"
-    t.datetime "failed_at"
     t.string "locked_by"
+    t.integer "priority", default: 0, null: false
     t.string "queue"
-    t.datetime "created_at"
+    t.datetime "run_at"
     t.datetime "updated_at"
-    t.string "cron"
     t.index ["priority", "run_at"], name: "delayed_jobs_priority"
   end
 
   create_table "deleted_dossiers", force: :cascade do |t|
-    t.bigint "procedure_id"
-    t.bigint "dossier_id"
-    t.datetime "deleted_at"
-    t.string "state"
     t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.string "reason"
-    t.bigint "user_id"
+    t.datetime "deleted_at"
+    t.bigint "dossier_id"
     t.bigint "groupe_instructeur_id"
+    t.bigint "procedure_id"
+    t.string "reason"
     t.bigint "revision_id"
+    t.string "state"
+    t.datetime "updated_at", null: false
+    t.bigint "user_id"
     t.index ["deleted_at"], name: "index_deleted_dossiers_on_deleted_at"
     t.index ["dossier_id"], name: "index_deleted_dossiers_on_dossier_id", unique: true
     t.index ["procedure_id"], name: "index_deleted_dossiers_on_procedure_id"
   end
 
   create_table "dossier_operation_logs", force: :cascade do |t|
-    t.string "operation", null: false
-    t.bigint "dossier_id"
-    t.bigint "instructeur_id"
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
     t.boolean "automatic_operation", default: false, null: false
-    t.datetime "keep_until"
-    t.datetime "executed_at"
-    t.text "digest"
     t.bigint "bill_signature_id"
+    t.datetime "created_at", null: false
+    t.text "digest"
+    t.bigint "dossier_id"
+    t.datetime "executed_at"
+    t.bigint "instructeur_id"
+    t.datetime "keep_until"
+    t.string "operation", null: false
+    t.datetime "updated_at", null: false
     t.index ["bill_signature_id"], name: "index_dossier_operation_logs_on_bill_signature_id"
     t.index ["dossier_id"], name: "index_dossier_operation_logs_on_dossier_id"
     t.index ["instructeur_id"], name: "index_dossier_operation_logs_on_instructeur_id"
@@ -277,64 +277,62 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
   end
 
   create_table "dossier_submitted_messages", force: :cascade do |t|
-    t.string "message_on_submit_by_usager"
     t.datetime "created_at", precision: 6, null: false
+    t.string "message_on_submit_by_usager"
     t.datetime "updated_at", precision: 6, null: false
   end
 
   create_table "dossier_transfer_logs", force: :cascade do |t|
+    t.datetime "created_at", precision: 6, null: false
+    t.bigint "dossier_id", null: false
     t.string "from", null: false
     t.string "to", null: false
-    t.bigint "dossier_id", null: false
-    t.datetime "created_at", precision: 6, null: false
     t.datetime "updated_at", precision: 6, null: false
     t.index ["dossier_id"], name: "index_dossier_transfer_logs_on_dossier_id"
   end
 
   create_table "dossier_transfers", force: :cascade do |t|
-    t.string "email", null: false
     t.datetime "created_at", precision: 6, null: false
+    t.string "email", null: false
     t.datetime "updated_at", precision: 6, null: false
     t.index ["email"], name: "index_dossier_transfers_on_email"
   end
 
   create_table "dossiers", id: :serial, force: :cascade do |t|
-    t.boolean "autorisation_donnees"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.string "state"
-    t.integer "user_id"
+    t.string "api_entreprise_job_exceptions", array: true
     t.boolean "archived", default: false
-    t.datetime "en_construction_at"
-    t.datetime "en_instruction_at"
-    t.datetime "processed_at"
-    t.text "motivation"
-    t.datetime "hidden_at"
-    t.text "search_terms"
-    t.text "private_search_terms"
-    t.bigint "groupe_instructeur_id"
+    t.boolean "autorisation_donnees"
     t.datetime "brouillon_close_to_expiration_notice_sent_at"
-    t.datetime "groupe_instructeur_updated_at"
+    t.interval "conservation_extension", default: "PT0S"
+    t.datetime "created_at"
+    t.datetime "declarative_triggered_at"
+    t.string "deleted_user_email_never_send"
+    t.datetime "depose_at"
+    t.bigint "dossier_transfer_id"
+    t.datetime "en_construction_at"
     t.datetime "en_construction_close_to_expiration_notice_sent_at"
     t.interval "en_construction_conservation_extension", default: "PT0S"
-    t.datetime "termine_close_to_expiration_notice_sent_at"
-    t.bigint "revision_id"
-    t.datetime "last_champ_updated_at"
-    t.datetime "last_champ_private_updated_at"
-    t.datetime "last_avis_updated_at"
-    t.datetime "last_commentaire_updated_at"
-    t.string "api_entreprise_job_exceptions", array: true
-    t.interval "conservation_extension", default: "PT0S"
-    t.string "deleted_user_email_never_send"
-    t.datetime "declarative_triggered_at"
-    t.bigint "dossier_transfer_id"
-    t.datetime "identity_updated_at"
-    t.datetime "depose_at"
-    t.datetime "hidden_by_user_at"
+    t.datetime "en_instruction_at"
+    t.bigint "groupe_instructeur_id"
+    t.datetime "groupe_instructeur_updated_at"
+    t.datetime "hidden_at"
     t.datetime "hidden_by_administration_at"
     t.string "hidden_by_reason"
-    t.index "to_tsvector('french'::regconfig, (search_terms || private_search_terms))", name: "index_dossiers_on_search_terms_private_search_terms", using: :gin
-    t.index "to_tsvector('french'::regconfig, search_terms)", name: "index_dossiers_on_search_terms", using: :gin
+    t.datetime "hidden_by_user_at"
+    t.datetime "identity_updated_at"
+    t.datetime "last_avis_updated_at"
+    t.datetime "last_champ_private_updated_at"
+    t.datetime "last_champ_updated_at"
+    t.datetime "last_commentaire_updated_at"
+    t.text "motivation"
+    t.text "private_search_terms"
+    t.datetime "processed_at"
+    t.bigint "revision_id"
+    t.text "search_terms"
+    t.string "state"
+    t.datetime "termine_close_to_expiration_notice_sent_at"
+    t.datetime "updated_at"
+    t.integer "user_id"
     t.index ["archived"], name: "index_dossiers_on_archived"
     t.index ["dossier_transfer_id"], name: "index_dossiers_on_dossier_transfer_id"
     t.index ["groupe_instructeur_id"], name: "index_dossiers_on_groupe_instructeur_id"
@@ -345,66 +343,66 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
   end
 
   create_table "drop_down_lists", id: :serial, force: :cascade do |t|
-    t.string "value"
-    t.integer "type_de_champ_id"
     t.datetime "created_at"
+    t.integer "type_de_champ_id"
     t.datetime "updated_at"
+    t.string "value"
     t.index ["type_de_champ_id"], name: "index_drop_down_lists_on_type_de_champ_id"
   end
 
   create_table "etablissements", id: :serial, force: :cascade do |t|
-    t.string "siret"
-    t.boolean "siege_social"
-    t.string "naf"
-    t.string "libelle_naf"
     t.string "adresse"
-    t.string "numero_voie"
-    t.string "type_voie"
-    t.string "nom_voie"
-    t.string "complement_adresse"
-    t.string "code_postal"
-    t.string "localite"
-    t.string "code_insee_localite"
-    t.integer "dossier_id"
-    t.string "entreprise_siren"
-    t.bigint "entreprise_capital_social"
-    t.string "entreprise_numero_tva_intracommunautaire"
-    t.string "entreprise_forme_juridique"
-    t.string "entreprise_forme_juridique_code"
-    t.string "entreprise_nom_commercial"
-    t.string "entreprise_raison_sociale"
-    t.string "entreprise_siret_siege_social"
-    t.string "entreprise_code_effectif_entreprise"
-    t.date "entreprise_date_creation"
-    t.string "entreprise_nom"
-    t.string "entreprise_prenom"
-    t.string "association_rna"
-    t.string "association_titre"
-    t.text "association_objet"
     t.date "association_date_creation"
     t.date "association_date_declaration"
     t.date "association_date_publication"
+    t.text "association_objet"
+    t.string "association_rna"
+    t.string "association_titre"
+    t.string "code_insee_localite"
+    t.string "code_postal"
+    t.string "complement_adresse"
     t.datetime "created_at"
-    t.datetime "updated_at"
     t.boolean "diffusable_commercialement"
-    t.string "entreprise_effectif_mois"
-    t.string "entreprise_effectif_annee"
-    t.decimal "entreprise_effectif_mensuel"
-    t.decimal "entreprise_effectif_annuel"
-    t.string "entreprise_effectif_annuel_annee"
+    t.integer "dossier_id"
+    t.string "enseigne"
     t.jsonb "entreprise_bilans_bdf"
     t.string "entreprise_bilans_bdf_monnaie"
-    t.string "enseigne"
+    t.bigint "entreprise_capital_social"
+    t.string "entreprise_code_effectif_entreprise"
+    t.date "entreprise_date_creation"
+    t.string "entreprise_effectif_annee"
+    t.decimal "entreprise_effectif_annuel"
+    t.string "entreprise_effectif_annuel_annee"
+    t.decimal "entreprise_effectif_mensuel"
+    t.string "entreprise_effectif_mois"
+    t.string "entreprise_forme_juridique"
+    t.string "entreprise_forme_juridique_code"
+    t.string "entreprise_nom"
+    t.string "entreprise_nom_commercial"
+    t.string "entreprise_numero_tva_intracommunautaire"
+    t.string "entreprise_prenom"
+    t.string "entreprise_raison_sociale"
+    t.string "entreprise_siren"
+    t.string "entreprise_siret_siege_social"
+    t.string "libelle_naf"
+    t.string "localite"
+    t.string "naf"
+    t.string "nom_voie"
+    t.string "numero_voie"
+    t.boolean "siege_social"
+    t.string "siret"
+    t.string "type_voie"
+    t.datetime "updated_at"
     t.index ["dossier_id"], name: "index_etablissements_on_dossier_id", unique: true
   end
 
   create_table "exercices", id: :serial, force: :cascade do |t|
     t.string "ca"
+    t.datetime "created_at"
     t.datetime "dateFinExercice"
+    t.datetime "date_fin_exercice"
     t.integer "date_fin_exercice_timestamp"
     t.integer "etablissement_id"
-    t.datetime "date_fin_exercice"
-    t.datetime "created_at"
     t.datetime "updated_at"
     t.index ["etablissement_id"], name: "index_exercices_on_etablissement_id"
   end
@@ -416,59 +414,59 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
   end
 
   create_table "experts_procedures", force: :cascade do |t|
-    t.bigint "expert_id", null: false
-    t.bigint "procedure_id", null: false
     t.boolean "allow_decision_access", default: false, null: false
     t.datetime "created_at", precision: 6, null: false
-    t.datetime "updated_at", precision: 6, null: false
+    t.bigint "expert_id", null: false
+    t.bigint "procedure_id", null: false
     t.datetime "revoked_at"
+    t.datetime "updated_at", precision: 6, null: false
     t.index ["expert_id", "procedure_id"], name: "index_experts_procedures_on_expert_id_and_procedure_id", unique: true
     t.index ["expert_id"], name: "index_experts_procedures_on_expert_id"
     t.index ["procedure_id"], name: "index_experts_procedures_on_procedure_id"
   end
 
   create_table "exports", force: :cascade do |t|
-    t.string "format", null: false
     t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
+    t.string "format", null: false
     t.text "key", null: false
     t.string "time_span_type", default: "everything", null: false
+    t.datetime "updated_at", null: false
     t.index ["format", "time_span_type", "key"], name: "index_exports_on_format_and_time_span_type_and_key", unique: true
   end
 
   create_table "exports_groupe_instructeurs", force: :cascade do |t|
+    t.datetime "created_at", null: false
     t.bigint "export_id", null: false
     t.bigint "groupe_instructeur_id", null: false
-    t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
   end
 
-  create_table "flipper_features", force: :cascade do |t|
-    t.string "key", null: false
+  create_table "flipper_features", id: false, force: :cascade do |t|
     t.datetime "created_at", null: false
+    t.bigserial "id", null: false
+    t.string "key", null: false
     t.datetime "updated_at", null: false
-    t.index ["key"], name: "index_flipper_features_on_key", unique: true
   end
 
   create_table "flipper_gates", force: :cascade do |t|
+    t.datetime "created_at", null: false
     t.string "feature_key", null: false
     t.string "key", null: false
-    t.string "value"
-    t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.string "value"
     t.index ["feature_key", "key", "value"], name: "index_flipper_gates_on_feature_key_and_key_and_value", unique: true
   end
 
   create_table "follows", id: :serial, force: :cascade do |t|
-    t.integer "instructeur_id", null: false
-    t.integer "dossier_id", null: false
-    t.datetime "demande_seen_at", null: false
     t.datetime "annotations_privees_seen_at", null: false
     t.datetime "avis_seen_at", null: false
-    t.datetime "messagerie_seen_at", null: false
     t.datetime "created_at"
-    t.datetime "updated_at"
+    t.datetime "demande_seen_at", null: false
+    t.integer "dossier_id", null: false
+    t.integer "instructeur_id", null: false
+    t.datetime "messagerie_seen_at", null: false
     t.datetime "unfollowed_at"
+    t.datetime "updated_at"
     t.index ["dossier_id"], name: "index_follows_on_dossier_id"
     t.index ["instructeur_id", "dossier_id", "unfollowed_at"], name: "uniqueness_index", unique: true
     t.index ["instructeur_id"], name: "index_follows_on_instructeur_id"
@@ -476,187 +474,187 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
   end
 
   create_table "france_connect_informations", id: :serial, force: :cascade do |t|
-    t.string "gender"
-    t.string "given_name"
-    t.string "family_name"
     t.date "birthdate"
     t.string "birthplace"
-    t.string "france_connect_particulier_id"
-    t.integer "user_id"
-    t.string "email_france_connect"
     t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
     t.jsonb "data"
+    t.string "email_france_connect"
+    t.string "family_name"
+    t.string "france_connect_particulier_id"
+    t.string "gender"
+    t.string "given_name"
     t.string "merge_token"
     t.datetime "merge_token_created_at"
+    t.datetime "updated_at", null: false
+    t.integer "user_id"
     t.index ["merge_token"], name: "index_france_connect_informations_on_merge_token"
     t.index ["user_id"], name: "index_france_connect_informations_on_user_id"
   end
 
   create_table "geo_areas", force: :cascade do |t|
-    t.string "source"
+    t.bigint "champ_id"
+    t.datetime "created_at"
+    t.string "geo_reference_id"
     t.jsonb "geometry"
     t.jsonb "properties"
-    t.bigint "champ_id"
-    t.string "geo_reference_id"
-    t.datetime "created_at"
+    t.string "source"
     t.datetime "updated_at"
     t.index ["champ_id"], name: "index_geo_areas_on_champ_id"
     t.index ["source"], name: "index_geo_areas_on_source"
   end
 
   create_table "groupe_instructeurs", force: :cascade do |t|
-    t.bigint "procedure_id", null: false
-    t.text "label", null: false
     t.datetime "created_at", null: false
+    t.text "label", null: false
+    t.bigint "procedure_id", null: false
     t.datetime "updated_at", null: false
     t.index ["procedure_id", "label"], name: "index_groupe_instructeurs_on_procedure_id_and_label", unique: true
     t.index ["procedure_id"], name: "index_groupe_instructeurs_on_procedure_id"
   end
 
   create_table "individuals", id: :serial, force: :cascade do |t|
-    t.string "nom"
-    t.string "prenom"
+    t.date "birthdate"
+    t.datetime "created_at"
     t.integer "dossier_id"
     t.string "gender"
-    t.datetime "created_at"
+    t.string "nom"
+    t.string "prenom"
     t.datetime "updated_at"
-    t.date "birthdate"
     t.index ["dossier_id"], name: "index_individuals_on_dossier_id", unique: true
   end
 
   create_table "initiated_mails", id: :serial, force: :cascade do |t|
-    t.string "subject"
     t.text "body"
-    t.integer "procedure_id"
     t.datetime "created_at", null: false
+    t.integer "procedure_id"
+    t.string "subject"
     t.datetime "updated_at", null: false
     t.index ["procedure_id"], name: "index_initiated_mails_on_procedure_id"
   end
 
   create_table "instructeurs", id: :serial, force: :cascade do |t|
+    t.string "agent_connect_id"
+    t.boolean "bypass_email_login_token", default: false, null: false
     t.datetime "created_at"
-    t.datetime "updated_at"
     t.text "encrypted_login_token"
     t.datetime "login_token_created_at"
-    t.boolean "bypass_email_login_token", default: false, null: false
-    t.string "agent_connect_id"
+    t.datetime "updated_at"
     t.bigint "user_id"
     t.index ["agent_connect_id"], name: "index_instructeurs_on_agent_connect_id", unique: true
   end
 
   create_table "invites", id: :serial, force: :cascade do |t|
+    t.datetime "created_at"
+    t.integer "dossier_id"
     t.string "email"
     t.string "email_sender"
-    t.integer "dossier_id"
-    t.integer "user_id"
-    t.datetime "created_at"
-    t.datetime "updated_at"
     t.text "message"
+    t.datetime "updated_at"
+    t.integer "user_id"
     t.index ["email", "dossier_id"], name: "index_invites_on_email_and_dossier_id", unique: true
   end
 
   create_table "merge_logs", force: :cascade do |t|
-    t.bigint "from_user_id", null: false
-    t.string "from_user_email", null: false
-    t.bigint "user_id", null: false
     t.datetime "created_at", precision: 6, null: false
+    t.string "from_user_email", null: false
+    t.bigint "from_user_id", null: false
     t.datetime "updated_at", precision: 6, null: false
+    t.bigint "user_id", null: false
     t.index ["user_id"], name: "index_merge_logs_on_user_id"
   end
 
   create_table "module_api_cartos", id: :serial, force: :cascade do |t|
-    t.integer "procedure_id"
-    t.boolean "use_api_carto", default: false
-    t.boolean "quartiers_prioritaires", default: false
     t.boolean "cadastre", default: false
     t.datetime "created_at"
-    t.datetime "updated_at"
     t.boolean "migrated"
+    t.integer "procedure_id"
+    t.boolean "quartiers_prioritaires", default: false
+    t.datetime "updated_at"
+    t.boolean "use_api_carto", default: false
     t.index ["procedure_id"], name: "index_module_api_cartos_on_procedure_id", unique: true
   end
 
   create_table "procedure_presentations", id: :serial, force: :cascade do |t|
     t.integer "assign_to_id"
-    t.jsonb "sort", default: {"order"=>"desc", "table"=>"notifications", "column"=>"notifications"}, null: false
-    t.jsonb "filters", default: {"tous"=>[], "suivis"=>[], "traites"=>[], "a-suivre"=>[], "archives"=>[], "expirant"=>[], "supprimes_recemment"=>[]}, null: false
     t.datetime "created_at"
-    t.datetime "updated_at"
     t.jsonb "displayed_fields", default: [{"label"=>"Demandeur", "table"=>"user", "column"=>"email"}], null: false
+    t.jsonb "filters", default: {"tous"=>[], "suivis"=>[], "traites"=>[], "a-suivre"=>[], "archives"=>[], "expirant"=>[], "supprimes_recemment"=>[]}, null: false
+    t.jsonb "sort", default: {"order"=>"desc", "table"=>"notifications", "column"=>"notifications"}, null: false
+    t.datetime "updated_at"
     t.index ["assign_to_id"], name: "index_procedure_presentations_on_assign_to_id", unique: true
   end
 
   create_table "procedure_revision_types_de_champ", force: :cascade do |t|
+    t.datetime "created_at", null: false
+    t.bigint "parent_id"
+    t.integer "position", null: false
     t.bigint "revision_id", null: false
     t.bigint "type_de_champ_id", null: false
-    t.integer "position", null: false
-    t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.bigint "parent_id"
     t.index ["parent_id"], name: "index_procedure_revision_types_de_champ_on_parent_id"
     t.index ["revision_id"], name: "index_procedure_revision_types_de_champ_on_revision_id"
     t.index ["type_de_champ_id"], name: "index_procedure_revision_types_de_champ_on_type_de_champ_id"
   end
 
   create_table "procedure_revisions", force: :cascade do |t|
-    t.bigint "procedure_id", null: false
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.datetime "published_at"
     t.bigint "attestation_template_id"
+    t.datetime "created_at", null: false
     t.bigint "dossier_submitted_message_id"
+    t.bigint "procedure_id", null: false
+    t.datetime "published_at"
+    t.datetime "updated_at", null: false
     t.index ["attestation_template_id"], name: "index_procedure_revisions_on_attestation_template_id"
     t.index ["dossier_submitted_message_id"], name: "index_procedure_revisions_on_dossier_submitted_message_id"
     t.index ["procedure_id"], name: "index_procedure_revisions_on_procedure_id"
   end
 
   create_table "procedures", id: :serial, force: :cascade do |t|
-    t.string "libelle"
-    t.string "description"
-    t.string "organisation"
-    t.string "direction"
-    t.string "lien_demarche"
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.boolean "euro_flag", default: false
-    t.boolean "cerfa_flag", default: false
-    t.string "lien_site_web"
-    t.string "lien_notice"
-    t.boolean "for_individual", default: false
-    t.date "auto_archive_on"
-    t.datetime "published_at"
-    t.datetime "hidden_at"
-    t.datetime "whitelisted_at"
-    t.boolean "ask_birthday", default: false, null: false
-    t.string "web_hook_url"
-    t.boolean "cloned_from_library", default: false
-    t.bigint "parent_procedure_id"
-    t.datetime "test_started_at"
     t.string "aasm_state", default: "brouillon"
-    t.bigint "service_id"
-    t.integer "duree_conservation_dossiers_dans_ds"
-    t.integer "duree_conservation_dossiers_hors_ds"
-    t.string "cadre_juridique"
-    t.boolean "juridique_required", default: true
-    t.boolean "durees_conservation_required", default: true
-    t.string "path", null: false
-    t.string "declarative_with_state"
-    t.text "monavis_embed"
-    t.text "routing_criteria_name", default: "Votre ville"
-    t.datetime "closed_at"
-    t.datetime "unpublished_at"
-    t.bigint "canonical_procedure_id"
-    t.string "api_entreprise_token"
-    t.bigint "draft_revision_id"
-    t.bigint "published_revision_id"
     t.boolean "allow_expert_review", default: true, null: false
-    t.boolean "experts_require_administrateur_invitation", default: false
-    t.string "encrypted_api_particulier_token"
+    t.string "api_entreprise_token"
     t.text "api_particulier_scopes", default: [], array: true
     t.jsonb "api_particulier_sources", default: {}
+    t.boolean "ask_birthday", default: false, null: false
+    t.date "auto_archive_on"
+    t.string "cadre_juridique"
+    t.bigint "canonical_procedure_id"
+    t.boolean "cerfa_flag", default: false
+    t.boolean "cloned_from_library", default: false
+    t.datetime "closed_at"
+    t.datetime "created_at", null: false
+    t.string "declarative_with_state"
+    t.string "description"
+    t.string "direction"
+    t.bigint "draft_revision_id"
+    t.integer "duree_conservation_dossiers_dans_ds"
+    t.integer "duree_conservation_dossiers_hors_ds"
+    t.boolean "durees_conservation_required", default: true
+    t.string "encrypted_api_particulier_token"
+    t.boolean "euro_flag", default: false
+    t.boolean "experts_require_administrateur_invitation", default: false
+    t.boolean "for_individual", default: false
+    t.datetime "hidden_at"
     t.boolean "instructeurs_self_management_enabled"
-    t.boolean "routing_enabled"
+    t.boolean "juridique_required", default: true
+    t.string "libelle"
+    t.string "lien_demarche"
+    t.string "lien_notice"
+    t.string "lien_site_web"
+    t.text "monavis_embed"
+    t.string "organisation"
+    t.bigint "parent_procedure_id"
+    t.string "path", null: false
     t.boolean "procedure_expires_when_termine_enabled", default: false
+    t.datetime "published_at"
+    t.bigint "published_revision_id"
+    t.text "routing_criteria_name", default: "Votre ville"
+    t.boolean "routing_enabled"
+    t.bigint "service_id"
+    t.datetime "test_started_at"
+    t.datetime "unpublished_at"
+    t.datetime "updated_at", null: false
+    t.string "web_hook_url"
+    t.datetime "whitelisted_at"
     t.bigint "zone_id"
     t.index ["api_particulier_sources"], name: "index_procedures_on_api_particulier_sources", using: :gin
     t.index ["declarative_with_state"], name: "index_procedures_on_declarative_with_state"
@@ -673,73 +671,73 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
 
   create_table "received_mails", id: :serial, force: :cascade do |t|
     t.text "body"
-    t.string "subject"
-    t.integer "procedure_id"
     t.datetime "created_at", null: false
+    t.integer "procedure_id"
+    t.string "subject"
     t.datetime "updated_at", null: false
     t.index ["procedure_id"], name: "index_received_mails_on_procedure_id"
   end
 
   create_table "refused_mails", id: :serial, force: :cascade do |t|
     t.text "body"
-    t.string "subject"
-    t.integer "procedure_id"
     t.datetime "created_at", null: false
+    t.integer "procedure_id"
+    t.string "subject"
     t.datetime "updated_at", null: false
     t.index ["procedure_id"], name: "index_refused_mails_on_procedure_id"
   end
 
   create_table "services", force: :cascade do |t|
-    t.string "type_organisme", null: false
-    t.string "nom", null: false
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
     t.bigint "administrateur_id"
-    t.string "organisme"
-    t.string "email"
-    t.string "telephone"
-    t.text "horaires"
     t.text "adresse"
+    t.datetime "created_at", null: false
+    t.string "email"
+    t.text "horaires"
+    t.string "nom", null: false
+    t.string "organisme"
+    t.string "telephone"
+    t.string "type_organisme", null: false
+    t.datetime "updated_at", null: false
     t.index ["administrateur_id", "nom"], name: "index_services_on_administrateur_id_and_nom", unique: true
     t.index ["administrateur_id"], name: "index_services_on_administrateur_id"
   end
 
   create_table "stats", force: :cascade do |t|
-    t.bigint "dossiers_not_brouillon", default: 0
+    t.bigint "administrations_partenaires", default: 0
+    t.datetime "created_at", precision: 6, null: false
     t.bigint "dossiers_brouillon", default: 0
-    t.bigint "dossiers_en_construction", default: 0
-    t.bigint "dossiers_en_instruction", default: 0
-    t.bigint "dossiers_termines", default: 0
+    t.jsonb "dossiers_cumulative", default: "{}", null: false
     t.bigint "dossiers_depose_avant_30_jours", default: 0
     t.bigint "dossiers_deposes_entre_60_et_30_jours", default: 0
-    t.bigint "administrations_partenaires", default: 0
-    t.jsonb "dossiers_cumulative", default: "{}", null: false
+    t.bigint "dossiers_en_construction", default: 0
+    t.bigint "dossiers_en_instruction", default: 0
     t.jsonb "dossiers_in_the_last_4_months", default: "{}", null: false
-    t.datetime "created_at", precision: 6, null: false
+    t.bigint "dossiers_not_brouillon", default: 0
+    t.bigint "dossiers_termines", default: 0
     t.datetime "updated_at", precision: 6, null: false
   end
 
   create_table "super_admins", id: :serial, force: :cascade do |t|
-    t.string "email", default: "", null: false
-    t.string "encrypted_password", default: "", null: false
-    t.string "reset_password_token"
-    t.datetime "reset_password_sent_at"
-    t.datetime "remember_created_at"
-    t.integer "sign_in_count", default: 0, null: false
-    t.datetime "current_sign_in_at"
-    t.datetime "last_sign_in_at"
-    t.string "current_sign_in_ip"
-    t.string "last_sign_in_ip"
+    t.integer "consumed_timestep"
     t.datetime "created_at"
-    t.datetime "updated_at"
-    t.integer "failed_attempts", default: 0, null: false
-    t.string "unlock_token"
-    t.datetime "locked_at"
+    t.datetime "current_sign_in_at"
+    t.string "current_sign_in_ip"
+    t.string "email", default: "", null: false
     t.string "encrypted_otp_secret"
     t.string "encrypted_otp_secret_iv"
     t.string "encrypted_otp_secret_salt"
-    t.integer "consumed_timestep"
+    t.string "encrypted_password", default: "", null: false
+    t.integer "failed_attempts", default: 0, null: false
+    t.datetime "last_sign_in_at"
+    t.string "last_sign_in_ip"
+    t.datetime "locked_at"
     t.boolean "otp_required_for_login"
+    t.datetime "remember_created_at"
+    t.datetime "reset_password_sent_at"
+    t.string "reset_password_token"
+    t.integer "sign_in_count", default: 0, null: false
+    t.string "unlock_token"
+    t.datetime "updated_at"
     t.index ["email"], name: "index_super_admins_on_email", unique: true
     t.index ["reset_password_token"], name: "index_super_admins_on_reset_password_token", unique: true
     t.index ["unlock_token"], name: "index_super_admins_on_unlock_token", unique: true
@@ -751,39 +749,38 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
 
   create_table "traitements", force: :cascade do |t|
     t.bigint "dossier_id"
-    t.string "motivation"
-    t.string "state"
-    t.datetime "processed_at"
     t.string "instructeur_email"
+    t.string "motivation"
     t.boolean "process_expired"
     t.boolean "process_expired_migrated", default: false
+    t.datetime "processed_at"
+    t.string "state"
     t.index ["dossier_id"], name: "index_traitements_on_dossier_id"
     t.index ["process_expired"], name: "index_traitements_on_process_expired"
   end
 
   create_table "trusted_device_tokens", force: :cascade do |t|
-    t.string "token", null: false
-    t.bigint "instructeur_id"
     t.datetime "created_at", null: false
+    t.bigint "instructeur_id"
+    t.string "token", null: false
     t.datetime "updated_at", null: false
     t.index ["instructeur_id"], name: "index_trusted_device_tokens_on_instructeur_id"
-    t.index ["token"], name: "index_trusted_device_tokens_on_token", unique: true
   end
 
   create_table "types_de_champ", id: :serial, force: :cascade do |t|
-    t.string "libelle"
-    t.string "type_champ"
-    t.integer "order_place"
-    t.text "description"
-    t.boolean "mandatory", default: false
-    t.boolean "private", default: false, null: false
     t.datetime "created_at"
-    t.datetime "updated_at"
-    t.jsonb "options"
-    t.bigint "stable_id"
-    t.bigint "parent_id"
-    t.bigint "revision_id"
+    t.text "description"
+    t.string "libelle"
+    t.boolean "mandatory", default: false
     t.boolean "migrated_parent"
+    t.jsonb "options"
+    t.integer "order_place"
+    t.bigint "parent_id"
+    t.boolean "private", default: false, null: false
+    t.bigint "revision_id"
+    t.bigint "stable_id"
+    t.string "type_champ"
+    t.datetime "updated_at"
     t.index ["parent_id"], name: "index_types_de_champ_on_parent_id"
     t.index ["private"], name: "index_types_de_champ_on_private"
     t.index ["revision_id"], name: "index_types_de_champ_on_revision_id"
@@ -791,32 +788,32 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
   end
 
   create_table "users", id: :serial, force: :cascade do |t|
-    t.string "email", default: "", null: false
-    t.string "encrypted_password", default: "", null: false
-    t.string "reset_password_token"
-    t.datetime "reset_password_sent_at"
-    t.datetime "remember_created_at"
-    t.integer "sign_in_count", default: 0, null: false
-    t.datetime "current_sign_in_at"
-    t.datetime "last_sign_in_at"
-    t.string "current_sign_in_ip"
-    t.string "last_sign_in_ip"
-    t.datetime "created_at"
-    t.datetime "updated_at"
-    t.string "siret"
-    t.string "loged_in_with_france_connect", default: "false"
+    t.bigint "administrateur_id"
+    t.datetime "confirmation_sent_at"
     t.string "confirmation_token"
     t.datetime "confirmed_at"
-    t.datetime "confirmation_sent_at"
-    t.text "unconfirmed_email"
-    t.integer "failed_attempts", default: 0, null: false
-    t.string "unlock_token"
-    t.datetime "locked_at"
-    t.bigint "instructeur_id"
-    t.bigint "administrateur_id"
+    t.datetime "created_at"
+    t.datetime "current_sign_in_at"
+    t.string "current_sign_in_ip"
+    t.string "email", default: "", null: false
+    t.string "encrypted_password", default: "", null: false
     t.bigint "expert_id"
+    t.integer "failed_attempts", default: 0, null: false
+    t.bigint "instructeur_id"
+    t.datetime "last_sign_in_at"
+    t.string "last_sign_in_ip"
     t.string "locale"
+    t.datetime "locked_at"
+    t.string "loged_in_with_france_connect", default: "false"
+    t.datetime "remember_created_at"
     t.bigint "requested_merge_into_id"
+    t.datetime "reset_password_sent_at"
+    t.string "reset_password_token"
+    t.integer "sign_in_count", default: 0, null: false
+    t.string "siret"
+    t.text "unconfirmed_email"
+    t.string "unlock_token"
+    t.datetime "updated_at"
     t.index ["administrateur_id"], name: "index_users_on_administrateur_id"
     t.index ["confirmation_token"], name: "index_users_on_confirmation_token", unique: true
     t.index ["email"], name: "index_users_on_email", unique: true
@@ -828,28 +825,28 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do
   end
 
   create_table "virus_scans", force: :cascade do |t|
+    t.string "blob_key"
+    t.bigint "champ_id"
+    t.datetime "created_at", null: false
     t.datetime "scanned_at"
     t.string "status"
-    t.bigint "champ_id"
-    t.string "blob_key"
-    t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["champ_id"], name: "index_virus_scans_on_champ_id"
   end
 
   create_table "without_continuation_mails", id: :serial, force: :cascade do |t|
     t.text "body"
-    t.string "subject"
-    t.integer "procedure_id"
     t.datetime "created_at", null: false
+    t.integer "procedure_id"
+    t.string "subject"
     t.datetime "updated_at", null: false
     t.index ["procedure_id"], name: "index_without_continuation_mails_on_procedure_id"
   end
 
   create_table "zones", force: :cascade do |t|
     t.string "acronym", null: false
-    t.string "label"
     t.datetime "created_at", precision: 6, null: false
+    t.string "label"
     t.datetime "updated_at", precision: 6, null: false
     t.index ["acronym"], name: "index_zones_on_acronym", unique: true
   end

From 5739150f158bc22b90e429ae2c04b26a3e0ea6fb Mon Sep 17 00:00:00 2001
From: Martin <martin@sharypic.com>
Date: Mon, 28 Feb 2022 13:16:27 +0100
Subject: [PATCH 4/4] feat(service/archive_uploader): add an archive uploader
 class to upload files thru a custom script which handle file encryption of
 massive file (bigger than 4Go)

Update doc/object-storange-and-data-encryption.md

Co-authored-by: LeSim <mail@simon.lehericey.net>

Update app/services/archive_uploader.rb

Co-authored-by: LeSim <mail@simon.lehericey.net>

Update doc/object-storange-and-data-encryption.md

Co-authored-by: Pierre de La Morinerie <kemenaran@gmail.com>

clean(doc): align document file name and document h1

clean(review): refactore based on various comments

clean(review): refactore based on various comments
---
 app/models/archive.rb                      |  2 +-
 app/services/archive_uploader.rb           | 85 ++++++++++++++++++++++
 app/services/procedure_archive_service.rb  |  9 +--
 config/env.example.optional                | 13 ++++
 doc/object-storange-and-data-encryption.md | 85 ++++++++++++++++++++++
 spec/services/archive_uploader_spec.rb     | 70 ++++++++++++++++++
 6 files changed, 257 insertions(+), 7 deletions(-)
 create mode 100644 app/services/archive_uploader.rb
 create mode 100644 doc/object-storange-and-data-encryption.md
 create mode 100644 spec/services/archive_uploader_spec.rb

diff --git a/app/models/archive.rb b/app/models/archive.rb
index 7ee93720f..42e0085ab 100644
--- a/app/models/archive.rb
+++ b/app/models/archive.rb
@@ -13,7 +13,7 @@
 class Archive < ApplicationRecord
   include AASM
 
-  RETENTION_DURATION = 1.week
+  RETENTION_DURATION = 4.days
 
   has_and_belongs_to_many :groupe_instructeurs
 
diff --git a/app/services/archive_uploader.rb b/app/services/archive_uploader.rb
new file mode 100644
index 000000000..1fb039600
--- /dev/null
+++ b/app/services/archive_uploader.rb
@@ -0,0 +1,85 @@
+class ArchiveUploader
+  # see: https://docs.ovh.com/fr/storage/pcs/capabilities-and-limitations/#max_file_size-5368709122-5gb
+  # officialy it's 5Gb. but let's avoid to reach the exact spot of the limit
+  # when file size is bigger, active storage expects the chunks + a manifest.
+  MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING = ENV.fetch('ACTIVE_STORAGE_FILE_SIZE_THRESHOLD_BEFORE_CUSTOM_UPLOAD') { 4.gigabytes }.to_i
+
+  def upload
+    uploaded_blob = create_and_upload_blob
+    begin
+      archive.file.purge if archive.file.attached?
+    rescue ActiveStorage::FileNotFoundError
+      archive.file.destroy
+      archive.file.detach
+    end
+    archive.reload
+    ActiveStorage::Attachment.create(
+      name: 'file',
+      record_type: 'Archive',
+      record_id: archive.id,
+      blob_id: uploaded_blob.id
+    )
+  end
+
+  private
+
+  attr_reader :procedure, :archive, :filepath
+
+  def create_and_upload_blob
+    if active_storage_service_local? || File.size(filepath) < MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING
+      upload_with_active_storage
+    else
+      upload_with_chunking_wrapper
+    end
+  end
+
+  def active_storage_service_local?
+    Rails.application.config.active_storage.service == :local
+  end
+
+  def upload_with_active_storage
+    params = blob_default_params(filepath).merge(io: File.open(filepath),
+                                                 identify: false)
+    blob = ActiveStorage::Blob.create_and_upload!(**params)
+    return blob
+  end
+
+  def upload_with_chunking_wrapper
+    params =  blob_default_params(filepath).merge(byte_size: File.size(filepath),
+                                                  checksum: Digest::SHA256.file(filepath).hexdigest)
+    blob = ActiveStorage::Blob.create_before_direct_upload!(**params)
+    if syscall_to_custom_uploader(blob)
+      return blob
+    else
+      blob.purge
+      fail "custom archive attachment failed, should it be retried ?"
+    end
+  end
+
+  # keeps consistency between ActiveStorage api calls (otherwise archives are not storaged in '/archives') :
+  # - create_and_upload, blob is attached by active storage
+  # - upload_with_chunking_wrapper, blob is attached by custom script
+  def blob_default_params(filepath)
+    {
+      key: namespaced_object_key,
+      filename: archive.filename(procedure),
+      content_type: 'application/zip',
+      metadata: { virus_scan_result: ActiveStorage::VirusScanner::SAFE }
+    }
+  end
+
+  # explicitely memoize so it keeps its consistency across many calls (Ex: retry)
+  def namespaced_object_key
+    @namespaced_object_key ||= "archives/#{Date.today.strftime("%Y-%m-%d")}/#{SecureRandom.uuid}"
+  end
+
+  def syscall_to_custom_uploader(blob)
+    system(ENV.fetch('ACTIVE_STORAGE_BIG_FILE_UPLOADER_WITH_ENCRYPTION_PATH').to_s, filepath, blob.key, exception: true)
+  end
+
+  def initialize(procedure:, archive:, filepath:)
+    @procedure = procedure
+    @archive = archive
+    @filepath = filepath
+  end
+end
diff --git a/app/services/procedure_archive_service.rb b/app/services/procedure_archive_service.rb
index f5561d426..3fd4c7211 100644
--- a/app/services/procedure_archive_service.rb
+++ b/app/services/procedure_archive_service.rb
@@ -34,12 +34,9 @@ class ProcedureArchiveService
     end
 
     attachments = create_list_of_attachments(dossiers)
-    download_and_zip(attachments) do |zip_file|
-      archive.file.attach(
-        io: File.open(zip_file),
-        filename: archive.filename(@procedure),
-        metadata: { virus_scan_result: ActiveStorage::VirusScanner::SAFE }
-      )
+    download_and_zip(attachments) do |zip_filepath|
+      ArchiveUploader.new(procedure: @procedure, archive: archive, filepath: zip_filepath)
+        .upload
     end
     archive.make_available!
     InstructeurMailer.send_archive(instructeur, @procedure, archive).deliver_later
diff --git a/config/env.example.optional b/config/env.example.optional
index 615162f9e..51b05c16d 100644
--- a/config/env.example.optional
+++ b/config/env.example.optional
@@ -99,3 +99,16 @@ MATOMO_IFRAME_URL="https://matomo.example.org/index.php?module=CoreAdminHome&act
 # Landing page sections
 # LANDING_TESTIMONIALS_ENABLED="enabled"
 # LANDING_USERS_ENABLED="enabled"
+
+# Archive creation options
+# when we create an archive of a Procedure, the worker uses this directory as a root in order to build our archives (archive are build within a tmp_dir in this dir)
+# ARCHIVE_CREATION_DIR='/tmp'
+# max parallel download when creating an archive
+# ARCHIVE_DOWNLOAD_MAX_PARALLEL=10
+
+# Archive when encryption of massive file options
+# depending on your object storage backend (ie: aws::s3/ovh::object_storage), it may requires a custom upload strategy for big file if you encrypt your files in case of data breach
+# suggested value is 4.gigabytes (4294967296)
+# ACTIVE_STORAGE_FILE_SIZE_THRESHOLD_BEFORE_CUSTOM_UPLOAD=4294967296
+# a custom script handling upload of big file
+# ACTIVE_STORAGE_BIG_FILE_UPLOADER_WITH_ENCRYPTION_PATH='/usr/local/bin/swift'
diff --git a/doc/object-storange-and-data-encryption.md b/doc/object-storange-and-data-encryption.md
new file mode 100644
index 000000000..f3bf4a395
--- /dev/null
+++ b/doc/object-storange-and-data-encryption.md
@@ -0,0 +1,85 @@
+# Object Storange And Data Encryption
+
+## Object Storage
+
+By default, demarches-simplifiees.fr uses an [OVH Object Storage](https://www.ovhcloud.com/en/public-cloud/object-storage/) backend. The hard-drives are encrypted at rest, but to protect user files even better, demarches-simplifiees.fr can also use an external encryption proxy, that will encrypt and decrypt files on the fly:
+
+* Encryption is done via our [proxy](https://github.com/betagouv/ds_proxy) when the file is uploaded by a client.
+* Decryption is done via the same proxy when the file is downloaded to a client
+
+### Object Storage limitation
+
+As an s3 compatible object storage backend, OVH Object Storage suffers the same limitations.
+
+One of them being that when you upload a file bigger than 5Go, it must be chunked into segments (see the [documentation](https://docs.ovh.com/fr/storage/pcs/capabilities-and-limitations/#max_file_size-5368709122-5gb)).
+
+This process to chunks the file in segment, then re-arrange it via a manifest. Unfortunately encryption can't work with this usecase.
+
+So we are using a custom script that wraps two call to our proxy in order to buffer all the chunks, encrypt/decrypt the whole. Here is an example
+
+```
+#!/usr/bin/env bash
+# wrapper script to encrypt and upload file received from archive
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# params
+# 1: filename
+# 2: key
+if ! [ "$#" -eq 2 ]; then
+    echo "usage: $0 <filename> <key>"
+    exit 1
+fi
+local_file_path=$1
+remote_basename=$(basename $local_file_path)
+key=$2
+
+# encrypt
+curl -s -XPUT http://ds_proxy_host:ds_proxy_port/local/encrypt/${remote_basename} --data-binary @${local_file_path}
+
+# get back encrypted file
+encrypted_filename="${local_file_path}.enc"
+curl -s http://ds_proxy_host:ds_proxy_port/local/encrypt/${remote_basename} -o ${encrypted_filename}
+
+# OVH openstack params
+os_tenant_name=os_tenant_name
+os_username=os_username
+os_password=os_password
+os_region_name=GRA
+
+# auth = https://auth.cloud.ovh.net/v3/
+# use haproxy proxy and not direct internet URL
+os_auth_url="os_auth_url"
+os_storage_url="os_storage_url" \
+container_name=container_name
+
+expiring_delay="$((60 * 60 * 24 * 4))" # 4 days
+
+# upload
+/usr/local/bin/swift \
+  --auth-version 3 \
+  --os-auth-url "$os_auth_url" \
+  --os-storage-url "$os_storage_url" \
+  --os-region-name "$os_region_name" \
+  --os-tenant-name "$os_tenant_name" \
+  --os-username "$os_username" \
+  --os-password "$os_password" \
+  upload \
+  --header "X-Delete-After: ${expiring_delay}" \
+  --segment-size "$((3 * 1024 * 1024 * 1024))" \
+  --header "Content-Disposition: filename=${remote_basename}" \
+  --object-name "${key}" \
+  "${container_name}" "${encrypted_filename}"
+
+swift_exit_code=$?
+
+# cleanup
+rm ${encrypted_filename}
+
+# return swift return code
+exit ${swift_exit_code}
+```
+
+
diff --git a/spec/services/archive_uploader_spec.rb b/spec/services/archive_uploader_spec.rb
new file mode 100644
index 000000000..c4b4bcfac
--- /dev/null
+++ b/spec/services/archive_uploader_spec.rb
@@ -0,0 +1,70 @@
+describe ProcedureArchiveService do
+  let(:procedure) { build(:procedure) }
+  let(:archive) { create(:archive) }
+  let(:file) { Tempfile.new }
+  let(:fixture_blob) { ActiveStorage::Blob.create_before_direct_upload!(filename: File.basename(file.path), byte_size: file.size, checksum: 'osf') }
+
+  let(:uploader) { ArchiveUploader.new(procedure: procedure, archive: archive, filepath: file.path) }
+
+  describe '.upload' do
+    context 'when active storage service is local' do
+      it 'uploads with upload_with_active_storage' do
+        expect(uploader).to receive(:active_storage_service_local?).and_return(true)
+        expect(uploader).to receive(:upload_with_active_storage).and_return(fixture_blob)
+        uploader.upload
+      end
+
+      it 'link the created blob as an attachment to the current archive instance' do
+        expect { uploader.upload }
+          .to change { ActiveStorage::Attachment.where(name: 'file', record_type: 'Archive', record_id: archive.id).count }.by(1)
+      end
+    end
+
+    context 'when active storage service is not local' do
+      before do
+        expect(uploader).to receive(:active_storage_service_local?).and_return(false)
+        expect(File).to receive(:size).with(file.path).and_return(filesize)
+      end
+
+      context 'when file is smaller than MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING' do
+        let(:filesize) { ArchiveUploader::MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING - 1 }
+
+        it 'uploads with upload_with_active_storage' do
+          expect(uploader).to receive(:upload_with_active_storage).and_return(fixture_blob)
+          uploader.upload
+        end
+      end
+
+      context 'when file is bigger than MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING' do
+        let(:filesize) { ArchiveUploader::MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING + 1 }
+
+        it 'uploads with upload_with_chunking_wrapper' do
+          expect(uploader).to receive(:upload_with_chunking_wrapper).and_return(fixture_blob)
+          uploader.upload
+        end
+
+        it 'link the created blob as an attachment to the current archive instance' do
+          expect(uploader).to receive(:upload_with_chunking_wrapper).and_return(fixture_blob)
+          expect { uploader.upload }
+            .to change { ActiveStorage::Attachment.where(name: 'file', record_type: 'Archive', record_id: archive.id).count }.by(1)
+        end
+      end
+    end
+  end
+
+  describe '.upload_with_chunking_wrapper' do
+    let(:fake_blob_checksum) { Digest::SHA256.file(file.path) }
+    let(:fake_blob_bytesize) { 100.gigabytes }
+
+    before do
+      expect(uploader).to receive(:syscall_to_custom_uploader).and_return(true)
+      expect(File).to receive(:size).with(file.path).and_return(fake_blob_bytesize)
+      expect(Digest::SHA256).to receive(:file).with(file.path).and_return(double(hexdigest: fake_blob_checksum.hexdigest))
+    end
+
+    it 'creates a blob' do
+      expect { uploader.send(:upload_with_chunking_wrapper) }
+        .to change { ActiveStorage::Blob.where(checksum: fake_blob_checksum.hexdigest, byte_size: fake_blob_bytesize).count }.by(1)
+    end
+  end
+end