diff --git a/Gemfile b/Gemfile index 0a5ce4a84..e210be466 100644 --- a/Gemfile +++ b/Gemfile @@ -80,6 +80,7 @@ gem 'sentry-ruby' gem 'sib-api-v3-sdk' gem 'skylight' gem 'spreadsheet_architect' +gem 'strong_migrations' # lint database migrations gem 'typhoeus' gem 'warden' gem 'webpacker' diff --git a/Gemfile.lock b/Gemfile.lock index 1a9ba160c..f2fbfeb94 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -691,6 +691,8 @@ GEM actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) + strong_migrations (0.8.0) + activerecord (>= 5.2) swd (1.3.0) activesupport (>= 3) attr_required (>= 0.0.5) @@ -875,6 +877,7 @@ DEPENDENCIES spreadsheet_architect spring spring-commands-rspec + strong_migrations timecop typhoeus vcr diff --git a/Rakefile b/Rakefile index f7a26ddaf..9e233babc 100644 --- a/Rakefile +++ b/Rakefile @@ -4,3 +4,6 @@ require File.expand_path('config/application', __dir__) Rails.application.load_tasks + +# Alphabetize schema.rb +task 'db:schema:dump': 'strong_migrations:alphabetize_columns' diff --git a/app/models/archive.rb b/app/models/archive.rb index 7ee93720f..42e0085ab 100644 --- a/app/models/archive.rb +++ b/app/models/archive.rb @@ -13,7 +13,7 @@ class Archive < ApplicationRecord include AASM - RETENTION_DURATION = 1.week + RETENTION_DURATION = 4.days has_and_belongs_to_many :groupe_instructeurs diff --git a/app/services/archive_uploader.rb b/app/services/archive_uploader.rb new file mode 100644 index 000000000..1fb039600 --- /dev/null +++ b/app/services/archive_uploader.rb @@ -0,0 +1,85 @@ +class ArchiveUploader + # see: https://docs.ovh.com/fr/storage/pcs/capabilities-and-limitations/#max_file_size-5368709122-5gb + # officialy it's 5Gb. but let's avoid to reach the exact spot of the limit + # when file size is bigger, active storage expects the chunks + a manifest. + MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING = ENV.fetch('ACTIVE_STORAGE_FILE_SIZE_THRESHOLD_BEFORE_CUSTOM_UPLOAD') { 4.gigabytes }.to_i + + def upload + uploaded_blob = create_and_upload_blob + begin + archive.file.purge if archive.file.attached? + rescue ActiveStorage::FileNotFoundError + archive.file.destroy + archive.file.detach + end + archive.reload + ActiveStorage::Attachment.create( + name: 'file', + record_type: 'Archive', + record_id: archive.id, + blob_id: uploaded_blob.id + ) + end + + private + + attr_reader :procedure, :archive, :filepath + + def create_and_upload_blob + if active_storage_service_local? || File.size(filepath) < MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING + upload_with_active_storage + else + upload_with_chunking_wrapper + end + end + + def active_storage_service_local? + Rails.application.config.active_storage.service == :local + end + + def upload_with_active_storage + params = blob_default_params(filepath).merge(io: File.open(filepath), + identify: false) + blob = ActiveStorage::Blob.create_and_upload!(**params) + return blob + end + + def upload_with_chunking_wrapper + params = blob_default_params(filepath).merge(byte_size: File.size(filepath), + checksum: Digest::SHA256.file(filepath).hexdigest) + blob = ActiveStorage::Blob.create_before_direct_upload!(**params) + if syscall_to_custom_uploader(blob) + return blob + else + blob.purge + fail "custom archive attachment failed, should it be retried ?" + end + end + + # keeps consistency between ActiveStorage api calls (otherwise archives are not storaged in '/archives') : + # - create_and_upload, blob is attached by active storage + # - upload_with_chunking_wrapper, blob is attached by custom script + def blob_default_params(filepath) + { + key: namespaced_object_key, + filename: archive.filename(procedure), + content_type: 'application/zip', + metadata: { virus_scan_result: ActiveStorage::VirusScanner::SAFE } + } + end + + # explicitely memoize so it keeps its consistency across many calls (Ex: retry) + def namespaced_object_key + @namespaced_object_key ||= "archives/#{Date.today.strftime("%Y-%m-%d")}/#{SecureRandom.uuid}" + end + + def syscall_to_custom_uploader(blob) + system(ENV.fetch('ACTIVE_STORAGE_BIG_FILE_UPLOADER_WITH_ENCRYPTION_PATH').to_s, filepath, blob.key, exception: true) + end + + def initialize(procedure:, archive:, filepath:) + @procedure = procedure + @archive = archive + @filepath = filepath + end +end diff --git a/app/services/procedure_archive_service.rb b/app/services/procedure_archive_service.rb index f5561d426..3fd4c7211 100644 --- a/app/services/procedure_archive_service.rb +++ b/app/services/procedure_archive_service.rb @@ -34,12 +34,9 @@ class ProcedureArchiveService end attachments = create_list_of_attachments(dossiers) - download_and_zip(attachments) do |zip_file| - archive.file.attach( - io: File.open(zip_file), - filename: archive.filename(@procedure), - metadata: { virus_scan_result: ActiveStorage::VirusScanner::SAFE } - ) + download_and_zip(attachments) do |zip_filepath| + ArchiveUploader.new(procedure: @procedure, archive: archive, filepath: zip_filepath) + .upload end archive.make_available! InstructeurMailer.send_archive(instructeur, @procedure, archive).deliver_later diff --git a/config/env.example.optional b/config/env.example.optional index 615162f9e..51b05c16d 100644 --- a/config/env.example.optional +++ b/config/env.example.optional @@ -99,3 +99,16 @@ MATOMO_IFRAME_URL="https://matomo.example.org/index.php?module=CoreAdminHome&act # Landing page sections # LANDING_TESTIMONIALS_ENABLED="enabled" # LANDING_USERS_ENABLED="enabled" + +# Archive creation options +# when we create an archive of a Procedure, the worker uses this directory as a root in order to build our archives (archive are build within a tmp_dir in this dir) +# ARCHIVE_CREATION_DIR='/tmp' +# max parallel download when creating an archive +# ARCHIVE_DOWNLOAD_MAX_PARALLEL=10 + +# Archive when encryption of massive file options +# depending on your object storage backend (ie: aws::s3/ovh::object_storage), it may requires a custom upload strategy for big file if you encrypt your files in case of data breach +# suggested value is 4.gigabytes (4294967296) +# ACTIVE_STORAGE_FILE_SIZE_THRESHOLD_BEFORE_CUSTOM_UPLOAD=4294967296 +# a custom script handling upload of big file +# ACTIVE_STORAGE_BIG_FILE_UPLOADER_WITH_ENCRYPTION_PATH='/usr/local/bin/swift' diff --git a/config/initializers/strong_migrations.rb b/config/initializers/strong_migrations.rb new file mode 100644 index 000000000..132263236 --- /dev/null +++ b/config/initializers/strong_migrations.rb @@ -0,0 +1,26 @@ +# Mark existing migrations as safe +StrongMigrations.start_after = 20220315125851 + +# Set timeouts for migrations +# If you use PgBouncer in transaction mode, delete these lines and set timeouts on the database user +StrongMigrations.lock_timeout = 10.seconds +StrongMigrations.statement_timeout = 1.hour + +# Analyze tables after indexes are added +# Outdated statistics can sometimes hurt performance +StrongMigrations.auto_analyze = true + +# Set the version of the production database +# so the right checks are run in development +# StrongMigrations.target_version = 10 + +# Add custom checks +# StrongMigrations.add_check do |method, args| +# if method == :add_index && args[0].to_s == "users" +# stop! "No more indexes on the users table" +# end +# end + +# Make some operations safe by default +# See https://github.com/ankane/strong_migrations#safe-by-default +# StrongMigrations.safe_by_default = true diff --git a/db/schema.rb b/db/schema.rb index 870305058..c5dac3a04 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -17,35 +17,35 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do enable_extension "unaccent" create_table "action_text_rich_texts", force: :cascade do |t| - t.string "name", null: false t.text "body" - t.string "record_type", null: false - t.bigint "record_id", null: false t.datetime "created_at", null: false + t.string "name", null: false + t.bigint "record_id", null: false + t.string "record_type", null: false t.datetime "updated_at", null: false t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true end create_table "active_storage_attachments", force: :cascade do |t| - t.string "name", null: false - t.string "record_type", null: false - t.bigint "record_id", null: false t.bigint "blob_id", null: false t.datetime "created_at", null: false + t.string "name", null: false + t.bigint "record_id", null: false + t.string "record_type", null: false t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true end create_table "active_storage_blobs", force: :cascade do |t| - t.string "key", null: false - t.string "filename", null: false - t.string "content_type" - t.text "metadata" t.bigint "byte_size", null: false t.string "checksum", null: false + t.string "content_type" t.datetime "created_at", null: false - t.string "service_name", null: false + t.string "filename", null: false + t.string "key", null: false t.integer "lock_version" + t.text "metadata" + t.string "service_name", null: false t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true end @@ -56,17 +56,17 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do end create_table "administrateurs", id: :serial, force: :cascade do |t| - t.datetime "created_at" - t.datetime "updated_at" t.boolean "active", default: false + t.datetime "created_at" t.string "encrypted_token" + t.datetime "updated_at" t.bigint "user_id" end create_table "administrateurs_instructeurs", id: false, force: :cascade do |t| t.integer "administrateur_id" - t.integer "instructeur_id" t.datetime "created_at" + t.integer "instructeur_id" t.datetime "updated_at" t.index ["administrateur_id"], name: "index_administrateurs_instructeurs_on_administrateur_id" t.index ["instructeur_id", "administrateur_id"], name: "unique_couple_administrateur_instructeur", unique: true @@ -75,97 +75,97 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do create_table "administrateurs_procedures", id: false, force: :cascade do |t| t.bigint "administrateur_id", null: false - t.bigint "procedure_id", null: false t.datetime "created_at", null: false - t.datetime "updated_at", null: false t.boolean "manager" + t.bigint "procedure_id", null: false + t.datetime "updated_at", null: false t.index ["administrateur_id", "procedure_id"], name: "index_unique_admin_proc_couple", unique: true t.index ["administrateur_id"], name: "index_administrateurs_procedures_on_administrateur_id" t.index ["procedure_id"], name: "index_administrateurs_procedures_on_procedure_id" end create_table "archives", force: :cascade do |t| - t.string "status", null: false - t.date "month" - t.string "time_span_type", null: false t.datetime "created_at", precision: 6, null: false - t.datetime "updated_at", precision: 6, null: false t.text "key", null: false + t.date "month" + t.string "status", null: false + t.string "time_span_type", null: false + t.datetime "updated_at", precision: 6, null: false t.index ["key", "time_span_type", "month"], name: "index_archives_on_key_and_time_span_type_and_month", unique: true end create_table "archives_groupe_instructeurs", force: :cascade do |t| t.bigint "archive_id", null: false - t.bigint "groupe_instructeur_id", null: false t.datetime "created_at", precision: 6, null: false + t.bigint "groupe_instructeur_id", null: false t.datetime "updated_at", precision: 6, null: false t.index ["archive_id"], name: "index_archives_groupe_instructeurs_on_archive_id" t.index ["groupe_instructeur_id"], name: "index_archives_groupe_instructeurs_on_groupe_instructeur_id" end create_table "assign_tos", id: :serial, force: :cascade do |t| - t.integer "instructeur_id" t.datetime "created_at" - t.datetime "updated_at" - t.bigint "groupe_instructeur_id" - t.boolean "weekly_email_notifications_enabled", default: true, null: false t.boolean "daily_email_notifications_enabled", default: false, null: false - t.boolean "instant_email_message_notifications_enabled", default: false, null: false + t.bigint "groupe_instructeur_id" t.boolean "instant_email_dossier_notifications_enabled", default: false, null: false + t.boolean "instant_email_message_notifications_enabled", default: false, null: false + t.integer "instructeur_id" + t.datetime "updated_at" + t.boolean "weekly_email_notifications_enabled", default: true, null: false t.index ["groupe_instructeur_id", "instructeur_id"], name: "unique_couple_groupe_instructeur_instructeur", unique: true t.index ["groupe_instructeur_id"], name: "index_assign_tos_on_groupe_instructeur_id" t.index ["instructeur_id"], name: "index_assign_tos_on_instructeur_id" end create_table "attestation_templates", id: :serial, force: :cascade do |t| - t.text "title" - t.text "body" - t.text "footer" t.boolean "activated" + t.text "body" t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.text "footer" t.integer "procedure_id" + t.text "title" + t.datetime "updated_at", null: false t.index ["procedure_id"], name: "index_attestation_templates_on_procedure_id", unique: true end create_table "attestations", id: :serial, force: :cascade do |t| - t.string "title" - t.integer "dossier_id", null: false t.datetime "created_at", null: false + t.integer "dossier_id", null: false + t.string "title" t.datetime "updated_at", null: false t.index ["dossier_id"], name: "index_attestations_on_dossier_id" end create_table "avis", id: :serial, force: :cascade do |t| - t.string "email" - t.text "introduction" t.text "answer" - t.integer "dossier_id" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false t.integer "claimant_id", null: false - t.boolean "confidentiel", default: false, null: false - t.datetime "revoked_at" - t.bigint "experts_procedure_id" t.string "claimant_type" + t.boolean "confidentiel", default: false, null: false + t.datetime "created_at", null: false + t.integer "dossier_id" + t.string "email" + t.bigint "experts_procedure_id" + t.text "introduction" + t.datetime "revoked_at" + t.datetime "updated_at", null: false t.index ["claimant_id"], name: "index_avis_on_claimant_id" t.index ["dossier_id"], name: "index_avis_on_dossier_id" t.index ["experts_procedure_id"], name: "index_avis_on_experts_procedure_id" end create_table "bill_signatures", force: :cascade do |t| - t.string "digest" t.datetime "created_at", null: false + t.string "digest" t.datetime "updated_at", null: false end create_table "bulk_messages", force: :cascade do |t| t.text "body", null: false + t.datetime "created_at", precision: 6, null: false t.integer "dossier_count" t.string "dossier_state" - t.datetime "sent_at", null: false t.bigint "instructeur_id", null: false - t.datetime "created_at", precision: 6, null: false + t.datetime "sent_at", null: false t.datetime "updated_at", precision: 6, null: false end @@ -178,21 +178,21 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do end create_table "champs", id: :serial, force: :cascade do |t| - t.string "value" - t.integer "type_de_champ_id" - t.integer "dossier_id" - t.string "type" t.datetime "created_at" - t.datetime "updated_at" - t.boolean "private", default: false, null: false - t.integer "etablissement_id" - t.bigint "parent_id" - t.integer "row" t.jsonb "data" + t.integer "dossier_id" + t.integer "etablissement_id" t.string "external_id" t.string "fetch_external_data_exceptions", array: true - t.jsonb "value_json" + t.bigint "parent_id" + t.boolean "private", default: false, null: false t.datetime "rebased_at" + t.integer "row" + t.string "type" + t.integer "type_de_champ_id" + t.datetime "updated_at" + t.string "value" + t.jsonb "value_json" t.index ["dossier_id"], name: "index_champs_on_dossier_id" t.index ["etablissement_id"], name: "index_champs_on_etablissement_id" t.index ["parent_id"], name: "index_champs_on_parent_id" @@ -204,23 +204,23 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do create_table "closed_mails", id: :serial, force: :cascade do |t| t.text "body" - t.string "subject" - t.integer "procedure_id" t.datetime "created_at", null: false + t.integer "procedure_id" + t.string "subject" t.datetime "updated_at", null: false t.index ["procedure_id"], name: "index_closed_mails_on_procedure_id" end create_table "commentaires", id: :serial, force: :cascade do |t| - t.string "email" - t.datetime "created_at", null: false t.string "body" + t.datetime "created_at", null: false + t.datetime "discarded_at" t.integer "dossier_id" + t.string "email" + t.bigint "expert_id" + t.bigint "instructeur_id" t.datetime "updated_at", null: false t.bigint "user_id" - t.bigint "instructeur_id" - t.bigint "expert_id" - t.datetime "discarded_at" t.index ["dossier_id"], name: "index_commentaires_on_dossier_id" t.index ["expert_id"], name: "index_commentaires_on_expert_id" t.index ["instructeur_id"], name: "index_commentaires_on_instructeur_id" @@ -228,48 +228,48 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do end create_table "delayed_jobs", id: :serial, force: :cascade do |t| - t.integer "priority", default: 0, null: false t.integer "attempts", default: 0, null: false + t.datetime "created_at" + t.string "cron" + t.datetime "failed_at" t.text "handler", null: false t.text "last_error" - t.datetime "run_at" t.datetime "locked_at" - t.datetime "failed_at" t.string "locked_by" + t.integer "priority", default: 0, null: false t.string "queue" - t.datetime "created_at" + t.datetime "run_at" t.datetime "updated_at" - t.string "cron" t.index ["priority", "run_at"], name: "delayed_jobs_priority" end create_table "deleted_dossiers", force: :cascade do |t| - t.bigint "procedure_id" - t.bigint "dossier_id" - t.datetime "deleted_at" - t.string "state" t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.string "reason" - t.bigint "user_id" + t.datetime "deleted_at" + t.bigint "dossier_id" t.bigint "groupe_instructeur_id" + t.bigint "procedure_id" + t.string "reason" t.bigint "revision_id" + t.string "state" + t.datetime "updated_at", null: false + t.bigint "user_id" t.index ["deleted_at"], name: "index_deleted_dossiers_on_deleted_at" t.index ["dossier_id"], name: "index_deleted_dossiers_on_dossier_id", unique: true t.index ["procedure_id"], name: "index_deleted_dossiers_on_procedure_id" end create_table "dossier_operation_logs", force: :cascade do |t| - t.string "operation", null: false - t.bigint "dossier_id" - t.bigint "instructeur_id" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false t.boolean "automatic_operation", default: false, null: false - t.datetime "keep_until" - t.datetime "executed_at" - t.text "digest" t.bigint "bill_signature_id" + t.datetime "created_at", null: false + t.text "digest" + t.bigint "dossier_id" + t.datetime "executed_at" + t.bigint "instructeur_id" + t.datetime "keep_until" + t.string "operation", null: false + t.datetime "updated_at", null: false t.index ["bill_signature_id"], name: "index_dossier_operation_logs_on_bill_signature_id" t.index ["dossier_id"], name: "index_dossier_operation_logs_on_dossier_id" t.index ["instructeur_id"], name: "index_dossier_operation_logs_on_instructeur_id" @@ -277,64 +277,62 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do end create_table "dossier_submitted_messages", force: :cascade do |t| - t.string "message_on_submit_by_usager" t.datetime "created_at", precision: 6, null: false + t.string "message_on_submit_by_usager" t.datetime "updated_at", precision: 6, null: false end create_table "dossier_transfer_logs", force: :cascade do |t| + t.datetime "created_at", precision: 6, null: false + t.bigint "dossier_id", null: false t.string "from", null: false t.string "to", null: false - t.bigint "dossier_id", null: false - t.datetime "created_at", precision: 6, null: false t.datetime "updated_at", precision: 6, null: false t.index ["dossier_id"], name: "index_dossier_transfer_logs_on_dossier_id" end create_table "dossier_transfers", force: :cascade do |t| - t.string "email", null: false t.datetime "created_at", precision: 6, null: false + t.string "email", null: false t.datetime "updated_at", precision: 6, null: false t.index ["email"], name: "index_dossier_transfers_on_email" end create_table "dossiers", id: :serial, force: :cascade do |t| - t.boolean "autorisation_donnees" - t.datetime "created_at" - t.datetime "updated_at" - t.string "state" - t.integer "user_id" + t.string "api_entreprise_job_exceptions", array: true t.boolean "archived", default: false - t.datetime "en_construction_at" - t.datetime "en_instruction_at" - t.datetime "processed_at" - t.text "motivation" - t.datetime "hidden_at" - t.text "search_terms" - t.text "private_search_terms" - t.bigint "groupe_instructeur_id" + t.boolean "autorisation_donnees" t.datetime "brouillon_close_to_expiration_notice_sent_at" - t.datetime "groupe_instructeur_updated_at" + t.interval "conservation_extension", default: "PT0S" + t.datetime "created_at" + t.datetime "declarative_triggered_at" + t.string "deleted_user_email_never_send" + t.datetime "depose_at" + t.bigint "dossier_transfer_id" + t.datetime "en_construction_at" t.datetime "en_construction_close_to_expiration_notice_sent_at" t.interval "en_construction_conservation_extension", default: "PT0S" - t.datetime "termine_close_to_expiration_notice_sent_at" - t.bigint "revision_id" - t.datetime "last_champ_updated_at" - t.datetime "last_champ_private_updated_at" - t.datetime "last_avis_updated_at" - t.datetime "last_commentaire_updated_at" - t.string "api_entreprise_job_exceptions", array: true - t.interval "conservation_extension", default: "PT0S" - t.string "deleted_user_email_never_send" - t.datetime "declarative_triggered_at" - t.bigint "dossier_transfer_id" - t.datetime "identity_updated_at" - t.datetime "depose_at" - t.datetime "hidden_by_user_at" + t.datetime "en_instruction_at" + t.bigint "groupe_instructeur_id" + t.datetime "groupe_instructeur_updated_at" + t.datetime "hidden_at" t.datetime "hidden_by_administration_at" t.string "hidden_by_reason" - t.index "to_tsvector('french'::regconfig, (search_terms || private_search_terms))", name: "index_dossiers_on_search_terms_private_search_terms", using: :gin - t.index "to_tsvector('french'::regconfig, search_terms)", name: "index_dossiers_on_search_terms", using: :gin + t.datetime "hidden_by_user_at" + t.datetime "identity_updated_at" + t.datetime "last_avis_updated_at" + t.datetime "last_champ_private_updated_at" + t.datetime "last_champ_updated_at" + t.datetime "last_commentaire_updated_at" + t.text "motivation" + t.text "private_search_terms" + t.datetime "processed_at" + t.bigint "revision_id" + t.text "search_terms" + t.string "state" + t.datetime "termine_close_to_expiration_notice_sent_at" + t.datetime "updated_at" + t.integer "user_id" t.index ["archived"], name: "index_dossiers_on_archived" t.index ["dossier_transfer_id"], name: "index_dossiers_on_dossier_transfer_id" t.index ["groupe_instructeur_id"], name: "index_dossiers_on_groupe_instructeur_id" @@ -345,66 +343,66 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do end create_table "drop_down_lists", id: :serial, force: :cascade do |t| - t.string "value" - t.integer "type_de_champ_id" t.datetime "created_at" + t.integer "type_de_champ_id" t.datetime "updated_at" + t.string "value" t.index ["type_de_champ_id"], name: "index_drop_down_lists_on_type_de_champ_id" end create_table "etablissements", id: :serial, force: :cascade do |t| - t.string "siret" - t.boolean "siege_social" - t.string "naf" - t.string "libelle_naf" t.string "adresse" - t.string "numero_voie" - t.string "type_voie" - t.string "nom_voie" - t.string "complement_adresse" - t.string "code_postal" - t.string "localite" - t.string "code_insee_localite" - t.integer "dossier_id" - t.string "entreprise_siren" - t.bigint "entreprise_capital_social" - t.string "entreprise_numero_tva_intracommunautaire" - t.string "entreprise_forme_juridique" - t.string "entreprise_forme_juridique_code" - t.string "entreprise_nom_commercial" - t.string "entreprise_raison_sociale" - t.string "entreprise_siret_siege_social" - t.string "entreprise_code_effectif_entreprise" - t.date "entreprise_date_creation" - t.string "entreprise_nom" - t.string "entreprise_prenom" - t.string "association_rna" - t.string "association_titre" - t.text "association_objet" t.date "association_date_creation" t.date "association_date_declaration" t.date "association_date_publication" + t.text "association_objet" + t.string "association_rna" + t.string "association_titre" + t.string "code_insee_localite" + t.string "code_postal" + t.string "complement_adresse" t.datetime "created_at" - t.datetime "updated_at" t.boolean "diffusable_commercialement" - t.string "entreprise_effectif_mois" - t.string "entreprise_effectif_annee" - t.decimal "entreprise_effectif_mensuel" - t.decimal "entreprise_effectif_annuel" - t.string "entreprise_effectif_annuel_annee" + t.integer "dossier_id" + t.string "enseigne" t.jsonb "entreprise_bilans_bdf" t.string "entreprise_bilans_bdf_monnaie" - t.string "enseigne" + t.bigint "entreprise_capital_social" + t.string "entreprise_code_effectif_entreprise" + t.date "entreprise_date_creation" + t.string "entreprise_effectif_annee" + t.decimal "entreprise_effectif_annuel" + t.string "entreprise_effectif_annuel_annee" + t.decimal "entreprise_effectif_mensuel" + t.string "entreprise_effectif_mois" + t.string "entreprise_forme_juridique" + t.string "entreprise_forme_juridique_code" + t.string "entreprise_nom" + t.string "entreprise_nom_commercial" + t.string "entreprise_numero_tva_intracommunautaire" + t.string "entreprise_prenom" + t.string "entreprise_raison_sociale" + t.string "entreprise_siren" + t.string "entreprise_siret_siege_social" + t.string "libelle_naf" + t.string "localite" + t.string "naf" + t.string "nom_voie" + t.string "numero_voie" + t.boolean "siege_social" + t.string "siret" + t.string "type_voie" + t.datetime "updated_at" t.index ["dossier_id"], name: "index_etablissements_on_dossier_id", unique: true end create_table "exercices", id: :serial, force: :cascade do |t| t.string "ca" + t.datetime "created_at" t.datetime "dateFinExercice" + t.datetime "date_fin_exercice" t.integer "date_fin_exercice_timestamp" t.integer "etablissement_id" - t.datetime "date_fin_exercice" - t.datetime "created_at" t.datetime "updated_at" t.index ["etablissement_id"], name: "index_exercices_on_etablissement_id" end @@ -416,59 +414,59 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do end create_table "experts_procedures", force: :cascade do |t| - t.bigint "expert_id", null: false - t.bigint "procedure_id", null: false t.boolean "allow_decision_access", default: false, null: false t.datetime "created_at", precision: 6, null: false - t.datetime "updated_at", precision: 6, null: false + t.bigint "expert_id", null: false + t.bigint "procedure_id", null: false t.datetime "revoked_at" + t.datetime "updated_at", precision: 6, null: false t.index ["expert_id", "procedure_id"], name: "index_experts_procedures_on_expert_id_and_procedure_id", unique: true t.index ["expert_id"], name: "index_experts_procedures_on_expert_id" t.index ["procedure_id"], name: "index_experts_procedures_on_procedure_id" end create_table "exports", force: :cascade do |t| - t.string "format", null: false t.datetime "created_at", null: false - t.datetime "updated_at", null: false + t.string "format", null: false t.text "key", null: false t.string "time_span_type", default: "everything", null: false + t.datetime "updated_at", null: false t.index ["format", "time_span_type", "key"], name: "index_exports_on_format_and_time_span_type_and_key", unique: true end create_table "exports_groupe_instructeurs", force: :cascade do |t| + t.datetime "created_at", null: false t.bigint "export_id", null: false t.bigint "groupe_instructeur_id", null: false - t.datetime "created_at", null: false t.datetime "updated_at", null: false end - create_table "flipper_features", force: :cascade do |t| - t.string "key", null: false + create_table "flipper_features", id: false, force: :cascade do |t| t.datetime "created_at", null: false + t.bigserial "id", null: false + t.string "key", null: false t.datetime "updated_at", null: false - t.index ["key"], name: "index_flipper_features_on_key", unique: true end create_table "flipper_gates", force: :cascade do |t| + t.datetime "created_at", null: false t.string "feature_key", null: false t.string "key", null: false - t.string "value" - t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.string "value" t.index ["feature_key", "key", "value"], name: "index_flipper_gates_on_feature_key_and_key_and_value", unique: true end create_table "follows", id: :serial, force: :cascade do |t| - t.integer "instructeur_id", null: false - t.integer "dossier_id", null: false - t.datetime "demande_seen_at", null: false t.datetime "annotations_privees_seen_at", null: false t.datetime "avis_seen_at", null: false - t.datetime "messagerie_seen_at", null: false t.datetime "created_at" - t.datetime "updated_at" + t.datetime "demande_seen_at", null: false + t.integer "dossier_id", null: false + t.integer "instructeur_id", null: false + t.datetime "messagerie_seen_at", null: false t.datetime "unfollowed_at" + t.datetime "updated_at" t.index ["dossier_id"], name: "index_follows_on_dossier_id" t.index ["instructeur_id", "dossier_id", "unfollowed_at"], name: "uniqueness_index", unique: true t.index ["instructeur_id"], name: "index_follows_on_instructeur_id" @@ -476,187 +474,187 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do end create_table "france_connect_informations", id: :serial, force: :cascade do |t| - t.string "gender" - t.string "given_name" - t.string "family_name" t.date "birthdate" t.string "birthplace" - t.string "france_connect_particulier_id" - t.integer "user_id" - t.string "email_france_connect" t.datetime "created_at", null: false - t.datetime "updated_at", null: false t.jsonb "data" + t.string "email_france_connect" + t.string "family_name" + t.string "france_connect_particulier_id" + t.string "gender" + t.string "given_name" t.string "merge_token" t.datetime "merge_token_created_at" + t.datetime "updated_at", null: false + t.integer "user_id" t.index ["merge_token"], name: "index_france_connect_informations_on_merge_token" t.index ["user_id"], name: "index_france_connect_informations_on_user_id" end create_table "geo_areas", force: :cascade do |t| - t.string "source" + t.bigint "champ_id" + t.datetime "created_at" + t.string "geo_reference_id" t.jsonb "geometry" t.jsonb "properties" - t.bigint "champ_id" - t.string "geo_reference_id" - t.datetime "created_at" + t.string "source" t.datetime "updated_at" t.index ["champ_id"], name: "index_geo_areas_on_champ_id" t.index ["source"], name: "index_geo_areas_on_source" end create_table "groupe_instructeurs", force: :cascade do |t| - t.bigint "procedure_id", null: false - t.text "label", null: false t.datetime "created_at", null: false + t.text "label", null: false + t.bigint "procedure_id", null: false t.datetime "updated_at", null: false t.index ["procedure_id", "label"], name: "index_groupe_instructeurs_on_procedure_id_and_label", unique: true t.index ["procedure_id"], name: "index_groupe_instructeurs_on_procedure_id" end create_table "individuals", id: :serial, force: :cascade do |t| - t.string "nom" - t.string "prenom" + t.date "birthdate" + t.datetime "created_at" t.integer "dossier_id" t.string "gender" - t.datetime "created_at" + t.string "nom" + t.string "prenom" t.datetime "updated_at" - t.date "birthdate" t.index ["dossier_id"], name: "index_individuals_on_dossier_id", unique: true end create_table "initiated_mails", id: :serial, force: :cascade do |t| - t.string "subject" t.text "body" - t.integer "procedure_id" t.datetime "created_at", null: false + t.integer "procedure_id" + t.string "subject" t.datetime "updated_at", null: false t.index ["procedure_id"], name: "index_initiated_mails_on_procedure_id" end create_table "instructeurs", id: :serial, force: :cascade do |t| + t.string "agent_connect_id" + t.boolean "bypass_email_login_token", default: false, null: false t.datetime "created_at" - t.datetime "updated_at" t.text "encrypted_login_token" t.datetime "login_token_created_at" - t.boolean "bypass_email_login_token", default: false, null: false - t.string "agent_connect_id" + t.datetime "updated_at" t.bigint "user_id" t.index ["agent_connect_id"], name: "index_instructeurs_on_agent_connect_id", unique: true end create_table "invites", id: :serial, force: :cascade do |t| + t.datetime "created_at" + t.integer "dossier_id" t.string "email" t.string "email_sender" - t.integer "dossier_id" - t.integer "user_id" - t.datetime "created_at" - t.datetime "updated_at" t.text "message" + t.datetime "updated_at" + t.integer "user_id" t.index ["email", "dossier_id"], name: "index_invites_on_email_and_dossier_id", unique: true end create_table "merge_logs", force: :cascade do |t| - t.bigint "from_user_id", null: false - t.string "from_user_email", null: false - t.bigint "user_id", null: false t.datetime "created_at", precision: 6, null: false + t.string "from_user_email", null: false + t.bigint "from_user_id", null: false t.datetime "updated_at", precision: 6, null: false + t.bigint "user_id", null: false t.index ["user_id"], name: "index_merge_logs_on_user_id" end create_table "module_api_cartos", id: :serial, force: :cascade do |t| - t.integer "procedure_id" - t.boolean "use_api_carto", default: false - t.boolean "quartiers_prioritaires", default: false t.boolean "cadastre", default: false t.datetime "created_at" - t.datetime "updated_at" t.boolean "migrated" + t.integer "procedure_id" + t.boolean "quartiers_prioritaires", default: false + t.datetime "updated_at" + t.boolean "use_api_carto", default: false t.index ["procedure_id"], name: "index_module_api_cartos_on_procedure_id", unique: true end create_table "procedure_presentations", id: :serial, force: :cascade do |t| t.integer "assign_to_id" - t.jsonb "sort", default: {"order"=>"desc", "table"=>"notifications", "column"=>"notifications"}, null: false - t.jsonb "filters", default: {"tous"=>[], "suivis"=>[], "traites"=>[], "a-suivre"=>[], "archives"=>[], "expirant"=>[], "supprimes_recemment"=>[]}, null: false t.datetime "created_at" - t.datetime "updated_at" t.jsonb "displayed_fields", default: [{"label"=>"Demandeur", "table"=>"user", "column"=>"email"}], null: false + t.jsonb "filters", default: {"tous"=>[], "suivis"=>[], "traites"=>[], "a-suivre"=>[], "archives"=>[], "expirant"=>[], "supprimes_recemment"=>[]}, null: false + t.jsonb "sort", default: {"order"=>"desc", "table"=>"notifications", "column"=>"notifications"}, null: false + t.datetime "updated_at" t.index ["assign_to_id"], name: "index_procedure_presentations_on_assign_to_id", unique: true end create_table "procedure_revision_types_de_champ", force: :cascade do |t| + t.datetime "created_at", null: false + t.bigint "parent_id" + t.integer "position", null: false t.bigint "revision_id", null: false t.bigint "type_de_champ_id", null: false - t.integer "position", null: false - t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.bigint "parent_id" t.index ["parent_id"], name: "index_procedure_revision_types_de_champ_on_parent_id" t.index ["revision_id"], name: "index_procedure_revision_types_de_champ_on_revision_id" t.index ["type_de_champ_id"], name: "index_procedure_revision_types_de_champ_on_type_de_champ_id" end create_table "procedure_revisions", force: :cascade do |t| - t.bigint "procedure_id", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.datetime "published_at" t.bigint "attestation_template_id" + t.datetime "created_at", null: false t.bigint "dossier_submitted_message_id" + t.bigint "procedure_id", null: false + t.datetime "published_at" + t.datetime "updated_at", null: false t.index ["attestation_template_id"], name: "index_procedure_revisions_on_attestation_template_id" t.index ["dossier_submitted_message_id"], name: "index_procedure_revisions_on_dossier_submitted_message_id" t.index ["procedure_id"], name: "index_procedure_revisions_on_procedure_id" end create_table "procedures", id: :serial, force: :cascade do |t| - t.string "libelle" - t.string "description" - t.string "organisation" - t.string "direction" - t.string "lien_demarche" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.boolean "euro_flag", default: false - t.boolean "cerfa_flag", default: false - t.string "lien_site_web" - t.string "lien_notice" - t.boolean "for_individual", default: false - t.date "auto_archive_on" - t.datetime "published_at" - t.datetime "hidden_at" - t.datetime "whitelisted_at" - t.boolean "ask_birthday", default: false, null: false - t.string "web_hook_url" - t.boolean "cloned_from_library", default: false - t.bigint "parent_procedure_id" - t.datetime "test_started_at" t.string "aasm_state", default: "brouillon" - t.bigint "service_id" - t.integer "duree_conservation_dossiers_dans_ds" - t.integer "duree_conservation_dossiers_hors_ds" - t.string "cadre_juridique" - t.boolean "juridique_required", default: true - t.boolean "durees_conservation_required", default: true - t.string "path", null: false - t.string "declarative_with_state" - t.text "monavis_embed" - t.text "routing_criteria_name", default: "Votre ville" - t.datetime "closed_at" - t.datetime "unpublished_at" - t.bigint "canonical_procedure_id" - t.string "api_entreprise_token" - t.bigint "draft_revision_id" - t.bigint "published_revision_id" t.boolean "allow_expert_review", default: true, null: false - t.boolean "experts_require_administrateur_invitation", default: false - t.string "encrypted_api_particulier_token" + t.string "api_entreprise_token" t.text "api_particulier_scopes", default: [], array: true t.jsonb "api_particulier_sources", default: {} + t.boolean "ask_birthday", default: false, null: false + t.date "auto_archive_on" + t.string "cadre_juridique" + t.bigint "canonical_procedure_id" + t.boolean "cerfa_flag", default: false + t.boolean "cloned_from_library", default: false + t.datetime "closed_at" + t.datetime "created_at", null: false + t.string "declarative_with_state" + t.string "description" + t.string "direction" + t.bigint "draft_revision_id" + t.integer "duree_conservation_dossiers_dans_ds" + t.integer "duree_conservation_dossiers_hors_ds" + t.boolean "durees_conservation_required", default: true + t.string "encrypted_api_particulier_token" + t.boolean "euro_flag", default: false + t.boolean "experts_require_administrateur_invitation", default: false + t.boolean "for_individual", default: false + t.datetime "hidden_at" t.boolean "instructeurs_self_management_enabled" - t.boolean "routing_enabled" + t.boolean "juridique_required", default: true + t.string "libelle" + t.string "lien_demarche" + t.string "lien_notice" + t.string "lien_site_web" + t.text "monavis_embed" + t.string "organisation" + t.bigint "parent_procedure_id" + t.string "path", null: false t.boolean "procedure_expires_when_termine_enabled", default: false + t.datetime "published_at" + t.bigint "published_revision_id" + t.text "routing_criteria_name", default: "Votre ville" + t.boolean "routing_enabled" + t.bigint "service_id" + t.datetime "test_started_at" + t.datetime "unpublished_at" + t.datetime "updated_at", null: false + t.string "web_hook_url" + t.datetime "whitelisted_at" t.bigint "zone_id" t.index ["api_particulier_sources"], name: "index_procedures_on_api_particulier_sources", using: :gin t.index ["declarative_with_state"], name: "index_procedures_on_declarative_with_state" @@ -673,73 +671,73 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do create_table "received_mails", id: :serial, force: :cascade do |t| t.text "body" - t.string "subject" - t.integer "procedure_id" t.datetime "created_at", null: false + t.integer "procedure_id" + t.string "subject" t.datetime "updated_at", null: false t.index ["procedure_id"], name: "index_received_mails_on_procedure_id" end create_table "refused_mails", id: :serial, force: :cascade do |t| t.text "body" - t.string "subject" - t.integer "procedure_id" t.datetime "created_at", null: false + t.integer "procedure_id" + t.string "subject" t.datetime "updated_at", null: false t.index ["procedure_id"], name: "index_refused_mails_on_procedure_id" end create_table "services", force: :cascade do |t| - t.string "type_organisme", null: false - t.string "nom", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false t.bigint "administrateur_id" - t.string "organisme" - t.string "email" - t.string "telephone" - t.text "horaires" t.text "adresse" + t.datetime "created_at", null: false + t.string "email" + t.text "horaires" + t.string "nom", null: false + t.string "organisme" + t.string "telephone" + t.string "type_organisme", null: false + t.datetime "updated_at", null: false t.index ["administrateur_id", "nom"], name: "index_services_on_administrateur_id_and_nom", unique: true t.index ["administrateur_id"], name: "index_services_on_administrateur_id" end create_table "stats", force: :cascade do |t| - t.bigint "dossiers_not_brouillon", default: 0 + t.bigint "administrations_partenaires", default: 0 + t.datetime "created_at", precision: 6, null: false t.bigint "dossiers_brouillon", default: 0 - t.bigint "dossiers_en_construction", default: 0 - t.bigint "dossiers_en_instruction", default: 0 - t.bigint "dossiers_termines", default: 0 + t.jsonb "dossiers_cumulative", default: "{}", null: false t.bigint "dossiers_depose_avant_30_jours", default: 0 t.bigint "dossiers_deposes_entre_60_et_30_jours", default: 0 - t.bigint "administrations_partenaires", default: 0 - t.jsonb "dossiers_cumulative", default: "{}", null: false + t.bigint "dossiers_en_construction", default: 0 + t.bigint "dossiers_en_instruction", default: 0 t.jsonb "dossiers_in_the_last_4_months", default: "{}", null: false - t.datetime "created_at", precision: 6, null: false + t.bigint "dossiers_not_brouillon", default: 0 + t.bigint "dossiers_termines", default: 0 t.datetime "updated_at", precision: 6, null: false end create_table "super_admins", id: :serial, force: :cascade do |t| - t.string "email", default: "", null: false - t.string "encrypted_password", default: "", null: false - t.string "reset_password_token" - t.datetime "reset_password_sent_at" - t.datetime "remember_created_at" - t.integer "sign_in_count", default: 0, null: false - t.datetime "current_sign_in_at" - t.datetime "last_sign_in_at" - t.string "current_sign_in_ip" - t.string "last_sign_in_ip" + t.integer "consumed_timestep" t.datetime "created_at" - t.datetime "updated_at" - t.integer "failed_attempts", default: 0, null: false - t.string "unlock_token" - t.datetime "locked_at" + t.datetime "current_sign_in_at" + t.string "current_sign_in_ip" + t.string "email", default: "", null: false t.string "encrypted_otp_secret" t.string "encrypted_otp_secret_iv" t.string "encrypted_otp_secret_salt" - t.integer "consumed_timestep" + t.string "encrypted_password", default: "", null: false + t.integer "failed_attempts", default: 0, null: false + t.datetime "last_sign_in_at" + t.string "last_sign_in_ip" + t.datetime "locked_at" t.boolean "otp_required_for_login" + t.datetime "remember_created_at" + t.datetime "reset_password_sent_at" + t.string "reset_password_token" + t.integer "sign_in_count", default: 0, null: false + t.string "unlock_token" + t.datetime "updated_at" t.index ["email"], name: "index_super_admins_on_email", unique: true t.index ["reset_password_token"], name: "index_super_admins_on_reset_password_token", unique: true t.index ["unlock_token"], name: "index_super_admins_on_unlock_token", unique: true @@ -751,39 +749,38 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do create_table "traitements", force: :cascade do |t| t.bigint "dossier_id" - t.string "motivation" - t.string "state" - t.datetime "processed_at" t.string "instructeur_email" + t.string "motivation" t.boolean "process_expired" t.boolean "process_expired_migrated", default: false + t.datetime "processed_at" + t.string "state" t.index ["dossier_id"], name: "index_traitements_on_dossier_id" t.index ["process_expired"], name: "index_traitements_on_process_expired" end create_table "trusted_device_tokens", force: :cascade do |t| - t.string "token", null: false - t.bigint "instructeur_id" t.datetime "created_at", null: false + t.bigint "instructeur_id" + t.string "token", null: false t.datetime "updated_at", null: false t.index ["instructeur_id"], name: "index_trusted_device_tokens_on_instructeur_id" - t.index ["token"], name: "index_trusted_device_tokens_on_token", unique: true end create_table "types_de_champ", id: :serial, force: :cascade do |t| - t.string "libelle" - t.string "type_champ" - t.integer "order_place" - t.text "description" - t.boolean "mandatory", default: false - t.boolean "private", default: false, null: false t.datetime "created_at" - t.datetime "updated_at" - t.jsonb "options" - t.bigint "stable_id" - t.bigint "parent_id" - t.bigint "revision_id" + t.text "description" + t.string "libelle" + t.boolean "mandatory", default: false t.boolean "migrated_parent" + t.jsonb "options" + t.integer "order_place" + t.bigint "parent_id" + t.boolean "private", default: false, null: false + t.bigint "revision_id" + t.bigint "stable_id" + t.string "type_champ" + t.datetime "updated_at" t.index ["parent_id"], name: "index_types_de_champ_on_parent_id" t.index ["private"], name: "index_types_de_champ_on_private" t.index ["revision_id"], name: "index_types_de_champ_on_revision_id" @@ -791,32 +788,32 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do end create_table "users", id: :serial, force: :cascade do |t| - t.string "email", default: "", null: false - t.string "encrypted_password", default: "", null: false - t.string "reset_password_token" - t.datetime "reset_password_sent_at" - t.datetime "remember_created_at" - t.integer "sign_in_count", default: 0, null: false - t.datetime "current_sign_in_at" - t.datetime "last_sign_in_at" - t.string "current_sign_in_ip" - t.string "last_sign_in_ip" - t.datetime "created_at" - t.datetime "updated_at" - t.string "siret" - t.string "loged_in_with_france_connect", default: "false" + t.bigint "administrateur_id" + t.datetime "confirmation_sent_at" t.string "confirmation_token" t.datetime "confirmed_at" - t.datetime "confirmation_sent_at" - t.text "unconfirmed_email" - t.integer "failed_attempts", default: 0, null: false - t.string "unlock_token" - t.datetime "locked_at" - t.bigint "instructeur_id" - t.bigint "administrateur_id" + t.datetime "created_at" + t.datetime "current_sign_in_at" + t.string "current_sign_in_ip" + t.string "email", default: "", null: false + t.string "encrypted_password", default: "", null: false t.bigint "expert_id" + t.integer "failed_attempts", default: 0, null: false + t.bigint "instructeur_id" + t.datetime "last_sign_in_at" + t.string "last_sign_in_ip" t.string "locale" + t.datetime "locked_at" + t.string "loged_in_with_france_connect", default: "false" + t.datetime "remember_created_at" t.bigint "requested_merge_into_id" + t.datetime "reset_password_sent_at" + t.string "reset_password_token" + t.integer "sign_in_count", default: 0, null: false + t.string "siret" + t.text "unconfirmed_email" + t.string "unlock_token" + t.datetime "updated_at" t.index ["administrateur_id"], name: "index_users_on_administrateur_id" t.index ["confirmation_token"], name: "index_users_on_confirmation_token", unique: true t.index ["email"], name: "index_users_on_email", unique: true @@ -828,28 +825,28 @@ ActiveRecord::Schema.define(version: 2022_03_15_113510) do end create_table "virus_scans", force: :cascade do |t| + t.string "blob_key" + t.bigint "champ_id" + t.datetime "created_at", null: false t.datetime "scanned_at" t.string "status" - t.bigint "champ_id" - t.string "blob_key" - t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["champ_id"], name: "index_virus_scans_on_champ_id" end create_table "without_continuation_mails", id: :serial, force: :cascade do |t| t.text "body" - t.string "subject" - t.integer "procedure_id" t.datetime "created_at", null: false + t.integer "procedure_id" + t.string "subject" t.datetime "updated_at", null: false t.index ["procedure_id"], name: "index_without_continuation_mails_on_procedure_id" end create_table "zones", force: :cascade do |t| t.string "acronym", null: false - t.string "label" t.datetime "created_at", precision: 6, null: false + t.string "label" t.datetime "updated_at", precision: 6, null: false t.index ["acronym"], name: "index_zones_on_acronym", unique: true end diff --git a/doc/object-storange-and-data-encryption.md b/doc/object-storange-and-data-encryption.md new file mode 100644 index 000000000..f3bf4a395 --- /dev/null +++ b/doc/object-storange-and-data-encryption.md @@ -0,0 +1,85 @@ +# Object Storange And Data Encryption + +## Object Storage + +By default, demarches-simplifiees.fr uses an [OVH Object Storage](https://www.ovhcloud.com/en/public-cloud/object-storage/) backend. The hard-drives are encrypted at rest, but to protect user files even better, demarches-simplifiees.fr can also use an external encryption proxy, that will encrypt and decrypt files on the fly: + +* Encryption is done via our [proxy](https://github.com/betagouv/ds_proxy) when the file is uploaded by a client. +* Decryption is done via the same proxy when the file is downloaded to a client + +### Object Storage limitation + +As an s3 compatible object storage backend, OVH Object Storage suffers the same limitations. + +One of them being that when you upload a file bigger than 5Go, it must be chunked into segments (see the [documentation](https://docs.ovh.com/fr/storage/pcs/capabilities-and-limitations/#max_file_size-5368709122-5gb)). + +This process to chunks the file in segment, then re-arrange it via a manifest. Unfortunately encryption can't work with this usecase. + +So we are using a custom script that wraps two call to our proxy in order to buffer all the chunks, encrypt/decrypt the whole. Here is an example + +``` +#!/usr/bin/env bash +# wrapper script to encrypt and upload file received from archive + +set -o errexit +set -o pipefail +set -o nounset + +# params +# 1: filename +# 2: key +if ! [ "$#" -eq 2 ]; then + echo "usage: $0 " + exit 1 +fi +local_file_path=$1 +remote_basename=$(basename $local_file_path) +key=$2 + +# encrypt +curl -s -XPUT http://ds_proxy_host:ds_proxy_port/local/encrypt/${remote_basename} --data-binary @${local_file_path} + +# get back encrypted file +encrypted_filename="${local_file_path}.enc" +curl -s http://ds_proxy_host:ds_proxy_port/local/encrypt/${remote_basename} -o ${encrypted_filename} + +# OVH openstack params +os_tenant_name=os_tenant_name +os_username=os_username +os_password=os_password +os_region_name=GRA + +# auth = https://auth.cloud.ovh.net/v3/ +# use haproxy proxy and not direct internet URL +os_auth_url="os_auth_url" +os_storage_url="os_storage_url" \ +container_name=container_name + +expiring_delay="$((60 * 60 * 24 * 4))" # 4 days + +# upload +/usr/local/bin/swift \ + --auth-version 3 \ + --os-auth-url "$os_auth_url" \ + --os-storage-url "$os_storage_url" \ + --os-region-name "$os_region_name" \ + --os-tenant-name "$os_tenant_name" \ + --os-username "$os_username" \ + --os-password "$os_password" \ + upload \ + --header "X-Delete-After: ${expiring_delay}" \ + --segment-size "$((3 * 1024 * 1024 * 1024))" \ + --header "Content-Disposition: filename=${remote_basename}" \ + --object-name "${key}" \ + "${container_name}" "${encrypted_filename}" + +swift_exit_code=$? + +# cleanup +rm ${encrypted_filename} + +# return swift return code +exit ${swift_exit_code} +``` + + diff --git a/spec/lib/database/migration_helpers_spec.rb b/spec/lib/database/migration_helpers_spec.rb index 1ca896cfb..033b4b41a 100644 --- a/spec/lib/database/migration_helpers_spec.rb +++ b/spec/lib/database/migration_helpers_spec.rb @@ -5,11 +5,7 @@ describe Database::MigrationHelpers do before(:all) do ActiveRecord::Migration.suppress_messages do - ActiveRecord::Migration.create_table "test_labels", force: true do |t| - t.string :label - t.integer :user_id - end - ActiveRecord::Migration.create_table "test_labels", force: true do |t| + ActiveRecord::Migration.create_table "test_labels" do |t| t.string :label t.integer :user_id end @@ -103,13 +99,13 @@ describe Database::MigrationHelpers do before(:all) do ActiveRecord::Migration.suppress_messages do - ActiveRecord::Migration.create_table "test_physicians", force: true do |t| + ActiveRecord::Migration.create_table "test_physicians" do |t| t.string :name end - ActiveRecord::Migration.create_table "test_patients", force: true do |t| + ActiveRecord::Migration.create_table "test_patients" do |t| t.string :name end - ActiveRecord::Migration.create_table "test_appointments", id: false, force: true do |t| + ActiveRecord::Migration.create_table "test_appointments", id: false do |t| t.integer :test_physician_id t.integer :test_patient_id t.datetime :datetime diff --git a/spec/services/archive_uploader_spec.rb b/spec/services/archive_uploader_spec.rb new file mode 100644 index 000000000..c4b4bcfac --- /dev/null +++ b/spec/services/archive_uploader_spec.rb @@ -0,0 +1,70 @@ +describe ProcedureArchiveService do + let(:procedure) { build(:procedure) } + let(:archive) { create(:archive) } + let(:file) { Tempfile.new } + let(:fixture_blob) { ActiveStorage::Blob.create_before_direct_upload!(filename: File.basename(file.path), byte_size: file.size, checksum: 'osf') } + + let(:uploader) { ArchiveUploader.new(procedure: procedure, archive: archive, filepath: file.path) } + + describe '.upload' do + context 'when active storage service is local' do + it 'uploads with upload_with_active_storage' do + expect(uploader).to receive(:active_storage_service_local?).and_return(true) + expect(uploader).to receive(:upload_with_active_storage).and_return(fixture_blob) + uploader.upload + end + + it 'link the created blob as an attachment to the current archive instance' do + expect { uploader.upload } + .to change { ActiveStorage::Attachment.where(name: 'file', record_type: 'Archive', record_id: archive.id).count }.by(1) + end + end + + context 'when active storage service is not local' do + before do + expect(uploader).to receive(:active_storage_service_local?).and_return(false) + expect(File).to receive(:size).with(file.path).and_return(filesize) + end + + context 'when file is smaller than MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING' do + let(:filesize) { ArchiveUploader::MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING - 1 } + + it 'uploads with upload_with_active_storage' do + expect(uploader).to receive(:upload_with_active_storage).and_return(fixture_blob) + uploader.upload + end + end + + context 'when file is bigger than MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING' do + let(:filesize) { ArchiveUploader::MAX_FILE_SIZE_FOR_BACKEND_BEFORE_CHUNKING + 1 } + + it 'uploads with upload_with_chunking_wrapper' do + expect(uploader).to receive(:upload_with_chunking_wrapper).and_return(fixture_blob) + uploader.upload + end + + it 'link the created blob as an attachment to the current archive instance' do + expect(uploader).to receive(:upload_with_chunking_wrapper).and_return(fixture_blob) + expect { uploader.upload } + .to change { ActiveStorage::Attachment.where(name: 'file', record_type: 'Archive', record_id: archive.id).count }.by(1) + end + end + end + end + + describe '.upload_with_chunking_wrapper' do + let(:fake_blob_checksum) { Digest::SHA256.file(file.path) } + let(:fake_blob_bytesize) { 100.gigabytes } + + before do + expect(uploader).to receive(:syscall_to_custom_uploader).and_return(true) + expect(File).to receive(:size).with(file.path).and_return(fake_blob_bytesize) + expect(Digest::SHA256).to receive(:file).with(file.path).and_return(double(hexdigest: fake_blob_checksum.hexdigest)) + end + + it 'creates a blob' do + expect { uploader.send(:upload_with_chunking_wrapper) } + .to change { ActiveStorage::Blob.where(checksum: fake_blob_checksum.hexdigest, byte_size: fake_blob_bytesize).count }.by(1) + end + end +end