diff --git a/Gemfile b/Gemfile index ee2bc8fc2..708603839 100644 --- a/Gemfile +++ b/Gemfile @@ -84,6 +84,7 @@ gem 'spreadsheet_architect' gem 'strong_migrations' # lint database migrations gem 'turbo-rails' gem 'typhoeus' +gem 'ulid-ruby', require: 'ulid' gem 'view_component' gem 'vite_rails' gem 'warden' diff --git a/Gemfile.lock b/Gemfile.lock index 12840698a..185a37b6a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -726,6 +726,7 @@ GEM ethon (>= 0.9.0) tzinfo (2.0.5) concurrent-ruby (~> 1.0) + ulid-ruby (1.0.2) unf (0.1.4) unf_ext unf_ext (0.0.7.7) @@ -915,6 +916,7 @@ DEPENDENCIES timecop turbo-rails typhoeus + ulid-ruby vcr view_component vite_rails diff --git a/app/jobs/migrations/backfill_row_id_job.rb b/app/jobs/migrations/backfill_row_id_job.rb new file mode 100644 index 000000000..30abf408c --- /dev/null +++ b/app/jobs/migrations/backfill_row_id_job.rb @@ -0,0 +1,7 @@ +class Migrations::BackfillRowIdJob < ApplicationJob + def perform(batch) + batch.each do |(row_id, champ_ids)| + Champ.where(id: champ_ids).update_all(row_id:) + end + end +end diff --git a/app/models/champ.rb b/app/models/champ.rb index 63cfe1bac..50d4bf3b2 100644 --- a/app/models/champ.rb +++ b/app/models/champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champ < ApplicationRecord @@ -221,7 +222,7 @@ class Champ < ApplicationRecord end def clone - champ_attributes = [:parent_id, :private, :row, :type, :type_de_champ_id] + champ_attributes = [:parent_id, :private, :row, :row_id, :type, :type_de_champ_id] value_attributes = private? ? [] : [:value, :value_json, :data, :external_id] relationships = private? ? [] : [:etablissement, :geo_areas] diff --git a/app/models/champs/address_champ.rb b/app/models/champs/address_champ.rb index 51c474432..640701e3c 100644 --- a/app/models/champs/address_champ.rb +++ b/app/models/champs/address_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::AddressChamp < Champs::TextChamp diff --git a/app/models/champs/annuaire_education_champ.rb b/app/models/champs/annuaire_education_champ.rb index 4a5504062..1befb2d53 100644 --- a/app/models/champs/annuaire_education_champ.rb +++ b/app/models/champs/annuaire_education_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::AnnuaireEducationChamp < Champs::TextChamp diff --git a/app/models/champs/carte_champ.rb b/app/models/champs/carte_champ.rb index 2a2827b2e..3493753f5 100644 --- a/app/models/champs/carte_champ.rb +++ b/app/models/champs/carte_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::CarteChamp < Champ diff --git a/app/models/champs/checkbox_champ.rb b/app/models/champs/checkbox_champ.rb index 521e188fc..52d579f7d 100644 --- a/app/models/champs/checkbox_champ.rb +++ b/app/models/champs/checkbox_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::CheckboxChamp < Champs::BooleanChamp diff --git a/app/models/champs/civilite_champ.rb b/app/models/champs/civilite_champ.rb index 7f1fa39a4..a8810418b 100644 --- a/app/models/champs/civilite_champ.rb +++ b/app/models/champs/civilite_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::CiviliteChamp < Champ diff --git a/app/models/champs/cnaf_champ.rb b/app/models/champs/cnaf_champ.rb index 56a82695f..3185a45ee 100644 --- a/app/models/champs/cnaf_champ.rb +++ b/app/models/champs/cnaf_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::CnafChamp < Champs::TextChamp diff --git a/app/models/champs/commune_champ.rb b/app/models/champs/commune_champ.rb index 68ed289e4..0a65c703d 100644 --- a/app/models/champs/commune_champ.rb +++ b/app/models/champs/commune_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::CommuneChamp < Champs::TextChamp diff --git a/app/models/champs/date_champ.rb b/app/models/champs/date_champ.rb index cadf77e21..18510a0af 100644 --- a/app/models/champs/date_champ.rb +++ b/app/models/champs/date_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::DateChamp < Champ diff --git a/app/models/champs/datetime_champ.rb b/app/models/champs/datetime_champ.rb index 6fbe10110..fccde406f 100644 --- a/app/models/champs/datetime_champ.rb +++ b/app/models/champs/datetime_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::DatetimeChamp < Champ diff --git a/app/models/champs/decimal_number_champ.rb b/app/models/champs/decimal_number_champ.rb index df3b06d02..f068afa3b 100644 --- a/app/models/champs/decimal_number_champ.rb +++ b/app/models/champs/decimal_number_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::DecimalNumberChamp < Champ diff --git a/app/models/champs/departement_champ.rb b/app/models/champs/departement_champ.rb index 61358021c..1a888139e 100644 --- a/app/models/champs/departement_champ.rb +++ b/app/models/champs/departement_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::DepartementChamp < Champs::TextChamp diff --git a/app/models/champs/dgfip_champ.rb b/app/models/champs/dgfip_champ.rb index 34d6c4f90..c69568289 100644 --- a/app/models/champs/dgfip_champ.rb +++ b/app/models/champs/dgfip_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::DgfipChamp < Champs::TextChamp diff --git a/app/models/champs/dossier_link_champ.rb b/app/models/champs/dossier_link_champ.rb index bcfe09dee..e0922108e 100644 --- a/app/models/champs/dossier_link_champ.rb +++ b/app/models/champs/dossier_link_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::DossierLinkChamp < Champ diff --git a/app/models/champs/drop_down_list_champ.rb b/app/models/champs/drop_down_list_champ.rb index 9e14a551d..8e0342efd 100644 --- a/app/models/champs/drop_down_list_champ.rb +++ b/app/models/champs/drop_down_list_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::DropDownListChamp < Champ diff --git a/app/models/champs/email_champ.rb b/app/models/champs/email_champ.rb index 422ffdc92..725e992d8 100644 --- a/app/models/champs/email_champ.rb +++ b/app/models/champs/email_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::EmailChamp < Champs::TextChamp diff --git a/app/models/champs/explication_champ.rb b/app/models/champs/explication_champ.rb index beda61f52..1c7483cf1 100644 --- a/app/models/champs/explication_champ.rb +++ b/app/models/champs/explication_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::ExplicationChamp < Champs::TextChamp diff --git a/app/models/champs/header_section_champ.rb b/app/models/champs/header_section_champ.rb index 08977d9f6..d2193d2e0 100644 --- a/app/models/champs/header_section_champ.rb +++ b/app/models/champs/header_section_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::HeaderSectionChamp < Champ diff --git a/app/models/champs/iban_champ.rb b/app/models/champs/iban_champ.rb index 239e1e812..457e66843 100644 --- a/app/models/champs/iban_champ.rb +++ b/app/models/champs/iban_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::IbanChamp < Champ diff --git a/app/models/champs/integer_number_champ.rb b/app/models/champs/integer_number_champ.rb index 8e22cb62f..3b1d04ac8 100644 --- a/app/models/champs/integer_number_champ.rb +++ b/app/models/champs/integer_number_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::IntegerNumberChamp < Champ diff --git a/app/models/champs/linked_drop_down_list_champ.rb b/app/models/champs/linked_drop_down_list_champ.rb index c582390fe..3861a7325 100644 --- a/app/models/champs/linked_drop_down_list_champ.rb +++ b/app/models/champs/linked_drop_down_list_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::LinkedDropDownListChamp < Champ diff --git a/app/models/champs/mesri_champ.rb b/app/models/champs/mesri_champ.rb index 2945da743..937932c7a 100644 --- a/app/models/champs/mesri_champ.rb +++ b/app/models/champs/mesri_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::MesriChamp < Champs::TextChamp diff --git a/app/models/champs/multiple_drop_down_list_champ.rb b/app/models/champs/multiple_drop_down_list_champ.rb index 6376635f7..96c442193 100644 --- a/app/models/champs/multiple_drop_down_list_champ.rb +++ b/app/models/champs/multiple_drop_down_list_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::MultipleDropDownListChamp < Champ diff --git a/app/models/champs/number_champ.rb b/app/models/champs/number_champ.rb index a43bd6347..7e23e3176 100644 --- a/app/models/champs/number_champ.rb +++ b/app/models/champs/number_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::NumberChamp < Champ diff --git a/app/models/champs/pays_champ.rb b/app/models/champs/pays_champ.rb index 2da10cff4..751afa7a5 100644 --- a/app/models/champs/pays_champ.rb +++ b/app/models/champs/pays_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::PaysChamp < Champs::TextChamp diff --git a/app/models/champs/phone_champ.rb b/app/models/champs/phone_champ.rb index a5e9e712f..160f746f3 100644 --- a/app/models/champs/phone_champ.rb +++ b/app/models/champs/phone_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::PhoneChamp < Champs::TextChamp diff --git a/app/models/champs/piece_justificative_champ.rb b/app/models/champs/piece_justificative_champ.rb index 4ae2e7fc2..1683e9362 100644 --- a/app/models/champs/piece_justificative_champ.rb +++ b/app/models/champs/piece_justificative_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::PieceJustificativeChamp < Champ diff --git a/app/models/champs/pole_emploi_champ.rb b/app/models/champs/pole_emploi_champ.rb index 3008d02cf..33874cf36 100644 --- a/app/models/champs/pole_emploi_champ.rb +++ b/app/models/champs/pole_emploi_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::PoleEmploiChamp < Champs::TextChamp diff --git a/app/models/champs/region_champ.rb b/app/models/champs/region_champ.rb index 3426b337c..2bdb0b182 100644 --- a/app/models/champs/region_champ.rb +++ b/app/models/champs/region_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::RegionChamp < Champs::TextChamp diff --git a/app/models/champs/repetition_champ.rb b/app/models/champs/repetition_champ.rb index c6d5b50fe..bb8c0782b 100644 --- a/app/models/champs/repetition_champ.rb +++ b/app/models/champs/repetition_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::RepetitionChamp < Champ @@ -32,8 +33,9 @@ class Champs::RepetitionChamp < Champ added_champs = [] transaction do row = (blank? ? -1 : champs.last.row) + 1 + row_id = ULID.generate revision.children_of(type_de_champ).each do |type_de_champ| - added_champs << type_de_champ.champ.build(row: row) + added_champs << type_de_champ.champ.build(row:, row_id:) end self.champs << added_champs end diff --git a/app/models/champs/rna_champ.rb b/app/models/champs/rna_champ.rb index aea1ebcc0..d3e5507fe 100644 --- a/app/models/champs/rna_champ.rb +++ b/app/models/champs/rna_champ.rb @@ -19,6 +19,7 @@ # external_id :string # parent_id :bigint # type_de_champ_id :integer not null +# row_id :string # class Champs::RNAChamp < Champ validates :value, allow_blank: true, format: { diff --git a/app/models/champs/siret_champ.rb b/app/models/champs/siret_champ.rb index 37bc6a9e7..581cacff5 100644 --- a/app/models/champs/siret_champ.rb +++ b/app/models/champs/siret_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::SiretChamp < Champ diff --git a/app/models/champs/text_champ.rb b/app/models/champs/text_champ.rb index aaf83a9f2..336ae75a3 100644 --- a/app/models/champs/text_champ.rb +++ b/app/models/champs/text_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::TextChamp < Champ diff --git a/app/models/champs/textarea_champ.rb b/app/models/champs/textarea_champ.rb index b26cc5494..d5f34efcd 100644 --- a/app/models/champs/textarea_champ.rb +++ b/app/models/champs/textarea_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::TextareaChamp < Champs::TextChamp diff --git a/app/models/champs/titre_identite_champ.rb b/app/models/champs/titre_identite_champ.rb index 196546e24..bfd2e9c16 100644 --- a/app/models/champs/titre_identite_champ.rb +++ b/app/models/champs/titre_identite_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::TitreIdentiteChamp < Champ diff --git a/app/models/champs/yes_no_champ.rb b/app/models/champs/yes_no_champ.rb index 482c3087f..77c80a3db 100644 --- a/app/models/champs/yes_no_champ.rb +++ b/app/models/champs/yes_no_champ.rb @@ -18,6 +18,7 @@ # etablissement_id :integer # external_id :string # parent_id :bigint +# row_id :string # type_de_champ_id :integer # class Champs::YesNoChamp < Champs::BooleanChamp diff --git a/app/models/concerns/dossier_rebase_concern.rb b/app/models/concerns/dossier_rebase_concern.rb index 92e7d338b..8e4c192ca 100644 --- a/app/models/concerns/dossier_rebase_concern.rb +++ b/app/models/concerns/dossier_rebase_concern.rb @@ -112,8 +112,8 @@ module DossierRebaseConcern .where(type_de_champ: { stable_id: parent_stable_id }) champs_repetition.each do |champ_repetition| - champ_repetition.champs.map(&:row).uniq.each do |row| - create_champ(target_coordinate, champ_repetition, row: row) + champ_repetition.champs.index_by(&:row).each do |(row, champ)| + create_champ(target_coordinate, champ_repetition, row:, row_id: champ.row_id) end end else @@ -121,9 +121,8 @@ module DossierRebaseConcern end end - def create_champ(target_coordinate, parent, row: nil) - params = { revision: target_coordinate.revision } - params[:row] = row if row + def create_champ(target_coordinate, parent, row: nil, row_id: nil) + params = { revision: target_coordinate.revision, row:, row_id: }.compact champ = target_coordinate .type_de_champ .build_champ(params) diff --git a/db/migrate/20221215131001_add_row_id_to_champs.rb b/db/migrate/20221215131001_add_row_id_to_champs.rb new file mode 100644 index 000000000..b3f034ea8 --- /dev/null +++ b/db/migrate/20221215131001_add_row_id_to_champs.rb @@ -0,0 +1,5 @@ +class AddRowIdToChamps < ActiveRecord::Migration[6.1] + def change + add_column :champs, :row_id, :string + end +end diff --git a/db/migrate/20221215131122_add_row_id_index_to_champs.rb b/db/migrate/20221215131122_add_row_id_index_to_champs.rb new file mode 100644 index 000000000..deae5352b --- /dev/null +++ b/db/migrate/20221215131122_add_row_id_index_to_champs.rb @@ -0,0 +1,7 @@ +class AddRowIdIndexToChamps < ActiveRecord::Migration[6.1] + disable_ddl_transaction! + + def change + add_index :champs, :row_id, algorithm: :concurrently + end +end diff --git a/db/schema.rb b/db/schema.rb index 1557d1975..262138d5a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -227,6 +227,7 @@ ActiveRecord::Schema.define(version: 2022_12_27_084442) do t.boolean "private", default: false, null: false t.datetime "rebased_at" t.integer "row" + t.string "row_id" t.string "type" t.integer "type_de_champ_id", null: false t.datetime "updated_at" @@ -237,6 +238,7 @@ ActiveRecord::Schema.define(version: 2022_12_27_084442) do t.index ["parent_id"], name: "index_champs_on_parent_id" t.index ["private"], name: "index_champs_on_private" t.index ["row"], name: "index_champs_on_row" + t.index ["row_id"], name: "index_champs_on_row_id" t.index ["type"], name: "index_champs_on_type" t.index ["type_de_champ_id", "dossier_id", "row"], name: "index_champs_on_type_de_champ_id_and_dossier_id_and_row", unique: true t.index ["type_de_champ_id"], name: "index_champs_on_type_de_champ_id" diff --git a/lib/tasks/deployment/20221215135522_backfill_row_id_on_champs.rake b/lib/tasks/deployment/20221215135522_backfill_row_id_on_champs.rake new file mode 100644 index 000000000..4bb96b576 --- /dev/null +++ b/lib/tasks/deployment/20221215135522_backfill_row_id_on_champs.rake @@ -0,0 +1,39 @@ +namespace :after_party do + desc 'Deployment task: backfill_row_id_on_champs' + task backfill_row_id_on_champs: :environment do + puts "Running deploy task 'backfill_row_id_on_champs'" + + champs_to_fill = Champ.where(row_id: nil).where.not(parent_id: nil) + progress = ProgressReport.new(champs_to_fill.count) + + champs = champs_to_fill.pluck(:parent_id, :row, :id) + pp "found #{champs.size} champs to fill" + + row_ids = Champ.where.not(row_id: nil) + .distinct(:row_id) + .pluck(:parent_id, :row, :row_id) + .map { [[_1.first, _1.second], _1.last] }.to_h + pp "found #{row_ids.size} existing row ids" + + champs_by_row = champs.group_by { [_1.first, _1.second] }.transform_values { _1.map(&:last) } + pp "found #{champs_by_row.size} rows to fill" + + champs_by_row.to_a.sort_by(&:first).in_groups_of(5_000) do |batch| + batch = batch.lazy.compact.flat_map do |(key, champs)| + row_ids[key] ||= ULID.generate + champs&.map { [row_ids[key], _1] } || [] + end.group_by(&:first).transform_values { _1.map(&:last) } + + Migrations::BackfillRowIdJob.perform_later(batch) + + progress.inc(batch.size) + end + + progress.finish + + # Update task as completed. If you remove the line below, the task will + # run with every deploy (or every time you call after_party:run). + AfterParty::TaskRecord + .create version: AfterParty::TaskRecorder.new(__FILE__).timestamp + end +end