Merge pull request #8295 from tchak/refactor-repetitions-row-id

refactor(repetition): add row_id to champs

Gemfile

@@ -84,6 +84,7 @@ gem 'spreadsheet_architect'
 gem 'strong_migrations' # lint database migrations
 gem 'turbo-rails'
 gem 'typhoeus'
+gem 'ulid-ruby', require: 'ulid'
 gem 'view_component'
 gem 'vite_rails'
 gem 'warden'

Gemfile.lock

@@ -726,6 +726,7 @@ GEM
       ethon (>= 0.9.0)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
+    ulid-ruby (1.0.2)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
@@ -915,6 +916,7 @@ DEPENDENCIES
   timecop
   turbo-rails
   typhoeus
+  ulid-ruby
   vcr
   view_component
   vite_rails

app/jobs/migrations/backfill_row_id_job.rb

@@ -0,0 +1,7 @@
+class Migrations::BackfillRowIdJob < ApplicationJob
+  def perform(batch)
+    batch.each do |(row_id, champ_ids)|
+      Champ.where(id: champ_ids).update_all(row_id:)
+    end
+  end
+end

app/models/champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champ < ApplicationRecord
@@ -221,7 +222,7 @@ class Champ < ApplicationRecord
   end
 
   def clone
-    champ_attributes = [:parent_id, :private, :row, :type, :type_de_champ_id]
+    champ_attributes = [:parent_id, :private, :row, :row_id, :type, :type_de_champ_id]
     value_attributes = private? ? [] : [:value, :value_json, :data, :external_id]
     relationships = private? ? [] : [:etablissement, :geo_areas]
 

app/models/champs/address_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::AddressChamp < Champs::TextChamp

app/models/champs/annuaire_education_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::AnnuaireEducationChamp < Champs::TextChamp

app/models/champs/carte_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::CarteChamp < Champ

app/models/champs/checkbox_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::CheckboxChamp < Champs::BooleanChamp

app/models/champs/civilite_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::CiviliteChamp < Champ

app/models/champs/cnaf_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::CnafChamp < Champs::TextChamp

app/models/champs/commune_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::CommuneChamp < Champs::TextChamp

app/models/champs/date_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::DateChamp < Champ

app/models/champs/datetime_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::DatetimeChamp < Champ

app/models/champs/decimal_number_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::DecimalNumberChamp < Champ

app/models/champs/departement_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::DepartementChamp < Champs::TextChamp

app/models/champs/dgfip_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::DgfipChamp < Champs::TextChamp

app/models/champs/dossier_link_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::DossierLinkChamp < Champ

app/models/champs/drop_down_list_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::DropDownListChamp < Champ

app/models/champs/email_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::EmailChamp < Champs::TextChamp

app/models/champs/explication_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::ExplicationChamp < Champs::TextChamp

app/models/champs/header_section_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::HeaderSectionChamp < Champ

app/models/champs/iban_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::IbanChamp < Champ

app/models/champs/integer_number_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::IntegerNumberChamp < Champ

app/models/champs/linked_drop_down_list_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::LinkedDropDownListChamp < Champ

app/models/champs/mesri_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::MesriChamp < Champs::TextChamp

app/models/champs/multiple_drop_down_list_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::MultipleDropDownListChamp < Champ

app/models/champs/number_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::NumberChamp < Champ

app/models/champs/pays_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::PaysChamp < Champs::TextChamp

app/models/champs/phone_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::PhoneChamp < Champs::TextChamp

app/models/champs/piece_justificative_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::PieceJustificativeChamp < Champ

app/models/champs/pole_emploi_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::PoleEmploiChamp < Champs::TextChamp

app/models/champs/region_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::RegionChamp < Champs::TextChamp

app/models/champs/repetition_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::RepetitionChamp < Champ
@@ -32,8 +33,9 @@ class Champs::RepetitionChamp < Champ
     added_champs = []
     transaction do
       row = (blank? ? -1 : champs.last.row) + 1
+      row_id = ULID.generate
       revision.children_of(type_de_champ).each do |type_de_champ|
-        added_champs << type_de_champ.champ.build(row: row)
+        added_champs << type_de_champ.champ.build(row:, row_id:)
       end
       self.champs << added_champs
     end

app/models/champs/rna_champ.rb

@@ -19,6 +19,7 @@
 #  external_id                    :string
 #  parent_id                      :bigint
 #  type_de_champ_id               :integer          not null
+#  row_id                         :string
 #
 class Champs::RNAChamp < Champ
   validates :value, allow_blank: true, format: {

app/models/champs/siret_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::SiretChamp < Champ

app/models/champs/text_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::TextChamp < Champ

app/models/champs/textarea_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::TextareaChamp < Champs::TextChamp

app/models/champs/titre_identite_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::TitreIdentiteChamp < Champ

app/models/champs/yes_no_champ.rb

@@ -18,6 +18,7 @@
 #  etablissement_id               :integer
 #  external_id                    :string
 #  parent_id                      :bigint
+#  row_id                         :string
 #  type_de_champ_id               :integer
 #
 class Champs::YesNoChamp < Champs::BooleanChamp

app/models/concerns/dossier_rebase_concern.rb

@@ -112,8 +112,8 @@ module DossierRebaseConcern
         .where(type_de_champ: { stable_id: parent_stable_id })
 
       champs_repetition.each do |champ_repetition|
-        champ_repetition.champs.map(&:row).uniq.each do |row|
+        champ_repetition.champs.index_by(&:row).each do |(row, champ)|
-          create_champ(target_coordinate, champ_repetition, row: row)
+          create_champ(target_coordinate, champ_repetition, row:, row_id: champ.row_id)
         end
       end
     else
@@ -121,9 +121,8 @@ module DossierRebaseConcern
     end
   end
 
-  def create_champ(target_coordinate, parent, row: nil)
+  def create_champ(target_coordinate, parent, row: nil, row_id: nil)
-    params = { revision: target_coordinate.revision }
+    params = { revision: target_coordinate.revision, row:, row_id: }.compact
-    params[:row] = row if row
     champ = target_coordinate
       .type_de_champ
       .build_champ(params)

db/migrate/20221215131001_add_row_id_to_champs.rb

@@ -0,0 +1,5 @@
+class AddRowIdToChamps < ActiveRecord::Migration[6.1]
+  def change
+    add_column :champs, :row_id, :string
+  end
+end

db/migrate/20221215131122_add_row_id_index_to_champs.rb

@@ -0,0 +1,7 @@
+class AddRowIdIndexToChamps < ActiveRecord::Migration[6.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :champs, :row_id, algorithm: :concurrently
+  end
+end

db/schema.rb

@@ -227,6 +227,7 @@ ActiveRecord::Schema.define(version: 2022_12_27_084442) do
     t.boolean "private", default: false, null: false
     t.datetime "rebased_at"
     t.integer "row"
+    t.string "row_id"
     t.string "type"
     t.integer "type_de_champ_id", null: false
     t.datetime "updated_at"
@@ -237,6 +238,7 @@ ActiveRecord::Schema.define(version: 2022_12_27_084442) do
     t.index ["parent_id"], name: "index_champs_on_parent_id"
     t.index ["private"], name: "index_champs_on_private"
     t.index ["row"], name: "index_champs_on_row"
+    t.index ["row_id"], name: "index_champs_on_row_id"
     t.index ["type"], name: "index_champs_on_type"
     t.index ["type_de_champ_id", "dossier_id", "row"], name: "index_champs_on_type_de_champ_id_and_dossier_id_and_row", unique: true
     t.index ["type_de_champ_id"], name: "index_champs_on_type_de_champ_id"

lib/tasks/deployment/20221215135522_backfill_row_id_on_champs.rake

@@ -0,0 +1,39 @@
+namespace :after_party do
+  desc 'Deployment task: backfill_row_id_on_champs'
+  task backfill_row_id_on_champs: :environment do
+    puts "Running deploy task 'backfill_row_id_on_champs'"
+
+    champs_to_fill = Champ.where(row_id: nil).where.not(parent_id: nil)
+    progress = ProgressReport.new(champs_to_fill.count)
+
+    champs = champs_to_fill.pluck(:parent_id, :row, :id)
+    pp "found #{champs.size} champs to fill"
+
+    row_ids = Champ.where.not(row_id: nil)
+      .distinct(:row_id)
+      .pluck(:parent_id, :row, :row_id)
+      .map { [[_1.first, _1.second], _1.last] }.to_h
+    pp "found #{row_ids.size} existing row ids"
+
+    champs_by_row = champs.group_by { [_1.first, _1.second] }.transform_values { _1.map(&:last) }
+    pp "found #{champs_by_row.size} rows to fill"
+
+    champs_by_row.to_a.sort_by(&:first).in_groups_of(5_000) do |batch|
+      batch = batch.lazy.compact.flat_map do |(key, champs)|
+        row_ids[key] ||= ULID.generate
+        champs&.map { [row_ids[key], _1] } || []
+      end.group_by(&:first).transform_values { _1.map(&:last) }
+
+      Migrations::BackfillRowIdJob.perform_later(batch)
+
+      progress.inc(batch.size)
+    end
+
+    progress.finish
+
+    # Update task as completed.  If you remove the line below, the task will
+    # run with every deploy (or every time you call after_party:run).
+    AfterParty::TaskRecord
+      .create version: AfterParty::TaskRecorder.new(__FILE__).timestamp
+  end
+end