diff --git a/app/jobs/migrations/backfill_dossier_repetition_job.rb b/app/jobs/migrations/backfill_dossier_repetition_job.rb index 827777f7a..6c03b5dc5 100644 --- a/app/jobs/migrations/backfill_dossier_repetition_job.rb +++ b/app/jobs/migrations/backfill_dossier_repetition_job.rb @@ -3,9 +3,12 @@ class Migrations::BackfillDossierRepetitionJob < ApplicationJob Dossier.where(id: dossier_ids) .includes(:champs, revision: :types_de_champ) .find_each do |dossier| - dossier.revision + dossier + .revision .types_de_champ - .filter { _1.type_champ == 'repetition' } + .filter do |type_de_champ| + type_de_champ.type_champ == 'repetition' && dossier.champs.none? { _1.type_de_champ_id == type_de_champ.id } + end .each do |type_de_champ| dossier.champs << type_de_champ.champ.build end diff --git a/lib/tasks/deployment/20230201090534_backfill_dossiers_repetitions.rake b/lib/tasks/deployment/20230201090534_backfill_dossiers_repetitions.rake deleted file mode 100644 index fb1a2453a..000000000 --- a/lib/tasks/deployment/20230201090534_backfill_dossiers_repetitions.rake +++ /dev/null @@ -1,29 +0,0 @@ -namespace :after_party do - desc 'Deployment task: backfill_dossiers_repetitions' - task backfill_dossiers_repetitions: :environment do - puts "Running deploy task 'backfill_dossiers_repetitions'" - - revision_ids = ProcedureRevision.joins(:types_de_champ).where(types_de_champ: { type_champ: :repetition }).distinct.pluck(:id) - dossier_ids = Dossier.where(revision_id: revision_ids).pluck(:id) - - progress = ProgressReport.new(dossier_ids.size) - dossier_ids_to_fix = [] - dossier_ids.in_groups_of(10000, false) do |dossier_ids| - dossier_ids_with_repetition = Champ.where(dossier_id: dossier_ids, type: 'Champs::RepetitionChamp').pluck(:dossier_id).uniq - dossier_ids_to_fix += dossier_ids - dossier_ids_with_repetition - progress.inc(10000) - end - progress.finish - - pp "fixing #{dossier_ids_to_fix.size} dossiers" - - dossier_ids_to_fix.in_groups_of(100, false) do |dossier_ids| - Migrations::BackfillDossierRepetitionJob.perform_later(dossier_ids) - end - - # Update task as completed. If you remove the line below, the task will - # run with every deploy (or every time you call after_party:run). - AfterParty::TaskRecord - .create version: AfterParty::TaskRecorder.new(__FILE__).timestamp - end -end diff --git a/lib/tasks/deployment/20230201090535_backfill_dossiers_repetitions.rake b/lib/tasks/deployment/20230201090535_backfill_dossiers_repetitions.rake new file mode 100644 index 000000000..cc525ab8b --- /dev/null +++ b/lib/tasks/deployment/20230201090535_backfill_dossiers_repetitions.rake @@ -0,0 +1,55 @@ +namespace :after_party do + desc 'Deployment task: backfill_dossiers_repetitions' + task backfill_dossiers_repetitions: :environment do + puts "Running deploy task 'backfill_dossiers_repetitions'" + + revision_ids = ProcedureRevision.joins(:types_de_champ).where(types_de_champ: { type_champ: :repetition }).distinct.pluck(:id) + dossier_ids = Dossier.where(revision_id: revision_ids).pluck(:id) + + progress = ProgressReport.new(dossier_ids.size) + batch_size = 7000 + dossier_ids_to_fix = [] + dossier_ids.in_groups_of(batch_size, false) do |dossier_ids| + # map dossier ids to revision ids + revision_id_by_dossier_id = Dossier.where(id: dossier_ids) + .pluck(:id, :revision_id) + .index_by(&:first) + .transform_values(&:last) + + # map revision ids to type de champ repetition ids + type_de_champ_ids_by_revision_id = ProcedureRevisionTypeDeChamp.joins(:type_de_champ) + .where(types_de_champ: { type_champ: :repetition }, revision_id: revision_id_by_dossier_id.values.uniq) + .pluck(:revision_id, :type_de_champ_id) + .uniq + .group_by(&:first) + .transform_values { _1.map(&:last) } + + # find all dossier ids where all repetition type de champ have a champ + dossier_ids_with_all_repetition_champs = Champ.where(dossier_id: dossier_ids, type: 'Champs::RepetitionChamp') + .pluck(:dossier_id, :type_de_champ_id) + .group_by(&:first) + .transform_values { _1.map(&:last) } + .filter_map do |dossier_id, type_de_champ_ids| + revision_id = revision_id_by_dossier_id[dossier_id] + if type_de_champ_ids.size >= type_de_champ_ids_by_revision_id[revision_id].size + dossier_id + end + end + + dossier_ids_to_fix += dossier_ids - dossier_ids_with_all_repetition_champs + progress.inc(batch_size) + end + progress.finish + + pp "fixing #{dossier_ids_to_fix.size} dossiers" + + dossier_ids_to_fix.in_groups_of(300, false) do |dossier_ids| + Migrations::BackfillDossierRepetitionJob.perform_later(dossier_ids) + end + + # Update task as completed. If you remove the line below, the task will + # run with every deploy (or every time you call after_party:run). + AfterParty::TaskRecord + .create version: AfterParty::TaskRecorder.new(__FILE__).timestamp + end +end