feat(Maintenance.communes): backfill missing external_id for communes champs in error

app/tasks/maintenance/fix_champs_commune_having_value_but_not_external_id_task.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Maintenance
+  # some of our Champs::CommuneChamp had been corrupted, ie: missing external_id
+  # this tasks fix this issue
+  class FixChampsCommuneHavingValueButNotExternalIdTask < MaintenanceTasks::Task
+    DEFAULT_INSTRUCTEUR_EMAIL = ENV.fetch('DEFAULT_INSTRUCTEUR_EMAIL') { CONTACT_EMAIL }
+
+    def collection
+      Champs::CommuneChamp.where.not(value: nil)
+    end
+
+    def process(champ)
+      return if !fixable?(champ)
+
+      response = APIGeoService.commune_by_name_or_postal_code(champ.value)
+      if !response.success?
+        notify("Strange case of existing commune not requestable", champ)
+      else
+        results = JSON.parse(response.body, symbolize_names: true)
+        formated_results = APIGeoService.format_commune_response(results, true)
+        case formated_results.size
+        when 1
+          champ.code = formated_results.first[:value]
+          champ.save!
+        else # otherwise, we can't find the expected departement
+          champ.code_departement = nil
+          champ.code_postal = nil
+          champ.external_id = nil
+          champ.value = nil
+          champ.save(validate: false)
+
+          ask_user_correction(champ)
+        end
+      end
+    end
+
+    def count
+      # 2.4M champs, count is not an option
+    end
+
+    private
+
+    def ask_user_correction(champ)
+      dossier = champ.dossier
+
+      commentaire = CommentaireService.build(current_instructeur, dossier, { body: "Suite à un problème technique, Veuillez re-remplir le champs : #{champ.libelle}" })
+      dossier.flag_as_pending_correction!(commentaire, :incomplete)
+    end
+
+    def current_instructeur
+      user = User.find_by(email: DEFAULT_INSTRUCTEUR_EMAIL)
+      user ||= User.create(email: DEFAULT_INSTRUCTEUR_EMAIL,
+                           password: Random.srand,
+                           confirmed_at: Time.zone.now,
+                           email_verified_at: Time.zone.now)
+      instructeur = user.instructeur
+      instructeur ||= user.create_instructeur!
+
+      instructeur
+    end
+
+    def fixable?(champ)
+      champ.value.present? && [champ.dossier.en_instruction? || champ.dossier.en_construction?]
+    end
+
+    def notify(message, champ) = Sentry.capture_message(message, extra: { champ: })
+  end
+end

spec/fixtures/cassettes/fix-champs-commune-with-more-than-one-results.yml

@@ -0,0 +1,41 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://geo.api.gouv.fr/communes?boost=population&limit=100&nom=Marseille&type=commune-actuelle,arrondissement-municipal
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Server:
+      - nginx/1.10.3 (Ubuntu)
+      Date:
+      - Fri, 06 Sep 2024 07:25:27 GMT
+      Content-Type:
+      - application/json; charset=utf-8
+      Content-Length:
+      - '3583'
+      Vary:
+      - Accept-Encoding
+      - Origin
+      X-Powered-By:
+      - Express
+      Etag:
+      - W/"dff-vG1Dx3YDZB27TTb5VLsh9AEwHbw"
+      Strict-Transport-Security:
+      - max-age=15552000
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        W3sibm9tIjoiTWFyc2VpbGxlIiwiY29kZSI6IjEzMDU1IiwiY29kZURlcGFydGVtZW50IjoiMTMiLCJzaXJlbiI6IjIxMTMwMDU1MyIsImNvZGVFcGNpIjoiMjAwMDU0ODA3IiwiY29kZVJlZ2lvbiI6IjkzIiwiY29kZXNQb3N0YXV4IjpbIjEzMDAxIiwiMTMwMDIiLCIxMzAwMyIsIjEzMDA0IiwiMTMwMDUiLCIxMzAwNiIsIjEzMDA3IiwiMTMwMDgiLCIxMzAwOSIsIjEzMDEwIiwiMTMwMTEiLCIxMzAxMiIsIjEzMDEzIiwiMTMwMTQiLCIxMzAxNSIsIjEzMDE2Il0sInBvcHVsYXRpb24iOjg3MzA3NiwiX3Njb3JlIjo0Ljg4ODM3MDE4MzkyOTEwMn0seyJub20iOiJNYXJzZWlsbGV0dGUiLCJjb2RlIjoiMTEyMjAiLCJjb2RlRGVwYXJ0ZW1lbnQiOiIxMSIsInNpcmVuIjoiMjExMTAyMjA3IiwiY29kZUVwY2kiOiIyMDAwMzU3MTUiLCJjb2RlUmVnaW9uIjoiNzYiLCJjb2Rlc1Bvc3RhdXgiOlsiMTE4MDAiXSwicG9wdWxhdGlvbiI6NzAwLCJfc2NvcmUiOjAuNjE1Njg0Njg2ODA1MzMxN30seyJub20iOiJNYXJzZWlsbGUgMTNlIEFycm9uZGlzc2VtZW50IiwiY29kZSI6IjEzMjEzIiwiY29kZURlcGFydGVtZW50IjoiMTMiLCJjb2RlUmVnaW9uIjoiOTMiLCJjb2Rlc1Bvc3RhdXgiOlsiMTMwMTMiXSwicG9wdWxhdGlvbiI6OTIyNjEsIl9zY29yZSI6MC41OTc4NjI1OTk0MDkxODg0fSx7Im5vbSI6Ik1hcnNlaWxsZSA4ZSBBcnJvbmRpc3NlbWVudCIsImNvZGUiOiIxMzIwOCIsImNvZGVEZXBhcnRlbWVudCI6IjEzIiwiY29kZVJlZ2lvbiI6IjkzIiwiY29kZXNQb3N0YXV4IjpbIjEzMDA4Il0sInBvcHVsYXRpb24iOjgyNjA5LCJfc2NvcmUiOjAuNTY3ODQ4MzQ4OTM5Nzg3NX0seyJub20iOiJNYXJzZWlsbGUgMTVlIEFycm9uZGlzc2VtZW50IiwiY29kZSI6IjEzMjE1IiwiY29kZURlcGFydGVtZW50IjoiMTMiLCJjb2RlUmVnaW9uIjoiOTMiLCJjb2Rlc1Bvc3RhdXgiOlsiMTMwMTUiXSwicG9wdWxhdGlvbiI6Nzk2NTYsIl9zY29yZSI6MC41NTg2NjU1ODA0MzIxMDYxfSx7Im5vbSI6Ik1hcnNlaWxsZSA5ZSBBcnJvbmRpc3NlbWVudCIsImNvZGUiOiIxMzIwOSIsImNvZGVEZXBhcnRlbWVudCI6IjEzIiwiY29kZVJlZ2lvbiI6IjkzIiwiY29kZXNQb3N0YXV4IjpbIjEzMDA5Il0sInBvcHVsYXRpb24iOjc3MTA2LCJfc2NvcmUiOjAuNTUwNzM1OTk3MDYxMDk3OH0seyJub20iOiJNYXJzZWlsbGUgMTJlIEFycm9uZGlzc2VtZW50IiwiY29kZSI6IjEzMjEyIiwiY29kZURlcGFydGVtZW50IjoiMTMiLCJjb2RlUmVnaW9uIjoiOTMiLCJjb2Rlc1Bvc3RhdXgiOlsiMTMwMTIiXSwicG9wdWxhdGlvbiI6NjMxMDgsIl9zY29yZSI6MC41MDcyMDcyNDg4MTUwNjg1fSx7Im5vbSI6Ik1hcnNlaWxsZSAxNGUgQXJyb25kaXNzZW1lbnQiLCJjb2RlIjoiMTMyMTQiLCJjb2RlRGVwYXJ0ZW1lbnQiOiIxMyIsImNvZGVSZWdpb24iOiI5MyIsImNvZGVzUG9zdGF1eCI6WyIxMzAxNCJdLCJwb3B1bGF0aW9uIjo1OTk0OCwiX3Njb3JlIjowLjQ5NzM4MDc4NDcxNjA5MzU0fSx7Im5vbSI6Ik1hcnNlaWxsZSAxMGUgQXJyb25kaXNzZW1lbnQiLCJjb2RlIjoiMTMyMTAiLCJjb2RlRGVwYXJ0ZW1lbnQiOiIxMyIsImNvZGVSZWdpb24iOiI5MyIsImNvZGVzUG9zdGF1eCI6WyIxMzAxMCJdLCJwb3B1bGF0aW9uIjo1OTAwMiwiX3Njb3JlIjowLjQ5NDQzOTA2NDc2NzQ3NjM0fSx7Im5vbSI6Ik1hcnNlaWxsZSAxMWUgQXJyb25kaXNzZW1lbnQiLCJjb2RlIjoiMTMyMTEiLCJjb2RlRGVwYXJ0ZW1lbnQiOiIxMyIsImNvZGVSZWdpb24iOiI5MyIsImNvZGVzUG9zdGF1eCI6WyIxMzAxMSJdLCJwb3B1bGF0aW9uIjo1NzkyNCwiX3Njb3JlIjowLjQ5MTA4Njg3MjI2Nzg4OTMzfSx7Im5vbSI6Ik1hcnNlaWxsZSAzZSBBcnJvbmRpc3NlbWVudCIsImNvZGUiOiIxMzIwMyIsImNvZGVEZXBhcnRlbWVudCI6IjEzIiwiY29kZVJlZ2lvbiI6IjkzIiwiY29kZXNQb3N0YXV4IjpbIjEzMDAzIl0sInBvcHVsYXRpb24iOjUzMTE1LCJfc2NvcmUiOjAuNDc2MTMyNjEwOTIyMzI4OTV9LHsibm9tIjoiTWFyc2VpbGxlcy1sw6hzLUF1YmlnbnkiLCJjb2RlIjoiMTgxMzkiLCJjb2RlRGVwYXJ0ZW1lbnQiOiIxOCIsInNpcmVuIjoiMjExODAxMzk0IiwiY29kZUVwY2kiOiIyMDAwMTE3ODEiLCJjb2RlUmVnaW9uIjoiMjQiLCJjb2Rlc1Bvc3RhdXgiOlsiMTgzMjAiXSwicG9wdWxhdGlvbiI6NjQ5LCJfc2NvcmUiOjAuNDcyMzE5NTc0NzcwNjA2OH0seyJub20iOiJNYXJzZWlsbGUgNGUgQXJyb25kaXNzZW1lbnQiLCJjb2RlIjoiMTMyMDQiLCJjb2RlRGVwYXJ0ZW1lbnQiOiIxMyIsImNvZGVSZWdpb24iOiI5MyIsImNvZGVzUG9zdGF1eCI6WyIxMzAwNCJdLCJwb3B1bGF0aW9uIjo1MDA2OCwiX3Njb3JlIjowLjQ2NjY1NzUzNjIwNDEwODR9LHsibm9tIjoiTWFyc2VpbGxlIDVlIEFycm9uZGlzc2VtZW50IiwiY29kZSI6IjEzMjA1IiwiY29kZURlcGFydGVtZW50IjoiMTMiLCJjb2RlUmVnaW9uIjoiOTMiLCJjb2Rlc1Bvc3RhdXgiOlsiMTMwMDUiXSwicG9wdWxhdGlvbiI6NDU0NDksIl9zY29yZSI6MC40NTIyOTQxMDY1NjA3MDE1N30seyJub20iOiJNYXJzZWlsbGUgNmUgQXJyb25kaXNzZW1lbnQiLCJjb2RlIjoiMTMyMDYiLCJjb2RlRGVwYXJ0ZW1lbnQiOiIxMyIsImNvZGVSZWdpb24iOiI5MyIsImNvZGVzUG9zdGF1eCI6WyIxMzAwNiJdLCJwb3B1bGF0aW9uIjozOTY0NywiX3Njb3JlIjowLjQzNDI1MTk3MjE2MTI1NDMzfSx7Im5vbSI6Ik1hcnNlaWxsZSA3ZSBBcnJvbmRpc3NlbWVudCIsImNvZGUiOiIxMzIwNyIsImNvZGVEZXBhcnRlbWVudCI6IjEzIiwiY29kZVJlZ2lvbiI6IjkzIiwiY29kZXNQb3N0YXV4IjpbIjEzMDA3Il0sInBvcHVsYXRpb24iOjM0NjMzLCJfc2NvcmUiOjAuNDE4NjYwMjM0NTA1NDc1N30seyJub20iOiJNYXJzZWlsbGUgMWVyIEFycm9uZGlzc2VtZW50IiwiY29kZSI6IjEzMjAxIiwiY29kZURlcGFydGVtZW50IjoiMTMiLCJjb2RlUmVnaW9uIjoiOTMiLCJjb2Rlc1Bvc3RhdXgiOlsiMTMwMDEiXSwicG9wdWxhdGlvbiI6Mzk0MzYsIl9zY29yZSI6MC4zODc5OTk5NzIyNDQ0ODU1N30seyJub20iOiJNYXJzZWlsbGUgMmUgQXJyb25kaXNzZW1lbnQiLCJjb2RlIjoiMTMyMDIiLCJjb2RlRGVwYXJ0ZW1lbnQiOiIxMyIsImNvZGVSZWdpb24iOiI5MyIsImNvZGVzUG9zdGF1eCI6WyIxMzAwMiJdLCJwb3B1bGF0aW9uIjoyMzYyNywiX3Njb3JlIjowLjM4NDQzNTUzMDc0ODA5NjJ9LHsibm9tIjoiTWFyc2VpbGxlIDE2ZSBBcnJvbmRpc3NlbWVudCIsImNvZGUiOiIxMzIxNiIsImNvZGVEZXBhcnRlbWVudCI6IjEzIiwiY29kZVJlZ2lvbiI6IjkzIiwiY29kZXNQb3N0YXV4IjpbIjEzMDE2Il0sInBvcHVsYXRpb24iOjE1NDg3LCJfc2NvcmUiOjAuMzU5MTIzMDU2NzcxNjIyNjV9LHsibm9tIjoiTWFyc2VpbGxlLWVuLUJlYXV2YWlzaXMiLCJjb2RlIjoiNjAzODciLCJjb2RlRGVwYXJ0ZW1lbnQiOiI2MCIsInNpcmVuIjoiMjE2MDAzODMwIiwiY29kZUVwY2kiOiIyNDYwMDA4NDgiLCJjb2RlUmVnaW9uIjoiMzIiLCJjb2Rlc1Bvc3RhdXgiOlsiNjA2OTAiXSwicG9wdWxhdGlvbiI6MTQ0MywiX3Njb3JlIjowLjMwMTMyNjM4Mzc4MTY5NjZ9XQ==
+  recorded_at: Fri, 06 Sep 2024 07:25:27 GMT
+recorded_with: VCR 6.2.0

spec/fixtures/cassettes/fix-champs-commune-with-one-results.yml

@@ -0,0 +1,41 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://geo.api.gouv.fr/communes?boost=population&limit=100&nom=Coye-la-For%C3%AAt&type=commune-actuelle,arrondissement-municipal
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Server:
+      - nginx/1.10.3 (Ubuntu)
+      Date:
+      - Fri, 06 Sep 2024 07:25:28 GMT
+      Content-Type:
+      - application/json; charset=utf-8
+      Content-Length:
+      - '197'
+      Vary:
+      - Accept-Encoding
+      - Origin
+      X-Powered-By:
+      - Express
+      Etag:
+      - W/"c5-7Mng3tlbwfBH/m/FGIzzGMvfw/8"
+      Strict-Transport-Security:
+      - max-age=15552000
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        W3sibm9tIjoiQ295ZS1sYS1Gb3LDqnQiLCJjb2RlIjoiNjAxNzIiLCJjb2RlRGVwYXJ0ZW1lbnQiOiI2MCIsInNpcmVuIjoiMjE2MDAxNzE5IiwiY29kZUVwY2kiOiIyNDYwMDA3NjQiLCJjb2RlUmVnaW9uIjoiMzIiLCJjb2Rlc1Bvc3RhdXgiOlsiNjA1ODAiXSwicG9wdWxhdGlvbiI6Mzk1MCwiX3Njb3JlIjowLjA3MTQ1MzgwNDc4MTQ5OTQ2fV0=
+  recorded_at: Fri, 06 Sep 2024 07:25:28 GMT
+recorded_with: VCR 6.2.0

spec/tasks/maintenance/fix_champs_commune_having_value_but_not_external_id_task_spec.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+module Maintenance
+  RSpec.describe FixChampsCommuneHavingValueButNotExternalIdTask do
+    describe "#process" do
+      let(:procedure) { create(:procedure, types_de_champ_public: [{ type: :communes }]) }
+      let(:dossier) { create(:dossier, state, :with_populated_champs, procedure:) }
+      let(:champ) { dossier.champs.first }
+      subject(:process) { described_class.process(champ) }
+
+      context 'when search find one result', vcr: { cassette_name: 'fix-champs-commune-with-one-results' } do
+        let(:state) { [:en_instruction, :en_construction].sample }
+        let!(:expected_external_id) { champ.external_id }
+        before { champ.update_column(:external_id, nil) }
+
+        it "backfill external_id" do
+          expect { subject }.to change { champ.reload.external_id }.from(nil).to(expected_external_id)
+        end
+      end
+
+      context 'when search find 0 or more than 1 results', vcr: { cassette_name: 'fix-champs-commune-with-more-than-one-results' } do
+        let(:instructeur) { create(:instructeur, user: create(:user, email: Maintenance::FixChampsCommuneHavingValueButNotExternalIdTask::DEFAULT_INSTRUCTEUR_EMAIL)) }
+        before do
+          instructeur
+          champ.update_columns(external_id: nil, value: 'Marseille') # more than one marseille in france
+        end
+
+        context 'en_instruction (go back to en_construction!), send comment' do
+          let(:state) { [:en_instruction, :en_construction].sample }
+
+          it 'flags as pending correction' do
+            expect { subject }.to change { champ.reload.value }.from('Marseille').to(nil)
+            expect(Commentaire.first.instructeur).to eq(instructeur)
+            expect(champ.dossier.state).to eq("en_construction")
+          end
+        end
+
+        context 'when champs will passthru validator (ie: state is brouillon)' \
+                'or champ belongs to dossier.termine (ie: state is accepte, refuse or classer_sans_suite)' do
+          let(:state) { [:brouillon, :accepte, :refuse, :sans_suite].sample }
+
+          it "skips backfill as well as asks for correction" do
+            expect { subject }.not_to change { champ.reload }
+          end
+        end
+      end
+    end
+  end
+end