This commit is contained in:
Paul Chavard 2023-07-27 15:16:29 +02:00
parent 9c8b015b45
commit 56d3b98007
48 changed files with 193 additions and 46 deletions

View file

@ -1,5 +1,24 @@
= @form.text_field(:value, input_opts(id: @champ.input_id, aria: { describedby: @champ.describedby_id }, placeholder: t(".placeholder"), data: { controller: 'turbo-input', turbo_input_load_on_connect_value: @champ.prefilled? && @champ.value.present? && @champ.etablissement.blank?, turbo_input_url_value: champs_siret_path(@champ.id) }, required: @champ.required?, pattern: "[0-9]{14}", title: t(".title"), class: "width-33-desktop", maxlength: 14))
.spinner.right.hidden
= @form.text_field(:value, input_opts(id: @champ.input_id, aria: { describedby: @champ.describedby_id }, placeholder: t(".placeholder"), required: @champ.required?, pattern: "[0-9]{14}", title: t(".title"), class: "width-33-desktop", maxlength: 14))
- if @champ.fetch_external_data_pending?
.spinner.right
.siret-info{ id: dom_id(@champ, :siret_info) }
- if @champ.etablissement.present?
= render EditableChamp::EtablissementTitreComponent.new(etablissement: @champ.etablissement)
- elsif @champ.fetch_external_data_retryable_exception? || @champ.fetch_external_data_final_exception?
- case @champ.external_data_exceptions.dig(:primary_endpoint, :type)
- when :invalid_length
%p.fr-error-text= t('errors.messages.invalid_siret_length')
- when :invalid_checksum
%p.fr-error-text= t('errors.messages.invalid_siret_checksum')
- when :not_found
%p.fr-error-text
Nous navons pas trouvé détablissement correspondant à ce numéro de SIRET.
= link_to('Plus dinformations', t("links.common.faq.erreur_siret_url"), **external_link_attributes)
- when :network_error
%p.fr-error-text= t('errors.messages.siret_network_error')
- when :api_entreprise_down
%p.fr-error-text= t('errors.messages.api_entreprise_down')

View file

@ -276,6 +276,8 @@ module Users
if errors.present?
flash.now.alert = errors
else
@dossier.champs_public_all.each(&:fetch_external_data_later)
end
respond_to do |format|

View file

@ -13,6 +13,8 @@ class ApplicationJob < ActiveJob::Base
Sentry.set_tags(dossier: arg.id, procedure: arg.procedure.id)
when Procedure
Sentry.set_tags(procedure: arg.id)
when Champ
Sentry.set_tags(champ: arg.id)
end
end

View file

@ -5,7 +5,6 @@ class ChampFetchExternalDataJob < ApplicationJob
return if champ.external_id != external_id
return if champ.data.present?
Sentry.set_tags(champ: champ.id)
Sentry.set_extras(external_id:)
result = champ.fetch_external_data
@ -13,12 +12,17 @@ class ChampFetchExternalDataJob < ApplicationJob
if result.is_a?(Dry::Monads::Result)
case result
in Success(data)
pp "success #{data}"
champ.update_with_external_data!(data:)
in Failure(retryable: true, reason:)
champ.log_fetch_external_data_exception(reason)
throw reason
error = result.failure
error.attempts = executions
pp "retryable failure #{error}"
champ.log_fetch_external_data_exception(error)
raise reason
in Failure(retryable: false, reason:)
champ.log_fetch_external_data_exception(reason)
pp "failure #{result.failure}"
champ.log_fetch_external_data_exception(result.failure)
Sentry.capture_exception(reason)
end
elsif result.present?

View file

@ -29,7 +29,7 @@ class API::Client
end
OK = Data.define(:body, :response)
Error = Data.define(:type, :code, :retryable, :reason)
Error = Data.define(:type, :code, :retryable, :attempts, :reason)
def handle_response(response, schema:)
if response.success?
@ -43,17 +43,17 @@ class API::Client
if !schema || schema.valid?(body)
Success(OK[body.deep_symbolize_keys, response])
else
Failure(Error[:schema, response.code, false, SchemaError.new(schema.validate(body))])
Failure(Error[:schema, response.code, false, 0, SchemaError.new(schema.validate(body))])
end
in Failure(reason)
Failure(Error[:json, response.code, false, reason])
Failure(Error[:json, response.code, false, 0, reason])
end
elsif response.timed_out?
Failure(Error[:timeout, response.code, true, HTTPError.new(response)])
Failure(Error[:timeout, response.code, true, 0, HTTPError.new(response)])
elsif response.code != 0
Failure(Error[:http, response.code, true, HTTPError.new(response)])
Failure(Error[:http, response.code, true, 0, HTTPError.new(response)])
else
Failure(Error[:network, response.code, true, HTTPError.new(response)])
Failure(Error[:network, response.code, true, 0, HTTPError.new(response)])
end
end

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean
# private :boolean default(FALSE), not null
@ -103,7 +104,7 @@ class Champ < ApplicationRecord
before_validation :set_dossier_id, if: :needs_dossier_id?
before_save :cleanup_if_empty
before_save :normalize
after_update_commit :fetch_external_data_later
#after_update_commit :fetch_external_data_later
def public?
!private?
@ -214,32 +215,63 @@ class Champ < ApplicationRecord
"#{html_id}-description" if description.present?
end
def log_fetch_external_data_exception(exception)
exceptions = self.fetch_external_data_exceptions ||= []
exceptions << exception.inspect
update_column(:fetch_external_data_exceptions, exceptions)
def log_fetch_external_data_exception(error)
self.external_data_exceptions ||= {}
self.external_data_exceptions[:primary_endpoint] = {
type: error.type,
code: error.code,
retryable: error.retryable,
attempts: error.attempts
}
self.external_id = nil if !error.retryable
save!
end
def fetch_external_data?
def fetchable_external_data?
false
end
def poll_external_data?
def pollable_external_data?
false
end
def fetch_external_data_pending?
# We don't have a good mechanism right now to know if the last fetch has errored. So, in order
# to ensure we don't poll to infinity, we stop after 5 minutes no matter what.
fetch_external_data? && poll_external_data? && external_id.present? && data.nil? && updated_at > 5.minutes.ago
return false unless fetchable_external_data?
return false unless pollable_external_data?
return false unless external_data_requested?
return false if fetch_external_data_final_exception?
return true if fetch_external_data_retryable_exception?
data.nil?
end
def external_data_requested?
external_id.present?
end
def fetch_external_data_retryable_exception?
if external_data_exceptions.present? && external_data_exceptions[:primary_endpoint].present?
external_data_exceptions.dig(:primary_endpoint, :retryable) && external_data_exceptionsdig(:primary_endpoint, :attempts) < 5
end
end
def fetch_external_data_final_exception?
if external_data_exceptions.present? && external_data_exceptions[:primary_endpoint].present?
!external_data_exceptions.dig(:primary_endpoint, :retryable)
end
end
def fetch_external_data
raise NotImplemented.new(:fetch_external_data)
end
def fetch_external_data_error_message
raise NotImplemented.new(:fetch_external_data_error_message)
end
def update_with_external_data!(data:)
update!(data: data)
update!(data: data,
external_data_exceptions: external_data_exceptions.merge(primary_endpoint: nil))
end
def clone(fork = false)
@ -260,6 +292,12 @@ class Champ < ApplicationRecord
public? && dossier.champ_forked_with_changes?(self)
end
def fetch_external_data_later
if fetchable_external_data? && external_id.present? && data.nil?
ChampFetchExternalDataJob.perform_later(self, external_id)
end
end
private
def html_id
@ -280,12 +318,6 @@ class Champ < ApplicationRecord
end
end
def fetch_external_data_later
if fetch_external_data? && external_id.present? && data.nil?
ChampFetchExternalDataJob.perform_later(self, external_id)
end
end
def normalize
return if value.nil?
return if value.present? && !value.include?("\u0000")

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null
@ -57,7 +58,7 @@ class Champs::AddressChamp < Champs::TextChamp
address_label
end
def fetch_external_data?
def fetchable_external_data?
true
end

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null
@ -21,7 +22,7 @@
# type_de_champ_id :integer
#
class Champs::AnnuaireEducationChamp < Champs::TextChamp
def fetch_external_data?
def fetchable_external_data?
true
end

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null
@ -31,7 +32,7 @@ class Champs::CnafChamp < Champs::TextChamp
external_id.nil?
end
def fetch_external_data?
def fetchable_external_data?
true
end

View file

@ -1,3 +1,26 @@
# == Schema Information
#
# Table name: champs
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null
# rebased_at :datetime
# type :string
# value :string
# value_json :jsonb
# created_at :datetime
# updated_at :datetime
# dossier_id :integer
# etablissement_id :integer
# external_id :string
# parent_id :bigint
# row_id :string
# type_de_champ_id :integer
#
class Champs::COJOChamp < Champ
store_accessor :value_json, :accreditation_number, :accreditation_birthdate
store_accessor :data, :accreditation_success, :accreditation_first_name, :accreditation_last_name
@ -22,11 +45,11 @@ class Champs::COJOChamp < Champ
accreditation_success != true
end
def fetch_external_data?
def fetchable_external_data?
true
end
def poll_external_data?
def pollable_external_data?
true
end

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null
@ -31,7 +32,7 @@ class Champs::DgfipChamp < Champs::TextChamp
external_id.nil?
end
def fetch_external_data?
def fetchable_external_data?
true
end

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null
@ -28,7 +29,7 @@ class Champs::MesriChamp < Champs::TextChamp
external_id.nil?
end
def fetch_external_data?
def fetchable_external_data?
true
end

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null
@ -28,7 +29,7 @@ class Champs::PoleEmploiChamp < Champs::TextChamp
external_id.nil?
end
def fetch_external_data?
def fetchable_external_data?
true
end

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null
@ -23,6 +24,8 @@
class Champs::SiretChamp < Champ
include SiretChampEtablissementFetchableConcern
after_validation :update_external_id
def search_terms
etablissement.present? ? etablissement.search_terms : [value]
end
@ -30,4 +33,23 @@ class Champs::SiretChamp < Champ
def mandatory_blank?
mandatory? && Siret.new(siret: value).invalid?
end
def fetchable_external_data?
true
end
def pollable_external_data?
true
end
def fetch_external_data
fetch_etablissement!(external_id, dossier.user)
end
def update_external_id
if value.present?
self.external_id = value
#self.etablissement = nil
end
end
end

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -4,6 +4,7 @@
#
# id :integer not null, primary key
# data :jsonb
# external_data_exceptions :jsonb
# fetch_external_data_exceptions :string is an Array
# prefilled :boolean default(FALSE)
# private :boolean default(FALSE), not null

View file

@ -1,7 +1,6 @@
module SiretChampEtablissementFetchableConcern
extend ActiveSupport::Concern
attr_reader :etablissement_fetch_error_key
include Dry::Monads[:result]
def fetch_etablissement!(siret, user)
return clear_etablissement!(:empty) if siret.empty?
@ -10,14 +9,15 @@ module SiretChampEtablissementFetchableConcern
return clear_etablissement!(:not_found) unless (etablissement = APIEntrepriseService.create_etablissement(self, siret, user&.id)) # i18n-tasks-use t('errors.messages.siret_not_found')
update!(etablissement: etablissement)
Success({ siret: })
rescue => error
if error.try(:network_error?) && !APIEntrepriseService.api_up?
# TODO: notify ops
update!(
etablissement: APIEntrepriseService.create_etablissement_as_degraded_mode(self, siret, user.id)
)
@etablissement_fetch_error_key = :api_entreprise_down
false
Failure(API::Client::Error[:api_entreprise_down, 0, true, 0, error])
else
Sentry.capture_exception(error, extra: { dossier_id: dossier_id, siret: siret })
clear_etablissement!(:network_error) # i18n-tasks-use t('errors.messages.siret_network_error')
@ -26,14 +26,15 @@ module SiretChampEtablissementFetchableConcern
private
def clear_etablissement!(error_key)
@etablissement_fetch_error_key = error_key
class SIRETError < StandardError
end
def clear_etablissement!(error_key)
etablissement_to_destroy = etablissement
update!(etablissement: nil)
etablissement_to_destroy&.destroy
false
Failure(API::Client::Error[error_key, 0, false, 0, SIRETError.new('error!')])
end
def invalid_because?(siret, criteria)

View file

@ -20,7 +20,7 @@ class COJOService
accreditation_last_name: accreditation_success ? body[:lastName] : nil
})
in Failure(code:, reason:) if code.in?(401..403)
Failure(API::Client::Error[:unauthorized, code, false, reason])
Failure(API::Client::Error[:unauthorized, code, false, 0, reason])
else
result
end

View file

@ -0,0 +1,5 @@
class AddFetchExternalDataFailureToChamps < ActiveRecord::Migration[7.0]
def change
add_column :champs, :external_data_exceptions, :jsonb
end
end

View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.0].define(version: 2023_07_18_113920) do
ActiveRecord::Schema[7.0].define(version: 2023_07_19_113920) do
# These are extensions that must be enabled in order to support this database
enable_extension "pgcrypto"
enable_extension "plpgsql"
@ -224,6 +224,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_07_18_113920) do
t.jsonb "data"
t.integer "dossier_id"
t.integer "etablissement_id"
t.jsonb "external_data_exceptions"
t.string "external_id"
t.string "fetch_external_data_exceptions", array: true
t.bigint "parent_id"