From 868decd06eb442f787cd73505f2f684e32670916 Mon Sep 17 00:00:00 2001 From: clemkeirua Date: Tue, 2 Jun 2020 15:03:37 +0200 Subject: [PATCH 1/4] add overide for fog openstack v3 --- config/initializers/active_storage.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/config/initializers/active_storage.rb b/config/initializers/active_storage.rb index e20e48620..364050bd0 100644 --- a/config/initializers/active_storage.rb +++ b/config/initializers/active_storage.rb @@ -40,3 +40,26 @@ module Fog::OpenStack::Auth::Catalog end end end + +require 'fog/openstack/auth/catalog/v3' +module Fog::OpenStack::Auth::Catalog + class V3 + def endpoint_url(endpoint, interface) + url = endpoint["#{interface}URL"] + + if interface == 'public' + publicize(url) + else + url + end + end + + private + + def publicize(url) + search = %r{^https://[^/]+/} + replace = "#{ENV['DS_PROXY_URL']}/" + url.gsub(search, replace) + end + end +end From 705487dab65367c9fb43654a9a521e395e042f18 Mon Sep 17 00:00:00 2001 From: Christophe Robillard Date: Tue, 2 Jun 2020 20:13:38 +0200 Subject: [PATCH 2/4] define max_attempts for jobs --- app/jobs/api_entreprise/association_job.rb | 2 +- app/jobs/api_entreprise/attestation_fiscale_job.rb | 2 +- app/jobs/api_entreprise/attestation_sociale_job.rb | 2 +- app/jobs/api_entreprise/bilans_bdf_job.rb | 2 +- app/jobs/api_entreprise/effectifs_annuels_job.rb | 2 +- app/jobs/api_entreprise/effectifs_job.rb | 2 +- app/jobs/api_entreprise/entreprise_job.rb | 2 +- app/jobs/api_entreprise/exercices_job.rb | 2 +- app/jobs/api_entreprise/job.rb | 6 ++++++ app/jobs/application_job.rb | 6 ++++++ config/env.example | 4 ++++ 11 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 app/jobs/api_entreprise/job.rb diff --git a/app/jobs/api_entreprise/association_job.rb b/app/jobs/api_entreprise/association_job.rb index 2924cd142..fe2716c31 100644 --- a/app/jobs/api_entreprise/association_job.rb +++ b/app/jobs/api_entreprise/association_job.rb @@ -1,4 +1,4 @@ -class ApiEntreprise::AssociationJob < ApplicationJob +class ApiEntreprise::AssociationJob < ApiEntreprise::Job def perform(etablissement_id, procedure_id) etablissement = Etablissement.find(etablissement_id) etablissement_params = ApiEntreprise::RNAAdapter.new(etablissement.siret, procedure_id).to_params diff --git a/app/jobs/api_entreprise/attestation_fiscale_job.rb b/app/jobs/api_entreprise/attestation_fiscale_job.rb index 328c8b1cf..aaa3c8cd3 100644 --- a/app/jobs/api_entreprise/attestation_fiscale_job.rb +++ b/app/jobs/api_entreprise/attestation_fiscale_job.rb @@ -1,4 +1,4 @@ -class ApiEntreprise::AttestationFiscaleJob < ApplicationJob +class ApiEntreprise::AttestationFiscaleJob < ApiEntreprise::Job def perform(etablissement_id, procedure_id, user_id) etablissement = Etablissement.find(etablissement_id) etablissement_params = ApiEntreprise::AttestationFiscaleAdapter.new(etablissement.siret, procedure_id, user_id).to_params diff --git a/app/jobs/api_entreprise/attestation_sociale_job.rb b/app/jobs/api_entreprise/attestation_sociale_job.rb index bc499049e..581ed8037 100644 --- a/app/jobs/api_entreprise/attestation_sociale_job.rb +++ b/app/jobs/api_entreprise/attestation_sociale_job.rb @@ -1,4 +1,4 @@ -class ApiEntreprise::AttestationSocialeJob < ApplicationJob +class ApiEntreprise::AttestationSocialeJob < ApiEntreprise::Job def perform(etablissement_id, procedure_id) etablissement = Etablissement.find(etablissement_id) etablissement_params = ApiEntreprise::AttestationSocialeAdapter.new(etablissement.siret, procedure_id).to_params diff --git a/app/jobs/api_entreprise/bilans_bdf_job.rb b/app/jobs/api_entreprise/bilans_bdf_job.rb index 34839a0f7..61c198ab7 100644 --- a/app/jobs/api_entreprise/bilans_bdf_job.rb +++ b/app/jobs/api_entreprise/bilans_bdf_job.rb @@ -1,4 +1,4 @@ -class ApiEntreprise::BilansBdfJob < ApplicationJob +class ApiEntreprise::BilansBdfJob < ApiEntreprise::Job def perform(etablissement_id, procedure_id) etablissement = Etablissement.find(etablissement_id) etablissement_params = ApiEntreprise::BilansBdfAdapter.new(etablissement.siret, procedure_id).to_params diff --git a/app/jobs/api_entreprise/effectifs_annuels_job.rb b/app/jobs/api_entreprise/effectifs_annuels_job.rb index 2220d484a..d0c1c497e 100644 --- a/app/jobs/api_entreprise/effectifs_annuels_job.rb +++ b/app/jobs/api_entreprise/effectifs_annuels_job.rb @@ -1,4 +1,4 @@ -class ApiEntreprise::EffectifsAnnuelsJob < ApplicationJob +class ApiEntreprise::EffectifsAnnuelsJob < ApiEntreprise::Job def perform(etablissement_id, procedure_id) etablissement = Etablissement.find(etablissement_id) etablissement_params = ApiEntreprise::EffectifsAnnuelsAdapter.new(etablissement.siret, procedure_id).to_params diff --git a/app/jobs/api_entreprise/effectifs_job.rb b/app/jobs/api_entreprise/effectifs_job.rb index faf1690e7..e668ef191 100644 --- a/app/jobs/api_entreprise/effectifs_job.rb +++ b/app/jobs/api_entreprise/effectifs_job.rb @@ -1,4 +1,4 @@ -class ApiEntreprise::EffectifsJob < ApplicationJob +class ApiEntreprise::EffectifsJob < ApiEntreprise::Job def perform(etablissement_id, procedure_id) etablissement = Etablissement.find(etablissement_id) etablissement_params = ApiEntreprise::EffectifsAdapter.new(etablissement.siret, procedure_id, *get_current_valid_month_for_effectif).to_params diff --git a/app/jobs/api_entreprise/entreprise_job.rb b/app/jobs/api_entreprise/entreprise_job.rb index e22443a2d..02cf5caaf 100644 --- a/app/jobs/api_entreprise/entreprise_job.rb +++ b/app/jobs/api_entreprise/entreprise_job.rb @@ -1,4 +1,4 @@ -class ApiEntreprise::EntrepriseJob < ApplicationJob +class ApiEntreprise::EntrepriseJob < ApiEntreprise::Job def perform(etablissement_id, procedure_id) etablissement = Etablissement.find(etablissement_id) etablissement_params = ApiEntreprise::EntrepriseAdapter.new(etablissement.siret, procedure_id).to_params diff --git a/app/jobs/api_entreprise/exercices_job.rb b/app/jobs/api_entreprise/exercices_job.rb index caf20a86b..fd8b5c682 100644 --- a/app/jobs/api_entreprise/exercices_job.rb +++ b/app/jobs/api_entreprise/exercices_job.rb @@ -1,4 +1,4 @@ -class ApiEntreprise::ExercicesJob < ApplicationJob +class ApiEntreprise::ExercicesJob < ApiEntreprise::Job def perform(etablissement_id, procedure_id) etablissement = Etablissement.find(etablissement_id) etablissement_params = ApiEntreprise::ExercicesAdapter.new(etablissement.siret, procedure_id).to_params diff --git a/app/jobs/api_entreprise/job.rb b/app/jobs/api_entreprise/job.rb new file mode 100644 index 000000000..45d21b1ad --- /dev/null +++ b/app/jobs/api_entreprise/job.rb @@ -0,0 +1,6 @@ +class ApiEntreprise::Job < ApplicationJob + DEFAULT_MAX_ATTEMPTS_API_ENTREPRISE_JOBS = 5 + def max_attempts + ENV[MAX_ATTEMPTS_API_ENTREPRISE_JOBS].to_i || DEFAULT_MAX_ATTEMPTS_API_ENTREPRISE_JOBS + end +end diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb index 7538ed4f5..d7de32266 100644 --- a/app/jobs/application_job.rb +++ b/app/jobs/application_job.rb @@ -1,4 +1,6 @@ class ApplicationJob < ActiveJob::Base + DEFAULT_MAX_ATTEMPTS_JOBS = 25 + before_perform do |job| Rails.logger.info("#{job.class.name} started at #{Time.zone.now}") end @@ -18,4 +20,8 @@ class ApplicationJob < ActiveJob::Base def error(job, exception) Raven.capture_exception(exception) end + + def max_attempts + ENV["MAX_ATTEMPTS_JOBS"].to_i || DEFAULT_MAX_ATTEMPTS_JOBS + end end diff --git a/config/env.example b/config/env.example index 7b3515bd5..64d3a8f6a 100644 --- a/config/env.example +++ b/config/env.example @@ -108,3 +108,7 @@ UNIVERSIGN_USERPWD="" # API Geo / Adresse API_ADRESSE_URL="https://api-adresse.data.gouv.fr" API_GEO_URL="https://geo.api.gouv.fr" + +# Modifier le nb de tentatives de relance de job si echec +# MAX_ATTEMPTS_JOBS=25 +# MAX_ATTEMPTS_API_ENTREPRISE_JOBS=5 From d06867822bc65f4ff00fc57deff890e8f21416e4 Mon Sep 17 00:00:00 2001 From: Christophe Robillard Date: Wed, 3 Jun 2020 15:15:55 +0200 Subject: [PATCH 3/4] reorder bilans bdf csv --- app/models/etablissement.rb | 31 ++++++++++++++++- spec/models/etablissement_spec.rb | 56 ++++++++++++++++++++++++++++--- 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/app/models/etablissement.rb b/app/models/etablissement.rb index 8dce7b4fa..6f3afcb08 100644 --- a/app/models/etablissement.rb +++ b/app/models/etablissement.rb @@ -140,7 +140,7 @@ class Etablissement < ApplicationRecord end def entreprise_bilans_bdf_to_csv - headers = entreprise_bilans_bdf.flat_map(&:keys).uniq + headers = bilans_headers.concat(bilans_new_keys) data = entreprise_bilans_bdf.map do |bilan| headers.map { |h| bilan[h] } end @@ -149,6 +149,11 @@ class Etablissement < ApplicationRecord private + def bilans_new_keys + keys = entreprise_bilans_bdf.flat_map(&:keys).uniq + keys - bilans_headers + end + def dossier_id_for_export if dossier_id dossier_id.to_s @@ -160,4 +165,28 @@ class Etablissement < ApplicationRecord def libelle_for_export champ&.libelle || 'Dossier' end + + def bilans_headers + [ + "date_arret_exercice", "duree_exercice", "chiffre_affaires_ht", "evolution_chiffre_affaires_ht", + "valeur_ajoutee_bdf", "evolution_valeur_ajoutee_bdf", "excedent_brut_exploitation", + "evolution_excedent_brut_exploitation", "resultat_exercice", "evolution_resultat_exercice", + "capacite_autofinancement", "evolution_capacite_autofinancement", "fonds_roulement_net_global", + "evolution_fonds_roulement_net_global", "besoin_en_fonds_de_roulement", "evolution_besoin_en_fonds_de_roulement", + "ratio_fonds_roulement_net_global_sur_besoin_en_fonds_de_roulement", + "evolution_ratio_fonds_roulement_net_global_sur_besoin_en_fonds_de_roulement", "disponibilites", + "evolution_disponibilites", "capital_social_inclus_dans_capitaux_propres_et_assimiles", + "evolution_capital_social_inclus_dans_capitaux_propres_et_assimiles", "capitaux_propres_et_assimiles", + "evolution_capitaux_propres_et_assimiles", "autres_fonds_propres", "evolution_autres_fonds_propres", + "total_provisions_pour_risques_et_charges", "evolution_total_provisions_pour_risques_et_charges", + "dettes1_emprunts_obligataires_et_convertibles", "evolution_dettes1_emprunts_obligataires_et_convertibles", + "dettes2_autres_emprunts_obligataires", "evolution_dettes2_autres_emprunts_obligataires", + "dettes3_emprunts_et_dettes_aupres_des_etablissements_de_credit", + "evolution_dettes3_emprunts_et_dettes_aupres_des_etablissements_de_credit", + "dettes4_maturite_a_un_an_au_plus", "evolution_dettes4_maturite_a_un_an_au_plus", + "emprunts_et_dettes_financieres_divers", "evolution_emprunts_et_dettes_financieres_divers", + "total_dettes_stables", "evolution_total_dettes_stables", "groupes_et_associes", + "evolution_groupes_et_associes", "total_passif", "evolution_total_passif" + ] + end end diff --git a/spec/models/etablissement_spec.rb b/spec/models/etablissement_spec.rb index c329a519e..89d9e6161 100644 --- a/spec/models/etablissement_spec.rb +++ b/spec/models/etablissement_spec.rb @@ -38,14 +38,38 @@ describe Etablissement do describe '.entreprise_bilans_bdf_to_csv' do let(:etablissement) { build(:etablissement, entreprise_bilans_bdf: bilans) } + let(:ordered_headers) { + [ + "date_arret_exercice", "duree_exercice", "chiffre_affaires_ht", "evolution_chiffre_affaires_ht", + "valeur_ajoutee_bdf", "evolution_valeur_ajoutee_bdf", "excedent_brut_exploitation", + "evolution_excedent_brut_exploitation", "resultat_exercice", "evolution_resultat_exercice", + "capacite_autofinancement", "evolution_capacite_autofinancement", "fonds_roulement_net_global", + "evolution_fonds_roulement_net_global", "besoin_en_fonds_de_roulement", "evolution_besoin_en_fonds_de_roulement", + "ratio_fonds_roulement_net_global_sur_besoin_en_fonds_de_roulement", + "evolution_ratio_fonds_roulement_net_global_sur_besoin_en_fonds_de_roulement", "disponibilites", + "evolution_disponibilites", "capital_social_inclus_dans_capitaux_propres_et_assimiles", + "evolution_capital_social_inclus_dans_capitaux_propres_et_assimiles", "capitaux_propres_et_assimiles", + "evolution_capitaux_propres_et_assimiles", "autres_fonds_propres", "evolution_autres_fonds_propres", + "total_provisions_pour_risques_et_charges", "evolution_total_provisions_pour_risques_et_charges", + "dettes1_emprunts_obligataires_et_convertibles", "evolution_dettes1_emprunts_obligataires_et_convertibles", + "dettes2_autres_emprunts_obligataires", "evolution_dettes2_autres_emprunts_obligataires", + "dettes3_emprunts_et_dettes_aupres_des_etablissements_de_credit", + "evolution_dettes3_emprunts_et_dettes_aupres_des_etablissements_de_credit", + "dettes4_maturite_a_un_an_au_plus", "evolution_dettes4_maturite_a_un_an_au_plus", + "emprunts_et_dettes_financieres_divers", "evolution_emprunts_et_dettes_financieres_divers", + "total_dettes_stables", "evolution_total_dettes_stables", "groupes_et_associes", + "evolution_groupes_et_associes", "total_passif", "evolution_total_passif" + ] + } let(:bilans) do [ { "total_passif": "1200", - "chiffres_affaires_ht": "40000" + "chiffre_affaires_ht": "40000" }, { "total_passif": "0", + "new_key": "50", "evolution_total_dettes_stables": "30" } ] @@ -53,10 +77,32 @@ describe Etablissement do subject { etablissement.entreprise_bilans_bdf_to_csv.split("\n") } - it "build a csv with all keys" do - expect(subject[0].split(',').sort).to eq(["total_passif", "chiffres_affaires_ht", "evolution_total_dettes_stables"].sort) - expect(subject[1].split(',')).to eq(["1200", "40000"]) - expect(subject[2].split(',')).to eq(["0", "", "30"]) + it "build a csv with keys in right order" do + headers = subject[0].split(',') + expect(headers).to eq(ordered_headers.concat(["new_key"])) + end + + it "build a csv with good values" do + bilans_h = csv_to_array_of_hash(subject) + expect(bilans_h[0]["total_passif"]).to eq("1200") + expect(bilans_h[0]["chiffre_affaires_ht"]).to eq("40000") + expect(bilans_h[1]["evolution_total_dettes_stables"]).to eq("30") + expect(bilans_h[1]["new_key"]).to eq("50") end end + + private + + def csv_to_array_of_hash(lines) + headers = lines.shift.split(',') + lines.map { |line| line_to_hash(line, headers) } + end + + def line_to_hash(line, headers) + bilan = {} + line.split(',').each_with_index do |value, index| + bilan[headers[index]] = value + end + bilan + end end From 6535324ae94a0a31f60cf0e64d74a35a0b6edb2a Mon Sep 17 00:00:00 2001 From: Christophe Robillard Date: Thu, 4 Jun 2020 09:18:01 +0200 Subject: [PATCH 4/4] ask effectifs only for 2020/02 --- app/jobs/api_entreprise/effectifs_job.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/jobs/api_entreprise/effectifs_job.rb b/app/jobs/api_entreprise/effectifs_job.rb index e668ef191..4ca19ab8d 100644 --- a/app/jobs/api_entreprise/effectifs_job.rb +++ b/app/jobs/api_entreprise/effectifs_job.rb @@ -1,7 +1,8 @@ class ApiEntreprise::EffectifsJob < ApiEntreprise::Job def perform(etablissement_id, procedure_id) etablissement = Etablissement.find(etablissement_id) - etablissement_params = ApiEntreprise::EffectifsAdapter.new(etablissement.siret, procedure_id, *get_current_valid_month_for_effectif).to_params + # effectifs endpoint currently only works when asking for february 2020 month + etablissement_params = ApiEntreprise::EffectifsAdapter.new(etablissement.siret, procedure_id, "2020", "02").to_params etablissement.update!(etablissement_params) end