Merge pull request #5227 from betagouv/dev

2020-06-04-01

app/jobs/api_entreprise/association_job.rb

@@ -1,4 +1,4 @@
-class ApiEntreprise::AssociationJob < ApplicationJob
+class ApiEntreprise::AssociationJob < ApiEntreprise::Job
   def perform(etablissement_id, procedure_id)
     etablissement = Etablissement.find(etablissement_id)
     etablissement_params = ApiEntreprise::RNAAdapter.new(etablissement.siret, procedure_id).to_params

app/jobs/api_entreprise/attestation_fiscale_job.rb

@@ -1,4 +1,4 @@
-class ApiEntreprise::AttestationFiscaleJob < ApplicationJob
+class ApiEntreprise::AttestationFiscaleJob < ApiEntreprise::Job
   def perform(etablissement_id, procedure_id, user_id)
     etablissement = Etablissement.find(etablissement_id)
     etablissement_params = ApiEntreprise::AttestationFiscaleAdapter.new(etablissement.siret, procedure_id, user_id).to_params

app/jobs/api_entreprise/attestation_sociale_job.rb

@@ -1,4 +1,4 @@
-class ApiEntreprise::AttestationSocialeJob < ApplicationJob
+class ApiEntreprise::AttestationSocialeJob < ApiEntreprise::Job
   def perform(etablissement_id, procedure_id)
     etablissement = Etablissement.find(etablissement_id)
     etablissement_params = ApiEntreprise::AttestationSocialeAdapter.new(etablissement.siret, procedure_id).to_params

app/jobs/api_entreprise/bilans_bdf_job.rb

@@ -1,4 +1,4 @@
-class ApiEntreprise::BilansBdfJob < ApplicationJob
+class ApiEntreprise::BilansBdfJob < ApiEntreprise::Job
   def perform(etablissement_id, procedure_id)
     etablissement = Etablissement.find(etablissement_id)
     etablissement_params = ApiEntreprise::BilansBdfAdapter.new(etablissement.siret, procedure_id).to_params

app/jobs/api_entreprise/effectifs_annuels_job.rb

@@ -1,4 +1,4 @@
-class ApiEntreprise::EffectifsAnnuelsJob < ApplicationJob
+class ApiEntreprise::EffectifsAnnuelsJob < ApiEntreprise::Job
   def perform(etablissement_id, procedure_id)
     etablissement = Etablissement.find(etablissement_id)
     etablissement_params = ApiEntreprise::EffectifsAnnuelsAdapter.new(etablissement.siret, procedure_id).to_params

app/jobs/api_entreprise/effectifs_job.rb

@@ -1,7 +1,8 @@
-class ApiEntreprise::EffectifsJob < ApplicationJob
+class ApiEntreprise::EffectifsJob < ApiEntreprise::Job
   def perform(etablissement_id, procedure_id)
     etablissement = Etablissement.find(etablissement_id)
-    etablissement_params = ApiEntreprise::EffectifsAdapter.new(etablissement.siret, procedure_id, *get_current_valid_month_for_effectif).to_params
+    # effectifs endpoint currently only works when asking for february 2020 month
+    etablissement_params = ApiEntreprise::EffectifsAdapter.new(etablissement.siret, procedure_id, "2020", "02").to_params
     etablissement.update!(etablissement_params)
   end
 

app/jobs/api_entreprise/entreprise_job.rb

@@ -1,4 +1,4 @@
-class ApiEntreprise::EntrepriseJob < ApplicationJob
+class ApiEntreprise::EntrepriseJob < ApiEntreprise::Job
   def perform(etablissement_id, procedure_id)
     etablissement = Etablissement.find(etablissement_id)
     etablissement_params = ApiEntreprise::EntrepriseAdapter.new(etablissement.siret, procedure_id).to_params

app/jobs/api_entreprise/exercices_job.rb

@@ -1,4 +1,4 @@
-class ApiEntreprise::ExercicesJob < ApplicationJob
+class ApiEntreprise::ExercicesJob < ApiEntreprise::Job
   def perform(etablissement_id, procedure_id)
     etablissement = Etablissement.find(etablissement_id)
     etablissement_params = ApiEntreprise::ExercicesAdapter.new(etablissement.siret, procedure_id).to_params

app/jobs/api_entreprise/job.rb

@@ -0,0 +1,6 @@
+class ApiEntreprise::Job < ApplicationJob
+  DEFAULT_MAX_ATTEMPTS_API_ENTREPRISE_JOBS = 5
+  def max_attempts
+    ENV[MAX_ATTEMPTS_API_ENTREPRISE_JOBS].to_i || DEFAULT_MAX_ATTEMPTS_API_ENTREPRISE_JOBS
+  end
+end

app/jobs/application_job.rb

@@ -1,4 +1,6 @@
 class ApplicationJob < ActiveJob::Base
+  DEFAULT_MAX_ATTEMPTS_JOBS = 25
+
   before_perform do |job|
     Rails.logger.info("#{job.class.name} started at #{Time.zone.now}")
   end
@@ -18,4 +20,8 @@ class ApplicationJob < ActiveJob::Base
   def error(job, exception)
     Raven.capture_exception(exception)
   end
+
+  def max_attempts
+    ENV["MAX_ATTEMPTS_JOBS"].to_i || DEFAULT_MAX_ATTEMPTS_JOBS
+  end
 end

app/models/etablissement.rb

@@ -140,7 +140,7 @@ class Etablissement < ApplicationRecord
   end
 
   def entreprise_bilans_bdf_to_csv
-    headers = entreprise_bilans_bdf.flat_map(&:keys).uniq
+    headers = bilans_headers.concat(bilans_new_keys)
     data = entreprise_bilans_bdf.map do |bilan|
       headers.map { |h| bilan[h] }
     end
@@ -149,6 +149,11 @@ class Etablissement < ApplicationRecord
 
   private
 
+  def bilans_new_keys
+    keys = entreprise_bilans_bdf.flat_map(&:keys).uniq
+    keys - bilans_headers
+  end
+
   def dossier_id_for_export
     if dossier_id
       dossier_id.to_s
@@ -160,4 +165,28 @@ class Etablissement < ApplicationRecord
   def libelle_for_export
     champ&.libelle || 'Dossier'
   end
+
+  def bilans_headers
+    [
+      "date_arret_exercice", "duree_exercice", "chiffre_affaires_ht", "evolution_chiffre_affaires_ht",
+      "valeur_ajoutee_bdf", "evolution_valeur_ajoutee_bdf", "excedent_brut_exploitation",
+      "evolution_excedent_brut_exploitation", "resultat_exercice", "evolution_resultat_exercice",
+      "capacite_autofinancement", "evolution_capacite_autofinancement", "fonds_roulement_net_global",
+      "evolution_fonds_roulement_net_global", "besoin_en_fonds_de_roulement", "evolution_besoin_en_fonds_de_roulement",
+      "ratio_fonds_roulement_net_global_sur_besoin_en_fonds_de_roulement",
+      "evolution_ratio_fonds_roulement_net_global_sur_besoin_en_fonds_de_roulement", "disponibilites",
+      "evolution_disponibilites", "capital_social_inclus_dans_capitaux_propres_et_assimiles",
+      "evolution_capital_social_inclus_dans_capitaux_propres_et_assimiles", "capitaux_propres_et_assimiles",
+      "evolution_capitaux_propres_et_assimiles", "autres_fonds_propres", "evolution_autres_fonds_propres",
+      "total_provisions_pour_risques_et_charges", "evolution_total_provisions_pour_risques_et_charges",
+      "dettes1_emprunts_obligataires_et_convertibles", "evolution_dettes1_emprunts_obligataires_et_convertibles",
+      "dettes2_autres_emprunts_obligataires", "evolution_dettes2_autres_emprunts_obligataires",
+      "dettes3_emprunts_et_dettes_aupres_des_etablissements_de_credit",
+      "evolution_dettes3_emprunts_et_dettes_aupres_des_etablissements_de_credit",
+      "dettes4_maturite_a_un_an_au_plus", "evolution_dettes4_maturite_a_un_an_au_plus",
+      "emprunts_et_dettes_financieres_divers", "evolution_emprunts_et_dettes_financieres_divers",
+      "total_dettes_stables", "evolution_total_dettes_stables", "groupes_et_associes",
+      "evolution_groupes_et_associes", "total_passif", "evolution_total_passif"
+    ]
+  end
 end

config/env.example

@@ -108,3 +108,7 @@ UNIVERSIGN_USERPWD=""
 # API Geo / Adresse
 API_ADRESSE_URL="https://api-adresse.data.gouv.fr"
 API_GEO_URL="https://geo.api.gouv.fr"
+
+# Modifier le nb de tentatives de relance de job si echec
+# MAX_ATTEMPTS_JOBS=25
+# MAX_ATTEMPTS_API_ENTREPRISE_JOBS=5

config/initializers/active_storage.rb

@@ -40,3 +40,26 @@ module Fog::OpenStack::Auth::Catalog
     end
   end
 end
+
+require 'fog/openstack/auth/catalog/v3'
+module Fog::OpenStack::Auth::Catalog
+  class V3
+    def endpoint_url(endpoint, interface)
+      url = endpoint["#{interface}URL"]
+
+      if interface == 'public'
+        publicize(url)
+      else
+        url
+      end
+    end
+
+    private
+
+    def publicize(url)
+      search = %r{^https://[^/]+/}
+      replace = "#{ENV['DS_PROXY_URL']}/"
+      url.gsub(search, replace)
+    end
+  end
+end

spec/models/etablissement_spec.rb

@@ -38,14 +38,38 @@ describe Etablissement do
 
   describe '.entreprise_bilans_bdf_to_csv' do
     let(:etablissement) { build(:etablissement, entreprise_bilans_bdf: bilans) }
+    let(:ordered_headers) {
+      [
+        "date_arret_exercice", "duree_exercice", "chiffre_affaires_ht", "evolution_chiffre_affaires_ht",
+        "valeur_ajoutee_bdf", "evolution_valeur_ajoutee_bdf", "excedent_brut_exploitation",
+        "evolution_excedent_brut_exploitation", "resultat_exercice", "evolution_resultat_exercice",
+        "capacite_autofinancement", "evolution_capacite_autofinancement", "fonds_roulement_net_global",
+        "evolution_fonds_roulement_net_global", "besoin_en_fonds_de_roulement", "evolution_besoin_en_fonds_de_roulement",
+        "ratio_fonds_roulement_net_global_sur_besoin_en_fonds_de_roulement",
+        "evolution_ratio_fonds_roulement_net_global_sur_besoin_en_fonds_de_roulement", "disponibilites",
+        "evolution_disponibilites", "capital_social_inclus_dans_capitaux_propres_et_assimiles",
+        "evolution_capital_social_inclus_dans_capitaux_propres_et_assimiles", "capitaux_propres_et_assimiles",
+        "evolution_capitaux_propres_et_assimiles", "autres_fonds_propres", "evolution_autres_fonds_propres",
+        "total_provisions_pour_risques_et_charges", "evolution_total_provisions_pour_risques_et_charges",
+        "dettes1_emprunts_obligataires_et_convertibles", "evolution_dettes1_emprunts_obligataires_et_convertibles",
+        "dettes2_autres_emprunts_obligataires", "evolution_dettes2_autres_emprunts_obligataires",
+        "dettes3_emprunts_et_dettes_aupres_des_etablissements_de_credit",
+        "evolution_dettes3_emprunts_et_dettes_aupres_des_etablissements_de_credit",
+        "dettes4_maturite_a_un_an_au_plus", "evolution_dettes4_maturite_a_un_an_au_plus",
+        "emprunts_et_dettes_financieres_divers", "evolution_emprunts_et_dettes_financieres_divers",
+        "total_dettes_stables", "evolution_total_dettes_stables", "groupes_et_associes",
+        "evolution_groupes_et_associes", "total_passif", "evolution_total_passif"
+      ]
+    }
     let(:bilans) do
       [
         {
           "total_passif": "1200",
-          "chiffres_affaires_ht": "40000"
+          "chiffre_affaires_ht": "40000"
         },
         {
           "total_passif": "0",
+          "new_key": "50",
           "evolution_total_dettes_stables": "30"
         }
       ]
@@ -53,10 +77,32 @@ describe Etablissement do
 
     subject { etablissement.entreprise_bilans_bdf_to_csv.split("\n") }
 
-    it "build a csv with all keys" do
+    it "build a csv with keys in right order" do
-      expect(subject[0].split(',').sort).to eq(["total_passif", "chiffres_affaires_ht", "evolution_total_dettes_stables"].sort)
+      headers = subject[0].split(',')
-      expect(subject[1].split(',')).to eq(["1200", "40000"])
+      expect(headers).to eq(ordered_headers.concat(["new_key"]))
-      expect(subject[2].split(',')).to eq(["0", "", "30"])
     end
+
+    it "build a csv with good values" do
+      bilans_h = csv_to_array_of_hash(subject)
+      expect(bilans_h[0]["total_passif"]).to eq("1200")
+      expect(bilans_h[0]["chiffre_affaires_ht"]).to eq("40000")
+      expect(bilans_h[1]["evolution_total_dettes_stables"]).to eq("30")
+      expect(bilans_h[1]["new_key"]).to eq("50")
+    end
+  end
+
+  private
+
+  def csv_to_array_of_hash(lines)
+    headers = lines.shift.split(',')
+    lines.map { |line| line_to_hash(line, headers) }
+  end
+
+  def line_to_hash(line, headers)
+    bilan = {}
+    line.split(',').each_with_index do |value, index|
+      bilan[headers[index]] = value
+    end
+    bilan
   end
 end