From 5c9f2e87837ce39874da6a42ce94bb5ceaeae1aa Mon Sep 17 00:00:00 2001 From: Paul Chavard Date: Wed, 13 Jan 2021 18:58:59 +0100 Subject: [PATCH] Add api education adapter and job --- Gemfile | 1 + Gemfile.lock | 10 + app/jobs/annuaire_education_update_job.rb | 16 ++ .../annuaire_education_adapter.rb | 36 ++++ app/lib/api_education/api.rb | 22 +++ .../etablissement-annuaire-education.json | 185 ++++++++++++++++++ config/env.example | 3 + config/initializers/urls.rb | 1 + config/secrets.yml | 1 + .../api_education/annuaire_education.json | 91 +++++++++ .../annuaire_education_invalid.json | 58 ++++++ .../annuaire_education_adapter_spec.rb | 29 +++ 12 files changed, 453 insertions(+) create mode 100644 app/jobs/annuaire_education_update_job.rb create mode 100644 app/lib/api_education/annuaire_education_adapter.rb create mode 100644 app/lib/api_education/api.rb create mode 100644 app/schemas/etablissement-annuaire-education.json create mode 100644 spec/fixtures/files/api_education/annuaire_education.json create mode 100644 spec/fixtures/files/api_education/annuaire_education_invalid.json create mode 100644 spec/lib/api_education/annuaire_education_adapter_spec.rb diff --git a/Gemfile b/Gemfile index 50ba4515f..dd0a1f48a 100644 --- a/Gemfile +++ b/Gemfile @@ -46,6 +46,7 @@ gem 'http_accept_language' gem 'iban-tools' gem 'image_processing' gem 'jquery-rails' # Use jquery as the JavaScript library +gem 'json_schemer' gem 'jwt' gem 'kaminari', '1.2.1' # Pagination gem 'lograge' diff --git a/Gemfile.lock b/Gemfile.lock index 6e8c7cb6c..ac1c5d752 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -230,6 +230,8 @@ GEM railties (>= 3.2) dry-inflector (0.2.0) dumb_delegator (0.8.1) + ecma-re-validator (0.2.1) + regexp_parser (~> 1.2) em-websocket (0.5.1) eventmachine (>= 0.12.9) http_parser.rb (~> 0.6.0) @@ -339,6 +341,7 @@ GEM rainbow rubocop (>= 0.50.0) sysexits (~> 1.1) + hana (1.3.7) hashdiff (1.0.1) hashie (4.1.0) html2haml (2.2.0) @@ -370,6 +373,11 @@ GEM activesupport (>= 4.2) aes_key_wrap bindata + json_schemer (0.2.16) + ecma-re-validator (~> 0.2) + hana (~> 1.3) + regexp_parser (~> 1.5) + uri_template (~> 0.7) jsonapi-renderer (0.2.2) jwt (2.2.2) kaminari (1.2.1) @@ -721,6 +729,7 @@ GEM unf_ext unf_ext (0.0.7.7) unicode-display_width (1.7.0) + uri_template (0.7.0) validate_email (0.1.6) activemodel (>= 3.0) mail (>= 2.2.5) @@ -835,6 +844,7 @@ DEPENDENCIES iban-tools image_processing jquery-rails + json_schemer jwt kaminari (= 1.2.1) launchy diff --git a/app/jobs/annuaire_education_update_job.rb b/app/jobs/annuaire_education_update_job.rb new file mode 100644 index 000000000..b312a0fd7 --- /dev/null +++ b/app/jobs/annuaire_education_update_job.rb @@ -0,0 +1,16 @@ +class AnnuaireEducationUpdateJob < ApplicationJob + def perform(champ) + search_term = champ.value + + if search_term.present? + data = ApiEducation::AnnuaireEducationAdapter.new(search_term).to_params + + if data.present? + champ.data = data + else + champ.value = nil + end + champ.save! + end + end +end diff --git a/app/lib/api_education/annuaire_education_adapter.rb b/app/lib/api_education/annuaire_education_adapter.rb new file mode 100644 index 000000000..46bf3b977 --- /dev/null +++ b/app/lib/api_education/annuaire_education_adapter.rb @@ -0,0 +1,36 @@ +require 'json_schemer' + +class ApiEducation::AnnuaireEducationAdapter + class InvalidSchemaError < ::StandardError + def initialize(errors) + super(errors.map(&:to_json).join("\n")) + end + end + + def initialize(search_term) + @search_term = search_term + end + + def to_params + record = data_source[:records].first + if record.present? + properties = record[:fields].merge({ geometry: record[:geometry] }).deep_stringify_keys + if schemer.valid?(properties) + properties + else + errors = schemer.validate(properties).to_a + raise InvalidSchemaError.new(errors) + end + end + end + + private + + def data_source + @data_source ||= JSON.parse(ApiEducation::API.search_annuaire_education(@search_term), symbolize_names: true) + end + + def schemer + @schemer ||= JSONSchemer.schema(Rails.root.join('app/schemas/etablissement-annuaire-education.json')) + end +end diff --git a/app/lib/api_education/api.rb b/app/lib/api_education/api.rb new file mode 100644 index 000000000..b113ebce9 --- /dev/null +++ b/app/lib/api_education/api.rb @@ -0,0 +1,22 @@ +class ApiEducation::API + class ResourceNotFound < StandardError + end + + def self.search_annuaire_education(search_term) + call([API_EDUCATION_URL, 'search'].join('/'), 'fr-en-annuaire-education', { q: search_term }) + end + + private + + def self.call(url, dataset, params) + response = Typhoeus.get(url, params: { rows: 1, dataset: dataset }.merge(params)) + + if response.success? + response.body + else + message = response.code == 0 ? response.return_message : response.code.to_s + Rails.logger.error "[ApiEducation] Error on #{url}: #{message}" + raise ResourceNotFound + end + end +end diff --git a/app/schemas/etablissement-annuaire-education.json b/app/schemas/etablissement-annuaire-education.json new file mode 100644 index 000000000..704776700 --- /dev/null +++ b/app/schemas/etablissement-annuaire-education.json @@ -0,0 +1,185 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://demarches-simplifiees.fr/etablissement-annuaire-education.schema.json", + "title": "Établissement annuaire education", + "type": "object", + "properties": { + "date_ouverture": { + "type": "date" + }, + "ministere_tutelle": { + "type": "string" + }, + "statut_public_prive": { + "type": "string" + }, + "libelle_region": { + "type": "string" + }, + "telephone": { + "type": "string" + }, + "date_maj_ligne": { + "type": "string" + }, + "libelle_nature": { + "type": "string" + }, + "etat": { + "type": "string" + }, + "type_etablissement": { + "type": "string" + }, + "identifiant_de_l_etablissement": { + "type": "string" + }, + "code_region": { + "type": "string" + }, + "siren_siret": { + "type": "string" + }, + "mail": { + "type": "string" + }, + "nom_circonscription": { + "type": "string" + }, + "nom_commune": { + "type": "string" + }, + "adresse_3": { + "type": "string" + }, + "adresse_1": { + "type": "string" + }, + "nombre_d_eleves": { + "type": "integer", + "minimum": 0 + }, + "code_commune": { + "type": "string" + }, + "code_departement": { + "type": "string" + }, + "precision_localisation": { + "type": "string" + }, + "type_contrat_prive": { + "type": "string" + }, + "code_postal": { + "type": "string" + }, + "libelle_departement": { + "type": "string" + }, + "code_academie": { + "type": "string" + }, + "libelle_academie": { + "type": "string" + }, + "nom_etablissement": { + "type": "string" + }, + "epsg_origine": { + "type": "string" + }, + "pial": { + "type": "string" + }, + "code_nature": { + "type": "integer", + "minimum": 0 + }, + "code_type_contrat_prive": { + "type": "integer", + "minimum": 0 + }, + "ecole_elementaire": { + "enum": [0, 1] + }, + "hebergement": { + "enum": [0, 1] + }, + "rpi_concentre": { + "enum": [0, 1] + }, + "ulis": { + "enum": [0, 1] + }, + "restauration": { + "enum": [0, 1] + }, + "ecole_maternelle": { + "enum": [0, 1] + }, + "multi_uai": { + "enum": [0, 1] + }, + "section_theatre": { + "enum": ["0", "1"] + }, + "section_internationale": { + "enum": ["0", "1"] + }, + "post_bac": { + "enum": ["0", "1"] + }, + "section_cinema": { + "enum": ["0", "1"] + }, + "section_europeenne": { + "enum": ["0", "1"] + }, + "lycee_des_metiers": { + "enum": ["0", "1"] + }, + "voie_professionnelle": { + "enum": ["0", "1"] + }, + "lycee_militaire": { + "enum": ["0", "1"] + }, + "section_sport": { + "enum": ["0", "1"] + }, + "voie_technologique": { + "enum": ["0", "1"] + }, + "section_arts": { + "enum": ["0", "1"] + }, + "lycee_agricole": { + "enum": ["0", "1"] + }, + "apprentissage": { + "enum": ["0", "1"] + }, + "voie_generale": { + "enum": ["0", "1"] + } + }, + "required": [ + "identifiant_de_l_etablissement", + "nom_etablissement", + "statut_public_prive", + "type_contrat_prive", + "nom_commune", + "code_commune", + "nombre_d_eleves", + "siren_siret", + "libelle_academie", + "code_academie", + "libelle_nature", + "code_nature", + "adresse_1", + "code_postal", + "libelle_region", + "code_region" + ] +} diff --git a/config/env.example b/config/env.example index f58214188..9ba82ab8a 100644 --- a/config/env.example +++ b/config/env.example @@ -112,6 +112,9 @@ UNIVERSIGN_USERPWD="" API_ADRESSE_URL="https://api-adresse.data.gouv.fr" API_GEO_URL="https://geo.api.gouv.fr" +# API Education +API_EDUCATION_URL="https://data.education.gouv.fr/api/records/1.0" + # Modifier le nb de tentatives de relance de job si echec # MAX_ATTEMPTS_JOBS=25 # MAX_ATTEMPTS_API_ENTREPRISE_JOBS=5 diff --git a/config/initializers/urls.rb b/config/initializers/urls.rb index 84053627e..4c0e5e10c 100644 --- a/config/initializers/urls.rb +++ b/config/initializers/urls.rb @@ -2,6 +2,7 @@ # API URLs API_CARTO_URL = ENV.fetch("API_CARTO_URL", "https://sandbox.geo.api.gouv.fr/apicarto") API_ENTREPRISE_URL = ENV.fetch("API_ENTREPRISE_URL", "https://entreprise.api.gouv.fr/v2") +API_EDUCATION_URL = ENV.fetch("API_EDUCATION_URL", "https://data.education.gouv.fr/api/records/1.0") HELPSCOUT_API_URL = ENV.fetch("HELPSCOUT_API_URL", "https://api.helpscout.net/v2") PIPEDRIVE_API_URL = ENV.fetch("PIPEDRIVE_API_URL", "https://api.pipedrive.com/v1") SENDINBLUE_API_URL = ENV.fetch("SENDINBLUE_API_URL", "https://in-automate.sendinblue.com/api/v2") diff --git a/config/secrets.yml b/config/secrets.yml index a244b6a0f..f521a1edf 100644 --- a/config/secrets.yml +++ b/config/secrets.yml @@ -60,6 +60,7 @@ defaults: &defaults autocomplete: api_geo_url: <%= ENV['API_GEO_URL'] %> api_adresse_url: <%= ENV['API_ADRESSE_URL'] %> + api_education_url: <%= ENV['API_EDUCATION_URL'] %> diff --git a/spec/fixtures/files/api_education/annuaire_education.json b/spec/fixtures/files/api_education/annuaire_education.json new file mode 100644 index 000000000..d4389f3c7 --- /dev/null +++ b/spec/fixtures/files/api_education/annuaire_education.json @@ -0,0 +1,91 @@ +{ + "nhits": 1, + "parameters": { + "dataset": "fr-en-annuaire-education", + "q": "0050009H", + "timezone": "UTC", + "rows": 1, + "start": 0, + "format": "json" + }, + "records": [ + { + "datasetid": "fr-en-annuaire-education", + "recordid": "0fd6a58dc6c7e5346e3bd62ada6216f476ebeeac", + "fields": { + "section_arts": "0", + "lycee_agricole": "0", + "apprentissage": "1", + "voie_generale": "0", + "ministere_tutelle": "MINISTERE DE L'EDUCATION NATIONALE", + "statut_public_prive": "Public", + "libelle_region": "Provence-Alpes-Côte d'Azur", + "telephone": "04 92 56 56 10", + "date_maj_ligne": "2021-01-12", + "hebergement": 1, + "libelle_nature": "LYCEE PROFESSIONNEL", + "lycee_militaire": "0", + "section_sport": "0", + "voie_technologique": "0", + "fiche_onisep": "http://geolocalisation.onisep.fr/05-hautes-alpes/gap/lycee/lycee-professionnel-sevigne.html", + "etat": "OUVERT", + "web": "www.lyc-sevigne.ac-aix-marseille.fr", + "rpi_concentre": 0, + "identifiant_de_l_etablissement": "0050009H", + "code_region": "93", + "ulis": 1, + "restauration": 1, + "code_departement": "05", + "date_ouverture": "1965-05-01", + "voie_professionnelle": "1", + "greta": "1", + "coordy_origine": 6389542.2, + "siren_siret": "19050009000013", + "mail": "ce.0050009H@ac-aix-marseille.fr", + "type_contrat_prive": "SANS OBJET", + "nom_commune": "Gap", + "segpa": "0", + "adresse_3": "05000 GAP", + "fax": "04 92 56 56 30", + "type_etablissement": "Lycée", + "nombre_d_eleves": 451, + "libelle_zone_animation_pedagogique": "GAP", + "code_commune": "05061", + "latitude": 44.562179100043984, + "section_theatre": "0", + "section_internationale": "0", + "post_bac": "1", + "precision_localisation": "Numéro de rue", + "multi_uai": 0, + "code_postal": "05000", + "libelle_departement": "Hautes-Alpes", + "section_cinema": "0", + "section_europeenne": "1", + "libelle_academie": "Aix-Marseille", + "longitude": 6.0746303437466915, + "code_zone_animation_pedagogique": "02104", + "code_academie": "02", + "lycee_des_metiers": "1", + "epsg_origine": "EPSG:2154", + "adresse_1": "6 rue Jean Macé", + "nom_etablissement": "Lycée professionnel Sévigné", + "pial": "0050009H", + "code_nature": 320, + "position": [ + 44.562179100043984, + 6.0746303437466915 + ], + "code_type_contrat_prive": 99, + "coordx_origine": 944110.8 + }, + "geometry": { + "type": "Point", + "coordinates": [ + 6.0746303437466915, + 44.562179100043984 + ] + }, + "record_timestamp": "2021-01-12T18:03:00+00:00" + } + ] +} diff --git a/spec/fixtures/files/api_education/annuaire_education_invalid.json b/spec/fixtures/files/api_education/annuaire_education_invalid.json new file mode 100644 index 000000000..1bb9609c8 --- /dev/null +++ b/spec/fixtures/files/api_education/annuaire_education_invalid.json @@ -0,0 +1,58 @@ +{ + "nhits": 1, + "parameters": { + "dataset": "fr-en-annuaire-education", + "q": "0050009H", + "timezone": "UTC", + "rows": 1, + "start": 0, + "format": "json" + }, + "records": [ + { + "datasetid": "fr-en-annuaire-education", + "recordid": "0fd6a58dc6c7e5346e3bd62ada6216f476ebeeac", + "fields": { + "section_arts": "0", + "lycee_agricole": "0", + "apprentissage": "1", + "voie_generale": "0", + "ministere_tutelle": "MINISTERE DE L'EDUCATION NATIONALE", + "statut_public_prive": "Public", + "libelle_region": "Provence-Alpes-Côte d'Azur", + "telephone": "04 92 56 56 10", + "date_maj_ligne": "2021-01-12", + "hebergement": 1, + "libelle_nature": "LYCEE PROFESSIONNEL", + "lycee_militaire": "0", + "section_sport": "0", + "voie_technologique": "0", + "fiche_onisep": "http://geolocalisation.onisep.fr/05-hautes-alpes/gap/lycee/lycee-professionnel-sevigne.html", + "etat": "OUVERT", + "web": "www.lyc-sevigne.ac-aix-marseille.fr", + "rpi_concentre": 0, + "code_region": "93", + "ulis": 1, + "restauration": 1, + "code_departement": "05", + "date_ouverture": "1965-05-01", + "voie_professionnelle": "1", + "greta": "1", + "coordy_origine": 6389542.2, + "siren_siret": "19050009000013", + "mail": "ce.0050009H@ac-aix-marseille.fr", + "type_contrat_prive": "SANS OBJET", + "nom_commune": "Gap", + "segpa": "0" + }, + "geometry": { + "type": "Point", + "coordinates": [ + 6.0746303437466915, + 44.562179100043984 + ] + }, + "record_timestamp": "2021-01-12T18:03:00+00:00" + } + ] +} diff --git a/spec/lib/api_education/annuaire_education_adapter_spec.rb b/spec/lib/api_education/annuaire_education_adapter_spec.rb new file mode 100644 index 000000000..a428c3d22 --- /dev/null +++ b/spec/lib/api_education/annuaire_education_adapter_spec.rb @@ -0,0 +1,29 @@ +describe ApiEducation::AnnuaireEducationAdapter do + let(:search_term) { '0050009H' } + let(:adapter) { described_class.new(search_term) } + subject { adapter.to_params } + + before do + stub_request(:get, /https:\/\/data.education.gouv.fr\/api\/records\/1.0/) + .to_return(body: body, status: status) + end + + context "when responds with valid schema" do + let(:body) { File.read('spec/fixtures/files/api_education/annuaire_education.json') } + let(:status) { 200 } + + it '#to_params return vaid hash' do + expect(subject).to be_an_instance_of(Hash) + expect(subject['identifiant_de_l_etablissement']).to eq(search_term) + end + end + + context "when responds with invalid schema" do + let(:body) { File.read('spec/fixtures/files/api_education/annuaire_education_invalid.json') } + let(:status) { 200 } + + it '#to_params raise exception' do + expect { subject }.to raise_exception(ApiEducation::AnnuaireEducationAdapter::InvalidSchemaError) + end + end +end