DGNum/demarches-normaliennes
15
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Merge pull request #3568 from betagouv/frederic/sendinblue

Browse source 
Send administrateur usage statistics in nightly batches rather than JS
This commit is contained in:
Frederic Merizen 2019-03-08 16:43:06 +01:00 committed by GitHub
commit 80b2f2988e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 466 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -64,6 +64,7 @@ En local, un utilisateur de test est créé automatiquement, avec les identifian
AutoArchiveProcedureJob.set(cron: "* * * * *").perform_later
WeeklyOverviewJob.set(cron: "0 8 * * 0").perform_later
AutoReceiveDossiersForProcedureJob.set(cron: "* * * * *").perform_later(procedure_declaratoire_id, Dossier.states.fetch(:en_instruction))
SendinblueUpdateAdministrateursJob.set(cron: "0 10 * * *").perform_later
FindDubiousProceduresJob.set(cron: "0 0 * * *").perform_later
Administrateurs::ActivateBeforeExpirationJob.set(cron: "0 8 * * *").perform_later
WarnExpiringDossiersJob.set(cron: "0 0 1 * *").perform_later

5
app/jobs/update_administrateur_usage_statistics_job.rb Normal file
View file

@ -0,0 +1,5 @@
class UpdateAdministrateurUsageStatisticsJob < ApplicationJob
def perform
AdministrateurUsageStatisticsService.new.update_administrateurs
end
end

73
app/lib/sendinblue/api.rb Normal file
View file

@ -0,0 +1,73 @@
class Sendinblue::Api
def self.new_properly_configured!
api = self.new
if !api.properly_configured?
raise StandardError, 'Sendinblue API is not properly configured'
end
api
end
def initialize
@failures = []
end
def properly_configured?
client_key.present?
end
def identify(email, attributes = {})
req = api_request('identify', email: email, attributes: attributes)
req.on_complete do |response|
if !response.success?
push_failure("Error while updating identity for administrateur '#{email}' in Sendinblue: #{response.response_code} '#{response.body}'")
end
end
hydra.queue(req)
end
def run
hydra.run
@hydra = nil
flush_failures
end
private
def hydra
@hydra ||= Typhoeus::Hydra.new
end
def push_failure(failure)
@failures << failure
end
def flush_failures
failures = @failures
@failures = []
if failures.present?
raise StandardError, failures.join(', ')
end
end
def api_request(path, body)
url = "#{SENDINBLUE_API_URL}/#{path}"
Typhoeus::Request.new(
url,
method: :post,
body: body.to_json,
headers: headers
)
end
def headers
{
'ma-key': client_key,
'Content-Type': 'application/json; charset=UTF-8'
}
end
def client_key
Rails.application.secrets.sendinblue[:client_key]
end
end

166
app/services/administrateur_usage_statistics_service.rb Normal file
View file

@ -0,0 +1,166 @@
# Note: this class uses a `synthetic_state` for Dossier, that diverges from the standard state:
# - 'termine' is the synthetic_state for all dossiers
# whose state is 'accepte', 'refuse' or 'sans_suite',
# even when `archive` is true
# - 'archive' is the synthetic_state for all dossiers
# where archive is true,
# except those whose synthetic_state is already 'termine'
# - For all other dossiers, the synthetic_state and the state are the same
class AdministrateurUsageStatisticsService
def update_administrateurs
Administrateur.find_each do |administrateur|
stats = administrateur_stats(administrateur)
api.identify(administrateur.email, stats)
end
api.run
end
private
def api
@api ||= Sendinblue::Api.new_properly_configured!
end
def administrateur_stats(administrateur)
nb_dossiers_by_procedure_id = nb_dossiers_by_procedure_id(administrateur.id)
nb_dossiers_by_synthetic_state = nb_dossiers_by_synthetic_state(administrateur.id)
result = {
ds_sign_in_count: administrateur.sign_in_count,
ds_created_at: administrateur.created_at,
ds_active: administrateur.active,
ds_id: administrateur.id,
ds_features: administrateur.features.to_json,
nb_services: nb_services_by_administrateur_id[administrateur.id],
nb_instructeurs: nb_instructeurs_by_administrateur_id[administrateur.id],
ds_nb_demarches_actives: nb_demarches_by_administrateur_id_and_state[[administrateur.id, "publiee"]],
ds_nb_demarches_archives: nb_demarches_by_administrateur_id_and_state[[administrateur.id, "archivee"]],
ds_nb_demarches_brouillons: nb_demarches_by_administrateur_id_and_state[[administrateur.id, "brouillon"]],
nb_demarches_test: nb_dossiers_by_procedure_id
.select { |procedure_id, count| count > 0 && is_brouillon(procedure_id) }
.count,
nb_demarches_prod: nb_dossiers_by_procedure_id
.reject { |procedure_id, count| count == 0 || is_brouillon(procedure_id) }
.count,
nb_demarches_prod_20: nb_dossiers_by_procedure_id
.reject { |procedure_id, count| count < 20 || is_brouillon(procedure_id) }
.count,
nb_dossiers: nb_dossiers_by_procedure_id
.reject { |procedure_id, _count| is_brouillon(procedure_id) }
.map { |_procedure_id, count| count }
.sum,
nb_dossiers_max: nb_dossiers_by_procedure_id
.reject { |procedure_id, _count| is_brouillon(procedure_id) }
.map { |_procedure_id, count| count }
.max || 0,
nb_dossiers_traite: nb_dossiers_by_synthetic_state['termine'],
nb_dossiers_dossier_en_instruction: nb_dossiers_by_synthetic_state['en_instruction']
}
if administrateur.current_sign_in_at.present?
result[:ds_current_sign_in_at] = administrateur.current_sign_in_at
end
if administrateur.last_sign_in_at.present?
result[:ds_last_sign_in_at] = administrateur.last_sign_in_at
end
result
end
# Returns a hash { procedure_id => dossier_count }:
# - The keys are the ids of procedures owned by administrateur_id
# - The values are the number of dossiers for that procedure.
# Brouillons, and dossiers that are 'archive' but not 'termine', are not counted.
def nb_dossiers_by_procedure_id(administrateur_id)
with_default(
0,
nb_dossiers_by_administrateur_id_and_procedure_id_and_synthetic_state[administrateur_id]
.map do |procedure_id, nb_dossiers_by_synthetic_state|
[
procedure_id,
nb_dossiers_by_synthetic_state
.reject { |synthetic_state, _count| ['brouillon', 'archive'].include?(synthetic_state) }
.map { |_synthetic_state, count| count }
.sum
]
end
.to_h
)
end
# Returns a hash { synthetic_state => dossier_count }
# - The keys are dossier synthetic_states (see class comment)
# - The values are the number of dossiers in that synthetic state, for procedures owned by `administrateur_id`
# Dossier on procedures en test are not counted
def nb_dossiers_by_synthetic_state(administrateur_id)
with_default(
0,
nb_dossiers_by_administrateur_id_and_procedure_id_and_synthetic_state[administrateur_id]
.reject { |procedure_id, _nb_dossiers_by_synthetic_state| is_brouillon(procedure_id) }
.flat_map { |_procedure_id, nb_dossiers_by_synthetic_state| nb_dossiers_by_synthetic_state.to_a }
.group_by { |synthetic_state, _count| synthetic_state }
.map { |synthetic_state, synthetic_states_and_counts| [synthetic_state, synthetic_states_and_counts.map { |_synthetic_state, count| count }.sum] }
.to_h
)
end
def nb_demarches_by_administrateur_id_and_state
@nb_demarches_by_administrateur_id_and_state ||= with_default(0, Procedure.group(:administrateur_id, :aasm_state).count)
end
def nb_services_by_administrateur_id
@nb_services_by_administrateur_id ||= with_default(0, Service.group(:administrateur_id).count)
end
def nb_instructeurs_by_administrateur_id
@nb_instructeurs_by_administrateur_id ||= with_default(0, Administrateur.joins(:gestionnaires).group(:administrateur_id).count)
end
def nb_dossiers_by_administrateur_id_and_procedure_id_and_synthetic_state
if @nb_dossiers_by_administrateur_id_and_procedure_id_and_synthetic_state.present?
return @nb_dossiers_by_administrateur_id_and_procedure_id_and_synthetic_state
end
result = {}
Dossier
.joins(:procedure)
.group(
:administrateur_id,
:procedure_id,
<<~EOSQL
CASE
WHEN state IN('accepte', 'refuse', 'sans_suite') THEN 'termine'
WHEN archived THEN 'archive'
ELSE state
END
EOSQL
)
.count
.each do |(administrateur_id, procedure_id, synthetic_state), count|
result.deep_merge!(
{ administrateur_id => { procedure_id => { synthetic_state => count } } }
)
end
@nb_dossiers_by_administrateur_id_and_procedure_id_and_synthetic_state =
with_default({}, result)
end
def is_brouillon(procedure_id)
procedure_states[procedure_id] == 'brouillon'
end
def procedure_states
@procedure_states ||= Procedure.pluck(:id, :aasm_state).to_h
end
def with_default(default, hash)
hash.default = default
hash
end
end

27
app/views/layouts/_sendinblue.html.haml
View file

@ -1,27 +0,0 @@
:javascript
(function() {
window.sib = { equeue: [], client_key: "pcxtf4lpkka986pf4l1kt" };
/* OPTIONAL: email for identify request*/
window.sib.email_id = '#{current_administrateur.email}';
window.sendinblue = {}; for (var j = ['track', 'identify', 'trackLink', 'page'], i = 0; i < j.length; i++) { (function(k) { window.sendinblue[k] = function() { var arg = Array.prototype.slice.call(arguments); (window.sib[k] || function() { var t = {}; t[k] = arg; window.sib.equeue.push(t);})(arg[0], arg[1], arg[2]);};})(j[i]);}var n = document.createElement("script"),i = document.getElementsByTagName("script")[0]; n.type = "text/javascript", n.id = "sendinblue-js", n.async = !0, n.src = "https://sibautomation.com/sa.js?key=" + window.sib.client_key, i.parentNode.insertBefore(n, i), window.sendinblue.page();
})();
sendinblue.identify('#{current_administrateur.email}', {
'DS_NB_DEMARCHES_BROUILLONS': '#{current_administrateur.procedures.brouillons.count}',
'DS_NB_DEMARCHES_ACTIVES': '#{current_administrateur.procedures.publiees.count}',
'DS_NB_DEMARCHES_ARCHIVES': '#{current_administrateur.procedures.archivees.count}',
'DS_SIGN_IN_COUNT' : '#{current_administrateur.sign_in_count}',
'DS_CREATED_AT' : '#{current_administrateur.created_at}',
'DS_ACTIVE' : '#{current_administrateur.active}',
'DS_ID' : '#{current_administrateur.id}'
// Dans l'ideal :
// 'nom' : //pour personnaliser les emails
// 'prenom' : //pour personnaliser les emails
// 'nb_demarches_prod' : // Avec plus de 20 dossiers
// 'nb_demarches_test' : // Avec entre 1 et 20 dossiers
// 'nb_services' : //combien de service ?
// 'nb_instructeurs' : //combien d'instructeur en tout ?
// 'nb_dossiers' : //combien de dossier en tout ?
});

1
app/views/layouts/application_old.html.haml
View file

@ -17,7 +17,6 @@
= csrf_meta_tags
= render partial: "layouts/matomo"
= render partial: "layouts/sendinblue"
:javascript
DATA = [{

1
config/initializers/urls.rb
View file

@ -6,6 +6,7 @@ API_GEO_URL = ENV.fetch("API_GEO_URL", "https://geo.api.gouv.fr")
API_GEO_SANDBOX_URL = ENV.fetch("API_GEO_SANDBOX_URL", "https://sandbox.geo.api.gouv.fr")
HELPSCOUT_API_URL = ENV.fetch("HELPSCOUT_API_URL", "https://api.helpscout.net/v2")
PIPEDRIVE_API_URL = ENV.fetch("PIPEDRIVE_API_URL", "https://api.pipedrive.com/v1")
SENDINBLUE_API_URL = ENV.fetch("SENDINBLUE_API_URL", "https://in-automate.sendinblue.com/api/v2")
# Internal URLs
FOG_BASE_URL = "https://static.demarches-simplifiees.fr"

2
config/secrets.yml
View file

@ -51,6 +51,8 @@ defaults: &defaults
client_id: <%= ENV['HELPSCOUT_CLIENT_ID'] %>
client_secret: <%= ENV['HELPSCOUT_CLIENT_SECRET'] %>
webhook_secret: <%= ENV['HELPSCOUT_WEBHOOK_SECRET'] %>
sendinblue:
client_key: <%= ENV['SENDINBLUE_CLIENT_KEY'] %>
development:
<<: *defaults

218
spec/services/administrateur_usage_statistics_service_spec.rb Normal file
View file

@ -0,0 +1,218 @@
require 'spec_helper'
describe AdministrateurUsageStatisticsService do
describe '#administrateur_stats' do
let(:service) { AdministrateurUsageStatisticsService.new }
subject { service.send(:administrateur_stats, administrateur) }
before { Timecop.freeze(Time.zone.now) }
after { Timecop.return }
context 'for an administrateur that has nothing' do
let(:administrateur) { create(:administrateur) }
it do
is_expected.to eq(
ds_sign_in_count: 0,
ds_created_at: Time.zone.now,
ds_active: false,
ds_id: administrateur.id,
ds_features: "{}",
nb_services: 0,
nb_instructeurs: 0,
ds_nb_demarches_actives: 0,
ds_nb_demarches_archives: 0,
ds_nb_demarches_brouillons: 0,
nb_demarches_test: 0,
nb_demarches_prod: 0,
nb_demarches_prod_20: 0,
nb_dossiers: 0,
nb_dossiers_max: 0,
nb_dossiers_traite: 0,
nb_dossiers_dossier_en_instruction: 0
)
end
end
context 'for an administrateur that has plenty of things' do
let(:administrateur) do
create(:administrateur,
sign_in_count: 17,
current_sign_in_at: Time.zone.local(2019, 3, 7),
last_sign_in_at: Time.zone.local(2019, 2, 27),
active: true,
features: { holy_hand_grenade_of_antioch: true },
services: [create(:service)],
gestionnaires: [create(:gestionnaire)])
end
it do
is_expected.to include(
ds_sign_in_count: 17,
ds_current_sign_in_at: Time.zone.local(2019, 3, 7),
ds_last_sign_in_at: Time.zone.local(2019, 2, 27),
ds_created_at: Time.zone.now,
ds_active: true,
ds_id: administrateur.id,
ds_features: { holy_hand_grenade_of_antioch: true }.to_json,
nb_services: 1,
nb_instructeurs: 1
)
end
end
context 'counting procedures and dossiers' do
let(:administrateur) do
create(:administrateur, procedures: [procedure])
end
context 'with a freshly active procedure' do
let(:procedure) { create(:procedure, aasm_state: 'publiee') }
it do
is_expected.to include(
ds_nb_demarches_actives: 1,
ds_nb_demarches_archives: 0,
ds_nb_demarches_brouillons: 0,
nb_demarches_test: 0,
nb_demarches_prod: 0,
nb_demarches_prod_20: 0,
nb_dossiers: 0,
nb_dossiers_max: 0,
nb_dossiers_traite: 0,
nb_dossiers_dossier_en_instruction: 0
)
end
end
context 'with a procedure archivee' do
let(:procedure) { create(:procedure, aasm_state: 'archivee', dossiers: dossiers) }
let(:dossiers) do
(1..7).flat_map do
[
create(:dossier, :en_construction),
create(:dossier, :en_instruction),
create(:dossier, :accepte)
]
end
end
it do
is_expected.to include(
ds_nb_demarches_actives: 0,
ds_nb_demarches_archives: 1,
ds_nb_demarches_brouillons: 0,
nb_demarches_test: 0,
nb_demarches_prod: 1,
nb_demarches_prod_20: 1,
nb_dossiers: 21,
nb_dossiers_max: 21,
nb_dossiers_traite: 7,
nb_dossiers_dossier_en_instruction: 7
)
end
end
context 'with a procedure brouillon' do
let(:procedure) { create(:procedure) }
it do
is_expected.to include(
ds_nb_demarches_actives: 0,
ds_nb_demarches_archives: 0,
ds_nb_demarches_brouillons: 1,
nb_demarches_test: 0,
nb_demarches_prod: 0,
nb_demarches_prod_20: 0,
nb_dossiers: 0,
nb_dossiers_max: 0,
nb_dossiers_traite: 0,
nb_dossiers_dossier_en_instruction: 0
)
end
end
context 'with a procedure en test' do
let(:procedure) { create(:procedure, dossiers: dossiers) }
let(:dossiers) do
(1..7).flat_map do
[
create(:dossier, :en_construction),
create(:dossier, :en_instruction),
create(:dossier, :accepte)
]
end
end
it do
is_expected.to include(
ds_nb_demarches_actives: 0,
ds_nb_demarches_archives: 0,
ds_nb_demarches_brouillons: 1,
nb_demarches_test: 1,
nb_demarches_prod: 0,
nb_demarches_prod_20: 0,
nb_dossiers: 0,
nb_dossiers_max: 0,
nb_dossiers_traite: 0,
nb_dossiers_dossier_en_instruction: 0
)
end
end
context 'with a procedure en prod' do
let(:procedure) { create(:procedure, aasm_state: 'publiee', dossiers: dossiers) }
let(:dossiers) do
[
create(:dossier, :en_construction),
create(:dossier, :en_instruction),
create(:dossier, :accepte)
]
end
it do
is_expected.to include(
ds_nb_demarches_actives: 1,
ds_nb_demarches_archives: 0,
ds_nb_demarches_brouillons: 0,
nb_demarches_test: 0,
nb_demarches_prod: 1,
nb_demarches_prod_20: 0,
nb_dossiers: 3,
nb_dossiers_max: 3,
nb_dossiers_traite: 1,
nb_dossiers_dossier_en_instruction: 1
)
end
end
context 'with a procedure en prod and more than 20 dossiers' do
let(:procedure) { create(:procedure, aasm_state: 'publiee', dossiers: dossiers) }
let(:dossiers) do
(1..7).flat_map do
[
create(:dossier, :en_construction),
create(:dossier, :en_instruction),
create(:dossier, :accepte)
]
end
end
it do
is_expected.to include(
ds_nb_demarches_actives: 1,
ds_nb_demarches_archives: 0,
ds_nb_demarches_brouillons: 0,
nb_demarches_test: 0,
nb_demarches_prod: 1,
nb_demarches_prod_20: 1,
nb_dossiers: 21,
nb_dossiers_max: 21,
nb_dossiers_traite: 7,
nb_dossiers_dossier_en_instruction: 7
)
end
end
end
end
end