Merge pull request #9708 from mfo/US/better-estimation-on-treatment-time

ETQ usager, l information de durée de traitement des 90 percentile peut être trompeur

app/components/procedure/estimated_delay_component.rb

@@ -0,0 +1,26 @@
+class Procedure::EstimatedDelayComponent < ApplicationComponent
+  delegate :distance_of_time_in_words, to: :helpers
+
+  def initialize(procedure:)
+    @procedure = procedure
+    @fastest, @mean, @slow = @procedure.stats_usual_traitement_time
+  end
+
+  def estimation_present?
+    @fastest && @mean && @slow
+  end
+
+  def render?
+    estimation_present?
+  end
+
+  def cleaned_nearby_estimation
+    [@fastest, @mean, @slow]
+      .map { distance_of_time_in_words(_1) }
+      .uniq
+      .zip(['fast_html', 'mean_html', 'slow_html'])
+      .each do |estimation, i18n_key|
+        yield(estimation, i18n_key)
+      end
+  end
+end

app/components/procedure/estimated_delay_component/estimated_delay_component.en.yml

@@ -0,0 +1,6 @@
+---
+en:
+  explanation: "Based on %{percentile}% during the last %{nb_recent_dossiers} days, the instruction time is :"
+  fast_html: "<strong>In the best cast scenario</strong> : <strong>%{estimation}</strong>."
+  mean_html: "If your file <strong>requires minor adjustments</strong>, the instruction time is  <strong>%{estimation}</strong>."
+  slow_html: "If you file <strong>is missing some information</strong> which requires a lot of exchanges with the administration, the instruction time is around <strong>%{estimation}</strong>."

app/components/procedure/estimated_delay_component/estimated_delay_component.fr.yml

@@ -0,0 +1,6 @@
+---
+fr:
+  explanation: Selon nos estimations, à partir des délais d’instruction constatés sur %{percentile}% des demandes qui ont été traitées lors des %{nb_recent_dossiers} derniers jours, les délais d’instruction sont les suivants
+  fast_html: "<strong>Dans le meilleur des cas</strong>, le délai d’instruction est : <strong>%{estimation}</strong>."
+  mean_html: "Les <strong>dossiers demandant quelques échanges</strong> le délai d’instruction est d‘environ : <strong>%{estimation}</strong>."
+  slow_html: "Si votre <strong>dossier est incomplet</strong> ou qu’il faut beaucoup d’échanges avec l’administration, le délai d’instruction est d’environ <strong>%{estimation}</strong>."

app/components/procedure/estimated_delay_component/estimated_delay_component.html.haml

@@ -0,0 +1,5 @@
+%p= t('.explanation', percentile: ProcedureStatsConcern::USUAL_TRAITEMENT_TIME_PERCENTILE, nb_recent_dossiers: ProcedureStatsConcern::NB_DAYS_RECENT_DOSSIERS)
+%ul
+  - cleaned_nearby_estimation do |estimation, i18n_key|
+    %li= t(".#{i18n_key}", estimation: estimation)
+

app/models/concerns/procedure_stats_concern.rb

@@ -83,14 +83,18 @@ module ProcedureStatsConcern
 
   def usual_traitement_time_for_recent_dossiers(nb_days)
     now = Time.zone.now
+    clusters_count = 3
+
     traitement_time =
       traitement_times((now - nb_days.days)..now)
         .map { |times| times[:processed_at] - times[:depose_at] }
-        .percentile(USUAL_TRAITEMENT_TIME_PERCENTILE)
-        .ceil
-
-    traitement_time = nil if traitement_time == 0
-    traitement_time
+        .sort
+    if traitement_time.size >= clusters_count
+      traitement_time.each_slice((traitement_time.size.to_f / clusters_count.to_f).ceil)
+        .map { _1.percentile(USUAL_TRAITEMENT_TIME_PERCENTILE) }
+    else
+      nil
+    end
   end
 
   private

app/views/shared/_procedure_description.html.haml

@@ -64,14 +64,15 @@
               %ul
                 = render partial: "shared/procedure_pieces_jointes_list", collection: procedure.pieces_jointes_list_with_conditionnal, as: :pj
 
-    - if procedure.stats_usual_traitement_time
+    - estimated_delay_component = Procedure::EstimatedDelayComponent.new(procedure: procedure)
+    - if estimated_delay_component.render?
       %section.fr-accordion
         %h2.fr-accordion__title
           %button.fr-accordion__btn{ "aria-controls" => "accordion-117", "aria-expanded" => "false" }
             = t('shared.procedure_description.usual_traitement_time_title')
 
         #accordion-117.fr-collapse
-          = t('shared.procedure_description.usual_traitement_time_detail_html', traitement_time: distance_of_time_in_words(procedure.stats_usual_traitement_time), percentile: ProcedureStatsConcern::USUAL_TRAITEMENT_TIME_PERCENTILE, days: ProcedureStatsConcern::NB_DAYS_RECENT_DOSSIERS, href: statistiques_path(procedure.path) )
+          = render estimated_delay_component
 
 
     - if procedure.persisted? && procedure.estimated_duration_visible?

app/views/users/dossiers/show/_estimated_delay.html.haml

@@ -1,11 +0,0 @@
-/ FIXME: remove the custom procedure switch at some point
-- procedure_id_for_which_we_hide_the_estimated_delay = 6547
-- procedure_path_for_which_we_hide_the_estimated_delay = 'deposer-une-offre-de-stage'
-- show_time_means = procedure.id != procedure_id_for_which_we_hide_the_estimated_delay && procedure.path != procedure_path_for_which_we_hide_the_estimated_delay
-
-- cache(procedure.id, expires_in: 1.day) do
-  - if procedure.usual_traitement_time_for_recent_dossiers(ProcedureStatsConcern::NB_DAYS_RECENT_DOSSIERS) && show_time_means
-    %p
-      Habituellement, les dossiers de cette démarche sont traités dans un délai de #{distance_of_time_in_words(procedure.usual_traitement_time_for_recent_dossiers(ProcedureStatsConcern::NB_DAYS_RECENT_DOSSIERS))}.
-    %p
-      Cette estimation est calculée automatiquement à partir des délais d’instruction constatés sur #{ProcedureStatsConcern::USUAL_TRAITEMENT_TIME_PERCENTILE}% des demandes qui ont été traitées lors des #{ProcedureStatsConcern::NB_DAYS_RECENT_DOSSIERS} derniers jours. Le délai réel peut être différent, en fonction du type de démarche (par exemple pour un appel à projet avec date de décision fixe).

app/views/users/dossiers/show/_status_overview.html.haml

@@ -33,8 +33,7 @@
           %p{ role: 'status' }
             = t('views.users.dossiers.show.status_overview.en_construction_html')
 
-          = render partial: 'users/dossiers/show/estimated_delay', locals: { procedure: dossier.procedure }
-
+          = render Procedure::EstimatedDelayComponent.new(procedure: dossier.procedure)
           %p
             = t('views.users.dossiers.show.status_overview.use_mailbox_for_questions_html', mailbox_url: messagerie_dossier_url(dossier))
 
@@ -50,7 +49,7 @@
                 %p
                   = t('views.users.dossiers.show.status_overview.delay_text_sva_svr', date: l(dossier.sva_svr_decision_on, format: :long))
 
-          = render partial: 'users/dossiers/show/estimated_delay', locals: { procedure: dossier.procedure }
+          = render Procedure::EstimatedDelayComponent.new(procedure: dossier.procedure)
 
           %p
             = t('views.users.dossiers.show.status_overview.use_mailbox_for_questions_html', mailbox_url: messagerie_dossier_url(dossier))

config/locales/en.yml

@@ -860,7 +860,6 @@ en:
       estimated_fill_duration_title: What is the procedure estimated fill time ?
       estimated_fill_duration_detail: "The fill time is etimated to %{estimated_minutes} min. This period may vary depending on the options you choose"
       usual_traitement_time_title: What are the processing times for this procedure?
-      usual_traitement_time_detail_html: Usually, files in this procedure are processed within %{traitement_time}. This estimate is calculated automatically from the processing times observed on %{percentile}% of requests that have been processed during the last %{days} days. Actual lead time may vary. For more information, you can <a href="%{href}" target="_blank" rel="noopener noreferrer">consult the statistics page</a> of this procedure.
       pieces_jointes : What are the required attachments ?
       pieces_jointes_conditionnal_list_title : Attachments list according to your situation
       sva_svr_title: "This procedure applies the « %{rule} »"

config/locales/fr.yml

@@ -916,7 +916,6 @@ fr:
       estimated_fill_duration_title: Quelle est la durée de remplissage de la démarche ?
       estimated_fill_duration_detail: "La durée de remplissage est estimée à %{estimated_minutes} min. Ce délai peut varier selon les options que vous choisirez."
       usual_traitement_time_title: Quels sont les délais d'instruction pour cette démarche ?
-      usual_traitement_time_detail_html: Habituellement, les dossiers de cette démarche sont traités dans un délai de %{traitement_time}. Cette estimation est calculée automatiquement à partir des délais d’instruction constatés sur %{percentile}% des demandes qui ont été traitées lors des %{days} derniers jours. Le délai réel peut être différent. Pour plus d'informations, vous pouvez <a href="%{href}" target="_blank" rel="noopener noreferrer">consulter la page statistiques</a> de cette démarche.
       pieces_jointes : Quelles sont les pièces justificatives à fournir ?
       pieces_jointes_conditionnal_list_title : Liste des pièces en fonction de votre situation
       sva_svr_title: "Cette démarche applique le « %{rule} »"

spec/fixtures/files/data/treatment-expected-3months.csv

@@ -0,0 +1,166 @@
+treatment_time
+1203843.06283
+673451.312319
+1552383.903455
+1558932.226583
+78953.851181
+5873677.701851
+4835744.936453
+1631478.524708
+861503.515445
+6814713.490218
+1461462.194324
+7948780.040199
+5339562.163289
+4815573.982363
+2557727.157024
+734848.454544
+5608902.330823
+146232.296199
+1703064.586672
+6547384.633624
+8727001.10326
+11834102.923026
+4765722.551394
+7144458.281005
+12078699.718379
+10791621.642477
+9728695.298227
+8898231.452671
+8614031.387295
+3612278.486039
+3099530.130012
+251825.410353
+2323903.490576
+4740543.429931
+5479723.048413
+58456.557576
+15816134.927362
+14083018.694661
+13925115.674794
+14413258.730125
+13219627.40563
+12167836.514329
+10288196.54087
+9159560.364292
+8452343.420395
+7226105.156653
+6986605.850082
+6139436.648255
+367809.55081
+7494981.090698
+15099034.271656
+6795865.968545
+7530658.801911
+23062760.185918
+22481020.792829
+23600464.237565
+19860640.860815
+11560589.34153
+21182717.435281
+19344094.290496
+5725859.591795
+6041115.639448
+9151354.506646
+17115471.478197
+15889250.745431
+18317558.238709
+15377601.995007
+6902877.478941
+11910474.059424
+8887242.141369
+6883053.792433
+5704549.604276
+8451867.368427
+12354634.38643
+6833405.134158
+11743220.126897
+6976430.091759
+15987516.272823
+6403848.206615
+8888113.588524
+10704916.153456
+27218018.181598
+8363801.252001
+12879049.401543
+7276014.459025
+12597161.777912
+21336664.044928
+12845340.622423
+24995902.606571
+23254108.742917
+7506117.642471
+13905245.60054
+12287247.584552
+7352936.085941
+8566220.354052
+10368657.535501
+15906330.563098
+16002838.219734
+14700391.249919
+7070984.790126
+22365269.429071
+21765196.514427
+9914694.677426
+14689961.266011
+12964639.144383
+9778265.323858
+9328862.93155
+7619712.137327
+7600640.729466
+12973111.352368
+17080325.525328
+23336659.500634
+20052186.759343
+12976004.639476
+12977253.811306
+17094274.254913
+8798280.910486
+11916285.159021
+26429950.828791
+14076903.134024
+9196105.575002
+21768324.914312
+14062259.428949
+7657750.048284
+10725178.088743
+23787478.797878
+12878477.046701
+20224285.919799
+23093716.442285
+22803621.363356
+13294707.956396
+12436442.520349
+17099624.683714
+12887043.034098
+8488204.538622
+12681098.717788
+12878734.794363
+13023470.726935
+14490874.679906
+35305778.218032
+19238942.783451
+21400362.163813
+23249302.30862
+18038933.602232
+23404216.157602
+26773977.72012
+31718870.185568
+23221152.814961
+23240562.752522
+17721166.26928
+30761746.125519
+35223871.718381
+18456657.228819
+25996946.138323
+16948792.627604
+27124016.406777
+34395629.288828
+16479826.146673
+36630962.342172
+18931647.947773
+31700548.203521
+33533741.312976
+31470438.299417
+19465934.747886
+20581103.945882

spec/models/concern/procedure_stats_concern_spec.rb

@@ -30,7 +30,7 @@ describe ProcedureStatsConcern do
       Timecop.freeze(Time.utc(2019, 6, 1, 12, 0))
 
       delays.each do |delay|
-        create_dossier(construction_date: 1.week.ago - delay, instruction_date: 1.week.ago - delay + 12.hours, processed_date: 1.week.ago)
+        create_dossier(depose_at: 1.week.ago - delay, en_instruction_at: 1.week.ago - delay + 12.hours, processed_at: 1.week.ago)
       end
     end
 
@@ -40,37 +40,59 @@ describe ProcedureStatsConcern do
       let(:delays) { [1.day, 2.days, 2.days, 2.days, 2.days, 3.days, 3.days, 3.days, 3.days, 12.days] }
 
       it 'returns a time representative of the dossier instruction delay' do
-        expect(procedure.usual_traitement_time_for_recent_dossiers(30)).to be_between(3.days, 4.days)
+        expect(procedure.usual_traitement_time_for_recent_dossiers(30)[0]).to be_between(1.day, 2.days)
+        expect(procedure.usual_traitement_time_for_recent_dossiers(30)[1]).to be_between(2.days, 3.days)
+        expect(procedure.usual_traitement_time_for_recent_dossiers(30)[2]).to be_between(11.days, 12.days)
       end
     end
 
     context 'when there are very old dossiers' do
-      let(:delays) { [2.days, 2.days] }
-      let!(:old_dossier) { create_dossier(construction_date: 3.months.ago, instruction_date: 2.months.ago, processed_date: 2.months.ago) }
+      let(:delays) { [1.day, 2.days, 3.days, 3.days, 4.days] }
+      let!(:old_dossier) { create_dossier(depose_at: 3.months.ago, en_instruction_at: 2.months.ago, processed_at: 2.months.ago) }
 
       it 'ignores dossiers older than 1 month' do
-        expect(procedure.usual_traitement_time_for_recent_dossiers(30)).to be_within(1.hour).of(2.days)
+        expect(procedure.usual_traitement_time_for_recent_dossiers(30)[0]).to be_between(1.day, 2.days)
+        expect(procedure.usual_traitement_time_for_recent_dossiers(30)[1]).to be_between(2.days, 3.days)
+        expect(procedure.usual_traitement_time_for_recent_dossiers(30)[2]).to be_between(3.days, 4.days)
       end
     end
 
     context 'when there is a dossier with bad data' do
-      let(:delays) { [2.days, 2.days] }
-      let!(:bad_dossier) { create_dossier(construction_date: nil, instruction_date: nil, processed_date: 10.days.ago) }
+      let(:delays) { [1.day, 2.days, 3.days, 3.days, 4.days] }
+      let!(:bad_dossier) { create_dossier(depose_at: nil, en_instruction_at: nil, processed_at: 10.days.ago) }
 
       it 'ignores bad dossiers' do
-        expect(procedure.usual_traitement_time_for_recent_dossiers(30)).to be_within(1.hour).of(2.days)
+        expect(procedure.usual_traitement_time_for_recent_dossiers(30)[0]).to be_between(21.hours, 36.hours.days)
+        expect(procedure.usual_traitement_time_for_recent_dossiers(30)[1]).to be_between(2.days, 3.days)
+        expect(procedure.usual_traitement_time_for_recent_dossiers(30)[2]).to be_between(3.days, 4.days)
       end
     end
 
     context 'when there is only one processed dossier' do
       let(:delays) { [1.day] }
-      it { expect(procedure.usual_traitement_time_for_recent_dossiers(30)).to be_within(1.hour).of(1.day) }
+      it { expect(procedure.usual_traitement_time_for_recent_dossiers(30)).to be_nil }
     end
 
     context 'where there is no processed dossier' do
       let(:delays) { [] }
       it { expect(procedure.usual_traitement_time_for_recent_dossiers(30)).to eq nil }
     end
+
+    context 'with real data' do
+      include ActionView::Helpers::DateHelper
+      let(:delays) { [] }
+      before do
+        csv = CSV.read(Rails.root.join('spec/fixtures/files/data/treatment-expected-3months.csv'))
+        traitement_times = csv[1..] # strip header
+          .flatten
+          .map { { processed_at: _1.to_f, depose_at: 0 } }
+        allow(procedure).to receive(:traitement_times).and_return(traitement_times)
+      end
+
+      it 'works' do
+        expect(procedure.usual_traitement_time_for_recent_dossiers(30).map { distance_of_time_in_words(_1) }).to eq(["3 mois", "6 mois", "environ un an"])
+      end
+    end
   end
 
   describe '.usual_traitement_time_by_month_in_days' do
@@ -79,8 +101,8 @@ describe ProcedureStatsConcern do
     def create_dossiers(delays_by_month)
       delays_by_month.each_with_index do |delays, index|
         delays.each do |delay|
-          processed_date = (index.months + 1.week).ago
-          create_dossier(construction_date: processed_date - delay, instruction_date: processed_date - delay + 12.hours, processed_date: processed_date)
+          processed_at = (index.months + 1.week).ago
+          create_dossier(depose_at: processed_at - delay, en_instruction_at: processed_at - delay + 12.hours, processed_at: processed_at)
         end
       end
     end
@@ -112,7 +134,7 @@ describe ProcedureStatsConcern do
 
   private
 
-  def create_dossier(construction_date:, instruction_date:, processed_date:)
-    dossier = create(:dossier, :accepte, procedure: procedure, depose_at: construction_date, en_instruction_at: instruction_date, processed_at: processed_date)
+  def create_dossier(depose_at:, en_instruction_at:, processed_at:)
+    dossier = create(:dossier, :accepte, procedure: procedure, depose_at:, en_instruction_at:, processed_at:)
   end
 end

spec/system/users/dossier_details_spec.rb

@@ -22,13 +22,11 @@ describe 'Dossier details:' do
   end
 
   describe "the user can see the mean time they are expected to wait" do
-    let(:other_dossier) { create(:dossier, :accepte, :with_individual, procedure: procedure, depose_at: 10.days.ago, en_instruction_at: 9.days.ago, processed_at: Time.zone.now) }
-
     context "when the dossier is in construction" do
       it "displays the estimated wait duration" do
-        other_dossier
+        allow_any_instance_of(Procedure).to receive(:stats_usual_traitement_time).and_return([1.day, 1.day, 1.day])
         visit dossier_path(dossier)
-        expect(page).to have_text("Habituellement, les dossiers de cette démarche sont traités dans un délai de 10 jours.")
+        expect(page).to have_text("Dans le meilleur des cas, le délai d’instruction est : 1 jour")
       end
     end
 
@@ -36,9 +34,9 @@ describe 'Dossier details:' do
       let(:dossier) { create(:dossier, :en_instruction, :with_individual, :with_commentaires, user: user, procedure: procedure) }
 
       it "displays the estimated wait duration" do
-        other_dossier
+        allow_any_instance_of(Procedure).to receive(:stats_usual_traitement_time).and_return([1.day, 1.day, 1.day])
         visit dossier_path(dossier)
-        expect(page).to have_text("Habituellement, les dossiers de cette démarche sont traités dans un délai de 10 jours.")
+        expect(page).to have_text("Dans le meilleur des cas, le délai d’instruction est : 1 jour")
       end
     end
   end

spec/views/shared/_procedure_description.html.haml_spec.rb

@@ -49,13 +49,13 @@ describe 'shared/_procedure_description', type: :view do
 
   context 'when procedure has usual_traitement_time' do
     before do
-      allow(procedure).to receive(:stats_usual_traitement_time).and_return(1.day)
+      allow(procedure).to receive(:stats_usual_traitement_time).and_return([1.day, 1.day, 1.day])
     end
 
     it 'shows a usual traitement text' do
       subject
       expect(rendered).to have_text("Quels sont les délais d'instruction pour cette démarche ?")
-      expect(rendered).to have_text("Habituellement, les dossiers de cette démarche sont traités dans un délai de 1 jour.")
+      expect(rendered).to have_text("Dans le meilleur des cas, le délai d’instruction est : 1 jour.")
     end
   end
 

spec/views/users/dossiers/show/_status_overview.html.haml_spec.rb

@@ -1,5 +1,5 @@
 describe 'users/dossiers/show/_status_overview', type: :view do
-  before { allow(dossier.procedure).to receive(:usual_traitement_time_for_recent_dossiers).and_return(1.day) }
+  before { allow(dossier.procedure).to receive(:usual_traitement_time_for_recent_dossiers).and_return([1.day, 2.days, 3.days]) }
 
   subject! { render 'users/dossiers/show/status_overview', dossier: dossier }
 
@@ -34,8 +34,14 @@ describe 'users/dossiers/show/_status_overview', type: :view do
       expect(rendered).to have_timeline_item('.termine').inactive
     end
 
-    it { is_expected.to have_selector('.status-explanation .en-construction') }
-    it { is_expected.to have_text('Habituellement, les dossiers de cette démarche sont traités dans un délai de 1 jour') }
+    it 'works' do
+      subject
+      expect(subject).to have_selector('.status-explanation .en-construction')
+      expect(subject).to have_text('Selon nos estimations, à partir des délais d’instruction constatés')
+      expect(subject).to have_text("Dans le meilleur des cas, le délai d’instruction est : 1 jour.")
+      expect(subject).to have_text("Les dossiers demandant quelques échanges le délai d’instruction est d‘environ : 2 jours.")
+      expect(subject).to have_text("Si votre dossier est incomplet ou qu’il faut beaucoup d’échanges avec l’administration, le délai d’instruction est d’environ 3 jours.")
+    end
   end
 
   context 'when en instruction' do
@@ -48,8 +54,13 @@ describe 'users/dossiers/show/_status_overview', type: :view do
       expect(rendered).to have_timeline_item('.termine').inactive
     end
 
-    it { is_expected.to have_selector('.status-explanation .en-instruction') }
-    it { is_expected.to have_text('Habituellement, les dossiers de cette démarche sont traités dans un délai de 1 jour') }
+    it 'works' do
+      expect(subject).to have_selector('.status-explanation .en-instruction')
+      expect(subject).to have_text('Selon nos estimations, à partir des délais d’instruction constatés')
+      expect(subject).to have_text("Dans le meilleur des cas, le délai d’instruction est : 1 jour.")
+      expect(subject).to have_text("Les dossiers demandant quelques échanges le délai d’instruction est d‘environ : 2 jours.")
+      expect(subject).to have_text("Si votre dossier est incomplet ou qu’il faut beaucoup d’échanges avec l’administration, le délai d’instruction est d’environ 3 jours.")
+    end
   end
 
   context 'when accepté' do