Merge pull request #6246 from betagouv/6207-optim-poids-archives

6207 optime la page qui liste les archives

app/assets/stylesheets/archive.scss

@@ -0,0 +1,15 @@
+@import "constants";
+
+table.archive-table {
+  .text-right {
+    text-align: right;
+  }
+
+  .center {
+    text-align: center;
+  }
+
+  td {
+    padding: 3 * $default-spacer $default-spacer;
+  }
+}

app/controllers/instructeurs/archives_controller.rb

@@ -4,12 +4,14 @@ module Instructeurs
 
     def index
       @procedure = procedure
+      @average_dossier_weight = procedure.average_dossier_weight
 
-      @archivable_months = archivable_months
-      @dossiers_termines = @procedure.dossiers.state_termine
-      @poids_total = ProcedureArchiveService.procedure_files_size(@procedure)
-      groupe_instructeur = current_instructeur.groupe_instructeurs.where(procedure: @procedure.id).first
-      @archives = Archive.for_groupe_instructeur(groupe_instructeur)
+      @count_dossiers_termines_by_month = Traitement.count_dossiers_termines_by_month(groupe_instructeurs)
+      @nb_dossiers_termines = @count_dossiers_termines_by_month.sum { |count_by_month| count_by_month["count"] }
+
+      @archives = Archive
+        .for_groupe_instructeur(groupe_instructeurs)
+        .to_a
     end
 
     def create
@@ -35,21 +37,20 @@ module Instructeurs
       end
     end
 
-    def archivable_months
-      start_date = procedure.published_at.to_date
-      end_date = Time.zone.now.to_date
+    def procedure_id
+      params[:procedure_id]
+    end
 
-      (start_date...end_date)
-        .map(&:beginning_of_month)
-        .uniq
-        .reverse
+    def groupe_instructeurs
+      current_instructeur
+        .groupe_instructeurs
+        .where(procedure_id: procedure_id)
     end
 
     def procedure
       current_instructeur
         .procedures
-        .for_download
-        .find(params[:procedure_id])
+        .find(procedure_id)
     end
   end
 end

app/helpers/archive_helper.rb

@@ -2,4 +2,12 @@ module ArchiveHelper
   def can_generate_archive?(dossiers_termines, poids_total)
     dossiers_termines.count < 100 && poids_total < 1.gigabyte
   end
+
+  def estimate_weight(archive, nb_dossiers_termines, average_dossier_weight)
+    if archive.present? && archive.available?
+      archive.file.byte_size
+    else
+      nb_dossiers_termines * average_dossier_weight
+    end
+  end
 end

app/models/procedure.rb

@@ -688,6 +688,20 @@ class Procedure < ApplicationRecord
     draft_revision.deep_clone(include: [:revision_types_de_champ, :revision_types_de_champ_private])
   end
 
+  def average_dossier_weight
+    if dossiers.termine.any?
+      dossiers_sample = dossiers.termine.limit(100)
+      total_size = Champ
+        .includes(piece_justificative_file_attachment: :blob)
+        .where(type: Champs::PieceJustificativeChamp.to_s, dossier: dossiers_sample)
+        .sum('active_storage_blobs.byte_size')
+
+      total_size / dossiers_sample.length
+    else
+      nil
+    end
+  end
+
   private
 
   def before_publish

app/models/traitement.rb

@@ -18,4 +18,21 @@ class Traitement < ApplicationRecord
       .where('dossiers.state' => Dossier::TERMINE)
       .where("traitements.processed_at + (procedures.duree_conservation_dossiers_dans_ds * INTERVAL '1 month') - INTERVAL :expires_in < :now", { now: Time.zone.now, expires_in: Dossier::INTERVAL_BEFORE_EXPIRATION })
   end
+
+  def self.count_dossiers_termines_by_month(groupe_instructeurs)
+    last_traitements_per_dossier = Traitement
+      .select('max(traitements.processed_at) as processed_at')
+      .where(dossier: Dossier.termine.where(groupe_instructeur: groupe_instructeurs))
+      .group(:dossier_id)
+      .to_sql
+
+    sql = <<~EOF
+      select date_trunc('month', r1.processed_at) as month, count(r1.processed_at)
+      from (#{last_traitements_per_dossier}) as r1
+      group by date_trunc('month', r1.processed_at)
+      order by month desc
+    EOF
+
+    ActiveRecord::Base.connection.execute(sql)
+  end
 end

app/views/instructeurs/archives/index.html.haml

@@ -5,7 +5,7 @@
                     'Archives'] }
 
 .container
-  %h1 Archives
+  %h1.mb-2 Archives
 
   .card.featured
     .card-title Gestion de vos archives
@@ -21,29 +21,25 @@
       = link_to 'la documentation', ARCHIVAGE_DOC_URL
       afin de voir les options à votre disposition pour mettre en place un système d’archive.
 
-  %table.table.hoverable
+  %table.table.hoverable.archive-table
     %thead
       %tr
         %th  
-        %th Nombre de dossiers terminés
-        %th Poids estimé
-        %th Télécharger
+        %th.text-right Nombre de dossiers terminés
+        %th.text-right Poids estimé
+        %th.center Télécharger
 
     %tbody
-    - if can_generate_archive?(@dossiers_termines, @poids_total)
       %tr
-        - matching_archive = @archives.find_by(time_span_type: 'everything')
+        - matching_archive = @archives.find { |archive| archive.time_span_type == 'everything' }
+        - weight = estimate_weight(matching_archive, @nb_dossiers_termines, @average_dossier_weight)
         %td
           Tous les dossiers
-        %td
-          = @dossiers_termines.count
-        %td
-          - if matching_archive.present? && matching_archive.available?
-            - weight = matching_archive.file.byte_size
-          - else
-            - weight = @poids_total
+        %td.text-right
+          = @nb_dossiers_termines
+        %td.text-right
           = number_to_human_size(weight)
-        %td
+        %td.center
           - if matching_archive.try(&:available?)
             = link_to url_for(matching_archive.file), class: 'button primary' do
               %span.icon.download-white
@@ -51,41 +47,40 @@
           - elsif matching_archive.try(&:pending?)
             %span.icon.retry
             = t(:archive_pending_html, created_period: time_ago_in_words(matching_archive.created_at), scope: [:instructeurs, :procedure])
-          - elsif @dossiers_termines.count > 0
+          - elsif @nb_dossiers_termines == 0
+            Rien à télécharger !
+          - elsif weight < 1.gigabyte
             = link_to instructeur_archives_path(@procedure, type: 'everything'), method: :post, class: "button" do
               %span.icon.new-folder
               Demander la création
           - else
-            Rien à télécharger !
-    - @archivable_months.each do |month|
-      - dossiers_termines = @procedure.dossiers.processed_in_month(month)
-      - nb_dossiers_termines = dossiers_termines.count
-      - matching_archive = @archives.find_by(time_span_type: 'monthly', month: month)
+            Archive trop volumineuse
+    - @count_dossiers_termines_by_month.each do |count_by_month|
+      - month = count_by_month["month"].to_date
+      - nb_dossiers_termines = count_by_month["count"]
+      - matching_archive = @archives.find { |archive| archive.time_span_type == 'monthly' && archive.month == month }
+      - weight = estimate_weight(matching_archive, nb_dossiers_termines, @average_dossier_weight)
+
       %tr
         %td
-          = I18n.l(month, format: "%B %Y")
-        %td
+          = I18n.l(month, format: "%B %Y").capitalize
+        %td.text-right
           = nb_dossiers_termines
-        %td
-          - if matching_archive.present? && matching_archive.available?
-            - weight = matching_archive.file.byte_size
-          - else
-            - weight = ProcedureArchiveService::dossiers_files_size(dossiers_termines)
+        %td.text-right
           = number_to_human_size(weight)
-        %td
-          - if nb_dossiers_termines > 0
-            - if matching_archive.present?
-              - if matching_archive.status == 'generated' && matching_archive.file.attached?
-                = link_to url_for(matching_archive.file), class: 'button primary' do
-                  %span.icon.download-white
-                  = t(:archive_ready_html, generated_period: time_ago_in_words(matching_archive.updated_at), scope: [:instructeurs, :procedure])
-              - else
-                %span.icon.retry
-                = t(:archive_pending_html, created_period: time_ago_in_words(matching_archive.created_at), scope: [:instructeurs, :procedure])
+        %td.center
+          - if matching_archive.present?
+            - if matching_archive.status == 'generated' && matching_archive.file.attached?
+              = link_to url_for(matching_archive.file), class: 'button primary' do
+                %span.icon.download-white
+                = t(:archive_ready_html, generated_period: time_ago_in_words(matching_archive.updated_at), scope: [:instructeurs, :procedure])
             - else
-              = link_to instructeur_archives_path(@procedure, type:'monthly', month: month.strftime('%Y-%m')), method: :post, class: "button" do
-                %span.icon.new-folder
-                Démander la création
+              %span.icon.retry
+              = t(:archive_pending_html, created_period: time_ago_in_words(matching_archive.created_at), scope: [:instructeurs, :procedure])
+          - elsif weight < 1.gigabyte
+            = link_to instructeur_archives_path(@procedure, type:'monthly', month: month.strftime('%Y-%m')), method: :post, class: "button" do
+              %span.icon.new-folder
+              Démander la création
           - else
-            Rien à télécharger !
+            Archive trop volumineuse
 

spec/controllers/instructeurs/archives_controller_spec.rb

@@ -25,7 +25,7 @@ describe Instructeurs::ArchivesController, type: :controller do
     it 'displays archives' do
       get :index, { params: { procedure_id: procedure1.id } }
 
-      expect(assigns(:dossiers_termines).size).to eq(3)
+      expect(assigns(:nb_dossiers_termines).size).to eq(8)
       expect(assigns(:archives)).to eq([archive1])
     end
   end

spec/factories/champ.rb

@@ -138,8 +138,12 @@ FactoryBot.define do
     factory :champ_piece_justificative, class: 'Champs::PieceJustificativeChamp' do
       type_de_champ { association :type_de_champ_piece_justificative, procedure: dossier.procedure }
 
-      after(:build) do |champ, _evaluator|
-        champ.piece_justificative_file.attach(io: StringIO.new("toto"), filename: "toto.txt", content_type: "text/plain")
+      transient do
+        size { 4 }
+      end
+
+      after(:build) do |champ, evaluator|
+        champ.piece_justificative_file.attach(io: StringIO.new("x" * evaluator.size), filename: "toto.txt", content_type: "text/plain")
       end
     end
 

spec/helpers/archive_helper_spec.rb

@@ -0,0 +1,24 @@
+describe ArchiveHelper, type: :helper do
+  describe ".estimate_weight" do
+    let(:nb_dossiers_termines) { 5 }
+    let(:average_dossier_weight) { 2 }
+
+    context 'when archive exist and available' do
+      let(:archive) { build(:archive, :generated) }
+      before do
+        allow_any_instance_of(Archive).to receive(:available?).and_return(true)
+      end
+
+      it 'returns real archive weight' do
+        expect(estimate_weight(archive, nb_dossiers_termines, average_dossier_weight)).to eq nil
+      end
+    end
+
+    context 'when archive has not been created' do
+      let(:archive) { nil }
+      it 'returns estimation' do
+        expect(estimate_weight(archive, nb_dossiers_termines, average_dossier_weight)).to eq 10
+      end
+    end
+  end
+end

spec/models/procedure_spec.rb

@@ -1069,4 +1069,25 @@ describe Procedure do
       expect(procedure.destroy).to be_truthy
     end
   end
+
+  describe '#average_dossier_weight' do
+    let(:procedure) { create(:procedure, :published) }
+
+    before do
+      create_dossier_with_pj_of_size(4, procedure)
+      create_dossier_with_pj_of_size(5, procedure)
+      create_dossier_with_pj_of_size(6, procedure)
+    end
+
+    it 'estimates average dossier weight' do
+      expect(procedure.reload.average_dossier_weight).to eq 5
+    end
+
+    private
+
+    def create_dossier_with_pj_of_size(size, procedure)
+      dossier = create(:dossier, :accepte, procedure: procedure)
+      create(:champ_piece_justificative, size: size, dossier: dossier)
+    end
+  end
 end

spec/models/traitement_spec.rb

@@ -0,0 +1,37 @@
+describe Traitement do
+  describe '#count_dossiers_termines_by_month' do
+    let(:procedure) { create(:procedure, :published, groupe_instructeurs: [groupe_instructeurs]) }
+    let(:groupe_instructeurs) { create(:groupe_instructeur) }
+    let(:result) { Traitement.count_dossiers_termines_by_month(groupe_instructeurs) }
+
+    before do
+      create_dossier_for_month(procedure, 2021, 3)
+      create_dossier_for_month(procedure, 2021, 3)
+      create_dossier_for_month(procedure, 2021, 2)
+      Timecop.freeze(Time.zone.local(2021, 3, 5))
+    end
+
+    it 'count dossiers_termines by month' do
+      expect(count_for_month(result, 3)).to eq 2
+      expect(count_for_month(result, 2)).to eq 1
+    end
+
+    it 'returns descending order by month' do
+      expect(result[0]["month"].to_date.month).to eq 3
+      expect(result[1]["month"].to_date.month).to eq 2
+    end
+  end
+
+  private
+
+  def count_for_month(count_dossiers_termines_by_month, month)
+    count_dossiers_termines_by_month.find do |count_by_month|
+      count_by_month["month"].to_date.month == month
+    end["count"]
+  end
+
+  def create_dossier_for_month(procedure, year, month)
+    Timecop.freeze(Time.zone.local(year, month, 5))
+    create(:dossier, :accepte, :with_attestation, procedure: procedure)
+  end
+end