Merge pull request #6800 from betagouv/main

2022-01-05-01

Gemfile

@@ -47,6 +47,7 @@ gem 'i18n_data'
 gem 'i18n-tasks', require: false
 gem 'iban-tools'
 gem 'image_processing'
+gem 'invisible_captcha'
 gem 'json_schemer'
 gem 'jwt'
 gem 'kaminari', '1.2.1' # Pagination

Gemfile.lock

@@ -21,40 +21,40 @@ GEM
     aasm (5.2.0)
       concurrent-ruby (~> 1.0)
     acsv (0.0.1)
-    actioncable (6.1.4.1)
+    actioncable (6.1.4.4)
-      actionpack (= 6.1.4.1)
+      actionpack (= 6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.4.1)
+    actionmailbox (6.1.4.4)
-      actionpack (= 6.1.4.1)
+      actionpack (= 6.1.4.4)
-      activejob (= 6.1.4.1)
+      activejob (= 6.1.4.4)
-      activerecord (= 6.1.4.1)
+      activerecord (= 6.1.4.4)
-      activestorage (= 6.1.4.1)
+      activestorage (= 6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
       mail (>= 2.7.1)
-    actionmailer (6.1.4.1)
+    actionmailer (6.1.4.4)
-      actionpack (= 6.1.4.1)
+      actionpack (= 6.1.4.4)
-      actionview (= 6.1.4.1)
+      actionview (= 6.1.4.4)
-      activejob (= 6.1.4.1)
+      activejob (= 6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.4.1)
+    actionpack (6.1.4.4)
-      actionview (= 6.1.4.1)
+      actionview (= 6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.4.1)
+    actiontext (6.1.4.4)
-      actionpack (= 6.1.4.1)
+      actionpack (= 6.1.4.4)
-      activerecord (= 6.1.4.1)
+      activerecord (= 6.1.4.4)
-      activestorage (= 6.1.4.1)
+      activestorage (= 6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
       nokogiri (>= 1.8.5)
-    actionview (6.1.4.1)
+    actionview (6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -72,26 +72,26 @@ GEM
       activemodel (>= 5.2.0)
       activestorage (>= 5.2.0)
       activesupport (>= 5.2.0)
-    activejob (6.1.4.1)
+    activejob (6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
       globalid (>= 0.3.6)
-    activemodel (6.1.4.1)
+    activemodel (6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
-    activerecord (6.1.4.1)
+    activerecord (6.1.4.4)
-      activemodel (= 6.1.4.1)
+      activemodel (= 6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
-    activestorage (6.1.4.1)
+    activestorage (6.1.4.4)
-      actionpack (= 6.1.4.1)
+      actionpack (= 6.1.4.4)
-      activejob (= 6.1.4.1)
+      activejob (= 6.1.4.4)
-      activerecord (= 6.1.4.1)
+      activerecord (= 6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
       marcel (~> 1.0.0)
       mini_mime (>= 1.1.0)
     activestorage-openstack (1.5.1)
       fog-openstack (~> 1.0)
       marcel
       rails (>= 5.2.2)
-    activesupport (6.1.4.1)
+    activesupport (6.1.4.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -295,7 +295,7 @@ GEM
       raabro (~> 1.4)
     geo_coord (0.2.0)
     geocoder (1.6.5)
-    globalid (0.5.2)
+    globalid (1.0.0)
       activesupport (>= 5.0)
     gon (6.4.0)
       actionpack (>= 3.0.20)
@@ -369,6 +369,8 @@ GEM
     image_processing (1.12.1)
       mini_magick (>= 4.9.5, < 5)
       ruby-vips (>= 2.0.17, < 3)
+    invisible_captcha (2.0.0)
+      rails (>= 5.0)
     ipaddress (0.8.3)
     jquery-rails (4.4.0)
       rails-dom-testing (>= 1, < 3)
@@ -424,7 +426,7 @@ GEM
       activesupport (>= 3.1.0)
       rack (>= 1.4.0)
       rest-client (>= 2.0.0)
-    marcel (1.0.1)
+    marcel (1.0.2)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
@@ -513,20 +515,20 @@ GEM
       rack
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (6.1.4.1)
+    rails (6.1.4.4)
-      actioncable (= 6.1.4.1)
+      actioncable (= 6.1.4.4)
-      actionmailbox (= 6.1.4.1)
+      actionmailbox (= 6.1.4.4)
-      actionmailer (= 6.1.4.1)
+      actionmailer (= 6.1.4.4)
-      actionpack (= 6.1.4.1)
+      actionpack (= 6.1.4.4)
-      actiontext (= 6.1.4.1)
+      actiontext (= 6.1.4.4)
-      actionview (= 6.1.4.1)
+      actionview (= 6.1.4.4)
-      activejob (= 6.1.4.1)
+      activejob (= 6.1.4.4)
-      activemodel (= 6.1.4.1)
+      activemodel (= 6.1.4.4)
-      activerecord (= 6.1.4.1)
+      activerecord (= 6.1.4.4)
-      activestorage (= 6.1.4.1)
+      activestorage (= 6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
       bundler (>= 1.15.0)
-      railties (= 6.1.4.1)
+      railties (= 6.1.4.4)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
@@ -545,9 +547,9 @@ GEM
     rails-i18n (6.0.0)
       i18n (>= 0.7, < 2)
       railties (>= 6.0.0, < 7)
-    railties (6.1.4.1)
+    railties (6.1.4.4)
-      actionpack (= 6.1.4.1)
+      actionpack (= 6.1.4.4)
-      activesupport (= 6.1.4.1)
+      activesupport (= 6.1.4.4)
       method_source
       rake (>= 0.13)
       thor (~> 1.0)
@@ -706,9 +708,9 @@ GEM
     sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
-    sprockets-rails (3.2.2)
+    sprockets-rails (3.4.2)
-      actionpack (>= 4.0)
+      actionpack (>= 5.2)
-      activesupport (>= 4.0)
+      activesupport (>= 5.2)
       sprockets (>= 3.0.0)
     swd (1.3.0)
       activesupport (>= 3)
@@ -718,7 +720,7 @@ GEM
     temple (0.8.2)
     terminal-table (3.0.0)
       unicode-display_width (~> 1.1, >= 1.1.1)
-    thor (1.1.0)
+    thor (1.2.1)
     thread_safe (0.3.6)
     tilt (2.0.10)
     timecop (0.9.4)
@@ -774,7 +776,7 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.5.1)
+    zeitwerk (2.5.3)
     zip_tricks (5.6.0)
     zxcvbn-ruby (1.2.0)
 
@@ -838,6 +840,7 @@ DEPENDENCIES
   i18n_data
   iban-tools
   image_processing
+  invisible_captcha
   json_schemer
   jwt
   kaminari (= 1.2.1)

app/controllers/ping_controller.rb

@@ -4,7 +4,7 @@ class PingController < ApplicationController
       status_code = if File.file?(Rails.root.join("maintenance"))
         # See https://cbonte.github.io/haproxy-dconv/2.0/configuration.html#4.2-http-check%20disable-on-404
         :not_found
-      elsif (ActiveRecord::Base.connected?)
+      elsif (ActiveRecord::Base.connection.execute('select 1 as test;').first['test'] == 1)
         :ok
       else
         :internal_server_error

app/controllers/support_controller.rb

@@ -1,4 +1,6 @@
 class SupportController < ApplicationController
+  invisible_captcha only: [:create], on_spam: :redirect_to_root
+
   def index
     setup_context
   end
@@ -92,4 +94,8 @@ class SupportController < ApplicationController
   def email
     current_user&.email || params[:email]
   end
+
+  def redirect_to_root
+    redirect_to root_path, alert: t('invisible_captcha.custom_message')
+  end
 end

app/lib/active_storage/downloadable_file.rb

@@ -1,20 +1,5 @@
+
 class ActiveStorage::DownloadableFile
-  # https://edgeapi.rubyonrails.org/classes/ActiveStorage/Blob.html#method-i-download
-  def self.download(attachment:, destination_path:, in_chunk: true)
-    byte_written = 0
-
-    File.open(destination_path, mode: 'wb') do |fd| # we expact a path as string, so we can recreate the file (ex: failure/retry on former existing fd)
-      if in_chunk
-        attachment.download do |chunk|
-          byte_written += fd.write(chunk)
-        end
-      else
-        byte_written = fd.write(attachment.download)
-      end
-    end
-    byte_written
-  end
-
   def self.create_list_from_dossier(dossier, for_expert = false)
     dossier_export = PiecesJustificativesService.generate_dossier_export(dossier)
     pjs = [dossier_export] + PiecesJustificativesService.liste_documents(dossier, for_expert)

app/lib/download_manager/parallel_download_queue.rb

@@ -0,0 +1,61 @@
+module DownloadManager
+  class ParallelDownloadQueue
+    include Utils::Retryable
+    DOWNLOAD_MAX_PARALLEL = ENV.fetch('DOWNLOAD_MAX_PARALLEL') { 10 }
+
+    attr_accessor :attachments,
+                  :destination,
+                  :on_error
+
+    def initialize(attachments, destination)
+      @attachments = attachments
+      @destination = destination
+    end
+
+    def download_all
+      hydra = Typhoeus::Hydra.new(max_concurrency: DOWNLOAD_MAX_PARALLEL)
+
+      attachments.map do |attachment, path|
+        begin
+          with_retry(max_attempt: 1) do
+            download_one(attachment: attachment,
+                         path_in_download_dir: path,
+                         http_client: hydra)
+          end
+        rescue => e
+          on_error.call(attachment, path, e)
+        end
+      end
+      hydra.run
+    end
+
+    # rubocop:disable Style/AutoResourceCleanup
+    # can't be used with typhoeus, otherwise block is closed before the request is run by hydra
+    def download_one(attachment:, path_in_download_dir:, http_client:)
+      attachment_path = File.join(destination, path_in_download_dir)
+      attachment_dir = File.dirname(attachment_path)
+
+      FileUtils.mkdir_p(attachment_dir) if !Dir.exist?(attachment_dir) # defensive, do not write in undefined dir
+      if attachment.is_a?(PiecesJustificativesService::FakeAttachment)
+        File.write(attachment_path, attachment.file.read, mode: 'wb')
+      else
+        request = Typhoeus::Request.new(attachment.url)
+        fd = File.open(attachment_path, mode: 'wb')
+        request.on_body do |chunk|
+          fd.write(chunk)
+        end
+        request.on_complete do |response|
+          fd.close
+          unless response.success?
+            raise 'ko'
+          end
+        end
+        http_client.queue(request)
+      end
+    rescue
+      File.delete(attachment_path) if File.exist?(attachment_path) # -> case of retries failed, must cleanup partialy downloaded file
+      raise
+    end
+    # rubocop:enable Style/AutoResourceCleanup
+  end
+end

app/lib/download_manager/procedure_attachments_export.rb

@@ -0,0 +1,34 @@
+module DownloadManager
+  class ProcedureAttachmentsExport
+    delegate :destination, to: :@queue
+
+    attr_reader :queue
+    attr_accessor :errors
+
+    def initialize(procedure, attachments, destination)
+      @procedure = procedure
+      @errors = {}
+      @queue = ParallelDownloadQueue.new(attachments, destination)
+      @queue.on_error = proc do |_attachment, path, error|
+        errors[path] = true
+        Rails.logger.error("Fail to download filename #{path} in procedure##{@procedure.id}, reason: #{error}")
+      end
+     end
+
+    def download_all
+      @queue.download_all
+      write_report if !errors.empty?
+    end
+
+    private
+
+    def write_report
+      manifest_path = File.join(destination, 'LISEZMOI.txt')
+      manifest_content = errors.map do |file_basename, _failed|
+                                                      "Impossible de récupérer le fichier #{file_basename}"
+                                                    end
+        .join("\n")
+      File.write(manifest_path, manifest_content)
+    end
+  end
+end

app/services/procedure_archive_service.rb

@@ -1,7 +1,6 @@
 require 'tempfile'
 
 class ProcedureArchiveService
-  include Utils::Retryable
   ARCHIVE_CREATION_DIR = ENV.fetch('ARCHIVE_CREATION_DIR') { '/tmp' }
 
   def initialize(procedure)
@@ -37,7 +36,6 @@ class ProcedureArchiveService
       archive.file.attach(
         io: File.open(zip_file),
         filename: archive.filename(@procedure),
-        # we don't want to run virus scanner on this file
         metadata: { virus_scan_result: ActiveStorage::VirusScanner::SAFE }
       )
     end
@@ -104,32 +102,12 @@ class ProcedureArchiveService
         FileUtils.remove_entry_secure(archive_dir) if Dir.exist?(archive_dir)
         Dir.mkdir(archive_dir)
 
-        bug_reports = ''
+        download_manager = DownloadManager::ProcedureAttachmentsExport.new(@procedure, attachments, archive_dir)
-        attachments.each do |attachment, path|
+        download_manager.download_all
-          attachment_path = File.join(archive_dir, path)
-          attachment_dir = File.dirname(attachment_path)
 
-          FileUtils.mkdir_p(attachment_dir) if !Dir.exist?(attachment_dir)
-          begin
-            with_retry(max_attempt: 1) do
-              ActiveStorage::DownloadableFile.download(attachment: attachment,
-                                                       destination_path: attachment_path,
-                                                       in_chunk: true)
-            end
-          rescue => e
-            Rails.logger.error("Fail to download filename #{File.basename(attachment_path)} in procedure##{@procedure.id}, reason: #{e}")
-            File.delete(attachment_path) if File.exist?(attachment_path)
-            bug_reports += "Impossible de récupérer le fichier #{File.basename(attachment_path)}\n"
-          end
-        end
-
-        if !bug_reports.empty?
-          File.write(File.join(archive_dir, 'LISEZMOI.txt'), bug_reports)
-        end
-
-        File.delete(zip_path) if File.exist?(zip_path)
         Dir.chdir(tmp_dir) do
-          system 'zip', '-r', zip_path, zip_root_folder
+          File.delete(zip_path) if File.exist?(zip_path)
+          system 'zip', '-0', '-r', zip_path, zip_root_folder
         end
         yield(zip_path)
       ensure
@@ -152,8 +130,8 @@ class ProcedureArchiveService
   def self.attachments_from_champs_piece_justificative(champs)
     champs
       .filter { |c| c.type_champ == TypeDeChamp.type_champs.fetch(:piece_justificative) }
-      .filter { |pj| pj.piece_justificative_file.attached? }
       .map(&:piece_justificative_file)
+      .filter(&:attached?)
   end
 
   def self.liste_pieces_justificatives_for_archive(dossier)

app/views/administrateurs/procedures/apercu.html.haml

@@ -2,7 +2,8 @@
   .container
     %h1.page-title Prévisualisation de la démarche « #{@dossier.procedure.libelle} »
 
-    %ul.tabs
+    %nav.tabs
+      %ul
         = tab_item('le dossier',
           apercu_admin_procedure_path(@dossier.procedure, tab: 'dossier'),
           active: @tab == 'dossier')

app/views/administrateurs/procedures/index.html.haml

@@ -3,7 +3,8 @@
     = link_to "Nouvelle Démarche", new_from_existing_admin_procedures_path, id: 'new-procedure', class: 'button primary'
   .container
 
-    %ul.tabs
+    %nav.tabs
+      %ul
         = tab_item(t('pluralize.published', count: @procedures_publiees.count), admin_procedures_path(statut: 'publiees'), active: @statut == 'publiees', badge: number_with_html_delimiter(@procedures_publiees_count))
         = tab_item('En test', admin_procedures_path(statut: 'brouillons'), active: @statut == 'brouillons', badge: number_with_html_delimiter(@procedures_draft_count))
         = tab_item(t('pluralize.closed', count: @procedures_closed.count), admin_procedures_path(statut: 'archivees'), active: @statut == 'archivees', badge: number_with_html_delimiter(@procedures_closed_count))

app/views/experts/avis/_header.html.haml

@@ -15,7 +15,8 @@
             - else
               %p.menu-item Le téléchargement des pièces jointes est désactivé pour les dossiers de plus de #{number_to_human_size Dossier::TAILLE_MAX_ZIP}.
 
-    %ul.tabs
+    %nav.tabs
+      %ul
         = dynamic_tab_item('Demande', expert_avis_path(avis.procedure, avis))
         = dynamic_tab_item('Avis', instruction_expert_avis_path(avis.procedure, avis), notification: avis.answer.blank?)
         = dynamic_tab_item('Messagerie', messagerie_expert_avis_path(avis.procedure, avis))

app/views/experts/avis/procedure.html.haml

@@ -11,7 +11,8 @@
       .procedure-header
         %h1= procedure_libelle @procedure
 
-        %ul.tabs
+        %nav.tabs
+          %ul
             = tab_item('avis à donner',
               procedure_expert_avis_index_path(statut: Instructeurs::AvisController::A_DONNER_STATUS),
               active: @statut == Instructeurs::AvisController::A_DONNER_STATUS,

app/views/instructeurs/avis/_header.html.haml

@@ -5,7 +5,8 @@
       %li= link_to(dossier.procedure.libelle, procedure_instructeur_avis_index_path(avis.procedure))
       %li= link_to("Dossier nº #{dossier.id}", instructeur_avis_path(avis.procedure, avis))
 
-    %ul.tabs
+    %nav.tabs
+      %ul
         = dynamic_tab_item('Demande', instructeur_avis_path(avis.procedure, avis))
         = dynamic_tab_item('Avis', instruction_instructeur_avis_path(avis.procedure, avis), notification: avis.answer.blank?)
         = dynamic_tab_item('Messagerie', messagerie_instructeur_avis_path(avis.procedure, avis))

app/views/instructeurs/avis/procedure.html.haml

@@ -11,7 +11,8 @@
       .procedure-header
         %h1= procedure_libelle @procedure
 
-        %ul.tabs
+        %nav.tabs
+          %ul
             = tab_item('avis à donner',
               procedure_instructeur_avis_index_path(statut: Instructeurs::AvisController::A_DONNER_STATUS),
               active: @statut == Instructeurs::AvisController::A_DONNER_STATUS,

app/views/instructeurs/dossiers/_header.html.haml

@@ -15,7 +15,8 @@
 
     = render(partial: 'instructeurs/dossiers/expiration_banner', locals: {dossier: dossier})
 
-    %ul.tabs
+    %nav.tabs
+      %ul
         - notifications_summary = current_instructeur.notifications_for_dossier(dossier)
 
         = dynamic_tab_item('Demande',

app/views/instructeurs/procedures/_tabs.html.haml

@@ -1,4 +1,5 @@
-%ul.tabs.mt-3
+%nav.tabs.mt-3
+  %ul
     = tab_item('à suivre',
       instructeur_procedure_path(procedure, statut: 'a-suivre'),
       active: statut == 'a-suivre',

app/views/root/patron.html.haml

@@ -244,7 +244,8 @@
   .sub-header
     .container
       Titre
-      %ul.tabs
+      %nav.tabs
+        %ul
           = tab_item("Onglet actif", "#", active: true)
           = tab_item("Onglet inactif", "#")
           = tab_item("Onglet avec badge", "#", badge: 2)

app/views/shared/_tab_item.html.haml

@@ -1,7 +1,7 @@
 %li{ class: (active ? 'active' : nil) }
   - if notification
     %span.notifications{ 'aria-label': 'notifications' }
-  = link_to(url) do
+  = link_to(url, 'aria-current': active ? 'page' : nil) do
     - if badge.present?
       %span.badge= badge
     = label

app/views/support/index.html.haml

@@ -25,6 +25,9 @@
         = label_tag :type do
           = t('.your_question')
         = hidden_field_tag :type, params[:type]
+
+      = invisible_captcha
+
       %dl
         - @options.each do |(question, question_type, link)|
           %dt

app/views/users/dossiers/index.html.haml

@@ -14,7 +14,8 @@
 
     - else
       %h1.page-title= t('views.users.dossiers.index.dossiers')
-      %ul.tabs
+      %nav.tabs
+        %ul
           - if @user_dossiers.present?
             = tab_item(t('pluralize.en_cours', count: @user_dossiers.count),
               dossiers_path(statut: 'en-cours'),

app/views/users/dossiers/show/_header.html.haml

@@ -26,7 +26,8 @@
               = link_to t('views.users.dossiers.show.header.print_dossier'), dossier_path(dossier, format: :pdf), target: "_blank", rel: "noopener", class: "menu-item menu-link"
 
 
-    %ul.tabs
+    %nav.tabs
+      %ul
         = dynamic_tab_item(t('views.users.dossiers.show.header.summary'), dossier_path(dossier))
         = dynamic_tab_item(t('views.users.dossiers.show.header.request'), [demande_dossier_path(dossier), modifier_dossier_path(dossier)])
         = dynamic_tab_item(t('views.users.dossiers.show.header.mailbox'), messagerie_dossier_path(dossier))

config/env.example

@@ -122,3 +122,5 @@ API_EDUCATION_URL="https://data.education.gouv.fr/api/records/1.0"
 
 # Clé de chriffrement des données sensibles en base
 ENCRYPTION_SERVICE_SALT=""
+
+INVISIBLE_CAPTCHA_SECRET="kikooloool"

config/initializers/invisible_captcha.rb

@@ -0,0 +1,12 @@
+InvisibleCaptcha.setup do |config|
+  # config.honeypots           << ['more', 'fake', 'attribute', 'names']
+  # config.visual_honeypots      = false
+  # config.timestamp_threshold   = 2
+  config.timestamp_enabled     = !Rails.env.test?
+  # config.injectable_styles     = false
+  config.spinner_enabled       = !Rails.env.test?
+
+  # Leave these unset if you want to use I18n (see below)
+  # config.sentence_for_humans     = 'If you are a human, ignore this field'
+  # config.timestamp_error_message = 'Sorry, that was too quick! Please resubmit.'
+end

config/locales/en.yml

@@ -30,6 +30,9 @@
 # available at http://guides.rubyonrails.org/i18n.html.
 
 en:
+  invisible_captcha:
+    custom_message: 'If you are a human, ignore this field'
+
   help: 'Help'
   utils:
     'yes': Yes

config/locales/fr.yml

@@ -20,6 +20,9 @@
 # available at http://guides.rubyonrails.org/i18n.html.
 
 fr:
+  invisible_captcha:
+    custom_message: 'Si vous êtes un humain, veuillez ignorer ce champs'
+
   help: 'Aide'
   utils:
     'yes': Oui

spec/controllers/ping_controller_spec.rb

@@ -6,10 +6,10 @@ describe PingController, type: :controller do
 
     context 'when base is un-plug' do
       before do
-        allow(ActiveRecord::Base).to receive(:connected?).and_return(false)
+        allow(ActiveRecord::Base).to receive(:connected?).and_raise(ActiveRecord::ConnectionTimeoutError)
       end
 
-      it { expect(subject.status).to eq 500 }
+      it { expect { subject }.to raise_error(ActiveRecord::ConnectionTimeoutError) }
     end
 
     context 'when a maintenance file is present' do

spec/controllers/support_controller_spec.rb

@@ -51,9 +51,12 @@ describe SupportController, type: :controller do
 
     describe "send form" do
       subject do
-        post :create, params: { subject: 'bonjour', text: 'un message' }
+        post :create, params: params
       end
 
+      context "when invisible captcha is ignored" do
+        let(:params) { { subject: 'bonjour', text: 'un message' } }
+
         it 'creates a conversation on HelpScout' do
           expect_any_instance_of(Helpscout::FormAdapter).to receive(:send_form).and_return(true)
 
@@ -114,6 +117,15 @@ describe SupportController, type: :controller do
           end
         end
       end
+
+      context "when invisible captcha is filled" do
+        let(:params) { { subject: 'bonjour', text: 'un message', InvisibleCaptcha.honeypots.sample => 'boom' } }
+        it 'does not create a conversation on HelpScout' do
+          expect { subject }.not_to change(Commentaire, :count)
+          expect(flash[:alert]).to eq(I18n.t('invisible_captcha.custom_message'))
+        end
+      end
+    end
   end
 
   context 'signed out' do

spec/fixtures/cassettes/archive/file_to_get.yml

@@ -0,0 +1,51 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://file.to/get.ext
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Last-Modified:
+      - Thu, 16 Dec 2021 13:04:07 GMT
+      X-Trans-Id:
+      - tx62bf43b03d7e4b60b3f25-0061d45dbd
+      Accept-Ranges:
+      - bytes
+      Expires:
+      - Tue, 04 Jan 2022 15:20:54 GMT
+      X-Openstack-Request-Id:
+      - tx62bf43b03d7e4b60b3f25-0061d45dbd
+      Content-Type:
+      - image/png
+      Date:
+      - Tue, 04 Jan 2022 14:46:21 GMT
+      X-Iplb-Request-Id:
+      - 877D6D0C:B8E4_5762BBC9:01BB_61D45DBD_104936A5:293F4
+      X-Timestamp:
+      - '1639659846.52947'
+      Etag:
+      - 49961feab1c277af65fcb876c379cebf
+      X-Iplb-Instance:
+      - '42085'
+      Content-Length:
+      - '494761'
+      Content-Disposition:
+      - inline; filename="Screen Shot 2021-12-09 at 9.42.44 AM.png"; filename*=UTF-8''Screen%20Shot%202021-12-09%20at%209.42.44%20AM.png
+      Strict-Transport-Security:
+      - max-age=63072000
+    body:
+      encoding: ASCII-8BIT
+      string: ''
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+recorded_with: VCR 6.0.0

spec/fixtures/cassettes/archive/file_to_get_typhoeus.yml

@@ -0,0 +1,145 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://i.etsystatic.com/6212702/r/il/744d2c/470726480/il_1588xN.470726480_bpk5.jpg
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Cache-Control:
+      - public, max-age=365000000, immutable
+      Content-Type:
+      - image/jpeg
+      Etag:
+      - '"J9Qt3QnUKZIbmKehfH+pmv/rYxafyM81ENfBKsCN6Qw"'
+      Expires:
+      - Mon, 02 Jan 2023 05:24:30 GMT
+      Fastly-Io-Info:
+      - ifsz=14677 idim=600x600 ifmt=jpeg ofsz=43339 odim=1588x1588 ofmt=jpeg
+      Fastly-Stats:
+      - io=1
+      Server:
+      - UploadServer
+      X-Goog-Generation:
+      - '1514228438620441'
+      X-Goog-Hash:
+      - crc32c=ZohETA==
+      - md5=iKPGsaOoUN0hhNZYYzYDLQ==
+      X-Goog-Metageneration:
+      - '1'
+      X-Goog-Storage-Class:
+      - MULTI_REGIONAL
+      X-Goog-Stored-Content-Encoding:
+      - identity
+      X-Goog-Stored-Content-Length:
+      - '14677'
+      X-Guploader-Uploadid:
+      - ADPycdvXmKF1KUStMVeN1v5TUKBQA_YezSueBDRwp4qiVKTn5IDoW7f3_t6_tyvJwjkOoUE4lO1cC_NxSl-LkM5ukthJcv6JkA
+      Via:
+      - 1.1 varnish, 1.1 varnish
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Tue, 04 Jan 2022 14:59:40 GMT
+      Age:
+      - '207310'
+      X-Served-By:
+      - cache-mdw17340-MDW, cache-cdg20755-CDG
+      X-Cache:
+      - HIT, HIT
+      X-Cache-Hits:
+      - 1, 1
+      X-Timer:
+      - S1641308380.238785,VS0,VE1
+      Vary:
+      - Accept
+      Strict-Transport-Security:
+      - max-age=300
+      Content-Length:
+      - '43339'
+    body:
+      encoding: ASCII-8BIT
+      string: ''
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+- request:
+    method: get
+    uri: https://i.etsystatic.com/6212702/r/il/744d2c/470726480/il_1588xN.470726480_bpk5.jpg
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Cache-Control:
+      - public, max-age=365000000, immutable
+      Content-Type:
+      - image/jpeg
+      Etag:
+      - '"J9Qt3QnUKZIbmKehfH+pmv/rYxafyM81ENfBKsCN6Qw"'
+      Expires:
+      - Mon, 02 Jan 2023 05:24:30 GMT
+      Fastly-Io-Info:
+      - ifsz=14677 idim=600x600 ifmt=jpeg ofsz=43339 odim=1588x1588 ofmt=jpeg
+      Fastly-Stats:
+      - io=1
+      Server:
+      - UploadServer
+      X-Goog-Generation:
+      - '1514228438620441'
+      X-Goog-Hash:
+      - crc32c=ZohETA==
+      - md5=iKPGsaOoUN0hhNZYYzYDLQ==
+      X-Goog-Metageneration:
+      - '1'
+      X-Goog-Storage-Class:
+      - MULTI_REGIONAL
+      X-Goog-Stored-Content-Encoding:
+      - identity
+      X-Goog-Stored-Content-Length:
+      - '14677'
+      X-Guploader-Uploadid:
+      - ADPycdvXmKF1KUStMVeN1v5TUKBQA_YezSueBDRwp4qiVKTn5IDoW7f3_t6_tyvJwjkOoUE4lO1cC_NxSl-LkM5ukthJcv6JkA
+      Via:
+      - 1.1 varnish, 1.1 varnish
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Tue, 04 Jan 2022 14:59:40 GMT
+      Age:
+      - '207310'
+      X-Served-By:
+      - cache-mdw17340-MDW, cache-cdg20737-CDG
+      X-Cache:
+      - HIT, HIT
+      X-Cache-Hits:
+      - 1, 1
+      X-Timer:
+      - S1641308380.241689,VS0,VE1
+      Vary:
+      - Accept
+      Strict-Transport-Security:
+      - max-age=300
+      Content-Length:
+      - '43339'
+    body:
+      encoding: ASCII-8BIT
+      string: ''
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+recorded_with: VCR 6.0.0

spec/lib/download_manager/parallel_download_queue_spec.rb

@@ -0,0 +1,44 @@
+describe DownloadManager::ParallelDownloadQueue do
+  let(:test_dir) { Dir.mktmpdir(nil, Dir.tmpdir) }
+  let(:download_to_dir) { test_dir }
+  before do
+    downloadable_manager.on_error = proc { |_, _, _| }
+  end
+
+  after { FileUtils.remove_entry_secure(test_dir) if Dir.exist?(test_dir) }
+
+  let(:downloadable_manager) { DownloadManager::ParallelDownloadQueue.new([attachment], download_to_dir) }
+  describe '#download_one' do
+    subject { downloadable_manager.download_one(attachment: attachment, path_in_download_dir: destination, http_client: double) }
+
+    let(:destination) { 'lol.png' }
+    let(:attachment) do
+      PiecesJustificativesService::FakeAttachment.new(
+        file: StringIO.new('coucou'),
+        filename: "export-dossier.pdf",
+        name: 'pdf_export_for_instructeur',
+        id: 1,
+        created_at: Time.zone.now
+      )
+    end
+
+    context 'with a PiecesJustificativesService::FakeAttachment and it works' do
+      it 'write attachment.file to disk' do
+        target = File.join(download_to_dir, destination)
+        expect { subject }.to change { File.exist?(target) }
+        attachment.file.rewind
+        expect(attachment.file.read).to eq(File.read(target))
+      end
+    end
+
+    context 'with a PiecesJustificativesService::FakeAttachment and it fails' do
+      it 'write attachment.file to disk' do
+        expect(attachment.file).to receive(:read).and_raise("boom")
+        target = File.join(download_to_dir, destination)
+        expect { subject }.to raise_error(StandardError)
+        expect(File.exist?(target)).to be_falsey
+        # expect(downloadable_manager.errors).to have_key(destination)
+      end
+    end
+  end
+end

spec/services/procedure_archive_service_spec.rb

@@ -128,10 +128,14 @@ describe ProcedureArchiveService do
       let(:archive) { create(:archive, time_span_type: 'monthly', status: 'pending', month: date_month) }
       let(:year) { 2021 }
       let(:mailer) { double('mailer', deliver_later: true) }
-
+      before do
+        allow_any_instance_of(ActiveStorage::Attached::One).to receive(:url).and_return("http://file.to/get.ext")
+      end
       it 'collect files' do
         expect(InstructeurMailer).to receive(:send_archive).and_return(mailer)
+        VCR.use_cassette('archive/file_to_get') do
           service.collect_files_archive(archive, instructeur)
+        end
 
         archive.file.open do |f|
           files = ZipTricks::FileReader.read_zip_structure(io: f)
@@ -207,11 +211,16 @@ describe ProcedureArchiveService do
     context 'for all months' do
       let(:archive) { create(:archive, time_span_type: 'everything', status: 'pending') }
       let(:mailer) { double('mailer', deliver_later: true) }
+      before do
+        allow_any_instance_of(ActiveStorage::Attached::One).to receive(:url).and_return("https://i.etsystatic.com/6212702/r/il/744d2c/470726480/il_1588xN.470726480_bpk5.jpg")
+      end
 
       it 'collect files' do
         expect(InstructeurMailer).to receive(:send_archive).and_return(mailer)
 
+        VCR.use_cassette('archive/file_to_get_typhoeus') do
           service.collect_files_archive(archive, instructeur)
+        end
 
         archive = Archive.last
         archive.file.open do |f|
@@ -234,6 +243,36 @@ describe ProcedureArchiveService do
     end
   end
 
+  describe '#download_and_zip' do
+    it 'create a tmpdir while block is running' do
+      previous_dir_list = Dir.entries(ProcedureArchiveService::ARCHIVE_CREATION_DIR)
+
+      service.send(:download_and_zip, []) do |_zip_file|
+        new_dir_list = Dir.entries(ProcedureArchiveService::ARCHIVE_CREATION_DIR)
+        expect(previous_dir_list).not_to eq(new_dir_list)
+      end
+    end
+
+    it 'cleans up its tmpdir after block execution' do
+      expect { service.send(:download_and_zip, []) { |zip_file| } }
+        .not_to change { Dir.entries(ProcedureArchiveService::ARCHIVE_CREATION_DIR) }
+    end
+
+    it 'creates a zip with zip utility' do
+      expected_zip_path = File.join(ProcedureArchiveService::ARCHIVE_CREATION_DIR, "#{service.send(:zip_root_folder)}.zip")
+      expect(service).to receive(:system).with('zip', '-0', '-r', expected_zip_path, an_instance_of(String))
+      service.send(:download_and_zip, []) { |zip_path| }
+    end
+
+    it 'cleans up its generated zip' do
+      expected_zip_path = File.join(ProcedureArchiveService::ARCHIVE_CREATION_DIR, "#{service.send(:zip_root_folder)}.zip")
+      service.send(:download_and_zip, []) do |_zip_path|
+        expect(File.exist?(expected_zip_path)).to be_truthy
+      end
+      expect(File.exist?(expected_zip_path)).to be_falsey
+    end
+  end
+
   private
 
   def create_dossier_for_month(year, month)

spec/views/users/dossiers/index.html.haml_spec.rb

@@ -56,7 +56,7 @@ describe 'users/dossiers/index.html.haml', type: :view do
     end
 
     it 'n’affiche la barre d’onglets' do
-      expect(rendered).to have_selector('ul.tabs')
+      expect(rendered).to have_selector('nav.tabs')
     end
   end
 
@@ -68,9 +68,9 @@ describe 'users/dossiers/index.html.haml', type: :view do
     end
 
     it 'affiche la barre d’onglets' do
-      expect(rendered).to have_selector('ul.tabs')
+      expect(rendered).to have_selector('nav.tabs')
-      expect(rendered).to have_selector('ul.tabs li', count: 4)
+      expect(rendered).to have_selector('nav.tabs li', count: 4)
-      expect(rendered).to have_selector('ul.tabs li.active', count: 1)
+      expect(rendered).to have_selector('nav.tabs li.active', count: 1)
     end
   end
 

spec/views/users/dossiers/show/_header.html.haml_spec.rb

@@ -12,7 +12,7 @@ describe 'users/dossiers/show/header.html.haml', type: :view do
     expect(rendered).to have_text("Dossier nº #{dossier.id}")
     expect(rendered).to have_text("en construction")
 
-    expect(rendered).to have_selector("ul.tabs")
+    expect(rendered).to have_selector("nav.tabs")
     expect(rendered).to have_link("Résumé", href: dossier_path(dossier))
     expect(rendered).to have_link("Demande", href: demande_dossier_path(dossier))
   end