extract download all attachments in dedicated class using async/async-http for better perf

Gemfile

@@ -10,6 +10,8 @@ gem 'administrate'
 gem 'administrate-field-enum' # Allow using Field::Enum in administrate
 gem 'after_party'
 gem 'anchored'
+gem 'async'
+gem 'async-http'
 gem 'bcrypt'
 gem 'bootsnap', '>= 1.4.4', require: false # Reduces boot times through caching; required in config/boot.rb
 gem 'browser'

Gemfile.lock

@@ -118,6 +118,21 @@ GEM
       activerecord (>= 3.2, < 7.0)
       rake (>= 10.4, < 14.0)
     ast (2.4.2)
+    async (1.30.1)
+      console (~> 1.10)
+      nio4r (~> 2.3)
+      timers (~> 4.1)
+    async-http (0.56.5)
+      async (>= 1.25)
+      async-io (>= 1.28)
+      async-pool (>= 0.2)
+      protocol-http (~> 0.22.0)
+      protocol-http1 (~> 0.14.0)
+      protocol-http2 (~> 0.14.0)
+    async-io (1.32.2)
+      async
+    async-pool (0.3.9)
+      async (>= 1.25)
     attr_encrypted (3.1.0)
       encryptor (~> 3.0.0)
     attr_required (1.0.1)
@@ -183,6 +198,8 @@ GEM
       descendants_tracker (~> 0.0.1)
     concurrent-ruby (1.1.9)
     connection_pool (2.2.3)
+    console (1.14.0)
+      fiber-local
     crack (0.4.5)
       rexml
     crass (1.0.6)
@@ -268,6 +285,7 @@ GEM
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
     ffi (1.15.4)
+    fiber-local (1.0.0)
     flipper (0.20.3)
     flipper-active_record (0.20.3)
       activerecord (>= 5.0, < 7)
@@ -485,6 +503,13 @@ GEM
       actionmailer (>= 3)
       premailer (~> 1.7, >= 1.7.9)
     promise.rb (0.7.4)
+    protocol-hpack (1.4.2)
+    protocol-http (0.22.5)
+    protocol-http1 (0.14.2)
+      protocol-http (~> 0.22)
+    protocol-http2 (0.14.2)
+      protocol-hpack (~> 1.4)
+      protocol-http (~> 0.18)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -724,6 +749,7 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.10)
     timecop (0.9.4)
+    timers (4.3.3)
     ttfunk (1.7.0)
     typhoeus (1.4.0)
       ethon (>= 0.9.0)
@@ -795,6 +821,8 @@ DEPENDENCIES
   after_party
   anchored
   annotate
+  async
+  async-http
   axe-core-rspec
   bcrypt
   bootsnap (>= 1.4.4)

app/lib/active_storage/download_manager.rb

@@ -0,0 +1,87 @@
+require 'async'
+require 'async/barrier'
+require 'async/http/internet'
+
+class ActiveStorage::DownloadManager
+  include Utils::Retryable
+  DOWNLOAD_MAX_PARALLEL = ENV.fetch('DOWNLOAD_MAX_PARALLEL') { 10 }
+
+  attr_reader :download_to_dir, :errors
+
+  def download_all(attachments:, on_failure:)
+    Async do
+      internet = Async::HTTP::Internet.new
+      barrier = Async::Barrier.new
+      semaphore = Async::Semaphore.new(DOWNLOAD_MAX_PARALLEL, parent: barrier)
+
+      attachments.map do |attachment, path|
+        semaphore.async do
+          begin
+            with_retry(max_attempt: 1) do
+              download_one(attachment: attachment,
+                           path_in_download_dir: path,
+                           async_internet: internet)
+            end
+          rescue => e
+            on_failure.call(attachment, path, e)
+          end
+        end
+      end
+      barrier.wait
+      write_error_manifest if !errors.empty?
+    ensure
+      internet&.close
+    end
+  end
+
+  # beware, must be re-entrant because retryable
+  def download_one(attachment:, path_in_download_dir:, async_internet:)
+    byte_written = 0
+    attachment_path = File.join(download_to_dir, path_in_download_dir)
+    attachment_dir = File.dirname(attachment_path)
+
+    FileUtils.mkdir_p(attachment_dir) if !Dir.exist?(attachment_dir) # defensive, do not write in undefined dir
+    if attachment.is_a?(PiecesJustificativesService::FakeAttachment)
+      byte_written = File.write(attachment_path, attachment.file.read, mode: 'wb')
+    else
+      response = async_internet.get(attachment.url)
+      File.open(attachment_path, mode: 'wb') do |fd|
+        response.body.each do |chunk|
+          byte_written = byte_written + fd.write(chunk)
+        end
+        response.body.close
+      end
+    end
+    track_retryable_download_state(attachment_path: attachment_path, state: true) # -> fail once, success after -> no failure
+    byte_written
+  rescue
+    track_retryable_download_state(attachment_path: attachment_path, state: false) #
+    File.delete(attachment_path) if File.exist?(attachment_path) # -> case of retries failed, must cleanup partialy downloaded file
+    raise
+  end
+
+  private
+
+  def initialize(download_to_dir:)
+    @download_to_dir = download_to_dir
+    @errors = {}
+  end
+
+  def track_retryable_download_state(attachment_path:, state:)
+    key = File.basename(attachment_path)
+    if state
+      errors.delete(key) # do not keep track of success, otherwise errors map grows
+    else
+      errors[key] = state
+    end
+  end
+
+  def write_error_manifest
+    manifest_path = File.join(download_to_dir, 'LISEZMOI.txt')
+    manifest_content = errors.map do |file_basename, _failed|
+                                                    "Impossible de récupérer le fichier #{file_basename}"
+                                                  end
+      .join("\n")
+    File.write(manifest_path, manifest_content)
+  end
+end

app/lib/active_storage/downloadable_file.rb

@@ -1,20 +1,5 @@
+
 class ActiveStorage::DownloadableFile
-  # https://edgeapi.rubyonrails.org/classes/ActiveStorage/Blob.html#method-i-download
-  def self.download(attachment:, destination_path:, in_chunk: true)
-    byte_written = 0
-
-    File.open(destination_path, mode: 'wb') do |fd| # we expact a path as string, so we can recreate the file (ex: failure/retry on former existing fd)
-      if in_chunk
-        attachment.download do |chunk|
-          byte_written += fd.write(chunk)
-        end
-      else
-        byte_written = fd.write(attachment.download)
-      end
-    end
-    byte_written
-  end
-
   def self.create_list_from_dossier(dossier, for_expert = false)
     dossier_export = PiecesJustificativesService.generate_dossier_export(dossier)
     pjs = [dossier_export] + PiecesJustificativesService.liste_documents(dossier, for_expert)

app/services/procedure_archive_service.rb

@@ -1,7 +1,6 @@
 require 'tempfile'
 
 class ProcedureArchiveService
-  include Utils::Retryable
   ARCHIVE_CREATION_DIR = ENV.fetch('ARCHIVE_CREATION_DIR') { '/tmp' }
 
   def initialize(procedure)
@@ -37,7 +36,6 @@ class ProcedureArchiveService
       archive.file.attach(
         io: File.open(zip_file),
         filename: archive.filename(@procedure),
-        # we don't want to run virus scanner on this file
         metadata: { virus_scan_result: ActiveStorage::VirusScanner::SAFE }
       )
     end
@@ -104,32 +102,16 @@ class ProcedureArchiveService
         FileUtils.remove_entry_secure(archive_dir) if Dir.exist?(archive_dir)
         Dir.mkdir(archive_dir)
 
-        bug_reports = ''
+        ActiveStorage::DownloadManager
-        attachments.each do |attachment, path|
+          .new(download_to_dir: archive_dir)
-          attachment_path = File.join(archive_dir, path)
+          .download_all(attachments: attachments,
-          attachment_dir = File.dirname(attachment_path)
+                        on_failure: proc { |_attachment, path, error|
+                         Rails.logger.error("Fail to download filename #{path} in procedure##{@procedure.id}, reason: #{error}")
+                       })
 
-          FileUtils.mkdir_p(attachment_dir) if !Dir.exist?(attachment_dir)
-          begin
-            with_retry(max_attempt: 1) do
-              ActiveStorage::DownloadableFile.download(attachment: attachment,
-                                                       destination_path: attachment_path,
-                                                       in_chunk: true)
-            end
-          rescue => e
-            Rails.logger.error("Fail to download filename #{File.basename(attachment_path)} in procedure##{@procedure.id}, reason: #{e}")
-            File.delete(attachment_path) if File.exist?(attachment_path)
-            bug_reports += "Impossible de récupérer le fichier #{File.basename(attachment_path)}\n"
-          end
-        end
-
-        if !bug_reports.empty?
-          File.write(File.join(archive_dir, 'LISEZMOI.txt'), bug_reports)
-        end
-
-        File.delete(zip_path) if File.exist?(zip_path)
         Dir.chdir(tmp_dir) do
-          system 'zip', '-r', zip_path, zip_root_folder
+          File.delete(zip_path) if File.exist?(zip_path)
+          system 'zip', '-0', '-r', zip_path, zip_root_folder
         end
         yield(zip_path)
       ensure
@@ -152,8 +134,8 @@ class ProcedureArchiveService
   def self.attachments_from_champs_piece_justificative(champs)
     champs
       .filter { |c| c.type_champ == TypeDeChamp.type_champs.fetch(:piece_justificative) }
-      .filter { |pj| pj.piece_justificative_file.attached? }
       .map(&:piece_justificative_file)
+      .filter(&:attached?)
   end
 
   def self.liste_pieces_justificatives_for_archive(dossier)

spec/fixtures/cassettes/archive/file_to_get.yml

@@ -0,0 +1,374 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://file.to/get.ext
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip
+      - identity
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Date:
+      - Thu, 16 Dec 2021 14:54:34 GMT
+      Content-Type:
+      - text/html
+      Connection:
+      - keep-alive
+      Last-Modified:
+      - Wed, 09 Dec 2020 11:18:24 GMT
+      X-Amz-Error-Code:
+      - NoSuchKey
+      X-Amz-Error-Message:
+      - The specified key does not exist.
+      X-Amz-Error-Detail-Key:
+      - get.ext
+      X-Amz-Request-Id:
+      - HKBMJ96T2RQAW8AC
+      X-Amz-Id-2:
+      - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo=
+      Cf-Cache-Status:
+      - DYNAMIC
+      Report-To:
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}'
+      Nel:
+      - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
+      Server:
+      - cloudflare
+      Cf-Ray:
+      - 6be8bf678a7b4031-CDG
+      Content-Encoding:
+      - gzip
+      Alt-Svc:
+      - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443";
+        ma=86400
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA=
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+- request:
+    method: get
+    uri: http://file.to/get.ext
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip
+      - identity
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Date:
+      - Thu, 16 Dec 2021 14:54:34 GMT
+      Content-Type:
+      - text/html
+      Connection:
+      - keep-alive
+      Last-Modified:
+      - Wed, 09 Dec 2020 11:18:24 GMT
+      X-Amz-Error-Code:
+      - NoSuchKey
+      X-Amz-Error-Message:
+      - The specified key does not exist.
+      X-Amz-Error-Detail-Key:
+      - get.ext
+      X-Amz-Request-Id:
+      - HKBMJ96T2RQAW8AC
+      X-Amz-Id-2:
+      - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo=
+      Cf-Cache-Status:
+      - DYNAMIC
+      Report-To:
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}'
+      Nel:
+      - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
+      Server:
+      - cloudflare
+      Cf-Ray:
+      - 6be8bf678a7b4031-CDG
+      Content-Encoding:
+      - gzip
+      Alt-Svc:
+      - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443";
+        ma=86400
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA=
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+- request:
+    method: get
+    uri: http://file.to/get.ext
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip
+      - identity
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Date:
+      - Thu, 16 Dec 2021 14:54:34 GMT
+      Content-Type:
+      - text/html
+      Connection:
+      - keep-alive
+      Last-Modified:
+      - Wed, 09 Dec 2020 11:18:24 GMT
+      X-Amz-Error-Code:
+      - NoSuchKey
+      X-Amz-Error-Message:
+      - The specified key does not exist.
+      X-Amz-Error-Detail-Key:
+      - get.ext
+      X-Amz-Request-Id:
+      - HKBMJ96T2RQAW8AC
+      X-Amz-Id-2:
+      - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo=
+      Cf-Cache-Status:
+      - DYNAMIC
+      Report-To:
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}'
+      Nel:
+      - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
+      Server:
+      - cloudflare
+      Cf-Ray:
+      - 6be8bf678a7b4031-CDG
+      Content-Encoding:
+      - gzip
+      Alt-Svc:
+      - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443";
+        ma=86400
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA=
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+- request:
+    method: get
+    uri: http://file.to/get.ext
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip
+      - identity
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Date:
+      - Thu, 16 Dec 2021 14:54:34 GMT
+      Content-Type:
+      - text/html
+      Connection:
+      - keep-alive
+      Last-Modified:
+      - Wed, 09 Dec 2020 11:18:24 GMT
+      X-Amz-Error-Code:
+      - NoSuchKey
+      X-Amz-Error-Message:
+      - The specified key does not exist.
+      X-Amz-Error-Detail-Key:
+      - get.ext
+      X-Amz-Request-Id:
+      - HKBMJ96T2RQAW8AC
+      X-Amz-Id-2:
+      - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo=
+      Cf-Cache-Status:
+      - DYNAMIC
+      Report-To:
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}'
+      Nel:
+      - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
+      Server:
+      - cloudflare
+      Cf-Ray:
+      - 6be8bf678a7b4031-CDG
+      Content-Encoding:
+      - gzip
+      Alt-Svc:
+      - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443";
+        ma=86400
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA=
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+- request:
+    method: get
+    uri: http://file.to/get.ext
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip
+      - identity
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Date:
+      - Thu, 16 Dec 2021 14:54:34 GMT
+      Content-Type:
+      - text/html
+      Connection:
+      - keep-alive
+      Last-Modified:
+      - Wed, 09 Dec 2020 11:18:24 GMT
+      X-Amz-Error-Code:
+      - NoSuchKey
+      X-Amz-Error-Message:
+      - The specified key does not exist.
+      X-Amz-Error-Detail-Key:
+      - get.ext
+      X-Amz-Request-Id:
+      - HKBMJ96T2RQAW8AC
+      X-Amz-Id-2:
+      - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo=
+      Cf-Cache-Status:
+      - DYNAMIC
+      Report-To:
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}'
+      Nel:
+      - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
+      Server:
+      - cloudflare
+      Cf-Ray:
+      - 6be8bf678a7b4031-CDG
+      Content-Encoding:
+      - gzip
+      Alt-Svc:
+      - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443";
+        ma=86400
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA=
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+- request:
+    method: get
+    uri: http://file.to/get.ext
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip
+      - identity
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Date:
+      - Thu, 16 Dec 2021 14:54:34 GMT
+      Content-Type:
+      - text/html
+      Connection:
+      - keep-alive
+      Last-Modified:
+      - Wed, 09 Dec 2020 11:18:24 GMT
+      X-Amz-Error-Code:
+      - NoSuchKey
+      X-Amz-Error-Message:
+      - The specified key does not exist.
+      X-Amz-Error-Detail-Key:
+      - get.ext
+      X-Amz-Request-Id:
+      - HKBMJ96T2RQAW8AC
+      X-Amz-Id-2:
+      - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo=
+      Cf-Cache-Status:
+      - DYNAMIC
+      Report-To:
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}'
+      Nel:
+      - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
+      Server:
+      - cloudflare
+      Cf-Ray:
+      - 6be8bf678a7b4031-CDG
+      Content-Encoding:
+      - gzip
+      Alt-Svc:
+      - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443";
+        ma=86400
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA=
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+- request:
+    method: get
+    uri: http://file.to/get.ext
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip
+      - identity
+  response:
+    status:
+      code: 404
+      message: Not Found
+    headers:
+      Date:
+      - Thu, 16 Dec 2021 14:54:34 GMT
+      Content-Type:
+      - text/html
+      Connection:
+      - keep-alive
+      Last-Modified:
+      - Wed, 09 Dec 2020 11:18:24 GMT
+      X-Amz-Error-Code:
+      - NoSuchKey
+      X-Amz-Error-Message:
+      - The specified key does not exist.
+      X-Amz-Error-Detail-Key:
+      - get.ext
+      X-Amz-Request-Id:
+      - HKBMJ96T2RQAW8AC
+      X-Amz-Id-2:
+      - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo=
+      Cf-Cache-Status:
+      - DYNAMIC
+      Report-To:
+      - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}'
+      Nel:
+      - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}'
+      Server:
+      - cloudflare
+      Cf-Ray:
+      - 6be8bf678a7b4031-CDG
+      Content-Encoding:
+      - gzip
+      Alt-Svc:
+      - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443";
+        ma=86400
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA=
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+recorded_with: VCR 6.0.0

spec/lib/active_storage/download_manager_spec.rb

@@ -0,0 +1,42 @@
+describe ActiveStorage::DownloadManager do
+  let(:test_dir) { Dir.mktmpdir(nil, Dir.tmpdir) }
+  let(:download_to_dir) { test_dir }
+  after { FileUtils.remove_entry_secure(test_dir) if Dir.exist?(test_dir) }
+
+  let(:downloadable_manager) { ActiveStorage::DownloadManager.new(download_to_dir: download_to_dir) }
+
+  describe '#download_one' do
+    subject { downloadable_manager.download_one(attachment: attachment, path_in_download_dir: path_in_download_dir, async_internet: double) }
+
+    let(:path_in_download_dir) { 'lol.png' }
+    let(:attachment) do
+      PiecesJustificativesService::FakeAttachment.new(
+        file: StringIO.new('coucou'),
+        filename: "export-dossier.pdf",
+        name: 'pdf_export_for_instructeur',
+        id: 1,
+        created_at: Time.zone.now
+      )
+    end
+
+    context 'with a PiecesJustificativesService::FakeAttachment and it works' do
+      it 'write attachment.file to disk' do
+        target = File.join(download_to_dir, path_in_download_dir)
+        expect { subject }.to change { File.exist?(target) }
+        attachment.file.rewind
+        expect(attachment.file.read).to eq(File.read(target))
+        expect(downloadable_manager.errors).not_to have_key(path_in_download_dir)
+      end
+    end
+
+    context 'with a PiecesJustificativesService::FakeAttachment and it fails' do
+      it 'write attachment.file to disk' do
+        expect(attachment.file).to receive(:read).and_raise("boom")
+        target = File.join(download_to_dir, path_in_download_dir)
+        expect { subject }.to raise_error(StandardError)
+        expect(File.exist?(target)).to be_falsey
+        expect(downloadable_manager.errors).to have_key(path_in_download_dir)
+      end
+    end
+  end
+end

spec/services/procedure_archive_service_spec.rb

@@ -128,10 +128,14 @@ describe ProcedureArchiveService do
       let(:archive) { create(:archive, time_span_type: 'monthly', status: 'pending', month: date_month) }
       let(:year) { 2021 }
       let(:mailer) { double('mailer', deliver_later: true) }
-
+      before do
+        allow_any_instance_of(ActiveStorage::Attached::One).to receive(:url).and_return("http://file.to/get.ext")
+      end
       it 'collect files' do
         expect(InstructeurMailer).to receive(:send_archive).and_return(mailer)
-        service.collect_files_archive(archive, instructeur)
+        VCR.use_cassette('archive/file.to.get') do
+          service.collect_files_archive(archive, instructeur)
+        end
 
         archive.file.open do |f|
           files = ZipTricks::FileReader.read_zip_structure(io: f)
@@ -207,11 +211,16 @@ describe ProcedureArchiveService do
     context 'for all months' do
       let(:archive) { create(:archive, time_span_type: 'everything', status: 'pending') }
       let(:mailer) { double('mailer', deliver_later: true) }
+      before do
+       allow_any_instance_of(ActiveStorage::Attached::One).to receive(:url).and_return("http://file.to/get.ext")
+     end
 
       it 'collect files' do
         expect(InstructeurMailer).to receive(:send_archive).and_return(mailer)
 
-        service.collect_files_archive(archive, instructeur)
+        VCR.use_cassette('archive/file.to.get') do
+          service.collect_files_archive(archive, instructeur)
+        end
 
         archive = Archive.last
         archive.file.open do |f|
@@ -234,6 +243,36 @@ describe ProcedureArchiveService do
     end
   end
 
+  describe '#download_and_zip' do
+    it 'create a tmpdir while block is running' do
+      previous_dir_list = Dir.entries(ProcedureArchiveService::ARCHIVE_CREATION_DIR)
+
+      service.send(:download_and_zip, []) do |_zip_file|
+        new_dir_list = Dir.entries(ProcedureArchiveService::ARCHIVE_CREATION_DIR)
+        expect(previous_dir_list).not_to eq(new_dir_list)
+      end
+    end
+
+    it 'cleans up its tmpdir after block execution' do
+      expect { service.send(:download_and_zip, []) { |zip_file| } }
+        .not_to change { Dir.entries(ProcedureArchiveService::ARCHIVE_CREATION_DIR) }
+    end
+
+    it 'creates a zip with zip utility' do
+      expected_zip_path = File.join(ProcedureArchiveService::ARCHIVE_CREATION_DIR, "#{service.send(:zip_root_folder)}.zip")
+      expect(service).to receive(:system).with('zip', '-0', '-r', expected_zip_path, an_instance_of(String))
+      service.send(:download_and_zip, []) { |zip_path| }
+    end
+
+    it 'cleans up its generated zip' do
+      expected_zip_path = File.join(ProcedureArchiveService::ARCHIVE_CREATION_DIR, "#{service.send(:zip_root_folder)}.zip")
+      service.send(:download_and_zip, []) do |_zip_path|
+        expect(File.exist?(expected_zip_path)).to be_truthy
+      end
+      expect(File.exist?(expected_zip_path)).to be_falsey
+    end
+  end
+
   private
 
   def create_dossier_for_month(year, month)