fix(parallel_download_queue): tophoeus does not like raise from a request handler [crash straight to first frame]

app/lib/download_manager/parallel_download_queue.rb

@@ -1,6 +1,5 @@
 module DownloadManager
   class ParallelDownloadQueue
-    include Utils::Retryable
     DOWNLOAD_MAX_PARALLEL = ENV.fetch('DOWNLOAD_MAX_PARALLEL') { 10 }
 
     attr_accessor :attachments,
@@ -17,11 +16,9 @@ module DownloadManager
 
       attachments.map do |attachment, path|
         begin
-          with_retry(max_attempt: 1) do
-            download_one(attachment: attachment,
-                         path_in_download_dir: path,
-                         http_client: hydra)
-          end
+          download_one(attachment: attachment,
+                       path_in_download_dir: path,
+                       http_client: hydra)
         rescue => e
           on_error.call(attachment, path, e)
         end
@@ -47,14 +44,12 @@ module DownloadManager
         request.on_complete do |response|
           fd.close
           unless response.success?
-            raise 'ko'
+            File.delete(attachment_path) if File.exist?(attachment_path) # -> case of retries failed, must cleanup partialy downloaded file
+            on_error.call(attachment, path_in_download_dir, response.code)
           end
         end
         http_client.queue(request)
       end
-    rescue
-      File.delete(attachment_path) if File.exist?(attachment_path) # -> case of retries failed, must cleanup partialy downloaded file
-      raise
     end
     # rubocop:enable Style/AutoResourceCleanup
   end

app/lib/download_manager/procedure_attachments_export.rb

@@ -9,18 +9,20 @@ module DownloadManager
       @procedure = procedure
       @errors = {}
       @queue = ParallelDownloadQueue.new(attachments, destination)
-      @queue.on_error = proc do |_attachment, path, error|
-        errors[path] = true
+      @queue.on_error = proc do |attachment, path, error|
+        errors[path] = [attachment, path]
         Rails.logger.error("Fail to download filename #{path} in procedure##{@procedure.id}, reason: #{error}")
       end
-     end
-
-    def download_all
-      @queue.download_all
-      write_report if !errors.empty?
     end
 
-    private
+    def download_all(attempt_left: 1)
+      @queue.download_all
+      if !errors.empty? && attempt_left.positive?
+        retryable_queue = self.class.new(@procedure, errors.values, destination)
+        retryable_queue.download_all(attempt_left: 0)
+        retryable_queue.write_report if !retryable_queue.errors.empty?
+      end
+    end
 
     def write_report
       manifest_path = File.join(destination, 'LISEZMOI.txt')

app/lib/utils/retryable.rb

@@ -1,20 +0,0 @@
-module Utils
-  module Retryable
-    # usage:
-    # max_attempt : retry count
-    # errors : only retry those errors
-    # with_retry(max_attempt: 10, errors: [StandardError]) do
-    #   do_something_which_can_fail
-    # end
-    def with_retry(max_attempt: 1, errors: [StandardError], &block)
-      limiter = 0
-      begin
-        yield
-      rescue *errors
-        limiter += 1
-        retry if limiter <= max_attempt
-        raise
-      end
-    end
-  end
-end