feat(async_backend): switch to typhoeus

Gemfile

@@ -10,8 +10,6 @@ gem 'administrate'
 gem 'administrate-field-enum' # Allow using Field::Enum in administrate
 gem 'after_party'
 gem 'anchored'
-gem 'async'
-gem 'async-http'
 gem 'bcrypt'
 gem 'bootsnap', '>= 1.4.4', require: false # Reduces boot times through caching; required in config/boot.rb
 gem 'browser'

Gemfile.lock

@@ -118,21 +118,6 @@ GEM
       activerecord (>= 3.2, < 7.0)
       rake (>= 10.4, < 14.0)
     ast (2.4.2)
-    async (1.30.1)
-      console (~> 1.10)
-      nio4r (~> 2.3)
-      timers (~> 4.1)
-    async-http (0.56.5)
-      async (>= 1.25)
-      async-io (>= 1.28)
-      async-pool (>= 0.2)
-      protocol-http (~> 0.22.0)
-      protocol-http1 (~> 0.14.0)
-      protocol-http2 (~> 0.14.0)
-    async-io (1.32.2)
-      async
-    async-pool (0.3.9)
-      async (>= 1.25)
     attr_encrypted (3.1.0)
       encryptor (~> 3.0.0)
     attr_required (1.0.1)
@@ -198,8 +183,6 @@ GEM
       descendants_tracker (~> 0.0.1)
     concurrent-ruby (1.1.9)
     connection_pool (2.2.3)
-    console (1.14.0)
-      fiber-local
     crack (0.4.5)
       rexml
     crass (1.0.6)
@@ -285,7 +268,6 @@ GEM
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
     ffi (1.15.4)
-    fiber-local (1.0.0)
     flipper (0.20.3)
     flipper-active_record (0.20.3)
       activerecord (>= 5.0, < 7)
@@ -503,13 +485,6 @@ GEM
       actionmailer (>= 3)
       premailer (~> 1.7, >= 1.7.9)
     promise.rb (0.7.4)
-    protocol-hpack (1.4.2)
-    protocol-http (0.22.5)
-    protocol-http1 (0.14.2)
-      protocol-http (~> 0.22)
-    protocol-http2 (0.14.2)
-      protocol-hpack (~> 1.4)
-      protocol-http (~> 0.18)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -749,7 +724,6 @@ GEM
     thread_safe (0.3.6)
     tilt (2.0.10)
     timecop (0.9.4)
-    timers (4.3.3)
     ttfunk (1.7.0)
     typhoeus (1.4.0)
       ethon (>= 0.9.0)
@@ -821,8 +795,6 @@ DEPENDENCIES
   after_party
   anchored
   annotate
-  async
-  async-http
   axe-core-rspec
   bcrypt
   bootsnap (>= 1.4.4)

app/lib/download_manager/parallel_download_queue.rb

@@ -1,7 +1,3 @@
-require 'async'
-require 'async/barrier'
-require 'async/http/internet'
-
 module DownloadManager
   class ParallelDownloadQueue
     include Utils::Retryable
@@ -17,51 +13,49 @@ module DownloadManager
     end
 
     def download_all
-      Async do
+      hydra = Typhoeus::Hydra.new(max_concurrency: DOWNLOAD_MAX_PARALLEL)
-        http_client = Async::HTTP::Internet.new
-        barrier = Async::Barrier.new
-        semaphore = Async::Semaphore.new(DOWNLOAD_MAX_PARALLEL, parent: barrier)
 
-        attachments.map do |attachment, path|
+      attachments.map do |attachment, path|
-          semaphore.async do
+        begin
-            begin
+          with_retry(max_attempt: 1) do
-              with_retry(max_attempt: 1) do
+            download_one(attachment: attachment,
-                download_one(attachment: attachment,
+                         path_in_download_dir: path,
-                             path_in_download_dir: path,
+                         http_client: hydra)
-                             http_client: http_client)
-              end
-            rescue => e
-              on_error.call(attachment, path, e)
-            end
           end
+        rescue => e
+          on_error.call(attachment, path, e)
         end
-        barrier.wait
-      ensure
-        http_client&.close
       end
+      hydra.run
     end
 
+    # rubocop:disable Style/AutoResourceCleanup
+    # can't be used with typhoeus, otherwise block is closed before the request is run by hydra
     def download_one(attachment:, path_in_download_dir:, http_client:)
-      byte_written = 0
       attachment_path = File.join(destination, path_in_download_dir)
       attachment_dir = File.dirname(attachment_path)
 
       FileUtils.mkdir_p(attachment_dir) if !Dir.exist?(attachment_dir) # defensive, do not write in undefined dir
       if attachment.is_a?(PiecesJustificativesService::FakeAttachment)
-        byte_written = File.write(attachment_path, attachment.file.read, mode: 'wb')
+        File.write(attachment_path, attachment.file.read, mode: 'wb')
       else
-        response = http_client.get(attachment.url)
+        request = Typhoeus::Request.new(attachment.url)
-        File.open(attachment_path, mode: 'wb') do |fd|
+        fd = File.open(attachment_path, mode: 'wb')
-          response.body.each do |chunk|
+        request.on_body do |chunk|
-            byte_written = byte_written + fd.write(chunk)
+          fd.write(chunk)
-          end
-          response.body.close
         end
+        request.on_complete do |response|
+          fd.close
+          unless response.success?
+            raise 'ko'
+          end
+        end
+        http_client.queue(request)
       end
-      byte_written
     rescue
       File.delete(attachment_path) if File.exist?(attachment_path) # -> case of retries failed, must cleanup partialy downloaded file
       raise
     end
+    # rubocop:enable Style/AutoResourceCleanup
   end
 end

spec/fixtures/cassettes/archive/file_to_get_typhoeus.yml

@@ -0,0 +1,145 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://i.etsystatic.com/6212702/r/il/744d2c/470726480/il_1588xN.470726480_bpk5.jpg
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Cache-Control:
+      - public, max-age=365000000, immutable
+      Content-Type:
+      - image/jpeg
+      Etag:
+      - '"J9Qt3QnUKZIbmKehfH+pmv/rYxafyM81ENfBKsCN6Qw"'
+      Expires:
+      - Mon, 02 Jan 2023 05:24:30 GMT
+      Fastly-Io-Info:
+      - ifsz=14677 idim=600x600 ifmt=jpeg ofsz=43339 odim=1588x1588 ofmt=jpeg
+      Fastly-Stats:
+      - io=1
+      Server:
+      - UploadServer
+      X-Goog-Generation:
+      - '1514228438620441'
+      X-Goog-Hash:
+      - crc32c=ZohETA==
+      - md5=iKPGsaOoUN0hhNZYYzYDLQ==
+      X-Goog-Metageneration:
+      - '1'
+      X-Goog-Storage-Class:
+      - MULTI_REGIONAL
+      X-Goog-Stored-Content-Encoding:
+      - identity
+      X-Goog-Stored-Content-Length:
+      - '14677'
+      X-Guploader-Uploadid:
+      - ADPycdvXmKF1KUStMVeN1v5TUKBQA_YezSueBDRwp4qiVKTn5IDoW7f3_t6_tyvJwjkOoUE4lO1cC_NxSl-LkM5ukthJcv6JkA
+      Via:
+      - 1.1 varnish, 1.1 varnish
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Tue, 04 Jan 2022 14:59:40 GMT
+      Age:
+      - '207310'
+      X-Served-By:
+      - cache-mdw17340-MDW, cache-cdg20755-CDG
+      X-Cache:
+      - HIT, HIT
+      X-Cache-Hits:
+      - 1, 1
+      X-Timer:
+      - S1641308380.238785,VS0,VE1
+      Vary:
+      - Accept
+      Strict-Transport-Security:
+      - max-age=300
+      Content-Length:
+      - '43339'
+    body:
+      encoding: ASCII-8BIT
+      string: ''
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+- request:
+    method: get
+    uri: https://i.etsystatic.com/6212702/r/il/744d2c/470726480/il_1588xN.470726480_bpk5.jpg
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Cache-Control:
+      - public, max-age=365000000, immutable
+      Content-Type:
+      - image/jpeg
+      Etag:
+      - '"J9Qt3QnUKZIbmKehfH+pmv/rYxafyM81ENfBKsCN6Qw"'
+      Expires:
+      - Mon, 02 Jan 2023 05:24:30 GMT
+      Fastly-Io-Info:
+      - ifsz=14677 idim=600x600 ifmt=jpeg ofsz=43339 odim=1588x1588 ofmt=jpeg
+      Fastly-Stats:
+      - io=1
+      Server:
+      - UploadServer
+      X-Goog-Generation:
+      - '1514228438620441'
+      X-Goog-Hash:
+      - crc32c=ZohETA==
+      - md5=iKPGsaOoUN0hhNZYYzYDLQ==
+      X-Goog-Metageneration:
+      - '1'
+      X-Goog-Storage-Class:
+      - MULTI_REGIONAL
+      X-Goog-Stored-Content-Encoding:
+      - identity
+      X-Goog-Stored-Content-Length:
+      - '14677'
+      X-Guploader-Uploadid:
+      - ADPycdvXmKF1KUStMVeN1v5TUKBQA_YezSueBDRwp4qiVKTn5IDoW7f3_t6_tyvJwjkOoUE4lO1cC_NxSl-LkM5ukthJcv6JkA
+      Via:
+      - 1.1 varnish, 1.1 varnish
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Tue, 04 Jan 2022 14:59:40 GMT
+      Age:
+      - '207310'
+      X-Served-By:
+      - cache-mdw17340-MDW, cache-cdg20737-CDG
+      X-Cache:
+      - HIT, HIT
+      X-Cache-Hits:
+      - 1, 1
+      X-Timer:
+      - S1641308380.241689,VS0,VE1
+      Vary:
+      - Accept
+      Strict-Transport-Security:
+      - max-age=300
+      Content-Length:
+      - '43339'
+    body:
+      encoding: ASCII-8BIT
+      string: ''
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+recorded_with: VCR 6.0.0

spec/services/procedure_archive_service_spec.rb

@@ -212,13 +212,13 @@ describe ProcedureArchiveService do
       let(:archive) { create(:archive, time_span_type: 'everything', status: 'pending') }
       let(:mailer) { double('mailer', deliver_later: true) }
       before do
-       allow_any_instance_of(ActiveStorage::Attached::One).to receive(:url).and_return("http://file.to/get.ext")
+        allow_any_instance_of(ActiveStorage::Attached::One).to receive(:url).and_return("https://i.etsystatic.com/6212702/r/il/744d2c/470726480/il_1588xN.470726480_bpk5.jpg")
-     end
+      end
 
       it 'collect files' do
         expect(InstructeurMailer).to receive(:send_archive).and_return(mailer)
 
-        VCR.use_cassette('archive/file_to_get') do
+        VCR.use_cassette('archive/file_to_get_typhoeus') do
           service.collect_files_archive(archive, instructeur)
         end