feat(async_backend): switch to typhoeus

2022-01-04 16:15:28 +01:00 · 2022-01-04 16:15:28 +01:00 · 3eb1c1a421
commit 3eb1c1a421
parent ce1b189dcd
6 changed files with 210 additions and 4187 deletions
--- a/2
+++ b/2
@ -10,8 +10,6 @@ gem 'administrate'
 gem 'administrate-field-enum' # Allow using Field::Enum in administrate
 gem 'after_party'
 gem 'anchored'
-gem 'async'
-gem 'async-http'
 gem 'bcrypt'
 gem 'bootsnap', '>= 1.4.4', require: false # Reduces boot times through caching; required in config/boot.rb
 gem 'browser'
--- a/Gemfile.lock
+++ b/Gemfile.lock
@ -118,21 +118,6 @@ GEM
      activerecord (>= 3.2, < 7.0)
      rake (>= 10.4, < 14.0)
    ast (2.4.2)
-    async (1.30.1)
-      console (~> 1.10)
-      nio4r (~> 2.3)
-      timers (~> 4.1)
-    async-http (0.56.5)
-      async (>= 1.25)
-      async-io (>= 1.28)
-      async-pool (>= 0.2)
-      protocol-http (~> 0.22.0)
-      protocol-http1 (~> 0.14.0)
-      protocol-http2 (~> 0.14.0)
-    async-io (1.32.2)
-      async
-    async-pool (0.3.9)
-      async (>= 1.25)
    attr_encrypted (3.1.0)
      encryptor (~> 3.0.0)
    attr_required (1.0.1)
@ -198,8 +183,6 @@ GEM
      descendants_tracker (~> 0.0.1)
    concurrent-ruby (1.1.9)
    connection_pool (2.2.3)
-    console (1.14.0)
-      fiber-local
    crack (0.4.5)
      rexml
    crass (1.0.6)
@ -285,7 +268,6 @@ GEM
    faraday-patron (1.0.0)
    faraday-rack (1.0.0)
    ffi (1.15.4)
-    fiber-local (1.0.0)
    flipper (0.20.3)
    flipper-active_record (0.20.3)
      activerecord (>= 5.0, < 7)
@ -503,13 +485,6 @@ GEM
      actionmailer (>= 3)
      premailer (~> 1.7, >= 1.7.9)
    promise.rb (0.7.4)
-    protocol-hpack (1.4.2)
-    protocol-http (0.22.5)
-    protocol-http1 (0.14.2)
-      protocol-http (~> 0.22)
-    protocol-http2 (0.14.2)
-      protocol-hpack (~> 1.4)
-      protocol-http (~> 0.18)
    pry (0.13.1)
      coderay (~> 1.1)
      method_source (~> 1.0)
@ -749,7 +724,6 @@ GEM
    thread_safe (0.3.6)
    tilt (2.0.10)
    timecop (0.9.4)
-    timers (4.3.3)
    ttfunk (1.7.0)
    typhoeus (1.4.0)
      ethon (>= 0.9.0)
@ -821,8 +795,6 @@ DEPENDENCIES
  after_party
  anchored
  annotate
-  async
-  async-http
  axe-core-rspec
  bcrypt
  bootsnap (>= 1.4.4)
--- a/app/lib/download_manager/parallel_download_queue.rb
+++ b/app/lib/download_manager/parallel_download_queue.rb
@ -1,7 +1,3 @@
-require 'async'
-require 'async/barrier'
-require 'async/http/internet'
-
 module DownloadManager
  class ParallelDownloadQueue
    include Utils::Retryable
@ -17,51 +13,49 @@ module DownloadManager
    end

    def download_all
-      Async do
-        http_client = Async::HTTP::Internet.new
-        barrier = Async::Barrier.new
-        semaphore = Async::Semaphore.new(DOWNLOAD_MAX_PARALLEL, parent: barrier)
+      hydra = Typhoeus::Hydra.new(max_concurrency: DOWNLOAD_MAX_PARALLEL)

-        attachments.map do |attachment, path|
-          semaphore.async do
-            begin
-              with_retry(max_attempt: 1) do
-                download_one(attachment: attachment,
-                             path_in_download_dir: path,
-                             http_client: http_client)
-              end
-            rescue => e
-              on_error.call(attachment, path, e)
-            end
+      attachments.map do |attachment, path|
+        begin
+          with_retry(max_attempt: 1) do
+            download_one(attachment: attachment,
+                         path_in_download_dir: path,
+                         http_client: hydra)
          end
+        rescue => e
+          on_error.call(attachment, path, e)
        end
-        barrier.wait
-      ensure
-        http_client&.close
      end
+      hydra.run
    end

+    # rubocop:disable Style/AutoResourceCleanup
+    # can't be used with typhoeus, otherwise block is closed before the request is run by hydra
    def download_one(attachment:, path_in_download_dir:, http_client:)
-      byte_written = 0
      attachment_path = File.join(destination, path_in_download_dir)
      attachment_dir = File.dirname(attachment_path)

      FileUtils.mkdir_p(attachment_dir) if !Dir.exist?(attachment_dir) # defensive, do not write in undefined dir
      if attachment.is_a?(PiecesJustificativesService::FakeAttachment)
-        byte_written = File.write(attachment_path, attachment.file.read, mode: 'wb')
+        File.write(attachment_path, attachment.file.read, mode: 'wb')
      else
-        response = http_client.get(attachment.url)
-        File.open(attachment_path, mode: 'wb') do |fd|
-          response.body.each do |chunk|
-            byte_written = byte_written + fd.write(chunk)
-          end
-          response.body.close
+        request = Typhoeus::Request.new(attachment.url)
+        fd = File.open(attachment_path, mode: 'wb')
+        request.on_body do |chunk|
+          fd.write(chunk)
        end
+        request.on_complete do |response|
+          fd.close
+          unless response.success?
+            raise 'ko'
+          end
+        end
+        http_client.queue(request)
      end
-      byte_written
    rescue
      File.delete(attachment_path) if File.exist?(attachment_path) # -> case of retries failed, must cleanup partialy downloaded file
      raise
    end
+    # rubocop:enable Style/AutoResourceCleanup
  end
 end
--- a/spec/fixtures/cassettes/archive/file_to_get.yml
+++ b/spec/fixtures/cassettes/archive/file_to_get.yml
--- a/spec/fixtures/cassettes/archive/file_to_get_typhoeus.yml
+++ b/spec/fixtures/cassettes/archive/file_to_get_typhoeus.yml
@ -0,0 +1,145 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: https://i.etsystatic.com/6212702/r/il/744d2c/470726480/il_1588xN.470726480_bpk5.jpg
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Cache-Control:
+      - public, max-age=365000000, immutable
+      Content-Type:
+      - image/jpeg
+      Etag:
+      - '"J9Qt3QnUKZIbmKehfH+pmv/rYxafyM81ENfBKsCN6Qw"'
+      Expires:
+      - Mon, 02 Jan 2023 05:24:30 GMT
+      Fastly-Io-Info:
+      - ifsz=14677 idim=600x600 ifmt=jpeg ofsz=43339 odim=1588x1588 ofmt=jpeg
+      Fastly-Stats:
+      - io=1
+      Server:
+      - UploadServer
+      X-Goog-Generation:
+      - '1514228438620441'
+      X-Goog-Hash:
+      - crc32c=ZohETA==
+      - md5=iKPGsaOoUN0hhNZYYzYDLQ==
+      X-Goog-Metageneration:
+      - '1'
+      X-Goog-Storage-Class:
+      - MULTI_REGIONAL
+      X-Goog-Stored-Content-Encoding:
+      - identity
+      X-Goog-Stored-Content-Length:
+      - '14677'
+      X-Guploader-Uploadid:
+      - ADPycdvXmKF1KUStMVeN1v5TUKBQA_YezSueBDRwp4qiVKTn5IDoW7f3_t6_tyvJwjkOoUE4lO1cC_NxSl-LkM5ukthJcv6JkA
+      Via:
+      - 1.1 varnish, 1.1 varnish
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Tue, 04 Jan 2022 14:59:40 GMT
+      Age:
+      - '207310'
+      X-Served-By:
+      - cache-mdw17340-MDW, cache-cdg20755-CDG
+      X-Cache:
+      - HIT, HIT
+      X-Cache-Hits:
+      - 1, 1
+      X-Timer:
+      - S1641308380.238785,VS0,VE1
+      Vary:
+      - Accept
+      Strict-Transport-Security:
+      - max-age=300
+      Content-Length:
+      - '43339'
+    body:
+      encoding: ASCII-8BIT
+      string: ''
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+- request:
+    method: get
+    uri: https://i.etsystatic.com/6212702/r/il/744d2c/470726480/il_1588xN.470726480_bpk5.jpg
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Cache-Control:
+      - public, max-age=365000000, immutable
+      Content-Type:
+      - image/jpeg
+      Etag:
+      - '"J9Qt3QnUKZIbmKehfH+pmv/rYxafyM81ENfBKsCN6Qw"'
+      Expires:
+      - Mon, 02 Jan 2023 05:24:30 GMT
+      Fastly-Io-Info:
+      - ifsz=14677 idim=600x600 ifmt=jpeg ofsz=43339 odim=1588x1588 ofmt=jpeg
+      Fastly-Stats:
+      - io=1
+      Server:
+      - UploadServer
+      X-Goog-Generation:
+      - '1514228438620441'
+      X-Goog-Hash:
+      - crc32c=ZohETA==
+      - md5=iKPGsaOoUN0hhNZYYzYDLQ==
+      X-Goog-Metageneration:
+      - '1'
+      X-Goog-Storage-Class:
+      - MULTI_REGIONAL
+      X-Goog-Stored-Content-Encoding:
+      - identity
+      X-Goog-Stored-Content-Length:
+      - '14677'
+      X-Guploader-Uploadid:
+      - ADPycdvXmKF1KUStMVeN1v5TUKBQA_YezSueBDRwp4qiVKTn5IDoW7f3_t6_tyvJwjkOoUE4lO1cC_NxSl-LkM5ukthJcv6JkA
+      Via:
+      - 1.1 varnish, 1.1 varnish
+      Accept-Ranges:
+      - bytes
+      Date:
+      - Tue, 04 Jan 2022 14:59:40 GMT
+      Age:
+      - '207310'
+      X-Served-By:
+      - cache-mdw17340-MDW, cache-cdg20737-CDG
+      X-Cache:
+      - HIT, HIT
+      X-Cache-Hits:
+      - 1, 1
+      X-Timer:
+      - S1641308380.241689,VS0,VE1
+      Vary:
+      - Accept
+      Strict-Transport-Security:
+      - max-age=300
+      Content-Length:
+      - '43339'
+    body:
+      encoding: ASCII-8BIT
+      string: ''
+  recorded_at: Wed, 04 Mar 2020 23:00:00 GMT
+recorded_with: VCR 6.0.0
--- a/spec/services/procedure_archive_service_spec.rb
+++ b/spec/services/procedure_archive_service_spec.rb
@ -212,13 +212,13 @@ describe ProcedureArchiveService do
      let(:archive) { create(:archive, time_span_type: 'everything', status: 'pending') }
      let(:mailer) { double('mailer', deliver_later: true) }
      before do
-       allow_any_instance_of(ActiveStorage::Attached::One).to receive(:url).and_return("http://file.to/get.ext")
-     end
+        allow_any_instance_of(ActiveStorage::Attached::One).to receive(:url).and_return("https://i.etsystatic.com/6212702/r/il/744d2c/470726480/il_1588xN.470726480_bpk5.jpg")
+      end

      it 'collect files' do
        expect(InstructeurMailer).to receive(:send_archive).and_return(mailer)

-        VCR.use_cassette('archive/file_to_get') do
+        VCR.use_cassette('archive/file_to_get_typhoeus') do
          service.collect_files_archive(archive, instructeur)
        end