diff --git a/Gemfile b/Gemfile index 3377bc729..e48d1916f 100644 --- a/Gemfile +++ b/Gemfile @@ -10,6 +10,8 @@ gem 'administrate' gem 'administrate-field-enum' # Allow using Field::Enum in administrate gem 'after_party' gem 'anchored' +gem 'async' +gem 'async-http' gem 'bcrypt' gem 'bootsnap', '>= 1.4.4', require: false # Reduces boot times through caching; required in config/boot.rb gem 'browser' diff --git a/Gemfile.lock b/Gemfile.lock index 809cb6fd6..041198279 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -118,6 +118,21 @@ GEM activerecord (>= 3.2, < 7.0) rake (>= 10.4, < 14.0) ast (2.4.2) + async (1.30.1) + console (~> 1.10) + nio4r (~> 2.3) + timers (~> 4.1) + async-http (0.56.5) + async (>= 1.25) + async-io (>= 1.28) + async-pool (>= 0.2) + protocol-http (~> 0.22.0) + protocol-http1 (~> 0.14.0) + protocol-http2 (~> 0.14.0) + async-io (1.32.2) + async + async-pool (0.3.9) + async (>= 1.25) attr_encrypted (3.1.0) encryptor (~> 3.0.0) attr_required (1.0.1) @@ -183,6 +198,8 @@ GEM descendants_tracker (~> 0.0.1) concurrent-ruby (1.1.9) connection_pool (2.2.3) + console (1.14.0) + fiber-local crack (0.4.5) rexml crass (1.0.6) @@ -268,6 +285,7 @@ GEM faraday-patron (1.0.0) faraday-rack (1.0.0) ffi (1.15.4) + fiber-local (1.0.0) flipper (0.20.3) flipper-active_record (0.20.3) activerecord (>= 5.0, < 7) @@ -485,6 +503,13 @@ GEM actionmailer (>= 3) premailer (~> 1.7, >= 1.7.9) promise.rb (0.7.4) + protocol-hpack (1.4.2) + protocol-http (0.22.5) + protocol-http1 (0.14.2) + protocol-http (~> 0.22) + protocol-http2 (0.14.2) + protocol-hpack (~> 1.4) + protocol-http (~> 0.18) pry (0.13.1) coderay (~> 1.1) method_source (~> 1.0) @@ -724,6 +749,7 @@ GEM thread_safe (0.3.6) tilt (2.0.10) timecop (0.9.4) + timers (4.3.3) ttfunk (1.7.0) typhoeus (1.4.0) ethon (>= 0.9.0) @@ -795,6 +821,8 @@ DEPENDENCIES after_party anchored annotate + async + async-http axe-core-rspec bcrypt bootsnap (>= 1.4.4) diff --git a/app/lib/active_storage/download_manager.rb b/app/lib/active_storage/download_manager.rb new file mode 100644 index 000000000..807502a3b --- /dev/null +++ b/app/lib/active_storage/download_manager.rb @@ -0,0 +1,87 @@ +require 'async' +require 'async/barrier' +require 'async/http/internet' + +class ActiveStorage::DownloadManager + include Utils::Retryable + DOWNLOAD_MAX_PARALLEL = ENV.fetch('DOWNLOAD_MAX_PARALLEL') { 10 } + + attr_reader :download_to_dir, :errors + + def download_all(attachments:, on_failure:) + Async do + internet = Async::HTTP::Internet.new + barrier = Async::Barrier.new + semaphore = Async::Semaphore.new(DOWNLOAD_MAX_PARALLEL, parent: barrier) + + attachments.map do |attachment, path| + semaphore.async do + begin + with_retry(max_attempt: 1) do + download_one(attachment: attachment, + path_in_download_dir: path, + async_internet: internet) + end + rescue => e + on_failure.call(attachment, path, e) + end + end + end + barrier.wait + write_error_manifest if !errors.empty? + ensure + internet&.close + end + end + + # beware, must be re-entrant because retryable + def download_one(attachment:, path_in_download_dir:, async_internet:) + byte_written = 0 + attachment_path = File.join(download_to_dir, path_in_download_dir) + attachment_dir = File.dirname(attachment_path) + + FileUtils.mkdir_p(attachment_dir) if !Dir.exist?(attachment_dir) # defensive, do not write in undefined dir + if attachment.is_a?(PiecesJustificativesService::FakeAttachment) + byte_written = File.write(attachment_path, attachment.file.read, mode: 'wb') + else + response = async_internet.get(attachment.url) + File.open(attachment_path, mode: 'wb') do |fd| + response.body.each do |chunk| + byte_written = byte_written + fd.write(chunk) + end + response.body.close + end + end + track_retryable_download_state(attachment_path: attachment_path, state: true) # -> fail once, success after -> no failure + byte_written + rescue + track_retryable_download_state(attachment_path: attachment_path, state: false) # + File.delete(attachment_path) if File.exist?(attachment_path) # -> case of retries failed, must cleanup partialy downloaded file + raise + end + + private + + def initialize(download_to_dir:) + @download_to_dir = download_to_dir + @errors = {} + end + + def track_retryable_download_state(attachment_path:, state:) + key = File.basename(attachment_path) + if state + errors.delete(key) # do not keep track of success, otherwise errors map grows + else + errors[key] = state + end + end + + def write_error_manifest + manifest_path = File.join(download_to_dir, 'LISEZMOI.txt') + manifest_content = errors.map do |file_basename, _failed| + "Impossible de récupérer le fichier #{file_basename}" + end + .join("\n") + File.write(manifest_path, manifest_content) + end +end diff --git a/app/lib/active_storage/downloadable_file.rb b/app/lib/active_storage/downloadable_file.rb index 577d52c3e..eca681943 100644 --- a/app/lib/active_storage/downloadable_file.rb +++ b/app/lib/active_storage/downloadable_file.rb @@ -1,20 +1,5 @@ + class ActiveStorage::DownloadableFile - # https://edgeapi.rubyonrails.org/classes/ActiveStorage/Blob.html#method-i-download - def self.download(attachment:, destination_path:, in_chunk: true) - byte_written = 0 - - File.open(destination_path, mode: 'wb') do |fd| # we expact a path as string, so we can recreate the file (ex: failure/retry on former existing fd) - if in_chunk - attachment.download do |chunk| - byte_written += fd.write(chunk) - end - else - byte_written = fd.write(attachment.download) - end - end - byte_written - end - def self.create_list_from_dossier(dossier, for_expert = false) dossier_export = PiecesJustificativesService.generate_dossier_export(dossier) pjs = [dossier_export] + PiecesJustificativesService.liste_documents(dossier, for_expert) diff --git a/app/services/procedure_archive_service.rb b/app/services/procedure_archive_service.rb index 30c13c02b..1d5ed6c28 100644 --- a/app/services/procedure_archive_service.rb +++ b/app/services/procedure_archive_service.rb @@ -1,7 +1,6 @@ require 'tempfile' class ProcedureArchiveService - include Utils::Retryable ARCHIVE_CREATION_DIR = ENV.fetch('ARCHIVE_CREATION_DIR') { '/tmp' } def initialize(procedure) @@ -37,7 +36,6 @@ class ProcedureArchiveService archive.file.attach( io: File.open(zip_file), filename: archive.filename(@procedure), - # we don't want to run virus scanner on this file metadata: { virus_scan_result: ActiveStorage::VirusScanner::SAFE } ) end @@ -104,32 +102,16 @@ class ProcedureArchiveService FileUtils.remove_entry_secure(archive_dir) if Dir.exist?(archive_dir) Dir.mkdir(archive_dir) - bug_reports = '' - attachments.each do |attachment, path| - attachment_path = File.join(archive_dir, path) - attachment_dir = File.dirname(attachment_path) + ActiveStorage::DownloadManager + .new(download_to_dir: archive_dir) + .download_all(attachments: attachments, + on_failure: proc { |_attachment, path, error| + Rails.logger.error("Fail to download filename #{path} in procedure##{@procedure.id}, reason: #{error}") + }) - FileUtils.mkdir_p(attachment_dir) if !Dir.exist?(attachment_dir) - begin - with_retry(max_attempt: 1) do - ActiveStorage::DownloadableFile.download(attachment: attachment, - destination_path: attachment_path, - in_chunk: true) - end - rescue => e - Rails.logger.error("Fail to download filename #{File.basename(attachment_path)} in procedure##{@procedure.id}, reason: #{e}") - File.delete(attachment_path) if File.exist?(attachment_path) - bug_reports += "Impossible de récupérer le fichier #{File.basename(attachment_path)}\n" - end - end - - if !bug_reports.empty? - File.write(File.join(archive_dir, 'LISEZMOI.txt'), bug_reports) - end - - File.delete(zip_path) if File.exist?(zip_path) Dir.chdir(tmp_dir) do - system 'zip', '-r', zip_path, zip_root_folder + File.delete(zip_path) if File.exist?(zip_path) + system 'zip', '-0', '-r', zip_path, zip_root_folder end yield(zip_path) ensure @@ -152,8 +134,8 @@ class ProcedureArchiveService def self.attachments_from_champs_piece_justificative(champs) champs .filter { |c| c.type_champ == TypeDeChamp.type_champs.fetch(:piece_justificative) } - .filter { |pj| pj.piece_justificative_file.attached? } .map(&:piece_justificative_file) + .filter(&:attached?) end def self.liste_pieces_justificatives_for_archive(dossier) diff --git a/spec/fixtures/cassettes/archive/file_to_get.yml b/spec/fixtures/cassettes/archive/file_to_get.yml new file mode 100644 index 000000000..e18ecf6a6 --- /dev/null +++ b/spec/fixtures/cassettes/archive/file_to_get.yml @@ -0,0 +1,374 @@ +--- +http_interactions: +- request: + method: get + uri: http://file.to/get.ext + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip + - identity + response: + status: + code: 404 + message: Not Found + headers: + Date: + - Thu, 16 Dec 2021 14:54:34 GMT + Content-Type: + - text/html + Connection: + - keep-alive + Last-Modified: + - Wed, 09 Dec 2020 11:18:24 GMT + X-Amz-Error-Code: + - NoSuchKey + X-Amz-Error-Message: + - The specified key does not exist. + X-Amz-Error-Detail-Key: + - get.ext + X-Amz-Request-Id: + - HKBMJ96T2RQAW8AC + X-Amz-Id-2: + - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo= + Cf-Cache-Status: + - DYNAMIC + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}' + Nel: + - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}' + Server: + - cloudflare + Cf-Ray: + - 6be8bf678a7b4031-CDG + Content-Encoding: + - gzip + Alt-Svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443"; + ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA= + recorded_at: Wed, 04 Mar 2020 23:00:00 GMT +- request: + method: get + uri: http://file.to/get.ext + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip + - identity + response: + status: + code: 404 + message: Not Found + headers: + Date: + - Thu, 16 Dec 2021 14:54:34 GMT + Content-Type: + - text/html + Connection: + - keep-alive + Last-Modified: + - Wed, 09 Dec 2020 11:18:24 GMT + X-Amz-Error-Code: + - NoSuchKey + X-Amz-Error-Message: + - The specified key does not exist. + X-Amz-Error-Detail-Key: + - get.ext + X-Amz-Request-Id: + - HKBMJ96T2RQAW8AC + X-Amz-Id-2: + - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo= + Cf-Cache-Status: + - DYNAMIC + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}' + Nel: + - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}' + Server: + - cloudflare + Cf-Ray: + - 6be8bf678a7b4031-CDG + Content-Encoding: + - gzip + Alt-Svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443"; + ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA= + recorded_at: Wed, 04 Mar 2020 23:00:00 GMT +- request: + method: get + uri: http://file.to/get.ext + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip + - identity + response: + status: + code: 404 + message: Not Found + headers: + Date: + - Thu, 16 Dec 2021 14:54:34 GMT + Content-Type: + - text/html + Connection: + - keep-alive + Last-Modified: + - Wed, 09 Dec 2020 11:18:24 GMT + X-Amz-Error-Code: + - NoSuchKey + X-Amz-Error-Message: + - The specified key does not exist. + X-Amz-Error-Detail-Key: + - get.ext + X-Amz-Request-Id: + - HKBMJ96T2RQAW8AC + X-Amz-Id-2: + - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo= + Cf-Cache-Status: + - DYNAMIC + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}' + Nel: + - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}' + Server: + - cloudflare + Cf-Ray: + - 6be8bf678a7b4031-CDG + Content-Encoding: + - gzip + Alt-Svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443"; + ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA= + recorded_at: Wed, 04 Mar 2020 23:00:00 GMT +- request: + method: get + uri: http://file.to/get.ext + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip + - identity + response: + status: + code: 404 + message: Not Found + headers: + Date: + - Thu, 16 Dec 2021 14:54:34 GMT + Content-Type: + - text/html + Connection: + - keep-alive + Last-Modified: + - Wed, 09 Dec 2020 11:18:24 GMT + X-Amz-Error-Code: + - NoSuchKey + X-Amz-Error-Message: + - The specified key does not exist. + X-Amz-Error-Detail-Key: + - get.ext + X-Amz-Request-Id: + - HKBMJ96T2RQAW8AC + X-Amz-Id-2: + - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo= + Cf-Cache-Status: + - DYNAMIC + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}' + Nel: + - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}' + Server: + - cloudflare + Cf-Ray: + - 6be8bf678a7b4031-CDG + Content-Encoding: + - gzip + Alt-Svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443"; + ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA= + recorded_at: Wed, 04 Mar 2020 23:00:00 GMT +- request: + method: get + uri: http://file.to/get.ext + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip + - identity + response: + status: + code: 404 + message: Not Found + headers: + Date: + - Thu, 16 Dec 2021 14:54:34 GMT + Content-Type: + - text/html + Connection: + - keep-alive + Last-Modified: + - Wed, 09 Dec 2020 11:18:24 GMT + X-Amz-Error-Code: + - NoSuchKey + X-Amz-Error-Message: + - The specified key does not exist. + X-Amz-Error-Detail-Key: + - get.ext + X-Amz-Request-Id: + - HKBMJ96T2RQAW8AC + X-Amz-Id-2: + - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo= + Cf-Cache-Status: + - DYNAMIC + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}' + Nel: + - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}' + Server: + - cloudflare + Cf-Ray: + - 6be8bf678a7b4031-CDG + Content-Encoding: + - gzip + Alt-Svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443"; + ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA= + recorded_at: Wed, 04 Mar 2020 23:00:00 GMT +- request: + method: get + uri: http://file.to/get.ext + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip + - identity + response: + status: + code: 404 + message: Not Found + headers: + Date: + - Thu, 16 Dec 2021 14:54:34 GMT + Content-Type: + - text/html + Connection: + - keep-alive + Last-Modified: + - Wed, 09 Dec 2020 11:18:24 GMT + X-Amz-Error-Code: + - NoSuchKey + X-Amz-Error-Message: + - The specified key does not exist. + X-Amz-Error-Detail-Key: + - get.ext + X-Amz-Request-Id: + - HKBMJ96T2RQAW8AC + X-Amz-Id-2: + - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo= + Cf-Cache-Status: + - DYNAMIC + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}' + Nel: + - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}' + Server: + - cloudflare + Cf-Ray: + - 6be8bf678a7b4031-CDG + Content-Encoding: + - gzip + Alt-Svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443"; + ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA= + recorded_at: Wed, 04 Mar 2020 23:00:00 GMT +- request: + method: get + uri: http://file.to/get.ext + body: + encoding: US-ASCII + string: '' + headers: + Accept-Encoding: + - gzip + - identity + response: + status: + code: 404 + message: Not Found + headers: + Date: + - Thu, 16 Dec 2021 14:54:34 GMT + Content-Type: + - text/html + Connection: + - keep-alive + Last-Modified: + - Wed, 09 Dec 2020 11:18:24 GMT + X-Amz-Error-Code: + - NoSuchKey + X-Amz-Error-Message: + - The specified key does not exist. + X-Amz-Error-Detail-Key: + - get.ext + X-Amz-Request-Id: + - HKBMJ96T2RQAW8AC + X-Amz-Id-2: + - lFbJHZ+IZgA6xRrWR6SAgzb3zJEpGG4QcZ2I02NStguP2FsR3uhfV2iRztXf87V2q2aFErr3xlo= + Cf-Cache-Status: + - DYNAMIC + Report-To: + - '{"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v3?s=ytSvF9iiC%2FRp%2F%2B6vRVKR6EHExByUK5rJrQLyEtdJnqAG2lCjS63dZqb%2Fr1aZ75xn%2BRhIxPDoY95Yb2mPbMz8FPaqePZdVc1BKWhUWjiIkfBAlS4bfoVxpQMSwzSN1VYOOkBIFQXn"}],"group":"cf-nel","max_age":604800}' + Nel: + - '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}' + Server: + - cloudflare + Cf-Ray: + - 6be8bf678a7b4031-CDG + Content-Encoding: + - gzip + Alt-Svc: + - h3=":443"; ma=86400, h3-29=":443"; ma=86400, h3-28=":443"; ma=86400, h3-27=":443"; + ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + H4sIAAAAAAAAA2RTUW/UMAx+51eYIvFEm8vGuHG0RXDsBAJpEzqE4GVKE5d6a5OS+Dp6vx6l17GbeIr9yd9nf5aTP/1wud7+uLqAhru2fJIfHoC8QWViAJB3yAoa5j7F3zsaikQ7y2g55bHHBOasSBj/sIgCb0A3ygfk4tt2k54nxzpWdVgkA+Fd7zwfse/IcFMYHEhjOiUvgCwxqTYNWrVYyHshJm6x3FCLsHXwHatAjLk4wFNJdCDuLeSVM+NMbWS5+fTlAraXD7xGlnlfbhsKEAGgANwg9N716HkEV0OuoPFYF0lcw0qIPSKFTLtOJOXPGMPadb2yYy5UmT2UC21sqn+RaAV2itq0945RMzn7bIkLI2t5IquFOcXFS1nJSi4X5mwhjZTyNCnXzrLSDLsQZaF2HqpdIIshgHZxOhWVwHnonEcgWzvfTViWiz66+oo1erjy7gY1h9VjI2ElROM6vCVOterQq9nSR9fhZ2JYT+DU/D+iavtGZRXtRVK+i3FFe3jfOn2rG0U2kqYZDA0QeGyxSAyFvlXjyjqLSZkH7alnMIpVqmsVRquLpFZtwASC10fbO1QGcabPjFmenAvdup2pW+UxDayY9Lxeg9oZzDqy2U1Iynxm/usVL3Y+1Bs1qAM6d7s3NshM2/1+WsX+OspnfdO/JVPIk+X5ciFfnb5+fofV9SPkuJkwNBzd4eH8cnH4W38BAAD//wMAJ4ThOHMDAAA= + recorded_at: Wed, 04 Mar 2020 23:00:00 GMT +recorded_with: VCR 6.0.0 diff --git a/spec/lib/active_storage/download_manager_spec.rb b/spec/lib/active_storage/download_manager_spec.rb new file mode 100644 index 000000000..aabb1ba92 --- /dev/null +++ b/spec/lib/active_storage/download_manager_spec.rb @@ -0,0 +1,42 @@ +describe ActiveStorage::DownloadManager do + let(:test_dir) { Dir.mktmpdir(nil, Dir.tmpdir) } + let(:download_to_dir) { test_dir } + after { FileUtils.remove_entry_secure(test_dir) if Dir.exist?(test_dir) } + + let(:downloadable_manager) { ActiveStorage::DownloadManager.new(download_to_dir: download_to_dir) } + + describe '#download_one' do + subject { downloadable_manager.download_one(attachment: attachment, path_in_download_dir: path_in_download_dir, async_internet: double) } + + let(:path_in_download_dir) { 'lol.png' } + let(:attachment) do + PiecesJustificativesService::FakeAttachment.new( + file: StringIO.new('coucou'), + filename: "export-dossier.pdf", + name: 'pdf_export_for_instructeur', + id: 1, + created_at: Time.zone.now + ) + end + + context 'with a PiecesJustificativesService::FakeAttachment and it works' do + it 'write attachment.file to disk' do + target = File.join(download_to_dir, path_in_download_dir) + expect { subject }.to change { File.exist?(target) } + attachment.file.rewind + expect(attachment.file.read).to eq(File.read(target)) + expect(downloadable_manager.errors).not_to have_key(path_in_download_dir) + end + end + + context 'with a PiecesJustificativesService::FakeAttachment and it fails' do + it 'write attachment.file to disk' do + expect(attachment.file).to receive(:read).and_raise("boom") + target = File.join(download_to_dir, path_in_download_dir) + expect { subject }.to raise_error(StandardError) + expect(File.exist?(target)).to be_falsey + expect(downloadable_manager.errors).to have_key(path_in_download_dir) + end + end + end +end diff --git a/spec/services/procedure_archive_service_spec.rb b/spec/services/procedure_archive_service_spec.rb index e66dd1d89..e3ceda65f 100644 --- a/spec/services/procedure_archive_service_spec.rb +++ b/spec/services/procedure_archive_service_spec.rb @@ -128,10 +128,14 @@ describe ProcedureArchiveService do let(:archive) { create(:archive, time_span_type: 'monthly', status: 'pending', month: date_month) } let(:year) { 2021 } let(:mailer) { double('mailer', deliver_later: true) } - + before do + allow_any_instance_of(ActiveStorage::Attached::One).to receive(:url).and_return("http://file.to/get.ext") + end it 'collect files' do expect(InstructeurMailer).to receive(:send_archive).and_return(mailer) - service.collect_files_archive(archive, instructeur) + VCR.use_cassette('archive/file.to.get') do + service.collect_files_archive(archive, instructeur) + end archive.file.open do |f| files = ZipTricks::FileReader.read_zip_structure(io: f) @@ -207,11 +211,16 @@ describe ProcedureArchiveService do context 'for all months' do let(:archive) { create(:archive, time_span_type: 'everything', status: 'pending') } let(:mailer) { double('mailer', deliver_later: true) } + before do + allow_any_instance_of(ActiveStorage::Attached::One).to receive(:url).and_return("http://file.to/get.ext") + end it 'collect files' do expect(InstructeurMailer).to receive(:send_archive).and_return(mailer) - service.collect_files_archive(archive, instructeur) + VCR.use_cassette('archive/file.to.get') do + service.collect_files_archive(archive, instructeur) + end archive = Archive.last archive.file.open do |f| @@ -234,6 +243,36 @@ describe ProcedureArchiveService do end end + describe '#download_and_zip' do + it 'create a tmpdir while block is running' do + previous_dir_list = Dir.entries(ProcedureArchiveService::ARCHIVE_CREATION_DIR) + + service.send(:download_and_zip, []) do |_zip_file| + new_dir_list = Dir.entries(ProcedureArchiveService::ARCHIVE_CREATION_DIR) + expect(previous_dir_list).not_to eq(new_dir_list) + end + end + + it 'cleans up its tmpdir after block execution' do + expect { service.send(:download_and_zip, []) { |zip_file| } } + .not_to change { Dir.entries(ProcedureArchiveService::ARCHIVE_CREATION_DIR) } + end + + it 'creates a zip with zip utility' do + expected_zip_path = File.join(ProcedureArchiveService::ARCHIVE_CREATION_DIR, "#{service.send(:zip_root_folder)}.zip") + expect(service).to receive(:system).with('zip', '-0', '-r', expected_zip_path, an_instance_of(String)) + service.send(:download_and_zip, []) { |zip_path| } + end + + it 'cleans up its generated zip' do + expected_zip_path = File.join(ProcedureArchiveService::ARCHIVE_CREATION_DIR, "#{service.send(:zip_root_folder)}.zip") + service.send(:download_and_zip, []) do |_zip_path| + expect(File.exist?(expected_zip_path)).to be_truthy + end + expect(File.exist?(expected_zip_path)).to be_falsey + end + end + private def create_dossier_for_month(year, month)