DGNum/demarches-normaliennes
13
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Merge pull request #7058 from betagouv/fix_fd_leak_in_archive_creation

Browse source 
fix(parallel_download_queue): avoid fd leak in archive creation
This commit is contained in:
mfo 2022-03-18 08:36:27 +01:00 • committed by GitHub
commit f91cf5ab8a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/lib/download_manager/parallel_download_queue.rb
View file

@ -26,7 +26,6 @@ module DownloadManager
hydra.run hydra.run
end end
# rubocop:disable Style/AutoResourceCleanup
# can't be used with typhoeus, otherwise block is closed before the request is run by hydra # can't be used with typhoeus, otherwise block is closed before the request is run by hydra
def download_one(attachment:, path_in_download_dir:, http_client:) def download_one(attachment:, path_in_download_dir:, http_client:)
attachment_path = File.join(destination, path_in_download_dir) attachment_path = File.join(destination, path_in_download_dir)
@ -37,12 +36,10 @@ module DownloadManager
File.write(attachment_path, attachment.file.read, mode: 'wb') File.write(attachment_path, attachment.file.read, mode: 'wb')
else else
request = Typhoeus::Request.new(attachment.url) request = Typhoeus::Request.new(attachment.url)
fd = File.open(attachment_path, mode: 'wb')
request.on_body do |chunk| request.on_body do |chunk|
fd.write(chunk) File.write(attachment_path, chunk, mode: 'a+b')
end end
request.on_complete do |response| request.on_complete do |response|
fd.close
unless response.success? unless response.success?
File.delete(attachment_path) if File.exist?(attachment_path) # -> case of retries failed, must cleanup partialy downloaded file File.delete(attachment_path) if File.exist?(attachment_path) # -> case of retries failed, must cleanup partialy downloaded file
on_error.call(attachment, path_in_download_dir, response.code) on_error.call(attachment, path_in_download_dir, response.code)
@ -51,6 +48,5 @@ module DownloadManager
http_client.queue(request) http_client.queue(request)
end end
end end
# rubocop:enable Style/AutoResourceCleanup
end end
end end