DGNum/demarches-normaliennes
13
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

extract download all attachments in dedicated class using async/async-http for better perf

Browse source
This commit is contained in:
Martin 2021-12-13 14:35:13 +01:00
parent 441d730b8e
commit 2ed9cccba0
8 changed files with 585 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

87
app/lib/active_storage/download_manager.rb Normal file
View file

@ -0,0 +1,87 @@
require 'async'
require 'async/barrier'
require 'async/http/internet'
class ActiveStorage::DownloadManager
include Utils::Retryable
DOWNLOAD_MAX_PARALLEL = ENV.fetch('DOWNLOAD_MAX_PARALLEL') { 10 }
attr_reader :download_to_dir, :errors
def download_all(attachments:, on_failure:)
Async do
internet = Async::HTTP::Internet.new
barrier = Async::Barrier.new
semaphore = Async::Semaphore.new(DOWNLOAD_MAX_PARALLEL, parent: barrier)
attachments.map do |attachment, path|
semaphore.async do
begin
with_retry(max_attempt: 1) do
download_one(attachment: attachment,
path_in_download_dir: path,
async_internet: internet)
end
rescue => e
on_failure.call(attachment, path, e)
end
end
end
barrier.wait
write_error_manifest if !errors.empty?
ensure
internet&.close
end
end
# beware, must be re-entrant because retryable
def download_one(attachment:, path_in_download_dir:, async_internet:)
byte_written = 0
attachment_path = File.join(download_to_dir, path_in_download_dir)
attachment_dir = File.dirname(attachment_path)
FileUtils.mkdir_p(attachment_dir) if !Dir.exist?(attachment_dir) # defensive, do not write in undefined dir
if attachment.is_a?(PiecesJustificativesService::FakeAttachment)
byte_written = File.write(attachment_path, attachment.file.read, mode: 'wb')
else
response = async_internet.get(attachment.url)
File.open(attachment_path, mode: 'wb') do |fd|
response.body.each do |chunk|
byte_written = byte_written + fd.write(chunk)
end
response.body.close
end
end
track_retryable_download_state(attachment_path: attachment_path, state: true) # -> fail once, success after -> no failure
byte_written
rescue
track_retryable_download_state(attachment_path: attachment_path, state: false) #
File.delete(attachment_path) if File.exist?(attachment_path) # -> case of retries failed, must cleanup partialy downloaded file
raise
end
private
def initialize(download_to_dir:)
@download_to_dir = download_to_dir
@errors = {}
end
def track_retryable_download_state(attachment_path:, state:)
key = File.basename(attachment_path)
if state
errors.delete(key) # do not keep track of success, otherwise errors map grows
else
errors[key] = state
end
end
def write_error_manifest
manifest_path = File.join(download_to_dir, 'LISEZMOI.txt')
manifest_content = errors.map do |file_basename, _failed|
"Impossible de récupérer le fichier #{file_basename}"
end
.join("\n")
File.write(manifest_path, manifest_content)
end
end

17
app/lib/active_storage/downloadable_file.rb
View file

@ -1,20 +1,5 @@
class ActiveStorage::DownloadableFile
# https://edgeapi.rubyonrails.org/classes/ActiveStorage/Blob.html#method-i-download
def self.download(attachment:, destination_path:, in_chunk: true)
byte_written = 0
File.open(destination_path, mode: 'wb') do |fd| # we expact a path as string, so we can recreate the file (ex: failure/retry on former existing fd)
if in_chunk
attachment.download do |chunk|
byte_written += fd.write(chunk)
end
else
byte_written = fd.write(attachment.download)
end
end
byte_written
end
def self.create_list_from_dossier(dossier, for_expert = false)
dossier_export = PiecesJustificativesService.generate_dossier_export(dossier)
pjs = [dossier_export] + PiecesJustificativesService.liste_documents(dossier, for_expert)

36
app/services/procedure_archive_service.rb
View file

@ -1,7 +1,6 @@
require 'tempfile'
class ProcedureArchiveService
include Utils::Retryable
ARCHIVE_CREATION_DIR = ENV.fetch('ARCHIVE_CREATION_DIR') { '/tmp' }
def initialize(procedure)
@ -37,7 +36,6 @@ class ProcedureArchiveService
archive.file.attach(
io: File.open(zip_file),
filename: archive.filename(@procedure),
# we don't want to run virus scanner on this file
metadata: { virus_scan_result: ActiveStorage::VirusScanner::SAFE }
)
end
@ -104,32 +102,16 @@ class ProcedureArchiveService
FileUtils.remove_entry_secure(archive_dir) if Dir.exist?(archive_dir)
Dir.mkdir(archive_dir)
bug_reports = ''
attachments.each do |attachment, path|
attachment_path = File.join(archive_dir, path)
attachment_dir = File.dirname(attachment_path)
ActiveStorage::DownloadManager
.new(download_to_dir: archive_dir)
.download_all(attachments: attachments,
on_failure: proc { |_attachment, path, error|
Rails.logger.error("Fail to download filename #{path} in procedure##{@procedure.id}, reason: #{error}")
})
FileUtils.mkdir_p(attachment_dir) if !Dir.exist?(attachment_dir)
begin
with_retry(max_attempt: 1) do
ActiveStorage::DownloadableFile.download(attachment: attachment,
destination_path: attachment_path,
in_chunk: true)
end
rescue => e
Rails.logger.error("Fail to download filename #{File.basename(attachment_path)} in procedure##{@procedure.id}, reason: #{e}")
File.delete(attachment_path) if File.exist?(attachment_path)
bug_reports += "Impossible de récupérer le fichier #{File.basename(attachment_path)}\n"
end
end
if !bug_reports.empty?
File.write(File.join(archive_dir, 'LISEZMOI.txt'), bug_reports)
end
File.delete(zip_path) if File.exist?(zip_path)
Dir.chdir(tmp_dir) do
system 'zip', '-r', zip_path, zip_root_folder
File.delete(zip_path) if File.exist?(zip_path)
system 'zip', '-0', '-r', zip_path, zip_root_folder
end
yield(zip_path)
ensure
@ -152,8 +134,8 @@ class ProcedureArchiveService
def self.attachments_from_champs_piece_justificative(champs)
champs
.filter { |c| c.type_champ == TypeDeChamp.type_champs.fetch(:piece_justificative) }
.filter { |pj| pj.piece_justificative_file.attached? }
.map(&:piece_justificative_file)
.filter(&:attached?)
end
def self.liste_pieces_justificatives_for_archive(dossier)