demarches-normaliennes/spec/services/procedure_archive_service_spec.rb
simon lehericey f0b0e7fd9a Switch to usage of zip unix binary to create archive. Also use a dedicated queue for DelayedJob
use dedicated archives queue

As the used disk space will increase, we want a fined grain control

move zip logic in dedicated method

zip

wip

wip

fix(spec): pass spec in green

tech(improvements): avoid File.delete(folder), favor FileUtils.remove_entry_secure which is safer. Also wrap most of code that open file within blocks so it is cleaned when the block ends. Lastly use  attachement.download to avoid big memory pressure [download in chunk, write in chunk] otherwise big file [124>1GO] are loaded in memory. what if we run multiple jobs/download in parallel ?

fix(spec): try to retry with grace

clean(procedure_archive_service_spec.rb): better retry [avoid to rewrite on open file]

lint(things): everything
2021-12-13 16:37:04 +01:00

153 lines
6 KiB
Ruby

describe ProcedureArchiveService do
let(:procedure) { create(:procedure, :published) }
let(:instructeur) { create(:instructeur) }
let(:service) { ProcedureArchiveService.new(procedure) }
let(:year) { 2020 }
let(:month) { 3 }
let(:date_month) { Date.strptime("#{year}-#{month}", "%Y-%m") }
describe '#create_pending_archive' do
context 'for a specific month' do
it 'creates a pending archive' do
archive = service.create_pending_archive(instructeur, 'monthly', date_month)
expect(archive.time_span_type).to eq 'monthly'
expect(archive.month).to eq date_month
expect(archive.pending?).to be_truthy
end
end
context 'for all months' do
it 'creates a pending archive' do
archive = service.create_pending_archive(instructeur, 'everything')
expect(archive.time_span_type).to eq 'everything'
expect(archive.month).to eq nil
expect(archive.pending?).to be_truthy
end
end
end
describe '#collect_files_archive' do
let!(:dossier) { create_dossier_for_month(year, month) }
let!(:dossier_2020) { create_dossier_for_month(2020, month) }
after { Timecop.return }
context 'for a specific month' do
let(:archive) { create(:archive, time_span_type: 'monthly', status: 'pending', month: date_month) }
let(:year) { 2021 }
let(:mailer) { double('mailer', deliver_later: true) }
it 'collect files' do
expect(InstructeurMailer).to receive(:send_archive).and_return(mailer)
service.collect_files_archive(archive, instructeur)
archive.file.open do |f|
files = ZipTricks::FileReader.read_zip_structure(io: f)
structure = [
"procedure-#{procedure.id}/",
"procedure-#{procedure.id}/dossier-#{dossier.id}/",
"procedure-#{procedure.id}/dossier-#{dossier.id}/pieces_justificatives/",
"procedure-#{procedure.id}/dossier-#{dossier.id}/pieces_justificatives/attestation-dossier--05-03-2021-00-00-#{dossier.attestation.pdf.id % 10000}.pdf",
"procedure-#{procedure.id}/dossier-#{dossier.id}/export-#{dossier.id}-05-03-2021-00-00-#{dossier.id}.pdf"
]
expect(files.map(&:filename)).to match_array(structure)
end
expect(archive.file.attached?).to be_truthy
end
context 'with a missing file' do
let(:pj) do
PiecesJustificativesService::FakeAttachment.new(
file: StringIO.new('coucou'),
filename: "export-dossier.pdf",
name: 'pdf_export_for_instructeur',
id: 1,
created_at: Time.zone.now
)
end
let(:bad_pj) do
PiecesJustificativesService::FakeAttachment.new(
file: nil,
filename: "cni.png",
name: 'cni.png',
id: 2,
created_at: Time.zone.now
)
end
let(:documents) { [pj, bad_pj] }
before do
allow(PiecesJustificativesService).to receive(:liste_documents).and_return(documents)
end
it 'collect files without raising exception' do
expect { service.collect_files_archive(archive, instructeur) }.not_to raise_exception
end
it 'add bug report to archive' do
service.collect_files_archive(archive, instructeur)
archive.file.open do |f|
files = ZipTricks::FileReader.read_zip_structure(io: f)
structure = [
"procedure-#{procedure.id}/",
"procedure-#{procedure.id}/dossier-#{dossier.id}/",
"procedure-#{procedure.id}/dossier-#{dossier.id}/export-dossier-05-03-2020-00-00-1.pdf",
"procedure-#{procedure.id}/dossier-#{dossier.id}/pieces_justificatives/",
"procedure-#{procedure.id}/dossier-#{dossier.id}/export-#{dossier.id}-05-03-2021-00-00-#{dossier.id}.pdf",
"procedure-#{procedure.id}/LISEZMOI.txt"
]
expect(files.map(&:filename)).to match_array(structure)
expect(extract(f, files.last)).to match(/Impossible de .* .*cni.*png/)
end
end
end
end
context 'for all months' do
let(:archive) { create(:archive, time_span_type: 'everything', status: 'pending') }
let(:mailer) { double('mailer', deliver_later: true) }
it 'collect files' do
expect(InstructeurMailer).to receive(:send_archive).and_return(mailer)
service.collect_files_archive(archive, instructeur)
archive = Archive.last
archive.file.open do |f|
files = ZipTricks::FileReader.read_zip_structure(io: f)
structure = [
"procedure-#{procedure.id}/",
"procedure-#{procedure.id}/dossier-#{dossier.id}/",
"procedure-#{procedure.id}/dossier-#{dossier.id}/pieces_justificatives/",
"procedure-#{procedure.id}/dossier-#{dossier.id}/pieces_justificatives/attestation-dossier--05-03-2020-00-00-#{dossier.attestation.pdf.id % 10000}.pdf",
"procedure-#{procedure.id}/dossier-#{dossier.id}/export-#{dossier.id}-05-03-2020-00-00-#{dossier.id}.pdf",
"procedure-#{procedure.id}/dossier-#{dossier_2020.id}/",
"procedure-#{procedure.id}/dossier-#{dossier_2020.id}/export-#{dossier_2020.id}-05-03-2020-00-00-#{dossier_2020.id}.pdf",
"procedure-#{procedure.id}/dossier-#{dossier_2020.id}/pieces_justificatives/",
"procedure-#{procedure.id}/dossier-#{dossier_2020.id}/pieces_justificatives/attestation-dossier--05-03-2020-00-00-#{dossier_2020.attestation.pdf.id % 10000}.pdf"
]
expect(files.map(&:filename)).to match_array(structure)
end
expect(archive.file.attached?).to be_truthy
end
end
end
private
def create_dossier_for_month(year, month)
Timecop.freeze(Time.zone.local(year, month, 5))
create(:dossier, :accepte, :with_attestation, procedure: procedure)
end
def extract(zip_file, zip_entry)
extractor = zip_entry.extractor_from(zip_file)
extractor.extract
end
end