From f40d96fbd20c33df69d5fb9f5ed97aa99e38af30 Mon Sep 17 00:00:00 2001 From: Christophe Robillard Date: Thu, 29 Apr 2021 17:29:47 +0200 Subject: [PATCH 1/2] =?UTF-8?q?Revert=20"Revert=20"Export=20de=20tous=20le?= =?UTF-8?q?s=20dossier=20d'une=20d=C3=A9marche""?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit d9a588b52e781f7de508c4f6cca018c9d4b87168. --- .../instructeurs/archives_controller.rb | 55 +++++++++++ .../instructeurs/dossiers_controller.rb | 7 -- app/helpers/archive_helper.rb | 5 + app/jobs/archive_creation_job.rb | 7 ++ app/jobs/cron/purge_stale_archives_job.rb | 7 ++ app/lib/active_storage/downloadable_file.rb | 20 ++-- app/mailers/instructeur_mailer.rb | 10 ++ app/models/archive.rb | 71 +++++++++++++++ app/models/dossier.rb | 7 +- app/models/instructeur.rb | 1 + app/models/procedure.rb | 14 +++ app/services/pieces_justificatives_service.rb | 14 ++- app/services/procedure_archive_service.rb | 72 +++++++++++++++ .../instructeur_mailer/send_archive.html.haml | 21 +++++ .../instructeurs/archives/create.js.haml | 1 + .../instructeurs/archives/index.html.haml | 91 +++++++++++++++++++ .../procedures/_download_dossiers.html.haml | 3 + config/locales/views/instructeurs/fr.yml | 3 + config/routes.rb | 2 + ...8_create_archive_for_groupe_instructeur.rb | 17 ++++ ..._type_to_to_time_span_type_for_archives.rb | 5 + .../20210427124500_add_key_to_archives.rb | 6 ++ db/schema.rb | 23 ++++- .../instructeurs/archives_controller_spec.rb | 56 ++++++++++++ .../instructeurs/dossiers_controller_spec.rb | 7 -- spec/factories/archive.rb | 15 +++ spec/features/instructeurs/expert_spec.rb | 2 + .../features/instructeurs/instruction_spec.rb | 18 ++-- spec/models/archive_spec.rb | 51 +++++++++++ .../pieces_justificatives_service_spec.rb | 7 +- .../procedure_archive_service_spec.rb | 83 +++++++++++++++++ .../_download_dossiers.html.haml_spec.rb | 10 ++ 32 files changed, 675 insertions(+), 36 deletions(-) create mode 100644 app/controllers/instructeurs/archives_controller.rb create mode 100644 app/helpers/archive_helper.rb create mode 100644 app/jobs/archive_creation_job.rb create mode 100644 app/jobs/cron/purge_stale_archives_job.rb create mode 100644 app/models/archive.rb create mode 100644 app/services/procedure_archive_service.rb create mode 100644 app/views/instructeur_mailer/send_archive.html.haml create mode 100644 app/views/instructeurs/archives/create.js.haml create mode 100644 app/views/instructeurs/archives/index.html.haml create mode 100644 db/migrate/20201104163658_create_archive_for_groupe_instructeur.rb create mode 100644 db/migrate/20210427112642_rename_content_type_to_to_time_span_type_for_archives.rb create mode 100644 db/migrate/20210427124500_add_key_to_archives.rb create mode 100644 spec/controllers/instructeurs/archives_controller_spec.rb create mode 100644 spec/factories/archive.rb create mode 100644 spec/models/archive_spec.rb create mode 100644 spec/services/procedure_archive_service_spec.rb diff --git a/app/controllers/instructeurs/archives_controller.rb b/app/controllers/instructeurs/archives_controller.rb new file mode 100644 index 000000000..2917520be --- /dev/null +++ b/app/controllers/instructeurs/archives_controller.rb @@ -0,0 +1,55 @@ +module Instructeurs + class ArchivesController < InstructeurController + before_action :ensure_procedure_enabled + + def index + @procedure = procedure + + @archivable_months = archivable_months + @dossiers_termines = @procedure.dossiers.state_termine + @poids_total = ProcedureArchiveService.procedure_files_size(@procedure) + groupe_instructeur = current_instructeur.groupe_instructeurs.where(procedure: @procedure.id).first + @archives = Archive.for_groupe_instructeur(groupe_instructeur) + end + + def create + type = params[:type] + month = Date.strptime(params[:month], '%Y-%m') if params[:month].present? + + archive = ProcedureArchiveService.new(procedure).create_pending_archive(current_instructeur, type, month) + if archive.pending? + ArchiveCreationJob.perform_later(procedure, archive, current_instructeur) + flash[:notice] = "Votre demande a été prise en compte. Selon le nombre de dossiers, cela peut prendre quelques minutes. Vous recevrez un courriel lorsque le fichier sera disponible." + else + flash[:notice] = "Cette archive a déjà été générée." + end + redirect_to instructeur_archives_path(procedure) + end + + private + + def ensure_procedure_enabled + if !procedure.feature_enabled?(:archive_zip_globale) || procedure.brouillon? + flash[:alert] = "L'accès aux archives n'est pas disponible pour cette démarche, merci d'en faire la demande à l'équipe de démarches simplifiees" + return redirect_to instructeur_procedure_path(procedure) + end + end + + def archivable_months + start_date = procedure.published_at.to_date + end_date = Time.zone.now.to_date + + (start_date...end_date) + .map(&:beginning_of_month) + .uniq + .reverse + end + + def procedure + current_instructeur + .procedures + .for_download + .find(params[:procedure_id]) + end + end +end diff --git a/app/controllers/instructeurs/dossiers_controller.rb b/app/controllers/instructeurs/dossiers_controller.rb index 37b7e1d4c..e6fb16d21 100644 --- a/app/controllers/instructeurs/dossiers_controller.rb +++ b/app/controllers/instructeurs/dossiers_controller.rb @@ -213,7 +213,6 @@ module Instructeurs def telecharger_pjs return head(:forbidden) if !dossier.attachments_downloadable? - generate_pdf_for_instructeur_export files = ActiveStorage::DownloadableFile.create_list_from_dossier(dossier) zipline(files, "dossier-#{dossier.id}.zip") @@ -239,12 +238,6 @@ module Instructeurs .find(params[:dossier_id]) end - def generate_pdf_for_instructeur_export - @include_infos_administration = true - pdf = render_to_string(template: 'dossiers/show', formats: [:pdf]) - dossier.pdf_export_for_instructeur.attach(io: StringIO.open(pdf), filename: "export-#{dossier.id}.pdf", content_type: 'application/pdf') - end - def commentaire_params params.require(:commentaire).permit(:body, :piece_jointe) end diff --git a/app/helpers/archive_helper.rb b/app/helpers/archive_helper.rb new file mode 100644 index 000000000..b349b8793 --- /dev/null +++ b/app/helpers/archive_helper.rb @@ -0,0 +1,5 @@ +module ArchiveHelper + def can_generate_archive?(dossiers_termines, poids_total) + dossiers_termines.count < 100 && poids_total < 1.gigabyte + end +end diff --git a/app/jobs/archive_creation_job.rb b/app/jobs/archive_creation_job.rb new file mode 100644 index 000000000..255af07db --- /dev/null +++ b/app/jobs/archive_creation_job.rb @@ -0,0 +1,7 @@ +class ArchiveCreationJob < ApplicationJob + def perform(procedure, archive, instructeur) + ProcedureArchiveService + .new(procedure) + .collect_files_archive(archive, instructeur) + end +end diff --git a/app/jobs/cron/purge_stale_archives_job.rb b/app/jobs/cron/purge_stale_archives_job.rb new file mode 100644 index 000000000..4ce48784a --- /dev/null +++ b/app/jobs/cron/purge_stale_archives_job.rb @@ -0,0 +1,7 @@ +class Cron::PurgeStaleArchivesJob < Cron::CronJob + self.schedule_expression = "every 5 minutes" + + def perform + Archive.stale.destroy_all + end +end diff --git a/app/lib/active_storage/downloadable_file.rb b/app/lib/active_storage/downloadable_file.rb index db86efc73..d8ca5cf4c 100644 --- a/app/lib/active_storage/downloadable_file.rb +++ b/app/lib/active_storage/downloadable_file.rb @@ -1,14 +1,12 @@ class ActiveStorage::DownloadableFile def self.create_list_from_dossier(dossier) pjs = PiecesJustificativesService.liste_pieces_justificatives(dossier) - files = pjs.map do |piece_justificative| + pjs.map do |piece_justificative| [ piece_justificative, - self.timestamped_filename(piece_justificative) + "dossier-#{dossier.id}/#{self.timestamped_filename(piece_justificative)}" ] end - files << [dossier.pdf_export_for_instructeur, self.timestamped_filename(dossier.pdf_export_for_instructeur)] - files end private @@ -22,19 +20,23 @@ class ActiveStorage::DownloadableFile timestamp = attachment.created_at.strftime("%d-%m-%Y-%H-%M") id = attachment.id % 10000 - "#{folder}/#{basename}-#{timestamp}-#{id}#{extension}" + [folder, "#{basename}-#{timestamp}-#{id}#{extension}"].join end def self.folder(attachment) + if attachment.name == 'pdf_export_for_instructeur' + return '' + end + case attachment.record_type when 'Dossier' - 'dossier' + 'dossier/' when 'DossierOperationLog', 'BillSignature' - 'horodatage' + 'horodatage/' when 'Commentaire' - 'messagerie' + 'messagerie/' else - 'pieces_justificatives' + 'pieces_justificatives/' end end diff --git a/app/mailers/instructeur_mailer.rb b/app/mailers/instructeur_mailer.rb index 89c58864c..55332cf38 100644 --- a/app/mailers/instructeur_mailer.rb +++ b/app/mailers/instructeur_mailer.rb @@ -1,5 +1,7 @@ # Preview all emails at http://localhost:3000/rails/mailers/instructeur_mailer class InstructeurMailer < ApplicationMailer + helper MailerHelper + layout 'mailers/layout' def user_to_instructeur(email) @@ -42,4 +44,12 @@ class InstructeurMailer < ApplicationMailer mail(to: instructeur.email, subject: subject) end + + def send_archive(instructeur, procedure, archive) + @archive = archive + @procedure = procedure + subject = "Votre archive est disponible" + + mail(to: instructeur.email, subject: subject) + end end diff --git a/app/models/archive.rb b/app/models/archive.rb new file mode 100644 index 000000000..7ee93720f --- /dev/null +++ b/app/models/archive.rb @@ -0,0 +1,71 @@ +# == Schema Information +# +# Table name: archives +# +# id :bigint not null, primary key +# key :text not null +# month :date +# status :string not null +# time_span_type :string not null +# created_at :datetime not null +# updated_at :datetime not null +# +class Archive < ApplicationRecord + include AASM + + RETENTION_DURATION = 1.week + + has_and_belongs_to_many :groupe_instructeurs + + has_one_attached :file + + scope :stale, -> { where('updated_at < ?', (Time.zone.now - RETENTION_DURATION)) } + scope :for_groupe_instructeur, -> (groupe_instructeur) { + joins(:archives_groupe_instructeurs) + .where( + archives_groupe_instructeurs: { groupe_instructeur: groupe_instructeur } + ) + } + + enum time_span_type: { + everything: 'everything', + monthly: 'monthly' + } + + enum status: { + pending: 'pending', + generated: 'generated' + } + + aasm whiny_persistence: true, column: :status, enum: true do + state :pending, initial: true + state :generated + + event :make_available do + transitions from: :pending, to: :generated + end + end + + def available? + status == 'generated' && file.attached? + end + + def filename(procedure) + if time_span_type == 'everything' + "procedure-#{procedure.id}.zip" + else + "procedure-#{procedure.id}-mois-#{I18n.l(month, format: '%Y-%m')}.zip" + end + end + + def self.find_or_create_archive(time_span_type, month, groupe_instructeurs) + create_with(groupe_instructeurs: groupe_instructeurs) + .create_or_find_by(time_span_type: time_span_type, month: month, key: generate_cache_key(groupe_instructeurs)) + end + + private + + def self.generate_cache_key(groupe_instructeurs) + groupe_instructeurs.map(&:id).sort.join('-') + end +end diff --git a/app/models/dossier.rb b/app/models/dossier.rb index a0add0355..8b8a1e60e 100644 --- a/app/models/dossier.rb +++ b/app/models/dossier.rb @@ -172,7 +172,12 @@ class Dossier < ApplicationRecord scope :en_construction, -> { not_archived.state_en_construction } scope :en_instruction, -> { not_archived.state_en_instruction } scope :termine, -> { not_archived.state_termine } - scope :downloadable_sorted, -> { + scope :processed_in_month, -> (month) do + state_termine + .joins(:traitements) + .where(traitements: { processed_at: month.beginning_of_month..month.end_of_month }) + end + scope :downloadable_sorted, -> { state_not_brouillon .includes( :user, diff --git a/app/models/instructeur.rb b/app/models/instructeur.rb index f43eda8fb..d3e2fc21b 100644 --- a/app/models/instructeur.rb +++ b/app/models/instructeur.rb @@ -25,6 +25,7 @@ class Instructeur < ApplicationRecord has_many :followed_dossiers, through: :follows, source: :dossier has_many :previously_followed_dossiers, -> { distinct }, through: :previous_follows, source: :dossier has_many :trusted_device_tokens, dependent: :destroy + has_many :archives has_one :user, dependent: :nullify diff --git a/app/models/procedure.rb b/app/models/procedure.rb index ea0b4e570..b4760c353 100644 --- a/app/models/procedure.rb +++ b/app/models/procedure.rb @@ -156,6 +156,20 @@ class Procedure < ApplicationRecord includes(:draft_revision, :published_revision, administrateurs: :user) } + scope :for_download, -> { + includes( + :groupe_instructeurs, + dossiers: { + champs: [ + piece_justificative_file_attachment: :blob, + champs: [ + piece_justificative_file_attachment: :blob + ] + ] + } + ) + } + validates :libelle, presence: true, allow_blank: false, allow_nil: false validates :description, presence: true, allow_blank: false, allow_nil: false validates :administrateurs, presence: true diff --git a/app/services/pieces_justificatives_service.rb b/app/services/pieces_justificatives_service.rb index a5903531b..f3cd8303c 100644 --- a/app/services/pieces_justificatives_service.rb +++ b/app/services/pieces_justificatives_service.rb @@ -1,10 +1,11 @@ class PiecesJustificativesService def self.liste_pieces_justificatives(dossier) + dossier_export = generate_dossier_export(dossier) pjs_champs = pjs_for_champs(dossier) pjs_commentaires = pjs_for_commentaires(dossier) pjs_dossier = pjs_for_dossier(dossier) - (pjs_champs + pjs_commentaires + pjs_dossier) + ([dossier_export] + pjs_champs + pjs_commentaires + pjs_dossier) .filter(&:attached?) end @@ -43,6 +44,17 @@ class PiecesJustificativesService private + def self.generate_dossier_export(dossier) + pdf = ApplicationController + .render(template: 'dossiers/show', formats: [:pdf], + assigns: { + include_infos_administration: true, + dossier: dossier + }) + dossier.pdf_export_for_instructeur.attach(io: StringIO.open(pdf), filename: "export-#{dossier.id}.pdf", content_type: 'application/pdf') + dossier.pdf_export_for_instructeur + end + def self.pjs_for_champs(dossier) allowed_champs = dossier.champs + dossier.champs_private diff --git a/app/services/procedure_archive_service.rb b/app/services/procedure_archive_service.rb new file mode 100644 index 000000000..b26757210 --- /dev/null +++ b/app/services/procedure_archive_service.rb @@ -0,0 +1,72 @@ +require 'tempfile' + +class ProcedureArchiveService + def initialize(procedure) + @procedure = procedure + end + + def create_pending_archive(instructeur, type, month = nil) + groupe_instructeurs = instructeur + .groupe_instructeurs + .where(procedure: @procedure) + + Archive.find_or_create_archive(type, month, groupe_instructeurs) + end + + def collect_files_archive(archive, instructeur) + if archive.time_span_type == 'everything' + dossiers = @procedure.dossiers.state_termine + else + dossiers = @procedure.dossiers.processed_in_month(archive.month) + end + + files = create_list_of_attachments(dossiers) + + tmp_file = Tempfile.new(['tc', '.zip']) + + Zip::OutputStream.open(tmp_file) do |zipfile| + files.each do |attachment, pj_filename| + zipfile.put_next_entry(pj_filename) + zipfile.puts(attachment.download) + end + end + + archive.file.attach(io: File.open(tmp_file), filename: archive.filename(@procedure)) + tmp_file.delete + archive.make_available! + InstructeurMailer.send_archive(instructeur, @procedure, archive).deliver_later + end + + def self.procedure_files_size(procedure) + dossiers_files_size(procedure.dossiers) + end + + def self.dossiers_files_size(dossiers) + dossiers.map do |dossier| + liste_pieces_justificatives_for_archive(dossier).sum(&:byte_size) + end.sum + end + + private + + def create_list_of_attachments(dossiers) + dossiers.flat_map do |dossier| + ActiveStorage::DownloadableFile.create_list_from_dossier(dossier) + end + end + + def self.attachments_from_champs_piece_justificative(champs) + champs + .filter { |c| c.type_champ == TypeDeChamp.type_champs.fetch(:piece_justificative) } + .filter { |pj| pj.piece_justificative_file.attached? } + .map(&:piece_justificative_file) + end + + def self.liste_pieces_justificatives_for_archive(dossier) + champs_blocs_repetables = dossier.champs + .filter { |c| c.type_champ == TypeDeChamp.type_champs.fetch(:repetition) } + .flat_map(&:champs) + + attachments_from_champs_piece_justificative(champs_blocs_repetables + dossier.champs) + end +end diff --git a/app/views/instructeur_mailer/send_archive.html.haml b/app/views/instructeur_mailer/send_archive.html.haml new file mode 100644 index 000000000..11857c69f --- /dev/null +++ b/app/views/instructeur_mailer/send_archive.html.haml @@ -0,0 +1,21 @@ +- content_for(:title, 'Votre archive est disponible') + +%p + Bonjour, + +%p + Votre archive pour la démarche + = link_to("#{@procedure.id} − #{@procedure.libelle}", instructeur_procedure_url(@procedure.id)) + est disponible. + Vous pouvez la télécharger dans votre espace de gestion des archives. + +%p + = round_button('Consulter mes archives', instructeur_archives_url(@procedure), :primary) + +%p + Ce fichier est + %b valide une semaine + et peut-être téléchargé + %b plusieurs fois. + += render partial: "layouts/mailers/signature" diff --git a/app/views/instructeurs/archives/create.js.haml b/app/views/instructeurs/archives/create.js.haml new file mode 100644 index 000000000..7fe9f7f0b --- /dev/null +++ b/app/views/instructeurs/archives/create.js.haml @@ -0,0 +1 @@ += render_flash(sticky: true) diff --git a/app/views/instructeurs/archives/index.html.haml b/app/views/instructeurs/archives/index.html.haml new file mode 100644 index 000000000..82ec92f1d --- /dev/null +++ b/app/views/instructeurs/archives/index.html.haml @@ -0,0 +1,91 @@ +- content_for(:title, "Archives pour #{@procedure.libelle}") + += render partial: 'new_administrateur/breadcrumbs', + locals: { steps: [link_to(@procedure.libelle, instructeur_procedure_path(@procedure)), + 'Archives'] } + +.container + %h1 Archives + + .card.featured + .card-title Gestion de vos archives + %p + Vous pouvez télécharger les archives des dossiers terminés depuis la publication de la procédure au format Zip. + + %p + Cet export contient les demande déposée par l'usager et la liste des pièces justificatives transmises. + + %p + Cet export n'est pas possible pour le moment pour les démarches à forte volumétrie. + Nous vous invitons à regarder + = link_to 'la documentation', ARCHIVAGE_DOC_URL + afin de voir les options à votre disposition pour mettre en place un système d'archive. + + %table.table.hoverable + %thead + %tr + %th   + %th Nombre de dossiers terminés + %th Poids estimé + %th Télécharger + + %tbody + - if can_generate_archive?(@dossiers_termines, @poids_total) + %tr + - matching_archive = @archives.find_by(time_span_type: 'everything') + %td + Tous les dossiers + %td + = @dossiers_termines.count + %td + - if matching_archive.present? && matching_archive.available? + - weight = matching_archive.file.byte_size + - else + - weight = @poids_total + = number_to_human_size(weight) + %td + - if matching_archive.try(&:available?) + = link_to url_for(matching_archive.file), class: 'button primary' do + %span.icon.download-white + = t(:archive_ready_html, generated_period: time_ago_in_words(matching_archive.updated_at), scope: [:instructeurs, :procedure]) + - elsif matching_archive.try(&:pending?) + %span.icon.retry + = t(:archive_pending_html, created_period: time_ago_in_words(matching_archive.created_at), scope: [:instructeurs, :procedure]) + - elsif @dossiers_termines.count > 0 + = link_to instructeur_archives_path(@procedure, type: 'everything'), method: :post, class: "button" do + %span.icon.new-folder + Demander la création + - else + Rien à télécharger ! + - @archivable_months.each do |month| + - dossiers_termines = @procedure.dossiers.processed_in_month(month) + - nb_dossiers_termines = dossiers_termines.count + - matching_archive = @archives.find_by(time_span_type: 'monthly', month: month) + %tr + %td + = I18n.l(month, format: "%B %Y") + %td + = nb_dossiers_termines + %td + - if matching_archive.present? && matching_archive.available? + - weight = matching_archive.file.byte_size + - else + - weight = ProcedureArchiveService::dossiers_files_size(dossiers_termines) + = number_to_human_size(weight) + %td + - if nb_dossiers_termines > 0 + - if matching_archive.present? + - if matching_archive.status == 'generated' && matching_archive.file.attached? + = link_to url_for(matching_archive.file), class: 'button primary' do + %span.icon.download-white + = t(:archive_ready_html, generated_period: time_ago_in_words(matching_archive.updated_at), scope: [:instructeurs, :procedure]) + - else + %span.icon.retry + = t(:archive_pending_html, created_period: time_ago_in_words(matching_archive.created_at), scope: [:instructeurs, :procedure]) + - else + = link_to instructeur_archives_path(@procedure, type:'monthly', month: month.strftime('%Y-%m')), method: :post, class: "button" do + %span.icon.new-folder + Démander la création + - else + Rien à télécharger ! + diff --git a/app/views/instructeurs/procedures/_download_dossiers.html.haml b/app/views/instructeurs/procedures/_download_dossiers.html.haml index a465eb9c9..b3be64c10 100644 --- a/app/views/instructeurs/procedures/_download_dossiers.html.haml +++ b/app/views/instructeurs/procedures/_download_dossiers.html.haml @@ -16,3 +16,6 @@ - else %span{ 'data-export-poll-url': download_export_instructeur_procedure_path(procedure, export_format: format, no_progress_notification: true) } = t(:export_pending_html, export_time: time_ago_in_words(export.created_at), export_format: ".#{format}", scope: [:instructeurs, :procedure]) + - if procedure.feature_enabled?(:archive_zip_globale) + %li + = link_to t(:download_archive, scope: [:instructeurs, :procedure]), instructeur_archives_path(procedure) diff --git a/config/locales/views/instructeurs/fr.yml b/config/locales/views/instructeurs/fr.yml index 3cc5fabdc..ef2b1987b 100644 --- a/config/locales/views/instructeurs/fr.yml +++ b/config/locales/views/instructeurs/fr.yml @@ -8,3 +8,6 @@ fr: ods_html: Demander un export au format .ods export_ready_html: Télécharger l’export au format %{export_format}
(généré il y a %{export_time}) export_pending_html: Un export au format %{export_format} est en train d’être généré
(demandé il y a %{export_time}) + download_archive: Télécharger une archive au format .zip de tous les dossiers et leurs pièces jointes + archive_pending_html: Archive en cours de création
(demandée il y a %{created_period}) + archive_ready_html: Télécharger l'archive
(demandée il y a %{generated_period}) diff --git a/config/routes.rb b/config/routes.rb index bb953ffa4..acdac88ea 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -383,6 +383,8 @@ Rails.application.routes.draw do get 'telecharger_pjs' => 'dossiers#telecharger_pjs' end end + + resources :archives, only: [:index, :create, :show], controller: 'archives' end end get "recherche" => "recherche#index" diff --git a/db/migrate/20201104163658_create_archive_for_groupe_instructeur.rb b/db/migrate/20201104163658_create_archive_for_groupe_instructeur.rb new file mode 100644 index 000000000..51793681c --- /dev/null +++ b/db/migrate/20201104163658_create_archive_for_groupe_instructeur.rb @@ -0,0 +1,17 @@ +class CreateArchiveForGroupeInstructeur < ActiveRecord::Migration[6.0] + def change + create_table :archives do |t| + t.string :status, null: false + t.date :month + t.string :content_type, null: false + t.timestamps + end + + create_table "archives_groupe_instructeurs", force: :cascade do |t| + t.belongs_to :archive, foreign_key: true, null: false + t.belongs_to :groupe_instructeur, foreign_key: true, null: false + + t.timestamps + end + end +end diff --git a/db/migrate/20210427112642_rename_content_type_to_to_time_span_type_for_archives.rb b/db/migrate/20210427112642_rename_content_type_to_to_time_span_type_for_archives.rb new file mode 100644 index 000000000..be6d5a404 --- /dev/null +++ b/db/migrate/20210427112642_rename_content_type_to_to_time_span_type_for_archives.rb @@ -0,0 +1,5 @@ +class RenameContentTypeToToTimeSpanTypeForArchives < ActiveRecord::Migration[6.1] + def change + rename_column :archives, :content_type, :time_span_type + end +end diff --git a/db/migrate/20210427124500_add_key_to_archives.rb b/db/migrate/20210427124500_add_key_to_archives.rb new file mode 100644 index 000000000..08559a0dc --- /dev/null +++ b/db/migrate/20210427124500_add_key_to_archives.rb @@ -0,0 +1,6 @@ +class AddKeyToArchives < ActiveRecord::Migration[6.1] + def change + add_column :archives, :key, :text, null: false + add_index :archives, [:key, :time_span_type, :month], unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 956e5a302..eefe0112f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_04_27_120002) do +ActiveRecord::Schema.define(version: 2021_04_27_124500) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -81,6 +81,25 @@ ActiveRecord::Schema.define(version: 2021_04_27_120002) do t.index ["procedure_id"], name: "index_administrateurs_procedures_on_procedure_id" end + create_table "archives", force: :cascade do |t| + t.string "status", null: false + t.date "month" + t.string "time_span_type", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.text "key", null: false + t.index ["key", "time_span_type", "month"], name: "index_archives_on_key_and_time_span_type_and_month", unique: true + end + + create_table "archives_groupe_instructeurs", force: :cascade do |t| + t.bigint "archive_id", null: false + t.bigint "groupe_instructeur_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["archive_id"], name: "index_archives_groupe_instructeurs_on_archive_id" + t.index ["groupe_instructeur_id"], name: "index_archives_groupe_instructeurs_on_groupe_instructeur_id" + end + create_table "assign_tos", id: :serial, force: :cascade do |t| t.integer "instructeur_id" t.datetime "created_at" @@ -735,6 +754,8 @@ ActiveRecord::Schema.define(version: 2021_04_27_120002) do end add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + add_foreign_key "archives_groupe_instructeurs", "archives" + add_foreign_key "archives_groupe_instructeurs", "groupe_instructeurs" add_foreign_key "assign_tos", "groupe_instructeurs" add_foreign_key "attestation_templates", "procedures" add_foreign_key "attestations", "dossiers" diff --git a/spec/controllers/instructeurs/archives_controller_spec.rb b/spec/controllers/instructeurs/archives_controller_spec.rb new file mode 100644 index 000000000..7ce7e4d22 --- /dev/null +++ b/spec/controllers/instructeurs/archives_controller_spec.rb @@ -0,0 +1,56 @@ +describe Instructeurs::ArchivesController, type: :controller do + let(:procedure1) { create(:procedure, :published, groupe_instructeurs: [gi1]) } + let(:procedure2) { create(:procedure, :published, groupe_instructeurs: [gi2]) } + let!(:instructeur) { create(:instructeur, groupe_instructeurs: [gi1, gi2]) } + let!(:archive1) { create(:archive, :generated, groupe_instructeurs: [gi1]) } + let!(:archive2) { create(:archive, :generated, groupe_instructeurs: [gi2]) } + let(:gi1) { create(:groupe_instructeur) } + let(:gi2) { create(:groupe_instructeur) } + + before do + sign_in(instructeur.user) + Flipper.enable(:archive_zip_globale, procedure1) + end + + after { Timecop.return } + + describe '#index' do + before do + create_dossier_for_month(procedure1, 2021, 3) + create_dossier_for_month(procedure1, 2021, 3) + create_dossier_for_month(procedure1, 2021, 2) + Timecop.freeze(Time.zone.local(2021, 3, 5)) + end + + it 'displays archives' do + get :index, { params: { procedure_id: procedure1.id } } + + expect(assigns(:dossiers_termines).size).to eq(3) + expect(assigns(:archives)).to eq([archive1]) + end + end + + describe '#create' do + let(:month) { '21-03' } + let(:date_month) { Date.strptime(month, "%Y-%m") } + let(:archive) { create(:archive) } + let(:subject) do + post :create, { + params: { procedure_id: procedure1.id, type: 'monthly', month: month } + } + end + + it "performs archive creation job" do + allow_any_instance_of(ProcedureArchiveService).to receive(:create_pending_archive).and_return(archive) + expect { subject }.to have_enqueued_job(ArchiveCreationJob).with(procedure1, archive, instructeur) + expect(flash.notice).to include("Votre demande a été prise en compte") + end + end + + private + + def create_dossier_for_month(procedure, year, month) + Timecop.freeze(Time.zone.local(year, month, 5)) + create(:dossier, :accepte, :with_attestation, procedure: procedure) + end +end diff --git a/spec/controllers/instructeurs/dossiers_controller_spec.rb b/spec/controllers/instructeurs/dossiers_controller_spec.rb index b31cb81c0..ab6d9ff5a 100644 --- a/spec/controllers/instructeurs/dossiers_controller_spec.rb +++ b/spec/controllers/instructeurs/dossiers_controller_spec.rb @@ -712,13 +712,6 @@ describe Instructeurs::DossiersController, type: :controller do dossier_id: dossier.id } end - - context 'when zip download is disabled through flipflop' do - it 'is forbidden' do - subject - expect(response).to have_http_status(:forbidden) - end - end end describe "#delete_dossier" do diff --git a/spec/factories/archive.rb b/spec/factories/archive.rb new file mode 100644 index 000000000..41fbc2542 --- /dev/null +++ b/spec/factories/archive.rb @@ -0,0 +1,15 @@ +FactoryBot.define do + factory :archive do + time_span_type { 'everything' } + groupe_instructeurs { [association(:groupe_instructeur)] } + key { 'unique-key' } + + trait :pending do + status { 'pending' } + end + + trait :generated do + status { 'generated' } + end + end +end diff --git a/spec/features/instructeurs/expert_spec.rb b/spec/features/instructeurs/expert_spec.rb index d361b3cf3..a384687b6 100644 --- a/spec/features/instructeurs/expert_spec.rb +++ b/spec/features/instructeurs/expert_spec.rb @@ -12,6 +12,8 @@ feature 'Inviting an expert:', js: true do context 'as an Instructeur' do scenario 'I can invite an expert' do + allow(ClamavService).to receive(:safe_file?).and_return(true) + # assign instructeur to linked dossier instructeur.assign_to_procedure(linked_dossier.procedure) diff --git a/spec/features/instructeurs/instruction_spec.rb b/spec/features/instructeurs/instruction_spec.rb index e24f3c48c..257ca2a73 100644 --- a/spec/features/instructeurs/instruction_spec.rb +++ b/spec/features/instructeurs/instruction_spec.rb @@ -165,10 +165,10 @@ feature 'Instructing a dossier:', js: true do expect(DownloadHelpers.download).to include "dossier-#{dossier.id}.zip" expect(files.size).to be 3 - expect(files[0].filename.include?('piece_justificative_0')).to be_truthy - expect(files[0].uncompressed_size).to be File.size(path) - expect(files[1].filename.include?('horodatage/operation')).to be_truthy - expect(files[2].filename.include?('dossier/export')).to be_truthy + expect(files[0].filename.include?('export')).to be_truthy + expect(files[1].filename.include?('piece_justificative_0')).to be_truthy + expect(files[1].uncompressed_size).to be File.size(path) + expect(files[2].filename.include?('horodatage/operation')).to be_truthy end scenario 'A instructeur can download an archive containing several identical attachments' do @@ -180,13 +180,13 @@ feature 'Instructing a dossier:', js: true do expect(DownloadHelpers.download).to include "dossier-#{dossier.id}.zip" expect(files.size).to be 4 - expect(files[0].filename.include?('piece_justificative_0')).to be_truthy + expect(files[0].filename.include?('export')).to be_truthy expect(files[1].filename.include?('piece_justificative_0')).to be_truthy - expect(files[0].filename).not_to eq files[1].filename - expect(files[0].uncompressed_size).to be File.size(path) + expect(files[2].filename.include?('piece_justificative_0')).to be_truthy + expect(files[1].filename).not_to eq files[2].filename expect(files[1].uncompressed_size).to be File.size(path) - expect(files[2].filename.include?('horodatage/operation')).to be_truthy - expect(files[3].filename.include?('dossier/export')).to be_truthy + expect(files[2].uncompressed_size).to be File.size(path) + expect(files[3].filename.include?('horodatage/operation')).to be_truthy end before { DownloadHelpers.clear_downloads } diff --git a/spec/models/archive_spec.rb b/spec/models/archive_spec.rb new file mode 100644 index 000000000..5753340d3 --- /dev/null +++ b/spec/models/archive_spec.rb @@ -0,0 +1,51 @@ +describe Dossier do + include ActiveJob::TestHelper + + before { Timecop.freeze(Time.zone.now) } + after { Timecop.return } + + let(:archive) { create(:archive) } + + describe 'scopes' do + describe 'staled' do + let(:recent_archive) { create(:archive) } + let(:staled_archive) { create(:archive, updated_at: (Archive::RETENTION_DURATION + 2).days.ago) } + + subject do + archive; recent_archive; staled_archive + Archive.stale + end + + it { is_expected.to match_array([staled_archive]) } + end + end + + describe '.status' do + it { expect(archive.status).to eq('pending') } + end + + describe '#make_available!' do + before { archive.make_available! } + it { expect(archive.status).to eq('generated') } + end + + describe '#available?' do + subject { archive.available? } + context 'without attachment' do + let(:archive) { create(:archive, file: nil) } + it { is_expected.to eq(false) } + end + + context 'with an attachment' do + context 'when the attachment was created but the process was not over' do + let(:archive) { create(:archive, :pending, file: Rack::Test::UploadedFile.new('spec/fixtures/files/file.pdf', 'application/pdf')) } + it { is_expected.to eq(false) } + end + + context 'when the attachment was created but the process was not over' do + let(:archive) { create(:archive, :generated, file: Rack::Test::UploadedFile.new('spec/fixtures/files/file.pdf', 'application/pdf')) } + it { is_expected.to eq(true) } + end + end + end +end diff --git a/spec/services/pieces_justificatives_service_spec.rb b/spec/services/pieces_justificatives_service_spec.rb index d0ad55499..fc151f908 100644 --- a/spec/services/pieces_justificatives_service_spec.rb +++ b/spec/services/pieces_justificatives_service_spec.rb @@ -16,7 +16,12 @@ describe PiecesJustificativesService do # to be exported it 'ensures no titre identite is given' do expect(champ_identite.piece_justificative_file).to be_attached - expect(subject).to eq([]) + expect(subject.any? { |piece| piece.name == 'piece_justificative_file' }).to be_falsy + end + + it 'returns export pdf of the dossier' do + expect(champ_identite.piece_justificative_file).to be_attached + expect(subject.any? { |piece| piece.name == 'pdf_export_for_instructeur' }).to be_truthy end end end diff --git a/spec/services/procedure_archive_service_spec.rb b/spec/services/procedure_archive_service_spec.rb new file mode 100644 index 000000000..80e013f74 --- /dev/null +++ b/spec/services/procedure_archive_service_spec.rb @@ -0,0 +1,83 @@ +describe ProcedureArchiveService do + let(:procedure) { create(:procedure, :published) } + let(:instructeur) { create(:instructeur) } + let(:service) { ProcedureArchiveService.new(procedure) } + let(:year) { 2020 } + let(:month) { 3 } + let(:date_month) { Date.strptime("#{year}-#{month}", "%Y-%m") } + describe '#create_pending_archive' do + context 'for a specific month' do + it 'creates a pending archive' do + archive = service.create_pending_archive(instructeur, 'monthly', date_month) + + expect(archive.time_span_type).to eq 'monthly' + expect(archive.month).to eq date_month + expect(archive.pending?).to be_truthy + end + end + + context 'for all months' do + it 'creates a pending archive' do + archive = service.create_pending_archive(instructeur, 'everything') + + expect(archive.time_span_type).to eq 'everything' + expect(archive.month).to eq nil + expect(archive.pending?).to be_truthy + end + end + end + + describe '#collect_files_archive' do + before do + create_dossier_for_month(year, month) + create_dossier_for_month(2020, month) + end + + after { Timecop.return } + + context 'for a specific month' do + let(:archive) { create(:archive, time_span_type: 'monthly', status: 'pending', month: date_month) } + let(:year) { 2021 } + let(:mailer) { double('mailer', deliver_later: true) } + + it 'collect files' do + expect(InstructeurMailer).to receive(:send_archive).and_return(mailer) + + service.collect_files_archive(archive, instructeur) + + archive.file.open do |f| + files = ZipTricks::FileReader.read_zip_structure(io: f) + expect(files.size).to be 2 + expect(files.first.filename).to include("export") + expect(files.last.filename).to include("attestation") + end + expect(archive.file.attached?).to be_truthy + end + end + + context 'for all months' do + let(:archive) { create(:archive, time_span_type: 'everything', status: 'pending') } + let(:mailer) { double('mailer', deliver_later: true) } + + it 'collect files' do + expect(InstructeurMailer).to receive(:send_archive).and_return(mailer) + + service.collect_files_archive(archive, instructeur) + + archive = Archive.last + archive.file.open do |f| + files = ZipTricks::FileReader.read_zip_structure(io: f) + expect(files.size).to be 4 + end + expect(archive.file.attached?).to be_truthy + end + end + end + + private + + def create_dossier_for_month(year, month) + Timecop.freeze(Time.zone.local(year, month, 5)) + create(:dossier, :accepte, :with_attestation, procedure: procedure) + end +end diff --git a/spec/views/instructeur/procedures/_download_dossiers.html.haml_spec.rb b/spec/views/instructeur/procedures/_download_dossiers.html.haml_spec.rb index 947ce178f..8fb8f346e 100644 --- a/spec/views/instructeur/procedures/_download_dossiers.html.haml_spec.rb +++ b/spec/views/instructeur/procedures/_download_dossiers.html.haml_spec.rb @@ -12,5 +12,15 @@ describe 'instructeurs/procedures/_download_dossiers.html.haml', type: :view do context "when procedure has at least 1 dossier" do let(:dossier_count) { 1 } it { is_expected.to include("Télécharger tous les dossiers") } + + context "With zip archive enabled" do + before { Flipper.enable(:archive_zip_globale, procedure) } + it { is_expected.to include("Télécharger une archive au format .zip") } + end + + context "With zip archive disabled" do + before { Flipper.disable(:archive_zip_globale, procedure) } + it { is_expected.not_to include("Télécharger une archive au format .zip") } + end end end From 3d04221dabdab8e671dbd80e52c938aef731cf0a Mon Sep 17 00:00:00 2001 From: Christophe Robillard Date: Thu, 29 Apr 2021 16:16:12 +0200 Subject: [PATCH 2/2] generate pdf export for dossier only on demand --- app/lib/active_storage/downloadable_file.rb | 3 ++- app/services/pieces_justificatives_service.rb | 7 +++---- spec/services/pieces_justificatives_service_spec.rb | 5 ++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/lib/active_storage/downloadable_file.rb b/app/lib/active_storage/downloadable_file.rb index d8ca5cf4c..25813202b 100644 --- a/app/lib/active_storage/downloadable_file.rb +++ b/app/lib/active_storage/downloadable_file.rb @@ -1,6 +1,7 @@ class ActiveStorage::DownloadableFile def self.create_list_from_dossier(dossier) - pjs = PiecesJustificativesService.liste_pieces_justificatives(dossier) + dossier_export = PiecesJustificativesService.generate_dossier_export(dossier) + pjs = [dossier_export] + PiecesJustificativesService.liste_pieces_justificatives(dossier) pjs.map do |piece_justificative| [ piece_justificative, diff --git a/app/services/pieces_justificatives_service.rb b/app/services/pieces_justificatives_service.rb index f3cd8303c..3c20ab32d 100644 --- a/app/services/pieces_justificatives_service.rb +++ b/app/services/pieces_justificatives_service.rb @@ -1,11 +1,10 @@ class PiecesJustificativesService def self.liste_pieces_justificatives(dossier) - dossier_export = generate_dossier_export(dossier) pjs_champs = pjs_for_champs(dossier) pjs_commentaires = pjs_for_commentaires(dossier) pjs_dossier = pjs_for_dossier(dossier) - ([dossier_export] + pjs_champs + pjs_commentaires + pjs_dossier) + (pjs_champs + pjs_commentaires + pjs_dossier) .filter(&:attached?) end @@ -42,8 +41,6 @@ class PiecesJustificativesService end end - private - def self.generate_dossier_export(dossier) pdf = ApplicationController .render(template: 'dossiers/show', formats: [:pdf], @@ -55,6 +52,8 @@ class PiecesJustificativesService dossier.pdf_export_for_instructeur end + private + def self.pjs_for_champs(dossier) allowed_champs = dossier.champs + dossier.champs_private diff --git a/spec/services/pieces_justificatives_service_spec.rb b/spec/services/pieces_justificatives_service_spec.rb index fc151f908..13d843563 100644 --- a/spec/services/pieces_justificatives_service_spec.rb +++ b/spec/services/pieces_justificatives_service_spec.rb @@ -19,9 +19,8 @@ describe PiecesJustificativesService do expect(subject.any? { |piece| piece.name == 'piece_justificative_file' }).to be_falsy end - it 'returns export pdf of the dossier' do - expect(champ_identite.piece_justificative_file).to be_attached - expect(subject.any? { |piece| piece.name == 'pdf_export_for_instructeur' }).to be_truthy + it "doesn't return export pdf of the dossier" do + expect(subject.any? { |piece| piece.name == 'pdf_export_for_instructeur' }).to be_falsy end end end