Merge pull request #4371 from Keirua/feat/export-asynchrone

Export asynchrone d'une procédure

app/controllers/instructeurs/procedures_controller.rb

@@ -205,6 +205,23 @@ module Instructeurs
       end
     end
 
+    def download_export
+      export_format = params[:export_format]
+
+      if procedure.should_generate_export?(export_format)
+        procedure.queue_export(current_instructeur, export_format)
+
+        respond_to do |format|
+          format.js do
+            flash.notice = "Nous générons cet export. Lorsque celui-ci sera disponible, vous recevrez une notification par email accompagnée d'un lien de téléchargement."
+            @procedure = procedure
+          end
+        end
+      else
+        redirect_to url_for(procedure.export_file(export_format))
+      end
+    end
+
     def email_notifications
       @procedure = procedure
       @assign_to = assign_to

app/jobs/export_procedure_job.rb

@@ -0,0 +1,6 @@
+class ExportProcedureJob < ApplicationJob
+  def perform(procedure, instructeur, export_format)
+    procedure.prepare_export_download(export_format)
+    InstructeurMailer.notify_procedure_export_available(instructeur, procedure, export_format).deliver_later
+  end
+end

app/mailers/instructeur_mailer.rb

@@ -42,4 +42,12 @@ class InstructeurMailer < ApplicationMailer
 
     mail(to: instructeur.email, subject: subject)
   end
+
+  def notify_procedure_export_available(instructeur, procedure, export_format)
+    @procedure = procedure
+    @export_format = export_format
+    subject = "Votre export de la démarche nº #{procedure.id} est disponible"
+
+    mail(to: instructeur.email, subject: subject)
+  end
 end

app/models/procedure.rb

@@ -6,6 +6,7 @@ class Procedure < ApplicationRecord
   include ProcedureStatsConcern
 
   MAX_DUREE_CONSERVATION = 36
+  MAX_DUREE_CONSERVATION_EXPORT = 3.hours
 
   has_many :types_de_champ, -> { root.public_only.ordered }, inverse_of: :procedure, dependent: :destroy
   has_many :types_de_champ_private, -> { root.private_only.ordered }, class_name: 'TypeDeChamp', inverse_of: :procedure, dependent: :destroy
@@ -35,6 +36,10 @@ class Procedure < ApplicationRecord
   has_one_attached :notice
   has_one_attached :deliberation
 
+  has_one_attached :csv_export_file
+  has_one_attached :xlsx_export_file
+  has_one_attached :ods_export_file
+
   accepts_nested_attributes_for :types_de_champ, reject_if: proc { |attributes| attributes['libelle'].blank? }, allow_destroy: true
   accepts_nested_attributes_for :types_de_champ_private, reject_if: proc { |attributes| attributes['libelle'].blank? }, allow_destroy: true
 
@@ -128,11 +133,88 @@ class Procedure < ApplicationRecord
     end
   end
 
+  def csv_export_stale?
+    !csv_export_file.attached? || csv_export_file.created_at < MAX_DUREE_CONSERVATION_EXPORT.ago
+  end
+
+  def xlsx_export_stale?
+    !xlsx_export_file.attached? || xlsx_export_file.created_at < MAX_DUREE_CONSERVATION_EXPORT.ago
+  end
+
+  def ods_export_stale?
+    !ods_export_file.attached? || ods_export_file.created_at < MAX_DUREE_CONSERVATION_EXPORT.ago
+  end
+
+  def should_generate_export?(format)
+    case format.to_sym
+    when :csv
+      return csv_export_stale? && !csv_export_queued?
+    when :xlsx
+      return xlsx_export_stale? && !xlsx_export_queued?
+    when :ods
+      return ods_export_stale? && !ods_export_queued?
+    end
+    false
+  end
+
+  def export_file(export_format)
+    case export_format.to_sym
+    when :csv
+      csv_export_file
+    when :xlsx
+      xlsx_export_file
+    when :ods
+      ods_export_file
+    end
+  end
+
+  def queue_export(instructeur, export_format)
+    ExportProcedureJob.perform_now(self, instructeur, export_format)
+    case export_format.to_sym
+    when :csv
+      update(csv_export_queued: true)
+    when :xlsx
+      update(xlsx_export_queued: true)
+    when :ods
+      update(ods_export_queued: true)
+    end
+  end
+
+  def prepare_export_download(format)
+    service = ProcedureExportV2Service.new(self, self.dossiers)
+    filename = export_filename(format)
+
+    case format.to_sym
+    when :csv
+      csv_export_file.attach(
+        io: StringIO.new(service.to_csv),
+        filename: filename,
+        content_type: 'text/csv'
+      )
+      update(csv_export_queued: false)
+    when :xlsx
+      xlsx_export_file.attach(
+        io: StringIO.new(service.to_xlsx),
+        filename: filename,
+        content_type: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+      )
+      update(xlsx_export_queued: false)
+    when :ods
+      ods_export_file.attach(
+        io: StringIO.new(service.to_ods),
+        filename: filename,
+        content_type: 'application/vnd.oasis.opendocument.spreadsheet'
+      )
+      update(ods_export_queued: false)
+    end
+  end
+
   def reset!
     if locked?
       raise "Can not reset a locked procedure."
     else
       groupe_instructeurs.each { |gi| gi.dossiers.destroy_all }
+      purge_export_files
     end
   end
 
@@ -174,6 +256,14 @@ class Procedure < ApplicationRecord
     procedure.blank? || administrateur.owns?(procedure)
   end
 
+  def purge_export_files
+    xlsx_export_file.purge_later
+    ods_export_file.purge_later
+    csv_export_file.purge_later
+
+    update(csv_export_queued: false, xlsx_export_queued: false, ods_export_queued: false)
+  end
+
   def locked?
     publiee_ou_archivee?
   end
@@ -513,12 +603,14 @@ class Procedure < ApplicationRecord
 
   def after_archive
     update!(archived_at: Time.zone.now)
+    purge_export_files
   end
 
   def after_hide
     now = Time.zone.now
     update!(hidden_at: now)
     dossiers.update_all(hidden_at: now)
+    purge_export_files
   end
 
   def after_draft

app/views/instructeur_mailer/notify_procedure_export_available.html.haml

@@ -0,0 +1,11 @@
+%p
+  Bonjour,
+
+%p
+  Votre export des dossiers de la démarche nº #{@procedure.id} « #{@procedure.libelle} » au format #{@export_format} est prêt.
+
+%p
+  Cliquez sur le lien ci-dessous pour le télécharger :
+  = link_to('Télécharger l\'export des dossiers', download_export_instructeur_procedure_url(@procedure, :export_format => @export_format))
+
+= render partial: "layouts/mailers/signature"

app/views/instructeurs/procedures/_download_dossiers.html.haml

@@ -4,14 +4,34 @@
       Télécharger tous les dossiers
     - old_format_limit_date = Date.parse("Oct 31 2019")
     - export_v1_enabled = old_format_limit_date > Time.zone.today
+    - export_v2_enabled = Flipper[:procedure_export_v2_enabled] || !export_v1_enabled
     .dropdown-content.fade-in-down{ style: !export_v1_enabled ? 'width: 330px' : '' }
       %ul.dropdown-items
-        %li
-          = link_to "Au format .xlsx", procedure_dossiers_download_path(procedure, format: :xlsx, version: 'v2'), target: "_blank", rel: "noopener"
-        %li
-          = link_to "Au format .ods", procedure_dossiers_download_path(procedure, format: :ods, version: 'v2'), target: "_blank", rel: "noopener"
-        %li
-          = link_to "Au format .csv", procedure_dossiers_download_path(procedure, format: :csv, version: 'v2'), target: "_blank", rel: "noopener"
+        - if export_v2_enabled
+          %li
+            - if procedure.xlsx_export_stale?
+              - if procedure.xlsx_export_queued?
+                L'export au format .xlsx est en cours de préparation, vous recevrez un email lorsqu'il sera disponible.
+              - else
+                = link_to "Exporter au format .xlsx", download_export_instructeur_procedure_path(procedure, export_format: :xlsx), remote: true
+            - else
+              = link_to "Au format .xlsx", url_for(procedure.xlsx_export_file), target: "_blank", rel: "noopener"
+          %li
+            - if procedure.ods_export_stale?
+              - if procedure.ods_export_queued?
+                L'export au format .ods est en cours de préparation, vous recevrez un email lorsqu'il sera disponible.
+              - else
+                = link_to "Préparer le téléchargement de l'export au format .ods", download_export_instructeur_procedure_path(procedure, export_format: :ods), remote: true
+            - else
+              = link_to "Au format .ods", url_for(procedure.ods_export_file), target: "_blank", rel: "noopener"
+          %li
+            - if procedure.csv_export_stale?
+              - if procedure.csv_export_queued?
+                L'export au format .csv est en cours de préparation, vous recevrez un email lorsqu'il sera disponible.
+              - else
+                = link_to "Préparer le téléchargement de l'export au format .csv", download_export_instructeur_procedure_path(procedure, export_format: :csv), remote: true
+            - else
+              = link_to "Au format .csv", url_for(procedure.csv_export_file), target: "_blank", rel: "noopener"
         - if export_v1_enabled
           - old_format_message = "(ancien format, jusqu’au #{old_format_limit_date.strftime('%d/%m/%Y')})"
           %li

app/views/instructeurs/procedures/download_export.js.erb

@@ -0,0 +1,2 @@
+<%= render_to_element('.procedure-actions', partial: "download_dossiers", locals: { procedure: @procedure }) %>
+<%= render_flash %>

config/initializers/flipper.rb

@@ -34,6 +34,7 @@ features = [
   :mini_profiler,
   :operation_log_serialize_subject,
   :pre_maintenance_mode,
+  :procedure_export_v2_enabled,
   :xray
 ]
 

config/routes.rb

@@ -293,6 +293,7 @@ Rails.application.routes.draw do
         post 'add_filter'
         get 'remove_filter' => 'procedures#remove_filter', as: 'remove_filter'
         get 'download_dossiers'
+        get 'download_export'
         get 'stats'
         get 'email_notifications'
         patch 'update_email_notifications'

db/migrate/20190709140415_add_export_queued_to_procedures.rb

@@ -0,0 +1,7 @@
+class AddExportQueuedToProcedures < ActiveRecord::Migration[5.2]
+  def change
+    add_column :procedures, :csv_export_queued, :boolean
+    add_column :procedures, :xlsx_export_queued, :boolean
+    add_column :procedures, :ods_export_queued, :boolean
+  end
+end

db/schema.rb

@@ -487,6 +487,9 @@ ActiveRecord::Schema.define(version: 2019_10_14_160538) do
     t.string "declarative_with_state"
     t.text "monavis_embed"
     t.text "routing_criteria_name"
+    t.boolean "csv_export_queued"
+    t.boolean "xlsx_export_queued"
+    t.boolean "ods_export_queued"
     t.index ["declarative_with_state"], name: "index_procedures_on_declarative_with_state"
     t.index ["hidden_at"], name: "index_procedures_on_hidden_at"
     t.index ["parent_procedure_id"], name: "index_procedures_on_parent_procedure_id"

spec/factories/procedure.rb

@@ -242,5 +242,47 @@ FactoryBot.define do
         end
       end
     end
+
+    trait :with_csv_export_file do
+      after(:create) do |procedure, _evaluator|
+        procedure.csv_export_file.attach(io: StringIO.new("some csv data"), filename: "export.csv", content_type: "text/plain")
+        procedure.csv_export_file.update(created_at: 5.minutes.ago)
+      end
+    end
+
+    trait :with_stale_csv_export_file do
+      after(:create) do |procedure, _evaluator|
+        procedure.csv_export_file.attach(io: StringIO.new("some csv data"), filename: "export.csv", content_type: "text/plain")
+        procedure.csv_export_file.update(created_at: 4.hours.ago)
+      end
+    end
+
+    trait :with_ods_export_file do
+      after(:create) do |procedure, _evaluator|
+        procedure.ods_export_file.attach(io: StringIO.new("some ods data"), filename: "export.ods", content_type: "text/plain")
+        procedure.ods_export_file.update(created_at: 5.minutes.ago)
+      end
+    end
+
+    trait :with_stale_ods_export_file do
+      after(:create) do |procedure, _evaluator|
+        procedure.ods_export_file.attach(io: StringIO.new("some ods data"), filename: "export.ods", content_type: "text/plain")
+        procedure.ods_export_file.update(created_at: 4.hours.ago)
+      end
+    end
+
+    trait :with_xlsx_export_file do
+      after(:create) do |procedure, _evaluator|
+        procedure.xlsx_export_file.attach(io: StringIO.new("some xlsx data"), filename: "export.xlsx", content_type: "text/plain")
+        procedure.xlsx_export_file.update(created_at: 5.minutes.ago)
+      end
+    end
+
+    trait :with_stale_xlsx_export_file do
+      after(:create) do |procedure, _evaluator|
+        procedure.xlsx_export_file.attach(io: StringIO.new("some xlsx data"), filename: "export.xlsx", content_type: "text/plain")
+        procedure.xlsx_export_file.update(created_at: 4.hours.ago)
+      end
+    end
   end
 end

spec/mailers/instructeur_mailer_spec.rb

@@ -46,4 +46,18 @@ RSpec.describe InstructeurMailer, type: :mailer do
       end
     end
   end
+
+  describe '#notify_procedure_export_available' do
+    let(:instructeur) { create(:instructeur) }
+    let(:procedure) { create(:procedure, :published, instructeurs: [instructeur]) }
+    let(:dossier) { create(:dossier, procedure: procedure) }
+    let(:format) { 'xlsx' }
+
+    context 'when the mail is sent' do
+      subject { described_class.notify_procedure_export_available(instructeur, procedure, format) }
+      it 'contains a download link' do
+        expect(subject.body).to include download_export_instructeur_procedure_url(procedure, :export_format => format)
+      end
+    end
+  end
 end

spec/mailers/previews/instructeur_mailer_preview.rb

@@ -37,6 +37,10 @@ class InstructeurMailerPreview < ActionMailer::Preview
     InstructeurMailer.send_notifications(instructeur, data)
   end
 
+  def notify_procedure_export_available
+    InstructeurMailer.notify_procedure_export_available(instructeur, procedure, 'xlsx')
+  end
+
   private
 
   def instructeur

spec/models/procedure_spec.rb

@@ -954,4 +954,165 @@ describe Procedure do
       it { is_expected.to be false }
     end
   end
+
+  describe '.ods_export_stale?' do
+    subject { procedure.ods_export_stale? }
+
+    context 'with no ods export' do
+      let(:procedure) { create(:procedure) }
+      it { is_expected.to be true }
+    end
+
+    context 'with a recent ods export' do
+      let(:procedure) { create(:procedure, :with_ods_export_file) }
+      it { is_expected.to be false }
+    end
+
+    context 'with an old ods export' do
+      let(:procedure) { create(:procedure, :with_stale_ods_export_file) }
+      it { is_expected.to be true }
+    end
+  end
+
+  describe '.csv_export_stale?' do
+    subject { procedure.csv_export_stale? }
+
+    context 'with no csv export' do
+      let(:procedure) { create(:procedure) }
+      it { is_expected.to be true }
+    end
+
+    context 'with a recent csv export' do
+      let(:procedure) { create(:procedure, :with_csv_export_file) }
+      it { is_expected.to be false }
+    end
+
+    context 'with an old csv export' do
+      let(:procedure) { create(:procedure, :with_stale_csv_export_file) }
+      it { is_expected.to be true }
+    end
+  end
+
+  describe '.xlsx_export_stale?' do
+    subject { procedure.xlsx_export_stale? }
+
+    context 'with no xlsx export' do
+      let(:procedure) { create(:procedure) }
+      it { is_expected.to be true }
+    end
+
+    context 'with a recent xlsx export' do
+      let(:procedure) { create(:procedure, :with_xlsx_export_file) }
+      it { is_expected.to be false }
+    end
+
+    context 'with an old xlsx export' do
+      let(:procedure) { create(:procedure, :with_stale_xlsx_export_file) }
+      it { is_expected.to be true }
+    end
+  end
+
+  describe '.should_generate_export?' do
+    context 'xlsx' do
+      subject { procedure.should_generate_export?('xlsx') }
+      context 'with no export' do
+        let(:procedure) { create(:procedure) }
+        it { is_expected.to be true }
+      end
+
+      context 'with a recent export' do
+        context 'when its not queued' do
+          let(:procedure) { create(:procedure, :with_xlsx_export_file, xlsx_export_queued: false) }
+          it { is_expected.to be false }
+        end
+
+        context 'when its already queued' do
+          let(:procedure) { create(:procedure, :with_xlsx_export_file, xlsx_export_queued: true) }
+          it { expect(procedure.xlsx_export_queued).to be true }
+          it { is_expected.to be false }
+        end
+      end
+
+      context 'with an old export' do
+        context 'when its not queued' do
+          let(:procedure) { create(:procedure, :with_stale_xlsx_export_file, xlsx_export_queued: false) }
+          it { is_expected.to be true }
+        end
+
+        context 'when its already queued' do
+          let(:procedure) { create(:procedure, :with_stale_xlsx_export_file, xlsx_export_queued: true) }
+          it { expect(procedure.xlsx_export_queued).to be true }
+          it { is_expected.to be false }
+        end
+      end
+    end
+
+    context 'csv' do
+      subject { procedure.should_generate_export?('csv') }
+      context 'with no export' do
+        let(:procedure) { create(:procedure) }
+        it { is_expected.to be true }
+      end
+
+      context 'with a recent export' do
+        context 'when its not queued' do
+          let(:procedure) { create(:procedure, :with_csv_export_file, csv_export_queued: false) }
+          it { is_expected.to be false }
+        end
+
+        context 'when its already queued' do
+          let(:procedure) { create(:procedure, :with_csv_export_file, csv_export_queued: true) }
+          it { expect(procedure.csv_export_queued).to be true }
+          it { is_expected.to be false }
+        end
+      end
+
+      context 'with an old export' do
+        context 'when its not queued' do
+          let(:procedure) { create(:procedure, :with_stale_csv_export_file, csv_export_queued: false) }
+          it { is_expected.to be true }
+        end
+
+        context 'when its already queued' do
+          let(:procedure) { create(:procedure, :with_stale_csv_export_file, csv_export_queued: true) }
+          it { expect(procedure.csv_export_queued).to be true }
+          it { is_expected.to be false }
+        end
+      end
+    end
+
+    context 'ods' do
+      subject { procedure.should_generate_export?('ods') }
+      context 'with no export' do
+        let(:procedure) { create(:procedure) }
+        it { is_expected.to be true }
+      end
+
+      context 'with a recent export' do
+        context 'when its not queued' do
+          let(:procedure) { create(:procedure, :with_ods_export_file, ods_export_queued: false) }
+          it { is_expected.to be false }
+        end
+
+        context 'when its already queued' do
+          let(:procedure) { create(:procedure, :with_ods_export_file, ods_export_queued: true) }
+          it { expect(procedure.ods_export_queued).to be true }
+          it { is_expected.to be false }
+        end
+      end
+
+      context 'with an old export' do
+        context 'when its not queued' do
+          let(:procedure) { create(:procedure, :with_stale_ods_export_file, ods_export_queued: false) }
+          it { is_expected.to be true }
+        end
+
+        context 'when its already queued' do
+          let(:procedure) { create(:procedure, :with_stale_ods_export_file, ods_export_queued: true) }
+          it { expect(procedure.ods_export_queued).to be true }
+          it { is_expected.to be false }
+        end
+      end
+    end
+  end
 end