Merge pull request #3923 from tchak/export-dossier

Export dossiers v2

Gemfile

@@ -103,6 +103,7 @@ group :development, :test do
   gem 'rspec-rails'
   gem 'rspec_junit_formatter', require: false
   gem 'ruby-debug-ide', require: false
+  gem 'simple_xlsx_reader'
   gem 'spring' # Spring speeds up development by keeping your application running in the background
   gem 'spring-commands-rspec'
 end

Gemfile.lock

@@ -576,6 +576,9 @@ GEM
     simple_form (4.1.0)
       actionpack (>= 5.0)
       activemodel (>= 5.0)
+    simple_xlsx_reader (1.0.4)
+      nokogiri
+      rubyzip
     sinatra (2.0.5)
       mustermann (~> 1.0)
       rack (~> 2.0)
@@ -747,6 +750,7 @@ DEPENDENCIES
   sentry-raven
   shoulda-matchers
   simple_form
+  simple_xlsx_reader
   skylight
   smart_listing
   spreadsheet_architect

app/controllers/gestionnaires/procedures_controller.rb

@@ -168,7 +168,7 @@ module Gestionnaires
     end
 
     def download_dossiers
-      options = params.permit(:limit, :since, tables: [])
+      options = params.permit(:version, :limit, :since, tables: [])
 
       respond_to do |format|
         format.csv do

app/helpers/procedure_helper.rb

@@ -54,6 +54,13 @@ module ProcedureHelper
     }
   end
 
+  def procedure_dossiers_download_path(procedure, format:, version:)
+    download_dossiers_gestionnaire_procedure_path(format: format,
+      procedure_id: procedure.id,
+      tables: [:etablissements],
+      version: version)
+  end
+
   private
 
   TOGGLES = {

app/models/avis.rb

@@ -37,6 +37,16 @@ class Avis < ApplicationRecord
     Avis.find_by(id: avis_id)&.email == email
   end
 
+  def spreadsheet_columns
+    [
+      ['Dossier ID', dossier_id.to_s],
+      ['Question / Introduction', :introduction],
+      ['Réponse', :answer],
+      ['Créé le', :created_at],
+      ['Répondu le', :updated_at]
+    ]
+  end
+
   private
 
   def notify_gestionnaire

app/models/champs/repetition_champ.rb

@@ -20,4 +20,25 @@ class Champs::RepetitionChamp < Champ
   def search_terms
     # The user cannot enter any information here so it doesn’t make much sense to search
   end
+
+  class Row < Hashie::Dash
+    property :index
+    property :dossier_id
+    property :champs
+
+    def spreadsheet_columns
+      [
+        ['Dossier ID', :dossier_id],
+        ['Ligne', :index]
+      ] + exported_champs
+    end
+
+    private
+
+    def exported_champs
+      champs.reject(&:exclude_from_export?).map do |champ|
+        [champ.libelle, champ.for_export]
+      end
+    end
+  end
 end

app/models/dossier.rb

@@ -59,7 +59,7 @@ class Dossier < ApplicationRecord
   scope :en_construction,             -> { not_archived.state_en_construction }
   scope :en_instruction,              -> { not_archived.state_en_instruction }
   scope :termine,                     -> { not_archived.state_termine }
-  scope :downloadable_sorted,         -> { state_not_brouillon.includes(:etablissement, :user, :individual, :followers_gestionnaires, champs: { etablissement: [], type_de_champ: :drop_down_list }, champs_private: { etablissement: [], type_de_champ: :drop_down_list }).order(en_construction_at: 'asc') }
+  scope :downloadable_sorted,         -> { state_not_brouillon.includes(:etablissement, :user, :individual, :followers_gestionnaires, :avis, champs: { etablissement: [:champ], type_de_champ: :drop_down_list }, champs_private: { etablissement: [:champ], type_de_champ: :drop_down_list }).order(en_construction_at: 'asc') }
   scope :en_cours,                    -> { not_archived.state_en_construction_ou_instruction }
   scope :without_followers,           -> { left_outer_joins(:follows).where(follows: { id: nil }) }
   scope :followed_by,                 -> (gestionnaire) { joins(:follows).where(follows: { gestionnaire: gestionnaire }) }
@@ -380,6 +380,37 @@ class Dossier < ApplicationRecord
     log_dossier_operation(avis.claimant, :demander_un_avis, avis)
   end
 
+  def spreadsheet_columns
+    [
+      ['ID', id.to_s],
+      ['Email', user.email],
+      ['Civilité', individual&.gender],
+      ['Nom', individual&.nom],
+      ['Prénom', individual&.prenom],
+      ['Date de naissance', individual&.birthdate],
+      ['Archivé', :archived],
+      ['État du dossier', I18n.t(state, scope: [:activerecord, :attributes, :dossier, :state])],
+      ['Dernière mise à jour le', :updated_at],
+      ['Passé en construction le', :en_instruction_at],
+      ['Passé en instruction le', :en_construction_at],
+      ['Traité le', :processed_at],
+      ['Motivation de la décision', :motivation],
+      ['Instructeurs', followers_gestionnaires.map(&:email).join(' ')]
+    ] + champs_for_export + annotations_for_export
+  end
+
+  def champs_for_export
+    champs.reject(&:exclude_from_export?).map do |champ|
+      [champ.libelle, champ.for_export]
+    end
+  end
+
+  def annotations_for_export
+    champs_private.reject(&:exclude_from_export?).map do |champ|
+      [champ.libelle, champ.for_export]
+    end
+  end
+
   private
 
   def log_dossier_operation(author, operation, subject = nil)

app/models/etablissement.rb

@@ -33,6 +33,43 @@ class Etablissement < ApplicationRecord
     ]
   end
 
+  def spreadsheet_columns
+    [
+      ['Dossier ID', :dossier_id_for_export],
+      ['Champ', :libelle_for_export],
+      ['Établissement SIRET', :siret],
+      ['Établissement siège social', :siege_social],
+      ['Établissement NAF', :naf],
+      ['Établissement libellé NAF', :libelle_naf],
+      ['Établissement Adresse', :adresse],
+      ['Établissement numero voie', :numero_voie],
+      ['Établissement type voie', :type_voie],
+      ['Établissement nom voie', :nom_voie],
+      ['Établissement complément adresse', :complement_adresse],
+      ['Établissement code postal', :code_postal],
+      ['Établissement localité', :localite],
+      ['Établissement code INSEE localité', :code_insee_localite],
+      ['Entreprise SIREN', :entreprise_siren],
+      ['Entreprise capital social', :entreprise_capital_social],
+      ['Entreprise numero TVA intracommunautaire', :entreprise_numero_tva_intracommunautaire],
+      ['Entreprise forme juridique', :entreprise_forme_juridique],
+      ['Entreprise forme juridique code', :entreprise_forme_juridique_code],
+      ['Entreprise nom commercial', :entreprise_nom_commercial],
+      ['Entreprise raison sociale', :entreprise_raison_sociale],
+      ['Entreprise SIRET siège social', :entreprise_siret_siege_social],
+      ['Entreprise code effectif entreprise', :entreprise_code_effectif_entreprise],
+      ['Entreprise date de création', :entreprise_date_creation],
+      ['Entreprise nom', :entreprise_nom],
+      ['Entreprise prénom', :entreprise_prenom],
+      ['Association RNA', :association_rna],
+      ['Association titre', :association_titre],
+      ['Association objet', :association_objet],
+      ['Association date de création', :association_date_creation],
+      ['Association date de déclaration', :association_date_declaration],
+      ['Association date de publication', :association_date_publication]
+    ]
+  end
+
   def siren
     entreprise_siren
   end
@@ -71,4 +108,18 @@ class Etablissement < ApplicationRecord
       inline_adresse: inline_adresse
     )
   end
+
+  private
+
+  def dossier_id_for_export
+    if dossier_id
+      dossier_id.to_s
+    elsif champ
+      champ.dossier_id.to_s
+    end
+  end
+
+  def libelle_for_export
+    champ&.libelle
+  end
 end

app/models/procedure.rb

@@ -296,7 +296,13 @@ class Procedure < ApplicationRecord
   end
 
   def export(options = {})
-    ProcedureExportService.new(self, **options.to_h.symbolize_keys)
+    version = options.delete(:version)
+    if version == 'v2'
+      options.delete(:tables)
+      ProcedureExportV2Service.new(self, **options.to_h.symbolize_keys)
+    else
+      ProcedureExportService.new(self, **options.to_h.symbolize_keys)
+    end
   end
 
   def to_csv(options = {})

app/services/procedure_export_v2_service.rb

@@ -0,0 +1,88 @@
+class ProcedureExportV2Service
+  attr_reader :dossiers
+
+  def initialize(procedure, ids: nil, since: nil, limit: nil)
+    @procedure = procedure
+    @dossiers = procedure.dossiers.downloadable_sorted
+    if ids
+      @dossiers = @dossiers.where(id: ids)
+    end
+    if since
+      @dossiers = @dossiers.since(since)
+    end
+    if limit
+      @dossiers = @dossiers.limit(limit)
+    end
+    @tables = [:dossiers, :etablissements, :avis] + champs_repetables_options
+  end
+
+  def to_csv(table = :dossiers)
+    SpreadsheetArchitect.to_csv(options_for(table))
+  end
+
+  def to_xlsx
+    # We recursively build multi page spreadsheet
+    @tables.reduce(nil) do |package, table|
+      SpreadsheetArchitect.to_axlsx_package(options_for(table), package)
+    end.to_stream.read
+  end
+
+  def to_ods
+    # We recursively build multi page spreadsheet
+    @tables.reduce(nil) do |spreadsheet, table|
+      SpreadsheetArchitect.to_rodf_spreadsheet(options_for(table), spreadsheet)
+    end.bytes
+  end
+
+  private
+
+  def etablissements
+    @etablissements ||= dossiers.flat_map do |dossier|
+      [dossier.champs, dossier.champs_private]
+        .flatten
+        .select { |champ| champ.is_a?(Champs::SiretChamp) }
+    end.map(&:etablissement).compact + dossiers.map(&:etablissement).compact
+  end
+
+  def avis
+    @avis ||= dossiers.flat_map(&:avis)
+  end
+
+  def champs_repetables
+    @champs_repetables ||= dossiers.flat_map do |dossier|
+      [dossier.champs, dossier.champs_private]
+        .flatten
+        .select { |champ| champ.is_a?(Champs::RepetitionChamp) }
+    end
+  end
+
+  def champs_repetables_options
+    champs_repetables.map do |champ|
+      [
+        champ.libelle,
+        champ.rows.each_with_index.map do |champs, index|
+          Champs::RepetitionChamp::Row.new(index: index + 1, dossier_id: champ.dossier_id.to_s, champs: champs)
+        end
+      ]
+    end
+  end
+
+  DEFAULT_STYLES = {
+    header_style: { background_color: "000000", color: "FFFFFF", font_size: 12, bold: true },
+    row_style: { background_color: nil, color: "000000", font_size: 12 }
+  }
+
+  def options_for(table)
+    case table
+    when :dossiers
+      { instances: dossiers.to_a, sheet_name: 'Dossiers' }.merge(DEFAULT_STYLES)
+    when :etablissements
+      { instances: etablissements.to_a, sheet_name: 'Etablissements' }.merge(DEFAULT_STYLES)
+    when :avis
+      { instances: avis.to_a, sheet_name: 'Avis' }.merge(DEFAULT_STYLES)
+    when Array
+      # We have to truncate the label here as spreadsheets have a (30 char) limit on length.
+      { instances: table.last, sheet_name: table.first.to_s.truncate(30) }.merge(DEFAULT_STYLES)
+    end
+  end
+end

app/views/gestionnaires/procedures/_download_dossiers.html.haml

@@ -2,11 +2,23 @@
   %span.dropdown
     %button.button.dropdown-button
       Télécharger tous les dossiers
-    .dropdown-content.fade-in-down
+    - old_format_limit_date = Date.parse("Oct 31 2019")
+    - export_v1_enabled = old_format_limit_date > Time.zone.today
+    - export_v2_enabled = Flipflop.procedure_export_v2_enabled? || !export_v1_enabled
+    - old_format_message = export_v1_enabled && export_v2_enabled ? "(ancien format, jusqu’au #{old_format_limit_date.strftime('%d/%m/%Y')})" : ''
+    .dropdown-content.fade-in-down{ style: export_v1_enabled && export_v2_enabled ? 'width: 330px' : '' }
       %ul.dropdown-items
-        %li
-          = link_to "Au format .csv", download_dossiers_gestionnaire_procedure_path(format: :csv, procedure_id: procedure.id), target: "_blank", rel: "noopener"
-        %li
-          = link_to "Au format .xlsx", download_dossiers_gestionnaire_procedure_path(format: :xlsx, procedure_id: procedure.id, tables: [:etablissements]), target: "_blank", rel: "noopener"
-        %li
-          = link_to "Au format .ods", download_dossiers_gestionnaire_procedure_path(format: :ods, procedure_id: procedure.id, tables: [:etablissements]), target: "_blank", rel: "noopener"
+        - if export_v2_enabled
+          %li
+            = link_to "Au format .xlsx", procedure_dossiers_download_path(procedure, format: :xlsx, version: 'v2'), target: "_blank", rel: "noopener"
+          %li
+            = link_to "Au format .ods", procedure_dossiers_download_path(procedure, format: :ods, version: 'v2'), target: "_blank", rel: "noopener"
+          %li
+            = link_to "Au format .csv", procedure_dossiers_download_path(procedure, format: :csv, version: 'v2'), target: "_blank", rel: "noopener"
+        - if export_v1_enabled
+          %li
+            = link_to "Au format .xlsx #{old_format_message}", procedure_dossiers_download_path(procedure, format: :xlsx, version: 'v1'), target: "_blank", rel: "noopener"
+          %li
+            = link_to "Au format .ods #{old_format_message}", procedure_dossiers_download_path(procedure, format: :ods, version: 'v1'), target: "_blank", rel: "noopener"
+          %li
+            = link_to "Au format .csv #{old_format_message}", procedure_dossiers_download_path(procedure, format: :csv, version: 'v1'), target: "_blank", rel: "noopener"

config/features.rb

@@ -16,6 +16,7 @@ Flipflop.configure do
   feature :web_hook
   feature :enable_email_login_token
 
+  feature :procedure_export_v2_enabled
   feature :operation_log_serialize_subject
 
   group :development do

lib/tasks/deployment/20190701131030_purge_unattached_piece_justificative.rake

@@ -16,6 +16,6 @@ namespace :after_party do
 
     # Update task as completed.  If you remove the line below, the task will
     # run with every deploy (or every time you call after_party:run).
-    AfterParty::TaskRecord.create version: '20190521131030'
+    AfterParty::TaskRecord.create version: '20190701131030'
   end
 end

spec/services/procedure_export_v2_service_spec.rb

@@ -0,0 +1,181 @@
+require 'spec_helper'
+
+describe ProcedureExportV2Service do
+  describe 'to_data' do
+    let(:procedure) { create(:procedure, :published, :with_all_champs) }
+    subject do
+      Tempfile.create do |f|
+        f << ProcedureExportV2Service.new(procedure).to_xlsx
+        f.rewind
+        SimpleXlsxReader.open(f.path)
+      end
+    end
+
+    let(:dossiers_sheet) { subject.sheets.first }
+    let(:etablissements_sheet) { subject.sheets.second }
+    let(:avis_sheet) { subject.sheets.third }
+    let(:repetition_sheet) { subject.sheets.fourth }
+
+    before do
+      # change one tdc place to check if the header is ordered
+      tdc_first = procedure.types_de_champ.first
+      tdc_last = procedure.types_de_champ.last
+
+      tdc_first.update(order_place: tdc_last.order_place + 1)
+      procedure.reload
+    end
+
+    context 'dossiers' do
+      it 'should have sheets' do
+        expect(subject.sheets.map(&:name)).to eq(['Dossiers', 'Etablissements', 'Avis'])
+      end
+    end
+
+    context 'with dossier' do
+      let!(:dossier) { create(:dossier, :en_instruction, :with_all_champs, :for_individual, procedure: procedure) }
+
+      it 'should have headers' do
+        expect(dossiers_sheet.headers).to eq([
+          "ID",
+          "Email",
+          "Civilité",
+          "Nom",
+          "Prénom",
+          "Date de naissance",
+          "Archivé",
+          "État du dossier",
+          "Dernière mise à jour le",
+          "Passé en construction le",
+          "Passé en instruction le",
+          "Traité le",
+          "Motivation de la décision",
+          "Instructeurs",
+          "textarea",
+          "date",
+          "datetime",
+          "number",
+          "decimal_number",
+          "integer_number",
+          "checkbox",
+          "civilite",
+          "email",
+          "phone",
+          "address",
+          "yes_no",
+          "simple_drop_down_list",
+          "multiple_drop_down_list",
+          "linked_drop_down_list",
+          "pays",
+          "regions",
+          "departements",
+          "engagement",
+          "dossier_link",
+          "piece_justificative",
+          "siret",
+          "carte",
+          "text"
+        ])
+      end
+
+      it 'should have data' do
+        expect(dossiers_sheet.data.size).to eq(1)
+        expect(etablissements_sheet.data.size).to eq(1)
+      end
+    end
+
+    context 'with etablissement' do
+      let!(:dossier) { create(:dossier, :en_instruction, :with_all_champs, :with_entreprise, procedure: procedure) }
+
+      it 'should have headers' do
+        expect(etablissements_sheet.headers).to eq([
+          "Dossier ID",
+          "Champ",
+          "Établissement SIRET",
+          "Établissement siège social",
+          "Établissement NAF",
+          "Établissement libellé NAF",
+          "Établissement Adresse",
+          "Établissement numero voie",
+          "Établissement type voie",
+          "Établissement nom voie",
+          "Établissement complément adresse",
+          "Établissement code postal",
+          "Établissement localité",
+          "Établissement code INSEE localité",
+          "Entreprise SIREN",
+          "Entreprise capital social",
+          "Entreprise numero TVA intracommunautaire",
+          "Entreprise forme juridique",
+          "Entreprise forme juridique code",
+          "Entreprise nom commercial",
+          "Entreprise raison sociale",
+          "Entreprise SIRET siège social",
+          "Entreprise code effectif entreprise",
+          "Entreprise date de création",
+          "Entreprise nom",
+          "Entreprise prénom",
+          "Association RNA",
+          "Association titre",
+          "Association objet",
+          "Association date de création",
+          "Association date de déclaration",
+          "Association date de publication"
+        ])
+      end
+
+      it 'should have data' do
+        expect(etablissements_sheet.data.size).to eq(2)
+      end
+    end
+
+    context 'with avis' do
+      let!(:dossier) { create(:dossier, :en_instruction, :with_all_champs, :for_individual, procedure: procedure) }
+      let!(:avis) { create(:avis, :with_answer, dossier: dossier) }
+
+      it 'should have headers' do
+        expect(avis_sheet.headers).to eq([
+          "Dossier ID",
+          "Question / Introduction",
+          "Réponse",
+          "Créé le",
+          "Répondu le"
+        ])
+      end
+
+      it 'should have data' do
+        expect(avis_sheet.data.size).to eq(1)
+      end
+    end
+
+    context 'with repetitions' do
+      let!(:dossier) { create(:dossier, :en_instruction, :with_all_champs, :for_individual, procedure: procedure) }
+      let(:champ_repetition) { dossier.champs.find { |champ| champ.type_champ == 'repetition' } }
+      let(:type_de_champ_text) { create(:type_de_champ_text, order_place: 0, parent: champ_repetition.type_de_champ) }
+      let(:type_de_champ_number) { create(:type_de_champ_number, order_place: 1, parent: champ_repetition.type_de_champ) }
+
+      before do
+        create(:champ_text, row: 0, type_de_champ: type_de_champ_text, parent: champ_repetition)
+        create(:champ_number, row: 0, type_de_champ: type_de_champ_number, parent: champ_repetition)
+        create(:champ_text, row: 1, type_de_champ: type_de_champ_text, parent: champ_repetition)
+        create(:champ_number, row: 1, type_de_champ: type_de_champ_number, parent: champ_repetition)
+      end
+
+      it 'should have sheets' do
+        expect(subject.sheets.map(&:name)).to eq(['Dossiers', 'Etablissements', 'Avis', champ_repetition.libelle])
+      end
+
+      it 'should have headers' do
+        expect(repetition_sheet.headers).to eq([
+          "Dossier ID",
+          "Ligne",
+          type_de_champ_text.libelle,
+          type_de_champ_number.libelle
+        ])
+      end
+
+      it 'should have data' do
+        expect(repetition_sheet.data.size).to eq(2)
+      end
+    end
+  end
+end