From d7d5d7eff48d4069efd96686c2a41304ce58915a Mon Sep 17 00:00:00 2001
From: simon lehericey <simon.lehericey@gmail.com>
Date: Wed, 3 Jan 2018 17:11:47 +0100
Subject: [PATCH 1/4] Rename new_admin_mailer to administration_mailer

---
 app/controllers/administrations_controller.rb                 | 2 +-
 app/mailers/{new_admin_mailer.rb => administration_mailer.rb} | 2 +-
 .../new_admin_email.text.erb                                  | 0
 spec/controllers/administrations_controller_spec.rb           | 4 ++--
 4 files changed, 4 insertions(+), 4 deletions(-)
 rename app/mailers/{new_admin_mailer.rb => administration_mailer.rb} (81%)
 rename app/views/{new_admin_mailer => administration_mailer}/new_admin_email.text.erb (100%)

diff --git a/app/controllers/administrations_controller.rb b/app/controllers/administrations_controller.rb
index c907a3b7a..f182fc0a7 100644
--- a/app/controllers/administrations_controller.rb
+++ b/app/controllers/administrations_controller.rb
@@ -18,7 +18,7 @@ class AdministrationsController < ApplicationController
 
     if admin.save
       flash.notice = "Administrateur créé"
-      NewAdminMailer.new_admin_email(admin, current_administration).deliver_now!
+      AdministrationMailer.new_admin_email(admin, current_administration).deliver_now!
     else
       flash.alert = admin.errors.full_messages
     end
diff --git a/app/mailers/new_admin_mailer.rb b/app/mailers/administration_mailer.rb
similarity index 81%
rename from app/mailers/new_admin_mailer.rb
rename to app/mailers/administration_mailer.rb
index 96d0e76d4..2d8f934cd 100644
--- a/app/mailers/new_admin_mailer.rb
+++ b/app/mailers/administration_mailer.rb
@@ -1,4 +1,4 @@
-class NewAdminMailer < ApplicationMailer
+class AdministrationMailer < ApplicationMailer
   def new_admin_email admin, administration
     @admin = admin
     @administration = administration
diff --git a/app/views/new_admin_mailer/new_admin_email.text.erb b/app/views/administration_mailer/new_admin_email.text.erb
similarity index 100%
rename from app/views/new_admin_mailer/new_admin_email.text.erb
rename to app/views/administration_mailer/new_admin_email.text.erb
diff --git a/spec/controllers/administrations_controller_spec.rb b/spec/controllers/administrations_controller_spec.rb
index 5ea846d1f..48b2ba8dc 100644
--- a/spec/controllers/administrations_controller_spec.rb
+++ b/spec/controllers/administrations_controller_spec.rb
@@ -35,8 +35,8 @@ describe AdministrationsController, type: :controller do
       end
 
       it 'alert new mail are send' do
-        expect(NewAdminMailer).to receive(:new_admin_email).and_return(NewAdminMailer)
-        expect(NewAdminMailer).to receive(:deliver_now!)
+        expect(AdministrationMailer).to receive(:new_admin_email).and_return(AdministrationMailer)
+        expect(AdministrationMailer).to receive(:deliver_now!)
         subject
       end
     end

From f1b7a03b63d83eba71859d4595b89604e4b5b4ef Mon Sep 17 00:00:00 2001
From: simon lehericey <simon.lehericey@gmail.com>
Date: Fri, 5 Jan 2018 11:16:50 +0100
Subject: [PATCH 2/4] [fix #1097] Add job to monitor procedures

---
 README.md                                     |  1 +
 app/jobs/find_dubious_procedures_job.rb       | 29 +++++++++++++
 app/mailers/administration_mailer.rb          |  8 ++++
 .../dubious_procedures.html.haml              |  7 +++
 spec/jobs/find_dubious_procedures_job_spec.rb | 43 +++++++++++++++++++
 .../previews/administration_mailer_preview.rb |  9 ++++
 6 files changed, 97 insertions(+)
 create mode 100644 app/jobs/find_dubious_procedures_job.rb
 create mode 100644 app/views/administration_mailer/dubious_procedures.html.haml
 create mode 100644 spec/jobs/find_dubious_procedures_job_spec.rb
 create mode 100644 spec/mailers/previews/administration_mailer_preview.rb

diff --git a/README.md b/README.md
index 7975e3ae4..26774a212 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,7 @@ client_secret: ''
     AutoArchiveProcedureJob.set(cron: "* * * * *").perform_later
     WeeklyOverviewJob.set(cron: "0 8 * * 0").perform_later
     AutoReceiveDossiersForProcedureJob.set(cron: "* * * * *").perform_later(procedure_declaratoire_id, "received")
+    FindDubiousProcedureJob.set(cron: "0 0 * * *").perform_later
 
 ## Exécution des tests (RSpec)
 
diff --git a/app/jobs/find_dubious_procedures_job.rb b/app/jobs/find_dubious_procedures_job.rb
new file mode 100644
index 000000000..100ee02f4
--- /dev/null
+++ b/app/jobs/find_dubious_procedures_job.rb
@@ -0,0 +1,29 @@
+class FindDubiousProceduresJob < ApplicationJob
+  queue_as :cron
+
+  FORBIDDEN_KEYWORDS = ['IBAN', 'NIR', 'NIRPP', 'race', 'religion',
+                        'carte bancaire', 'carte bleue', 'sécurité sociale']
+
+  def perform(*args)
+    # \\y is a word boundary
+    forbidden_regexp = FORBIDDEN_KEYWORDS
+      .map { |keyword| '\\y' + keyword + '\\y' }
+      .join('|')
+
+    # ~* -> case insensitive regexp match
+    # https://www.postgresql.org/docs/current/static/functions-matching.html#FUNCTIONS-POSIX-REGEXP
+    forbidden_tdcs = TypeDeChamp
+      .joins(:procedure)
+      .where("types_de_champ.libelle ~* '#{forbidden_regexp}'")
+      .where(type_champ: %w(text textarea))
+      .where(procedures: { archived_at: nil })
+
+    dubious_procedures_and_tdcs = forbidden_tdcs
+      .group_by(&:procedure_id)
+      .map { |_procedure_id, tdcs| [tdcs[0].procedure, tdcs] }
+
+    if dubious_procedures_and_tdcs.present?
+      AdministrationMailer.dubious_procedures(dubious_procedures_and_tdcs).deliver_now
+    end
+  end
+end
diff --git a/app/mailers/administration_mailer.rb b/app/mailers/administration_mailer.rb
index 2d8f934cd..6e5b54505 100644
--- a/app/mailers/administration_mailer.rb
+++ b/app/mailers/administration_mailer.rb
@@ -1,4 +1,6 @@
 class AdministrationMailer < ApplicationMailer
+  layout 'mailers/layout'
+
   def new_admin_email admin, administration
     @admin = admin
     @administration = administration
@@ -6,4 +8,10 @@ class AdministrationMailer < ApplicationMailer
     mail(to: 'tech@tps.apientreprise.fr',
          subject: "Création d'un compte Admin TPS")
   end
+
+  def dubious_procedures(procedures_and_type_de_champs)
+    @procedures_and_type_de_champs = procedures_and_type_de_champs
+    mail(to: 'tech@tps.apientreprise.fr',
+         subject: "[RGS] De nouvelles procédures comportent des champs interdits")
+  end
 end
diff --git a/app/views/administration_mailer/dubious_procedures.html.haml b/app/views/administration_mailer/dubious_procedures.html.haml
new file mode 100644
index 000000000..3f0d0d179
--- /dev/null
+++ b/app/views/administration_mailer/dubious_procedures.html.haml
@@ -0,0 +1,7 @@
+- content_for(:title, 'Liste de procédures douteuses')
+
+%ul
+  - @procedures_and_type_de_champs.each do |procedure, type_de_champs|
+    %li
+      Nº #{procedure.id}, #{procedure.libelle} :
+      %b= type_de_champs.map(&:libelle).join(', ')
diff --git a/spec/jobs/find_dubious_procedures_job_spec.rb b/spec/jobs/find_dubious_procedures_job_spec.rb
new file mode 100644
index 000000000..40222c2d4
--- /dev/null
+++ b/spec/jobs/find_dubious_procedures_job_spec.rb
@@ -0,0 +1,43 @@
+require 'rails_helper'
+
+RSpec.describe FindDubiousProceduresJob, type: :job do
+  describe 'perform' do
+    let(:mailer_double) { double('mailer', deliver_now: true) }
+    let(:procedure) { create(:procedure) }
+    let(:allowed_tdc) { create(:type_de_champ_public, libelle: 'fournir') }
+
+    before do
+      allow(AdministrationMailer).to receive(:dubious_procedures)
+        .and_return(mailer_double)
+
+      procedure.types_de_champ << tdcs
+      FindDubiousProceduresJob.new.perform
+    end
+
+    context 'with suspicious champs' do
+      let(:forbidden_tdcs) do
+        [create(:type_de_champ_public, libelle: 'donne ton iban, stp'),
+         create(:type_de_champ_public, libelle: "t'aurais une carte bancaire ?")]
+      end
+
+      let(:tdcs) { forbidden_tdcs + [allowed_tdc] }
+
+      it 'mails tech about the dubious procedure' do
+        expect(AdministrationMailer).to have_received(:dubious_procedures)
+          .with([[procedure, forbidden_tdcs]])
+      end
+
+      context 'and a archived procedure' do
+        let(:procedure) { create(:procedure, archived_at: DateTime.now) }
+
+        it { expect(AdministrationMailer).not_to have_received(:dubious_procedures) }
+      end
+    end
+
+    context 'with no suspicious champs' do
+      let(:tdcs) { [allowed_tdc] }
+
+      it { expect(AdministrationMailer).not_to receive(:dubious_procedures) }
+    end
+  end
+end
diff --git a/spec/mailers/previews/administration_mailer_preview.rb b/spec/mailers/previews/administration_mailer_preview.rb
new file mode 100644
index 000000000..d0a6e2554
--- /dev/null
+++ b/spec/mailers/previews/administration_mailer_preview.rb
@@ -0,0 +1,9 @@
+class AdministrationMailerPreview < ActionMailer::Preview
+  def dubious_procedures
+    procedures_and_champs = [
+      [Procedure.first, [TypeDeChamp.new(libelle: 'iban'), TypeDeChamp.new(libelle: 'religion')]],
+      [Procedure.last, [TypeDeChamp.new(libelle: 'iban'), TypeDeChamp.new(libelle: 'numéro de carte bleu')]]
+    ]
+    AdministrationMailer.dubious_procedures(procedures_and_champs)
+  end
+end

From 6a2b575027acb4f7bfa3a7c983db1587f67f7683 Mon Sep 17 00:00:00 2001
From: simon lehericey <simon.lehericey@gmail.com>
Date: Fri, 5 Jan 2018 17:08:43 +0100
Subject: [PATCH 3/4] [fix #1097] Add whitelist mechanisme to procedure

---
 app/jobs/find_dubious_procedures_job.rb                     | 2 +-
 ...20180105152235_add_whitelisted_at_column_to_procedure.rb | 5 +++++
 db/schema.rb                                                | 3 ++-
 spec/jobs/find_dubious_procedures_job_spec.rb               | 6 ++++++
 4 files changed, 14 insertions(+), 2 deletions(-)
 create mode 100644 db/migrate/20180105152235_add_whitelisted_at_column_to_procedure.rb

diff --git a/app/jobs/find_dubious_procedures_job.rb b/app/jobs/find_dubious_procedures_job.rb
index 100ee02f4..9f35006e8 100644
--- a/app/jobs/find_dubious_procedures_job.rb
+++ b/app/jobs/find_dubious_procedures_job.rb
@@ -16,7 +16,7 @@ class FindDubiousProceduresJob < ApplicationJob
       .joins(:procedure)
       .where("types_de_champ.libelle ~* '#{forbidden_regexp}'")
       .where(type_champ: %w(text textarea))
-      .where(procedures: { archived_at: nil })
+      .where(procedures: { archived_at: nil, whitelisted_at: nil })
 
     dubious_procedures_and_tdcs = forbidden_tdcs
       .group_by(&:procedure_id)
diff --git a/db/migrate/20180105152235_add_whitelisted_at_column_to_procedure.rb b/db/migrate/20180105152235_add_whitelisted_at_column_to_procedure.rb
new file mode 100644
index 000000000..e6d533cd3
--- /dev/null
+++ b/db/migrate/20180105152235_add_whitelisted_at_column_to_procedure.rb
@@ -0,0 +1,5 @@
+class AddWhitelistedAtColumnToProcedure < ActiveRecord::Migration[5.0]
+  def change
+    add_column :procedures, :whitelisted_at, :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 7829c49ef..7df78c179 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171214155554) do
+ActiveRecord::Schema.define(version: 20180105152235) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -424,6 +424,7 @@ ActiveRecord::Schema.define(version: 20171214155554) do
     t.datetime "published_at"
     t.datetime "hidden_at"
     t.datetime "archived_at"
+    t.datetime "whitelisted_at"
     t.index ["hidden_at"], name: "index_procedures_on_hidden_at", using: :btree
   end
 
diff --git a/spec/jobs/find_dubious_procedures_job_spec.rb b/spec/jobs/find_dubious_procedures_job_spec.rb
index 40222c2d4..a65813a1f 100644
--- a/spec/jobs/find_dubious_procedures_job_spec.rb
+++ b/spec/jobs/find_dubious_procedures_job_spec.rb
@@ -27,6 +27,12 @@ RSpec.describe FindDubiousProceduresJob, type: :job do
           .with([[procedure, forbidden_tdcs]])
       end
 
+      context 'and a whitelisted procedure' do
+        let(:procedure) { create(:procedure, whitelisted_at: DateTime.now) }
+
+        it { expect(AdministrationMailer).not_to have_received(:dubious_procedures) }
+      end
+
       context 'and a archived procedure' do
         let(:procedure) { create(:procedure, archived_at: DateTime.now) }
 

From 2bc61f9f3f75dc167597a3930194b209a26fe696 Mon Sep 17 00:00:00 2001
From: simon lehericey <simon.lehericey@gmail.com>
Date: Mon, 8 Jan 2018 14:59:23 +0100
Subject: [PATCH 4/4] [fix #1097] Accent insensitive search

---
 app/jobs/find_dubious_procedures_job.rb                  | 2 +-
 ...0180108132507_enable_unaccent_postgresql_extension.rb | 9 +++++++++
 db/schema.rb                                             | 3 ++-
 spec/jobs/find_dubious_procedures_job_spec.rb            | 2 +-
 4 files changed, 13 insertions(+), 3 deletions(-)
 create mode 100644 db/migrate/20180108132507_enable_unaccent_postgresql_extension.rb

diff --git a/app/jobs/find_dubious_procedures_job.rb b/app/jobs/find_dubious_procedures_job.rb
index 9f35006e8..5c6122db8 100644
--- a/app/jobs/find_dubious_procedures_job.rb
+++ b/app/jobs/find_dubious_procedures_job.rb
@@ -14,7 +14,7 @@ class FindDubiousProceduresJob < ApplicationJob
     # https://www.postgresql.org/docs/current/static/functions-matching.html#FUNCTIONS-POSIX-REGEXP
     forbidden_tdcs = TypeDeChamp
       .joins(:procedure)
-      .where("types_de_champ.libelle ~* '#{forbidden_regexp}'")
+      .where("unaccent(types_de_champ.libelle) ~* unaccent('#{forbidden_regexp}')")
       .where(type_champ: %w(text textarea))
       .where(procedures: { archived_at: nil, whitelisted_at: nil })
 
diff --git a/db/migrate/20180108132507_enable_unaccent_postgresql_extension.rb b/db/migrate/20180108132507_enable_unaccent_postgresql_extension.rb
new file mode 100644
index 000000000..e68c29332
--- /dev/null
+++ b/db/migrate/20180108132507_enable_unaccent_postgresql_extension.rb
@@ -0,0 +1,9 @@
+class EnableUnaccentPostgresqlExtension < ActiveRecord::Migration[5.0]
+  def up
+    execute 'CREATE EXTENSION unaccent;'
+  end
+
+  def down
+    execute 'DROP EXTENSION unaccent;'
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 7df78c179..fdbe707be 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,10 +10,11 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180105152235) do
+ActiveRecord::Schema.define(version: 20180108132507) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
+  enable_extension "unaccent"
 
   create_table "administrateurs", force: :cascade do |t|
     t.string   "email",                  default: "", null: false
diff --git a/spec/jobs/find_dubious_procedures_job_spec.rb b/spec/jobs/find_dubious_procedures_job_spec.rb
index a65813a1f..01776aee7 100644
--- a/spec/jobs/find_dubious_procedures_job_spec.rb
+++ b/spec/jobs/find_dubious_procedures_job_spec.rb
@@ -16,7 +16,7 @@ RSpec.describe FindDubiousProceduresJob, type: :job do
 
     context 'with suspicious champs' do
       let(:forbidden_tdcs) do
-        [create(:type_de_champ_public, libelle: 'donne ton iban, stp'),
+        [create(:type_de_champ_public, libelle: 'num de securite sociale, stp'),
          create(:type_de_champ_public, libelle: "t'aurais une carte bancaire ?")]
       end