Merge pull request #7495 from betagouv/feat/7451

OPEN DATA - Administrateurs, dossiers, instructeurs crées par mois

app/jobs/concerns/datagouv_cron_schedulable_concern.rb

@@ -0,0 +1,8 @@
+module DatagouvCronSchedulableConcern
+  extend ActiveSupport::Concern
+  class_methods do
+    def schedulable?
+      ENV.fetch('OPENDATA_ENABLED', nil) == 'enabled'
+    end
+  end
+end

app/jobs/cron/datagouv/account_by_month_job.rb

@@ -0,0 +1,19 @@
+class Cron::Datagouv::AccountByMonthJob < Cron::CronJob
+  include DatagouvCronSchedulableConcern
+  self.schedule_expression = "every month at 3:00"
+  FILE_NAME = "nb_comptes_crees_par_mois"
+
+  def perform(*args)
+    GenerateOpenDataCsvService.save_csv_to_tmp(FILE_NAME, data) do |file|
+      begin
+        APIDatagouv::API.upload(file, :statistics_dataset)
+      ensure
+        FileUtils.rm(file)
+      end
+    end
+  end
+
+  def data
+    User.where(created_at: 1.month.ago.beginning_of_month..1.month.ago.end_of_month).count
+  end
+end

app/jobs/cron/datagouv/administrateur_by_month_job.rb

@@ -0,0 +1,19 @@
+class Cron::Datagouv::AdministrateurByMonthJob < Cron::CronJob
+  include DatagouvCronSchedulableConcern
+  self.schedule_expression = "every month at 3:00"
+  FILE_NAME = "nb_administrateurs_crees_par_mois"
+
+  def perform(*args)
+    GenerateOpenDataCsvService.save_csv_to_tmp(FILE_NAME, data) do |file|
+      begin
+        APIDatagouv::API.upload(file, :statistics_dataset)
+      ensure
+        FileUtils.rm(file)
+      end
+    end
+  end
+
+  def data
+    Administrateur.where(created_at: 1.month.ago.beginning_of_month..1.month.ago.end_of_month).count
+  end
+end

app/jobs/cron/datagouv/export_and_publish_demarches_publiques_job.rb

@@ -1,4 +1,5 @@
 class Cron::Datagouv::ExportAndPublishDemarchesPubliquesJob < Cron::CronJob
+  include DatagouvCronSchedulableConcern
   self.schedule_expression = "every month at 3:00"
 
   def perform(*args)
@@ -10,13 +11,10 @@ class Cron::Datagouv::ExportAndPublishDemarchesPubliquesJob < Cron::CronJob
 
     begin
       DemarchesPubliquesExportService.new(gzip_filepath).call
-      APIDatagouv::API.upload(gzip_filepath)
+      io = File.new(gzip_filepath, 'r')
+      APIDatagouv::API.upload(io, :descriptif_demarches_dataset, :descriptif_demarches_resource)
     ensure
       FileUtils.rm(gzip_filepath)
     end
   end
-
-  def self.schedulable?
-    ENV.fetch('OPENDATA_ENABLED', nil) == 'enabled'
-  end
 end

app/jobs/cron/datagouv/file_by_month_job.rb

@@ -0,0 +1,19 @@
+class Cron::Datagouv::FileByMonthJob < Cron::CronJob
+  include DatagouvCronSchedulableConcern
+  self.schedule_expression = "every month at 3:00"
+  FILE_NAME = "nb_dossiers_crees_par_mois"
+
+  def perform(*args)
+    GenerateOpenDataCsvService.save_csv_to_tmp(FILE_NAME, data) do |file|
+      begin
+        APIDatagouv::API.upload(file, :statistics_dataset)
+      ensure
+        FileUtils.rm(file)
+      end
+    end
+  end
+
+  def data
+    Dossier.where(created_at: 1.month.ago.beginning_of_month..1.month.ago.end_of_month).count
+  end
+end

app/jobs/cron/datagouv/instructeur_by_month_job.rb

@@ -0,0 +1,19 @@
+class Cron::Datagouv::InstructeurByMonthJob < Cron::CronJob
+  include DatagouvCronSchedulableConcern
+  self.schedule_expression = "every month at 3:00"
+  FILE_NAME = "nb_instructeurs_crees_par_mois"
+
+  def perform(*args)
+    GenerateOpenDataCsvService.save_csv_to_tmp(FILE_NAME, data) do |file|
+      begin
+        APIDatagouv::API.upload(file, :statistics_dataset)
+      ensure
+        FileUtils.rm(file)
+      end
+    end
+  end
+
+  def data
+    Instructeur.where(created_at: 1.month.ago.beginning_of_month..1.month.ago.end_of_month).count
+  end
+end

app/jobs/cron/datagouv/instructeur_connected_by_month_job.rb

@@ -0,0 +1,19 @@
+class Cron::Datagouv::InstructeurConnectedByMonthJob < Cron::CronJob
+  include DatagouvCronSchedulableConcern
+  self.schedule_expression = "every month at 3:00"
+  FILE_NAME = "nb_instructeurs_connectes_par_mois"
+
+  def perform(*args)
+    GenerateOpenDataCsvService.save_csv_to_tmp(FILE_NAME, data) do |file|
+      begin
+        APIDatagouv::API.upload(file, :statistics_dataset)
+      ensure
+        FileUtils.rm(file)
+      end
+    end
+  end
+
+  def data
+    Instructeur.joins(:user).where(user: { last_sign_in_at: 1.month.ago.beginning_of_month..1.month.ago.end_of_month }).count
+  end
+end

app/jobs/cron/datagouv/procedure_by_month_job.rb

@@ -0,0 +1,19 @@
+class Cron::Datagouv::ProcedureByMonthJob < Cron::CronJob
+  include DatagouvCronSchedulableConcern
+  self.schedule_expression = "every month at 3:00"
+  FILE_NAME = "nb_procedures_creees_par_mois"
+
+  def perform(*args)
+    GenerateOpenDataCsvService.save_csv_to_tmp(FILE_NAME, data) do |file|
+      begin
+        APIDatagouv::API.upload(file, :statistics_dataset)
+      ensure
+        FileUtils.rm(file)
+      end
+    end
+  end
+
+  def data
+    Procedure.where(created_at: 1.month.ago.beginning_of_month..1.month.ago.end_of_month).count
+  end
+end

app/jobs/cron/datagouv/procedure_closed_by_month_job.rb

@@ -0,0 +1,19 @@
+class Cron::Datagouv::ProcedureClosedByMonthJob < Cron::CronJob
+  include DatagouvCronSchedulableConcern
+  self.schedule_expression = "every month at 3:00"
+  FILE_NAME = "nb_procedures_closes_par_mois"
+
+  def perform(*args)
+    GenerateOpenDataCsvService.save_csv_to_tmp(FILE_NAME, data) do |file|
+      begin
+        APIDatagouv::API.upload(file, :statistics_dataset)
+      ensure
+        FileUtils.rm(file)
+      end
+    end
+  end
+
+  def data
+    Procedure.where(closed_at: 1.month.ago.beginning_of_month..1.month.ago.end_of_month).count
+  end
+end

app/jobs/cron/datagouv/procedure_deleted_by_month_job.rb

@@ -0,0 +1,19 @@
+class Cron::Datagouv::ProcedureDeletedByMonthJob < Cron::CronJob
+  include DatagouvCronSchedulableConcern
+  self.schedule_expression = "every month at 3:00"
+  FILE_NAME = "nb_procedures_supprimees_par_mois"
+
+  def perform(*args)
+    GenerateOpenDataCsvService.save_csv_to_tmp(FILE_NAME, data) do |file|
+      begin
+        APIDatagouv::API.upload(file, :statistics_dataset)
+      ensure
+        FileUtils.rm(file)
+      end
+    end
+  end
+
+  def data
+    Procedure.where(hidden_at: 1.month.ago.beginning_of_month..1.month.ago.end_of_month).count
+  end
+end

app/jobs/cron/datagouv/user_connected_with_france_connect_by_month_job.rb

@@ -0,0 +1,19 @@
+class Cron::Datagouv::UserConnectedWithFranceConnectByMonthJob < Cron::CronJob
+  include DatagouvCronSchedulableConcern
+  self.schedule_expression = "every month at 3:00"
+  FILE_NAME = "nb_utilisateurs_connectes_france_connect_par_mois"
+
+  def perform(*args)
+    GenerateOpenDataCsvService.save_csv_to_tmp(FILE_NAME, data) do |file|
+      begin
+        APIDatagouv::API.upload(file, :statistics_dataset)
+      ensure
+        FileUtils.rm(file)
+      end
+    end
+  end
+
+  def data
+    User.where(created_at: 1.month.ago.beginning_of_month..1.month.ago.end_of_month, loged_in_with_france_connect: "particulier").count
+  end
+end

app/lib/api_datagouv/api.rb

@@ -11,10 +11,9 @@ class APIDatagouv::API
   end
 
   class << self
-    def upload(path)
-      io = File.new(path, 'r')
+    def upload(io, dataset, resource = nil)
       response = Typhoeus.post(
-        datagouv_upload_url,
+        datagouv_upload_url(dataset, resource),
         body: {
           file: io
         },
@@ -25,19 +24,27 @@ class APIDatagouv::API
       if response.success?
         response.body
       else
-        raise RequestFailed.new(datagouv_upload_url, response)
+        raise RequestFailed.new(datagouv_upload_url(dataset, resource), response)
       end
     end
 
     private
 
-    def datagouv_upload_url
+    def datagouv_upload_url(dataset, resource = nil)
+      if resource.present?
         [
           datagouv_secret[:api_url],
-        "/datasets/", datagouv_secret[:descriptif_demarches_dataset],
-        "/resources/", datagouv_secret[:descriptif_demarches_resource],
+          "/datasets/", datagouv_secret[dataset],
+          "/resources/", datagouv_secret[resource],
           "/upload/"
         ].join
+      else
+        [
+          datagouv_secret[:api_url],
+          "/datasets/", datagouv_secret[dataset],
+          "/upload/"
+        ].join
+      end
     end
 
     def datagouv_secret

app/services/generate_open_data_csv_service.rb

@@ -0,0 +1,19 @@
+class GenerateOpenDataCsvService
+  def self.save_csv_to_tmp(file_name, data)
+    f = Tempfile.create(["#{file_name}_#{date_last_month}", '.csv'], 'tmp')
+    f << generate_csv(file_name, data)
+    f.rewind
+    yield f if block_given?
+    f.close
+  end
+
+  def self.date_last_month
+    Date.today.prev_month.strftime("%B")
+  end
+
+  def self.generate_csv(file_name, data)
+    headers = ["mois", file_name]
+    data = [[date_last_month, data]]
+    SpreadsheetArchitect.to_csv(headers: headers, data: data)
+  end
+end

config/application.rb

@@ -22,6 +22,7 @@ module TPS
     Rails.autoloaders.main.ignore(Rails.root.join('lib/linters'))
     Rails.autoloaders.main.ignore(Rails.root.join('lib/tasks/task_helper.rb'))
     config.paths.add Rails.root.join('spec/mailers/previews').to_s, eager_load: true
+    config.autoload_paths << "#{Rails.root}/app/jobs/concerns"
 
     # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
     # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.

config/secrets.yml

@@ -83,6 +83,7 @@ defaults: &defaults
     api_url: <%= ENV['DATAGOUV_API_URL'] %>
     descriptif_demarches_dataset: <%= ENV['DATAGOUV_DESCRIPTIF_DEMARCHES_DATASET'] %>
     descriptif_demarches_resource: <%= ENV['DATAGOUV_DESCRIPTIF_DEMARCHES_RESOURCE'] %>
+    statistics_dataset: <%= ENV['DATAGOUV_STATISTICS_DATASET'] %>
 
 
 development:

spec/jobs/cron/datagouv/account_by_month_job_spec.rb

@@ -0,0 +1,19 @@
+RSpec.describe Cron::Datagouv::AccountByMonthJob, type: :job do
+  let!(:user) { create(:user, created_at: 1.month.ago) }
+  let(:status) { 200 }
+  let(:body) { "ok" }
+  let(:stub) { stub_request(:post, /https:\/\/www.data.gouv.fr\/api\/.*\/upload\//) }
+
+  describe 'perform' do
+    before do
+      stub
+    end
+
+    subject { Cron::Datagouv::AccountByMonthJob.perform_now }
+
+    it 'send POST request to datagouv' do
+      subject
+      expect(stub).to have_been_requested
+    end
+  end
+end

spec/jobs/cron/datagouv/administrateur_by_month_job_spec.rb

@@ -0,0 +1,19 @@
+RSpec.describe Cron::Datagouv::AdministrateurByMonthJob, type: :job do
+  let!(:administrateur) { create(:administrateur, created_at: 1.month.ago) }
+  let(:status) { 200 }
+  let(:body) { "ok" }
+  let(:stub) { stub_request(:post, /https:\/\/www.data.gouv.fr\/api\/.*\/upload\//) }
+
+  describe 'perform' do
+    before do
+      stub
+    end
+
+    subject { Cron::Datagouv::AdministrateurByMonthJob.perform_now }
+
+    it 'send POST request to datagouv' do
+      subject
+      expect(stub).to have_been_requested
+    end
+  end
+end

spec/jobs/cron/datagouv/file_by_month_job_spec.rb

@@ -0,0 +1,19 @@
+RSpec.describe Cron::Datagouv::FileByMonthJob, type: :job do
+  let!(:dossier) { create(:dossier, created_at: 1.month.ago) }
+  let(:status) { 200 }
+  let(:body) { "ok" }
+  let(:stub) { stub_request(:post, /https:\/\/www.data.gouv.fr\/api\/.*\/upload\//) }
+
+  describe 'perform' do
+    before do
+      stub
+    end
+
+    subject { Cron::Datagouv::FileByMonthJob.perform_now }
+
+    it 'send POST request to datagouv' do
+      subject
+      expect(stub).to have_been_requested
+    end
+  end
+end

spec/jobs/cron/datagouv/instructeur_by_month_job_spec.rb

@@ -0,0 +1,19 @@
+RSpec.describe Cron::Datagouv::InstructeurByMonthJob, type: :job do
+  let!(:instructeur) { create(:instructeur, created_at: 1.month.ago) }
+  let(:status) { 200 }
+  let(:body) { "ok" }
+  let(:stub) { stub_request(:post, /https:\/\/www.data.gouv.fr\/api\/.*\/upload\//) }
+
+  describe 'perform' do
+    before do
+      stub
+    end
+
+    subject { Cron::Datagouv::InstructeurByMonthJob.perform_now }
+
+    it 'send POST request to datagouv' do
+      subject
+      expect(stub).to have_been_requested
+    end
+  end
+end

spec/jobs/cron/datagouv/instructeur_connected_by_month_job_spec.rb

@@ -0,0 +1,20 @@
+RSpec.describe Cron::Datagouv::InstructeurConnectedByMonthJob, type: :job do
+  let!(:user) { create(:user) }
+  let!(:instructeur) { create(:instructeur, user: user, created_at: 1.month.ago) }
+  let(:status) { 200 }
+  let(:body) { "ok" }
+  let(:stub) { stub_request(:post, /https:\/\/www.data.gouv.fr\/api\/.*\/upload\//) }
+
+  describe 'perform' do
+    before do
+      stub
+    end
+
+    subject { Cron::Datagouv::InstructeurConnectedByMonthJob.perform_now }
+
+    it 'send POST request to datagouv' do
+      subject
+      expect(stub).to have_been_requested
+    end
+  end
+end

spec/jobs/cron/datagouv/procedure_by_month_job_spec.rb

@@ -0,0 +1,19 @@
+RSpec.describe Cron::Datagouv::ProcedureByMonthJob, type: :job do
+  let!(:procedure) { create(:procedure, created_at: 1.month.ago) }
+  let(:status) { 200 }
+  let(:body) { "ok" }
+  let(:stub) { stub_request(:post, /https:\/\/www.data.gouv.fr\/api\/.*\/upload\//) }
+
+  describe 'perform' do
+    before do
+      stub
+    end
+
+    subject { Cron::Datagouv::ProcedureByMonthJob.perform_now }
+
+    it 'send POST request to datagouv' do
+      subject
+      expect(stub).to have_been_requested
+    end
+  end
+end

spec/jobs/cron/datagouv/procedure_closed_by_month_job_spec.rb

@@ -0,0 +1,19 @@
+RSpec.describe Cron::Datagouv::ProcedureClosedByMonthJob, type: :job do
+  let!(:procedure) { create(:procedure, closed_at: 1.month.ago) }
+  let(:status) { 200 }
+  let(:body) { "ok" }
+  let(:stub) { stub_request(:post, /https:\/\/www.data.gouv.fr\/api\/.*\/upload\//) }
+
+  describe 'perform' do
+    before do
+      stub
+    end
+
+    subject { Cron::Datagouv::ProcedureClosedByMonthJob.perform_now }
+
+    it 'send POST request to datagouv' do
+      subject
+      expect(stub).to have_been_requested
+    end
+  end
+end

spec/jobs/cron/datagouv/procedure_deleted_by_month_job_spec.rb

@@ -0,0 +1,19 @@
+RSpec.describe Cron::Datagouv::ProcedureDeletedByMonthJob, type: :job do
+  let!(:procedure) { create(:procedure, hidden_at: 1.month.ago) }
+  let(:status) { 200 }
+  let(:body) { "ok" }
+  let(:stub) { stub_request(:post, /https:\/\/www.data.gouv.fr\/api\/.*\/upload\//) }
+
+  describe 'perform' do
+    before do
+      stub
+    end
+
+    subject { Cron::Datagouv::ProcedureDeletedByMonthJob.perform_now }
+
+    it 'send POST request to datagouv' do
+      subject
+      expect(stub).to have_been_requested
+    end
+  end
+end

spec/lib/api_datagouv/api_spec.rb

@@ -1,9 +1,12 @@
 describe APIDatagouv::API do
   describe '#upload' do
-    let(:subject) { APIDatagouv::API.upload(Tempfile.new.path) }
+    let(:dataset) { :descriptif_demarches_dataset }
+    let(:resource) { :descriptif_demarches_resource }
+    let(:datagouv_secret) { Rails.application.secrets.datagouv }
+    let(:subject) { APIDatagouv::API.upload(Tempfile.new, dataset, resource) }
 
     before do
-      stub_request(:post, /https:\/\/www.data.gouv.fr\/api/)
+      stub_request(:post, /https:\/\/www.data.gouv.fr\/api\/1\/datasets\/#{datagouv_secret[dataset]}\/resources\/#{datagouv_secret[resource]}\/upload\//)
         .to_return(body: body, status: status)
     end