diff --git a/app/controllers/manager/email_events_controller.rb b/app/controllers/manager/email_events_controller.rb
index bd7b59ff7..4a4cb74c2 100644
--- a/app/controllers/manager/email_events_controller.rb
+++ b/app/controllers/manager/email_events_controller.rb
@@ -1,4 +1,25 @@
module Manager
class EmailEventsController < Manager::ApplicationController
+ def index
+ @dolist_enabled = Dolist::API.new.properly_configured?
+
+ super
+ end
+
+ def generate_dolist_report
+ email = current_super_admin.email
+
+ DolistReportJob.perform_later(email)
+
+ respond_to do |format|
+ @message = "Le rapport sera envoyé sur #{email}. Il peut prendre plus d'1h pour être généré."
+
+ format.turbo_stream
+
+ format.html do
+ redirect_to manager_email_events_path, notice: @message
+ end
+ end
+ end
end
end
diff --git a/app/jobs/dolist_report_job.rb b/app/jobs/dolist_report_job.rb
new file mode 100644
index 000000000..8a7b38efe
--- /dev/null
+++ b/app/jobs/dolist_report_job.rb
@@ -0,0 +1,55 @@
+class DolistReportJob < ApplicationJob
+ # Consolidate random recent emails dispatched to Dolist with their statuses
+ # and send a report by email.
+ def perform(report_to, sample_size = 1000)
+ events = EmailEvent.dolist.dispatched.where(processed_at: 2.weeks.ago..).order("RANDOM()").limit(sample_size)
+
+ data = CSV.generate(headers: true) do |csv|
+ column_names = ["dispatched_at", "subject", "domain", "status", "delivered_at", "delay (min)"]
+ csv << column_names
+
+ events.each do |event|
+ report_event(csv, event)
+ end
+ end
+
+ tempfile = Tempfile.new("dolist_report.csv")
+ tempfile.write(data)
+ tempfile.rewind
+
+ SuperAdminMailer.dolist_report(report_to, tempfile.path).deliver_now
+ ensure
+ tempfile&.unlink
+ end
+
+ private
+
+ def report_event(csv, event)
+ wait_if_api_limit_reached
+
+ sent_email = event.match_dolist_email
+
+ delay = if sent_email
+ (sent_email.delivered_at.to_i - event.processed_at.to_i) / 60
+ end
+
+ csv << [
+ event.processed_at,
+ event.subject,
+ event.domain,
+ sent_email&.status,
+ sent_email&.delivered_at,
+ delay
+ ]
+ rescue StandardError => error
+ Sentry.capture_exception(error, extra: { event_id: event.id, api_limit_remaining: Dolist::API.limit_remaining, api_limit_reset_at: Dolist::API.limit_reset_at })
+ end
+
+ def wait_if_api_limit_reached
+ return unless Dolist::API.near_rate_limit?
+
+ Rails.logger.info("Dolist API rate limit reached, sleep until #{Dolist::API.limit_reset_at}")
+
+ Dolist::API.sleep_until_limit_reset
+ end
+end
diff --git a/app/lib/dolist/api.rb b/app/lib/dolist/api.rb
index b18ec5e49..8ae14c2f6 100644
--- a/app/lib/dolist/api.rb
+++ b/app/lib/dolist/api.rb
@@ -4,6 +4,27 @@ class Dolist::API
EMAIL_KEY = 7
DOLIST_WEB_DASHBOARD = "https://campaign.dolist.net/#/%{account_id}/contacts/%{contact_id}/sendings"
+ class_attribute :limit_remaining, :limit_reset_at
+
+ class << self
+ def save_rate_limit_headers(headers)
+ self.limit_remaining = headers["X-Rate-Limit-Remaining"].to_i
+ self.limit_reset_at = Time.zone.at(headers["X-Rate-Limit-Reset"].to_i / 1_000)
+ end
+
+ def near_rate_limit?
+ return if limit_remaining.nil?
+
+ limit_remaining < 20 # keep 20 requests for non background API calls
+ end
+
+ def sleep_until_limit_reset
+ return if limit_reset_at.nil? || limit_reset_at.past?
+
+ sleep (limit_reset_at - Time.zone.now).ceil
+ end
+ end
+
def properly_configured?
client_key.present?
end
@@ -49,9 +70,7 @@ class Dolist::API
Query: { FieldValueList: [{ ID: EMAIL_KEY, Value: email_address }] }
}.to_json
- response = Typhoeus.post(url, body: body, headers: headers)
-
- JSON.parse(response.response_body)["ID"]
+ post(url, body)["ID"]
end
# https://api.dolist.com/documentation/index.html#/b3A6Mzg0MTQ4MDk-recuperer-les-statistiques-des-envois-pour-un-contact
@@ -60,9 +79,15 @@ class Dolist::API
body = { SearchQuery: { ContactID: contact_id } }.to_json
- response = Typhoeus.post(url, body: body, headers: headers)
+ post(url, body)["ItemList"]
+ end
- JSON.parse(response.response_body)['ItemList']
+ def post(url, body)
+ response = Typhoeus.post(url, body:, headers:).tap do
+ self.class.save_rate_limit_headers(_1.headers)
+ end
+
+ JSON.parse(response.response_body)
end
def to_sent_mail(email_address, contact_id, dolist_message)
diff --git a/app/mailers/super_admin_mailer.rb b/app/mailers/super_admin_mailer.rb
new file mode 100644
index 000000000..75c838652
--- /dev/null
+++ b/app/mailers/super_admin_mailer.rb
@@ -0,0 +1,7 @@
+class SuperAdminMailer < ApplicationMailer
+ def dolist_report(to, csv_path)
+ attachments["dolist_report.csv"] = File.read(csv_path)
+
+ mail(to: to, subject: "Dolist report", body: "Ci-joint le rapport d'emails récents envoyés via Dolist.")
+ end
+end
diff --git a/app/models/email_event.rb b/app/models/email_event.rb
index 6a66ff907..aa13275a6 100644
--- a/app/models/email_event.rb
+++ b/app/models/email_event.rb
@@ -17,6 +17,9 @@ class EmailEvent < ApplicationRecord
dispatch_error: 'dispatch_error'
}
+ scope :dolist, -> { where(method: 'dolist') }
+ scope :sendinblue, -> { where(method: 'sendinblue') }
+
class << self
def create_from_message!(message, status:)
to = message.to || ["unset"] # no recipients when error occurs *before* setting to: in the mailer
@@ -34,4 +37,15 @@ class EmailEvent < ApplicationRecord
end
end
end
+
+ def match_dolist_email
+ return if to == "unset"
+
+ # subjects does not match, so compare to event time with tolerance
+ Dolist::API.new.sent_mails(to).sort_by(&:delivered_at).find { (processed_at..processed_at + 1.hour).cover?(_1.delivered_at) }
+ end
+
+ def domain
+ to.split("@").last
+ end
end
diff --git a/app/views/manager/application/_index_footer.html.erb b/app/views/manager/application/_index_footer.html.erb
new file mode 100644
index 000000000..e69de29bb
diff --git a/app/views/manager/application/index.html.erb b/app/views/manager/application/index.html.erb
new file mode 100644
index 000000000..9201f1d60
--- /dev/null
+++ b/app/views/manager/application/index.html.erb
@@ -0,0 +1,46 @@
+<%#
+# Override original index template augmented with an optional "index_footer" partial.
+# Update it from the template with the original content:
+# https://github.com/thoughtbot/administrate/blob/main/app/views/administrate/application/index.html.erb
+
+# Index
+This view is the template for the index page.
+It is responsible for rendering the search bar, header and pagination.
+It renders the `_table` partial to display details about the resources.
+## Local variables:
+- `page`:
+ An instance of [Administrate::Page::Collection][1].
+ Contains helper methods to help display a table,
+ and knows which attributes should be displayed in the resource's table.
+- `resources`:
+ An instance of `ActiveRecord::Relation` containing the resources
+ that match the user's search criteria.
+ By default, these resources are passed to the table partial to be displayed.
+- `search_term`:
+ A string containing the term the user has searched for, if any.
+- `show_search_bar`:
+ A boolean that determines if the search bar should be shown.
+[1]: http://www.rubydoc.info/gems/administrate/Administrate/Page/Collection
+%>
+<%=
+ render("index_header",
+ resources: resources,
+ search_term: search_term,
+ page: page,
+ show_search_bar: show_search_bar,
+ )
+%>
+
+
+ <%= render(
+ "collection",
+ collection_presenter: page,
+ collection_field_name: resource_name,
+ page: page,
+ resources: resources,
+ table_title: "page-title"
+ ) %>
+ <%= render("pagination", resources: resources) %>
+
+
+<%= render("index_footer") %>
diff --git a/app/views/manager/email_events/_index_footer.html.erb b/app/views/manager/email_events/_index_footer.html.erb
new file mode 100644
index 000000000..aa4f835cd
--- /dev/null
+++ b/app/views/manager/email_events/_index_footer.html.erb
@@ -0,0 +1,7 @@
+
diff --git a/app/views/manager/email_events/generate_dolist_report.turbo_stream.haml b/app/views/manager/email_events/generate_dolist_report.turbo_stream.haml
new file mode 100644
index 000000000..efe704452
--- /dev/null
+++ b/app/views/manager/email_events/generate_dolist_report.turbo_stream.haml
@@ -0,0 +1,2 @@
+= turbo_stream.morph "dolist-report-form" do
+ = @message
diff --git a/config/routes.rb b/config/routes.rb
index aedd12400..87b74e2b1 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -65,7 +65,9 @@ Rails.application.routes.draw do
resources :team_accounts, only: [:index, :show]
- resources :email_events, only: [:index, :show]
+ resources :email_events, only: [:index, :show] do
+ post :generate_dolist_report, on: :collection
+ end
resources :dubious_procedures, only: [:index]
resources :outdated_procedures, only: [:index] do
diff --git a/spec/factories/email_event.rb b/spec/factories/email_event.rb
new file mode 100644
index 000000000..198851eec
--- /dev/null
+++ b/spec/factories/email_event.rb
@@ -0,0 +1,12 @@
+FactoryBot.define do
+ factory :email_event do
+ to { "user@email.com" }
+ subject { "Thank you" }
+ processed_at { Time.zone.now }
+ status { "dispatched" }
+
+ trait :dolist do
+ add_attribute(:method) { "dolist" }
+ end
+ end
+end
diff --git a/spec/jobs/dolist_report_job_spec.rb b/spec/jobs/dolist_report_job_spec.rb
new file mode 100644
index 000000000..18b236d91
--- /dev/null
+++ b/spec/jobs/dolist_report_job_spec.rb
@@ -0,0 +1,40 @@
+require 'rails_helper'
+
+RSpec.describe DolistReportJob, type: :job do
+ let(:event1) { create(:email_event, :dolist, :dispatched, to: "you@blabla.com", processed_at: 1.minute.ago) }
+ let(:event2) { create(:email_event, :dolist, :dispatched) }
+
+ before do
+ emails = [
+ SentMail.new(
+ to: event1.to,
+ status: "delivered",
+ delivered_at: event1.processed_at + 1.minute
+ )
+ ]
+
+ allow_any_instance_of(Dolist::API).to receive(:sent_mails).with(event1.to).and_return(emails)
+ allow_any_instance_of(Dolist::API).to receive(:sent_mails).with(event2.to).and_return([])
+ end
+
+ subject(:perform_job) { described_class.perform_now("you@rocks.com") }
+
+ it "generates a csv file and send it by email" do
+ perform_job
+
+ email = ActionMailer::Base.deliveries.last
+ expect(email.to).to eq(["you@rocks.com"])
+ expect(email.attachments[0].filename).to eq("dolist_report.csv")
+
+ csv = CSV.parse(email.attachments[0].body.decoded, headers: true)
+ expect(csv.size).to eq(2)
+
+ # events were processed randomly, go back to a deterministic order
+ rows = csv.sort_by { _1["dispatched_at"] }
+
+ expect(rows[0]["domain"]).to eq("blabla.com")
+ expect(rows[0]["status"]).to eq("delivered")
+ expect(rows[0]["delay (min)"]).to eq("1")
+ expect(rows[1]["status"]).to be_nil
+ end
+end
diff --git a/spec/mailers/previews/super_admin_mailer_preview.rb b/spec/mailers/previews/super_admin_mailer_preview.rb
new file mode 100644
index 000000000..69ae5cdf0
--- /dev/null
+++ b/spec/mailers/previews/super_admin_mailer_preview.rb
@@ -0,0 +1,6 @@
+# Preview all emails at http://localhost:3000/rails/mailers/super_admin_mailer
+class SuperAdminMailerPreview < ActionMailer::Preview
+ def dolist_report
+ SuperAdminMailer.dolist_report("you@beta.gouv.fr", Rails.root.join("spec/fixtures/files/groupe_avec_caracteres_speciaux.csv"))
+ end
+end