Merge pull request #8451 from colinux/dolist-report-job

feat: ETQ super-admin je peux recevoir un rapport d'analyse d'emails envoyés avec Dolist

app/controllers/manager/email_events_controller.rb

@@ -1,4 +1,25 @@
 module Manager
   class EmailEventsController < Manager::ApplicationController
+    def index
+      @dolist_enabled = Dolist::API.new.properly_configured?
+
+      super
+    end
+
+    def generate_dolist_report
+      email = current_super_admin.email
+
+      DolistReportJob.perform_later(email)
+
+      respond_to do |format|
+        @message = "Le rapport sera envoyé sur #{email}. Il peut prendre plus d'1h pour être généré."
+
+        format.turbo_stream
+
+        format.html do
+          redirect_to manager_email_events_path, notice: @message
+        end
+      end
+    end
   end
 end

app/jobs/dolist_report_job.rb

@@ -0,0 +1,55 @@
+class DolistReportJob < ApplicationJob
+  # Consolidate random recent emails dispatched to Dolist with their statuses
+  # and send a report by email.
+  def perform(report_to, sample_size = 1000)
+    events = EmailEvent.dolist.dispatched.where(processed_at: 2.weeks.ago..).order("RANDOM()").limit(sample_size)
+
+    data = CSV.generate(headers: true) do |csv|
+      column_names = ["dispatched_at", "subject", "domain", "status", "delivered_at", "delay (min)"]
+      csv << column_names
+
+      events.each do |event|
+        report_event(csv, event)
+      end
+    end
+
+    tempfile = Tempfile.new("dolist_report.csv")
+    tempfile.write(data)
+    tempfile.rewind
+
+    SuperAdminMailer.dolist_report(report_to, tempfile.path).deliver_now
+  ensure
+    tempfile&.unlink
+  end
+
+  private
+
+  def report_event(csv, event)
+    wait_if_api_limit_reached
+
+    sent_email = event.match_dolist_email
+
+    delay = if sent_email
+      (sent_email.delivered_at.to_i - event.processed_at.to_i) / 60
+    end
+
+    csv << [
+      event.processed_at,
+      event.subject,
+      event.domain,
+      sent_email&.status,
+      sent_email&.delivered_at,
+      delay
+    ]
+  rescue StandardError => error
+    Sentry.capture_exception(error, extra: { event_id: event.id, api_limit_remaining: Dolist::API.limit_remaining, api_limit_reset_at: Dolist::API.limit_reset_at })
+  end
+
+  def wait_if_api_limit_reached
+    return unless Dolist::API.near_rate_limit?
+
+    Rails.logger.info("Dolist API rate limit reached, sleep until #{Dolist::API.limit_reset_at}")
+
+    Dolist::API.sleep_until_limit_reset
+  end
+end

app/lib/dolist/api.rb

@@ -4,6 +4,27 @@ class Dolist::API
   EMAIL_KEY = 7
   DOLIST_WEB_DASHBOARD = "https://campaign.dolist.net/#/%{account_id}/contacts/%{contact_id}/sendings"
 
+  class_attribute :limit_remaining, :limit_reset_at
+
+  class << self
+    def save_rate_limit_headers(headers)
+      self.limit_remaining = headers["X-Rate-Limit-Remaining"].to_i
+      self.limit_reset_at = Time.zone.at(headers["X-Rate-Limit-Reset"].to_i / 1_000)
+    end
+
+    def near_rate_limit?
+      return if limit_remaining.nil?
+
+      limit_remaining < 20 # keep 20 requests for non background API calls
+    end
+
+    def sleep_until_limit_reset
+      return if limit_reset_at.nil? || limit_reset_at.past?
+
+      sleep (limit_reset_at - Time.zone.now).ceil
+    end
+  end
+
   def properly_configured?
     client_key.present?
   end
@@ -49,9 +70,7 @@ class Dolist::API
       Query: { FieldValueList: [{ ID: EMAIL_KEY, Value: email_address }] }
     }.to_json
 
-    response = Typhoeus.post(url, body: body, headers: headers)
-
-    JSON.parse(response.response_body)["ID"]
+    post(url, body)["ID"]
   end
 
   # https://api.dolist.com/documentation/index.html#/b3A6Mzg0MTQ4MDk-recuperer-les-statistiques-des-envois-pour-un-contact
@@ -60,9 +79,15 @@ class Dolist::API
 
     body = { SearchQuery: { ContactID: contact_id } }.to_json
 
-    response = Typhoeus.post(url, body: body, headers: headers)
+    post(url, body)["ItemList"]
+  end
 
-    JSON.parse(response.response_body)['ItemList']
+  def post(url, body)
+    response = Typhoeus.post(url, body:, headers:).tap do
+      self.class.save_rate_limit_headers(_1.headers)
+    end
+
+    JSON.parse(response.response_body)
   end
 
   def to_sent_mail(email_address, contact_id, dolist_message)

app/mailers/super_admin_mailer.rb

@@ -0,0 +1,7 @@
+class SuperAdminMailer < ApplicationMailer
+  def dolist_report(to, csv_path)
+    attachments["dolist_report.csv"] = File.read(csv_path)
+
+    mail(to: to, subject: "Dolist report", body: "Ci-joint le rapport d'emails récents envoyés via Dolist.")
+  end
+end

app/models/email_event.rb

@@ -17,6 +17,9 @@ class EmailEvent < ApplicationRecord
     dispatch_error: 'dispatch_error'
   }
 
+  scope :dolist, -> { where(method: 'dolist') }
+  scope :sendinblue, -> { where(method: 'sendinblue') }
+
   class << self
     def create_from_message!(message, status:)
       to = message.to || ["unset"] # no recipients when error occurs *before* setting to: in the mailer
@@ -34,4 +37,15 @@ class EmailEvent < ApplicationRecord
       end
     end
   end
+
+  def match_dolist_email
+    return if to == "unset"
+
+    # subjects does not match, so compare to event time with tolerance
+    Dolist::API.new.sent_mails(to).sort_by(&:delivered_at).find { (processed_at..processed_at + 1.hour).cover?(_1.delivered_at) }
+  end
+
+  def domain
+    to.split("@").last
+  end
 end

app/views/manager/application/index.html.erb

@@ -0,0 +1,46 @@
+<%#
+# Override original index template augmented with an optional "index_footer" partial.
+# Update it from the template with the original content:
+# https://github.com/thoughtbot/administrate/blob/main/app/views/administrate/application/index.html.erb
+
+# Index
+This view is the template for the index page.
+It is responsible for rendering the search bar, header and pagination.
+It renders the `_table` partial to display details about the resources.
+## Local variables:
+- `page`:
+  An instance of [Administrate::Page::Collection][1].
+  Contains helper methods to help display a table,
+  and knows which attributes should be displayed in the resource's table.
+- `resources`:
+  An instance of `ActiveRecord::Relation` containing the resources
+  that match the user's search criteria.
+  By default, these resources are passed to the table partial to be displayed.
+- `search_term`:
+  A string containing the term the user has searched for, if any.
+- `show_search_bar`:
+  A boolean that determines if the search bar should be shown.
+[1]: http://www.rubydoc.info/gems/administrate/Administrate/Page/Collection
+%>
+<%=
+  render("index_header",
+    resources: resources,
+    search_term: search_term,
+    page: page,
+    show_search_bar: show_search_bar,
+  )
+%>
+
+<section class="main-content__body main-content__body--flush">
+  <%= render(
+    "collection",
+    collection_presenter: page,
+    collection_field_name: resource_name,
+    page: page,
+    resources: resources,
+    table_title: "page-title"
+  ) %>
+  <%= render("pagination", resources: resources) %>
+</section>
+
+<%= render("index_footer") %>

app/views/manager/email_events/_index_footer.html.erb

@@ -0,0 +1,7 @@
+<footer class="main-content__body">
+  <% if @dolist_enabled %>
+    <%= form_tag(generate_dolist_report_manager_email_events_path, id: "dolist-report-form", data: { turbo: true }) do %>
+      <button>Recevoir un rapport Dolist</button>
+    <% end %>
+  <% end %>
+</footer>

app/views/manager/email_events/generate_dolist_report.turbo_stream.haml

@@ -0,0 +1,2 @@
+= turbo_stream.morph "dolist-report-form" do
+  = @message

config/routes.rb

@@ -65,7 +65,9 @@ Rails.application.routes.draw do
 
     resources :team_accounts, only: [:index, :show]
 
-    resources :email_events, only: [:index, :show]
+    resources :email_events, only: [:index, :show] do
+      post :generate_dolist_report, on: :collection
+    end
 
     resources :dubious_procedures, only: [:index]
     resources :outdated_procedures, only: [:index] do

spec/factories/email_event.rb

@@ -0,0 +1,12 @@
+FactoryBot.define do
+  factory :email_event do
+    to { "user@email.com" }
+    subject { "Thank you" }
+    processed_at { Time.zone.now }
+    status { "dispatched" }
+
+    trait :dolist do
+      add_attribute(:method) { "dolist" }
+    end
+  end
+end

spec/jobs/dolist_report_job_spec.rb

@@ -0,0 +1,40 @@
+require 'rails_helper'
+
+RSpec.describe DolistReportJob, type: :job do
+  let(:event1) { create(:email_event, :dolist, :dispatched, to: "you@blabla.com", processed_at: 1.minute.ago) }
+  let(:event2) { create(:email_event, :dolist, :dispatched) }
+
+  before do
+    emails = [
+      SentMail.new(
+        to: event1.to,
+        status: "delivered",
+        delivered_at: event1.processed_at + 1.minute
+      )
+    ]
+
+    allow_any_instance_of(Dolist::API).to receive(:sent_mails).with(event1.to).and_return(emails)
+    allow_any_instance_of(Dolist::API).to receive(:sent_mails).with(event2.to).and_return([])
+  end
+
+  subject(:perform_job) { described_class.perform_now("you@rocks.com") }
+
+  it "generates a csv file and send it by email" do
+    perform_job
+
+    email = ActionMailer::Base.deliveries.last
+    expect(email.to).to eq(["you@rocks.com"])
+    expect(email.attachments[0].filename).to eq("dolist_report.csv")
+
+    csv = CSV.parse(email.attachments[0].body.decoded, headers: true)
+    expect(csv.size).to eq(2)
+
+    # events were processed randomly, go back to a deterministic order
+    rows = csv.sort_by { _1["dispatched_at"] }
+
+    expect(rows[0]["domain"]).to eq("blabla.com")
+    expect(rows[0]["status"]).to eq("delivered")
+    expect(rows[0]["delay (min)"]).to eq("1")
+    expect(rows[1]["status"]).to be_nil
+  end
+end

spec/mailers/previews/super_admin_mailer_preview.rb

@@ -0,0 +1,6 @@
+# Preview all emails at http://localhost:3000/rails/mailers/super_admin_mailer
+class SuperAdminMailerPreview < ActionMailer::Preview
+  def dolist_report
+    SuperAdminMailer.dolist_report("you@beta.gouv.fr", Rails.root.join("spec/fixtures/files/groupe_avec_caracteres_speciaux.csv"))
+  end
+end