feat(dolist): consolidate dispatched events with delivered emails

app/jobs/dolist_report_job.rb

@@ -0,0 +1,55 @@
+class DolistReportJob < ApplicationJob
+  # Consolidate random recent emails dispatched to Dolist with their statuses
+  # and send a report by email.
+  def perform(report_to, sample_size = 1000)
+    events = EmailEvent.dolist.dispatched.where(processed_at: 2.weeks.ago..).order("RANDOM()").limit(sample_size)
+
+    data = CSV.generate(headers: true) do |csv|
+      column_names = ["dispatched_at", "subject", "domain", "status", "delivered_at", "delay (min)"]
+      csv << column_names
+
+      events.each do |event|
+        report_event(csv, event)
+      end
+    end
+
+    tempfile = Tempfile.new("dolist_report.csv")
+    tempfile.write(data)
+    tempfile.rewind
+
+    SuperAdminMailer.dolist_report(report_to, tempfile.path).deliver_now
+  ensure
+    tempfile&.unlink
+  end
+
+  private
+
+  def report_event(csv, event)
+    wait_if_api_limit_reached
+
+    sent_email = event.match_dolist_email
+
+    delay = if sent_email
+      (sent_email.delivered_at.to_i - event.processed_at.to_i) / 60
+    end
+
+    csv << [
+      event.processed_at,
+      event.subject,
+      event.domain,
+      sent_email&.status,
+      sent_email&.delivered_at,
+      delay
+    ]
+  rescue StandardError => error
+    Sentry.capture_exception(error, extra: { event_id: event.id, api_limit_remaining: Dolist::API.limit_remaining, api_limit_reset_at: Dolist::API.limit_reset_at })
+  end
+
+  def wait_if_api_limit_reached
+    return unless Dolist::API.near_rate_limit?
+
+    Rails.logger.info("Dolist API rate limit reached, sleep until #{Dolist::API.limit_reset_at}")
+
+    Dolist::API.sleep_until_limit_reset
+  end
+end

app/lib/dolist/api.rb

@@ -4,6 +4,27 @@ class Dolist::API
   EMAIL_KEY = 7
   DOLIST_WEB_DASHBOARD = "https://campaign.dolist.net/#/%{account_id}/contacts/%{contact_id}/sendings"
 
+  class_attribute :limit_remaining, :limit_reset_at
+
+  class << self
+    def save_rate_limit_headers(headers)
+      self.limit_remaining = headers["X-Rate-Limit-Remaining"].to_i
+      self.limit_reset_at = Time.zone.at(headers["X-Rate-Limit-Reset"].to_i / 1_000)
+    end
+
+    def near_rate_limit?
+      return if limit_remaining.nil?
+
+      limit_remaining < 20 # keep 20 requests for non background API calls
+    end
+
+    def sleep_until_limit_reset
+      return if limit_reset_at.nil? || limit_reset_at.past?
+
+      sleep (limit_reset_at - Time.zone.now).ceil
+    end
+  end
+
   def properly_configured?
     client_key.present?
   end
@@ -49,9 +70,7 @@ class Dolist::API
       Query: { FieldValueList: [{ ID: EMAIL_KEY, Value: email_address }] }
     }.to_json
 
-    response = Typhoeus.post(url, body: body, headers: headers)
-
-    JSON.parse(response.response_body)["ID"]
+    post(url, body)["ID"]
   end
 
   # https://api.dolist.com/documentation/index.html#/b3A6Mzg0MTQ4MDk-recuperer-les-statistiques-des-envois-pour-un-contact
@@ -60,9 +79,15 @@ class Dolist::API
 
     body = { SearchQuery: { ContactID: contact_id } }.to_json
 
-    response = Typhoeus.post(url, body: body, headers: headers)
+    post(url, body)["ItemList"]
+  end
 
-    JSON.parse(response.response_body)['ItemList']
+  def post(url, body)
+    response = Typhoeus.post(url, body:, headers:).tap do
+      self.class.save_rate_limit_headers(_1.headers)
+    end
+
+    JSON.parse(response.response_body)
   end
 
   def to_sent_mail(email_address, contact_id, dolist_message)

app/mailers/super_admin_mailer.rb

@@ -0,0 +1,7 @@
+class SuperAdminMailer < ApplicationMailer
+  def dolist_report(to, csv_path)
+    attachments["dolist_report.csv"] = File.read(csv_path)
+
+    mail(to: to, subject: "Dolist report", body: "Ci-joint le rapport d'emails récents envoyés via Dolist.")
+  end
+end

app/models/email_event.rb

@@ -17,6 +17,9 @@ class EmailEvent < ApplicationRecord
     dispatch_error: 'dispatch_error'
   }
 
+  scope :dolist, -> { where(method: 'dolist') }
+  scope :sendinblue, -> { where(method: 'sendinblue') }
+
   class << self
     def create_from_message!(message, status:)
       to = message.to || ["unset"] # no recipients when error occurs *before* setting to: in the mailer
@@ -34,4 +37,15 @@ class EmailEvent < ApplicationRecord
       end
     end
   end
+
+  def match_dolist_email
+    return if to == "unset"
+
+    # subjects does not match, so compare to event time with tolerance
+    Dolist::API.new.sent_mails(to).sort_by(&:delivered_at).find { (processed_at..processed_at + 1.hour).cover?(_1.delivered_at) }
+  end
+
+  def domain
+    to.split("@").last
+  end
 end

spec/factories/email_event.rb

@@ -0,0 +1,12 @@
+FactoryBot.define do
+  factory :email_event do
+    to { "user@email.com" }
+    subject { "Thank you" }
+    processed_at { Time.zone.now }
+    status { "dispatched" }
+
+    trait :dolist do
+      add_attribute(:method) { "dolist" }
+    end
+  end
+end

spec/jobs/dolist_report_job_spec.rb

@@ -0,0 +1,40 @@
+require 'rails_helper'
+
+RSpec.describe DolistReportJob, type: :job do
+  let(:event1) { create(:email_event, :dolist, :dispatched, to: "you@blabla.com", processed_at: 1.minute.ago) }
+  let(:event2) { create(:email_event, :dolist, :dispatched) }
+
+  before do
+    emails = [
+      SentMail.new(
+        to: event1.to,
+        status: "delivered",
+        delivered_at: event1.processed_at + 1.minute
+      )
+    ]
+
+    allow_any_instance_of(Dolist::API).to receive(:sent_mails).with(event1.to).and_return(emails)
+    allow_any_instance_of(Dolist::API).to receive(:sent_mails).with(event2.to).and_return([])
+  end
+
+  subject(:perform_job) { described_class.perform_now("you@rocks.com") }
+
+  it "generates a csv file and send it by email" do
+    perform_job
+
+    email = ActionMailer::Base.deliveries.last
+    expect(email.to).to eq(["you@rocks.com"])
+    expect(email.attachments[0].filename).to eq("dolist_report.csv")
+
+    csv = CSV.parse(email.attachments[0].body.decoded, headers: true)
+    expect(csv.size).to eq(2)
+
+    # events were processed randomly, go back to a deterministic order
+    rows = csv.sort_by { _1["dispatched_at"] }
+
+    expect(rows[0]["domain"]).to eq("blabla.com")
+    expect(rows[0]["status"]).to eq("delivered")
+    expect(rows[0]["delay (min)"]).to eq("1")
+    expect(rows[1]["status"]).to be_nil
+  end
+end

spec/mailers/previews/super_admin_mailer_preview.rb

@@ -0,0 +1,6 @@
+# Preview all emails at http://localhost:3000/rails/mailers/super_admin_mailer
+class SuperAdminMailerPreview < ActionMailer::Preview
+  def dolist_report
+    SuperAdminMailer.dolist_report("you@beta.gouv.fr", Rails.root.join("spec/fixtures/files/groupe_avec_caracteres_speciaux.csv"))
+  end
+end