diff --git a/app/jobs/dolist_report_job.rb b/app/jobs/dolist_report_job.rb new file mode 100644 index 000000000..8a7b38efe --- /dev/null +++ b/app/jobs/dolist_report_job.rb @@ -0,0 +1,55 @@ +class DolistReportJob < ApplicationJob + # Consolidate random recent emails dispatched to Dolist with their statuses + # and send a report by email. + def perform(report_to, sample_size = 1000) + events = EmailEvent.dolist.dispatched.where(processed_at: 2.weeks.ago..).order("RANDOM()").limit(sample_size) + + data = CSV.generate(headers: true) do |csv| + column_names = ["dispatched_at", "subject", "domain", "status", "delivered_at", "delay (min)"] + csv << column_names + + events.each do |event| + report_event(csv, event) + end + end + + tempfile = Tempfile.new("dolist_report.csv") + tempfile.write(data) + tempfile.rewind + + SuperAdminMailer.dolist_report(report_to, tempfile.path).deliver_now + ensure + tempfile&.unlink + end + + private + + def report_event(csv, event) + wait_if_api_limit_reached + + sent_email = event.match_dolist_email + + delay = if sent_email + (sent_email.delivered_at.to_i - event.processed_at.to_i) / 60 + end + + csv << [ + event.processed_at, + event.subject, + event.domain, + sent_email&.status, + sent_email&.delivered_at, + delay + ] + rescue StandardError => error + Sentry.capture_exception(error, extra: { event_id: event.id, api_limit_remaining: Dolist::API.limit_remaining, api_limit_reset_at: Dolist::API.limit_reset_at }) + end + + def wait_if_api_limit_reached + return unless Dolist::API.near_rate_limit? + + Rails.logger.info("Dolist API rate limit reached, sleep until #{Dolist::API.limit_reset_at}") + + Dolist::API.sleep_until_limit_reset + end +end diff --git a/app/lib/dolist/api.rb b/app/lib/dolist/api.rb index b18ec5e49..8ae14c2f6 100644 --- a/app/lib/dolist/api.rb +++ b/app/lib/dolist/api.rb @@ -4,6 +4,27 @@ class Dolist::API EMAIL_KEY = 7 DOLIST_WEB_DASHBOARD = "https://campaign.dolist.net/#/%{account_id}/contacts/%{contact_id}/sendings" + class_attribute :limit_remaining, :limit_reset_at + + class << self + def save_rate_limit_headers(headers) + self.limit_remaining = headers["X-Rate-Limit-Remaining"].to_i + self.limit_reset_at = Time.zone.at(headers["X-Rate-Limit-Reset"].to_i / 1_000) + end + + def near_rate_limit? + return if limit_remaining.nil? + + limit_remaining < 20 # keep 20 requests for non background API calls + end + + def sleep_until_limit_reset + return if limit_reset_at.nil? || limit_reset_at.past? + + sleep (limit_reset_at - Time.zone.now).ceil + end + end + def properly_configured? client_key.present? end @@ -49,9 +70,7 @@ class Dolist::API Query: { FieldValueList: [{ ID: EMAIL_KEY, Value: email_address }] } }.to_json - response = Typhoeus.post(url, body: body, headers: headers) - - JSON.parse(response.response_body)["ID"] + post(url, body)["ID"] end # https://api.dolist.com/documentation/index.html#/b3A6Mzg0MTQ4MDk-recuperer-les-statistiques-des-envois-pour-un-contact @@ -60,9 +79,15 @@ class Dolist::API body = { SearchQuery: { ContactID: contact_id } }.to_json - response = Typhoeus.post(url, body: body, headers: headers) + post(url, body)["ItemList"] + end - JSON.parse(response.response_body)['ItemList'] + def post(url, body) + response = Typhoeus.post(url, body:, headers:).tap do + self.class.save_rate_limit_headers(_1.headers) + end + + JSON.parse(response.response_body) end def to_sent_mail(email_address, contact_id, dolist_message) diff --git a/app/mailers/super_admin_mailer.rb b/app/mailers/super_admin_mailer.rb new file mode 100644 index 000000000..75c838652 --- /dev/null +++ b/app/mailers/super_admin_mailer.rb @@ -0,0 +1,7 @@ +class SuperAdminMailer < ApplicationMailer + def dolist_report(to, csv_path) + attachments["dolist_report.csv"] = File.read(csv_path) + + mail(to: to, subject: "Dolist report", body: "Ci-joint le rapport d'emails récents envoyés via Dolist.") + end +end diff --git a/app/models/email_event.rb b/app/models/email_event.rb index 6a66ff907..aa13275a6 100644 --- a/app/models/email_event.rb +++ b/app/models/email_event.rb @@ -17,6 +17,9 @@ class EmailEvent < ApplicationRecord dispatch_error: 'dispatch_error' } + scope :dolist, -> { where(method: 'dolist') } + scope :sendinblue, -> { where(method: 'sendinblue') } + class << self def create_from_message!(message, status:) to = message.to || ["unset"] # no recipients when error occurs *before* setting to: in the mailer @@ -34,4 +37,15 @@ class EmailEvent < ApplicationRecord end end end + + def match_dolist_email + return if to == "unset" + + # subjects does not match, so compare to event time with tolerance + Dolist::API.new.sent_mails(to).sort_by(&:delivered_at).find { (processed_at..processed_at + 1.hour).cover?(_1.delivered_at) } + end + + def domain + to.split("@").last + end end diff --git a/spec/factories/email_event.rb b/spec/factories/email_event.rb new file mode 100644 index 000000000..198851eec --- /dev/null +++ b/spec/factories/email_event.rb @@ -0,0 +1,12 @@ +FactoryBot.define do + factory :email_event do + to { "user@email.com" } + subject { "Thank you" } + processed_at { Time.zone.now } + status { "dispatched" } + + trait :dolist do + add_attribute(:method) { "dolist" } + end + end +end diff --git a/spec/jobs/dolist_report_job_spec.rb b/spec/jobs/dolist_report_job_spec.rb new file mode 100644 index 000000000..18b236d91 --- /dev/null +++ b/spec/jobs/dolist_report_job_spec.rb @@ -0,0 +1,40 @@ +require 'rails_helper' + +RSpec.describe DolistReportJob, type: :job do + let(:event1) { create(:email_event, :dolist, :dispatched, to: "you@blabla.com", processed_at: 1.minute.ago) } + let(:event2) { create(:email_event, :dolist, :dispatched) } + + before do + emails = [ + SentMail.new( + to: event1.to, + status: "delivered", + delivered_at: event1.processed_at + 1.minute + ) + ] + + allow_any_instance_of(Dolist::API).to receive(:sent_mails).with(event1.to).and_return(emails) + allow_any_instance_of(Dolist::API).to receive(:sent_mails).with(event2.to).and_return([]) + end + + subject(:perform_job) { described_class.perform_now("you@rocks.com") } + + it "generates a csv file and send it by email" do + perform_job + + email = ActionMailer::Base.deliveries.last + expect(email.to).to eq(["you@rocks.com"]) + expect(email.attachments[0].filename).to eq("dolist_report.csv") + + csv = CSV.parse(email.attachments[0].body.decoded, headers: true) + expect(csv.size).to eq(2) + + # events were processed randomly, go back to a deterministic order + rows = csv.sort_by { _1["dispatched_at"] } + + expect(rows[0]["domain"]).to eq("blabla.com") + expect(rows[0]["status"]).to eq("delivered") + expect(rows[0]["delay (min)"]).to eq("1") + expect(rows[1]["status"]).to be_nil + end +end diff --git a/spec/mailers/previews/super_admin_mailer_preview.rb b/spec/mailers/previews/super_admin_mailer_preview.rb new file mode 100644 index 000000000..69ae5cdf0 --- /dev/null +++ b/spec/mailers/previews/super_admin_mailer_preview.rb @@ -0,0 +1,6 @@ +# Preview all emails at http://localhost:3000/rails/mailers/super_admin_mailer +class SuperAdminMailerPreview < ActionMailer::Preview + def dolist_report + SuperAdminMailer.dolist_report("you@beta.gouv.fr", Rails.root.join("spec/fixtures/files/groupe_avec_caracteres_speciaux.csv")) + end +end