diff --git a/app/controllers/manager/email_events_controller.rb b/app/controllers/manager/email_events_controller.rb index bd7b59ff7..4a4cb74c2 100644 --- a/app/controllers/manager/email_events_controller.rb +++ b/app/controllers/manager/email_events_controller.rb @@ -1,4 +1,25 @@ module Manager class EmailEventsController < Manager::ApplicationController + def index + @dolist_enabled = Dolist::API.new.properly_configured? + + super + end + + def generate_dolist_report + email = current_super_admin.email + + DolistReportJob.perform_later(email) + + respond_to do |format| + @message = "Le rapport sera envoyé sur #{email}. Il peut prendre plus d'1h pour être généré." + + format.turbo_stream + + format.html do + redirect_to manager_email_events_path, notice: @message + end + end + end end end diff --git a/app/jobs/dolist_report_job.rb b/app/jobs/dolist_report_job.rb new file mode 100644 index 000000000..8a7b38efe --- /dev/null +++ b/app/jobs/dolist_report_job.rb @@ -0,0 +1,55 @@ +class DolistReportJob < ApplicationJob + # Consolidate random recent emails dispatched to Dolist with their statuses + # and send a report by email. + def perform(report_to, sample_size = 1000) + events = EmailEvent.dolist.dispatched.where(processed_at: 2.weeks.ago..).order("RANDOM()").limit(sample_size) + + data = CSV.generate(headers: true) do |csv| + column_names = ["dispatched_at", "subject", "domain", "status", "delivered_at", "delay (min)"] + csv << column_names + + events.each do |event| + report_event(csv, event) + end + end + + tempfile = Tempfile.new("dolist_report.csv") + tempfile.write(data) + tempfile.rewind + + SuperAdminMailer.dolist_report(report_to, tempfile.path).deliver_now + ensure + tempfile&.unlink + end + + private + + def report_event(csv, event) + wait_if_api_limit_reached + + sent_email = event.match_dolist_email + + delay = if sent_email + (sent_email.delivered_at.to_i - event.processed_at.to_i) / 60 + end + + csv << [ + event.processed_at, + event.subject, + event.domain, + sent_email&.status, + sent_email&.delivered_at, + delay + ] + rescue StandardError => error + Sentry.capture_exception(error, extra: { event_id: event.id, api_limit_remaining: Dolist::API.limit_remaining, api_limit_reset_at: Dolist::API.limit_reset_at }) + end + + def wait_if_api_limit_reached + return unless Dolist::API.near_rate_limit? + + Rails.logger.info("Dolist API rate limit reached, sleep until #{Dolist::API.limit_reset_at}") + + Dolist::API.sleep_until_limit_reset + end +end diff --git a/app/lib/dolist/api.rb b/app/lib/dolist/api.rb index b18ec5e49..8ae14c2f6 100644 --- a/app/lib/dolist/api.rb +++ b/app/lib/dolist/api.rb @@ -4,6 +4,27 @@ class Dolist::API EMAIL_KEY = 7 DOLIST_WEB_DASHBOARD = "https://campaign.dolist.net/#/%{account_id}/contacts/%{contact_id}/sendings" + class_attribute :limit_remaining, :limit_reset_at + + class << self + def save_rate_limit_headers(headers) + self.limit_remaining = headers["X-Rate-Limit-Remaining"].to_i + self.limit_reset_at = Time.zone.at(headers["X-Rate-Limit-Reset"].to_i / 1_000) + end + + def near_rate_limit? + return if limit_remaining.nil? + + limit_remaining < 20 # keep 20 requests for non background API calls + end + + def sleep_until_limit_reset + return if limit_reset_at.nil? || limit_reset_at.past? + + sleep (limit_reset_at - Time.zone.now).ceil + end + end + def properly_configured? client_key.present? end @@ -49,9 +70,7 @@ class Dolist::API Query: { FieldValueList: [{ ID: EMAIL_KEY, Value: email_address }] } }.to_json - response = Typhoeus.post(url, body: body, headers: headers) - - JSON.parse(response.response_body)["ID"] + post(url, body)["ID"] end # https://api.dolist.com/documentation/index.html#/b3A6Mzg0MTQ4MDk-recuperer-les-statistiques-des-envois-pour-un-contact @@ -60,9 +79,15 @@ class Dolist::API body = { SearchQuery: { ContactID: contact_id } }.to_json - response = Typhoeus.post(url, body: body, headers: headers) + post(url, body)["ItemList"] + end - JSON.parse(response.response_body)['ItemList'] + def post(url, body) + response = Typhoeus.post(url, body:, headers:).tap do + self.class.save_rate_limit_headers(_1.headers) + end + + JSON.parse(response.response_body) end def to_sent_mail(email_address, contact_id, dolist_message) diff --git a/app/mailers/super_admin_mailer.rb b/app/mailers/super_admin_mailer.rb new file mode 100644 index 000000000..75c838652 --- /dev/null +++ b/app/mailers/super_admin_mailer.rb @@ -0,0 +1,7 @@ +class SuperAdminMailer < ApplicationMailer + def dolist_report(to, csv_path) + attachments["dolist_report.csv"] = File.read(csv_path) + + mail(to: to, subject: "Dolist report", body: "Ci-joint le rapport d'emails récents envoyés via Dolist.") + end +end diff --git a/app/models/email_event.rb b/app/models/email_event.rb index 6a66ff907..aa13275a6 100644 --- a/app/models/email_event.rb +++ b/app/models/email_event.rb @@ -17,6 +17,9 @@ class EmailEvent < ApplicationRecord dispatch_error: 'dispatch_error' } + scope :dolist, -> { where(method: 'dolist') } + scope :sendinblue, -> { where(method: 'sendinblue') } + class << self def create_from_message!(message, status:) to = message.to || ["unset"] # no recipients when error occurs *before* setting to: in the mailer @@ -34,4 +37,15 @@ class EmailEvent < ApplicationRecord end end end + + def match_dolist_email + return if to == "unset" + + # subjects does not match, so compare to event time with tolerance + Dolist::API.new.sent_mails(to).sort_by(&:delivered_at).find { (processed_at..processed_at + 1.hour).cover?(_1.delivered_at) } + end + + def domain + to.split("@").last + end end diff --git a/app/views/manager/application/_index_footer.html.erb b/app/views/manager/application/_index_footer.html.erb new file mode 100644 index 000000000..e69de29bb diff --git a/app/views/manager/application/index.html.erb b/app/views/manager/application/index.html.erb new file mode 100644 index 000000000..9201f1d60 --- /dev/null +++ b/app/views/manager/application/index.html.erb @@ -0,0 +1,46 @@ +<%# +# Override original index template augmented with an optional "index_footer" partial. +# Update it from the template with the original content: +# https://github.com/thoughtbot/administrate/blob/main/app/views/administrate/application/index.html.erb + +# Index +This view is the template for the index page. +It is responsible for rendering the search bar, header and pagination. +It renders the `_table` partial to display details about the resources. +## Local variables: +- `page`: + An instance of [Administrate::Page::Collection][1]. + Contains helper methods to help display a table, + and knows which attributes should be displayed in the resource's table. +- `resources`: + An instance of `ActiveRecord::Relation` containing the resources + that match the user's search criteria. + By default, these resources are passed to the table partial to be displayed. +- `search_term`: + A string containing the term the user has searched for, if any. +- `show_search_bar`: + A boolean that determines if the search bar should be shown. +[1]: http://www.rubydoc.info/gems/administrate/Administrate/Page/Collection +%> +<%= + render("index_header", + resources: resources, + search_term: search_term, + page: page, + show_search_bar: show_search_bar, + ) +%> + +
+ <%= render( + "collection", + collection_presenter: page, + collection_field_name: resource_name, + page: page, + resources: resources, + table_title: "page-title" + ) %> + <%= render("pagination", resources: resources) %> +
+ +<%= render("index_footer") %> diff --git a/app/views/manager/email_events/_index_footer.html.erb b/app/views/manager/email_events/_index_footer.html.erb new file mode 100644 index 000000000..aa4f835cd --- /dev/null +++ b/app/views/manager/email_events/_index_footer.html.erb @@ -0,0 +1,7 @@ + diff --git a/app/views/manager/email_events/generate_dolist_report.turbo_stream.haml b/app/views/manager/email_events/generate_dolist_report.turbo_stream.haml new file mode 100644 index 000000000..efe704452 --- /dev/null +++ b/app/views/manager/email_events/generate_dolist_report.turbo_stream.haml @@ -0,0 +1,2 @@ += turbo_stream.morph "dolist-report-form" do + = @message diff --git a/config/routes.rb b/config/routes.rb index aedd12400..87b74e2b1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -65,7 +65,9 @@ Rails.application.routes.draw do resources :team_accounts, only: [:index, :show] - resources :email_events, only: [:index, :show] + resources :email_events, only: [:index, :show] do + post :generate_dolist_report, on: :collection + end resources :dubious_procedures, only: [:index] resources :outdated_procedures, only: [:index] do diff --git a/spec/factories/email_event.rb b/spec/factories/email_event.rb new file mode 100644 index 000000000..198851eec --- /dev/null +++ b/spec/factories/email_event.rb @@ -0,0 +1,12 @@ +FactoryBot.define do + factory :email_event do + to { "user@email.com" } + subject { "Thank you" } + processed_at { Time.zone.now } + status { "dispatched" } + + trait :dolist do + add_attribute(:method) { "dolist" } + end + end +end diff --git a/spec/jobs/dolist_report_job_spec.rb b/spec/jobs/dolist_report_job_spec.rb new file mode 100644 index 000000000..18b236d91 --- /dev/null +++ b/spec/jobs/dolist_report_job_spec.rb @@ -0,0 +1,40 @@ +require 'rails_helper' + +RSpec.describe DolistReportJob, type: :job do + let(:event1) { create(:email_event, :dolist, :dispatched, to: "you@blabla.com", processed_at: 1.minute.ago) } + let(:event2) { create(:email_event, :dolist, :dispatched) } + + before do + emails = [ + SentMail.new( + to: event1.to, + status: "delivered", + delivered_at: event1.processed_at + 1.minute + ) + ] + + allow_any_instance_of(Dolist::API).to receive(:sent_mails).with(event1.to).and_return(emails) + allow_any_instance_of(Dolist::API).to receive(:sent_mails).with(event2.to).and_return([]) + end + + subject(:perform_job) { described_class.perform_now("you@rocks.com") } + + it "generates a csv file and send it by email" do + perform_job + + email = ActionMailer::Base.deliveries.last + expect(email.to).to eq(["you@rocks.com"]) + expect(email.attachments[0].filename).to eq("dolist_report.csv") + + csv = CSV.parse(email.attachments[0].body.decoded, headers: true) + expect(csv.size).to eq(2) + + # events were processed randomly, go back to a deterministic order + rows = csv.sort_by { _1["dispatched_at"] } + + expect(rows[0]["domain"]).to eq("blabla.com") + expect(rows[0]["status"]).to eq("delivered") + expect(rows[0]["delay (min)"]).to eq("1") + expect(rows[1]["status"]).to be_nil + end +end diff --git a/spec/mailers/previews/super_admin_mailer_preview.rb b/spec/mailers/previews/super_admin_mailer_preview.rb new file mode 100644 index 000000000..69ae5cdf0 --- /dev/null +++ b/spec/mailers/previews/super_admin_mailer_preview.rb @@ -0,0 +1,6 @@ +# Preview all emails at http://localhost:3000/rails/mailers/super_admin_mailer +class SuperAdminMailerPreview < ActionMailer::Preview + def dolist_report + SuperAdminMailer.dolist_report("you@beta.gouv.fr", Rails.root.join("spec/fixtures/files/groupe_avec_caracteres_speciaux.csv")) + end +end