Merge pull request #8571 from demarches-simplifiees/US/dolist-delivery-via-api-on-origin

amelioration(dolist): ajoute dolist_api a notre routage des emails.

app/jobs/cron/purge_old_email_event_job.rb

@@ -0,0 +1,7 @@
+class Cron::PurgeOldEmailEventJob < Cron::CronJob
+  self.schedule_expression = "every week at 3:00"
+
+  def perform
+    EmailEvent.outdated.destroy_all
+  end
+end

app/lib/balancer_delivery_method.rb

@@ -47,8 +47,12 @@ class BalancerDeliveryMethod
   def delivery_method(mail)
     return mail[FORCE_DELIVERY_METHOD_HEADER].value.to_sym if force_delivery_method?(mail)
 
-    @delivery_methods
+    compatible_delivery_methods_for(mail)
       .flat_map { |delivery_method, weight| [delivery_method] * weight }
       .sample(random: self.class.random)
   end
+
+  def compatible_delivery_methods_for(mail)
+    @delivery_methods.reject { |delivery_method, _weight| delivery_method.to_s == 'dolist_api' && !Dolist::API.sendable?(mail) }
+  end
 end

app/models/email_event.rb

@@ -10,15 +10,21 @@
 #  to           :string           not null
 #  created_at   :datetime         not null
 #  updated_at   :datetime         not null
+#  message_id   :string
 #
 class EmailEvent < ApplicationRecord
+  RETENTION_DURATION = 1.month
+
   enum status: {
     dispatched: 'dispatched',
     dispatch_error: 'dispatch_error'
   }
 
-  scope :dolist, -> { where(method: 'dolist') }
+  scope :dolist, -> { dolist_smtp.or(dolist_api) }
+  scope :dolist_smtp, -> { where(method: 'dolist_smtp') }
+  scope :dolist_api, -> { where(method: 'dolist_api') }
   scope :sendinblue, -> { where(method: 'sendinblue') }
+  scope :outdated, -> { where("created_at < ?", RETENTION_DURATION.ago) }
 
   class << self
     def create_from_message!(message, status:)
@@ -30,6 +36,7 @@ class EmailEvent < ApplicationRecord
           subject: message.subject || "",
           processed_at: message.date,
           method: ActionMailer::Base.delivery_methods.key(message.delivery_method.class),
+          message_id: message.message_id,
           status:
         )
       rescue StandardError => error

config/environments/production.rb

@@ -80,12 +80,13 @@ Rails.application.configure do
   else
     sendinblue_weigth = ENV.fetch('SENDINBLUE_BALANCING_VALUE') { 0 }.to_i
     dolist_weigth = ENV.fetch('DOLIST_BALANCING_VALUE') { 0 }.to_i
-
+    dolist_api_weight = ENV.fetch('DOLIST_API_BALANCING_VALUE') { 0 }.to_i
     ActionMailer::Base.add_delivery_method :balancer, BalancerDeliveryMethod
     config.action_mailer.balancer_settings = {
       sendinblue: sendinblue_weigth,
       dolist: dolist_weigth,
-      mailjet: 100 - (sendinblue_weigth + dolist_weigth)
+      dolist_api: dolist_api_weight,
+      mailjet: 100 - (sendinblue_weigth + dolist_weigth + dolist_api_weight)
     }
     config.action_mailer.delivery_method = :balancer
   end

config/initializers/dolist.rb

@@ -5,17 +5,29 @@ ActiveSupport.on_load(:action_mailer) do
         mail.from(ENV['DOLIST_NO_REPLY_EMAIL'])
         mail.sender(ENV['DOLIST_NO_REPLY_EMAIL'])
         mail['X-ACCOUNT-ID'] = Rails.application.secrets.dolist[:account_id]
-
         mail['X-Dolist-Sending-Type'] = 'TransactionalService' # send even if the target is not active
 
         super(mail)
       end
     end
+
+    class ApiSender
+      def initialize(mail); end
+
+      def deliver!(mail)
+        response = Dolist::API.new.send_email(mail)
+
+        if response&.dig("Result")
+          mail.message_id = response.dig("Result")
+        else
+          fail "DoList delivery error. Body: #{response}"
+        end
+      end
+    end
   end
 
-  ActionMailer::Base.add_delivery_method :dolist, Dolist::SMTP
-
-  ActionMailer::Base.dolist_settings = {
+  ActionMailer::Base.add_delivery_method :dolist_smtp, Dolist::SMTP
+  ActionMailer::Base.dolist_smtp_settings = {
     user_name: Rails.application.secrets.dolist[:username],
     password: Rails.application.secrets.dolist[:password],
     address: 'smtp.dolist.net',
@@ -23,4 +35,6 @@ ActiveSupport.on_load(:action_mailer) do
     authentication: 'plain',
     enable_starttls_auto: true
   }
+
+  ActionMailer::Base.add_delivery_method :dolist_api, Dolist::ApiSender
 end

db/migrate/20230203134127_add_message_id_to_email_event.rb

@@ -0,0 +1,5 @@
+class AddMessageIdToEmailEvent < ActiveRecord::Migration[6.1]
+  def change
+    add_column :email_events, :message_id, :string
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2023_01_31_172119) do
+ActiveRecord::Schema.define(version: 2023_02_03_134127) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
@@ -407,6 +407,7 @@ ActiveRecord::Schema.define(version: 2023_01_31_172119) do
 
   create_table "email_events", force: :cascade do |t|
     t.datetime "created_at", precision: 6, null: false
+    t.string "message_id"
     t.string "method", null: false
     t.datetime "processed_at"
     t.string "status", null: false

lib/tasks/deployment/20230203155423_rename_email_event_dolist_method.rake

@@ -0,0 +1,20 @@
+namespace :after_party do
+  desc 'Deployment task: rename_email_event_dolist_method'
+  task rename_email_event_dolist_method: :environment do
+    puts "Running deploy task 'rename_email_event_dolist_method'"
+
+    # Put your task implementation HERE.
+    email_events = EmailEvent.where(method: 'dolist')
+    progress = ProgressReport.new(email_events.count)
+    email_events.in_batches do |relation|
+      count = relation.count
+      relation.update_all(method: 'dolist_smtp')
+      progress.inc(count)
+    end
+    progress.finish
+    # Update task as completed.  If you remove the line below, the task will
+    # run with every deploy (or every time you call after_party:run).
+    AfterParty::TaskRecord
+      .create version: AfterParty::TaskRecorder.new(__FILE__).timestamp
+  end
+end

spec/factories/email_event.rb

@@ -6,7 +6,7 @@ FactoryBot.define do
     status { "dispatched" }
 
     trait :dolist do
-      add_attribute(:method) { "dolist" }
+      add_attribute(:method) { "dolist_smtp" }
     end
   end
 end

spec/jobs/cron/purge_old_email_event_job_spec.rb

@@ -0,0 +1,14 @@
+RSpec.describe Cron::PurgeOldEmailEventJob, type: :job do
+  describe 'perform' do
+    subject { Cron::PurgeOldEmailEventJob.perform_now }
+    let(:older_than_retention_duration) { create(:email_event, :dolist, created_at: Time.zone.now.utc - (EmailEvent::RETENTION_DURATION + 1.day)) }
+    let(:more_recent_than_retention_duraiton) { create(:email_event, :dolist, created_at: Time.zone.now.utc - (EmailEvent::RETENTION_DURATION - 1.day)) }
+    before do
+      older_than_retention_duration
+      more_recent_than_retention_duraiton
+    end
+
+    it { expect { subject }.to change { EmailEvent.count }.by(-1) }
+    it { expect { subject }.to change { EmailEvent.exists?(id: older_than_retention_duration.id) }.from(true).to(false) }
+  end
+end

spec/mailers/application_mailer_spec.rb

@@ -26,6 +26,36 @@ RSpec.describe ApplicationMailer, type: :mailer do
     end
   end
 
+  describe 'dealing with Dolist API error' do
+    let(:dossier) { create(:dossier, procedure: create(:simple_procedure)) }
+    before do
+      ActionMailer::Base.delivery_method = :dolist_api
+      api_error_response = { "ResponseStatus": { "ErrorCode": "Forbidden", "Message": "Blocked non authorized request", "Errors": [] } }
+      allow_any_instance_of(Dolist::API).to receive(:send_email).and_return(api_error_response)
+    end
+    subject { DossierMailer.with(dossier:).notify_new_draft.deliver_now }
+
+    it 'raise classic error to retry' do
+      expect { subject }.to raise_error(MailDeliveryError)
+      expect(EmailEvent.dolist_api.dispatch_error.count).to eq(1)
+    end
+  end
+
+  describe 'dealing with Dolist API success' do
+    let(:dossier) { create(:dossier, procedure: create(:simple_procedure)) }
+    let(:message_id) { "29d9b692-0374-4084-8434-d9cddbced205" }
+    before do
+      ActionMailer::Base.delivery_method = :dolist_api
+      api_success_response = { "Result" => message_id }
+      allow_any_instance_of(Dolist::API).to receive(:send_email).and_return(api_success_response)
+    end
+    subject { DossierMailer.with(dossier:).notify_new_draft.deliver_now }
+
+    it 'forward message ID to observer and EmailEvent.create_from_message!' do
+      expect { subject }.to change { EmailEvent.dolist_api.dispatched.where(message_id:).count }.to eq(1)
+    end
+  end
+
   describe 'EmailDeliveryObserver is invoked' do
     let(:user1) { create(:user) }
     let(:user2) { create(:user, email: "your@email.com") }