DGNum/demarches-normaliennes
15
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

fix: dolist module/class mismatch

Browse source
This commit is contained in:
Colin Darie 2023-04-24 16:10:12 +02:00
parent 2ad8ccc310
commit f7a80eb905
No known key found for this signature in database
GPG key ID: 4FB865FDBCA4BCC4
5 changed files with 308 additions and 304 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

538
app/lib/dolist/api.rb
View file

@ -1,314 +1,316 @@
require "support/jsv" require "support/jsv"
class Dolist::API module Dolist
CONTACT_URL = "https://apiv9.dolist.net/v1/contacts/read?AccountID=%{account_id}" class API
EMAIL_LOGS_URL = "https://apiv9.dolist.net/v1/statistics/email/sendings/transactional/search?AccountID=%{account_id}" CONTACT_URL = "https://apiv9.dolist.net/v1/contacts/read?AccountID=%{account_id}"
EMAIL_KEY = 7 EMAIL_LOGS_URL = "https://apiv9.dolist.net/v1/statistics/email/sendings/transactional/search?AccountID=%{account_id}"
STATUS_KEY = 72 EMAIL_KEY = 7
DOLIST_WEB_DASHBOARD = "https://campaign.dolist.net/#/%{account_id}/contacts/%{contact_id}/sendings" STATUS_KEY = 72
EMAIL_MESSAGES_ADRESSES_REPLIES = "https://apiv9.dolist.net/v1/email/messages/addresses/replies?AccountID=%{account_id}" DOLIST_WEB_DASHBOARD = "https://campaign.dolist.net/#/%{account_id}/contacts/%{contact_id}/sendings"
EMAIL_MESSAGES_ADRESSES_PACKSENDERS = "https://apiv9.dolist.net/v1/email/messages/addresses/packsenders?AccountID=%{account_id}" EMAIL_MESSAGES_ADRESSES_REPLIES = "https://apiv9.dolist.net/v1/email/messages/addresses/replies?AccountID=%{account_id}"
EMAIL_SENDING_TRANSACTIONAL = "https://apiv9.dolist.net/v1/email/sendings/transactional?AccountID=%{account_id}" EMAIL_MESSAGES_ADRESSES_PACKSENDERS = "https://apiv9.dolist.net/v1/email/messages/addresses/packsenders?AccountID=%{account_id}"
EMAIL_SENDING_TRANSACTIONAL_ATTACHMENT = "https://apiv9.dolist.net/v1/email/sendings/transactional/attachment?AccountID=%{account_id}" EMAIL_SENDING_TRANSACTIONAL = "https://apiv9.dolist.net/v1/email/sendings/transactional?AccountID=%{account_id}"
EMAIL_SENDING_TRANSACTIONAL_SEARCH = "https://apiv9.dolist.net/v1/email/sendings/transactional/search?AccountID=%{account_id}" EMAIL_SENDING_TRANSACTIONAL_ATTACHMENT = "https://apiv9.dolist.net/v1/email/sendings/transactional/attachment?AccountID=%{account_id}"
EMAIL_SENDING_TRANSACTIONAL_SEARCH = "https://apiv9.dolist.net/v1/email/sendings/transactional/search?AccountID=%{account_id}"
class_attribute :limit_remaining, :limit_reset_at class_attribute :limit_remaining, :limit_reset_at
# those code are just undocumented # those code are just undocumented
IGNORABLE_API_ERROR_CODE = [ IGNORABLE_API_ERROR_CODE = [
"458", "458",
"402" "402"
] ]
# see: https://usercampaign.dolist.net/wp-content/uploads/2022/12/Comprendre-les-Opt-out-tableau-v2.pdf # see: https://usercampaign.dolist.net/wp-content/uploads/2022/12/Comprendre-les-Opt-out-tableau-v2.pdf
IGNORABLE_CONTACT_STATUSES = [ IGNORABLE_CONTACT_STATUSES = [
"4", # Le serveur distant n'accepte pas le mail car il identifie que l’adresse e-mail est en erreur. "4", # Le serveur distant n'accepte pas le mail car il identifie que l’adresse e-mail est en erreur.
"7" # Suite à un envoi, le serveur distant accepte le mail dans un premier temps mais envoie une erreur définitive car l’adresse e-mail est en erreur. L'adresse e-mail n’existe pas ou n'existe plus. "7" # Suite à un envoi, le serveur distant accepte le mail dans un premier temps mais envoie une erreur définitive car l’adresse e-mail est en erreur. L'adresse e-mail n’existe pas ou n'existe plus.
] ]
class << self class << self
def save_rate_limit_headers(headers) def save_rate_limit_headers(headers)
self.limit_remaining = headers["X-Rate-Limit-Remaining"].to_i self.limit_remaining = headers["X-Rate-Limit-Remaining"].to_i
self.limit_reset_at = Time.zone.at(headers["X-Rate-Limit-Reset"].to_i / 1_000) self.limit_reset_at = Time.zone.at(headers["X-Rate-Limit-Reset"].to_i / 1_000)
end
def near_rate_limit?
return if limit_remaining.nil?
limit_remaining < 20 # keep 20 requests for non background API calls
end
def sleep_until_limit_reset
return if limit_reset_at.nil? || limit_reset_at.past?
sleep (limit_reset_at - Time.zone.now).ceil
end
def sendable?(mail)
return false if mail.to.blank? # recipient are mandatory
return false if mail.bcc.present? # no bcc support
# Mail having attachments are not yet supported in our account
mail.attachments.none? { !_1.inline? }
end
end end
def near_rate_limit? def properly_configured?
return if limit_remaining.nil? client_key.present?
limit_remaining < 20 # keep 20 requests for non background API calls
end end
def sleep_until_limit_reset def send_email(mail)
return if limit_reset_at.nil? || limit_reset_at.past? if mail.attachments.any? { !_1.inline? }
return send_email_with_attachment(mail)
end
sleep (limit_reset_at - Time.zone.now).ceil body = { "TransactionalSending": prepare_mail_body(mail) }
url = format_url(EMAIL_SENDING_TRANSACTIONAL)
post(url, body.to_json)
end end
def sendable?(mail) def send_email_with_attachment(mail)
return false if mail.to.blank? # recipient are mandatory uri = URI(format_url(EMAIL_SENDING_TRANSACTIONAL_ATTACHMENT))
return false if mail.bcc.present? # no bcc support
# Mail having attachments are not yet supported in our account request = Net::HTTP::Post.new(uri)
mail.attachments.none? { !_1.inline? }
end
end
def properly_configured? default_headers.each do |key, value|
client_key.present? next if key.to_s == "Content-Type"
end request[key] = value
end
def send_email(mail) boundary = "---011000010111000001101001" # any random string not present in the body
if mail.attachments.any? { !_1.inline? } request.content_type = "multipart/form-data; boundary=#{boundary}"
return send_email_with_attachment(mail)
end
body = { "TransactionalSending": prepare_mail_body(mail) } body = "--#{boundary}\r\n"
url = format_url(EMAIL_SENDING_TRANSACTIONAL) base64_files(mail.attachments).each do |file|
post(url, body.to_json) body << "Content-Disposition: form-data; name=\"#{file.field_name}\"; filename=\"#{file.filename}\"\r\n"
end body << "Content-Type: #{file.mime_type}\r\n"
body << "\r\n"
body << file.content
body << "\r\n"
end
def send_email_with_attachment(mail) body << "\r\n--#{boundary}\r\n"
uri = URI(format_url(EMAIL_SENDING_TRANSACTIONAL_ATTACHMENT)) body << "Content-Disposition: form-data; name=\"TransactionalSending\"\r\n"
body << "Content-Type: text/plain; charset=utf-8\r\n"
request = Net::HTTP::Post.new(uri)
default_headers.each do |key, value|
next if key.to_s == "Content-Type"
request[key] = value
end
boundary = "---011000010111000001101001" # any random string not present in the body
request.content_type = "multipart/form-data; boundary=#{boundary}"
body = "--#{boundary}\r\n"
base64_files(mail.attachments).each do |file|
body << "Content-Disposition: form-data; name=\"#{file.field_name}\"; filename=\"#{file.filename}\"\r\n"
body << "Content-Type: #{file.mime_type}\r\n"
body << "\r\n" body << "\r\n"
body << file.content body << prepare_mail_body(mail).to_jsv
body << "\r\n--#{boundary}--\r\n"
body << "\r\n" body << "\r\n"
request.body = body
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true
response = http.request(request)
if response.body.empty?
fail "Dolist API returned an empty response"
else
JSON.parse(response.body)
end
end end
body << "\r\n--#{boundary}\r\n" def sent_mails(email_address)
body << "Content-Disposition: form-data; name=\"TransactionalSending\"\r\n" contact_id = fetch_contact_id(email_address)
body << "Content-Type: text/plain; charset=utf-8\r\n" if contact_id.nil?
body << "\r\n" Rails.logger.info "Dolist::API: no contact found for email address '#{email_address}'"
body << prepare_mail_body(mail).to_jsv return []
end
body << "\r\n--#{boundary}--\r\n" dolist_messages = fetch_dolist_messages(contact_id)
body << "\r\n"
request.body = body dolist_messages.map { |m| to_sent_mail(email_address, contact_id, m) }
rescue StandardError => e
http = Net::HTTP.new(uri.host, uri.port) Rails.logger.error e.message
http.use_ssl = true []
response = http.request(request)
if response.body.empty?
fail "Dolist API returned an empty response"
else
JSON.parse(response.body)
end
end
def sent_mails(email_address)
contact_id = fetch_contact_id(email_address)
if contact_id.nil?
Rails.logger.info "Dolist::API: no contact found for email address '#{email_address}'"
return []
end end
dolist_messages = fetch_dolist_messages(contact_id) def senders
get format_url(EMAIL_MESSAGES_ADRESSES_PACKSENDERS)
dolist_messages.map { |m| to_sent_mail(email_address, contact_id, m) }
rescue StandardError => e
Rails.logger.error e.message
[]
end
def senders
get format_url(EMAIL_MESSAGES_ADRESSES_PACKSENDERS)
end
def replies
get format_url(EMAIL_MESSAGES_ADRESSES_REPLIES)
end
# Une adresse e-mail peut ne pas être adressable pour différentes raisons (injoignable, plainte pour spam, blocage d’un FAI).
# Dans ce cas l’API d’envoi transactionnel renvoie différentes erreurs.
# Pour connaitre exactement le statut d’une adresse, je vous invite à récupérer le champ 72 du contact à partir de son adresse e-mail avec la méthode https://api.dolist.com/documentation/index.html#/40e7751d00dc3-rechercher-un-contact
#
# La liste des différents statuts est disponible sur https://usercampaign.dolist.net/wp-content/uploads/2022/12/Comprendre-les-Opt-out-tableau-v2.pdf
def fetch_contact_status(email_address)
url = format(Dolist::API::CONTACT_URL, account_id: account_id)
body = {
Query: {
FieldValueList: [{ ID: 7, Value: email_address }],
OutputFieldIDList: [72]
}
}.to_json
post(url, body)["FieldList"].find { _1['ID'] == 72 }['Value']
end
def ignorable_error?(response, mail)
error_code = response&.dig("ResponseStatus", "ErrorCode")
invalid_contact_status = if ignorable_api_error_code?(error_code)
fetch_contact_status(mail.to.first)
else
nil
end
[error_code, invalid_contact_status]
end
private
def ignorable_api_error_code?(api_error_code)
IGNORABLE_API_ERROR_CODE.include?(api_error_code)
end
def ignorable_contact_status?(contact_status)
IGNORABLE_CONTACT_STATUSES.include?(contact_status)
end
def format_url(base)
format(base, account_id: account_id)
end
def sender_id
Rails.cache.fetch("dolist_api_sender_id", expires_in: 1.hour) do
senders.dig("ItemList", 0, "Sender", "ID")
end
end
def get(url)
response = Typhoeus.get(url, headers: default_headers).tap do
self.class.save_rate_limit_headers(_1.headers)
end end
JSON.parse(response.response_body) def replies
end get format_url(EMAIL_MESSAGES_ADRESSES_REPLIES)
def post(url, body)
response = Typhoeus.post(url, body:, headers: default_headers).tap do
self.class.save_rate_limit_headers(_1.headers)
end end
if response.response_body.empty? # Une adresse e-mail peut ne pas être adressable pour différentes raisons (injoignable, plainte pour spam, blocage d’un FAI).
fail "Empty response from Dolist API" # Dans ce cas l’API d’envoi transactionnel renvoie différentes erreurs.
else # Pour connaitre exactement le statut d’une adresse, je vous invite à récupérer le champ 72 du contact à partir de son adresse e-mail avec la méthode https://api.dolist.com/documentation/index.html#/40e7751d00dc3-rechercher-un-contact
#
# La liste des différents statuts est disponible sur https://usercampaign.dolist.net/wp-content/uploads/2022/12/Comprendre-les-Opt-out-tableau-v2.pdf
def fetch_contact_status(email_address)
url = format(Dolist::API::CONTACT_URL, account_id: account_id)
body = {
Query: {
FieldValueList: [{ ID: 7, Value: email_address }],
OutputFieldIDList: [72]
}
}.to_json
post(url, body)["FieldList"].find { _1['ID'] == 72 }['Value']
end
def ignorable_error?(response, mail)
error_code = response&.dig("ResponseStatus", "ErrorCode")
invalid_contact_status = if ignorable_api_error_code?(error_code)
fetch_contact_status(mail.to.first)
else
nil
end
[error_code, invalid_contact_status]
end
private
def ignorable_api_error_code?(api_error_code)
IGNORABLE_API_ERROR_CODE.include?(api_error_code)
end
def ignorable_contact_status?(contact_status)
IGNORABLE_CONTACT_STATUSES.include?(contact_status)
end
def format_url(base)
format(base, account_id: account_id)
end
def sender_id
Rails.cache.fetch("dolist_api_sender_id", expires_in: 1.hour) do
senders.dig("ItemList", 0, "Sender", "ID")
end
end
def get(url)
response = Typhoeus.get(url, headers: default_headers).tap do
self.class.save_rate_limit_headers(_1.headers)
end
JSON.parse(response.response_body) JSON.parse(response.response_body)
end end
end
def default_headers def post(url, body)
{ response = Typhoeus.post(url, body:, headers: default_headers).tap do
"Content-Type": 'application/json', self.class.save_rate_limit_headers(_1.headers)
"Accept": 'application/json', end
"X-API-Key": client_key
}
end
def client_key if response.response_body.empty?
Rails.application.secrets.dolist[:api_key] fail "Empty response from Dolist API"
end else
JSON.parse(response.response_body)
end
end
def account_id def default_headers
Rails.application.secrets.dolist[:account_id] {
end "Content-Type": 'application/json',
"Accept": 'application/json',
# https://api.dolist.com/documentation/index.html#/b3A6Mzg0MTQ0MDc-rechercher-un-contact "X-API-Key": client_key
def fetch_contact_id(email_address)
url = format(CONTACT_URL, account_id: account_id)
body = {
Query: { FieldValueList: [{ ID: EMAIL_KEY, Value: email_address }] }
}.to_json
post(url, body)["ID"]
end
# https://api.dolist.com/documentation/index.html#/b3A6Mzg0MTQ4MDk-recuperer-les-statistiques-des-envois-pour-un-contact
def fetch_dolist_messages(contact_id)
url = format(EMAIL_LOGS_URL, account_id: account_id)
body = { SearchQuery: { ContactID: contact_id } }.to_json
post(url, body)["ItemList"]
end
# see: https://api.dolist.com/documentation/index.html#/7edc2948ba01f-creer-un-envoi-transactionnel
def prepare_mail_body(mail)
{
"Type": "TransactionalService",
"Contact": {
"FieldList": [
{
"ID": EMAIL_KEY,
"Value": mail.to.first
}
]
},
"Message": {
"Name": mail['X-Dolist-Message-Name'].value,
"Subject": mail.subject,
"SenderID": sender_id,
"ForceHttp": false, # ForceHttp : force le tracking http non sécurisé (True/False).
"Format": "html",
"DisableOpenTracking": true, # DisableOpenTracking : désactivation du tracking d'ouverture (True/False).
"IsTrackingValidated": true # IsTrackingValidated : est-ce que le tracking de ce message est validé ? (True/False). Passez la valeur True pour un envoi transactionnel.
},
"MessageContent": {
"SourceCode": mail_source_code(mail),
"EncodingType": "UTF8",
"EnableTrackingDetection": false # EnableTrackingDetection : booléen pour l’activation du tracking personnalisé des liens des messages utilisés lors des précédents envois (True/False).
} }
}
end
def to_sent_mail(email_address, contact_id, dolist_message)
SentMail.new(
from: ENV['DOLIST_NO_REPLY_EMAIL'],
to: email_address,
subject: dolist_message['SendingName'],
delivered_at: Time.zone.parse(dolist_message['SendDate']),
status: status(dolist_message),
service_name: 'Dolist',
external_url: format(DOLIST_WEB_DASHBOARD, account_id: account_id, contact_id: contact_id)
)
end
def status(dolist_message)
case dolist_message.fetch_values('Status', 'IsDelivered')
in ['Sent', true]
"delivered"
in ['Sent', false]
"sent (delivered ?)"
in [status, _]
status
end end
end
def mail_source_code(mail) def client_key
if mail.html_part.nil? && mail.text_part.nil? Rails.application.secrets.dolist[:api_key]
mail.decoded
else
mail.html_part.body.decoded
end end
end
def base64_files(attachments) def account_id
attachments.map do |attachment| Rails.application.secrets.dolist[:account_id]
raise ArgumentError, "Dolist API does not support non PDF attachments. Given #{attachment.filename} which has mime_type=#{attachment.mime_type}" unless attachment.mime_type == "application/pdf" end
field_name = File.basename(attachment.filename, File.extname(attachment.filename)) # https://api.dolist.com/documentation/index.html#/b3A6Mzg0MTQ0MDc-rechercher-un-contact
attachment_content = attachment.body.decoded def fetch_contact_id(email_address)
attachment_base64 = Base64.strict_encode64(attachment_content) url = format(CONTACT_URL, account_id: account_id)
Dolist::Base64File.new(field_name:, filename: attachment.filename, mime_type: attachment.mime_type, content: attachment_base64) body = {
Query: { FieldValueList: [{ ID: EMAIL_KEY, Value: email_address }] }
}.to_json
post(url, body)["ID"]
end
# https://api.dolist.com/documentation/index.html#/b3A6Mzg0MTQ4MDk-recuperer-les-statistiques-des-envois-pour-un-contact
def fetch_dolist_messages(contact_id)
url = format(EMAIL_LOGS_URL, account_id: account_id)
body = { SearchQuery: { ContactID: contact_id } }.to_json
post(url, body)["ItemList"]
end
# see: https://api.dolist.com/documentation/index.html#/7edc2948ba01f-creer-un-envoi-transactionnel
def prepare_mail_body(mail)
{
"Type": "TransactionalService",
"Contact": {
"FieldList": [
{
"ID": EMAIL_KEY,
"Value": mail.to.first
}
]
},
"Message": {
"Name": mail['X-Dolist-Message-Name'].value,
"Subject": mail.subject,
"SenderID": sender_id,
"ForceHttp": false, # ForceHttp : force le tracking http non sécurisé (True/False).
"Format": "html",
"DisableOpenTracking": true, # DisableOpenTracking : désactivation du tracking d'ouverture (True/False).
"IsTrackingValidated": true # IsTrackingValidated : est-ce que le tracking de ce message est validé ? (True/False). Passez la valeur True pour un envoi transactionnel.
},
"MessageContent": {
"SourceCode": mail_source_code(mail),
"EncodingType": "UTF8",
"EnableTrackingDetection": false # EnableTrackingDetection : booléen pour l’activation du tracking personnalisé des liens des messages utilisés lors des précédents envois (True/False).
}
}
end
def to_sent_mail(email_address, contact_id, dolist_message)
SentMail.new(
from: ENV['DOLIST_NO_REPLY_EMAIL'],
to: email_address,
subject: dolist_message['SendingName'],
delivered_at: Time.zone.parse(dolist_message['SendDate']),
status: status(dolist_message),
service_name: 'Dolist',
external_url: format(DOLIST_WEB_DASHBOARD, account_id: account_id, contact_id: contact_id)
)
end
def status(dolist_message)
case dolist_message.fetch_values('Status', 'IsDelivered')
in ['Sent', true]
"delivered"
in ['Sent', false]
"sent (delivered ?)"
in [status, _]
status
end
end
def mail_source_code(mail)
if mail.html_part.nil? && mail.text_part.nil?
mail.decoded
else
mail.html_part.body.decoded
end
end
def base64_files(attachments)
attachments.map do |attachment|
raise ArgumentError, "Dolist API does not support non PDF attachments. Given #{attachment.filename} which has mime_type=#{attachment.mime_type}" unless attachment.mime_type == "application/pdf"
field_name = File.basename(attachment.filename, File.extname(attachment.filename))
attachment_content = attachment.body.decoded
attachment_base64 = Base64.strict_encode64(attachment_content)
Dolist::Base64File.new(field_name:, filename: attachment.filename, mime_type: attachment.mime_type, content: attachment_base64)
end
end end
end end
end end

21
app/lib/dolist/api_sender.rb Normal file
View file

@ -0,0 +1,21 @@
module Dolist
class APISender
def initialize(mail); end
def deliver!(mail)
client = Dolist::API.new
response = client.send_email(mail)
if response&.dig("Result")
mail.message_id = response.dig("Result")
else
_, invalid_contact_status = client.ignorable_error?(response, mail)
if invalid_contact_status
raise Dolist::IgnorableError.new("DoList delivery error. contact unreachable: #{invalid_contact_status}")
else
fail "DoList delivery error. Body: #{response}"
end
end
end
end
end

3
app/lib/dolist/ignorable_error.rb Normal file
View file

@ -0,0 +1,3 @@
module Dolist
class IgnorableError < StandardError; end
end

12
app/lib/dolist/smtp.rb Normal file
View file

@ -0,0 +1,12 @@
module Dolist
class SMTP < ::Mail::SMTP
def deliver!(mail)
mail.from(ENV['DOLIST_NO_REPLY_EMAIL'])
mail.sender(ENV['DOLIST_NO_REPLY_EMAIL'])
mail['X-ACCOUNT-ID'] = Rails.application.secrets.dolist[:account_id]
mail['X-Dolist-Sending-Type'] = 'TransactionalService' # send even if the target is not active
super(mail)
end
end
end

38
config/initializers/dolist.rb
View file

@ -1,39 +1,5 @@
ActiveSupport.on_load(:action_mailer) do ActiveSupport.on_load(:action_mailer) do
module Dolist require "dolist/smtp"
class IgnorableError < StandardError
end
class SMTP < ::Mail::SMTP
def deliver!(mail)
mail.from(ENV['DOLIST_NO_REPLY_EMAIL'])
mail.sender(ENV['DOLIST_NO_REPLY_EMAIL'])
mail['X-ACCOUNT-ID'] = Rails.application.secrets.dolist[:account_id]
mail['X-Dolist-Sending-Type'] = 'TransactionalService' # send even if the target is not active
super(mail)
end
end
class ApiSender
def initialize(mail); end
def deliver!(mail)
client = Dolist::API.new
response = client.send_email(mail)
if response&.dig("Result")
mail.message_id = response.dig("Result")
else
_, invalid_contact_status = client.ignorable_error?(response, mail)
if invalid_contact_status
raise Dolist::IgnorableError.new("DoList delivery error. contact unreachable: #{invalid_contact_status}")
else
fail "DoList delivery error. Body: #{response}"
end
end
end
end
end
ActionMailer::Base.add_delivery_method :dolist_smtp, Dolist::SMTP ActionMailer::Base.add_delivery_method :dolist_smtp, Dolist::SMTP
ActionMailer::Base.dolist_smtp_settings = { ActionMailer::Base.dolist_smtp_settings = {
@ -45,5 +11,5 @@ ActiveSupport.on_load(:action_mailer) do
enable_starttls_auto: true enable_starttls_auto: true
} }
ActionMailer::Base.add_delivery_method :dolist_api, Dolist::ApiSender ActionMailer::Base.add_delivery_method :dolist_api, Dolist::APISender
end end