From 41c196f2ec47949da0e4729c54c4126cdb19bf25 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Thu, 2 Feb 2023 14:50:26 +0100 Subject: [PATCH 1/8] feat(dolist): send email from API Co-authored-by: mfo --- app/lib/dolist/api.rb | 84 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/app/lib/dolist/api.rb b/app/lib/dolist/api.rb index 8ae14c2f6..3fbbe354e 100644 --- a/app/lib/dolist/api.rb +++ b/app/lib/dolist/api.rb @@ -3,6 +3,10 @@ class Dolist::API EMAIL_LOGS_URL = "https://apiv9.dolist.net/v1/statistics/email/sendings/transactional/search?AccountID=%{account_id}" EMAIL_KEY = 7 DOLIST_WEB_DASHBOARD = "https://campaign.dolist.net/#/%{account_id}/contacts/%{contact_id}/sendings" + EMAIL_MESSAGES_ADRESSES_REPLIES = "https://apiv9.dolist.net/v1/email/messages/addresses/replies?AccountID=%{account_id}" + EMAIL_MESSAGES_ADRESSES_PACKSENDERS = "https://apiv9.dolist.net/v1/email/messages/addresses/packsenders?AccountID=%{account_id}" + EMAIL_SENDING_TRANSACTIONAL = "https://apiv9.dolist.net/v1/email/sendings/transactional?AccountID=%{account_id}" + EMAIL_SENDING_TRANSACTIONAL_SEARCH = "https://apiv9.dolist.net/v1/email/sendings/transactional/search?AccountID=%{account_id}" class_attribute :limit_remaining, :limit_reset_at @@ -29,6 +33,40 @@ class Dolist::API client_key.present? end + def send_email(mail) + url = format(EMAIL_SENDING_TRANSACTIONAL, account_id: account_id) + + body = { + "TransactionalSending": { + "Type": "TransactionalService", + "Contact": { + "FieldList": [ + { + "ID": EMAIL_KEY, + "Value": mail.to.first + } + ] + }, + "Message": { + "Name": mail['X-Dolist-Message-Name'].value, + "Subject": mail.subject, + "SenderID": sender_id, + "ForceHttp": true, + "Format": "html", + "DisableOpenTracking": true, + "IsTrackingValidated": true + }, + "MessageContent": { + "SourceCode": mail_source_code(mail), + "EncodingType": "UTF8", + "EnableTrackingDetection": false + } + } + } + + post(url, body.to_json) + end + def sent_mails(email_address) contact_id = fetch_contact_id(email_address) if contact_id.nil? @@ -44,8 +82,38 @@ class Dolist::API [] end + def senders + url = format(EMAIL_MESSAGES_ADRESSES_PACKSENDERS, account_id: account_id) + get(url) + end + + def replies + url = format(EMAIL_MESSAGES_ADRESSES_REPLIES, account_id: account_id) + get(url) + end + private + def sender_id + senders.dig("ItemList", 0, "Sender", "ID") + end + + def get(url) + response = Typhoeus.get(url, headers:).tap do + self.class.save_rate_limit_headers(_1.headers) + end + + JSON.parse(response.response_body) + end + + def post(url, body) + response = Typhoeus.post(url, body:, headers:).tap do + self.class.save_rate_limit_headers(_1.headers) + end + + JSON.parse(response.response_body) + end + def headers { "Content-Type": 'application/json', @@ -82,14 +150,6 @@ class Dolist::API post(url, body)["ItemList"] end - def post(url, body) - response = Typhoeus.post(url, body:, headers:).tap do - self.class.save_rate_limit_headers(_1.headers) - end - - JSON.parse(response.response_body) - end - def to_sent_mail(email_address, contact_id, dolist_message) SentMail.new( from: ENV['DOLIST_NO_REPLY_EMAIL'], @@ -112,4 +172,12 @@ class Dolist::API status end end + + def mail_source_code(mail) + if mail.html_part.nil? && mail.text_part.nil? + mail.decoded + else + mail.html_part.body.decoded + end + end end From b0b7114c3bea301143e4c2ef58189fc15649ac6c Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Thu, 2 Feb 2023 14:50:43 +0100 Subject: [PATCH 2/8] feat: jsv support for primitives --- lib/support/jsv.rb | 9 +++ lib/support/jsv/core_ext/array.rb | 5 ++ lib/support/jsv/core_ext/false_class.rb | 5 ++ lib/support/jsv/core_ext/hash.rb | 11 ++++ lib/support/jsv/core_ext/number.rb | 5 ++ lib/support/jsv/core_ext/string.rb | 12 ++++ lib/support/jsv/core_ext/symbol.rb | 5 ++ lib/support/jsv/core_ext/true_class.rb | 5 ++ spec/lib/support/jsv_spec.rb | 75 +++++++++++++++++++++++++ 9 files changed, 132 insertions(+) create mode 100644 lib/support/jsv.rb create mode 100644 lib/support/jsv/core_ext/array.rb create mode 100644 lib/support/jsv/core_ext/false_class.rb create mode 100644 lib/support/jsv/core_ext/hash.rb create mode 100644 lib/support/jsv/core_ext/number.rb create mode 100644 lib/support/jsv/core_ext/string.rb create mode 100644 lib/support/jsv/core_ext/symbol.rb create mode 100644 lib/support/jsv/core_ext/true_class.rb create mode 100644 spec/lib/support/jsv_spec.rb diff --git a/lib/support/jsv.rb b/lib/support/jsv.rb new file mode 100644 index 000000000..0d09cf69d --- /dev/null +++ b/lib/support/jsv.rb @@ -0,0 +1,9 @@ +# spec by dolist https://api.dolist.com/ConvertJSON-to-JSV.html +# +require_relative "jsv/core_ext/array" +require_relative "jsv/core_ext/false_class" +require_relative "jsv/core_ext/hash" +require_relative "jsv/core_ext/number" +require_relative "jsv/core_ext/string" +require_relative "jsv/core_ext/symbol" +require_relative "jsv/core_ext/true_class" diff --git a/lib/support/jsv/core_ext/array.rb b/lib/support/jsv/core_ext/array.rb new file mode 100644 index 000000000..67c262513 --- /dev/null +++ b/lib/support/jsv/core_ext/array.rb @@ -0,0 +1,5 @@ +class Array + def to_jsv + "[" + reject(&:nil?).map(&:to_jsv).join(",") + "]" + end +end diff --git a/lib/support/jsv/core_ext/false_class.rb b/lib/support/jsv/core_ext/false_class.rb new file mode 100644 index 000000000..171eb991b --- /dev/null +++ b/lib/support/jsv/core_ext/false_class.rb @@ -0,0 +1,5 @@ +class TrueClass + def to_jsv + "True" + end +end diff --git a/lib/support/jsv/core_ext/hash.rb b/lib/support/jsv/core_ext/hash.rb new file mode 100644 index 000000000..3fc326552 --- /dev/null +++ b/lib/support/jsv/core_ext/hash.rb @@ -0,0 +1,11 @@ +class Hash + def to_jsv + js_array = filter_map do |key, value| + next if value.nil? # skip nil values + + "#{key.to_jsv}:#{value.to_jsv}" + end + + "{" + js_array.join(",") + "}" + end +end diff --git a/lib/support/jsv/core_ext/number.rb b/lib/support/jsv/core_ext/number.rb new file mode 100644 index 000000000..def9ee63d --- /dev/null +++ b/lib/support/jsv/core_ext/number.rb @@ -0,0 +1,5 @@ +class Numeric + def to_jsv + self + end +end diff --git a/lib/support/jsv/core_ext/string.rb b/lib/support/jsv/core_ext/string.rb new file mode 100644 index 000000000..fb1cc0a8a --- /dev/null +++ b/lib/support/jsv/core_ext/string.rb @@ -0,0 +1,12 @@ +class String + JSV_REGEX_SPECIAL_CHARS = /[\[\]\{\}"\,]/.freeze + + def to_jsv + double_quoted = self.gsub('"', '""') + if match?(JSV_REGEX_SPECIAL_CHARS) + "\"#{double_quoted}\"" + else + double_quoted + end + end +end diff --git a/lib/support/jsv/core_ext/symbol.rb b/lib/support/jsv/core_ext/symbol.rb new file mode 100644 index 000000000..44c26d6c3 --- /dev/null +++ b/lib/support/jsv/core_ext/symbol.rb @@ -0,0 +1,5 @@ +class Symbol + def to_jsv + to_s.to_jsv + end +end diff --git a/lib/support/jsv/core_ext/true_class.rb b/lib/support/jsv/core_ext/true_class.rb new file mode 100644 index 000000000..090ebe9f7 --- /dev/null +++ b/lib/support/jsv/core_ext/true_class.rb @@ -0,0 +1,5 @@ +class FalseClass + def to_jsv + "False" + end +end diff --git a/spec/lib/support/jsv_spec.rb b/spec/lib/support/jsv_spec.rb new file mode 100644 index 000000000..f3b9670d0 --- /dev/null +++ b/spec/lib/support/jsv_spec.rb @@ -0,0 +1,75 @@ +require_relative "../../../lib/support/jsv" + +describe ".to_jsv support" do + it "converts a Hash to JSV" do + expect({}.to_jsv).to eq("{}") + expect({ "a" => "b" }.to_jsv).to eq("{a:b}") + end + + it "converts an Array to JSV" do + expect([].to_jsv).to eq("[]") + expect(["a", "b"].to_jsv).to eq("[a,b]") + end + + it "converts a String to JSV" do + expect("".to_jsv).to eq("") + expect("a".to_jsv).to eq("a") + + # escape special characters + expect("a[b".to_jsv).to eq('"a[b"') + expect("a]b".to_jsv).to eq('"a]b"') + expect("a,b".to_jsv).to eq('"a,b"') + expect("a{b".to_jsv).to eq('"a{b"') + expect("a}b".to_jsv).to eq('"a}b"') + expect('a"b'.to_jsv).to eq('"a""b"') + end + + it "skip null values" do + expect({ "a" => nil }.to_jsv).to eq("{}") + expect([nil].to_jsv).to eq("[]") + end + + it "converts symbols like strings" do + expect({ a: :b }.to_jsv).to eq("{a:b}") + end + + it "converts booleans" do + expect(true.to_jsv).to eq("True") + expect(false.to_jsv).to eq("False") + end + + it "converts numbers" do + expect(1.to_jsv).to eq(1) + expect(3.14.to_jsv).to eq(3.14) + end + + it "converts nested structures" do + expect({ "a" => { "b" => "c" } }.to_jsv).to eq("{a:{b:c}}") + expect({ "a" => ["b", "c"] }.to_jsv).to eq("{a:[b,c]}") + end + + it "converts relastic structures" do + hash = { + Type: :TransactionalService, + "Contact": { + "FieldList": [ + { + "ID": 3, + "Value": "glou[0]" + } + ] + }, + "Message": { + "Subject": "You, and me", + "ForceHttp": true, + "IsTrackingValidated": false, + "IgnoreMe": nil, + "SourceCode": "

Un mail tout simple pour commencer

", + "SourceWithQuote": 'Ceci est une double quote: "' + } + } + + expected = '{Type:TransactionalService,Contact:{FieldList:[{ID:3,Value:"glou[0]"}]},Message:{Subject:"You, and me",ForceHttp:True,IsTrackingValidated:False,SourceCode:

Un mail tout simple pour commencer

,SourceWithQuote:"Ceci est une double quote: """}}' + expect(hash.to_jsv).to eq(expected) + end +end From 61cd9aa8c79e3924cbe074204abc096ca9b5b747 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Thu, 2 Feb 2023 11:18:10 +0100 Subject: [PATCH 3/8] feat(dolist): send mail having attachments --- app/lib/dolist/api.rb | 146 +++++++++++++++++++++++++--------- app/lib/dolist/base64_file.rb | 3 + 2 files changed, 112 insertions(+), 37 deletions(-) create mode 100644 app/lib/dolist/base64_file.rb diff --git a/app/lib/dolist/api.rb b/app/lib/dolist/api.rb index 3fbbe354e..0faa9f38e 100644 --- a/app/lib/dolist/api.rb +++ b/app/lib/dolist/api.rb @@ -1,3 +1,5 @@ +require "support/jsv" + class Dolist::API CONTACT_URL = "https://apiv9.dolist.net/v1/contacts/read?AccountID=%{account_id}" EMAIL_LOGS_URL = "https://apiv9.dolist.net/v1/statistics/email/sendings/transactional/search?AccountID=%{account_id}" @@ -6,6 +8,7 @@ class Dolist::API EMAIL_MESSAGES_ADRESSES_REPLIES = "https://apiv9.dolist.net/v1/email/messages/addresses/replies?AccountID=%{account_id}" EMAIL_MESSAGES_ADRESSES_PACKSENDERS = "https://apiv9.dolist.net/v1/email/messages/addresses/packsenders?AccountID=%{account_id}" EMAIL_SENDING_TRANSACTIONAL = "https://apiv9.dolist.net/v1/email/sendings/transactional?AccountID=%{account_id}" + EMAIL_SENDING_TRANSACTIONAL_ATTACHMENT = "https://apiv9.dolist.net/v1/email/sendings/transactional/attachment?AccountID=%{account_id}" EMAIL_SENDING_TRANSACTIONAL_SEARCH = "https://apiv9.dolist.net/v1/email/sendings/transactional/search?AccountID=%{account_id}" class_attribute :limit_remaining, :limit_reset_at @@ -34,39 +37,62 @@ class Dolist::API end def send_email(mail) - url = format(EMAIL_SENDING_TRANSACTIONAL, account_id: account_id) + if mail.attachments.any? { !_1.inline? } + return send_email_with_attachment(mail) + end - body = { - "TransactionalSending": { - "Type": "TransactionalService", - "Contact": { - "FieldList": [ - { - "ID": EMAIL_KEY, - "Value": mail.to.first - } - ] - }, - "Message": { - "Name": mail['X-Dolist-Message-Name'].value, - "Subject": mail.subject, - "SenderID": sender_id, - "ForceHttp": true, - "Format": "html", - "DisableOpenTracking": true, - "IsTrackingValidated": true - }, - "MessageContent": { - "SourceCode": mail_source_code(mail), - "EncodingType": "UTF8", - "EnableTrackingDetection": false - } - } - } + body = { "TransactionalSending": prepare_mail_body(mail) } + url = format_url(EMAIL_SENDING_TRANSACTIONAL) post(url, body.to_json) end + def send_email_with_attachment(mail) + uri = URI(format_url(EMAIL_SENDING_TRANSACTIONAL_ATTACHMENT)) + + request = Net::HTTP::Post.new(uri) + + default_headers.each do |key, value| + next if key.to_s == "Content-Type" + request[key] = value + end + + boundary = "---011000010111000001101001" # any random string not present in the body + request.content_type = "multipart/form-data; boundary=#{boundary}" + + body = "--#{boundary}\r\n" + + base64_files(mail.attachments).each do |file| + body << "Content-Disposition: form-data; name=\"#{file.field_name}\"; filename=\"#{file.filename}\"\r\n" + body << "Content-Type: #{file.mime_type}\r\n" + body << "\r\n" + body << file.content + body << "\r\n" + end + + body << "\r\n--#{boundary}\r\n" + body << "Content-Disposition: form-data; name=\"TransactionalSending\"\r\n" + body << "Content-Type: text/plain; charset=utf-8\r\n" + body << "\r\n" + body << prepare_mail_body(mail).to_jsv + + body << "\r\n--#{boundary}--\r\n" + body << "\r\n" + + request.body = body + + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + + response = http.request(request) + + if response.body.empty? + fail "Dolist API returned an empty response" + else + JSON.parse(response.body) + end + end + def sent_mails(email_address) contact_id = fetch_contact_id(email_address) if contact_id.nil? @@ -83,23 +109,25 @@ class Dolist::API end def senders - url = format(EMAIL_MESSAGES_ADRESSES_PACKSENDERS, account_id: account_id) - get(url) + get format_url(EMAIL_MESSAGES_ADRESSES_PACKSENDERS) end def replies - url = format(EMAIL_MESSAGES_ADRESSES_REPLIES, account_id: account_id) - get(url) + get format_url(EMAIL_MESSAGES_ADRESSES_REPLIES) end private + def format_url(base) + format(base, account_id: account_id) + end + def sender_id - senders.dig("ItemList", 0, "Sender", "ID") + @sender_id ||= senders.dig("ItemList", 0, "Sender", "ID") end def get(url) - response = Typhoeus.get(url, headers:).tap do + response = Typhoeus.get(url, headers: default_headers).tap do self.class.save_rate_limit_headers(_1.headers) end @@ -107,14 +135,18 @@ class Dolist::API end def post(url, body) - response = Typhoeus.post(url, body:, headers:).tap do + response = Typhoeus.post(url, body:, headers: default_headers).tap do self.class.save_rate_limit_headers(_1.headers) end - JSON.parse(response.response_body) + if response.response_body.empty? + fail "Empty response from Dolist API" + else + JSON.parse(response.response_body) + end end - def headers + def default_headers { "Content-Type": 'application/json', "Accept": 'application/json', @@ -150,6 +182,34 @@ class Dolist::API post(url, body)["ItemList"] end + def prepare_mail_body(mail) + { + "Type": "TransactionalService", + "Contact": { + "FieldList": [ + { + "ID": EMAIL_KEY, + "Value": mail.to.first + } + ] + }, + "Message": { + "Name": mail['X-Dolist-Message-Name'].value, + "Subject": mail.subject, + "SenderID": sender_id, + "ForceHttp": true, + "Format": "html", + "DisableOpenTracking": true, + "IsTrackingValidated": true + }, + "MessageContent": { + "SourceCode": mail_source_code(mail), + "EncodingType": "UTF8", + "EnableTrackingDetection": false + } + } + end + def to_sent_mail(email_address, contact_id, dolist_message) SentMail.new( from: ENV['DOLIST_NO_REPLY_EMAIL'], @@ -180,4 +240,16 @@ class Dolist::API mail.html_part.body.decoded end end + + def base64_files(attachments) + attachments.map do |attachment| + raise ArgumentError, "Dolist API does not support non PDF attachments. Given #{attachment.filename} which has mime_type=#{attachment.mime_type}" unless attachment.mime_type == "application/pdf" + + field_name = File.basename(attachment.filename, File.extname(attachment.filename)) + attachment_content = attachment.body.decoded + attachment_base64 = Base64.strict_encode64(attachment_content) + + Dolist::Base64File.new(field_name:, filename: attachment.filename, mime_type: attachment.mime_type, content: attachment_base64) + end + end end diff --git a/app/lib/dolist/base64_file.rb b/app/lib/dolist/base64_file.rb new file mode 100644 index 000000000..8afcf2024 --- /dev/null +++ b/app/lib/dolist/base64_file.rb @@ -0,0 +1,3 @@ +module Dolist + Base64File = Struct.new(:field_name, :filename, :mime_type, :content, keyword_init: true) +end From 7fa966548c466814c83b4993791dc5b7aa963d63 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Thu, 2 Feb 2023 15:58:32 +0100 Subject: [PATCH 4/8] fix(mailer): missing procedure logo for DossierMailer --- app/views/layouts/mailers/layout.html.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/views/layouts/mailers/layout.html.erb b/app/views/layouts/mailers/layout.html.erb index 6175243c6..70562a48b 100644 --- a/app/views/layouts/mailers/layout.html.erb +++ b/app/views/layouts/mailers/layout.html.erb @@ -108,6 +108,7 @@
+ <%= yield(:procedure_logo) %>
From 380a4232c6c78bd3a33fab7bc9a1847da11cebf1 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Thu, 2 Feb 2023 15:58:53 +0100 Subject: [PATCH 5/8] fix(mailer): preview for DossierMailer#notify_new_draft --- spec/mailers/previews/dossier_mailer_preview.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/mailers/previews/dossier_mailer_preview.rb b/spec/mailers/previews/dossier_mailer_preview.rb index 61fad6354..2ef877538 100644 --- a/spec/mailers/previews/dossier_mailer_preview.rb +++ b/spec/mailers/previews/dossier_mailer_preview.rb @@ -1,7 +1,7 @@ # Preview all emails at http://localhost:3000/rails/mailers/dossier_mailer class DossierMailerPreview < ActionMailer::Preview def notify_new_draft - DossierMailer.notify_new_draft(draft) + DossierMailer.with(dossier: draft).notify_new_draft end def notify_new_answer From 9641a40ea764d62131a8363be274e0ce5a3504fe Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Thu, 2 Feb 2023 15:59:18 +0100 Subject: [PATCH 6/8] chore(mailer): MailDeliveryError with original exception backtrace --- app/lib/mail_delivery_error.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/lib/mail_delivery_error.rb b/app/lib/mail_delivery_error.rb index 4ce1cffed..630496528 100644 --- a/app/lib/mail_delivery_error.rb +++ b/app/lib/mail_delivery_error.rb @@ -3,4 +3,10 @@ # so it would be shallowed otherwise. # # TODO: add a test which verify that the error will permit the job to retry -class MailDeliveryError < Exception; end # rubocop:disable Lint/InheritException +class MailDeliveryError < Exception # rubocop:disable Lint/InheritException + def initialize(original_exception) + super(original_exception.message) + + set_backtrace(original_exception.backtrace) + end +end From 6a3de1b57a40d1f6f295e4d220a997660bd8ec45 Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Thu, 2 Feb 2023 16:02:27 +0100 Subject: [PATCH 7/8] chore(dolist): cache sender_id so we don't request it at each send --- app/lib/dolist/api.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/lib/dolist/api.rb b/app/lib/dolist/api.rb index 0faa9f38e..e6ca3129e 100644 --- a/app/lib/dolist/api.rb +++ b/app/lib/dolist/api.rb @@ -123,7 +123,9 @@ class Dolist::API end def sender_id - @sender_id ||= senders.dig("ItemList", 0, "Sender", "ID") + Rails.cache.fetch("dolist_api_sender_id", expires_in: 1.hour) do + senders.dig("ItemList", 0, "Sender", "ID") + end end def get(url) From 6b011b8b44ef494324df28b1113d7d15c587070b Mon Sep 17 00:00:00 2001 From: Colin Darie Date: Thu, 2 Feb 2023 16:11:52 +0100 Subject: [PATCH 8/8] chore(dolist): helper so we know if mail is sendable by API --- app/lib/dolist/api.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/lib/dolist/api.rb b/app/lib/dolist/api.rb index e6ca3129e..08ba7d57f 100644 --- a/app/lib/dolist/api.rb +++ b/app/lib/dolist/api.rb @@ -30,6 +30,11 @@ class Dolist::API sleep (limit_reset_at - Time.zone.now).ceil end + + def sendable?(mail) + # Mail having attachments are not yet supported in our account + mail.attachments.none? { !_1.inline? } + end end def properly_configured?