diff --git a/app/lib/helpscout/api.rb b/app/lib/helpscout/api.rb index b3c4d63bb..bf5e9ed8b 100644 --- a/app/lib/helpscout/api.rb +++ b/app/lib/helpscout/api.rb @@ -11,6 +11,8 @@ class Helpscout::API RATELIMIT_KEY = "helpscout-rate-limit-remaining" + class RateLimitError < StandardError; end; + def ready? required_secrets = [ Rails.application.secrets.helpscout[:mailbox_id], @@ -66,10 +68,33 @@ class Helpscout::API [body[:_embedded][:conversations], body[:page]] end + def list_old_customers(before, page: 1) + body = { + page:, + query: "( + modifiedAt:[* TO #{before.iso8601}] + )", + sortField: "modifiedAt", + sortOrder: "desc" + } + + response = call_api(:get, "#{CUSTOMERS}?#{body.to_query}") + if !response.success? + raise StandardError, "Error while listing customers: #{response.response_code} '#{response.body}'" + end + + body = parse_response_body(response) + [body[:_embedded][:customers], body[:page]] + end + def delete_conversation(conversation_id) call_api(:delete, "#{CONVERSATIONS}/#{conversation_id}") end + def delete_customer(customer_id) + call_api(:delete, "#{CUSTOMERS}/#{customer_id}") + end + def add_phone_number(email, phone) query = CGI.escape("(email:#{email})") response = call_api(:get, "#{CUSTOMERS}?mailbox=#{user_support_mailbox_id}&query=#{query}") @@ -164,6 +189,10 @@ class Helpscout::API }) end.tap do |response| Rails.cache.write(RATELIMIT_KEY, response.headers["X-Ratelimit-Remaining-Minute"], expires_in: 1.minute) + + if response.response_code.to_i == 429 + raise RateLimitError + end end end diff --git a/app/tasks/maintenance/helpscout_delete_old_customers_task.rb b/app/tasks/maintenance/helpscout_delete_old_customers_task.rb new file mode 100644 index 000000000..32e241435 --- /dev/null +++ b/app/tasks/maintenance/helpscout_delete_old_customers_task.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Maintenance + class HelpscoutDeleteOldCustomersTask < MaintenanceTasks::Task + # Delete Helpscout customers not seen in the last 2 years + # with any conversations, and any data related with GPDR compliance. + # Respects the Helpscout API rate limit (200 calls per minute). + + MODIFIED_BEFORE = 2.years.freeze + + throttle_on(backoff: 1.minute) do + limit = Rails.cache.read(Helpscout::API::RATELIMIT_KEY) + limit.present? && limit.to_i <= 26 # check is made before each process but not before listing each page. External activity can affect the rate limit. + end + + def count + _customers, pagination = api.list_old_customers(modified_before) + + pagination[:totalElements] + end + + # Because customers are deleted progressively, + # ignore cursor and always pick the first page + def enumerator_builder(cursor:) + Enumerator.new do |yielder| + loop do + customers, pagination = api.list_old_customers(modified_before) + customers.each do |customer| + yielder.yield(customer[:id], nil) # don't care about cursor parameter + end + + # "number" is the current page (always 1 in our case) + # iterate until there are no remaining pages + break if pagination[:totalPages] == 0 || pagination[:totalPages] == pagination[:number] + end + end + end + + def process(customer_id) + api.delete_customer(customer_id) + rescue Helpscout::API::RateLimitError # despite throttle and counter, race conditions sometimes lead to rate limit hit + sleep 1.minute + retry + end + + private + + def api + @api ||= Helpscout::API.new + end + + def modified_before + MODIFIED_BEFORE.ago.utc.beginning_of_day + end + end +end diff --git a/spec/fixtures/cassettes/helpscout_list_old_customers.yml b/spec/fixtures/cassettes/helpscout_list_old_customers.yml new file mode 100644 index 000000000..11231cf04 --- /dev/null +++ b/spec/fixtures/cassettes/helpscout_list_old_customers.yml @@ -0,0 +1,153 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.helpscout.net/v2/oauth2/token + body: + encoding: UTF-8 + string: client_id=1234&client_secret=5678&grant_type=client_credentials + headers: + User-Agent: + - demarches-simplifiees.fr + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + Date: + - Tue, 11 Jun 2024 14:13:26 GMT + Content-Type: + - application/json; charset=utf-8 + Content-Length: + - '94' + Server: + - kong/0.14.1 + Cache-Control: + - no-store + Pragma: + - no-cache + Access-Control-Allow-Origin: + - "*" + Access-Control-Expose-Headers: + - Location,Resource-Id + body: + encoding: UTF-8 + string: '{"token_type":"bearer","access_token":"redacted","expires_in":172800}' + recorded_at: Wed, 05 Jun 2024 00:00:00 GMT +- request: + method: get + uri: https://api.helpscout.net/v2/customers?page=1&query=(%0A%20%20%20%20%20%20%20%20modifiedAt:%5B*%20TO%202022-06-05T00:00:00Z%5D%0A%20%20%20%20%20%20)&sortField=modifiedAt&sortOrder=desc + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - demarches-simplifiees.fr + Authorization: + - Bearer redacted + Content-Type: + - application/json; charset=UTF-8 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + Date: + - Tue, 11 Jun 2024 14:13:27 GMT + Content-Type: + - application/hal+json + X-Ratelimit-Limit-Minute: + - '200' + X-Ratelimit-Remaining-Minute: + - '199' + X-Content-Type-Options: + - nosniff + X-Xss-Protection: + - '0' + Cache-Control: + - no-cache, no-store, max-age=0, must-revalidate + Pragma: + - no-cache + Expires: + - '0' + X-Frame-Options: + - DENY + Access-Control-Allow-Origin: + - "*" + Access-Control-Expose-Headers: + - Location,Resource-Id + Correlation-Id: + - a9ca7664-2711-4c36-a092-73203365b474#13211836 + X-Kong-Upstream-Latency: + - '640' + X-Kong-Proxy-Latency: + - '3' + Via: + - kong/0.14.1 + body: + encoding: UTF-8 + string: '{"_embedded":{"customers":[{"id":553306602,"firstName":"Energie","lastName":"","gender":"Unknown","photoType":"default","photoUrl":"https://d33v4339jhl8k0.cloudfront.net/customer-avatar/07.png","createdAt":"2022-08-19T06:50:27Z","updatedAt":"2022-08-19T06:50:27Z","background":"","draft":false,"_embedded":{"emails":[{"id":699846839,"value":"adresse@email.com","type":"work"}],"phones":[],"chats":[],"social_profiles":[],"websites":[],"properties":[]},"_links":{"address":{"href":"https://api.helpscout.net/v2/customers/553306602/address"},"chats":{"href":"https://api.helpscout.net/v2/customers/553306602/chats"},"emails":{"href":"https://api.helpscout.net/v2/customers/553306602/emails"},"phones":{"href":"https://api.helpscout.net/v2/customers/553306602/phones"},"social-profiles":{"href":"https://api.helpscout.net/v2/customers/553306602/social-profiles"},"websites":{"href":"https://api.helpscout.net/v2/customers/553306602/websites"},"self":{"href":"https://api.helpscout.net/v2/customers/553306602"}}},{"id":552485177,"firstName":"Ars-Ara-Adeli","lastName":"","gender":"Unknown","photoType":"default","photoUrl":"https://d33v4339jhl8k0.cloudfront.net/customer-avatar/04.png","createdAt":"2022-08-16T07:46:44Z","updatedAt":"2022-08-22T06:33:23Z","background":"","draft":false,"_embedded":{"emails":[{"id":699054288,"value":"adresse@email.com","type":"work"}],"phones":[],"chats":[],"social_profiles":[],"websites":[],"properties":[]},"_links":{"address":{"href":"https://api.helpscout.net/v2/customers/552485177/address"},"chats":{"href":"https://api.helpscout.net/v2/customers/552485177/chats"},"emails":{"href":"https://api.helpscout.net/v2/customers/552485177/emails"},"phones":{"href":"https://api.helpscout.net/v2/customers/552485177/phones"},"social-profiles":{"href":"https://api.helpscout.net/v2/customers/552485177/social-profiles"},"websites":{"href":"https://api.helpscout.net/v2/customers/552485177/websites"},"self":{"href":"https://api.helpscout.net/v2/customers/552485177"}}}]},"_links":{"next":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=2"},"self":{"href":"https://api.helpscout.net/v2/customers?page=1\u0026query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])"},"first":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=1"},"last":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=403"},"page":{"href":"https://api.helpscout.net/v2/customers?page=1\u0026query=(modifiedAt:%5B*%20TO%202022-08-25T22:00:00Z%5D)"}},"page":{"size":2,"totalElements":4,"totalPages":2,"number":1}} + ' + recorded_at: Wed, 05 Jun 2024 00:00:00 GMT + +- request: + method: get + uri: https://api.helpscout.net/v2/customers?page=1&query=(%0A%20%20%20%20%20%20%20%20modifiedAt:%5B*%20TO%202022-06-05T00:00:00Z%5D%0A%20%20%20%20%20%20)&sortField=modifiedAt&sortOrder=desc + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - demarches-simplifiees.fr + Authorization: + - Bearer redacted + Content-Type: + - application/json; charset=UTF-8 + Expect: + - '' + response: + status: + code: 200 + message: '' + headers: + Date: + - Tue, 11 Jun 2024 14:13:27 GMT + Content-Type: + - application/hal+json + X-Ratelimit-Limit-Minute: + - '200' + X-Ratelimit-Remaining-Minute: + - '199' + X-Content-Type-Options: + - nosniff + X-Xss-Protection: + - '0' + Cache-Control: + - no-cache, no-store, max-age=0, must-revalidate + Pragma: + - no-cache + Expires: + - '0' + X-Frame-Options: + - DENY + Access-Control-Allow-Origin: + - "*" + Access-Control-Expose-Headers: + - Location,Resource-Id + Correlation-Id: + - a9ca7664-2711-4c36-a092-73203365b474#13211836 + X-Kong-Upstream-Latency: + - '640' + X-Kong-Proxy-Latency: + - '3' + Via: + - kong/0.14.1 + body: + encoding: UTF-8 + string: '{"_embedded":{"customers":[{"id":553306602,"firstName":"Energie","lastName":"","gender":"Unknown","photoType":"default","photoUrl":"https://d33v4339jhl8k0.cloudfront.net/customer-avatar/07.png","createdAt":"2022-08-19T06:50:27Z","updatedAt":"2022-08-19T06:50:27Z","background":"","draft":false,"_embedded":{"emails":[{"id":699846839,"value":"adresse@email.com","type":"work"}],"phones":[],"chats":[],"social_profiles":[],"websites":[],"properties":[]},"_links":{"address":{"href":"https://api.helpscout.net/v2/customers/553306602/address"},"chats":{"href":"https://api.helpscout.net/v2/customers/553306602/chats"},"emails":{"href":"https://api.helpscout.net/v2/customers/553306602/emails"},"phones":{"href":"https://api.helpscout.net/v2/customers/553306602/phones"},"social-profiles":{"href":"https://api.helpscout.net/v2/customers/553306602/social-profiles"},"websites":{"href":"https://api.helpscout.net/v2/customers/553306602/websites"},"self":{"href":"https://api.helpscout.net/v2/customers/553306602"}}},{"id":552485177,"firstName":"Ars-Ara-Adeli","lastName":"","gender":"Unknown","photoType":"default","photoUrl":"https://d33v4339jhl8k0.cloudfront.net/customer-avatar/04.png","createdAt":"2022-08-16T07:46:44Z","updatedAt":"2022-08-22T06:33:23Z","background":"","draft":false,"_embedded":{"emails":[{"id":699054288,"value":"adresse@email.com","type":"work"}],"phones":[],"chats":[],"social_profiles":[],"websites":[],"properties":[]},"_links":{"address":{"href":"https://api.helpscout.net/v2/customers/552485177/address"},"chats":{"href":"https://api.helpscout.net/v2/customers/552485177/chats"},"emails":{"href":"https://api.helpscout.net/v2/customers/552485177/emails"},"phones":{"href":"https://api.helpscout.net/v2/customers/552485177/phones"},"social-profiles":{"href":"https://api.helpscout.net/v2/customers/552485177/social-profiles"},"websites":{"href":"https://api.helpscout.net/v2/customers/552485177/websites"},"self":{"href":"https://api.helpscout.net/v2/customers/552485177"}}}]},"_links":{"next":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=2"},"self":{"href":"https://api.helpscout.net/v2/customers?page=1\u0026query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])"},"first":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=1"},"last":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=403"},"page":{"href":"https://api.helpscout.net/v2/customers?page=1\u0026query=(modifiedAt:%5B*%20TO%202022-08-25T22:00:00Z%5D)"}},"page":{"size":2,"totalElements":4,"totalPages":1,"number":1}} + ' + recorded_at: Wed, 05 Jun 2024 00:00:00 GMT diff --git a/spec/tasks/maintenance/helpscout_delete_old_customers_task_spec.rb b/spec/tasks/maintenance/helpscout_delete_old_customers_task_spec.rb new file mode 100644 index 000000000..8e3f7bd03 --- /dev/null +++ b/spec/tasks/maintenance/helpscout_delete_old_customers_task_spec.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +describe Maintenance::HelpscoutDeleteOldCustomersTask do + before do + mock_helpscout_secrets + travel_to DateTime.new(2024, 6, 5) + end + + subject do + described_class.new + end + + describe '#enumerator_builder' do + it "enumerates conversation ids" do + VCR.use_cassette("helpscout_list_old_customers") do |c| + ids = subject.enumerator_builder(cursor: 0).to_a + # Warning: calling a enumerable method always reinvoke the enumerable ! + # So immediately convert in array and run expectations on it + + # anonymize when recorded cassettes + c.new_recorded_interactions.each do |interaction| + interaction.request.body = anonymize_request(interaction) + + body = anonymize_response(interaction) + interaction.response.body = body.to_json + end + + expect(ids.count).to eq(4) # 2 first page + 2 next page + expect(ids[0][0]).to eq(553306602) + end + end + end + + def anonymize_response(interaction) + body = JSON.parse(interaction.response.body) + + Array(body.dig("_embedded", "customers")).each do |customer| + customer["emails"][0]["value"] = "adresse@email.com" + end + + body["access_token"] = "redacted" if body.key?("access_token") + + body + end + + def anonymize_request(interaction) + body = interaction.request.body + + return body unless body.include?("client_secret") + + URI.decode_www_form(body).to_h.merge("client_id" => "1234", "client_secret" => "5678").to_query + end + + def mock_helpscout_secrets + Rails.application.secrets.helpscout[:mailbox_id] = '9999' + Rails.application.secrets.helpscout[:client_id] = '1234' + Rails.application.secrets.helpscout[:client_secret] = '5678' + end +end