Merge pull request #10745 from colinux/helpscout-delete-old-customers

ETQ opérateur, je supprime les contacts HS > 2 ans

app/lib/helpscout/api.rb

@@ -11,6 +11,8 @@ class Helpscout::API
 
   RATELIMIT_KEY = "helpscout-rate-limit-remaining"
 
+  class RateLimitError < StandardError; end;
+
   def ready?
     required_secrets = [
       Rails.application.secrets.helpscout[:mailbox_id],
@@ -66,10 +68,33 @@ class Helpscout::API
     [body[:_embedded][:conversations], body[:page]]
   end
 
+  def list_old_customers(before, page: 1)
+    body = {
+      page:,
+      query: "(
+        modifiedAt:[* TO #{before.iso8601}]
+      )",
+      sortField: "modifiedAt",
+      sortOrder: "desc"
+    }
+
+    response = call_api(:get, "#{CUSTOMERS}?#{body.to_query}")
+    if !response.success?
+      raise StandardError, "Error while listing customers: #{response.response_code} '#{response.body}'"
+    end
+
+    body = parse_response_body(response)
+    [body[:_embedded][:customers], body[:page]]
+  end
+
   def delete_conversation(conversation_id)
     call_api(:delete, "#{CONVERSATIONS}/#{conversation_id}")
   end
 
+  def delete_customer(customer_id)
+    call_api(:delete, "#{CUSTOMERS}/#{customer_id}")
+  end
+
   def add_phone_number(email, phone)
     query = CGI.escape("(email:#{email})")
     response = call_api(:get, "#{CUSTOMERS}?mailbox=#{user_support_mailbox_id}&query=#{query}")
@@ -164,6 +189,10 @@ class Helpscout::API
       })
     end.tap do |response|
       Rails.cache.write(RATELIMIT_KEY, response.headers["X-Ratelimit-Remaining-Minute"], expires_in: 1.minute)
+
+      if response.response_code.to_i == 429
+        raise RateLimitError
+      end
     end
   end
 

app/tasks/maintenance/helpscout_delete_old_customers_task.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Maintenance
+  class HelpscoutDeleteOldCustomersTask < MaintenanceTasks::Task
+    # Delete Helpscout customers not seen in the last 2 years
+    # with any conversations, and any data related with GPDR compliance.
+    # Respects the Helpscout API rate limit (200 calls per minute).
+
+    MODIFIED_BEFORE = 2.years.freeze
+
+    throttle_on(backoff: 1.minute) do
+      limit = Rails.cache.read(Helpscout::API::RATELIMIT_KEY)
+      limit.present? && limit.to_i <= 26 # check is made before each process but not before listing each page. External activity can affect the rate limit.
+    end
+
+    def count
+      _customers, pagination = api.list_old_customers(modified_before)
+
+      pagination[:totalElements]
+    end
+
+    # Because customers are deleted progressively,
+    # ignore cursor and always pick the first page
+    def enumerator_builder(cursor:)
+      Enumerator.new do |yielder|
+        loop do
+          customers, pagination = api.list_old_customers(modified_before)
+          customers.each do |customer|
+            yielder.yield(customer[:id], nil) # don't care about cursor parameter
+          end
+
+          # "number" is the current page (always 1 in our case)
+          # iterate until there are no remaining pages
+          break if pagination[:totalPages] == 0 || pagination[:totalPages] == pagination[:number]
+        end
+      end
+    end
+
+    def process(customer_id)
+      api.delete_customer(customer_id)
+    rescue Helpscout::API::RateLimitError # despite throttle and counter, race conditions sometimes lead to rate limit hit
+      sleep 1.minute
+      retry
+    end
+
+    private
+
+    def api
+      @api ||= Helpscout::API.new
+    end
+
+    def modified_before
+      MODIFIED_BEFORE.ago.utc.beginning_of_day
+    end
+  end
+end

spec/fixtures/cassettes/helpscout_list_old_customers.yml

@@ -0,0 +1,153 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://api.helpscout.net/v2/oauth2/token
+    body:
+      encoding: UTF-8
+      string: client_id=1234&client_secret=5678&grant_type=client_credentials
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Date:
+      - Tue, 11 Jun 2024 14:13:26 GMT
+      Content-Type:
+      - application/json; charset=utf-8
+      Content-Length:
+      - '94'
+      Server:
+      - kong/0.14.1
+      Cache-Control:
+      - no-store
+      Pragma:
+      - no-cache
+      Access-Control-Allow-Origin:
+      - "*"
+      Access-Control-Expose-Headers:
+      - Location,Resource-Id
+    body:
+      encoding: UTF-8
+      string: '{"token_type":"bearer","access_token":"redacted","expires_in":172800}'
+  recorded_at: Wed, 05 Jun 2024 00:00:00 GMT
+- request:
+    method: get
+    uri: https://api.helpscout.net/v2/customers?page=1&query=(%0A%20%20%20%20%20%20%20%20modifiedAt:%5B*%20TO%202022-06-05T00:00:00Z%5D%0A%20%20%20%20%20%20)&sortField=modifiedAt&sortOrder=desc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Authorization:
+      - Bearer redacted
+      Content-Type:
+      - application/json; charset=UTF-8
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Date:
+      - Tue, 11 Jun 2024 14:13:27 GMT
+      Content-Type:
+      - application/hal+json
+      X-Ratelimit-Limit-Minute:
+      - '200'
+      X-Ratelimit-Remaining-Minute:
+      - '199'
+      X-Content-Type-Options:
+      - nosniff
+      X-Xss-Protection:
+      - '0'
+      Cache-Control:
+      - no-cache, no-store, max-age=0, must-revalidate
+      Pragma:
+      - no-cache
+      Expires:
+      - '0'
+      X-Frame-Options:
+      - DENY
+      Access-Control-Allow-Origin:
+      - "*"
+      Access-Control-Expose-Headers:
+      - Location,Resource-Id
+      Correlation-Id:
+      - a9ca7664-2711-4c36-a092-73203365b474#13211836
+      X-Kong-Upstream-Latency:
+      - '640'
+      X-Kong-Proxy-Latency:
+      - '3'
+      Via:
+      - kong/0.14.1
+    body:
+      encoding: UTF-8
+      string: '{"_embedded":{"customers":[{"id":553306602,"firstName":"Energie","lastName":"","gender":"Unknown","photoType":"default","photoUrl":"https://d33v4339jhl8k0.cloudfront.net/customer-avatar/07.png","createdAt":"2022-08-19T06:50:27Z","updatedAt":"2022-08-19T06:50:27Z","background":"","draft":false,"_embedded":{"emails":[{"id":699846839,"value":"adresse@email.com","type":"work"}],"phones":[],"chats":[],"social_profiles":[],"websites":[],"properties":[]},"_links":{"address":{"href":"https://api.helpscout.net/v2/customers/553306602/address"},"chats":{"href":"https://api.helpscout.net/v2/customers/553306602/chats"},"emails":{"href":"https://api.helpscout.net/v2/customers/553306602/emails"},"phones":{"href":"https://api.helpscout.net/v2/customers/553306602/phones"},"social-profiles":{"href":"https://api.helpscout.net/v2/customers/553306602/social-profiles"},"websites":{"href":"https://api.helpscout.net/v2/customers/553306602/websites"},"self":{"href":"https://api.helpscout.net/v2/customers/553306602"}}},{"id":552485177,"firstName":"Ars-Ara-Adeli","lastName":"","gender":"Unknown","photoType":"default","photoUrl":"https://d33v4339jhl8k0.cloudfront.net/customer-avatar/04.png","createdAt":"2022-08-16T07:46:44Z","updatedAt":"2022-08-22T06:33:23Z","background":"","draft":false,"_embedded":{"emails":[{"id":699054288,"value":"adresse@email.com","type":"work"}],"phones":[],"chats":[],"social_profiles":[],"websites":[],"properties":[]},"_links":{"address":{"href":"https://api.helpscout.net/v2/customers/552485177/address"},"chats":{"href":"https://api.helpscout.net/v2/customers/552485177/chats"},"emails":{"href":"https://api.helpscout.net/v2/customers/552485177/emails"},"phones":{"href":"https://api.helpscout.net/v2/customers/552485177/phones"},"social-profiles":{"href":"https://api.helpscout.net/v2/customers/552485177/social-profiles"},"websites":{"href":"https://api.helpscout.net/v2/customers/552485177/websites"},"self":{"href":"https://api.helpscout.net/v2/customers/552485177"}}}]},"_links":{"next":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=2"},"self":{"href":"https://api.helpscout.net/v2/customers?page=1\u0026query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])"},"first":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=1"},"last":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=403"},"page":{"href":"https://api.helpscout.net/v2/customers?page=1\u0026query=(modifiedAt:%5B*%20TO%202022-08-25T22:00:00Z%5D)"}},"page":{"size":2,"totalElements":4,"totalPages":2,"number":1}}
+      '
+  recorded_at: Wed, 05 Jun 2024 00:00:00 GMT
+
+- request:
+    method: get
+    uri: https://api.helpscout.net/v2/customers?page=1&query=(%0A%20%20%20%20%20%20%20%20modifiedAt:%5B*%20TO%202022-06-05T00:00:00Z%5D%0A%20%20%20%20%20%20)&sortField=modifiedAt&sortOrder=desc
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      User-Agent:
+      - demarches-simplifiees.fr
+      Authorization:
+      - Bearer redacted
+      Content-Type:
+      - application/json; charset=UTF-8
+      Expect:
+      - ''
+  response:
+    status:
+      code: 200
+      message: ''
+    headers:
+      Date:
+      - Tue, 11 Jun 2024 14:13:27 GMT
+      Content-Type:
+      - application/hal+json
+      X-Ratelimit-Limit-Minute:
+      - '200'
+      X-Ratelimit-Remaining-Minute:
+      - '199'
+      X-Content-Type-Options:
+      - nosniff
+      X-Xss-Protection:
+      - '0'
+      Cache-Control:
+      - no-cache, no-store, max-age=0, must-revalidate
+      Pragma:
+      - no-cache
+      Expires:
+      - '0'
+      X-Frame-Options:
+      - DENY
+      Access-Control-Allow-Origin:
+      - "*"
+      Access-Control-Expose-Headers:
+      - Location,Resource-Id
+      Correlation-Id:
+      - a9ca7664-2711-4c36-a092-73203365b474#13211836
+      X-Kong-Upstream-Latency:
+      - '640'
+      X-Kong-Proxy-Latency:
+      - '3'
+      Via:
+      - kong/0.14.1
+    body:
+      encoding: UTF-8
+      string: '{"_embedded":{"customers":[{"id":553306602,"firstName":"Energie","lastName":"","gender":"Unknown","photoType":"default","photoUrl":"https://d33v4339jhl8k0.cloudfront.net/customer-avatar/07.png","createdAt":"2022-08-19T06:50:27Z","updatedAt":"2022-08-19T06:50:27Z","background":"","draft":false,"_embedded":{"emails":[{"id":699846839,"value":"adresse@email.com","type":"work"}],"phones":[],"chats":[],"social_profiles":[],"websites":[],"properties":[]},"_links":{"address":{"href":"https://api.helpscout.net/v2/customers/553306602/address"},"chats":{"href":"https://api.helpscout.net/v2/customers/553306602/chats"},"emails":{"href":"https://api.helpscout.net/v2/customers/553306602/emails"},"phones":{"href":"https://api.helpscout.net/v2/customers/553306602/phones"},"social-profiles":{"href":"https://api.helpscout.net/v2/customers/553306602/social-profiles"},"websites":{"href":"https://api.helpscout.net/v2/customers/553306602/websites"},"self":{"href":"https://api.helpscout.net/v2/customers/553306602"}}},{"id":552485177,"firstName":"Ars-Ara-Adeli","lastName":"","gender":"Unknown","photoType":"default","photoUrl":"https://d33v4339jhl8k0.cloudfront.net/customer-avatar/04.png","createdAt":"2022-08-16T07:46:44Z","updatedAt":"2022-08-22T06:33:23Z","background":"","draft":false,"_embedded":{"emails":[{"id":699054288,"value":"adresse@email.com","type":"work"}],"phones":[],"chats":[],"social_profiles":[],"websites":[],"properties":[]},"_links":{"address":{"href":"https://api.helpscout.net/v2/customers/552485177/address"},"chats":{"href":"https://api.helpscout.net/v2/customers/552485177/chats"},"emails":{"href":"https://api.helpscout.net/v2/customers/552485177/emails"},"phones":{"href":"https://api.helpscout.net/v2/customers/552485177/phones"},"social-profiles":{"href":"https://api.helpscout.net/v2/customers/552485177/social-profiles"},"websites":{"href":"https://api.helpscout.net/v2/customers/552485177/websites"},"self":{"href":"https://api.helpscout.net/v2/customers/552485177"}}}]},"_links":{"next":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=2"},"self":{"href":"https://api.helpscout.net/v2/customers?page=1\u0026query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])"},"first":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=1"},"last":{"href":"https://api.helpscout.net/v2/customers?query=(modifiedAt:[* TO 2022-08-25T22:00:00Z])\u0026page=403"},"page":{"href":"https://api.helpscout.net/v2/customers?page=1\u0026query=(modifiedAt:%5B*%20TO%202022-08-25T22:00:00Z%5D)"}},"page":{"size":2,"totalElements":4,"totalPages":1,"number":1}}
+      '
+  recorded_at: Wed, 05 Jun 2024 00:00:00 GMT

spec/tasks/maintenance/helpscout_delete_old_customers_task_spec.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+describe Maintenance::HelpscoutDeleteOldCustomersTask do
+  before do
+    mock_helpscout_secrets
+    travel_to DateTime.new(2024, 6, 5)
+  end
+
+  subject do
+    described_class.new
+  end
+
+  describe '#enumerator_builder' do
+    it "enumerates conversation ids" do
+      VCR.use_cassette("helpscout_list_old_customers") do |c|
+        ids = subject.enumerator_builder(cursor: 0).to_a
+        # Warning: calling a enumerable method always reinvoke the enumerable !
+        # So immediately convert in array and run expectations on it
+
+        # anonymize when recorded cassettes
+        c.new_recorded_interactions.each do |interaction|
+          interaction.request.body = anonymize_request(interaction)
+
+          body = anonymize_response(interaction)
+          interaction.response.body = body.to_json
+        end
+
+        expect(ids.count).to eq(4) # 2 first page + 2 next page
+        expect(ids[0][0]).to eq(553306602)
+      end
+    end
+  end
+
+  def anonymize_response(interaction)
+    body = JSON.parse(interaction.response.body)
+
+    Array(body.dig("_embedded", "customers")).each do |customer|
+      customer["emails"][0]["value"] = "adresse@email.com"
+    end
+
+    body["access_token"] = "redacted" if body.key?("access_token")
+
+    body
+  end
+
+  def anonymize_request(interaction)
+    body = interaction.request.body
+
+    return body unless body.include?("client_secret")
+
+    URI.decode_www_form(body).to_h.merge("client_id" => "1234", "client_secret" => "5678").to_query
+  end
+
+  def mock_helpscout_secrets
+    Rails.application.secrets.helpscout[:mailbox_id] = '9999'
+    Rails.application.secrets.helpscout[:client_id] = '1234'
+    Rails.application.secrets.helpscout[:client_secret] = '5678'
+  end
+end