Merge pull request #1404 from betagouv/cellar_integration

Active storage and Clever Cloud Cellar integration

lib/active_storage/service/cellar_service.rb

@@ -0,0 +1,167 @@
+require 'base64'
+require 'net/http'
+require 'openssl'
+
+module ActiveStorage
+  class Service::CellarService < Service
+    def initialize(access_key_id:, secret_access_key:, bucket:, **)
+      @endpoint = URI::HTTPS.build(host: "#{bucket}.cellar.services.clever-cloud.com")
+      @access_key_id = access_key_id
+      @secret_access_key = secret_access_key
+      @bucket = bucket
+    end
+
+    def download(key)
+      # TODO: error handling
+      if block_given?
+        instrument :streaming_download, key: key do
+          http_start do |http|
+            http.request(get_request(key)) do |response|
+              response.read_body do |chunk|
+                yield(chunk.force_encoding(Encoding::BINARY))
+              end
+            end
+          end
+        end
+      else
+        instrument :download, key: key do
+          http_start do |http|
+            response = http.request(get_request(key))
+            if response.is_a?(Net::HTTPSuccess)
+              response.body.force_encoding(Encoding::BINARY)
+            end
+          end
+        end
+      end
+    end
+
+    def delete(key)
+      # TODO: error handling
+      instrument :delete, key: key do
+        http_start do |http|
+          perform_delete(http, key)
+        end
+      end
+    end
+
+    def delete_prefixed(prefix)
+      # TODO: error handling
+      # TODO: handle pagination if more than 1000 keys
+      instrument :delete_prefixed, prefix: prefix do
+        http_start do |http|
+          list_prefixed(http, prefix).each do |key|
+            # TODO: use bulk delete instead
+            perform_delete(http, key)
+          end
+        end
+      end
+    end
+
+    def url(key, expires_in:, filename:, disposition:, content_type:)
+      instrument :url, key: key do |payload|
+        generated_url = presigned_url(
+          method: 'GET',
+          key: key,
+          expires_in: expires_in,
+          "response-content-disposition": content_disposition_with(type: disposition, filename: filename),
+          "response-content-type": content_type
+        )
+        payload[:url] = generated_url
+        generated_url
+      end
+    end
+
+    def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
+      instrument :url, key: key do |payload|
+        generated_url = presigned_url(
+          method: 'PUT',
+          key: key,
+          expires_in: expires_in,
+          content_type: content_type,
+          checksum: checksum
+        )
+        payload[:url] = generated_url
+        generated_url
+      end
+    end
+
+    def headers_for_direct_upload(key, content_type:, checksum:, **)
+      { "Content-Type" => content_type, "Content-MD5" => checksum }
+    end
+
+    private
+
+    def http_start(&block)
+      Net::HTTP.start(@endpoint.host, @endpoint.port, use_ssl: true, &block)
+    end
+
+    def sign(request, key, checksum: '')
+      date = Time.now.httpdate
+      sig = signature(method: request.method, key: key, date: date, checksum: checksum)
+      request['date'] = date
+      request['authorization'] = "AWS #{@access_key_id}:#{sig}"
+    end
+
+    def presigned_url(method:, key:, expires_in:, content_type: '', checksum: '', **query_params)
+      expires = expires_in.from_now.to_i
+
+      query = query_params.merge({
+        AWSAccessKeyId: @access_key_id,
+        Expires: expires,
+        Signature: signature(method: method, key: key, expires: expires, content_type: content_type, checksum: checksum)
+      })
+
+      generated_url = URI::join(@endpoint, "/#{key}","?#{query.to_query}").to_s
+    end
+
+    def signature(method:, key:, expires: '', date: '', content_type: '', checksum: '')
+      canonicalized_amz_headers = ""
+      canonicalized_resource = "/#{@bucket}/#{key}"
+      string_to_sign = "#{method}\n#{checksum}\n#{content_type}\n#{expires}#{date}\n" +
+                       "#{canonicalized_amz_headers}#{canonicalized_resource}"
+      Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), @secret_access_key, string_to_sign)).strip
+    end
+
+    def list_prefixed(http, prefix)
+      request = Net::HTTP::Get.new(URI::join(@endpoint, "/?list-type=2&prefix=#{prefix}"))
+      sign(request, "")
+      response = http.request(request)
+      if response.is_a?(Net::HTTPSuccess)
+        parse_bucket_listing(response.body)
+      end
+    end
+
+    def parse_bucket_listing(bucket_listing_xml)
+      doc = Nokogiri::XML(bucket_listing_xml)
+      doc
+        .xpath('//xmlns:Contents/xmlns:Key')
+        .map{ |k| k.text }
+    end
+
+    def get_request(key)
+      request = Net::HTTP::Get.new(URI::join(@endpoint, "/#{key}"))
+      sign(request, key)
+      request
+    end
+
+    def bulk_deletion_request_body(keys)
+      builder = Nokogiri::XML::Builder.new(:encoding => 'UTF-8') do |xml|
+        xml.Delete do
+          xml.Quiet("true")
+          keys.each do |k|
+            xml.Object do
+              xml.Key(k)
+            end
+          end
+        end
+      end
+      builder.to_xml
+    end
+
+    def perform_delete(http, key)
+      request = Net::HTTP::Delete.new(URI::join(@endpoint, "/#{key}"))
+      sign(request, key)
+      http.request(request)
+    end
+  end
+end

spec/lib/active_storage/service/cellar_service_spec.rb

@@ -0,0 +1,166 @@
+require 'active_storage/service/cellar_service'
+require 'cgi'
+require 'net/http'
+require 'uri'
+
+describe 'CellarService' do
+  let(:cellar_service) do
+    # These are actual keys, but they’re safe to put here because
+    # - they never had any rights attached, and
+    # - the keys were revoked before copying them here
+
+    ActiveStorage::Service::CellarService.new(
+      access_key_id: 'AKIAJFTRSGRH3RXX6D5Q',
+      secret_access_key: '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',
+      bucket: 'rogets'
+    )
+  end
+
+  before { Timecop.freeze(Time.gm(2016, 10, 2)) }
+  after { Timecop.return }
+
+  describe 'signature generation' do
+    context 'for presigned URLs' do
+      subject do
+        cellar_service.send(
+          :signature,
+          {
+            method: 'GET',
+            key: 'fichier',
+            expires: 5.minutes.from_now.to_i
+          }
+        )
+      end
+
+      it { is_expected.to eq('nzCsB6cip8oofkuOdvvJs6FafkA=') }
+    end
+
+    context 'for server-side requests' do
+      subject do
+        Net::HTTP::Delete.new('https://rogets.cellar.services.clever-cloud.com/fichier')
+      end
+
+      before { cellar_service.send(:sign, subject, 'fichier') }
+
+      it { expect(subject['date']).to eq(Time.now.httpdate) }
+      it { expect(subject['authorization']).to eq('AWS AKIAJFTRSGRH3RXX6D5Q:nkvviwZYb1V9HDrKyJZmY3Z8sSA=') }
+    end
+  end
+
+  describe 'presigned url for download' do
+    subject do
+      URI.parse(
+        cellar_service.url(
+          'fichier',
+          expires_in: 5.minutes,
+          filename: ActiveStorage::Filename.new("toto.png"),
+          disposition: 'attachment',
+          content_type: 'image/png'
+        )
+      )
+    end
+
+    it do
+      is_expected.to have_attributes(
+        scheme: 'https',
+        host: 'rogets.cellar.services.clever-cloud.com',
+        path: '/fichier'
+      )
+    end
+
+    it do
+      expect(CGI::parse(subject.query)).to eq(
+        {
+          'AWSAccessKeyId' => ['AKIAJFTRSGRH3RXX6D5Q'],
+          'Expires' => ['1475366700'],
+          'Signature' => ['nzCsB6cip8oofkuOdvvJs6FafkA='],
+          'response-content-disposition' => ["attachment; filename=\"toto.png\"; filename*=UTF-8''toto.png"],
+          'response-content-type' => ['image/png'],
+        }
+      )
+    end
+  end
+
+  describe 'presigned url for direct upload' do
+    subject do
+      URI.parse(
+        cellar_service.url_for_direct_upload(
+          'fichier',
+          expires_in: 5.minutes,
+          content_type: 'image/png',
+          content_length: 2713,
+          checksum: 'DEADBEEF'
+        )
+      )
+    end
+
+    it do
+      is_expected.to have_attributes(
+        scheme: 'https',
+        host: 'rogets.cellar.services.clever-cloud.com',
+        path: '/fichier'
+      )
+    end
+
+    it do
+      expect(CGI::parse(subject.query)).to eq(
+        {
+          'AWSAccessKeyId' => ['AKIAJFTRSGRH3RXX6D5Q'],
+          'Expires' => ['1475366700'],
+          'Signature' => ['VwsX5nxGfTC3dxXjS6wSeU64r5o=']
+        }
+      )
+    end
+  end
+
+  describe 'parse_bucket_listing' do
+    let(:response) do
+      '<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+         <Name>example-bucket</Name>
+         <Prefix></Prefix>
+         <KeyCount>2</KeyCount>
+         <MaxKeys>1000</MaxKeys>
+         <Delimiter>/</Delimiter>
+         <IsTruncated>false</IsTruncated>
+         <Contents>
+           <Key>sample1.jpg</Key>
+           <LastModified>2011-02-26T01:56:20.000Z</LastModified>
+           <ETag>&quot;bf1d737a4d46a19f3bced6905cc8b902&quot;</ETag>
+           <Size>142863</Size>
+           <StorageClass>STANDARD</StorageClass>
+         </Contents>
+         <Contents>
+           <Key>sample2.jpg</Key>
+           <LastModified>2011-02-26T01:56:20.000Z</LastModified>
+           <ETag>&quot;bf1d737a4d46a19f3bced6905cc8b902&quot;</ETag>
+           <Size>142863</Size>
+           <StorageClass>STANDARD</StorageClass>
+         </Contents>
+       </ListBucketResult>'
+    end
+
+    subject { cellar_service.send(:parse_bucket_listing, response) }
+
+    it { is_expected.to eq(["sample1.jpg", "sample2.jpg"]) }
+  end
+
+  describe 'bulk_deletion_request_body' do
+    let(:expected_response) do
+      '<?xml version="1.0" encoding="UTF-8"?>
+<Delete>
+  <Quiet>true</Quiet>
+  <Object>
+    <Key>chapi</Key>
+  </Object>
+  <Object>
+    <Key>chapo</Key>
+  </Object>
+</Delete>
+'
+    end
+
+    subject { cellar_service.send(:bulk_deletion_request_body, ['chapi', 'chapo']) }
+
+    it { is_expected.to eq(expected_response) }
+  end
+end