Merge pull request #1469 from betagouv/cellar_improvements

Cellar improvements

lib/active_storage/service/cellar_service.rb

@@ -1,65 +1,53 @@
-require 'base64'
-require 'net/http'
-require 'openssl'
-
 module ActiveStorage
   class Service::CellarService < Service
     def initialize(access_key_id:, secret_access_key:, bucket:, **)
-      @endpoint = URI::HTTPS.build(host: "#{bucket}.cellar.services.clever-cloud.com")
+      @adapter = Cellar::CellarAdapter.new(access_key_id, secret_access_key, bucket)
-      @access_key_id = access_key_id
-      @secret_access_key = secret_access_key
-      @bucket = bucket
     end
 
-    def download(key)
+    def upload(key, io, checksum: nil)
-      # TODO: error handling
+      instrument :upload, key: key, checksum: checksum do
+        @adapter.session { |s| s.upload(key, io, checksum) }
+      end
+    end
+
+    def download(key, &block)
       if block_given?
         instrument :streaming_download, key: key do
-          http_start do |http|
+          @adapter.session { |s| s.download(key, &block) }
-            http.request(get_request(key)) do |response|
-              response.read_body do |chunk|
-                yield(chunk.force_encoding(Encoding::BINARY))
-              end
-            end
-          end
         end
       else
         instrument :download, key: key do
-          http_start do |http|
+          @adapter.session { |s| s.download(key) }
-            response = http.request(get_request(key))
-            if response.is_a?(Net::HTTPSuccess)
-              response.body.force_encoding(Encoding::BINARY)
-            end
-          end
         end
       end
     end
 
     def delete(key)
-      # TODO: error handling
       instrument :delete, key: key do
-        http_start do |http|
+        @adapter.session { |s| s.delete(key) }
-          perform_delete(http, key)
-        end
       end
     end
 
     def delete_prefixed(prefix)
-      # TODO: error handling
-      # TODO: handle pagination if more than 1000 keys
       instrument :delete_prefixed, prefix: prefix do
-        http_start do |http|
+        @adapter.session do |s|
-          list_prefixed(http, prefix).each do |key|
+          keys = s.list_prefixed(prefix)
-            # TODO: use bulk delete instead
+          s.delete_keys(keys)
-            perform_delete(http, key)
-          end
         end
       end
     end
 
+    def exist?(key)
+      instrument :exist, key: key do |payload|
+        answer = @adapter.session { |s| s.exist?(key) }
+        payload[:exist] = answer
+        answer
+      end
+    end
+
     def url(key, expires_in:, filename:, disposition:, content_type:)
       instrument :url, key: key do |payload|
-        generated_url = presigned_url(
+        generated_url = @adapter.presigned_url(
           method: 'GET',
           key: key,
           expires_in: expires_in,
@@ -73,7 +61,7 @@ module ActiveStorage
 
     def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
       instrument :url, key: key do |payload|
-        generated_url = presigned_url(
+        generated_url = @adapter.presigned_url(
           method: 'PUT',
           key: key,
           expires_in: expires_in,
@@ -88,80 +76,5 @@ module ActiveStorage
     def headers_for_direct_upload(key, content_type:, checksum:, **)
       { "Content-Type" => content_type, "Content-MD5" => checksum }
     end
-
-    private
-
-    def http_start(&block)
-      Net::HTTP.start(@endpoint.host, @endpoint.port, use_ssl: true, &block)
-    end
-
-    def sign(request, key, checksum: '')
-      date = Time.now.httpdate
-      sig = signature(method: request.method, key: key, date: date, checksum: checksum)
-      request['date'] = date
-      request['authorization'] = "AWS #{@access_key_id}:#{sig}"
-    end
-
-    def presigned_url(method:, key:, expires_in:, content_type: '', checksum: '', **query_params)
-      expires = expires_in.from_now.to_i
-
-      query = query_params.merge({
-        AWSAccessKeyId: @access_key_id,
-        Expires: expires,
-        Signature: signature(method: method, key: key, expires: expires, content_type: content_type, checksum: checksum)
-      })
-
-      generated_url = URI::join(@endpoint, "/#{key}","?#{query.to_query}").to_s
-    end
-
-    def signature(method:, key:, expires: '', date: '', content_type: '', checksum: '')
-      canonicalized_amz_headers = ""
-      canonicalized_resource = "/#{@bucket}/#{key}"
-      string_to_sign = "#{method}\n#{checksum}\n#{content_type}\n#{expires}#{date}\n" +
-                       "#{canonicalized_amz_headers}#{canonicalized_resource}"
-      Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), @secret_access_key, string_to_sign)).strip
-    end
-
-    def list_prefixed(http, prefix)
-      request = Net::HTTP::Get.new(URI::join(@endpoint, "/?list-type=2&prefix=#{prefix}"))
-      sign(request, "")
-      response = http.request(request)
-      if response.is_a?(Net::HTTPSuccess)
-        parse_bucket_listing(response.body)
-      end
-    end
-
-    def parse_bucket_listing(bucket_listing_xml)
-      doc = Nokogiri::XML(bucket_listing_xml)
-      doc
-        .xpath('//xmlns:Contents/xmlns:Key')
-        .map{ |k| k.text }
-    end
-
-    def get_request(key)
-      request = Net::HTTP::Get.new(URI::join(@endpoint, "/#{key}"))
-      sign(request, key)
-      request
-    end
-
-    def bulk_deletion_request_body(keys)
-      builder = Nokogiri::XML::Builder.new(:encoding => 'UTF-8') do |xml|
-        xml.Delete do
-          xml.Quiet("true")
-          keys.each do |k|
-            xml.Object do
-              xml.Key(k)
-            end
-          end
-        end
-      end
-      builder.to_xml
-    end
-
-    def perform_delete(http, key)
-      request = Net::HTTP::Delete.new(URI::join(@endpoint, "/#{key}"))
-      sign(request, key)
-      http.request(request)
-    end
   end
 end

lib/cellar/amazon_v2_request_signer.rb

@@ -0,0 +1,43 @@
+require 'base64'
+require 'openssl'
+
+module Cellar
+  class AmazonV2RequestSigner
+    def initialize(access_key_id, secret_access_key, bucket)
+      @access_key_id = access_key_id
+      @secret_access_key = secret_access_key
+      @bucket = bucket
+    end
+
+    def sign(request, key)
+      date = Time.now.httpdate
+      sig = signature(
+        method: request.method,
+        key: key,
+        date: date,
+        checksum: request['Content-MD5'] || '',
+        content_type: request.content_type || ''
+      )
+      request['date'] = date
+      request['authorization'] = "AWS #{@access_key_id}:#{sig}"
+    end
+
+    def url_signature_params(method:, key:, expires_in:, content_type: '', checksum: '')
+      expires = expires_in.from_now.to_i
+
+      {
+        AWSAccessKeyId: @access_key_id,
+        Expires: expires,
+        Signature: signature(method: method, key: key, expires: expires, content_type: content_type, checksum: checksum)
+      }
+    end
+
+    def signature(method:, key:, expires: '', date: '', content_type: '', checksum: '')
+      canonicalized_amz_headers = ""
+      canonicalized_resource = "/#{@bucket}/#{key}"
+      string_to_sign = "#{method}\n#{checksum}\n#{content_type}\n#{expires}#{date}\n" +
+                       "#{canonicalized_amz_headers}#{canonicalized_resource}"
+      Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), @secret_access_key, string_to_sign)).strip
+    end
+  end
+end

lib/cellar/cellar_adapter.rb

@@ -0,0 +1,146 @@
+require 'net/http'
+require 'openssl'
+
+module Cellar
+  class CellarAdapter
+    def initialize(access_key_id, secret_access_key, bucket)
+      @endpoint = URI::HTTPS.build(host: "#{bucket}.cellar.services.clever-cloud.com")
+      @signer = AmazonV2RequestSigner.new(access_key_id, secret_access_key, bucket)
+    end
+
+    def presigned_url(method:, key:, expires_in:, content_type: '', checksum: '', **query_params)
+      query = query_params.merge(
+        @signer.url_signature_params(
+          method: method,
+          key: key,
+          expires_in: expires_in,
+          content_type: content_type,
+          checksum: checksum
+        )
+      )
+
+      URI::join(@endpoint, "/#{key}", "?#{query.to_query}").to_s
+    end
+
+    def session
+      Net::HTTP.start(@endpoint.host, @endpoint.port, use_ssl: true) do |http|
+        yield Session.new(http, @signer)
+      end
+    end
+
+    class Session
+      def initialize(http, signer)
+        @http = http
+        @signer = signer
+      end
+
+      def upload(key, io, checksum)
+        with_io_length(io) do |io, length|
+          request = Net::HTTP::Put.new("/#{key}")
+          request.content_type = 'application/octet-stream'
+          request['Content-MD5'] = checksum
+          request['Content-Length'] = length
+          request.body_stream = io
+          @signer.sign(request, key)
+          @http.request(request)
+          # TODO: error handling
+        end
+      end
+
+      def download(key)
+        request = Net::HTTP::Get.new("/#{key}")
+        @signer.sign(request, key)
+        if block_given?
+          @http.request(request) do |response|
+            if response.is_a?(Net::HTTPSuccess)
+              response.read_body do |chunk|
+                yield(chunk.force_encoding(Encoding::BINARY))
+              end
+            else
+              # TODO: error handling
+            end
+          end
+        else
+          response = @http.request(request)
+          if response.is_a?(Net::HTTPSuccess)
+            response.body.force_encoding(Encoding::BINARY)
+          else
+            # TODO: error handling
+          end
+        end
+      end
+
+      def delete(key)
+        # TODO: error handling
+        request = Net::HTTP::Delete.new("/#{key}")
+        @signer.sign(request, key)
+        @http.request(request)
+      end
+
+      def list_prefixed(prefix)
+        request = Net::HTTP::Get.new("/?prefix=#{prefix}")
+        @signer.sign(request, "")
+        response = @http.request(request)
+        if response.is_a?(Net::HTTPSuccess)
+          parse_bucket_listing(response.body)
+        end
+      end
+
+      def delete_keys(keys)
+        request_body = bulk_deletion_request_body(keys)
+        request = Net::HTTP::Post.new("/?delete")
+        request.content_type = 'text/xml'
+        request['Content-MD5'] = Digest::MD5.base64digest(request_body)
+        request['Content-Length'] = request_body.length
+        request.body = request_body
+        @signer.sign(request, "?delete")
+        @http.request(request)
+      end
+
+      def exist?(key)
+        request = Net::HTTP::Head.new("/#{key}")
+        @signer.sign(request, key)
+        response = @http.request(request)
+        response.is_a?(Net::HTTPSuccess)
+      end
+
+      private
+
+      def parse_bucket_listing(bucket_listing_xml)
+        doc = Nokogiri::XML(bucket_listing_xml)
+        doc
+          .xpath('//xmlns:Contents/xmlns:Key')
+          .map{ |k| k.text }
+      end
+
+      def bulk_deletion_request_body(keys)
+        builder = Nokogiri::XML::Builder.new(:encoding => 'UTF-8') do |xml|
+          xml.Delete do
+            keys.each do |k|
+              xml.Object do
+                xml.Key(k)
+              end
+            end
+          end
+        end
+        builder.to_xml
+      end
+
+      def with_io_length(io)
+        if io.respond_to?(:size) && io.respond_to?(:pos)
+          yield(io, io.size - io.pos)
+        else
+          tmp_file = Tempfile.new('cellar_io_length')
+          begin
+            IO.copy_stream(io, tmp_file)
+            length = tmp_file.pos
+            tmp_file.rewind
+            yield(tmp_file, length)
+          ensure
+            tmp_file.close!
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/active_storage/service/cellar_service_spec.rb

@@ -19,34 +19,6 @@ describe 'CellarService' do
   before { Timecop.freeze(Time.gm(2016, 10, 2)) }
   after { Timecop.return }
 
-  describe 'signature generation' do
-    context 'for presigned URLs' do
-      subject do
-        cellar_service.send(
-          :signature,
-          {
-            method: 'GET',
-            key: 'fichier',
-            expires: 5.minutes.from_now.to_i
-          }
-        )
-      end
-
-      it { is_expected.to eq('nzCsB6cip8oofkuOdvvJs6FafkA=') }
-    end
-
-    context 'for server-side requests' do
-      subject do
-        Net::HTTP::Delete.new('https://rogets.cellar.services.clever-cloud.com/fichier')
-      end
-
-      before { cellar_service.send(:sign, subject, 'fichier') }
-
-      it { expect(subject['date']).to eq(Time.now.httpdate) }
-      it { expect(subject['authorization']).to eq('AWS AKIAJFTRSGRH3RXX6D5Q:nkvviwZYb1V9HDrKyJZmY3Z8sSA=') }
-    end
-  end
-
   describe 'presigned url for download' do
     subject do
       URI.parse(
@@ -112,55 +84,4 @@ describe 'CellarService' do
       )
     end
   end
-
-  describe 'parse_bucket_listing' do
-    let(:response) do
-      '<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
-         <Name>example-bucket</Name>
-         <Prefix></Prefix>
-         <KeyCount>2</KeyCount>
-         <MaxKeys>1000</MaxKeys>
-         <Delimiter>/</Delimiter>
-         <IsTruncated>false</IsTruncated>
-         <Contents>
-           <Key>sample1.jpg</Key>
-           <LastModified>2011-02-26T01:56:20.000Z</LastModified>
-           <ETag>&quot;bf1d737a4d46a19f3bced6905cc8b902&quot;</ETag>
-           <Size>142863</Size>
-           <StorageClass>STANDARD</StorageClass>
-         </Contents>
-         <Contents>
-           <Key>sample2.jpg</Key>
-           <LastModified>2011-02-26T01:56:20.000Z</LastModified>
-           <ETag>&quot;bf1d737a4d46a19f3bced6905cc8b902&quot;</ETag>
-           <Size>142863</Size>
-           <StorageClass>STANDARD</StorageClass>
-         </Contents>
-       </ListBucketResult>'
-    end
-
-    subject { cellar_service.send(:parse_bucket_listing, response) }
-
-    it { is_expected.to eq(["sample1.jpg", "sample2.jpg"]) }
-  end
-
-  describe 'bulk_deletion_request_body' do
-    let(:expected_response) do
-      '<?xml version="1.0" encoding="UTF-8"?>
-<Delete>
-  <Quiet>true</Quiet>
-  <Object>
-    <Key>chapi</Key>
-  </Object>
-  <Object>
-    <Key>chapo</Key>
-  </Object>
-</Delete>
-'
-    end
-
-    subject { cellar_service.send(:bulk_deletion_request_body, ['chapi', 'chapo']) }
-
-    it { is_expected.to eq(expected_response) }
-  end
 end

spec/lib/cellar/amazon_v2_request_signer_spec.rb

@@ -0,0 +1,43 @@
+require 'net/http'
+
+describe 'AmazonV2RequestSigner' do
+  let(:request_signer) do
+    # These are actual keys, but they’re safe to put here because
+    # - they never had any rights attached, and
+    # - the keys were revoked before copying them here
+
+    Cellar::AmazonV2RequestSigner.new(
+      'AKIAJFTRSGRH3RXX6D5Q',
+      '3/y/3Tf5zkfcrTaLFxyKB/oU2/7ay7/Dz8UdEHC7',
+      'rogets'
+    )
+  end
+
+  before { Timecop.freeze(Time.gm(2016, 10, 2)) }
+  after { Timecop.return }
+
+  describe 'signature generation' do
+    context 'for presigned URLs' do
+      subject do
+        request_signer.signature(
+          method: 'GET',
+          key: 'fichier',
+          expires: 5.minutes.from_now.to_i
+        )
+      end
+
+      it { is_expected.to eq('nzCsB6cip8oofkuOdvvJs6FafkA=') }
+    end
+
+    context 'for server-side requests' do
+      subject do
+        Net::HTTP::Delete.new('https://rogets.cellar.services.clever-cloud.com/fichier')
+      end
+
+      before { request_signer.sign(subject, 'fichier') }
+
+      it { expect(subject['date']).to eq(Time.now.httpdate) }
+      it { expect(subject['authorization']).to eq('AWS AKIAJFTRSGRH3RXX6D5Q:nkvviwZYb1V9HDrKyJZmY3Z8sSA=') }
+    end
+  end
+end

spec/lib/cellar/cellar_adapter_spec.rb

@@ -0,0 +1,56 @@
+describe 'CellarAdapter' do
+  let(:session) { Cellar::CellarAdapter::Session.new(nil, nil) }
+
+  before { Timecop.freeze(Time.gm(2016, 10, 2)) }
+  after { Timecop.return }
+
+  describe 'parse_bucket_listing' do
+    let(:response) do
+      '<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+         <Name>example-bucket</Name>
+         <Prefix></Prefix>
+         <KeyCount>2</KeyCount>
+         <MaxKeys>1000</MaxKeys>
+         <Delimiter>/</Delimiter>
+         <IsTruncated>false</IsTruncated>
+         <Contents>
+           <Key>sample1.jpg</Key>
+           <LastModified>2011-02-26T01:56:20.000Z</LastModified>
+           <ETag>&quot;bf1d737a4d46a19f3bced6905cc8b902&quot;</ETag>
+           <Size>142863</Size>
+           <StorageClass>STANDARD</StorageClass>
+         </Contents>
+         <Contents>
+           <Key>sample2.jpg</Key>
+           <LastModified>2011-02-26T01:56:20.000Z</LastModified>
+           <ETag>&quot;bf1d737a4d46a19f3bced6905cc8b902&quot;</ETag>
+           <Size>142863</Size>
+           <StorageClass>STANDARD</StorageClass>
+         </Contents>
+       </ListBucketResult>'
+    end
+
+    subject { session.send(:parse_bucket_listing, response) }
+
+    it { is_expected.to eq(["sample1.jpg", "sample2.jpg"]) }
+  end
+
+  describe 'bulk_deletion_request_body' do
+    let(:expected_response) do
+      '<?xml version="1.0" encoding="UTF-8"?>
+<Delete>
+  <Object>
+    <Key>chapi</Key>
+  </Object>
+  <Object>
+    <Key>chapo</Key>
+  </Object>
+</Delete>
+'
+    end
+
+    subject { session.send(:bulk_deletion_request_body, ['chapi', 'chapo']) }
+
+    it { is_expected.to eq(expected_response) }
+  end
+end