Merge pull request #1404 from betagouv/cellar_integration
Active storage and Clever Cloud Cellar integration
This commit is contained in:
commit
fcba5c8523
2 changed files with 333 additions and 0 deletions
167
lib/active_storage/service/cellar_service.rb
Normal file
167
lib/active_storage/service/cellar_service.rb
Normal file
|
@ -0,0 +1,167 @@
|
|||
require 'base64'
|
||||
require 'net/http'
|
||||
require 'openssl'
|
||||
|
||||
module ActiveStorage
|
||||
class Service::CellarService < Service
|
||||
def initialize(access_key_id:, secret_access_key:, bucket:, **)
|
||||
@endpoint = URI::HTTPS.build(host: "#{bucket}.cellar.services.clever-cloud.com")
|
||||
@access_key_id = access_key_id
|
||||
@secret_access_key = secret_access_key
|
||||
@bucket = bucket
|
||||
end
|
||||
|
||||
def download(key)
|
||||
# TODO: error handling
|
||||
if block_given?
|
||||
instrument :streaming_download, key: key do
|
||||
http_start do |http|
|
||||
http.request(get_request(key)) do |response|
|
||||
response.read_body do |chunk|
|
||||
yield(chunk.force_encoding(Encoding::BINARY))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
else
|
||||
instrument :download, key: key do
|
||||
http_start do |http|
|
||||
response = http.request(get_request(key))
|
||||
if response.is_a?(Net::HTTPSuccess)
|
||||
response.body.force_encoding(Encoding::BINARY)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def delete(key)
|
||||
# TODO: error handling
|
||||
instrument :delete, key: key do
|
||||
http_start do |http|
|
||||
perform_delete(http, key)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def delete_prefixed(prefix)
|
||||
# TODO: error handling
|
||||
# TODO: handle pagination if more than 1000 keys
|
||||
instrument :delete_prefixed, prefix: prefix do
|
||||
http_start do |http|
|
||||
list_prefixed(http, prefix).each do |key|
|
||||
# TODO: use bulk delete instead
|
||||
perform_delete(http, key)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def url(key, expires_in:, filename:, disposition:, content_type:)
|
||||
instrument :url, key: key do |payload|
|
||||
generated_url = presigned_url(
|
||||
method: 'GET',
|
||||
key: key,
|
||||
expires_in: expires_in,
|
||||
"response-content-disposition": content_disposition_with(type: disposition, filename: filename),
|
||||
"response-content-type": content_type
|
||||
)
|
||||
payload[:url] = generated_url
|
||||
generated_url
|
||||
end
|
||||
end
|
||||
|
||||
def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
|
||||
instrument :url, key: key do |payload|
|
||||
generated_url = presigned_url(
|
||||
method: 'PUT',
|
||||
key: key,
|
||||
expires_in: expires_in,
|
||||
content_type: content_type,
|
||||
checksum: checksum
|
||||
)
|
||||
payload[:url] = generated_url
|
||||
generated_url
|
||||
end
|
||||
end
|
||||
|
||||
def headers_for_direct_upload(key, content_type:, checksum:, **)
|
||||
{ "Content-Type" => content_type, "Content-MD5" => checksum }
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def http_start(&block)
|
||||
Net::HTTP.start(@endpoint.host, @endpoint.port, use_ssl: true, &block)
|
||||
end
|
||||
|
||||
def sign(request, key, checksum: '')
|
||||
date = Time.now.httpdate
|
||||
sig = signature(method: request.method, key: key, date: date, checksum: checksum)
|
||||
request['date'] = date
|
||||
request['authorization'] = "AWS #{@access_key_id}:#{sig}"
|
||||
end
|
||||
|
||||
def presigned_url(method:, key:, expires_in:, content_type: '', checksum: '', **query_params)
|
||||
expires = expires_in.from_now.to_i
|
||||
|
||||
query = query_params.merge({
|
||||
AWSAccessKeyId: @access_key_id,
|
||||
Expires: expires,
|
||||
Signature: signature(method: method, key: key, expires: expires, content_type: content_type, checksum: checksum)
|
||||
})
|
||||
|
||||
generated_url = URI::join(@endpoint, "/#{key}","?#{query.to_query}").to_s
|
||||
end
|
||||
|
||||
def signature(method:, key:, expires: '', date: '', content_type: '', checksum: '')
|
||||
canonicalized_amz_headers = ""
|
||||
canonicalized_resource = "/#{@bucket}/#{key}"
|
||||
string_to_sign = "#{method}\n#{checksum}\n#{content_type}\n#{expires}#{date}\n" +
|
||||
"#{canonicalized_amz_headers}#{canonicalized_resource}"
|
||||
Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), @secret_access_key, string_to_sign)).strip
|
||||
end
|
||||
|
||||
def list_prefixed(http, prefix)
|
||||
request = Net::HTTP::Get.new(URI::join(@endpoint, "/?list-type=2&prefix=#{prefix}"))
|
||||
sign(request, "")
|
||||
response = http.request(request)
|
||||
if response.is_a?(Net::HTTPSuccess)
|
||||
parse_bucket_listing(response.body)
|
||||
end
|
||||
end
|
||||
|
||||
def parse_bucket_listing(bucket_listing_xml)
|
||||
doc = Nokogiri::XML(bucket_listing_xml)
|
||||
doc
|
||||
.xpath('//xmlns:Contents/xmlns:Key')
|
||||
.map{ |k| k.text }
|
||||
end
|
||||
|
||||
def get_request(key)
|
||||
request = Net::HTTP::Get.new(URI::join(@endpoint, "/#{key}"))
|
||||
sign(request, key)
|
||||
request
|
||||
end
|
||||
|
||||
def bulk_deletion_request_body(keys)
|
||||
builder = Nokogiri::XML::Builder.new(:encoding => 'UTF-8') do |xml|
|
||||
xml.Delete do
|
||||
xml.Quiet("true")
|
||||
keys.each do |k|
|
||||
xml.Object do
|
||||
xml.Key(k)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
builder.to_xml
|
||||
end
|
||||
|
||||
def perform_delete(http, key)
|
||||
request = Net::HTTP::Delete.new(URI::join(@endpoint, "/#{key}"))
|
||||
sign(request, key)
|
||||
http.request(request)
|
||||
end
|
||||
end
|
||||
end
|
166
spec/lib/active_storage/service/cellar_service_spec.rb
Normal file
166
spec/lib/active_storage/service/cellar_service_spec.rb
Normal file
|
@ -0,0 +1,166 @@
|
|||
require 'active_storage/service/cellar_service'
|
||||
require 'cgi'
|
||||
require 'net/http'
|
||||
require 'uri'
|
||||
|
||||
describe 'CellarService' do
|
||||
let(:cellar_service) do
|
||||
# These are actual keys, but they’re safe to put here because
|
||||
# - they never had any rights attached, and
|
||||
# - the keys were revoked before copying them here
|
||||
|
||||
ActiveStorage::Service::CellarService.new(
|
||||
access_key_id: 'AKIAJFTRSGRH3RXX6D5Q',
|
||||
secret_access_key: '3/y/3Tf5zkfcrTaLFxyKB/oU2/7ay7/Dz8UdEHC7',
|
||||
bucket: 'rogets'
|
||||
)
|
||||
end
|
||||
|
||||
before { Timecop.freeze(Time.gm(2016, 10, 2)) }
|
||||
after { Timecop.return }
|
||||
|
||||
describe 'signature generation' do
|
||||
context 'for presigned URLs' do
|
||||
subject do
|
||||
cellar_service.send(
|
||||
:signature,
|
||||
{
|
||||
method: 'GET',
|
||||
key: 'fichier',
|
||||
expires: 5.minutes.from_now.to_i
|
||||
}
|
||||
)
|
||||
end
|
||||
|
||||
it { is_expected.to eq('nzCsB6cip8oofkuOdvvJs6FafkA=') }
|
||||
end
|
||||
|
||||
context 'for server-side requests' do
|
||||
subject do
|
||||
Net::HTTP::Delete.new('https://rogets.cellar.services.clever-cloud.com/fichier')
|
||||
end
|
||||
|
||||
before { cellar_service.send(:sign, subject, 'fichier') }
|
||||
|
||||
it { expect(subject['date']).to eq(Time.now.httpdate) }
|
||||
it { expect(subject['authorization']).to eq('AWS AKIAJFTRSGRH3RXX6D5Q:nkvviwZYb1V9HDrKyJZmY3Z8sSA=') }
|
||||
end
|
||||
end
|
||||
|
||||
describe 'presigned url for download' do
|
||||
subject do
|
||||
URI.parse(
|
||||
cellar_service.url(
|
||||
'fichier',
|
||||
expires_in: 5.minutes,
|
||||
filename: ActiveStorage::Filename.new("toto.png"),
|
||||
disposition: 'attachment',
|
||||
content_type: 'image/png'
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
it do
|
||||
is_expected.to have_attributes(
|
||||
scheme: 'https',
|
||||
host: 'rogets.cellar.services.clever-cloud.com',
|
||||
path: '/fichier'
|
||||
)
|
||||
end
|
||||
|
||||
it do
|
||||
expect(CGI::parse(subject.query)).to eq(
|
||||
{
|
||||
'AWSAccessKeyId' => ['AKIAJFTRSGRH3RXX6D5Q'],
|
||||
'Expires' => ['1475366700'],
|
||||
'Signature' => ['nzCsB6cip8oofkuOdvvJs6FafkA='],
|
||||
'response-content-disposition' => ["attachment; filename=\"toto.png\"; filename*=UTF-8''toto.png"],
|
||||
'response-content-type' => ['image/png'],
|
||||
}
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
describe 'presigned url for direct upload' do
|
||||
subject do
|
||||
URI.parse(
|
||||
cellar_service.url_for_direct_upload(
|
||||
'fichier',
|
||||
expires_in: 5.minutes,
|
||||
content_type: 'image/png',
|
||||
content_length: 2713,
|
||||
checksum: 'DEADBEEF'
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
it do
|
||||
is_expected.to have_attributes(
|
||||
scheme: 'https',
|
||||
host: 'rogets.cellar.services.clever-cloud.com',
|
||||
path: '/fichier'
|
||||
)
|
||||
end
|
||||
|
||||
it do
|
||||
expect(CGI::parse(subject.query)).to eq(
|
||||
{
|
||||
'AWSAccessKeyId' => ['AKIAJFTRSGRH3RXX6D5Q'],
|
||||
'Expires' => ['1475366700'],
|
||||
'Signature' => ['VwsX5nxGfTC3dxXjS6wSeU64r5o=']
|
||||
}
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
describe 'parse_bucket_listing' do
|
||||
let(:response) do
|
||||
'<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>example-bucket</Name>
|
||||
<Prefix></Prefix>
|
||||
<KeyCount>2</KeyCount>
|
||||
<MaxKeys>1000</MaxKeys>
|
||||
<Delimiter>/</Delimiter>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents>
|
||||
<Key>sample1.jpg</Key>
|
||||
<LastModified>2011-02-26T01:56:20.000Z</LastModified>
|
||||
<ETag>"bf1d737a4d46a19f3bced6905cc8b902"</ETag>
|
||||
<Size>142863</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>
|
||||
<Contents>
|
||||
<Key>sample2.jpg</Key>
|
||||
<LastModified>2011-02-26T01:56:20.000Z</LastModified>
|
||||
<ETag>"bf1d737a4d46a19f3bced6905cc8b902"</ETag>
|
||||
<Size>142863</Size>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
</Contents>
|
||||
</ListBucketResult>'
|
||||
end
|
||||
|
||||
subject { cellar_service.send(:parse_bucket_listing, response) }
|
||||
|
||||
it { is_expected.to eq(["sample1.jpg", "sample2.jpg"]) }
|
||||
end
|
||||
|
||||
describe 'bulk_deletion_request_body' do
|
||||
let(:expected_response) do
|
||||
'<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Delete>
|
||||
<Quiet>true</Quiet>
|
||||
<Object>
|
||||
<Key>chapi</Key>
|
||||
</Object>
|
||||
<Object>
|
||||
<Key>chapo</Key>
|
||||
</Object>
|
||||
</Delete>
|
||||
'
|
||||
end
|
||||
|
||||
subject { cellar_service.send(:bulk_deletion_request_body, ['chapi', 'chapo']) }
|
||||
|
||||
it { is_expected.to eq(expected_response) }
|
||||
end
|
||||
end
|
Loading…
Reference in a new issue