DGNum/demarches-normaliennes
15
1
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

Merge pull request #3176 from betagouv/frederic/drop-cellar

Browse source 
Drop CleverCloud Service for ActiveStorage
This commit is contained in:
Frederic Merizen 2018-12-19 10:40:06 +01:00 committed by GitHub
commit d2d24eacb6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 1 additions and 549 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
config/env.example
View file

@ -25,10 +25,6 @@ FOG_DIRECTORY=""
FOG_ENABLED=""
CARRIERWAVE_CACHE_DIR="/tmp/tps-local-cache"
CLEVER_CLOUD_ACCESS_KEY_ID=""
CLEVER_CLOUD_SECRET_ACCESS_KEY=""
CLEVER_CLOUD_BUCKET=""
FC_PARTICULIER_ID=""
FC_PARTICULIER_SECRET=""
FC_PARTICULIER_BASE_URL=""

5
config/storage.yml
View file

@ -4,11 +4,6 @@ local:
test:
service: Disk
root: <%= Rails.root.join("tmp/storage") %>
clever_cloud:
service: Cellar
access_key_id: <%= ENV['CLEVER_CLOUD_ACCESS_KEY_ID'] %>
secret_access_key: <%= ENV['CLEVER_CLOUD_SECRET_ACCESS_KEY'] %>
bucket: <%= ENV['CLEVER_CLOUD_BUCKET'] %>
openstack:
service: OpenStack
container: "<%= ENV['FOG_ACTIVESTORAGE_DIRECTORY'] %>"

86
lib/active_storage/service/cellar_service.rb
View file

@ -1,86 +0,0 @@
module ActiveStorage
class Service::CellarService < Service
def initialize(access_key_id:, secret_access_key:, bucket:, **)
@adapter = Cellar::CellarAdapter.new(access_key_id, secret_access_key, bucket)
end
def upload(key, io, checksum: nil, **)
instrument :upload, key: key, checksum: checksum do
@adapter.session { |s| s.upload(key, io, checksum) }
end
end
def download(key, &block)
if block_given?
instrument :streaming_download, key: key do
@adapter.session { |s| s.download(key, &block) }
end
else
instrument :download, key: key do
@adapter.session { |s| s.download(key) }
end
end
end
def download_chunk(key, range)
instrument :download_chunk, key: key, range: range do
@adapter.session { |s| s.download(key, range: range) }
end
end
def delete(key)
instrument :delete, key: key do
@adapter.session { |s| s.delete(key) }
end
end
def delete_prefixed(prefix)
instrument :delete_prefixed, prefix: prefix do
@adapter.session do |s|
keys = s.list_prefixed(prefix).map(&:first)
s.delete_keys(keys)
end
end
end
def exist?(key)
instrument :exist, key: key do |payload|
answer = @adapter.session { |s| s.exist?(key) }
payload[:exist] = answer
answer
end
end
def url(key, expires_in:, filename:, disposition:, content_type:)
instrument :url, key: key do |payload|
generated_url = @adapter.presigned_url(
method: 'GET',
key: key,
expires_in: expires_in,
"response-content-disposition": content_disposition_with(type: disposition, filename: filename),
"response-content-type": content_type
)
payload[:url] = generated_url
generated_url
end
end
def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:)
instrument :url, key: key do |payload|
generated_url = @adapter.presigned_url(
method: 'PUT',
key: key,
expires_in: expires_in,
content_type: content_type,
checksum: checksum
)
payload[:url] = generated_url
generated_url
end
end
def headers_for_direct_upload(key, content_type:, checksum:, **)
{ "Content-Type" => content_type, "Content-MD5" => checksum }
end
end
end

43
lib/cellar/amazon_v2_request_signer.rb
View file

@ -1,43 +0,0 @@
require 'base64'
require 'openssl'
module Cellar
class AmazonV2RequestSigner
def initialize(access_key_id, secret_access_key, bucket)
@access_key_id = access_key_id
@secret_access_key = secret_access_key
@bucket = bucket
end
def sign(request, key)
date = Time.zone.now.httpdate
sig = signature(
method: request.method,
key: key,
date: date,
checksum: request['Content-MD5'] || '',
content_type: request.content_type || ''
)
request['date'] = date
request['authorization'] = "AWS #{@access_key_id}:#{sig}"
end
def url_signature_params(method:, key:, expires_in:, content_type: '', checksum: '')
expires = expires_in.from_now.to_i
{
AWSAccessKeyId: @access_key_id,
Expires: expires,
Signature: signature(method: method, key: key, expires: expires, content_type: content_type, checksum: checksum)
}
end
def signature(method:, key:, expires: '', date: '', content_type: '', checksum: '')
canonicalized_amz_headers = ""
canonicalized_resource = "/#{@bucket}/#{key}"
string_to_sign = "#{method}\n#{checksum}\n#{content_type}\n#{expires}#{date}\n" +
"#{canonicalized_amz_headers}#{canonicalized_resource}"
Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), @secret_access_key, string_to_sign)).strip
end
end
end

183
lib/cellar/cellar_adapter.rb
View file

@ -1,183 +0,0 @@
require 'net/http'
require 'openssl'
module Cellar
class CellarAdapter
def initialize(access_key_id, secret_access_key, bucket)
@endpoint = URI::HTTPS.build(host: "#{bucket}.cellar.services.clever-cloud.com")
@signer = AmazonV2RequestSigner.new(access_key_id, secret_access_key, bucket)
end
def presigned_url(method:, key:, expires_in:, content_type: '', checksum: '', **query_params)
query = query_params.merge(
@signer.url_signature_params(
method: method,
key: key,
expires_in: expires_in,
content_type: content_type,
checksum: checksum
)
)
URI::join(@endpoint, "/#{key}", "?#{query.to_query}").to_s
end
def session
Net::HTTP.start(@endpoint.host, @endpoint.port, use_ssl: true) do |http|
yield Session.new(http, @signer)
end
end
class Session
def initialize(http, signer)
@http = http
@signer = signer
end
def upload(key, io, checksum)
with_io_length(io) do |io, length|
request = Net::HTTP::Put.new("/#{key}")
request.content_type = 'application/octet-stream'
request['Content-MD5'] = checksum
request['Content-Length'] = length
request.body_stream = io
@signer.sign(request, key)
@http.request(request)
# TODO: error handling
end
end
def download(key, range: nil)
request = Net::HTTP::Get.new("/#{key}")
if range.present?
add_range_header(request, range)
end
@signer.sign(request, key)
if block_given?
@http.request(request) do |response|
if response.is_a?(Net::HTTPSuccess)
response.read_body do |chunk|
yield(chunk.force_encoding(Encoding::BINARY))
end
else
# TODO: error handling
end
end
else
response = @http.request(request)
if response.is_a?(Net::HTTPSuccess)
response.body.force_encoding(Encoding::BINARY)
else
# TODO: error handling
end
end
end
def delete(key)
# TODO: error handling
request = Net::HTTP::Delete.new("/#{key}")
@signer.sign(request, key)
@http.request(request)
end
def list_prefixed(prefix)
result = []
marker = ''
begin
request = Net::HTTP::Get.new("/?prefix=#{prefix}&marker=#{marker}")
@signer.sign(request, "")
response = @http.request(request)
if response.is_a?(Net::HTTPSuccess)
(listing, truncated) = parse_bucket_listing(response.body)
result += listing
marker = listing.last.first
else
# TODO: error handling
return nil
end
end while truncated
result
end
def delete_keys(keys)
request_body = bulk_deletion_request_body(keys)
request = Net::HTTP::Post.new("/?delete")
request.content_type = 'text/xml'
request['Content-MD5'] = Digest::MD5.base64digest(request_body)
request['Content-Length'] = request_body.length
request.body = request_body
@signer.sign(request, "?delete")
@http.request(request)
end
def exist?(key)
request = Net::HTTP::Head.new("/#{key}")
@signer.sign(request, key)
response = @http.request(request)
response.is_a?(Net::HTTPSuccess)
end
def last_modified(key)
request = Net::HTTP::Head.new("/#{key}")
@signer.sign(request, key)
response = @http.request(request)
if response.is_a?(Net::HTTPSuccess)
Time.zone.parse(response['Last-Modified'])
end
end
private
def add_range_header(request, range)
bytes_end = range.exclude_end? ? range.end - 1 : range.end
request['range'] = "bytes=#{range.begin}-#{bytes_end}"
end
def parse_bucket_listing(bucket_listing_xml)
doc = Nokogiri::XML(bucket_listing_xml)
listing = doc
.xpath('//xmlns:Contents')
.map do |node|
[
node.xpath('xmlns:Key').text,
DateTime.iso8601(node.xpath('xmlns:LastModified').text)
]
end
truncated = doc.xpath('//xmlns:IsTruncated').text == 'true'
[listing, truncated]
end
def bulk_deletion_request_body(keys)
builder = Nokogiri::XML::Builder.new(:encoding => 'UTF-8') do |xml|
xml.Delete do
keys.each do |k|
xml.Object do
xml.Key(k)
end
end
end
end
builder.to_xml
end
def with_io_length(io)
if io.respond_to?(:size) && io.respond_to?(:pos)
yield(io, io.size - io.pos)
else
tmp_file = Tempfile.new('cellar_io_length')
begin
IO.copy_stream(io, tmp_file)
length = tmp_file.pos
tmp_file.rewind
yield(tmp_file, length)
ensure
tmp_file.close!
end
end
end
end
end
end

10
lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake
View file

@ -32,15 +32,7 @@ namespace :'2018_12_03_finish_piece_jointe_transfer' do
end
def old_pj_adapter
if !defined? @old_pj_adapter
@old_pj_adapter = Cellar::CellarAdapter.new(
ENV['CLEVER_CLOUD_ACCESS_KEY_ID'],
ENV['CLEVER_CLOUD_SECRET_ACCESS_KEY'],
ENV['CLEVER_CLOUD_BUCKET']
)
end
@old_pj_adapter
raise NotImplementedError, "No connection adapter for old PJ storage"
end
def new_pjs

87
spec/lib/active_storage/service/cellar_service_spec.rb
View file

@ -1,87 +0,0 @@
require 'active_storage/service/cellar_service'
require 'cgi'
require 'net/http'
require 'uri'
describe 'CellarService' do
let(:cellar_service) do
# These are actual keys, but they’re safe to put here because
# - they never had any rights attached, and
# - the keys were revoked before copying them here
ActiveStorage::Service::CellarService.new(
access_key_id: 'AKIAJFTRSGRH3RXX6D5Q',
secret_access_key: '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',
bucket: 'rogets'
)
end
before { Timecop.freeze(Time.gm(2016, 10, 2)) }
after { Timecop.return }
describe 'presigned url for download' do
subject do
URI.parse(
cellar_service.url(
'fichier',
expires_in: 5.minutes,
filename: ActiveStorage::Filename.new("toto.png"),
disposition: 'attachment',
content_type: 'image/png'
)
)
end
it do
is_expected.to have_attributes(
scheme: 'https',
host: 'rogets.cellar.services.clever-cloud.com',
path: '/fichier'
)
end
it do
expect(CGI::parse(subject.query)).to eq(
{
'AWSAccessKeyId' => ['AKIAJFTRSGRH3RXX6D5Q'],
'Expires' => ['1475366700'],
'Signature' => ['nzCsB6cip8oofkuOdvvJs6FafkA='],
'response-content-disposition' => ["attachment; filename=\"toto.png\"; filename*=UTF-8''toto.png"],
'response-content-type' => ['image/png']
}
)
end
end
describe 'presigned url for direct upload' do
subject do
URI.parse(
cellar_service.url_for_direct_upload(
'fichier',
expires_in: 5.minutes,
content_type: 'image/png',
content_length: 2713,
checksum: 'DEADBEEF'
)
)
end
it do
is_expected.to have_attributes(
scheme: 'https',
host: 'rogets.cellar.services.clever-cloud.com',
path: '/fichier'
)
end
it do
expect(CGI::parse(subject.query)).to eq(
{
'AWSAccessKeyId' => ['AKIAJFTRSGRH3RXX6D5Q'],
'Expires' => ['1475366700'],
'Signature' => ['VwsX5nxGfTC3dxXjS6wSeU64r5o=']
}
)
end
end
end

43
spec/lib/cellar/amazon_v2_request_signer_spec.rb
View file

@ -1,43 +0,0 @@
require 'net/http'
describe 'AmazonV2RequestSigner' do
let(:request_signer) do
# These are actual keys, but they’re safe to put here because
# - they never had any rights attached, and
# - the keys were revoked before copying them here
Cellar::AmazonV2RequestSigner.new(
'AKIAJFTRSGRH3RXX6D5Q',
'3/y/3Tf5zkfcrTaLFxyKB/oU2/7ay7/Dz8UdEHC7',
'rogets'
)
end
before { Timecop.freeze(Time.gm(2016, 10, 2)) }
after { Timecop.return }
describe 'signature generation' do
context 'for presigned URLs' do
subject do
request_signer.signature(
method: 'GET',
key: 'fichier',
expires: 5.minutes.from_now.to_i
)
end
it { is_expected.to eq('nzCsB6cip8oofkuOdvvJs6FafkA=') }
end
context 'for server-side requests' do
subject do
Net::HTTP::Delete.new('https://rogets.cellar.services.clever-cloud.com/fichier')
end
before { request_signer.sign(subject, 'fichier') }
it { expect(subject['date']).to eq(Time.zone.now.httpdate) }
it { expect(subject['authorization']).to eq('AWS AKIAJFTRSGRH3RXX6D5Q:nkvviwZYb1V9HDrKyJZmY3Z8sSA=') }
end
end
end

89
spec/lib/cellar/cellar_adapter_spec.rb
View file

@ -1,89 +0,0 @@
describe 'CellarAdapter' do
let(:session) { Cellar::CellarAdapter::Session.new(nil, nil) }
before { Timecop.freeze(Time.gm(2016, 10, 2)) }
after { Timecop.return }
describe 'add_range_header' do
let(:request) { Net::HTTP::Get.new('/whatever') }
before { session.send(:add_range_header, request, range) }
subject { request['range'] }
context 'with end included' do
let(:range) { 100..500 }
it { is_expected.to eq('bytes=100-500') }
end
context 'with end excluded' do
let(:range) { 10...50 }
it { is_expected.to eq('bytes=10-49') }
end
end
describe 'parse_bucket_listing' do
let(:response) do
<<~EOS
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>example-bucket</Name>
<Prefix></Prefix>
<KeyCount>2</KeyCount>
<MaxKeys>1000</MaxKeys>
<Delimiter>/</Delimiter>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>sample1.jpg</Key>
<LastModified>2011-02-26T01:56:20.000Z</LastModified>
<ETag>&quot;bf1d737a4d46a19f3bced6905cc8b902&quot;</ETag>
<Size>142863</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>sample2.jpg</Key>
<LastModified>2014-03-21T17:44:07.000Z</LastModified>
<ETag>&quot;bf1d737a4d46a19f3bced6905cc8b902&quot;</ETag>
<Size>142863</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
</ListBucketResult>'
EOS
end
subject { session.send(:parse_bucket_listing, response) }
it do
is_expected.to eq(
[
[
["sample1.jpg", DateTime.new(2011, 2, 26, 1, 56, 20, 0)],
["sample2.jpg", DateTime.new(2014, 3, 21, 17, 44, 7, 0)]
],
false
]
)
end
end
describe 'bulk_deletion_request_body' do
let(:expected_response) do
<<~EOS
<?xml version="1.0" encoding="UTF-8"?>
<Delete>
<Object>
<Key>chapi</Key>
</Object>
<Object>
<Key>chapo</Key>
</Object>
</Delete>
EOS
end
subject { session.send(:bulk_deletion_request_body, ['chapi', 'chapo']) }
it { is_expected.to eq(expected_response) }
end
end