From 832b4a61bcd8286a0b203f66ac159aeac871d704 Mon Sep 17 00:00:00 2001 From: Frederic Merizen Date: Tue, 11 Dec 2018 09:55:45 +0100 Subject: [PATCH] Drop CleverCloud Service for ActiveStorage --- config/env.example | 4 - config/storage.yml | 5 - lib/active_storage/service/cellar_service.rb | 86 -------- lib/cellar/amazon_v2_request_signer.rb | 43 ---- lib/cellar/cellar_adapter.rb | 183 ------------------ ...18_12_03_finish_piece_jointe_transfer.rake | 10 +- .../service/cellar_service_spec.rb | 87 --------- .../cellar/amazon_v2_request_signer_spec.rb | 43 ---- spec/lib/cellar/cellar_adapter_spec.rb | 89 --------- 9 files changed, 1 insertion(+), 549 deletions(-) delete mode 100644 lib/active_storage/service/cellar_service.rb delete mode 100644 lib/cellar/amazon_v2_request_signer.rb delete mode 100644 lib/cellar/cellar_adapter.rb delete mode 100644 spec/lib/active_storage/service/cellar_service_spec.rb delete mode 100644 spec/lib/cellar/amazon_v2_request_signer_spec.rb delete mode 100644 spec/lib/cellar/cellar_adapter_spec.rb diff --git a/config/env.example b/config/env.example index fdfef28c6..d3703ea5e 100644 --- a/config/env.example +++ b/config/env.example @@ -25,10 +25,6 @@ FOG_DIRECTORY="" FOG_ENABLED="" CARRIERWAVE_CACHE_DIR="/tmp/tps-local-cache" -CLEVER_CLOUD_ACCESS_KEY_ID="" -CLEVER_CLOUD_SECRET_ACCESS_KEY="" -CLEVER_CLOUD_BUCKET="" - FC_PARTICULIER_ID="" FC_PARTICULIER_SECRET="" FC_PARTICULIER_BASE_URL="" diff --git a/config/storage.yml b/config/storage.yml index 2d3348eab..0427a3f7a 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -4,11 +4,6 @@ local: test: service: Disk root: <%= Rails.root.join("tmp/storage") %> -clever_cloud: - service: Cellar - access_key_id: <%= ENV['CLEVER_CLOUD_ACCESS_KEY_ID'] %> - secret_access_key: <%= ENV['CLEVER_CLOUD_SECRET_ACCESS_KEY'] %> - bucket: <%= ENV['CLEVER_CLOUD_BUCKET'] %> openstack: service: OpenStack container: "<%= ENV['FOG_ACTIVESTORAGE_DIRECTORY'] %>" diff --git a/lib/active_storage/service/cellar_service.rb b/lib/active_storage/service/cellar_service.rb deleted file mode 100644 index 56b00e24b..000000000 --- a/lib/active_storage/service/cellar_service.rb +++ /dev/null @@ -1,86 +0,0 @@ -module ActiveStorage - class Service::CellarService < Service - def initialize(access_key_id:, secret_access_key:, bucket:, **) - @adapter = Cellar::CellarAdapter.new(access_key_id, secret_access_key, bucket) - end - - def upload(key, io, checksum: nil, **) - instrument :upload, key: key, checksum: checksum do - @adapter.session { |s| s.upload(key, io, checksum) } - end - end - - def download(key, &block) - if block_given? - instrument :streaming_download, key: key do - @adapter.session { |s| s.download(key, &block) } - end - else - instrument :download, key: key do - @adapter.session { |s| s.download(key) } - end - end - end - - def download_chunk(key, range) - instrument :download_chunk, key: key, range: range do - @adapter.session { |s| s.download(key, range: range) } - end - end - - def delete(key) - instrument :delete, key: key do - @adapter.session { |s| s.delete(key) } - end - end - - def delete_prefixed(prefix) - instrument :delete_prefixed, prefix: prefix do - @adapter.session do |s| - keys = s.list_prefixed(prefix).map(&:first) - s.delete_keys(keys) - end - end - end - - def exist?(key) - instrument :exist, key: key do |payload| - answer = @adapter.session { |s| s.exist?(key) } - payload[:exist] = answer - answer - end - end - - def url(key, expires_in:, filename:, disposition:, content_type:) - instrument :url, key: key do |payload| - generated_url = @adapter.presigned_url( - method: 'GET', - key: key, - expires_in: expires_in, - "response-content-disposition": content_disposition_with(type: disposition, filename: filename), - "response-content-type": content_type - ) - payload[:url] = generated_url - generated_url - end - end - - def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) - instrument :url, key: key do |payload| - generated_url = @adapter.presigned_url( - method: 'PUT', - key: key, - expires_in: expires_in, - content_type: content_type, - checksum: checksum - ) - payload[:url] = generated_url - generated_url - end - end - - def headers_for_direct_upload(key, content_type:, checksum:, **) - { "Content-Type" => content_type, "Content-MD5" => checksum } - end - end -end diff --git a/lib/cellar/amazon_v2_request_signer.rb b/lib/cellar/amazon_v2_request_signer.rb deleted file mode 100644 index 03bec0b04..000000000 --- a/lib/cellar/amazon_v2_request_signer.rb +++ /dev/null @@ -1,43 +0,0 @@ -require 'base64' -require 'openssl' - -module Cellar - class AmazonV2RequestSigner - def initialize(access_key_id, secret_access_key, bucket) - @access_key_id = access_key_id - @secret_access_key = secret_access_key - @bucket = bucket - end - - def sign(request, key) - date = Time.zone.now.httpdate - sig = signature( - method: request.method, - key: key, - date: date, - checksum: request['Content-MD5'] || '', - content_type: request.content_type || '' - ) - request['date'] = date - request['authorization'] = "AWS #{@access_key_id}:#{sig}" - end - - def url_signature_params(method:, key:, expires_in:, content_type: '', checksum: '') - expires = expires_in.from_now.to_i - - { - AWSAccessKeyId: @access_key_id, - Expires: expires, - Signature: signature(method: method, key: key, expires: expires, content_type: content_type, checksum: checksum) - } - end - - def signature(method:, key:, expires: '', date: '', content_type: '', checksum: '') - canonicalized_amz_headers = "" - canonicalized_resource = "/#{@bucket}/#{key}" - string_to_sign = "#{method}\n#{checksum}\n#{content_type}\n#{expires}#{date}\n" + - "#{canonicalized_amz_headers}#{canonicalized_resource}" - Base64.encode64(OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha1'), @secret_access_key, string_to_sign)).strip - end - end -end diff --git a/lib/cellar/cellar_adapter.rb b/lib/cellar/cellar_adapter.rb deleted file mode 100644 index 0cd7e88d0..000000000 --- a/lib/cellar/cellar_adapter.rb +++ /dev/null @@ -1,183 +0,0 @@ -require 'net/http' -require 'openssl' - -module Cellar - class CellarAdapter - def initialize(access_key_id, secret_access_key, bucket) - @endpoint = URI::HTTPS.build(host: "#{bucket}.cellar.services.clever-cloud.com") - @signer = AmazonV2RequestSigner.new(access_key_id, secret_access_key, bucket) - end - - def presigned_url(method:, key:, expires_in:, content_type: '', checksum: '', **query_params) - query = query_params.merge( - @signer.url_signature_params( - method: method, - key: key, - expires_in: expires_in, - content_type: content_type, - checksum: checksum - ) - ) - - URI::join(@endpoint, "/#{key}", "?#{query.to_query}").to_s - end - - def session - Net::HTTP.start(@endpoint.host, @endpoint.port, use_ssl: true) do |http| - yield Session.new(http, @signer) - end - end - - class Session - def initialize(http, signer) - @http = http - @signer = signer - end - - def upload(key, io, checksum) - with_io_length(io) do |io, length| - request = Net::HTTP::Put.new("/#{key}") - request.content_type = 'application/octet-stream' - request['Content-MD5'] = checksum - request['Content-Length'] = length - request.body_stream = io - @signer.sign(request, key) - @http.request(request) - # TODO: error handling - end - end - - def download(key, range: nil) - request = Net::HTTP::Get.new("/#{key}") - if range.present? - add_range_header(request, range) - end - @signer.sign(request, key) - if block_given? - @http.request(request) do |response| - if response.is_a?(Net::HTTPSuccess) - response.read_body do |chunk| - yield(chunk.force_encoding(Encoding::BINARY)) - end - else - # TODO: error handling - end - end - else - response = @http.request(request) - if response.is_a?(Net::HTTPSuccess) - response.body.force_encoding(Encoding::BINARY) - else - # TODO: error handling - end - end - end - - def delete(key) - # TODO: error handling - request = Net::HTTP::Delete.new("/#{key}") - @signer.sign(request, key) - @http.request(request) - end - - def list_prefixed(prefix) - result = [] - marker = '' - - begin - request = Net::HTTP::Get.new("/?prefix=#{prefix}&marker=#{marker}") - @signer.sign(request, "") - response = @http.request(request) - if response.is_a?(Net::HTTPSuccess) - (listing, truncated) = parse_bucket_listing(response.body) - result += listing - marker = listing.last.first - else - # TODO: error handling - return nil - end - end while truncated - - result - end - - def delete_keys(keys) - request_body = bulk_deletion_request_body(keys) - request = Net::HTTP::Post.new("/?delete") - request.content_type = 'text/xml' - request['Content-MD5'] = Digest::MD5.base64digest(request_body) - request['Content-Length'] = request_body.length - request.body = request_body - @signer.sign(request, "?delete") - @http.request(request) - end - - def exist?(key) - request = Net::HTTP::Head.new("/#{key}") - @signer.sign(request, key) - response = @http.request(request) - response.is_a?(Net::HTTPSuccess) - end - - def last_modified(key) - request = Net::HTTP::Head.new("/#{key}") - @signer.sign(request, key) - response = @http.request(request) - if response.is_a?(Net::HTTPSuccess) - Time.zone.parse(response['Last-Modified']) - end - end - - private - - def add_range_header(request, range) - bytes_end = range.exclude_end? ? range.end - 1 : range.end - - request['range'] = "bytes=#{range.begin}-#{bytes_end}" - end - - def parse_bucket_listing(bucket_listing_xml) - doc = Nokogiri::XML(bucket_listing_xml) - listing = doc - .xpath('//xmlns:Contents') - .map do |node| - [ - node.xpath('xmlns:Key').text, - DateTime.iso8601(node.xpath('xmlns:LastModified').text) - ] - end - truncated = doc.xpath('//xmlns:IsTruncated').text == 'true' - [listing, truncated] - end - - def bulk_deletion_request_body(keys) - builder = Nokogiri::XML::Builder.new(:encoding => 'UTF-8') do |xml| - xml.Delete do - keys.each do |k| - xml.Object do - xml.Key(k) - end - end - end - end - builder.to_xml - end - - def with_io_length(io) - if io.respond_to?(:size) && io.respond_to?(:pos) - yield(io, io.size - io.pos) - else - tmp_file = Tempfile.new('cellar_io_length') - begin - IO.copy_stream(io, tmp_file) - length = tmp_file.pos - tmp_file.rewind - yield(tmp_file, length) - ensure - tmp_file.close! - end - end - end - end - end -end diff --git a/lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake b/lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake index c11db190f..ab1790b36 100644 --- a/lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake +++ b/lib/tasks/2018_12_03_finish_piece_jointe_transfer.rake @@ -32,15 +32,7 @@ namespace :'2018_12_03_finish_piece_jointe_transfer' do end def old_pj_adapter - if !defined? @old_pj_adapter - @old_pj_adapter = Cellar::CellarAdapter.new( - ENV['CLEVER_CLOUD_ACCESS_KEY_ID'], - ENV['CLEVER_CLOUD_SECRET_ACCESS_KEY'], - ENV['CLEVER_CLOUD_BUCKET'] - ) - end - - @old_pj_adapter + raise NotImplementedError, "No connection adapter for old PJ storage" end def new_pjs diff --git a/spec/lib/active_storage/service/cellar_service_spec.rb b/spec/lib/active_storage/service/cellar_service_spec.rb deleted file mode 100644 index c9bcacda0..000000000 --- a/spec/lib/active_storage/service/cellar_service_spec.rb +++ /dev/null @@ -1,87 +0,0 @@ -require 'active_storage/service/cellar_service' -require 'cgi' -require 'net/http' -require 'uri' - -describe 'CellarService' do - let(:cellar_service) do - # These are actual keys, but they’re safe to put here because - # - they never had any rights attached, and - # - the keys were revoked before copying them here - - ActiveStorage::Service::CellarService.new( - access_key_id: 'AKIAJFTRSGRH3RXX6D5Q', - secret_access_key: '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', - bucket: 'rogets' - ) - end - - before { Timecop.freeze(Time.gm(2016, 10, 2)) } - after { Timecop.return } - - describe 'presigned url for download' do - subject do - URI.parse( - cellar_service.url( - 'fichier', - expires_in: 5.minutes, - filename: ActiveStorage::Filename.new("toto.png"), - disposition: 'attachment', - content_type: 'image/png' - ) - ) - end - - it do - is_expected.to have_attributes( - scheme: 'https', - host: 'rogets.cellar.services.clever-cloud.com', - path: '/fichier' - ) - end - - it do - expect(CGI::parse(subject.query)).to eq( - { - 'AWSAccessKeyId' => ['AKIAJFTRSGRH3RXX6D5Q'], - 'Expires' => ['1475366700'], - 'Signature' => ['nzCsB6cip8oofkuOdvvJs6FafkA='], - 'response-content-disposition' => ["attachment; filename=\"toto.png\"; filename*=UTF-8''toto.png"], - 'response-content-type' => ['image/png'] - } - ) - end - end - - describe 'presigned url for direct upload' do - subject do - URI.parse( - cellar_service.url_for_direct_upload( - 'fichier', - expires_in: 5.minutes, - content_type: 'image/png', - content_length: 2713, - checksum: 'DEADBEEF' - ) - ) - end - - it do - is_expected.to have_attributes( - scheme: 'https', - host: 'rogets.cellar.services.clever-cloud.com', - path: '/fichier' - ) - end - - it do - expect(CGI::parse(subject.query)).to eq( - { - 'AWSAccessKeyId' => ['AKIAJFTRSGRH3RXX6D5Q'], - 'Expires' => ['1475366700'], - 'Signature' => ['VwsX5nxGfTC3dxXjS6wSeU64r5o='] - } - ) - end - end -end diff --git a/spec/lib/cellar/amazon_v2_request_signer_spec.rb b/spec/lib/cellar/amazon_v2_request_signer_spec.rb deleted file mode 100644 index 9776b423c..000000000 --- a/spec/lib/cellar/amazon_v2_request_signer_spec.rb +++ /dev/null @@ -1,43 +0,0 @@ -require 'net/http' - -describe 'AmazonV2RequestSigner' do - let(:request_signer) do - # These are actual keys, but they’re safe to put here because - # - they never had any rights attached, and - # - the keys were revoked before copying them here - - Cellar::AmazonV2RequestSigner.new( - 'AKIAJFTRSGRH3RXX6D5Q', - '3/y/3Tf5zkfcrTaLFxyKB/oU2/7ay7/Dz8UdEHC7', - 'rogets' - ) - end - - before { Timecop.freeze(Time.gm(2016, 10, 2)) } - after { Timecop.return } - - describe 'signature generation' do - context 'for presigned URLs' do - subject do - request_signer.signature( - method: 'GET', - key: 'fichier', - expires: 5.minutes.from_now.to_i - ) - end - - it { is_expected.to eq('nzCsB6cip8oofkuOdvvJs6FafkA=') } - end - - context 'for server-side requests' do - subject do - Net::HTTP::Delete.new('https://rogets.cellar.services.clever-cloud.com/fichier') - end - - before { request_signer.sign(subject, 'fichier') } - - it { expect(subject['date']).to eq(Time.zone.now.httpdate) } - it { expect(subject['authorization']).to eq('AWS AKIAJFTRSGRH3RXX6D5Q:nkvviwZYb1V9HDrKyJZmY3Z8sSA=') } - end - end -end diff --git a/spec/lib/cellar/cellar_adapter_spec.rb b/spec/lib/cellar/cellar_adapter_spec.rb deleted file mode 100644 index c385a5ecf..000000000 --- a/spec/lib/cellar/cellar_adapter_spec.rb +++ /dev/null @@ -1,89 +0,0 @@ -describe 'CellarAdapter' do - let(:session) { Cellar::CellarAdapter::Session.new(nil, nil) } - - before { Timecop.freeze(Time.gm(2016, 10, 2)) } - after { Timecop.return } - - describe 'add_range_header' do - let(:request) { Net::HTTP::Get.new('/whatever') } - - before { session.send(:add_range_header, request, range) } - - subject { request['range'] } - - context 'with end included' do - let(:range) { 100..500 } - - it { is_expected.to eq('bytes=100-500') } - end - - context 'with end excluded' do - let(:range) { 10...50 } - - it { is_expected.to eq('bytes=10-49') } - end - end - - describe 'parse_bucket_listing' do - let(:response) do - <<~EOS - - example-bucket - - 2 - 1000 - / - false - - sample1.jpg - 2011-02-26T01:56:20.000Z - "bf1d737a4d46a19f3bced6905cc8b902" - 142863 - STANDARD - - - sample2.jpg - 2014-03-21T17:44:07.000Z - "bf1d737a4d46a19f3bced6905cc8b902" - 142863 - STANDARD - - ' - EOS - end - - subject { session.send(:parse_bucket_listing, response) } - - it do - is_expected.to eq( - [ - [ - ["sample1.jpg", DateTime.new(2011, 2, 26, 1, 56, 20, 0)], - ["sample2.jpg", DateTime.new(2014, 3, 21, 17, 44, 7, 0)] - ], - false - ] - ) - end - end - - describe 'bulk_deletion_request_body' do - let(:expected_response) do - <<~EOS - - - - chapi - - - chapo - - - EOS - end - - subject { session.send(:bulk_deletion_request_body, ['chapi', 'chapo']) } - - it { is_expected.to eq(expected_response) } - end -end