From 6a3d7251347ca2dc5e9436998afb6747841c550c Mon Sep 17 00:00:00 2001 From: Paul Chavard Date: Wed, 30 Oct 2019 12:11:45 +0100 Subject: [PATCH 1/3] Revert "Revert "Decommission ActiveStorage proxy service and use openstack service"" This reverts commit 71227be37f237fbf2ef374a623b6913d88b86d29. --- .../service/ds_proxy_service.rb | 57 ------------------- config/environments/production.rb | 2 +- config/initializers/active_storage.rb | 30 ++++++++++ config/storage.yml | 3 - 4 files changed, 31 insertions(+), 61 deletions(-) delete mode 100644 app/lib/active_storage/service/ds_proxy_service.rb diff --git a/app/lib/active_storage/service/ds_proxy_service.rb b/app/lib/active_storage/service/ds_proxy_service.rb deleted file mode 100644 index fe540d591..000000000 --- a/app/lib/active_storage/service/ds_proxy_service.rb +++ /dev/null @@ -1,57 +0,0 @@ -module ActiveStorage - # Wraps an ActiveStorage::Service to route direct upload and direct download URLs through our proxy, - # thus avoiding exposing the storage provider’s URL to our end-users. - class Service::DsProxyService < SimpleDelegator - attr_reader :wrapped - - def self.build(wrapped:, configurator:, **options) - new(wrapped: configurator.build(wrapped)) - end - - def initialize(wrapped:) - @wrapped = wrapped - super(wrapped) - end - - def url(*args) - url = wrapped.url(*args) - publicize(url) - end - - def url_for_direct_upload(*args) - url = wrapped.url_for_direct_upload(*args) - publicize(url) - end - - private - - def object_for(key, &block) - blob_url = url(key) - if block_given? - request = Typhoeus::Request.new(blob_url) - request.on_headers do |response| - if response.code != 200 - raise Fog::OpenStack::Storage::NotFound.new - end - end - request.on_body do |chunk| - yield chunk - end - request.run - else - response = Typhoeus.get(blob_url) - if response.success? - response - else - raise Fog::OpenStack::Storage::NotFound.new - end - end - end - - def publicize(url) - search = %r{^https://[^/]+/v1/AUTH_[a-f0-9]{32}} - replace = 'https://static.demarches-simplifiees.fr' - url.gsub(search, replace) - end - end -end diff --git a/config/environments/production.rb b/config/environments/production.rb index 0c93feb51..8837fb88b 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -93,7 +93,7 @@ Rails.application.configure do # the I18n.default_locale when a translation cannot be found). config.i18n.fallbacks = true - config.active_storage.service = :proxied + config.active_storage.service = :openstack # Send deprecation notices to registered listeners. config.active_support.deprecation = :notify diff --git a/config/initializers/active_storage.rb b/config/initializers/active_storage.rb index 7357c9267..1d600d89c 100644 --- a/config/initializers/active_storage.rb +++ b/config/initializers/active_storage.rb @@ -7,3 +7,33 @@ ActiveStorage::Service.url_expires_in = 1.hour # cleaner (as it allows to enqueue the virus scan on attachment creation, rather # than on blob creation). ActiveSupport.on_load(:active_storage_blob) { include BlobVirusScanner } + +# When an OpenStack service is initialized it makes a request to fetch +# `publicURL` to use for all operations. We intercept the method that reads +# this url and replace the host with DS_Proxy host. This way all the operation +# are performed through DS_Proxy. +# +# https://github.com/fog/fog-openstack/blob/37621bb1d5ca78d037b3c56bd307f93bba022ae1/lib/fog/openstack/auth/catalog/v2.rb#L16 +require 'fog/openstack/auth/catalog/v2' + +module Fog::OpenStack::Auth::Catalog + class V2 + def endpoint_url(endpoint, interface) + url = endpoint["#{interface}URL"] + + if interface == 'public' + publicize(url) + else + url + end + end + + private + + def publicize(url) + search = %r{^https://[^/]+/} + replace = 'https://static.demarches-simplifiees.fr/' + url.gsub(search, replace) + end + end +end diff --git a/config/storage.yml b/config/storage.yml index 11de850f6..0427a3f7a 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -4,9 +4,6 @@ local: test: service: Disk root: <%= Rails.root.join("tmp/storage") %> -proxied: - service: DsProxy - wrapped: openstack openstack: service: OpenStack container: "<%= ENV['FOG_ACTIVESTORAGE_DIRECTORY'] %>" From 5a87db9920e2d24fd110623c29d1b20efd1f3c0d Mon Sep 17 00:00:00 2001 From: Paul Chavard Date: Wed, 30 Oct 2019 12:11:53 +0100 Subject: [PATCH 2/3] Revert "Revert "Update activestorage-openstack"" This reverts commit c102dc63b77b1b9c1a6a080bd786709b180b887a. --- Gemfile | 2 +- Gemfile.lock | 18 ++++++------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/Gemfile b/Gemfile index 77978a26f..13763e699 100644 --- a/Gemfile +++ b/Gemfile @@ -4,7 +4,7 @@ gem 'aasm' gem 'actiontext', git: 'https://github.com/kobaltz/actiontext.git', branch: 'archive', require: 'action_text' # Port of ActionText to Rails 5 gem 'active_link_to' # Automatically set a class on active links gem 'active_model_serializers' -gem 'activestorage-openstack', git: 'https://github.com/fredZen/activestorage-openstack.git', branch: 'frederic/fix_upload_signature' +gem 'activestorage-openstack' gem 'administrate' gem 'after_party' gem 'anchored' diff --git a/Gemfile.lock b/Gemfile.lock index 6ac80df50..bbfe3c9ab 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,14 +1,3 @@ -GIT - remote: https://github.com/fredZen/activestorage-openstack.git - revision: c71d5107a51701eab9d9267dd0000e6c1cf3e39a - branch: frederic/fix_upload_signature - specs: - activestorage-openstack (0.5.0) - fog-openstack (~> 1.0) - marcel - mime-types - rails (~> 5.2.0) - GIT remote: https://github.com/kobaltz/actiontext.git revision: ef59c4ba99d1b7614dd47f5a294eef553224db88 @@ -75,6 +64,11 @@ GEM actionpack (= 5.2.2.1) activerecord (= 5.2.2.1) marcel (~> 0.3.1) + activestorage-openstack (1.0.0) + fog-openstack (~> 1.0) + marcel + mime-types + rails (<= 6) activesupport (5.2.2.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) @@ -717,7 +711,7 @@ DEPENDENCIES actiontext! active_link_to active_model_serializers - activestorage-openstack! + activestorage-openstack administrate after_party anchored From de8a1637072f102127c0170d83cffb26f784ebaf Mon Sep 17 00:00:00 2001 From: Paul Chavard Date: Wed, 30 Oct 2019 12:12:31 +0100 Subject: [PATCH 3/3] Bump openstack and fog-openstack --- Gemfile.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index bbfe3c9ab..7eaa820eb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -208,7 +208,7 @@ GEM ethon (0.11.0) ffi (>= 1.3.0) eventmachine (1.2.7) - excon (0.62.0) + excon (0.68.0) execjs (2.7.0) factory_bot (4.11.1) activesupport (>= 3.0.0) @@ -232,7 +232,7 @@ GEM fog-json (1.2.0) fog-core multi_json (~> 1.10) - fog-openstack (1.0.6) + fog-openstack (1.0.10) fog-core (~> 2.1) fog-json (>= 1.0) ipaddress (>= 0.8) @@ -319,7 +319,7 @@ GEM rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) - json (2.1.0) + json (2.2.0) json-jwt (1.10.0) activesupport (>= 4.2) aes_key_wrap @@ -369,16 +369,16 @@ GEM marcel (0.3.3) mimemagic (~> 0.3.2) method_source (0.9.2) - mime-types (3.2.2) + mime-types (3.3) mime-types-data (~> 3.2015) - mime-types-data (3.2018.0812) + mime-types-data (3.2019.1009) mimemagic (0.3.3) mini_mime (1.0.2) mini_portile2 (2.4.0) minitest (5.11.3) momentjs-rails (2.20.1) railties (>= 3.1) - multi_json (1.13.1) + multi_json (1.14.1) multi_xml (0.6.0) multipart-post (2.0.0) mustermann (1.0.3) @@ -419,7 +419,7 @@ GEM validate_email validate_url webfinger (>= 1.0.1) - openstack (3.3.20) + openstack (3.3.21) json orm_adapter (0.5.0) parallel (1.12.1)