From 4ceb0d5975048c3bd1a2764ceb3854c22c18f3f0 Mon Sep 17 00:00:00 2001 From: Guillaume Lazzara Date: Fri, 1 Jul 2016 14:45:07 +0200 Subject: [PATCH 1/9] Add link to features.yml in deployment --- config/deploy.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/config/deploy.rb b/config/deploy.rb index 6e25a2cf6..82ee49d66 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -65,6 +65,7 @@ set :shared_paths, [ 'config/database.yml', "config/fog_credentials.yml", 'config/initializers/secret_token.rb', + 'config/initializers/features.yml', "config/environments/#{ENV['to']}.rb", "config/initializers/token.rb", "config/initializers/super_admin.rb", From 35a48ad11c651ea566d4b6f552c566b6a389bc54 Mon Sep 17 00:00:00 2001 From: Guillaume Lazzara Date: Fri, 1 Jul 2016 14:45:24 +0200 Subject: [PATCH 2/9] Add task for cloud storage migration --- Gemfile | 2 + Gemfile.lock | 3 ++ lib/tasks/cloud_storage.rb | 89 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 lib/tasks/cloud_storage.rb diff --git a/Gemfile b/Gemfile index 12229eea2..eea36d384 100644 --- a/Gemfile +++ b/Gemfile @@ -90,6 +90,8 @@ gem 'as_csv' gem 'apipie-rails', '=0.3.1' gem "maruku" # for Markdown support in apipie +gem 'openstack' + group :test do gem 'capybara' gem 'factory_girl' diff --git a/Gemfile.lock b/Gemfile.lock index 31a9b9b55..f51bed510 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -391,6 +391,8 @@ GEM validate_email validate_url webfinger (>= 1.0.1) + openstack (2.0.2) + json orm_adapter (0.5.0) parallel (1.6.1) parallel_tests (1.9.0) @@ -659,6 +661,7 @@ DEPENDENCIES mina! nyan-cat-formatter openid_connect + openstack parallel_tests pg poltergeist diff --git a/lib/tasks/cloud_storage.rb b/lib/tasks/cloud_storage.rb new file mode 100644 index 000000000..743d35995 --- /dev/null +++ b/lib/tasks/cloud_storage.rb @@ -0,0 +1,89 @@ +namespace :cloudstorage do + + task init: :environment do + os_config = (YAML.load_file(Fog.credentials_path))['default'] + @os = OpenStack::Connection.create( + { + username: os_config['openstack_username'], + api_key: os_config['openstack_api_key'], + auth_method: "password", + auth_url: "https://auth.cloud.ovh.net/v2.0/", + authtenant_name: os_config['openstack_tenant'], + service_type: "object-store", + region: os_config['openstack_region'] + } + ) + @cont = @os.container(CarrierWave::Uploader::Base.fog_directory) + end + + desc 'Move local attestations on cloud storage' + task migrate: :environment do + puts 'Starting migration' + + Rake::Task['cloudstorage:init'].invoke + + [Cerfa, PieceJustificative, Procedure].each { |c| + c.all.each { |entry| + content = entry.content + content = entry.logo if c == Procedure + unless content.current_path.nil? || File.exist?(File.dirname(content.current_path) + '/uploaded') + secure_token = SecureRandom.uuid + filename = "#{entry.class.to_s.underscore}-#{secure_token}.pdf" + puts "Uploading #{content.current_path}" + @cont.create_object(filename, { content_type: "application/pdf"}, File.open(content.current_path)) + File.open(File.dirname(content.current_path) + '/uploaded', "w+"){ |f| f.write(File.basename(content.current_path)) } + entry.update_column(c == Procedure ? :logo : :content, filename) + end + } + } + + + puts 'Enf of migration' + end + + desc 'Clear documents in tenant and revert file entries in database' + task :revert do + Rake::Task['cloudstorage:init'].invoke + + @cont.objects.each { |object| + puts "Removing #{object}" + @cont.delete_object(object) + } + + [Cerfa, PieceJustificative, Procedure].each { |c| + c.all.each { |entry| + content = entry.content + content = entry.logo if c == Procedure + unless content.current_path.nil? + if File.exist?(File.dirname(content.current_path) + '/uploaded') + previous_filename = File.read(File.dirname(content.current_path) + '/uploaded') + entry.update_column(c == Procedure ? :logo : :content, filename) + puts "restoring #{content.current_path} db data to #{previous_filename}" + FileUtils.rm(File.dirname(content.current_path) + '/uploaded') + end + end + } + } + end + + + desc 'Clear old documents in tenant' + task :clear do + Rake::Task['cloudstorage:init'].invoke + + @cont.objects.each { |object| + puts "Removing #{object}" + @cont.delete_object(object) + } + end + + task :clear_old_objects do + Rake::Task['cloudstorage:init'].invoke + + @cont.objects_detail.each { |object, details| + last_modified = DateTime.parse(details[:last_modified]) + @cont.delete_object(object) unless last_modified.utc > (Time.now - 2.year).utc + } + end + +end \ No newline at end of file From 8e88e1489b7f6dbe69171f727b757c4a110a5344 Mon Sep 17 00:00:00 2001 From: Guillaume Lazzara Date: Mon, 4 Jul 2016 16:53:38 +0200 Subject: [PATCH 3/9] Fix cloud storage migration task --- config/initializers/openstack_version.rb | 1 + .../{cloud_storage.rb => cloud_storage.rake} | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 config/initializers/openstack_version.rb rename lib/tasks/{cloud_storage.rb => cloud_storage.rake} (75%) diff --git a/config/initializers/openstack_version.rb b/config/initializers/openstack_version.rb new file mode 100644 index 000000000..e71e1f19f --- /dev/null +++ b/config/initializers/openstack_version.rb @@ -0,0 +1 @@ +OpenStack::VERSION = 2.0 \ No newline at end of file diff --git a/lib/tasks/cloud_storage.rb b/lib/tasks/cloud_storage.rake similarity index 75% rename from lib/tasks/cloud_storage.rb rename to lib/tasks/cloud_storage.rake index 743d35995..a41b3d93f 100644 --- a/lib/tasks/cloud_storage.rb +++ b/lib/tasks/cloud_storage.rake @@ -22,22 +22,30 @@ namespace :cloudstorage do Rake::Task['cloudstorage:init'].invoke + error_count = 0 [Cerfa, PieceJustificative, Procedure].each { |c| c.all.each { |entry| - content = entry.content - content = entry.logo if c == Procedure + content = (c == Procedure)? entry.logo : entry.content unless content.current_path.nil? || File.exist?(File.dirname(content.current_path) + '/uploaded') secure_token = SecureRandom.uuid filename = "#{entry.class.to_s.underscore}-#{secure_token}.pdf" puts "Uploading #{content.current_path}" - @cont.create_object(filename, { content_type: "application/pdf"}, File.open(content.current_path)) - File.open(File.dirname(content.current_path) + '/uploaded', "w+"){ |f| f.write(File.basename(content.current_path)) } - entry.update_column(c == Procedure ? :logo : :content, filename) + begin + @cont.create_object(filename, { content_type: "application/pdf"}, File.open(content.current_path)) + File.open(File.dirname(content.current_path) + '/uploaded', "w+"){ |f| f.write(File.basename(content.current_path)) } + entry.update_column(c == Procedure ? :logo : :content, filename) + rescue Errno::ENOENT + puts "ERROR: #{content.current_path} does not exist!" + File.open('upload_errors.report', "a+"){ |f| f.write(content.current_path) } + error_count += 1 + end + else + puts "Skipping #{content.current_path}" end } } - + puts "There were #{error_count} errors while uploading files. See upload_errors.report file for details." puts 'Enf of migration' end From 47790e7ef12a43c02701c72ca571b6bfdf9bfb3e Mon Sep 17 00:00:00 2001 From: Guillaume Lazzara Date: Mon, 4 Jul 2016 17:47:34 +0200 Subject: [PATCH 4/9] Fix previous filename --- lib/tasks/cloud_storage.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/cloud_storage.rake b/lib/tasks/cloud_storage.rake index a41b3d93f..821866129 100644 --- a/lib/tasks/cloud_storage.rake +++ b/lib/tasks/cloud_storage.rake @@ -65,7 +65,7 @@ namespace :cloudstorage do unless content.current_path.nil? if File.exist?(File.dirname(content.current_path) + '/uploaded') previous_filename = File.read(File.dirname(content.current_path) + '/uploaded') - entry.update_column(c == Procedure ? :logo : :content, filename) + entry.update_column(c == Procedure ? :logo : :content, previous_filename) puts "restoring #{content.current_path} db data to #{previous_filename}" FileUtils.rm(File.dirname(content.current_path) + '/uploaded') end From 365ead6b5f460e17020bfa837c6bf078c72a498c Mon Sep 17 00:00:00 2001 From: Guillaume Lazzara Date: Mon, 4 Jul 2016 17:57:55 +0200 Subject: [PATCH 5/9] Fix access to content attribute --- lib/tasks/cloud_storage.rake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/tasks/cloud_storage.rake b/lib/tasks/cloud_storage.rake index 821866129..379fa081f 100644 --- a/lib/tasks/cloud_storage.rake +++ b/lib/tasks/cloud_storage.rake @@ -60,8 +60,7 @@ namespace :cloudstorage do [Cerfa, PieceJustificative, Procedure].each { |c| c.all.each { |entry| - content = entry.content - content = entry.logo if c == Procedure + content = (c == Procedure)? entry.logo : entry.content unless content.current_path.nil? if File.exist?(File.dirname(content.current_path) + '/uploaded') previous_filename = File.read(File.dirname(content.current_path) + '/uploaded') From f4cfae85817eef8d55129b7485c7ebac03d39f41 Mon Sep 17 00:00:00 2001 From: Guillaume Lazzara Date: Mon, 4 Jul 2016 18:00:15 +0200 Subject: [PATCH 6/9] Upload files with the correct extension --- lib/tasks/cloud_storage.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/cloud_storage.rake b/lib/tasks/cloud_storage.rake index 379fa081f..a04c81811 100644 --- a/lib/tasks/cloud_storage.rake +++ b/lib/tasks/cloud_storage.rake @@ -28,10 +28,10 @@ namespace :cloudstorage do content = (c == Procedure)? entry.logo : entry.content unless content.current_path.nil? || File.exist?(File.dirname(content.current_path) + '/uploaded') secure_token = SecureRandom.uuid - filename = "#{entry.class.to_s.underscore}-#{secure_token}.pdf" + filename = "#{entry.class.to_s.underscore}-#{secure_token}#{File.extname(content.current_path)}" puts "Uploading #{content.current_path}" begin - @cont.create_object(filename, { content_type: "application/pdf"}, File.open(content.current_path)) + @cont.create_object(filename, {}, File.open(content.current_path)) File.open(File.dirname(content.current_path) + '/uploaded', "w+"){ |f| f.write(File.basename(content.current_path)) } entry.update_column(c == Procedure ? :logo : :content, filename) rescue Errno::ENOENT From 360177ca4a7faf2e353eef464a3c2b7791fb55b8 Mon Sep 17 00:00:00 2001 From: Guillaume Lazzara Date: Mon, 4 Jul 2016 18:41:09 +0200 Subject: [PATCH 7/9] Update secure_token columns in cloud storage migration --- lib/tasks/cloud_storage.rake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tasks/cloud_storage.rake b/lib/tasks/cloud_storage.rake index a04c81811..80ed3293d 100644 --- a/lib/tasks/cloud_storage.rake +++ b/lib/tasks/cloud_storage.rake @@ -34,6 +34,7 @@ namespace :cloudstorage do @cont.create_object(filename, {}, File.open(content.current_path)) File.open(File.dirname(content.current_path) + '/uploaded', "w+"){ |f| f.write(File.basename(content.current_path)) } entry.update_column(c == Procedure ? :logo : :content, filename) + entry.update_column(c == Procedure ? :logo_secure_token : :content_secure_token, secure_token) rescue Errno::ENOENT puts "ERROR: #{content.current_path} does not exist!" File.open('upload_errors.report', "a+"){ |f| f.write(content.current_path) } @@ -65,6 +66,7 @@ namespace :cloudstorage do if File.exist?(File.dirname(content.current_path) + '/uploaded') previous_filename = File.read(File.dirname(content.current_path) + '/uploaded') entry.update_column(c == Procedure ? :logo : :content, previous_filename) + entry.update_column(c == Procedure ? :logo_secure_token : :content_secure_token, nil) puts "restoring #{content.current_path} db data to #{previous_filename}" FileUtils.rm(File.dirname(content.current_path) + '/uploaded') end From eb0edd0b07d31e01f97bee83db2662d1c4c12a70 Mon Sep 17 00:00:00 2001 From: Xavier J Date: Tue, 5 Jul 2016 16:09:03 +0200 Subject: [PATCH 8/9] Add RESTORE IN DATABASE in cloustorage migrate tasks --- lib/tasks/cloud_storage.rake | 38 +++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/lib/tasks/cloud_storage.rake b/lib/tasks/cloud_storage.rake index 80ed3293d..23ef7b5a4 100644 --- a/lib/tasks/cloud_storage.rake +++ b/lib/tasks/cloud_storage.rake @@ -25,23 +25,37 @@ namespace :cloudstorage do error_count = 0 [Cerfa, PieceJustificative, Procedure].each { |c| c.all.each { |entry| - content = (c == Procedure)? entry.logo : entry.content + content = (c == Procedure) ? entry.logo : entry.content unless content.current_path.nil? || File.exist?(File.dirname(content.current_path) + '/uploaded') secure_token = SecureRandom.uuid filename = "#{entry.class.to_s.underscore}-#{secure_token}#{File.extname(content.current_path)}" puts "Uploading #{content.current_path}" begin @cont.create_object(filename, {}, File.open(content.current_path)) - File.open(File.dirname(content.current_path) + '/uploaded', "w+"){ |f| f.write(File.basename(content.current_path)) } + + File.open(File.dirname(content.current_path) + '/uploaded', "w+") { |f| f.write(File.basename(content.current_path)) } + File.open(File.dirname(content.current_path) + '/filename_cloudstorage', "w+") { |f| f.write(filename) } + File.open(File.dirname(content.current_path) + '/secure_token_cloudstorage', "w+") { |f| f.write(secure_token) } + entry.update_column(c == Procedure ? :logo : :content, filename) entry.update_column(c == Procedure ? :logo_secure_token : :content_secure_token, secure_token) rescue Errno::ENOENT puts "ERROR: #{content.current_path} does not exist!" - File.open('upload_errors.report', "a+"){ |f| f.write(content.current_path) } + File.open('upload_errors.report', "a+") { |f| f.write(content.current_path) } error_count += 1 end else - puts "Skipping #{content.current_path}" + if !content.current_path.nil? && File.exist?(File.dirname(content.current_path) + '/uploaded') + filename = File.open(File.dirname(content.current_path) + '/filename_cloudstorage', "r").read + secure_token = File.open(File.dirname(content.current_path) + '/secure_token_cloudstorage', "r").read + + entry.update_column(c == Procedure ? :logo : :content, filename) + entry.update_column(c == Procedure ? :logo_secure_token : :content_secure_token, secure_token) + + puts "RESTORE IN DATABASE: #{filename} " + elsif !content.current_path.nil? + puts "Skipping #{content.current_path}" + end end } } @@ -54,21 +68,23 @@ namespace :cloudstorage do task :revert do Rake::Task['cloudstorage:init'].invoke - @cont.objects.each { |object| - puts "Removing #{object}" - @cont.delete_object(object) - } - [Cerfa, PieceJustificative, Procedure].each { |c| c.all.each { |entry| - content = (c == Procedure)? entry.logo : entry.content + content = (c == Procedure) ? entry.logo : entry.content unless content.current_path.nil? if File.exist?(File.dirname(content.current_path) + '/uploaded') previous_filename = File.read(File.dirname(content.current_path) + '/uploaded') + entry.update_column(c == Procedure ? :logo : :content, previous_filename) entry.update_column(c == Procedure ? :logo_secure_token : :content_secure_token, nil) + puts "restoring #{content.current_path} db data to #{previous_filename}" + + @cont.delete_object(File.open(File.dirname(content.current_path) + '/filename_cloudstorage', "r").read) + FileUtils.rm(File.dirname(content.current_path) + '/uploaded') + FileUtils.rm(File.dirname(content.current_path) + '/filename_cloudstorage') + FileUtils.rm(File.dirname(content.current_path) + '/secure_token_cloudstorage') end end } @@ -91,7 +107,7 @@ namespace :cloudstorage do @cont.objects_detail.each { |object, details| last_modified = DateTime.parse(details[:last_modified]) - @cont.delete_object(object) unless last_modified.utc > (Time.now - 2.year).utc + @cont.delete_object(object) unless last_modified.utc > (Time.now - 2.year).utc } end From 8eb604383c643abfcdefd977b522239535d492f6 Mon Sep 17 00:00:00 2001 From: Guillaume Lazzara Date: Tue, 5 Jul 2016 16:11:50 +0200 Subject: [PATCH 9/9] Add new deploy tasks --- Rakefile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Rakefile b/Rakefile index e9c354f09..7cde3d716 100644 --- a/Rakefile +++ b/Rakefile @@ -6,12 +6,28 @@ require File.expand_path('../config/application', __FILE__) Rails.application.load_tasks task :deploy do + domains = %w(37.187.249.111 149.202.72.152 149.202.198.6) + domains.each do |domain| + sh "mina deploy domain=#{domain}" + end +end + +task :deploy_ha do + domains = %w(149.202.72.152 149.202.198.6) + domains.each do |domain| + sh "mina deploy domain=#{domain}" + end +end + + +task :deploy_old do domains = %w(37.187.154.237 37.187.249.111) domains.each do |domain| sh "mina deploy domain=#{domain}" end end + task :deploy_test do domains = %w(192.168.0.116) branch = 'clamav'