From 608419701e0394413e55e693e4beb64101bc95a5 Mon Sep 17 00:00:00 2001 From: Mathieu Magnin Date: Tue, 26 Sep 2017 10:43:50 +0200 Subject: [PATCH 1/8] Add and install delayed_job gem --- Gemfile | 3 +++ Gemfile.lock | 6 +++++ bin/delayed_job | 5 +++++ .../20170926083816_create_delayed_jobs.rb | 22 +++++++++++++++++++ db/schema.rb | 17 +++++++++++++- 5 files changed, 52 insertions(+), 1 deletion(-) create mode 100755 bin/delayed_job create mode 100644 db/migrate/20170926083816_create_delayed_jobs.rb diff --git a/Gemfile b/Gemfile index a254eeb39..c1b53f322 100644 --- a/Gemfile +++ b/Gemfile @@ -101,6 +101,9 @@ gem 'sidekiq' gem 'sidekiq-cron', '~> 0.4.4' gem 'sinatra', git: 'https://github.com/sinatra/sinatra.git', require: false +# Cron jobs +gem 'delayed_job_active_record' + gem 'select2-rails' # PDF Generation diff --git a/Gemfile.lock b/Gemfile.lock index fda9782df..8861e6c2b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -130,6 +130,11 @@ GEM debug_inspector (0.0.2) deep_cloneable (2.2.2) activerecord (>= 3.1.0, < 5.2.0) + delayed_job (4.1.3) + activesupport (>= 3.0, < 5.2) + delayed_job_active_record (4.1.2) + activerecord (>= 3.0, < 5.2) + delayed_job (>= 3.0, < 5) devise (4.2.0) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -696,6 +701,7 @@ DEPENDENCIES copy_carrierwave_file database_cleaner deep_cloneable (~> 2.2.1) + delayed_job_active_record devise dotenv-rails draper (~> 3.0.0.pre1) diff --git a/bin/delayed_job b/bin/delayed_job new file mode 100755 index 000000000..edf195985 --- /dev/null +++ b/bin/delayed_job @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby + +require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment')) +require 'delayed/command' +Delayed::Command.new(ARGV).daemonize diff --git a/db/migrate/20170926083816_create_delayed_jobs.rb b/db/migrate/20170926083816_create_delayed_jobs.rb new file mode 100644 index 000000000..130a8d570 --- /dev/null +++ b/db/migrate/20170926083816_create_delayed_jobs.rb @@ -0,0 +1,22 @@ +class CreateDelayedJobs < ActiveRecord::Migration[5.0] + def self.up + create_table :delayed_jobs, force: true do |table| + table.integer :priority, default: 0, null: false # Allows some jobs to jump to the front of the queue + table.integer :attempts, default: 0, null: false # Provides for retries, but still fail eventually. + table.text :handler, null: false # YAML-encoded string of the object that will do work + table.text :last_error # reason for last failure (See Note below) + table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future. + table.datetime :locked_at # Set when a client is working on this object + table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead) + table.string :locked_by # Who is working on this object (if locked) + table.string :queue # The name of the queue this job is in + table.timestamps null: true + end + + add_index :delayed_jobs, [:priority, :run_at], name: "delayed_jobs_priority" + end + + def self.down + drop_table :delayed_jobs + end +end diff --git a/db/schema.rb b/db/schema.rb index ec4a5d01a..56e5f14e9 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170908101023) do +ActiveRecord::Schema.define(version: 20170926083816) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -162,6 +162,21 @@ ActiveRecord::Schema.define(version: 20170908101023) do t.index ["dossier_id"], name: "index_commentaires_on_dossier_id", using: :btree end + create_table "delayed_jobs", force: :cascade do |t| + t.integer "priority", default: 0, null: false + t.integer "attempts", default: 0, null: false + t.text "handler", null: false + t.text "last_error" + t.datetime "run_at" + t.datetime "locked_at" + t.datetime "failed_at" + t.string "locked_by" + t.string "queue" + t.datetime "created_at" + t.datetime "updated_at" + t.index ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree + end + create_table "dossiers", force: :cascade do |t| t.boolean "autorisation_donnees" t.integer "procedure_id" From 8965c668d9144aee5fe0c182a0db2ecae6313b83 Mon Sep 17 00:00:00 2001 From: Mathieu Magnin Date: Tue, 26 Sep 2017 11:32:34 +0200 Subject: [PATCH 2/8] Add and install delayed_cron_job gem --- Gemfile | 1 + Gemfile.lock | 3 +++ db/migrate/20170926092716_add_cron_to_delayed_jobs.rb | 9 +++++++++ db/schema.rb | 3 ++- 4 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20170926092716_add_cron_to_delayed_jobs.rb diff --git a/Gemfile b/Gemfile index c1b53f322..20d3d3890 100644 --- a/Gemfile +++ b/Gemfile @@ -103,6 +103,7 @@ gem 'sinatra', git: 'https://github.com/sinatra/sinatra.git', require: false # Cron jobs gem 'delayed_job_active_record' +gem 'delayed_cron_job' gem 'select2-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 8861e6c2b..4e96db14e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -130,6 +130,8 @@ GEM debug_inspector (0.0.2) deep_cloneable (2.2.2) activerecord (>= 3.1.0, < 5.2.0) + delayed_cron_job (0.7.2) + delayed_job (>= 4.1) delayed_job (4.1.3) activesupport (>= 3.0, < 5.2) delayed_job_active_record (4.1.2) @@ -701,6 +703,7 @@ DEPENDENCIES copy_carrierwave_file database_cleaner deep_cloneable (~> 2.2.1) + delayed_cron_job delayed_job_active_record devise dotenv-rails diff --git a/db/migrate/20170926092716_add_cron_to_delayed_jobs.rb b/db/migrate/20170926092716_add_cron_to_delayed_jobs.rb new file mode 100644 index 000000000..33e89d190 --- /dev/null +++ b/db/migrate/20170926092716_add_cron_to_delayed_jobs.rb @@ -0,0 +1,9 @@ +class AddCronToDelayedJobs < ActiveRecord::Migration[5.0] + def self.up + add_column :delayed_jobs, :cron, :string + end + + def self.down + remove_column :delayed_jobs, :cron + end +end diff --git a/db/schema.rb b/db/schema.rb index 56e5f14e9..e2bd74e20 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170926083816) do +ActiveRecord::Schema.define(version: 20170926092716) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -174,6 +174,7 @@ ActiveRecord::Schema.define(version: 20170926083816) do t.string "queue" t.datetime "created_at" t.datetime "updated_at" + t.string "cron" t.index ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree end From 68506fe2c74b144aae8fbb8e106f6ece45210c42 Mon Sep 17 00:00:00 2001 From: Mathieu Magnin Date: Tue, 26 Sep 2017 12:06:32 +0200 Subject: [PATCH 3/8] Add and install daemons gem --- Gemfile | 1 + Gemfile.lock | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Gemfile b/Gemfile index 20d3d3890..c5257b517 100644 --- a/Gemfile +++ b/Gemfile @@ -103,6 +103,7 @@ gem 'sinatra', git: 'https://github.com/sinatra/sinatra.git', require: false # Cron jobs gem 'delayed_job_active_record' +gem "daemons" gem 'delayed_cron_job' gem 'select2-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 4e96db14e..17ba3ad98 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -126,6 +126,7 @@ GEM carrierwave (>= 0.9) crack (0.4.3) safe_yaml (~> 1.0.0) + daemons (1.2.4) database_cleaner (1.5.3) debug_inspector (0.0.2) deep_cloneable (2.2.2) @@ -701,6 +702,7 @@ DEPENDENCIES chunky_png clamav-client copy_carrierwave_file + daemons database_cleaner deep_cloneable (~> 2.2.1) delayed_cron_job From b9869d861f413b91a27b08322f48af8d4fc940db Mon Sep 17 00:00:00 2001 From: Mathieu Magnin Date: Tue, 26 Sep 2017 12:04:05 +0200 Subject: [PATCH 4/8] Add and install delayed_job_web gem --- Gemfile | 1 + Gemfile.lock | 5 +++++ config/routes.rb | 1 + 3 files changed, 7 insertions(+) diff --git a/Gemfile b/Gemfile index c5257b517..be12e52ef 100644 --- a/Gemfile +++ b/Gemfile @@ -105,6 +105,7 @@ gem 'sinatra', git: 'https://github.com/sinatra/sinatra.git', require: false gem 'delayed_job_active_record' gem "daemons" gem 'delayed_cron_job' +gem "delayed_job_web" gem 'select2-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 17ba3ad98..950868119 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -138,6 +138,10 @@ GEM delayed_job_active_record (4.1.2) activerecord (>= 3.0, < 5.2) delayed_job (>= 3.0, < 5) + delayed_job_web (1.4) + activerecord (> 3.0.0) + delayed_job (> 2.0.3) + sinatra (>= 1.4.4) devise (4.2.0) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -707,6 +711,7 @@ DEPENDENCIES deep_cloneable (~> 2.2.1) delayed_cron_job delayed_job_active_record + delayed_job_web devise dotenv-rails draper (~> 3.0.0.pre1) diff --git a/config/routes.rb b/config/routes.rb index c0272c9ef..aac7387bf 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -50,6 +50,7 @@ Rails.application.routes.draw do require 'sidekiq/web' require 'sidekiq/cron/web' mount Sidekiq::Web => '/sidekiq' + match "/delayed_job" => DelayedJobWeb, :anchor => false, :via => [:get, :post] end end From fbad15bc84d0b49686802e9ca9f29523d9fc9cba Mon Sep 17 00:00:00 2001 From: Mathieu Magnin Date: Tue, 26 Sep 2017 14:12:55 +0200 Subject: [PATCH 5/8] Change AutoArchiveProcedureWorker for delayed_job --- app/workers/auto_archive_procedure_worker.rb | 10 ++++++++-- spec/workers/auto_archive_procedure_worker_spec.rb | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/app/workers/auto_archive_procedure_worker.rb b/app/workers/auto_archive_procedure_worker.rb index d9715b843..d8c6df0a1 100644 --- a/app/workers/auto_archive_procedure_worker.rb +++ b/app/workers/auto_archive_procedure_worker.rb @@ -1,7 +1,6 @@ class AutoArchiveProcedureWorker - include Sidekiq::Worker - def perform(*args) + Rails.logger.info("AutoArchiveProcedureWorker started at #{Time.now}") Procedure.publiees.where("auto_archive_on <= ?", Date.today).each do |procedure| procedure.dossiers.state_en_construction.each do |dossier| dossier.received! @@ -9,5 +8,12 @@ class AutoArchiveProcedureWorker procedure.archive end + Rails.logger.info("AutoArchiveProcedureWorker ended at #{Time.now}") end + + def queue_name + "cron" + end + + handle_asynchronously :perform end diff --git a/spec/workers/auto_archive_procedure_worker_spec.rb b/spec/workers/auto_archive_procedure_worker_spec.rb index ae47d6f42..af3fe73fe 100644 --- a/spec/workers/auto_archive_procedure_worker_spec.rb +++ b/spec/workers/auto_archive_procedure_worker_spec.rb @@ -1,6 +1,9 @@ require 'rails_helper' RSpec.describe AutoArchiveProcedureWorker, type: :worker do + before { Delayed::Worker.delay_jobs = false } + after { Delayed::Worker.delay_jobs = true } + let!(:procedure) { create(:procedure, published_at: Time.now, archived_at: nil, auto_archive_on: nil )} let!(:procedure_hier) { create(:procedure, published_at: Time.now, archived_at: nil, auto_archive_on: 1.day.ago )} let!(:procedure_aujourdhui) { create(:procedure, published_at: Time.now, archived_at: nil, auto_archive_on: Date.today )} From f46330be1a9fa2ca9816972ced579fc99e0b1590 Mon Sep 17 00:00:00 2001 From: Mathieu Magnin Date: Tue, 26 Sep 2017 14:58:51 +0200 Subject: [PATCH 6/8] Update README for delayed_job --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d395433ee..e2ea0ad8d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ Téléprocédures Simplifiées, ou TPS pour les intimes, est une plateforme 100 ### Tous environnements - postgresql -- redis ### Développement @@ -61,11 +60,13 @@ Afin de générer la BDD de l'application, il est nécessaire d'éxécuter les c ## Lancement de l'application - redis-server - sidekiq + bin/delayed_job run mailcatcher -f rails s +## Lancement des workers + + Delayed::Job.enqueue(AutoArchiveProcedureWorker.new, cron: "* * * * *") ## Exécution des tests (RSpec) From e753c67ad6af05ab5078f1a277f86514cfe3f3f8 Mon Sep 17 00:00:00 2001 From: Mathieu Magnin Date: Tue, 26 Sep 2017 15:32:03 +0200 Subject: [PATCH 7/8] Change WeeklyOverviewWorker for delayed_job --- README.md | 1 + app/workers/weekly_overview_worker.rb | 10 ++++++++-- spec/workers/weekly_overview_worker_spec.rb | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e2ea0ad8d..23c936df1 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Afin de générer la BDD de l'application, il est nécessaire d'éxécuter les c ## Lancement des workers Delayed::Job.enqueue(AutoArchiveProcedureWorker.new, cron: "* * * * *") + Delayed::Job.enqueue(WeeklyOverviewWorker.new, cron: "0 8 * * 0") ## Exécution des tests (RSpec) diff --git a/app/workers/weekly_overview_worker.rb b/app/workers/weekly_overview_worker.rb index 8f15194a7..2d3cf9260 100644 --- a/app/workers/weekly_overview_worker.rb +++ b/app/workers/weekly_overview_worker.rb @@ -1,7 +1,6 @@ class WeeklyOverviewWorker - include Sidekiq::Worker - def perform(*args) + Rails.logger.info("WeeklyOverviewWorker started at #{Time.now}") # Feature flipped to avoid mails in staging due to unprocessed dossier if Features.weekly_overview Gestionnaire.all @@ -9,5 +8,12 @@ class WeeklyOverviewWorker .reject { |_, overview| overview.nil? } .each { |gestionnaire, overview| GestionnaireMailer.last_week_overview(gestionnaire, overview).deliver_now } end + Rails.logger.info("WeeklyOverviewWorker ended at #{Time.now}") end + + def queue_name + "cron" + end + + handle_asynchronously :perform end diff --git a/spec/workers/weekly_overview_worker_spec.rb b/spec/workers/weekly_overview_worker_spec.rb index e2bff3e80..9568e1509 100644 --- a/spec/workers/weekly_overview_worker_spec.rb +++ b/spec/workers/weekly_overview_worker_spec.rb @@ -1,6 +1,9 @@ require 'rails_helper' RSpec.describe WeeklyOverviewWorker, type: :worker do + before { Delayed::Worker.delay_jobs = false } + after { Delayed::Worker.delay_jobs = true } + describe 'perform' do let!(:gestionnaire) { create(:gestionnaire) } let(:overview) { double('overview') } From 7b20395b638d43925ac52e02a2ec8db154219f96 Mon Sep 17 00:00:00 2001 From: Mathieu Magnin Date: Tue, 26 Sep 2017 15:50:39 +0200 Subject: [PATCH 8/8] Remove sidekiq traces --- Gemfile | 5 ----- Gemfile.lock | 38 +++++++--------------------------- config/deploy.rb | 5 ++--- config/initializers/sidekiq.rb | 6 ------ config/routes.rb | 3 --- config/sidekiq.yml | 7 ------- 6 files changed, 10 insertions(+), 54 deletions(-) delete mode 100644 config/initializers/sidekiq.rb delete mode 100644 config/sidekiq.yml diff --git a/Gemfile b/Gemfile index be12e52ef..a68d6e3f1 100644 --- a/Gemfile +++ b/Gemfile @@ -96,11 +96,6 @@ gem 'newrelic_rpm' gem 'scenic' -# Sidekiq -gem 'sidekiq' -gem 'sidekiq-cron', '~> 0.4.4' -gem 'sinatra', git: 'https://github.com/sinatra/sinatra.git', require: false - # Cron jobs gem 'delayed_job_active_record' gem "daemons" diff --git a/Gemfile.lock b/Gemfile.lock index 950868119..6ce018ead 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -7,18 +7,6 @@ GIT open4 (~> 1.3.4) rake -GIT - remote: https://github.com/sinatra/sinatra.git - revision: d0c4053fd459be9f2c207cfeec5c0606461c014b - specs: - rack-protection (2.0.0.rc1) - rack - sinatra (2.0.0.rc1) - mustermann (= 1.0.0) - rack (~> 2.0) - rack-protection (= 2.0.0.rc1) - tilt (~> 2.0) - GEM remote: https://rubygems.org/ specs: @@ -121,7 +109,6 @@ GEM execjs coffee-script-source (1.12.2) concurrent-ruby (1.0.5) - connection_pool (2.2.1) copy_carrierwave_file (1.3.0) carrierwave (>= 0.9) crack (0.4.3) @@ -408,7 +395,7 @@ GEM minitest (5.10.1) multi_json (1.12.1) multipart-post (2.0.0) - mustermann (1.0.0) + mustermann (1.0.1) nenv (0.3.0) netrc (0.11.0) newrelic_rpm (3.18.1.330) @@ -468,6 +455,8 @@ GEM httpclient (>= 2.4) multi_json (>= 1.3.6) rack (>= 1.1) + rack-protection (2.0.0) + rack rack-test (0.6.3) rack (>= 1.0) rails (5.0.0.1) @@ -511,8 +500,6 @@ GEM trollop (~> 2.1) rdoc (4.3.0) redis (3.3.3) - redis-namespace (1.5.3) - redis (~> 3.0, >= 3.0.4) ref (2.0.0) request_store (1.3.1) responders (2.3.0) @@ -561,8 +548,6 @@ GEM ruby_parser (3.8.3) sexp_processor (~> 4.1) rubyzip (1.0.0) - rufus-scheduler (3.3.4) - tzinfo safe_yaml (1.0.4) sass (3.4.22) sass-rails (5.0.6) @@ -589,18 +574,14 @@ GEM shellany (0.0.1) shoulda-matchers (3.1.1) activesupport (>= 4.0.0) - sidekiq (4.2.9) - concurrent-ruby (~> 1.0) - connection_pool (~> 2.2, >= 2.2.0) - rack-protection (>= 1.5.0) - redis (~> 3.2, >= 3.2.1) - sidekiq-cron (0.4.5) - redis-namespace (>= 1.5.2) - rufus-scheduler (>= 2.0.24) - sidekiq (>= 4.2.1) simple_form (3.4.0) actionpack (> 4, < 5.1) activemodel (> 4, < 5.1) + sinatra (2.0.0) + mustermann (~> 1.0) + rack (~> 2.0) + rack-protection (= 2.0.0) + tilt (~> 2.0) slop (3.6.0) smart_listing (1.2.0) coffee-rails @@ -759,10 +740,7 @@ DEPENDENCIES select2-rails sentry-raven shoulda-matchers - sidekiq - sidekiq-cron (~> 0.4.4) simple_form - sinatra! smart_listing spreadsheet_architect spring diff --git a/config/deploy.rb b/config/deploy.rb index c6c264d9c..0a48ab1a3 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -58,7 +58,6 @@ set :shared_paths, [ "config/fog_credentials.yml", 'config/initializers/secret_token.rb', 'config/initializers/features.yml', - 'config/initializers/sidekiq.rb', "config/environments/#{rails_env}.rb", "config/initializers/token.rb", "config/initializers/urls.rb", @@ -131,7 +130,7 @@ desc "Deploys the current version to the server." task :deploy => :environment do queue 'export PATH=$PATH:/usr/local/rbenv/bin:/usr/local/rbenv/shims' deploy do - queue %[sudo stop sidekiq_#{user!} || true] + queue %[sudo stop delayed_job_#{user!} || true] # Put things that will set up an empty directory into a fully set-up # instance of your project. invoke :'git:clone' @@ -142,7 +141,7 @@ task :deploy => :environment do to :launch do queue "/etc/init.d/#{user} upgrade " - queue! %[sudo start sidekiq_#{user!}] + queue! %[sudo start delayed_job_#{user!}] queue "cd #{deploy_to}/#{current_path}/" queue "bundle exec rake db:seed RAILS_ENV=#{rails_env}" diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb deleted file mode 100644 index a61ed5e57..000000000 --- a/config/initializers/sidekiq.rb +++ /dev/null @@ -1,6 +0,0 @@ -Sidekiq.configure_server do |config| - Sidekiq::Logging.logger = Rails.logger - - schedule_file = "config/schedule.yml" - Sidekiq::Cron::Job.load_from_hash YAML.load_file(schedule_file) -end diff --git a/config/routes.rb b/config/routes.rb index aac7387bf..dbaa51e86 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -47,9 +47,6 @@ Rails.application.routes.draw do authenticate :administration do resources :administrations, only: [:index, :create] namespace :administrations do - require 'sidekiq/web' - require 'sidekiq/cron/web' - mount Sidekiq::Web => '/sidekiq' match "/delayed_job" => DelayedJobWeb, :anchor => false, :via => [:get, :post] end end diff --git a/config/sidekiq.yml b/config/sidekiq.yml deleted file mode 100644 index 0fe0c4360..000000000 --- a/config/sidekiq.yml +++ /dev/null @@ -1,7 +0,0 @@ -:concurrency: 5 -staging: - :concurrency: 2 -production: - :concurrency: 2 -:queues: - - default