diff --git a/Gemfile b/Gemfile index a254eeb39..a68d6e3f1 100644 --- a/Gemfile +++ b/Gemfile @@ -96,10 +96,11 @@ gem 'newrelic_rpm' gem 'scenic' -# Sidekiq -gem 'sidekiq' -gem 'sidekiq-cron', '~> 0.4.4' -gem 'sinatra', git: 'https://github.com/sinatra/sinatra.git', require: false +# Cron jobs +gem 'delayed_job_active_record' +gem "daemons" +gem 'delayed_cron_job' +gem "delayed_job_web" gem 'select2-rails' diff --git a/Gemfile.lock b/Gemfile.lock index fda9782df..6ce018ead 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -7,18 +7,6 @@ GIT open4 (~> 1.3.4) rake -GIT - remote: https://github.com/sinatra/sinatra.git - revision: d0c4053fd459be9f2c207cfeec5c0606461c014b - specs: - rack-protection (2.0.0.rc1) - rack - sinatra (2.0.0.rc1) - mustermann (= 1.0.0) - rack (~> 2.0) - rack-protection (= 2.0.0.rc1) - tilt (~> 2.0) - GEM remote: https://rubygems.org/ specs: @@ -121,15 +109,26 @@ GEM execjs coffee-script-source (1.12.2) concurrent-ruby (1.0.5) - connection_pool (2.2.1) copy_carrierwave_file (1.3.0) carrierwave (>= 0.9) crack (0.4.3) safe_yaml (~> 1.0.0) + daemons (1.2.4) database_cleaner (1.5.3) debug_inspector (0.0.2) deep_cloneable (2.2.2) activerecord (>= 3.1.0, < 5.2.0) + delayed_cron_job (0.7.2) + delayed_job (>= 4.1) + delayed_job (4.1.3) + activesupport (>= 3.0, < 5.2) + delayed_job_active_record (4.1.2) + activerecord (>= 3.0, < 5.2) + delayed_job (>= 3.0, < 5) + delayed_job_web (1.4) + activerecord (> 3.0.0) + delayed_job (> 2.0.3) + sinatra (>= 1.4.4) devise (4.2.0) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -396,7 +395,7 @@ GEM minitest (5.10.1) multi_json (1.12.1) multipart-post (2.0.0) - mustermann (1.0.0) + mustermann (1.0.1) nenv (0.3.0) netrc (0.11.0) newrelic_rpm (3.18.1.330) @@ -456,6 +455,8 @@ GEM httpclient (>= 2.4) multi_json (>= 1.3.6) rack (>= 1.1) + rack-protection (2.0.0) + rack rack-test (0.6.3) rack (>= 1.0) rails (5.0.0.1) @@ -499,8 +500,6 @@ GEM trollop (~> 2.1) rdoc (4.3.0) redis (3.3.3) - redis-namespace (1.5.3) - redis (~> 3.0, >= 3.0.4) ref (2.0.0) request_store (1.3.1) responders (2.3.0) @@ -549,8 +548,6 @@ GEM ruby_parser (3.8.3) sexp_processor (~> 4.1) rubyzip (1.0.0) - rufus-scheduler (3.3.4) - tzinfo safe_yaml (1.0.4) sass (3.4.22) sass-rails (5.0.6) @@ -577,18 +574,14 @@ GEM shellany (0.0.1) shoulda-matchers (3.1.1) activesupport (>= 4.0.0) - sidekiq (4.2.9) - concurrent-ruby (~> 1.0) - connection_pool (~> 2.2, >= 2.2.0) - rack-protection (>= 1.5.0) - redis (~> 3.2, >= 3.2.1) - sidekiq-cron (0.4.5) - redis-namespace (>= 1.5.2) - rufus-scheduler (>= 2.0.24) - sidekiq (>= 4.2.1) simple_form (3.4.0) actionpack (> 4, < 5.1) activemodel (> 4, < 5.1) + sinatra (2.0.0) + mustermann (~> 1.0) + rack (~> 2.0) + rack-protection (= 2.0.0) + tilt (~> 2.0) slop (3.6.0) smart_listing (1.2.0) coffee-rails @@ -694,8 +687,12 @@ DEPENDENCIES chunky_png clamav-client copy_carrierwave_file + daemons database_cleaner deep_cloneable (~> 2.2.1) + delayed_cron_job + delayed_job_active_record + delayed_job_web devise dotenv-rails draper (~> 3.0.0.pre1) @@ -743,10 +740,7 @@ DEPENDENCIES select2-rails sentry-raven shoulda-matchers - sidekiq - sidekiq-cron (~> 0.4.4) simple_form - sinatra! smart_listing spreadsheet_architect spring diff --git a/README.md b/README.md index d395433ee..23c936df1 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ Téléprocédures Simplifiées, ou TPS pour les intimes, est une plateforme 100 ### Tous environnements - postgresql -- redis ### Développement @@ -61,11 +60,14 @@ Afin de générer la BDD de l'application, il est nécessaire d'éxécuter les c ## Lancement de l'application - redis-server - sidekiq + bin/delayed_job run mailcatcher -f rails s +## Lancement des workers + + Delayed::Job.enqueue(AutoArchiveProcedureWorker.new, cron: "* * * * *") + Delayed::Job.enqueue(WeeklyOverviewWorker.new, cron: "0 8 * * 0") ## Exécution des tests (RSpec) diff --git a/app/workers/auto_archive_procedure_worker.rb b/app/workers/auto_archive_procedure_worker.rb index d9715b843..d8c6df0a1 100644 --- a/app/workers/auto_archive_procedure_worker.rb +++ b/app/workers/auto_archive_procedure_worker.rb @@ -1,7 +1,6 @@ class AutoArchiveProcedureWorker - include Sidekiq::Worker - def perform(*args) + Rails.logger.info("AutoArchiveProcedureWorker started at #{Time.now}") Procedure.publiees.where("auto_archive_on <= ?", Date.today).each do |procedure| procedure.dossiers.state_en_construction.each do |dossier| dossier.received! @@ -9,5 +8,12 @@ class AutoArchiveProcedureWorker procedure.archive end + Rails.logger.info("AutoArchiveProcedureWorker ended at #{Time.now}") end + + def queue_name + "cron" + end + + handle_asynchronously :perform end diff --git a/app/workers/weekly_overview_worker.rb b/app/workers/weekly_overview_worker.rb index 8f15194a7..2d3cf9260 100644 --- a/app/workers/weekly_overview_worker.rb +++ b/app/workers/weekly_overview_worker.rb @@ -1,7 +1,6 @@ class WeeklyOverviewWorker - include Sidekiq::Worker - def perform(*args) + Rails.logger.info("WeeklyOverviewWorker started at #{Time.now}") # Feature flipped to avoid mails in staging due to unprocessed dossier if Features.weekly_overview Gestionnaire.all @@ -9,5 +8,12 @@ class WeeklyOverviewWorker .reject { |_, overview| overview.nil? } .each { |gestionnaire, overview| GestionnaireMailer.last_week_overview(gestionnaire, overview).deliver_now } end + Rails.logger.info("WeeklyOverviewWorker ended at #{Time.now}") end + + def queue_name + "cron" + end + + handle_asynchronously :perform end diff --git a/bin/delayed_job b/bin/delayed_job new file mode 100755 index 000000000..edf195985 --- /dev/null +++ b/bin/delayed_job @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby + +require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment')) +require 'delayed/command' +Delayed::Command.new(ARGV).daemonize diff --git a/config/deploy.rb b/config/deploy.rb index c6c264d9c..0a48ab1a3 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -58,7 +58,6 @@ set :shared_paths, [ "config/fog_credentials.yml", 'config/initializers/secret_token.rb', 'config/initializers/features.yml', - 'config/initializers/sidekiq.rb', "config/environments/#{rails_env}.rb", "config/initializers/token.rb", "config/initializers/urls.rb", @@ -131,7 +130,7 @@ desc "Deploys the current version to the server." task :deploy => :environment do queue 'export PATH=$PATH:/usr/local/rbenv/bin:/usr/local/rbenv/shims' deploy do - queue %[sudo stop sidekiq_#{user!} || true] + queue %[sudo stop delayed_job_#{user!} || true] # Put things that will set up an empty directory into a fully set-up # instance of your project. invoke :'git:clone' @@ -142,7 +141,7 @@ task :deploy => :environment do to :launch do queue "/etc/init.d/#{user} upgrade " - queue! %[sudo start sidekiq_#{user!}] + queue! %[sudo start delayed_job_#{user!}] queue "cd #{deploy_to}/#{current_path}/" queue "bundle exec rake db:seed RAILS_ENV=#{rails_env}" diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb deleted file mode 100644 index a61ed5e57..000000000 --- a/config/initializers/sidekiq.rb +++ /dev/null @@ -1,6 +0,0 @@ -Sidekiq.configure_server do |config| - Sidekiq::Logging.logger = Rails.logger - - schedule_file = "config/schedule.yml" - Sidekiq::Cron::Job.load_from_hash YAML.load_file(schedule_file) -end diff --git a/config/routes.rb b/config/routes.rb index c0272c9ef..dbaa51e86 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -47,9 +47,7 @@ Rails.application.routes.draw do authenticate :administration do resources :administrations, only: [:index, :create] namespace :administrations do - require 'sidekiq/web' - require 'sidekiq/cron/web' - mount Sidekiq::Web => '/sidekiq' + match "/delayed_job" => DelayedJobWeb, :anchor => false, :via => [:get, :post] end end diff --git a/config/sidekiq.yml b/config/sidekiq.yml deleted file mode 100644 index 0fe0c4360..000000000 --- a/config/sidekiq.yml +++ /dev/null @@ -1,7 +0,0 @@ -:concurrency: 5 -staging: - :concurrency: 2 -production: - :concurrency: 2 -:queues: - - default diff --git a/db/migrate/20170926083816_create_delayed_jobs.rb b/db/migrate/20170926083816_create_delayed_jobs.rb new file mode 100644 index 000000000..130a8d570 --- /dev/null +++ b/db/migrate/20170926083816_create_delayed_jobs.rb @@ -0,0 +1,22 @@ +class CreateDelayedJobs < ActiveRecord::Migration[5.0] + def self.up + create_table :delayed_jobs, force: true do |table| + table.integer :priority, default: 0, null: false # Allows some jobs to jump to the front of the queue + table.integer :attempts, default: 0, null: false # Provides for retries, but still fail eventually. + table.text :handler, null: false # YAML-encoded string of the object that will do work + table.text :last_error # reason for last failure (See Note below) + table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future. + table.datetime :locked_at # Set when a client is working on this object + table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead) + table.string :locked_by # Who is working on this object (if locked) + table.string :queue # The name of the queue this job is in + table.timestamps null: true + end + + add_index :delayed_jobs, [:priority, :run_at], name: "delayed_jobs_priority" + end + + def self.down + drop_table :delayed_jobs + end +end diff --git a/db/migrate/20170926092716_add_cron_to_delayed_jobs.rb b/db/migrate/20170926092716_add_cron_to_delayed_jobs.rb new file mode 100644 index 000000000..33e89d190 --- /dev/null +++ b/db/migrate/20170926092716_add_cron_to_delayed_jobs.rb @@ -0,0 +1,9 @@ +class AddCronToDelayedJobs < ActiveRecord::Migration[5.0] + def self.up + add_column :delayed_jobs, :cron, :string + end + + def self.down + remove_column :delayed_jobs, :cron + end +end diff --git a/db/schema.rb b/db/schema.rb index ec4a5d01a..e2bd74e20 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170908101023) do +ActiveRecord::Schema.define(version: 20170926092716) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -162,6 +162,22 @@ ActiveRecord::Schema.define(version: 20170908101023) do t.index ["dossier_id"], name: "index_commentaires_on_dossier_id", using: :btree end + create_table "delayed_jobs", force: :cascade do |t| + t.integer "priority", default: 0, null: false + t.integer "attempts", default: 0, null: false + t.text "handler", null: false + t.text "last_error" + t.datetime "run_at" + t.datetime "locked_at" + t.datetime "failed_at" + t.string "locked_by" + t.string "queue" + t.datetime "created_at" + t.datetime "updated_at" + t.string "cron" + t.index ["priority", "run_at"], name: "delayed_jobs_priority", using: :btree + end + create_table "dossiers", force: :cascade do |t| t.boolean "autorisation_donnees" t.integer "procedure_id" diff --git a/spec/workers/auto_archive_procedure_worker_spec.rb b/spec/workers/auto_archive_procedure_worker_spec.rb index ae47d6f42..af3fe73fe 100644 --- a/spec/workers/auto_archive_procedure_worker_spec.rb +++ b/spec/workers/auto_archive_procedure_worker_spec.rb @@ -1,6 +1,9 @@ require 'rails_helper' RSpec.describe AutoArchiveProcedureWorker, type: :worker do + before { Delayed::Worker.delay_jobs = false } + after { Delayed::Worker.delay_jobs = true } + let!(:procedure) { create(:procedure, published_at: Time.now, archived_at: nil, auto_archive_on: nil )} let!(:procedure_hier) { create(:procedure, published_at: Time.now, archived_at: nil, auto_archive_on: 1.day.ago )} let!(:procedure_aujourdhui) { create(:procedure, published_at: Time.now, archived_at: nil, auto_archive_on: Date.today )} diff --git a/spec/workers/weekly_overview_worker_spec.rb b/spec/workers/weekly_overview_worker_spec.rb index e2bff3e80..9568e1509 100644 --- a/spec/workers/weekly_overview_worker_spec.rb +++ b/spec/workers/weekly_overview_worker_spec.rb @@ -1,6 +1,9 @@ require 'rails_helper' RSpec.describe WeeklyOverviewWorker, type: :worker do + before { Delayed::Worker.delay_jobs = false } + after { Delayed::Worker.delay_jobs = true } + describe 'perform' do let!(:gestionnaire) { create(:gestionnaire) } let(:overview) { double('overview') }