From df222f18a3e2a9c3b48cd9db8d3569c078fe9fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20P=C3=A9pin?= Date: Fri, 4 Dec 2020 16:09:59 +0100 Subject: [PATCH 1/4] =?UTF-8?q?Update=20the=20vagrant=20config=20=E2=86=92?= =?UTF-8?q?=20should=20work=20now?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Vagrantfile | 42 +++------------ cof/settings/dev.py | 58 ++++++++++++++++++++ provisioning/bootstrap.sh | 96 ++++++++++++++++++++++------------ provisioning/daphne.service | 9 ++-- provisioning/nginx.conf | 54 +++++++------------ provisioning/prepare_django.sh | 7 ++- provisioning/worker.service | 6 +-- 7 files changed, 160 insertions(+), 112 deletions(-) create mode 100644 cof/settings/dev.py diff --git a/Vagrantfile b/Vagrantfile index e12a45ed..f34653a5 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,47 +1,19 @@ # -*- mode: ruby -*- # vi: set ft=ruby : -# All Vagrant configuration is done below. The "2" in Vagrant.configure -# configures the configuration version (we support older styles for -# backwards compatibility). Please don't change it unless you know what -# you're doing. +# Configuration de base pour GestioCOF. +# Voir https://docs.vagrantup.com pour plus d'informations. Vagrant.configure(2) do |config| - # The most common configuration options are documented and commented below. - # For a complete reference, please see the online documentation at - # https://docs.vagrantup.com. - - config.vm.box = "ubuntu/xenial64" + # On se base sur Debian 10 (Buster) pour avoir le même environnement qu'en + # production. + config.vm.box = "debian/contrib-buster64" # On associe le port 80 dans la machine virtuelle avec le port 8080 de notre # ordinateur, et le port 8000 avec le port 8000. config.vm.network :forwarded_port, guest: 80, host: 8080 config.vm.network :forwarded_port, guest: 8000, host: 8000 - # Create a private network, which allows host-only access to the machine - # using a specific IP. - # config.vm.network "private_network", ip: "192.168.33.10" - - # Provider-specific configuration so you can fine-tune various - # backing providers for Vagrant. These expose provider-specific options. - # Example for VirtualBox: - # - # config.vm.provider "virtualbox" do |vb| - # # Display the VirtualBox GUI when booting the machine - # vb.gui = true - # - # # Customize the amount of memory on the VM: - # vb.memory = "1024" - # end - # - # View the documentation for the provider you are using for more - # information on available options. - - # Enable provisioning with a shell script. Additional provisioners such as - # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the - # documentation for more information about their specific syntax and use. - # config.vm.provision "shell", inline: <<-SHELL - # sudo apt-get update - # sudo apt-get install -y apache2 - # SHELL + # Le restes de la configuration (installation de paquets, etc) est géré un + # script shell. config.vm.provision :shell, path: "provisioning/bootstrap.sh" end diff --git a/cof/settings/dev.py b/cof/settings/dev.py new file mode 100644 index 00000000..7e1a63a8 --- /dev/null +++ b/cof/settings/dev.py @@ -0,0 +1,58 @@ +"""Django local development settings.""" +import os + +from . import bds_prod +from .cof_prod import * # NOQA +from .cof_prod import INSTALLED_APPS, MIDDLEWARE, TESTING + +# --- +# Merge COF and BDS configs +# --- + +for app in bds_prod.INSTALLED_APPS: + if app not in INSTALLED_APPS: + INSTALLED_APPS.append(app) + +# --- +# Tweaks for debug/local development +# --- + +ALLOWED_HOSTS = [] + +DEBUG = True +EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" + +if TESTING: + PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] + +STATIC_URL = "/static/" +STATIC_ROOT = "/srv/gestiocof/static" +MEDIA_URL = "/media/" +MEDIA_ROOT = "/srv/gestiocof/media" + + +# --- +# Debug tool bar +# --- + + +def show_toolbar(request): + """ + On active la debug-toolbar en mode développement local sauf : + - dans l'admin où ça ne sert pas à grand chose; + - si la variable d'environnement DJANGO_NO_DDT est à 1 → ça permet de la désactiver + sans modifier ce fichier en exécutant `export DJANGO_NO_DDT=1` dans le terminal + qui lance `./manage.py runserver`. + + Autre side effect de cette fonction : on ne fait pas la vérification de INTERNAL_IPS + que ferait la debug-toolbar par défaut, ce qui la fait fonctionner aussi à + l'intérieur de Vagrant (comportement non testé depuis un moment…) + """ + env_no_ddt = bool(os.environ.get("DJANGO_NO_DDT", None)) + return DEBUG and not env_no_ddt and not request.path.startswith("/admin/") + + +if not TESTING: + INSTALLED_APPS += ["debug_toolbar"] + MIDDLEWARE = ["debug_toolbar.middleware.DebugToolbarMiddleware"] + MIDDLEWARE + DEBUG_TOOLBAR_CONFIG = {"SHOW_TOOLBAR_CALLBACK": show_toolbar} diff --git a/provisioning/bootstrap.sh b/provisioning/bootstrap.sh index cb6917a7..5e2e4c44 100644 --- a/provisioning/bootstrap.sh +++ b/provisioning/bootstrap.sh @@ -1,36 +1,60 @@ #!/bin/sh -# Stop if an error is encountered -set -e +# Arête le script quand : +# - une erreur survient +# - on essaie d'utiliser une variable non définie +# - on essaie d'écraser un fichier avec une redirection (>). +set -euC -# Configuration de la base de données. Le mot de passe est constant car c'est +# Configuration de la base de données, redis, Django, etc. +# Tous les mots de passe sont constant et en clair dans le fichier car c'est # pour une installation de dév locale qui ne sera accessible que depuis la # machine virtuelle. -DBUSER="cof_gestion" -DBNAME="cof_gestion" -DBPASSWD="4KZt3nGPLVeWSvtBZPSM3fSzXpzEU4" +readonly DBUSER="cof_gestion" +readonly DBNAME="cof_gestion" +readonly DBPASSWD="4KZt3nGPLVeWSvtBZPSM3fSzXpzEU4" +readonly REDIS_PASSWD="dummy" +readonly DJANGO_SETTINGS_MODULE="cof.settings.dev" + # Installation de paquets utiles -apt-get update && apt-get upgrade -y +apt-get update +# https://github.com/chef/bento/issues/661 +export DEBIAN_FRONTEND=noninteractive +apt-get -y upgrade + # -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \ + # upgrade apt-get install -y python3-pip python3-dev python3-venv libpq-dev postgresql \ - postgresql-contrib libjpeg-dev nginx git redis-server + postgresql-contrib libjpeg-dev nginx git redis-server \ + libldap2-dev libsasl2-dev slapd ldap-utils # Postgresql -sudo -u postgres createdb $DBNAME -sudo -u postgres createuser -SdR $DBUSER +pg_user_exists () { + sudo -u postgres psql postgres -tAc \ + "SELECT 1 FROM pg_roles WHERE rolname='$1'" \ + | grep -q '^1$' +} + +pg_db_exists () { + sudo -u postgres psql postgres -tAc \ + "SELECT 1 FROM pg_database WHERE datname='$1'" \ + | grep -q '^1$' +} + +pg_db_exists "$DBNAME" || sudo -u postgres createdb "$DBNAME" +pg_user_exists "$DBUSER" || sudo -u postgres createuser -SdR "$DBUSER" sudo -u postgres psql -c "ALTER USER $DBUSER WITH PASSWORD '$DBPASSWD';" sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE $DBNAME TO $DBUSER;" # Redis -REDIS_PASSWD="dummy" -redis-cli CONFIG SET requirepass $REDIS_PASSWD -redis-cli -a $REDIS_PASSWD CONFIG REWRITE +redis-cli CONFIG SET requirepass "$REDIS_PASSWD" +redis-cli -a "$REDIS_PASSWD" CONFIG REWRITE # Contenu statique mkdir -p /srv/gestiocof/media mkdir -p /srv/gestiocof/static -chown -R ubuntu:www-data /srv/gestiocof +chown -R vagrant:www-data /srv/gestiocof # Nginx ln -s -f /vagrant/provisioning/nginx.conf /etc/nginx/sites-enabled/gestiocof.conf @@ -38,36 +62,44 @@ rm -f /etc/nginx/sites-enabled/default systemctl reload nginx # Environnement virtuel python -sudo -H -u ubuntu python3 -m venv ~ubuntu/venv -sudo -H -u ubuntu ~ubuntu/venv/bin/pip install -U pip -sudo -H -u ubuntu ~ubuntu/venv/bin/pip install -r /vagrant/requirements-devel.txt +sudo -H -u vagrant python3 -m venv ~vagrant/venv +sudo -H -u vagrant ~vagrant/venv/bin/pip install -U pip +sudo -H -u vagrant ~vagrant/venv/bin/pip install \ + -r /vagrant/requirements-prod.txt \ + -r /vagrant/requirements-devel.txt \ # Préparation de Django cd /vagrant ln -s -f secret_example.py cof/settings/secret.py -sudo -H -u ubuntu \ - DJANGO_SETTINGS_MODULE='cof.settings.dev' \ - bash -c ". ~/venv/bin/activate && bash provisioning/prepare_django.sh" -/home/ubuntu/venv/bin/python manage.py collectstatic --noinput --settings cof.settings.dev +sudo -H -u vagrant \ + DJANGO_SETTINGS_MODULE="$DJANGO_SETTINGS_MODULE"\ + /bin/sh -c ". ~vagrant/venv/bin/activate && /bin/sh provisioning/prepare_django.sh" +~vagrant/venv/bin/python manage.py collectstatic \ + --noinput \ + --settings "$DJANGO_SETTINGS_MODULE" # Installation du cron pour les mails de rappels -sudo -H -u ubuntu crontab provisioning/cron.dev +# TODO: FIXME +# sudo -H -u vagrant crontab provisioning/cron.dev # Daphne + runworker -cp /vagrant/provisioning/daphne.service /etc/systemd/system/daphne.service -cp /vagrant/provisioning/worker.service /etc/systemd/system/worker.service -systemctl enable daphne.service -systemctl enable worker.service -systemctl start daphne.service -systemctl start worker.service +# TODO: explain +ln -sf /vagrant/provisioning/daphne.service /etc/systemd/system/daphne.service +ln -sf /vagrant/provisioning/worker.service /etc/systemd/system/worker.service +systemctl enable --now daphne.service +systemctl enable --now worker.service -# Mise en place du .bash_profile pour tout configurer lors du `vagrant ssh` -cat >> ~ubuntu/.bashrc < ~vagrant/.bash_aliases < Date: Fri, 4 Dec 2020 16:58:25 +0100 Subject: [PATCH 2/4] =?UTF-8?q?Vagrant:=20toutes=20les=20units=20systemd?= =?UTF-8?q?=20sont=20l=C3=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- provisioning/bootstrap.sh | 23 +++++++++++-------- provisioning/cron.dev | 10 -------- .../{nginx.conf => nginx/gestiocof.conf} | 0 provisioning/{ => systemd}/daphne.service | 2 +- provisioning/systemd/rappels.service | 8 +++++++ provisioning/systemd/rappels.timer | 9 ++++++++ provisioning/systemd/reventes.service | 8 +++++++ provisioning/systemd/reventes.timer | 9 ++++++++ provisioning/{ => systemd}/worker.service | 0 9 files changed, 49 insertions(+), 20 deletions(-) delete mode 100644 provisioning/cron.dev rename provisioning/{nginx.conf => nginx/gestiocof.conf} (100%) rename provisioning/{ => systemd}/daphne.service (91%) create mode 100644 provisioning/systemd/rappels.service create mode 100644 provisioning/systemd/rappels.timer create mode 100644 provisioning/systemd/reventes.service create mode 100644 provisioning/systemd/reventes.timer rename provisioning/{ => systemd}/worker.service (100%) diff --git a/provisioning/bootstrap.sh b/provisioning/bootstrap.sh index 5e2e4c44..9b2bf9f2 100644 --- a/provisioning/bootstrap.sh +++ b/provisioning/bootstrap.sh @@ -57,7 +57,7 @@ mkdir -p /srv/gestiocof/static chown -R vagrant:www-data /srv/gestiocof # Nginx -ln -s -f /vagrant/provisioning/nginx.conf /etc/nginx/sites-enabled/gestiocof.conf +ln -s -f /vagrant/provisioning/nginx/gestiocof.conf /etc/nginx/sites-enabled/gestiocof.conf rm -f /etc/nginx/sites-enabled/default systemctl reload nginx @@ -78,16 +78,21 @@ sudo -H -u vagrant \ --noinput \ --settings "$DJANGO_SETTINGS_MODULE" -# Installation du cron pour les mails de rappels -# TODO: FIXME -# sudo -H -u vagrant crontab provisioning/cron.dev - -# Daphne + runworker -# TODO: explain -ln -sf /vagrant/provisioning/daphne.service /etc/systemd/system/daphne.service -ln -sf /vagrant/provisioning/worker.service /etc/systemd/system/worker.service +# Quelques units systemd: +# - Daphne fait tourner le serveur asgi +# - worker = https://channels.readthedocs.io/en/stable/topics/worker.html +# - Mails de rappels du BdA +# - Mails de revente du BdA +ln -sf /vagrant/provisioning/systemd/daphne.service /etc/systemd/system/daphne.service +ln -sf /vagrant/provisioning/systemd/worker.service /etc/systemd/system/worker.service +ln -sf /vagrant/provisioning/systemd/reventes.service /etc/systemd/system/reventes.service +ln -sf /vagrant/provisioning/systemd/rappels.service /etc/systemd/system/rappels.service +ln -sf /vagrant/provisioning/systemd/reventes.timer /etc/systemd/system/reventes.timer +ln -sf /vagrant/provisioning/systemd/rappels.timer /etc/systemd/system/rappels.timer systemctl enable --now daphne.service systemctl enable --now worker.service +systemctl enable rappels.timer +systemctl enable reventes.timer # Configure le bash de l'utilisateur 'vagrant' pour utiliser le bon fichier de # settings et et bon virtualenv. diff --git a/provisioning/cron.dev b/provisioning/cron.dev deleted file mode 100644 index 896b5125..00000000 --- a/provisioning/cron.dev +++ /dev/null @@ -1,10 +0,0 @@ -# On utilise la version de développement de GestioCOF -DJANGO_SETTINGS_MODULE='cof.settings.dev' - -# Identifiants MySQL -DBUSER="cof_gestion" -DBNAME="cof_gestion" -DBPASSWD="4KZt3nGPLVeWSvtBZPSM3fSzXpzEU4" - -19 */12 * * * date >> /vagrant/rappels.log ; /ubuntu/home/venv/bin/python /vagrant/manage.py sendrappels >> /vagrant/rappels.log 2>&1 -*/5 * * * * /ubuntu/home/venv/bin/python /vagrant/manage.py manage_revente >> /vagrant/reventes.log 2>&1 diff --git a/provisioning/nginx.conf b/provisioning/nginx/gestiocof.conf similarity index 100% rename from provisioning/nginx.conf rename to provisioning/nginx/gestiocof.conf diff --git a/provisioning/daphne.service b/provisioning/systemd/daphne.service similarity index 91% rename from provisioning/daphne.service rename to provisioning/systemd/daphne.service index 451ac37c..a9c30008 100644 --- a/provisioning/daphne.service +++ b/provisioning/systemd/daphne.service @@ -1,4 +1,4 @@ -Description="GestioCOF" +Description="GestioCOF - Daphne" After=syslog.target After=network.target diff --git a/provisioning/systemd/rappels.service b/provisioning/systemd/rappels.service new file mode 100644 index 00000000..2d407d53 --- /dev/null +++ b/provisioning/systemd/rappels.service @@ -0,0 +1,8 @@ +[Unit] +Description=Envoi des mails de rappel des spectales BdA + +[Service] +Type=oneshot +User=vagrant +Environment="DJANGO_SETTINGS_MODULE=cof.settings.dev" +ExecStart=/home/vagrant/venv/bin/python /vagrant/manage.py sendrappels diff --git a/provisioning/systemd/rappels.timer b/provisioning/systemd/rappels.timer new file mode 100644 index 00000000..f05c54e0 --- /dev/null +++ b/provisioning/systemd/rappels.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Envoi des mails de rappel des spectales BdA + +[Timer] +OnBootSec=10min +OnUnitActiveSec=3h + +[Install] +WantedBy=timers.target diff --git a/provisioning/systemd/reventes.service b/provisioning/systemd/reventes.service new file mode 100644 index 00000000..bd1992f8 --- /dev/null +++ b/provisioning/systemd/reventes.service @@ -0,0 +1,8 @@ +[Unit] +Description=Envoi des mails de BdA-Revente + +[Service] +Type=oneshot +User=vagrant +Environment="DJANGO_SETTINGS_MODULE=cof.settings.dev" +ExecStart=/home/vagrant/venv/bin/python /vagrant/manage.py manage_reventes diff --git a/provisioning/systemd/reventes.timer b/provisioning/systemd/reventes.timer new file mode 100644 index 00000000..2ccaf7bf --- /dev/null +++ b/provisioning/systemd/reventes.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Envoi des mails de BdA-Revente + +[Timer] +OnBootSec=15min +OnUnitActiveSec=15min + +[Install] +WantedBy=timers.target diff --git a/provisioning/worker.service b/provisioning/systemd/worker.service similarity index 100% rename from provisioning/worker.service rename to provisioning/systemd/worker.service From 783fe1de3278dc3b19b06b0b64f5897190130ab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20P=C3=A9pin?= Date: Mon, 7 Dec 2020 19:58:00 +0100 Subject: [PATCH 3/4] =?UTF-8?q?Liste=20des=20paquets=20dans=20un=20fichier?= =?UTF-8?q?=20s=C3=A9par=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- provisioning/bootstrap.sh | 18 +++++++++++------- provisioning/packages.list | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 provisioning/packages.list diff --git a/provisioning/bootstrap.sh b/provisioning/bootstrap.sh index 9b2bf9f2..9659f89d 100644 --- a/provisioning/bootstrap.sh +++ b/provisioning/bootstrap.sh @@ -17,16 +17,20 @@ readonly REDIS_PASSWD="dummy" readonly DJANGO_SETTINGS_MODULE="cof.settings.dev" -# Installation de paquets utiles -apt-get update +# --- +# Installation des paquets systèmes +# --- + +get_packages_list () { + sed 's/#.*$//' /vagrant/provisioning/packages.list | grep -v '^ *$' +} + # https://github.com/chef/bento/issues/661 export DEBIAN_FRONTEND=noninteractive + +apt-get update apt-get -y upgrade - # -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \ - # upgrade -apt-get install -y python3-pip python3-dev python3-venv libpq-dev postgresql \ - postgresql-contrib libjpeg-dev nginx git redis-server \ - libldap2-dev libsasl2-dev slapd ldap-utils +get_packages_list | xargs apt-get install -y # Postgresql pg_user_exists () { diff --git a/provisioning/packages.list b/provisioning/packages.list new file mode 100644 index 00000000..34714442 --- /dev/null +++ b/provisioning/packages.list @@ -0,0 +1,25 @@ +# Python +python3-pip +python3-dev +python3-venv + +# Pour installer authens depuis git.eleves +git + +# Postgres +libpq-dev +postgresql +postgresql-contrib + +# Pour Pillow +libjpeg-dev + +# Outils de prod +nginx # Test +redis-server + +# Le LDAP +libldap2-dev +libsasl2-dev +slapd +ldap-utils From 0ce1e6258666c0d960e5b678f2e9765d1b79c92b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20P=C3=A9pin?= Date: Mon, 7 Dec 2020 20:04:19 +0100 Subject: [PATCH 4/4] =?UTF-8?q?Fichier=20bootstrap.sh=20mieux=20comment?= =?UTF-8?q?=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- provisioning/bootstrap.sh | 44 +++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/provisioning/bootstrap.sh b/provisioning/bootstrap.sh index 9659f89d..d6b8f914 100644 --- a/provisioning/bootstrap.sh +++ b/provisioning/bootstrap.sh @@ -32,6 +32,11 @@ apt-get update apt-get -y upgrade get_packages_list | xargs apt-get install -y + +# --- +# Configuration de la base de données +# --- + # Postgresql pg_user_exists () { sudo -u postgres psql postgres -tAc \ @@ -51,20 +56,24 @@ sudo -u postgres psql -c "ALTER USER $DBUSER WITH PASSWORD '$DBPASSWD';" sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE $DBNAME TO $DBUSER;" +# --- +# Configuration de redis (pour django-channels) +# --- + # Redis redis-cli CONFIG SET requirepass "$REDIS_PASSWD" redis-cli -a "$REDIS_PASSWD" CONFIG REWRITE -# Contenu statique + +# --- +# Préparation de Django +# --- + +# Dossiers pour le contenu statique mkdir -p /srv/gestiocof/media mkdir -p /srv/gestiocof/static chown -R vagrant:www-data /srv/gestiocof -# Nginx -ln -s -f /vagrant/provisioning/nginx/gestiocof.conf /etc/nginx/sites-enabled/gestiocof.conf -rm -f /etc/nginx/sites-enabled/default -systemctl reload nginx - # Environnement virtuel python sudo -H -u vagrant python3 -m venv ~vagrant/venv sudo -H -u vagrant ~vagrant/venv/bin/pip install -U pip @@ -82,7 +91,11 @@ sudo -H -u vagrant \ --noinput \ --settings "$DJANGO_SETTINGS_MODULE" -# Quelques units systemd: + +# --- +# Units systemd +# --- + # - Daphne fait tourner le serveur asgi # - worker = https://channels.readthedocs.io/en/stable/topics/worker.html # - Mails de rappels du BdA @@ -98,8 +111,12 @@ systemctl enable --now worker.service systemctl enable rappels.timer systemctl enable reventes.timer -# Configure le bash de l'utilisateur 'vagrant' pour utiliser le bon fichier de -# settings et et bon virtualenv. + +# --- +# Configuration du shell de l'utilisateur 'vagrant' pour utiliser le bon fichier +# de settings et et bon virtualenv. +# --- + # On utilise .bash_aliases au lieu de .bashrc pour ne pas écraser la # configuration par défaut. rm -f ~vagrant/.bash_aliases @@ -113,3 +130,12 @@ export DJANGO_SETTINGS_MODULE='$DJANGO_SETTINGS_MODULE' # On va dans /vagrant où se trouve le code de gestioCOF cd /vagrant EOF + + +# --- +# Configuration d'nginx +# --- + +ln -s -f /vagrant/provisioning/nginx/gestiocof.conf /etc/nginx/sites-enabled/gestiocof.conf +rm -f /etc/nginx/sites-enabled/default +systemctl reload nginx