fix(ds-fr): Update module

machines/nixos/compute01/ds-fr/default.nix

@@ -11,41 +11,47 @@
 
 let
   host = "demarches.dgnum.eu";
+  port = 3000;
 
-  dgn-id = "1fbe81d211b18dae7b9c1727362997c62636f24a";
+  dgn-id = "8dfdc60d1aa66e7206461ed7a49199f624a66b4e";
+  patch = pkgs.fetchurl {
+    url = "https://git.dgnum.eu/DGNum/demarches-normaliennes/commit/${dgn-id}.patch";
+    hash = "sha256-6JdbUf2fc79E5F1wtYFnP1JLGJffhGbjaxysRFr8xN4=";
+  };
 in
 {
   imports = [ ./module.nix ];
 
-  dgn-web.internalPorts.ds-fr = 3000;
+  dgn-web.internalPorts.ds-fr = port;
 
   services.demarches-simplifiees = {
     enable = true;
 
-    package =
-      ((import sources.nix-pkgs { inherit pkgs; }).demarches-simplifiees.override {
-        initialDeploymentDate = "20230923";
-      }).overrideAttrs
-        (old: {
-          dsModules = old.dsModules.overrideAttrs {
-            prePatch = ''
-              ${pkgs.lib.getExe pkgs.git} apply -p1 < ${
-                pkgs.fetchurl {
-                  url = "https://git.dgnum.eu/DGNum/demarches-normaliennes/commit/${dgn-id}.patch";
-                  hash = "sha256-aCq/WkV4+PUSIzXgznwm2sAcaz12Y1zmUbh7QoXoMsM=";
-                }
-              }
-            '';
-          };
-        });
+    package = (import sources.nix-pkgs { inherit pkgs; }).demarches-simplifiees.overrideAttrs (old: {
+      dsModules = old.dsModules.overrideAttrs {
+        prePatch = ''
+          ${pkgs.lib.getExe pkgs.git} apply -p1 < ${patch}
+        '';
+      };
 
-    secretFile = config.age.secrets."ds-fr-secret_file".path;
+      prePatch = ''
+        ${pkgs.lib.getExe pkgs.git} apply -p1 < ${patch}
+      '';
+
+      postPatch = ''
+        rm -f lib/tasks/deployment/20240830192553_backfill_hide_instructeurs_email.rake
+        rm -f lib/tasks/deployment/20240912151317_clean_virtual_column_from_procedure_presentation.rake
+        rm -f lib/tasks/deployment/20240920130741_migrate_procedure_presentation_to_columns.rake
+      '';
+    });
+
+    inherit host port;
+
+    environmentFile = config.age.secrets."ds-fr-secret_file".path;
 
     initialDeploymentDate = "20230923";
 
-    settings = {
-      APP_HOST = host;
-
+    environment = {
       # Disable France Connect and Agent Connect
       FRANCE_CONNECT_ENABLED = "disabled";
       AGENT_CONNECT_ENABLED = "disabled";
@@ -87,18 +93,10 @@ in
 
       RUBY_YJIT_ENABLE = "1";
 
-      STRICT_EMAIL_VALIDATION_STARTS_ON = "2024-02-23";
-      WEASYPRINT_URL = "http://127.0.0.1:5000/pdf";
-
-      # Customization
-      # HEADER_LOGO_SRC = "logo_ens_psl_couleur.png";
-      # HEADER_LOGO_ALT = "Par la Recherche, pour la Recherche";
-      # PROCEDURE_DEFAULT_LOGO_SRC = "logo_ens_psl_couleur.png";
+      STRICT_EMAIL_VALIDATION_STARTS_ON = "2024-12-18";
     };
   };
 
-  age-secrets.autoMatch = [ "ds-fr" ];
-
-  dgn-backups.jobs.ds-fr.settings.paths = [ "/var/lib/ds-fr" ];
+  dgn-backups.jobs.ds-fr.settings.paths = [ "/var/lib/demarches-simplifiees" ];
   dgn-backups.postgresDatabases = [ "ds-fr" ];
 }

machines/nixos/compute01/ds-fr/module.nix

@@ -1,5 +1,4 @@
-# Copyright Tom Hubrecht, (2023)
-# SPDX-FileCopyrightText: 2024 Tom Hubrecht <tom.hubrecht@dgnum.eu>
+# SPDX-FileCopyrightText: 2023-2024 Tom Hubrecht <tom.hubrecht@dgnum.eu>
 #
 # SPDX-License-Identifier: EUPL-1.2
 
@@ -7,192 +6,290 @@
   config,
   lib,
   pkgs,
+  utils,
   ...
 }:
 
 let
   inherit (lib)
+    getExe
+    getExe'
+    mapAttrs
     mkDefault
     mkEnableOption
     mkIf
     mkOption
-
+    mkPackageOption
     optional
-    optionalString
-
-    types
     ;
 
+  inherit (lib.types)
+    attrsOf
+    nullOr
+    oneOf
+    package
+    path
+    port
+    str
+    ;
+
+  inherit (utils) escapeSystemdExecArgs;
+
   cfg = config.services.demarches-simplifiees;
 
-  settingsFormat = pkgs.formats.keyValue { };
-
-  env = settingsFormat.generate "ds-fr-env" cfg.settings;
-
-  ds-fr = pkgs.writeShellScriptBin "ds-fr" ''
-    set -a
-    cd ${cfg.package}
-
-    ${optionalString (cfg.secretFile != null) "source ${cfg.secretFile}"}
-    source ${env}
-
-    BIN="$1"
-    shift
-
-    SUDO="exec"
-    if [[ $USER != ${cfg.user} ]]; then
-      SUDO='exec /run/wrappers/bin/sudo -u ${cfg.user} --preserve-env'
-    fi
-
-    $SUDO ${cfg.package}/bin/$BIN "$@"
-  '';
+  weasyprintEnv = pkgs.python3.withPackages (ps: [
+    ps.flask
+    ps.sentry-sdk
+    ps.weasyprint
+  ]);
 in
 {
   options.services.demarches-simplifiees = {
-    enable = mkEnableOption "demarches-simplifiees.";
+    enable = mkEnableOption "Démarches Simplifiées";
 
-    package = mkOption {
-      type = types.package;
-      default = pkgs.callPackage ./package { inherit (cfg) initialDeploymentDate dataDir logDir; };
+    package = mkPackageOption pkgs "demarches-simplifiees" { };
+
+    finalPackage = mkOption {
+      type = package;
+      default = cfg.package.override { inherit (cfg) initialDeploymentDate; };
     };
 
-    user = mkOption {
-      type = types.str;
-      default = "ds-fr";
-      description = "User account under which DS runs.";
+    host = mkOption {
+      type = str;
+      description = ''
+        Hostname of the web server.
+      '';
     };
 
-    group = mkOption {
-      type = types.str;
-      default = "ds-fr";
-      description = "Group account under which DS runs.";
+    port = mkOption {
+      type = port;
+      default = 3000;
+      description = ''
+        Listening port for the web server.
+      '';
     };
 
-    dataDir = mkOption {
-      type = types.str;
-      default = "/var/lib/ds-fr";
+    weasyprintPort = mkOption {
+      type = port;
+      default = 5000;
+      description = ''
+        Port of the weasyprint server.
+      '';
     };
 
-    logDir = mkOption {
-      type = types.str;
-      default = "/var/log/ds-fr";
+    environment = mkOption {
+      type = attrsOf (
+        nullOr (oneOf [
+          package
+          path
+          str
+        ])
+      );
+      description = ''
+        Evironment variables available to Démarches Simplifiées.
+      '';
     };
 
-    secretFile = mkOption {
-      type = types.nullOr types.path;
+    environmentFile = mkOption {
+      type = nullOr path;
       default = null;
+      description = ''
+        Path to a file containing environment variables.
+        Required secrets are `SECRET_KEY_BASE` and `OTP_SECRET_KEY`,
+        which can be generated using `rails secret`.
+      '';
     };
 
-    settings = mkOption { inherit (settingsFormat) type; };
-
     initialDeploymentDate = mkOption {
-      type = types.nullOr types.str;
+      type = nullOr str;
       default = null;
+      description = ''
+        Initial deployment date, used to ignore some migrations,
+        which are known to be buggy and are supposed to change old production data.
+      '';
+    };
+
+    interactScript = mkOption {
+      type = package;
+      default = pkgs.writeShellApplication {
+        name = "ds-fr";
+
+        runtimeInputs = [
+          cfg.finalPackage
+          config.systemd.package
+          pkgs.util-linux
+        ];
+        text = ''
+          MainPID=$(systemctl show -p MainPID --value demarches-simplifiees.service)
+
+          nsenter -e -a -w -t "$MainPID" -G follow -S follow "$@"
+        '';
+      };
+      description = ''
+        Script to run ds-fr tasks.
+      '';
     };
   };
 
   config = mkIf cfg.enable {
 
-    environment.systemPackages = [ ds-fr ];
-
-    systemd.tmpfiles.rules = [
-      "f '${cfg.logDir}/production.log' 0640 ${cfg.user} ${cfg.group} - -"
-      "f '${cfg.dataDir}/.env' 0600 ${cfg.user} ${cfg.group} - -"
-      "d '${cfg.dataDir}/tmp' 0700 ${cfg.user} ${cfg.group} 10d -"
-      "d '${cfg.dataDir}/storage' 0700 ${cfg.user} ${cfg.group} - -"
-    ];
-
-    systemd.services = {
-      ds-fr-setup = {
-        description = "Demarches Simplifiees setup";
-
-        wantedBy = [ "multi-user.target" ];
-        path = [
-          pkgs.bash
-          ds-fr
-        ];
-        after = [ "postgresql.service" ];
+    environment.systemPackages = [ cfg.interactScript ];
 
+    systemd.services =
+      let
         serviceConfig = {
-          Type = "oneshot";
-          User = cfg.user;
-          Group = cfg.group;
-          EnvironmentFile = [ env ] ++ (optional (cfg.secretFile != null) cfg.secretFile);
-          StateDirectory = mkIf (cfg.dataDir == "/var/lib/ds-fr") "ds-fr";
-          LogsDirectory = mkIf (cfg.logDir == "/var/log/ds-fr") "ds-fr";
+          User = "ds-fr";
+          DynamicUser = true;
+          EnvironmentFile = optional (cfg.environmentFile != null) cfg.environmentFile;
+          CacheDirectory = "demarches-simplifiees";
+          LogsDirectory = "demarches-simplifiees";
+          RuntimeDirectory = "demarches-simplifiees";
+          StateDirectory = "demarches-simplifiees";
+          WorkingDirectory = cfg.finalPackage;
+        };
+      in
+      {
+        demarches-simplifiees = {
+          description = "Démarches Simplifiées";
+
+          inherit (cfg) environment;
+
+          path = [
+            cfg.finalPackage
+            pkgs.imagemagick
+          ];
+
+          after = [
+            "network.target"
+            "postgresql.target"
+          ];
+          wantedBy = [ "multi-user.target" ];
+
+          preStart = ''
+            mkdir -p "$STATE_DIRECTORY/storage"
+
+            if [[ ! -f "$STATE_DIRECTORY/.version" ]]; then
+              # Run initial setup
+              rails db:environment:set
+              rails db:schema:load
+              rails db:seed
+              rails jobs:schedule
+              touch "$STATE_DIRECTORY/.version"
+            fi
+
+            if [[ $(cat "$STATE_DIRECTORY/.version") != "$__DS_VERSION" ]]; then
+              # Run migrations on version change
+              rake db:migrate
+              rake after_party:run
+              echo "$__DS_VERSION" > "$STATE_DIRECTORY/.version"
+            fi
+          '';
+
+          serviceConfig = serviceConfig // {
+            ExecStart = escapeSystemdExecArgs [
+              (getExe' cfg.finalPackage "rails")
+              "server"
+              "-b"
+              "127.0.0.1"
+              "-p"
+              cfg.port
+            ];
+          };
         };
 
-        script = ''
-          [[ ! -f ${cfg.dataDir}/.initial-migration ]] \
-            && ds-fr rails db:environment:set \
-            && ds-fr rails db:schema:load \
-            && ds-fr rails db:seed \
-            && touch ${cfg.dataDir}/.initial-migration
+        demarches-simplifiees-work = {
+          description = "Démarches Simplifiées work service";
 
-          ds-fr rake db:migrate
-          ds-fr rake after_party:run
-        '';
-      };
+          inherit (cfg) environment;
 
-      ds-fr-work = {
-        description = "Demarches Simplifiees work service";
+          after = [ "demarches-simplifiees.service" ];
+          wantedBy = [ "multi-user.target" ];
+          bindsTo = [ "demarches-simplifiees.service" ];
+          partOf = [ "demarches-simplifiees.service" ];
 
-        wantedBy = [
-          "multi-user.target"
-          "ds-fr.service"
-        ];
-        after = [
-          "network.target"
-          "ds-fr-setup.service"
-        ];
-        requires = [ "ds-fr-setup.service" ];
+          serviceConfig = serviceConfig // {
+            ExecStart = escapeSystemdExecArgs [
+              (getExe' cfg.finalPackage "rails")
+              "jobs:work"
+            ];
+          };
+        };
 
-        serviceConfig = {
-          ExecStart = "${ds-fr}/bin/ds-fr rails jobs:work";
-          EnvironmentFile = [ env ] ++ (optional (cfg.secretFile != null) cfg.secretFile);
-          User = cfg.user;
-          Group = cfg.group;
-          StateDirectory = mkIf (cfg.dataDir == "/var/lib/ds-fr") "ds-fr";
-          LogsDirectory = mkIf (cfg.logDir == "/var/log/ds-fr") "ds-fr";
+        weasyprint-server = {
+          description = "Weasyprint server";
+
+          wantedBy = [ "multi-user.target" ];
+
+          environment = {
+            BASE_URL = "https://${cfg.host}";
+            LOG_DIR = "/var/log/weasyprint";
+            UWSGI_PYTHONPATH = weasyprintEnv;
+            UWSGI_MODULE = "wgsi:app";
+          };
+
+          serviceConfig = {
+            DynamicUser = true;
+            Type = "notify";
+            WorkingDirectory = cfg.finalPackage.weasyprint_server;
+            LogsDirectory = "weasyprint";
+            ExecStart = escapeSystemdExecArgs [
+              (getExe (pkgs.uwsgi.override { plugins = [ "python3" ]; }))
+              "--http-socket"
+              "127.0.0.1:${builtins.toString cfg.weasyprintPort}"
+              "--processes=4"
+              "--enable-threads"
+            ];
+            NotifyAccess = "all";
+            KillSignal = "SIGQUIT";
+            ExecReload = "${getExe' pkgs.coreutils "kill"} -HUP $MainPID";
+            ExecStop = "${getExe' pkgs.coreutils "kill"} -INT $MainPID";
+
+            ProtectSystem = "full";
+            ProtectHome = true;
+            NoNewPrivileges = true;
+            PrivateDevices = true;
+          };
         };
       };
 
-      ds-fr = {
-        description = "Demarches Simplifiees web service";
-
-        wantedBy = [ "multi-user.target" ];
-        after = [
-          "network.target"
-          "ds-fr-setup.service"
-        ];
-        requires = [ "ds-fr-setup.service" ];
-        path = [ pkgs.imagemagick ];
-
-        serviceConfig = {
-          ExecStart = "${ds-fr}/bin/ds-fr rails server";
-          Environment = [ "RAILS_QUEUE_ADAPTER=delayed_job" ];
-          EnvironmentFile = [ env ] ++ (optional (cfg.secretFile != null) cfg.secretFile);
-          User = cfg.user;
-          Group = cfg.group;
-          StateDirectory = mkIf (cfg.dataDir == "/var/lib/ds-fr") "ds-fr";
-          LogsDirectory = mkIf (cfg.logDir == "/var/log/ds-fr") "ds-fr";
-        };
-      };
-    };
-
     services = {
-      demarches-simplifiees.settings =
-        (builtins.mapAttrs (_: mkDefault) {
-          RAILS_ENV = "production";
-          RAILS_ROOT = builtins.toString cfg.package;
-
+      demarches-simplifiees.environment =
+        # Hardcoded values
+        {
           # Application host name
           #
           # Examples:
           # * For local development: localhost:3000
           # * For preproduction: staging.ds.example.org
           # * For production: ds.example.org
-          APP_HOST = "localhost:3000";
+          APP_HOST = cfg.host;
+
+          # Database credentials
+          DB_DATABASE = "ds-fr";
+          DB_USERNAME = "ds-fr";
+          DB_HOST = "/run/postgresql";
+          DB_PORT = "5432";
+
+          # The variables must be present even if empty...
+          DB_PASSWORD = "";
+          DB_POOL = "";
+
+          # Jobs configuration
+          RAILS_QUEUE_ADAPTER = "delayed_job";
+
+          # Log on stdout
+          RAILS_LOG_TO_STDOUT = "true";
+
+          # Package version
+          __DS_VERSION = cfg.finalPackage.version;
+
+          # Weasyprint endpoint generating attestations v2
+          # See https://github.com/demarches-simplifiees/weasyprint_server
+          WEASYPRINT_URL = "http://127.0.0.1:${builtins.toString cfg.weasyprintPort}/pdf";
+        }
+        // (mapAttrs (_: mkDefault) {
+          RAILS_ENV = "production";
+          RAILS_ROOT = builtins.toString cfg.finalPackage;
 
           # Rails key for signing sensitive data
           # See https://guides.rubyonrails.org/security.html
@@ -227,18 +324,6 @@ in
           # SAML
           SAML_IDP_ENABLED = "disabled";
 
-          # External service: authentication through France Connect
-          FC_PARTICULIER_ID = "";
-          FC_PARTICULIER_SECRET = "";
-          FC_PARTICULIER_BASE_URL = "";
-
-          # External service: authentication through Agent Connect
-          AGENT_CONNECT_ID = "";
-          AGENT_CONNECT_SECRET = "";
-          AGENT_CONNECT_BASE_URL = "";
-          AGENT_CONNECT_JWKS = "";
-          AGENT_CONNECT_REDIRECT = "";
-
           # External service: integration with HelpScout (optional)
           HELPSCOUT_MAILBOX_ID = "";
           HELPSCOUT_CLIENT_ID = "";
@@ -288,9 +373,6 @@ in
           # https://api.gouv.fr/api/api-entreprise.html
           API_ENTREPRISE_KEY = "";
 
-          # External service: CRM for following admin accounts pipeline (specific to démarches-simplifiées.fr)
-          PIPEDRIVE_KEY = "";
-
           # Networks bypassing the email login token that verifies new devices, and rack-attack throttling
           TRUSTED_NETWORKS = "";
 
@@ -299,7 +381,7 @@ in
           #   "sXaot-fKhBlkI8qaSirQyuZbrpv5sVFoOturQ0pFEh0";
 
           # Enable or disable Lograge logs
-          LOGRAGE_ENABLED = "disabled";
+          LOGRAGE_ENABLED = "enabled";
 
           # Logs source for Lograge
           #
@@ -336,57 +418,42 @@ in
 
           # Siret number used for API Entreprise, by default we use SIRET from dinum
           API_ENTREPRISE_DEFAULT_SIRET = "put_your_own_siret";
-        })
-        // {
-          # Database credentials
-          DB_DATABASE = "ds-fr";
-          DB_USERNAME = cfg.user;
-          DB_PASSWORD = "";
-          DB_HOST = "/run/postgresql";
-          DB_POOL = "";
 
-          # Log on stdout
-          RAILS_LOG_TO_STDOUT = true;
-        };
+          # Date from which email validation requires a TLD in email adresses.
+          # This change had been introduced by : cc53946d221d6f64c365ad6c6c4c544802eb94b4
+          # Records (users, …) created before this date won't be affected. See #9978
+          # To set a date, we recommend using *the day after* you have deployed this commit,
+          # so existing records won't be invalid.
+          STRICT_EMAIL_VALIDATION_STARTS_ON = "2024-02-19";
+        });
 
       postgresql = {
         enable = true;
 
         ensureDatabases = [ "ds-fr" ];
 
-        ensureUsers = optional (cfg.user == "ds-fr") {
-          name = "ds-fr";
-          ensureDBOwnership = true;
-        };
+        ensureUsers = [
+          {
+            name = "ds-fr";
+            ensureDBOwnership = true;
+          }
+        ];
 
-        extraPlugins = with config.services.postgresql.package.pkgs; [ postgis ];
+        extensions = [ config.services.postgresql.package.pkgs.postgis ];
       };
 
       nginx = {
         enable = true;
 
-        virtualHosts.${cfg.settings.APP_HOST} = {
+        virtualHosts.${cfg.host} = {
           enableACME = true;
           forceSSL = true;
-          root = "${cfg.package}/public/";
+          root = "${cfg.finalPackage}/public/";
 
           locations."/".tryFiles = "$uri @proxy";
-          locations."@proxy" = {
-            proxyPass = "http://127.0.0.1:3000";
-          };
+          locations."@proxy".proxyPass = "http://127.0.0.1:${builtins.toString cfg.port}";
         };
       };
     };
-
-    users.users = mkIf (cfg.user == "ds-fr") {
-      ds-fr = {
-        inherit (cfg) group;
-
-        isSystemUser = true;
-        home = cfg.package;
-      };
-    };
-
-    users.groups.${cfg.group} = { };
   };
 }

npins/sources.json

@@ -262,9 +262,9 @@
         "url": "https://git.hubrecht.ovh/hubrecht/nix-pkgs"
       },
       "branch": "main",
-      "revision": "e8494b9d6110a97e2225b2fe43d29efa34cd9451",
+      "revision": "476e657d9c285d91638b2a7c2bbbd9e6f9d0cfd4",
       "url": null,
-      "hash": "1r2g3jdr311cn8y0cxvawc6qyp58lbydscp5hxadya2vl810vpln"
+      "hash": "1i1a46q2v465zfa8rcfk1xisb7ywd4as18q6n2842ncnm69fxqns"
     },
     "nix-reuse": {
       "type": "GitRelease",