# SPDX-FileCopyrightText: 2024 Tom Hubrecht # # SPDX-License-Identifier: EUPL-1.2 { config, lib, options, pkgs, utils, sources, ... }: let inherit (lib) attrNames composeManyExtensions concatLists concatMapAttrs filterAttrs getExe getExe' literalExpression mapAttrs mapAttrs' mapAttrsToList mkEnableOption mkIf mkMerge mkOption mkPackageOption nameValuePair optional optionalString optionals recursiveUpdate toUpper ; inherit (lib.types) attrs attrsOf bool enum functionTo ints lines listOf nullOr package path str submodule ; inherit (utils) escapeSystemdExecArgs; cfg = config.services.django-apps; # Alias the global config to allow its use when the identifier is shadowed config' = config; systemctl = getExe' config.systemd.package "systemctl"; in { options.services.django-apps = { enable = mkEnableOption "automatic django apps management"; webhook = { domain = mkOption { type = str; description = '' The domain where the webhook service will listen. ''; }; nginx = mkOption { type = nullOr options.services.nginx.virtualHosts.type.nestedTypes.elemType; default = null; description = '' With this option, you can customize the nginx virtualHost settings. ''; example = literalExpression '' { # To enable encryption and let Let's Encrypt take care of certificate forceSSL = true; enableACME = true; } ''; }; }; sites = mkOption { type = attrsOf ( submodule ( { name, config, ... }: { options = { source = mkOption { type = str; description = '' The URI where the source of the app can be publicly fetched via git. ''; }; branch = mkOption { type = str; default = "production"; description = '' Branch to follow for updates to the source. ''; }; domain = mkOption { type = str; description = '' The domain where the web app will be served. ''; }; nginx = mkOption { type = nullOr options.services.nginx.virtualHosts.type.nestedTypes.elemType; default = null; description = '' With this option, you can customize the nginx virtualHost settings. ''; example = literalExpression '' { # To enable encryption and let Let's Encrypt take care of certificate forceSSL = true; enableACME = true; } ''; }; admins = mkOption { type = attrsOf str; default = { }; description = '' Admins of this website, they will be added to the ADMINS credential. ''; example = { "Toto Example" = "toto@example.com"; }; }; serveMedia = mkOption { type = bool; default = true; description = "Wether to serve the MEDIA_ROOT directory with nginx."; }; env_prefix = mkOption { type = str; default = toUpper name; description = '' The prefix to use for environment settings declaration. ''; }; application = { type = mkOption { type = enum [ "asgi" "wsgi" "daphne" ]; default = "wsgi"; description = '' Specification for the django application. ''; }; module = mkOption { type = str; default = "app"; description = '' Name of the module containing the application interface. ''; }; settingsModule = mkOption { type = str; default = "${config.application.module}.settings"; description = '' The django settings module, will be passed as an environment variable to the app. ''; }; workers = mkOption { type = ints.positive; default = 4; description = '' Number of workers processes to use. ''; }; channelLayer = mkOption { type = str; default = "default"; description = '' Channel layer to use when running the application with daphne. ''; }; }; python = mkPackageOption pkgs "python3" { }; overlays = { kat-pkgs = mkOption { type = listOf str; default = [ ]; description = '' List of python packages to pull from [kat-pkgs](https://git.dgnum.eu/lbailly/kat-pkgs). ''; }; nix-pkgs = mkOption { type = listOf str; default = [ ]; description = '' List of python packages to pull from [nix-pkgs](https://git.hubrecht.ovh/hubrecht/nix-pkgs). ''; }; }; django = mkOption { type = functionTo package; default = ps: ps.django; defaultText = literalExpression "ps: ps.django"; description = '' The django version to use to run the app. ''; }; djangoEnv = mkOption { type = package; default = let overlays = (optional (config.overlays.nix-pkgs != [ ]) ( (import "${sources.nix-pkgs}/overlay.nix").mkOverlay { folder = "python-modules"; plist = config.overlays.nix-pkgs; } )) ++ (optional (config.overlays.kat-pkgs != [ ]) ( self: super: super.lib.genAttrs config.overlays.kat-pkgs ( name: self.callPackage "${sources.kat-pkgs}/python-pkgs/${name}.nix" { } ) )); in ( if (overlays != [ ]) then config.python.override { packageOverrides = composeManyExtensions overlays; } else config.python ).withPackages ( ps: [ (config.django ps) ] ++ (optional (config.application.type != "daphne") ps.gunicorn) ++ (optional (config.application.type == "daphne") ps.daphne) ++ (optional (config.application.type == "asgi") ps.uvicorn) ++ (optional (config.dbType == "postgresql") ps.psycopg) ++ (config.dependencies ps) ); description = '' The python version used to run the app, with the correct dependencies. ''; }; dependencies = mkOption { type = functionTo (listOf package); default = _: [ ]; example = literalExpression "ps: [ ps.requests ]"; description = '' Python dependencies of the app. ''; }; extraPackages = mkOption { type = listOf package; default = [ ]; description = '' Packages that will be added to the path of the app. ''; }; credentials = mkOption { type = attrsOf path; default = { }; description = '' The files containing credentials to pass through `LoadCredential` to the application. ''; }; environment = mkOption { type = attrsOf (pkgs.formats.json { }).type; default = { }; description = '' Environment variables to pass to the app. ''; }; managePath = mkOption { type = str; default = "manage.py"; description = '' Path to the manage.py file inside the source ''; }; extraServices = mkOption { type = attrs; default = { }; description = '' Extra services to run in parallel of the application. May be used to run background tasks and/or workers. ''; }; manageScript = mkOption { type = package; default = pkgs.writeShellApplication { name = "${name}-manage"; runtimeInputs = [ pkgs.util-linux config'.systemd.package config.djangoEnv ] ++ config.extraPackages; text = '' MainPID=$(systemctl show -p MainPID --value dj-${name}.service) nsenter -e -a -t "$MainPID" -G follow -S follow python /var/lib/django-apps/${name}/source/${config.managePath} "$@" ''; }; description = '' Script to run manage.py related tasks. ''; }; updateScript = mkOption { type = package; default = pkgs.writeShellApplication { name = "dj-${name}-update-source"; runtimeInputs = [ config.djangoEnv pkgs.git ]; text = '' git pull python3 ${config.managePath} migrate python3 ${config.managePath} collectstatic --no-input ${optionalString (config.extraUpdateSteps != null) config.extraUpdateSteps} ''; }; description = '' Script to run when updating the app source. ''; }; extraUpdateSteps = mkOption { type = nullOr lines; default = null; description = '' Steps taken during the update after the migration is done and the static files have been collected. ''; }; webHookSecret = mkOption { type = path; description = '' Path to the webhook secret. ''; }; dbType = mkOption { type = enum [ "manual" "postgresql" "sqlite" ]; default = "postgresql"; description = '' Which database backend to use, set to `manual` for custom declaration. ''; }; baseDirectory = mkOption { type = str; readOnly = true; default = "/var/lib/django-apps/${name}"; description = '' The directory containing the various files necessary for the website to function. ''; }; sourceDirectory = mkOption { type = str; readOnly = true; default = "${config.baseDirectory}/source"; description = '' The path where the source code of the django application will be stored. It is an absolute path. ''; }; staticDirectory = mkOption { type = str; default = "static"; description = '' Path to the staticfiles directory. This is relative to the base directory, e.g. the parent of the source directory. ''; }; mediaDirectory = mkOption { type = str; default = "media"; description = '' Path to the media files directory. This is relative to the base directory, e.g. the parent of the source directory. ''; }; }; } ) ); default = { }; description = '' The set of django websites to deploy. ''; }; }; config = mkIf cfg.enable { security.sudo.extraRules = [ { users = [ "webhook" ]; commands = builtins.map (name: { command = "${systemctl} start dj-${name}-update.service"; options = [ "NOPASSWD" ]; }) (attrNames cfg.sites); } ]; environment.systemPackages = mapAttrsToList (_: { manageScript, ... }: manageScript) cfg.sites; services = { webhook = { enable = true; package = pkgs.webhook.overrideAttrs (old: { patches = (old.patches or [ ]) ++ [ ./01-webhook.patch ]; }); # Only listen on localhost ip = "127.0.0.1"; verbose = false; hooksTemplated = mapAttrs' ( name: { branch, ... }: nameValuePair "dj-${name}" ( # Avoid issues when quoting "dj-name" through builtins.toJSON builtins.replaceStrings [ "\\" ] [ "" ] ( builtins.toJSON { id = "dj-${name}"; execute-command = "/run/wrappers/bin/sudo"; pass-arguments-to-command = builtins.map (name: { inherit name; source = "string"; }) [ systemctl "start" "dj-${name}-update.service" ]; # command-working-directory = "/var/lib/django-apps/${name}"; trigger-rule = { and = [ { "or" = [ { match = { type = "payload-hmac-sha256"; secret = ''{{ credential "dj-${name}" | js }}''; parameter = { source = "header"; name = "X-Hub-Signature-256"; }; }; } { match = { type = "value"; value = ''{{ credential "dj-${name}" | js }}''; parameter = { source = "header"; name = "X-Gitlab-Token"; }; }; } ]; } { match = { type = "value"; value = "refs/heads/${branch}"; parameter = { source = "payload"; name = "ref"; }; }; } ]; }; } ) ) ) cfg.sites; }; nginx = mkMerge [ (mkIf (cfg.webhook.nginx != null) { enable = true; virtualHosts = { ${cfg.webhook.domain} = mkMerge [ { locations."/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.webhook.port}"; } cfg.webhook.nginx ]; }; }) { virtualHosts = mapAttrs' ( name: { application, domain, nginx, serveMedia, ... }: nameValuePair domain ( recursiveUpdate { locations = { "/".proxyPass = if application.type == "daphne" then "http://unix:/run/django-apps/${name}/socket" else "http://unix:/run/django-apps/${name}.sock"; "/static/".root = "/run/django-apps/${name}"; "/media/".root = mkIf serveMedia "/run/django-apps/${name}"; }; } nginx ) ) cfg.sites; } ]; postgresql = let apps = builtins.map (name: "dj-${name}") ( attrNames (filterAttrs (_: { dbType, ... }: dbType == "postgresql") cfg.sites) ); in mkIf (apps != [ ]) { enable = true; ensureDatabases = apps; ensureUsers = builtins.map (name: { inherit name; ensureDBOwnership = true; }) apps; }; }; users = { users.nginx.extraGroups = [ "django-apps" ]; groups.django-apps = { }; }; systemd = { sockets = mapAttrs' ( name: _: nameValuePair "dj-${name}" { description = "Socket for the ${name} Django Application"; wantedBy = [ "sockets.target" ]; socketConfig = { ListenStream = "/run/django-apps/${name}.sock"; SocketMode = "600"; SocketUser = config'.services.nginx.user; }; } ) (filterAttrs (_: { application, ... }: application.type != "daphne") cfg.sites); mounts = concatLists ( mapAttrsToList ( name: { mediaDirectory, staticDirectory, ... }: [ { where = "/run/django-apps/${name}/static"; what = "/var/lib/django-apps/${name}/${staticDirectory}"; options = "bind"; after = [ "dj-${name}.service" ]; partOf = [ "dj-${name}.service" ]; upheldBy = [ "dj-${name}.service" ]; } { where = "/run/django-apps/${name}/media"; what = "/var/lib/django-apps/${name}/${mediaDirectory}"; options = "bind"; after = [ "dj-${name}.service" ]; partOf = [ "dj-${name}.service" ]; upheldBy = [ "dj-${name}.service" ]; } ] ) cfg.sites ); services = { webhook.serviceConfig.LoadCredential = mapAttrsToList ( name: { webHookSecret, ... }: "dj-${name}:${webHookSecret}" ) cfg.sites; } // (concatMapAttrs ( name: config: let mkDatabase = name: type: if type == "postgresql" then { ENGINE = "django.db.backends.postgresql"; NAME = "dj-${name}"; } else if type == "sqlite" then { ENGINE = "django.db.backends.sqlite3"; NAME = "/var/lib/django-apps/${name}/db.sqlite3"; } else throw "Invalid database type !"; # Systemd Service Configuration Group = "django-apps"; LoadCredential = mapAttrsToList (credential: path: "${credential}:${path}") config.credentials; RuntimeDirectory = "django-apps/${name}"; StateDirectory = "django-apps/${name}"; UMask = "0027"; User = "dj-${name}"; WorkingDirectory = "/var/lib/django-apps/${name}"; environment = let mkValue = v: if builtins.isString v then v else builtins.toJSON v; in (mapAttrs' (key: value: nameValuePair "${config.env_prefix}_${key}" (mkValue value)) { DATABASES = if (config.dbType != "manual") then { default = mkDatabase name config.dbType; } else null; STATIC_ROOT = "/var/lib/django-apps/${name}/${config.staticDirectory}"; MEDIA_ROOT = "/var/lib/django-apps/${name}/${config.mediaDirectory}"; ALLOWED_HOSTS = [ config.domain ]; ADMINS = mapAttrsToList (name: email: [ name email ]) ({ "Admins DGNum" = "admins+dj-${name}@dgnum.eu"; } // config.admins); }) // { DJANGO_SETTINGS_MODULE = config.application.settingsModule; } // (mapAttrs (_: mkValue) config.environment); path = config.extraPackages ++ [ config.djangoEnv ]; after = [ "network.target" ] ++ (optional (config.dbType == "postgresql") "postgresql.service"); in { "dj-${name}" = { inherit after environment path; preStart = '' if [ ! -f .initialized ]; then # The previous initialization might have failed, so restart from the beginning rm -rf source # We need to download the application source and run the migrations first ${lib.getExe pkgs.git} clone --single-branch --branch ${config.branch} ${config.source} source (cd source && python ${config.managePath} migrate --no-input && python ${config.managePath} collectstatic --no-input) touch .initialized fi # Create the necessary directory with the correct user/group mkdir -p ${config.mediaDirectory} ${config.staticDirectory} ''; requires = optional (config.application.type != "daphne") "dj-${name}.socket"; wantedBy = [ "multi-user.target" ]; script = mkIf (config.application.type == "daphne") '' cd source && ${getExe' config.djangoEnv "daphne"} \ -u /run/django-apps/${name}/socket \ ${config.application.module}.asgi:application ''; serviceConfig = { inherit Group LoadCredential RuntimeDirectory StateDirectory UMask User WorkingDirectory ; DynamicUser = true; ExecStart = mkIf (config.application.type != "daphne") ( escapeSystemdExecArgs ( [ (getExe' config.djangoEnv "gunicorn") "--workers" config.application.workers "--bind" "unix:/run/django-apps/${name}.sock" "--pythonpath" "source" ] ++ (optionals (config.application.type == "asgi") [ "--worker-class" "uvicorn.workers.UvicornWorker" ]) ++ [ "${config.application.module}.${config.application.type}" ] ) ); ExecReload = "${getExe' pkgs.coreutils "kill"} -s HUP $MAINPID"; KillMode = "mixed"; Type = mkIf (config.application.type != "daphne") "notify"; }; }; "dj-${name}-update" = { inherit environment path; serviceConfig = { inherit Group LoadCredential StateDirectory UMask User ; DynamicUser = true; ExecStart = "${getExe config.updateScript}"; Type = "oneshot"; WorkingDirectory = "/var/lib/django-apps/${name}/source"; }; unitConfig = { After = "dj-${name}.service"; Conflicts = "dj-${name}.service"; OnSuccess = "dj-${name}.service"; }; }; } // (mapAttrs' ( serviceName: serviceContent: nameValuePair "dj-${name}_${serviceName}" ( recursiveUpdate { inherit after environment path; partOf = [ "dj-${name}.service" ]; wantedBy = [ "multi-user.target" ]; upheldBy = [ "dj-${name}.service" ]; serviceConfig = { inherit Group LoadCredential RuntimeDirectory StateDirectory UMask User ; DynamicUser = true; }; } serviceContent ) ) config.extraServices) ) cfg.sites); }; dgn-backups = { # jobs = mapAttrs' ( # name: _: nameValuePair "dj-${name}" { settings.paths = [ "/var/lib/private/django-apps/${name}" ]; } # ) cfg.sites; postgresDatabases = builtins.map (name: "dj-${name}") ( attrNames (filterAttrs (_: { dbType, ... }: dbType == "postgresql") cfg.sites) ); }; }; }