diff --git a/meta/nodes.nix b/meta/nodes.nix index 445e2f1..fc1cf35 100644 --- a/meta/nodes.nix +++ b/meta/nodes.nix @@ -132,8 +132,6 @@ hashedPassword = "$y$j9T$Un/tcX5SPKNXG.sy/BcTa.$kyNHELjb1GAOWnauJfcjyVi5tacWcuEBKflZDCUC6x4"; - nix-modules = [ "services/django-apps" ]; - stateVersion = "24.05"; nixpkgs = "unstable"; vm-cluster = "Hyperviseur NPS"; diff --git a/modules/default.nix b/modules/default.nix index 684fc66..fa603cc 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -58,6 +58,7 @@ "dgn-ssh" "dgn-vm-variant" "dgn-web" + "django-apps" ]) ++ [ "${sources.agenix}/modules/age.nix" diff --git a/modules/django-apps/01-webhook.patch b/modules/django-apps/01-webhook.patch new file mode 100644 index 0000000..5d492b8 --- /dev/null +++ b/modules/django-apps/01-webhook.patch @@ -0,0 +1,67 @@ +diff --git a/internal/hook/hook.go b/internal/hook/hook.go +index 0510095..0347f26 100644 +--- a/internal/hook/hook.go ++++ b/internal/hook/hook.go +@@ -13,12 +13,12 @@ import ( + "errors" + "fmt" + "hash" +- "io/ioutil" + "log" + "math" + "net" + "net/textproto" + "os" ++ "path" + "reflect" + "regexp" + "strconv" +@@ -750,14 +750,18 @@ func (h *Hooks) LoadFromFile(path string, asTemplate bool) error { + } + + // parse hook file for hooks +- file, e := ioutil.ReadFile(path) ++ file, e := os.ReadFile(path) + + if e != nil { + return e + } + + if asTemplate { +- funcMap := template.FuncMap{"getenv": getenv} ++ funcMap := template.FuncMap{ ++ "cat": cat, ++ "credential": credential, ++ "getenv": getenv, ++ } + + tmpl, err := template.New("hooks").Funcs(funcMap).Parse(string(file)) + if err != nil { +@@ -956,3 +960,27 @@ func compare(a, b string) bool { + func getenv(s string) string { + return os.Getenv(s) + } ++ ++// cat provides a template function to retrieve content of files ++// Similarly to getenv, if no file is found, it returns the empty string ++func cat(s string) string { ++ data, e := os.ReadFile(s) ++ ++ if e != nil { ++ return "" ++ } ++ ++ return strings.TrimSuffix(string(data), "\n") ++} ++ ++// credential provides a template function to retreive secrets using systemd's LoadCredential mechanism ++func credential(s string) string { ++ dir := getenv("CREDENTIALS_DIRECTORY") ++ ++ // If no credential directory is found, fallback to the env variable ++ if dir == "" { ++ return getenv(s) ++ } ++ ++ return cat(path.Join(dir, s)) ++} diff --git a/modules/django-apps/default.nix b/modules/django-apps/default.nix new file mode 100644 index 0000000..438e5ac --- /dev/null +++ b/modules/django-apps/default.nix @@ -0,0 +1,710 @@ +{ + config, + lib, + options, + pkgs, + utils, + ... +}: + +let + inherit (lib) + attrNames + concatLists + concatMapAttrs + filterAttrs + getExe + getExe' + literalExpression + mapAttrs + mapAttrs' + mapAttrsToList + mkEnableOption + mkIf + mkMerge + mkOption + mkPackageOption + nameValuePair + optional + optionals + recursiveUpdate + toUpper + ; + + inherit (lib.types) + attrs + attrsOf + enum + functionTo + ints + listOf + nullOr + package + path + str + submodule + ; + + inherit (utils) escapeSystemdExecArgs; + + cfg = config.services.django-apps; + + # Alias the global config to allow its use when the identifier is shadowed + config' = config; + systemctl = getExe' config.systemd.package "systemctl"; +in + +{ + options.services.django-apps = { + enable = mkEnableOption "automatic django apps management"; + + webhook = { + domain = mkOption { + type = str; + description = '' + The domain where the webhook service will listen. + ''; + }; + + nginx = mkOption { + type = nullOr options.services.nginx.virtualHosts.type.nestedTypes.elemType; + default = null; + description = '' + With this option, you can customize the nginx virtualHost settings. + ''; + example = literalExpression '' + { + # To enable encryption and let Let's Encrypt take care of certificate + forceSSL = true; + enableACME = true; + } + ''; + }; + }; + + sites = mkOption { + type = attrsOf ( + submodule ( + { name, config, ... }: + { + options = { + source = mkOption { + type = str; + description = '' + The URI where the source of the app can be publicly fetched via git. + ''; + }; + + branch = mkOption { + type = str; + default = "production"; + description = '' + Branch to follow for updates to the source. + ''; + }; + + domain = mkOption { + type = str; + description = '' + The domain where the web app will be served. + ''; + }; + + nginx = mkOption { + type = nullOr options.services.nginx.virtualHosts.type.nestedTypes.elemType; + default = null; + description = '' + With this option, you can customize the nginx virtualHost settings. + ''; + example = literalExpression '' + { + # To enable encryption and let Let's Encrypt take care of certificate + forceSSL = true; + enableACME = true; + } + ''; + }; + + env_prefix = mkOption { + type = str; + default = toUpper name; + description = '' + The prefix to use for environment settings declaration. + ''; + }; + + application = { + type = mkOption { + type = enum [ + "asgi" + "wsgi" + "daphne" + ]; + default = "wsgi"; + description = '' + Specification for the django application. + ''; + }; + + module = mkOption { + type = str; + default = "app"; + description = '' + Name of the module containing the application interface. + ''; + }; + + settingsModule = mkOption { + type = str; + default = "${config.application.module}.settings"; + description = '' + The django settings module, will be passed as an environment variable to the app. + ''; + }; + + workers = mkOption { + type = ints.positive; + default = 4; + description = '' + Number of workers processes to use. + ''; + }; + + channelLayer = mkOption { + type = str; + default = "channel_layer"; + description = '' + Channel layer to use when running the application with daphne. + ''; + }; + }; + + python = mkPackageOption pkgs "python3" { }; + + django = mkOption { + type = functionTo package; + default = ps: ps.django; + defaultText = literalExpression "ps: ps.django"; + description = '' + The django version to use to run the app. + ''; + }; + + djangoEnv = mkOption { + type = package; + default = config.python.withPackages ( + ps: + [ (config.django ps) ] + ++ (optional (config.application.type != "daphne") ps.gunicorn) + ++ (optional (config.application.type == "asgi") ps.uvicorn) + ++ (optional (config.dbType == "postgresql") ps.psycopg) + ++ (config.dependencies ps) + ); + description = '' + The python version used to run the app, with the correct dependencies. + ''; + }; + + dependencies = mkOption { + type = functionTo (listOf package); + default = _: [ ]; + example = literalExpression "ps: [ ps.requests ]"; + description = '' + Python dependencies of the app. + ''; + }; + + extraPackages = mkOption { + type = listOf package; + default = [ ]; + description = '' + Packages that will be added to the path of the app. + ''; + }; + + credentials = mkOption { + type = attrsOf path; + default = { }; + description = '' + The files containing credentials to pass through `LoadCredential` to the application. + ''; + }; + + environment = mkOption { + type = attrsOf (pkgs.formats.json { }).type; + default = { }; + description = '' + Environment variables to pass to the app. + ''; + }; + + managePath = mkOption { + type = str; + default = "manage.py"; + description = '' + Path to the manage.py file inside the source + ''; + }; + + extraServices = mkOption { + type = attrs; + default = { }; + description = '' + Extra services to run in parallel of the application. + May be used to run background tasks and/or workers. + ''; + }; + + manageScript = mkOption { + type = package; + default = pkgs.writeShellApplication { + name = "${name}-manage"; + + runtimeInputs = [ + pkgs.util-linux + config'.systemd.package + + config.djangoEnv + ] ++ config.extraPackages; + text = '' + MainPID=$(systemctl show -p MainPID --value dj-${name}.service) + + nsenter -e -a -t "$MainPID" -G follow -S follow python /var/lib/django-apps/${name}/source/${config.managePath} "$@" + ''; + }; + description = '' + Script to run manage.py related tasks. + ''; + }; + + updateScript = mkOption { + type = package; + default = pkgs.writeShellApplication { + name = "dj-${name}-update-source"; + + runtimeInputs = [ + config.djangoEnv + + pkgs.git + ]; + + text = '' + git pull + python3 ${config.managePath} migrate + python3 ${config.managePath} collectstatic --no-input + ''; + }; + description = '' + Script to run when updating the app source. + ''; + }; + + webHookSecret = mkOption { + type = path; + description = '' + Path to the webhook secret. + ''; + }; + + dbType = mkOption { + type = enum [ + "manual" + "postgresql" + "sqlite" + ]; + default = "postgresql"; + description = '' + Which database backend to use, set to `manual` for custom declaration. + ''; + }; + + baseDirectory = mkOption { + type = str; + readOnly = true; + default = "/var/lib/django-apps/${name}"; + }; + + sourceDirectory = mkOption { + type = str; + readOnly = true; + default = "${config.baseDirectory}/source"; + }; + + staticDirectory = mkOption { + type = str; + default = "static"; + description = '' + Path to the staticfiles directory. + This is relative to the base directory, e.g. the parent of the source directory. + ''; + }; + + mediaDirectory = mkOption { + type = str; + default = "media"; + description = '' + Path to the media files directory. + This is relative to the base directory, e.g. the parent of the source directory. + ''; + }; + }; + } + ) + ); + }; + }; + + config = mkIf cfg.enable { + security.sudo.extraRules = [ + { + users = [ "webhook" ]; + commands = builtins.map (name: { + command = "${systemctl} start dj-${name}-update.service"; + options = [ "NOPASSWD" ]; + }) (attrNames cfg.sites); + } + ]; + + environment.systemPackages = mapAttrsToList (_: { manageScript, ... }: manageScript) cfg.sites; + + services = { + webhook = { + enable = true; + + package = pkgs.webhook.overrideAttrs (old: { + patches = (old.patches or [ ]) ++ [ ./01-webhook.patch ]; + }); + + # extraArgs = [ "-debug" ]; + + # Only listen on localhost + ip = "127.0.0.1"; + + hooksTemplated = mapAttrs' ( + name: + { branch, ... }: + nameValuePair "dj-${name}" ( + # Avoid issues when quoting "dj-name" through builtins.toJSON + builtins.replaceStrings [ "\\" ] [ "" ] ( + builtins.toJSON { + id = "dj-${name}"; + execute-command = "/run/wrappers/bin/sudo"; + pass-arguments-to-command = + builtins.map + (name: { + inherit name; + source = "string"; + }) + [ + systemctl + "start" + "dj-${name}-update.service" + ]; + # command-working-directory = "/var/lib/django-apps/${name}"; + trigger-rule = { + and = [ + { + or = [ + { + match = { + type = "payload-hmac-sha256"; + secret = ''{{ credential "dj-${name}" | js }}''; + parameter = { + source = "header"; + name = "X-Hub-Signature-256"; + }; + }; + } + { + match = { + type = "value"; + value = ''{{ credential "dj-${name}" | js }}''; + parameter = { + source = "header"; + name = "X-Gitlab-Token"; + }; + }; + } + ]; + } + { + match = { + type = "value"; + value = "refs/heads/${branch}"; + parameter = { + source = "payload"; + name = "ref"; + }; + }; + } + ]; + }; + } + ) + ) + ) cfg.sites; + }; + + nginx = mkMerge [ + (mkIf (cfg.webhook.nginx != null) { + enable = true; + + virtualHosts = { + ${cfg.webhook.domain} = mkMerge [ + { locations."/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.webhook.port}"; } + cfg.webhook.nginx + ]; + }; + }) + { + virtualHosts = mapAttrs' ( + name: + { domain, nginx, ... }: + nameValuePair domain ( + recursiveUpdate { + locations = { + "/".proxyPass = "http://unix:/run/django-apps/${name}.sock"; + "/static/".root = "/run/django-apps/${name}"; + "/media/".root = "/run/django-apps/${name}"; + }; + } nginx + ) + ) cfg.sites; + } + ]; + + postgresql = + let + apps = builtins.map (name: "dj-${name}") ( + attrNames (filterAttrs (_: { dbType, ... }: dbType == "postgresql") cfg.sites) + ); + in + mkIf (apps != [ ]) { + enable = true; + + ensureDatabases = apps; + ensureUsers = builtins.map (name: { + inherit name; + ensureDBOwnership = true; + }) apps; + }; + }; + + users = { + users.nginx.extraGroups = [ "django-apps" ]; + groups.django-apps = { }; + }; + + systemd = { + sockets = mapAttrs' ( + name: _: + nameValuePair "dj-${name}" { + description = "Socket for the ${name} Django Application"; + wantedBy = [ "sockets.target" ]; + + socketConfig = { + ListenStream = "/run/django-apps/${name}.sock"; + SocketMode = "600"; + SocketUser = config'.services.nginx.user; + }; + } + ) cfg.sites; + + mounts = concatLists ( + mapAttrsToList ( + name: + { mediaDirectory, staticDirectory, ... }: + [ + { + where = "/run/django-apps/${name}/static"; + what = "/var/lib/django-apps/${name}/${staticDirectory}"; + options = "bind"; + + after = [ "dj-${name}.service" ]; + partOf = [ "dj-${name}.service" ]; + upheldBy = [ "dj-${name}.service" ]; + } + + { + where = "/run/django-apps/${name}/media"; + what = "/var/lib/django-apps/${name}/${mediaDirectory}"; + options = "bind"; + + after = [ "dj-${name}.service" ]; + partOf = [ "dj-${name}.service" ]; + upheldBy = [ "dj-${name}.service" ]; + } + ] + ) cfg.sites + ); + + services = + { + webhook.serviceConfig.LoadCredential = mapAttrsToList ( + name: { webHookSecret, ... }: "dj-${name}:${webHookSecret}" + ) cfg.sites; + } + // (concatMapAttrs ( + name: config: + let + mkDatabase = + name: type: + if type == "postgresql" then + { + ENGINE = "django.db.backends.postgresql"; + NAME = "dj-${name}"; + } + else if type == "sqlite" then + { + ENGINE = "django.db.backends.sqlite3"; + NAME = "/var/lib/django-apps/${name}/db.sqlite3"; + } + else + throw "Invalid database type !"; + + # Systemd Service Configuration + Group = "django-apps"; + LoadCredential = mapAttrsToList (credential: path: "${credential}:${path}") config.credentials; + RuntimeDirectory = "django-apps/${name}"; + StateDirectory = "django-apps/${name}"; + UMask = "0027"; + User = "dj-${name}"; + WorkingDirectory = "/var/lib/django-apps/${name}"; + + environment = + let + mkValue = v: if builtins.isString v then v else builtins.toJSON v; + in + (mapAttrs' (key: value: nameValuePair "${config.env_prefix}_${key}" (mkValue value)) { + DATABASES = + if (config.dbType != "manual") then { default = mkDatabase name config.dbType; } else null; + STATIC_ROOT = "/var/lib/django-apps/${name}/${config.staticDirectory}"; + MEDIA_ROOT = "/var/lib/django-apps/${name}/${config.mediaDirectory}"; + ALLOWED_HOSTS = [ config.domain ]; + }) + // { + DJANGO_SETTINGS_MODULE = config.application.settingsModule; + } + // (mapAttrs (_: mkValue) config.environment); + path = config.extraPackages ++ [ config.djangoEnv ]; + after = [ "network.target" ] ++ (optional (config.dbType == "postgresql") "postgresql.service"); + in + { + "dj-${name}" = { + inherit after environment path; + + preStart = '' + if [ ! -f .initialized ]; then + # The previous initialization might have failed, so restart from the beginning + rm -rf source + + # We need to download the application source and run the migrations first + ${lib.getExe pkgs.git} clone --single-branch --branch ${config.branch} ${config.source} source + (cd source && python ${config.managePath} migrate --no-input && python ${config.managePath} collectstatic --no-input) + touch .initialized + fi + + # Create the necessary directory with the correct user/group + mkdir -p ${config.mediaDirectory} ${config.staticDirectory} + ''; + + requires = [ "dj-${name}.socket" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + inherit + Group + LoadCredential + RuntimeDirectory + StateDirectory + User + UMask + WorkingDirectory + ; + + DynamicUser = true; + ExecStart = escapeSystemdExecArgs ( + if (config.application.type == "daphne") then + [ + (getExe' config.djangoEnv "daphne") + "-u" + "/run/django-apps/${name}.sock" + "${config.application.module}.asgi:${config.application.channelLayer}" + ] + else + ( + [ + (getExe' config.djangoEnv "gunicorn") + "--workers" + config.application.workers + "--bind" + "unix:/run/django-apps/${name}.sock" + "--pythonpath" + "source" + ] + ++ (optionals (config.application.type == "asgi") [ + "--worker-class" + "uvicorn.workers.UvicornWorker" + ]) + ++ [ "${config.application.module}.${config.application.type}" ] + ) + ); + ExecReload = "${getExe' pkgs.coreutils "kill"} -s HUP $MAINPID"; + KillMode = "mixed"; + Type = mkIf (config.application.type != "daphne") "notify"; + }; + }; + + "dj-${name}-update" = { + inherit environment path; + + serviceConfig = { + inherit + Group + LoadCredential + StateDirectory + UMask + User + ; + + DynamicUser = true; + ExecStart = "${getExe config.updateScript}"; + Type = "oneshot"; + WorkingDirectory = "/var/lib/django-apps/${name}/source"; + }; + + unitConfig = { + After = "dj-${name}.service"; + Conflicts = "dj-${name}.service"; + }; + }; + } + // (mapAttrs' ( + serviceName: serviceContent: + nameValuePair "dj-${name}_${serviceName}" ( + recursiveUpdate { + inherit after environment path; + + partOf = [ "dj-${name}.service" ]; + wantedBy = [ "multi-user.target" ]; + upheldBy = [ "dj-${name}.service" ]; + + serviceConfig = { + inherit + Group + LoadCredential + RuntimeDirectory + StateDirectory + UMask + User + ; + + DynamicUser = true; + }; + } serviceContent + ) + ) config.extraServices) + ) cfg.sites); + }; + }; +}