cst1/infrastructure
1
0
Fork 0
forked from DGNum/infrastructure
Code Pull requests Activity

feat(django-apps): Internalize

Browse source
This commit is contained in:
Tom Hubrecht 2024-11-25 23:16:33 +01:00
parent aa154d1b1b
commit 96e8bfff5b
Signed by: thubrecht
SSH key fingerprint: SHA256:r+nK/SIcWlJ0zFZJGHtlAoRwq1Rm+WcKAm5ADYMoQPc
4 changed files with 778 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
meta/nodes.nix
View file

@ -132,8 +132,6 @@
hashedPassword = "$y$j9T$Un/tcX5SPKNXG.sy/BcTa.$kyNHELjb1GAOWnauJfcjyVi5tacWcuEBKflZDCUC6x4";
nix-modules = [ "services/django-apps" ];
stateVersion = "24.05";
nixpkgs = "unstable";
vm-cluster = "Hyperviseur NPS";

1
modules/default.nix
View file

@ -58,6 +58,7 @@
"dgn-ssh"
"dgn-vm-variant"
"dgn-web"
"django-apps"
])
++ [
"${sources.agenix}/modules/age.nix"

67
modules/django-apps/01-webhook.patch Normal file
View file

@ -0,0 +1,67 @@
diff --git a/internal/hook/hook.go b/internal/hook/hook.go
index 0510095..0347f26 100644
--- a/internal/hook/hook.go
+++ b/internal/hook/hook.go
@@ -13,12 +13,12 @@ import (
"errors"
"fmt"
"hash"
- "io/ioutil"
"log"
"math"
"net"
"net/textproto"
"os"
+ "path"
"reflect"
"regexp"
"strconv"
@@ -750,14 +750,18 @@ func (h *Hooks) LoadFromFile(path string, asTemplate bool) error {
}
// parse hook file for hooks
- file, e := ioutil.ReadFile(path)
+ file, e := os.ReadFile(path)
if e != nil {
return e
}
if asTemplate {
- funcMap := template.FuncMap{"getenv": getenv}
+ funcMap := template.FuncMap{
+ "cat": cat,
+ "credential": credential,
+ "getenv": getenv,
+ }
tmpl, err := template.New("hooks").Funcs(funcMap).Parse(string(file))
if err != nil {
@@ -956,3 +960,27 @@ func compare(a, b string) bool {
func getenv(s string) string {
return os.Getenv(s)
}
+
+// cat provides a template function to retrieve content of files
+// Similarly to getenv, if no file is found, it returns the empty string
+func cat(s string) string {
+ data, e := os.ReadFile(s)
+
+ if e != nil {
+ return ""
+ }
+
+ return strings.TrimSuffix(string(data), "\n")
+}
+
+// credential provides a template function to retreive secrets using systemd's LoadCredential mechanism
+func credential(s string) string {
+ dir := getenv("CREDENTIALS_DIRECTORY")
+
+ // If no credential directory is found, fallback to the env variable
+ if dir == "" {
+ return getenv(s)
+ }
+
+ return cat(path.Join(dir, s))
+}

710
modules/django-apps/default.nix Normal file
View file

@ -0,0 +1,710 @@
{
config,
lib,
options,
pkgs,
utils,
...
}:
let
inherit (lib)
attrNames
concatLists
concatMapAttrs
filterAttrs
getExe
getExe'
literalExpression
mapAttrs
mapAttrs'
mapAttrsToList
mkEnableOption
mkIf
mkMerge
mkOption
mkPackageOption
nameValuePair
optional
optionals
recursiveUpdate
toUpper
;
inherit (lib.types)
attrs
attrsOf
enum
functionTo
ints
listOf
nullOr
package
path
str
submodule
;
inherit (utils) escapeSystemdExecArgs;
cfg = config.services.django-apps;
# Alias the global config to allow its use when the identifier is shadowed
config' = config;
systemctl = getExe' config.systemd.package "systemctl";
in
{
options.services.django-apps = {
enable = mkEnableOption "automatic django apps management";
webhook = {
domain = mkOption {
type = str;
description = ''
The domain where the webhook service will listen.
'';
};
nginx = mkOption {
type = nullOr options.services.nginx.virtualHosts.type.nestedTypes.elemType;
default = null;
description = ''
With this option, you can customize the nginx virtualHost settings.
'';
example = literalExpression ''
{
# To enable encryption and let Let's Encrypt take care of certificate
forceSSL = true;
enableACME = true;
}
'';
};
};
sites = mkOption {
type = attrsOf (
submodule (
{ name, config, ... }:
{
options = {
source = mkOption {
type = str;
description = ''
The URI where the source of the app can be publicly fetched via git.
'';
};
branch = mkOption {
type = str;
default = "production";
description = ''
Branch to follow for updates to the source.
'';
};
domain = mkOption {
type = str;
description = ''
The domain where the web app will be served.
'';
};
nginx = mkOption {
type = nullOr options.services.nginx.virtualHosts.type.nestedTypes.elemType;
default = null;
description = ''
With this option, you can customize the nginx virtualHost settings.
'';
example = literalExpression ''
{
# To enable encryption and let Let's Encrypt take care of certificate
forceSSL = true;
enableACME = true;
}
'';
};
env_prefix = mkOption {
type = str;
default = toUpper name;
description = ''
The prefix to use for environment settings declaration.
'';
};
application = {
type = mkOption {
type = enum [
"asgi"
"wsgi"
"daphne"
];
default = "wsgi";
description = ''
Specification for the django application.
'';
};
module = mkOption {
type = str;
default = "app";
description = ''
Name of the module containing the application interface.
'';
};
settingsModule = mkOption {
type = str;
default = "${config.application.module}.settings";
description = ''
The django settings module, will be passed as an environment variable to the app.
'';
};
workers = mkOption {
type = ints.positive;
default = 4;
description = ''
Number of workers processes to use.
'';
};
channelLayer = mkOption {
type = str;
default = "channel_layer";
description = ''
Channel layer to use when running the application with daphne.
'';
};
};
python = mkPackageOption pkgs "python3" { };
django = mkOption {
type = functionTo package;
default = ps: ps.django;
defaultText = literalExpression "ps: ps.django";
description = ''
The django version to use to run the app.
'';
};
djangoEnv = mkOption {
type = package;
default = config.python.withPackages (
ps:
[ (config.django ps) ]
++ (optional (config.application.type != "daphne") ps.gunicorn)
++ (optional (config.application.type == "asgi") ps.uvicorn)
++ (optional (config.dbType == "postgresql") ps.psycopg)
++ (config.dependencies ps)
);
description = ''
The python version used to run the app, with the correct dependencies.
'';
};
dependencies = mkOption {
type = functionTo (listOf package);
default = _: [ ];
example = literalExpression "ps: [ ps.requests ]";
description = ''
Python dependencies of the app.
'';
};
extraPackages = mkOption {
type = listOf package;
default = [ ];
description = ''
Packages that will be added to the path of the app.
'';
};
credentials = mkOption {
type = attrsOf path;
default = { };
description = ''
The files containing credentials to pass through `LoadCredential` to the application.
'';
};
environment = mkOption {
type = attrsOf (pkgs.formats.json { }).type;
default = { };
description = ''
Environment variables to pass to the app.
'';
};
managePath = mkOption {
type = str;
default = "manage.py";
description = ''
Path to the manage.py file inside the source
'';
};
extraServices = mkOption {
type = attrs;
default = { };
description = ''
Extra services to run in parallel of the application.
May be used to run background tasks and/or workers.
'';
};
manageScript = mkOption {
type = package;
default = pkgs.writeShellApplication {
name = "${name}-manage";
runtimeInputs = [
pkgs.util-linux
config'.systemd.package
config.djangoEnv
] ++ config.extraPackages;
text = ''
MainPID=$(systemctl show -p MainPID --value dj-${name}.service)
nsenter -e -a -t "$MainPID" -G follow -S follow python /var/lib/django-apps/${name}/source/${config.managePath} "$@"
'';
};
description = ''
Script to run manage.py related tasks.
'';
};
updateScript = mkOption {
type = package;
default = pkgs.writeShellApplication {
name = "dj-${name}-update-source";
runtimeInputs = [
config.djangoEnv
pkgs.git
];
text = ''
git pull
python3 ${config.managePath} migrate
python3 ${config.managePath} collectstatic --no-input
'';
};
description = ''
Script to run when updating the app source.
'';
};
webHookSecret = mkOption {
type = path;
description = ''
Path to the webhook secret.
'';
};
dbType = mkOption {
type = enum [
"manual"
"postgresql"
"sqlite"
];
default = "postgresql";
description = ''
Which database backend to use, set to `manual` for custom declaration.
'';
};
baseDirectory = mkOption {
type = str;
readOnly = true;
default = "/var/lib/django-apps/${name}";
};
sourceDirectory = mkOption {
type = str;
readOnly = true;
default = "${config.baseDirectory}/source";
};
staticDirectory = mkOption {
type = str;
default = "static";
description = ''
Path to the staticfiles directory.
This is relative to the base directory, e.g. the parent of the source directory.
'';
};
mediaDirectory = mkOption {
type = str;
default = "media";
description = ''
Path to the media files directory.
This is relative to the base directory, e.g. the parent of the source directory.
'';
};
};
}
)
);
};
};
config = mkIf cfg.enable {
security.sudo.extraRules = [
{
users = [ "webhook" ];
commands = builtins.map (name: {
command = "${systemctl} start dj-${name}-update.service";
options = [ "NOPASSWD" ];
}) (attrNames cfg.sites);
}
];
environment.systemPackages = mapAttrsToList (_: { manageScript, ... }: manageScript) cfg.sites;
services = {
webhook = {
enable = true;
package = pkgs.webhook.overrideAttrs (old: {
patches = (old.patches or [ ]) ++ [ ./01-webhook.patch ];
});
# extraArgs = [ "-debug" ];
# Only listen on localhost
ip = "127.0.0.1";
hooksTemplated = mapAttrs' (
name:
{ branch, ... }:
nameValuePair "dj-${name}" (
# Avoid issues when quoting "dj-name" through builtins.toJSON
builtins.replaceStrings [ "\\" ] [ "" ] (
builtins.toJSON {
id = "dj-${name}";
execute-command = "/run/wrappers/bin/sudo";
pass-arguments-to-command =
builtins.map
(name: {
inherit name;
source = "string";
})
[
systemctl
"start"
"dj-${name}-update.service"
];
# command-working-directory = "/var/lib/django-apps/${name}";
trigger-rule = {
and = [
{
or = [
{
match = {
type = "payload-hmac-sha256";
secret = ''{{ credential "dj-${name}" | js }}'';
parameter = {
source = "header";
name = "X-Hub-Signature-256";
};
};
}
{
match = {
type = "value";
value = ''{{ credential "dj-${name}" | js }}'';
parameter = {
source = "header";
name = "X-Gitlab-Token";
};
};
}
];
}
{
match = {
type = "value";
value = "refs/heads/${branch}";
parameter = {
source = "payload";
name = "ref";
};
};
}
];
};
}
)
)
) cfg.sites;
};
nginx = mkMerge [
(mkIf (cfg.webhook.nginx != null) {
enable = true;
virtualHosts = {
${cfg.webhook.domain} = mkMerge [
{ locations."/".proxyPass = "http://127.0.0.1:${builtins.toString config.services.webhook.port}"; }
cfg.webhook.nginx
];
};
})
{
virtualHosts = mapAttrs' (
name:
{ domain, nginx, ... }:
nameValuePair domain (
recursiveUpdate {
locations = {
"/".proxyPass = "http://unix:/run/django-apps/${name}.sock";
"/static/".root = "/run/django-apps/${name}";
"/media/".root = "/run/django-apps/${name}";
};
} nginx
)
) cfg.sites;
}
];
postgresql =
let
apps = builtins.map (name: "dj-${name}") (
attrNames (filterAttrs (_: { dbType, ... }: dbType == "postgresql") cfg.sites)
);
in
mkIf (apps != [ ]) {
enable = true;
ensureDatabases = apps;
ensureUsers = builtins.map (name: {
inherit name;
ensureDBOwnership = true;
}) apps;
};
};
users = {
users.nginx.extraGroups = [ "django-apps" ];
groups.django-apps = { };
};
systemd = {
sockets = mapAttrs' (
name: _:
nameValuePair "dj-${name}" {
description = "Socket for the ${name} Django Application";
wantedBy = [ "sockets.target" ];
socketConfig = {
ListenStream = "/run/django-apps/${name}.sock";
SocketMode = "600";
SocketUser = config'.services.nginx.user;
};
}
) cfg.sites;
mounts = concatLists (
mapAttrsToList (
name:
{ mediaDirectory, staticDirectory, ... }:
[
{
where = "/run/django-apps/${name}/static";
what = "/var/lib/django-apps/${name}/${staticDirectory}";
options = "bind";
after = [ "dj-${name}.service" ];
partOf = [ "dj-${name}.service" ];
upheldBy = [ "dj-${name}.service" ];
}
{
where = "/run/django-apps/${name}/media";
what = "/var/lib/django-apps/${name}/${mediaDirectory}";
options = "bind";
after = [ "dj-${name}.service" ];
partOf = [ "dj-${name}.service" ];
upheldBy = [ "dj-${name}.service" ];
}
]
) cfg.sites
);
services =
{
webhook.serviceConfig.LoadCredential = mapAttrsToList (
name: { webHookSecret, ... }: "dj-${name}:${webHookSecret}"
) cfg.sites;
}
// (concatMapAttrs (
name: config:
let
mkDatabase =
name: type:
if type == "postgresql" then
{
ENGINE = "django.db.backends.postgresql";
NAME = "dj-${name}";
}
else if type == "sqlite" then
{
ENGINE = "django.db.backends.sqlite3";
NAME = "/var/lib/django-apps/${name}/db.sqlite3";
}
else
throw "Invalid database type !";
# Systemd Service Configuration
Group = "django-apps";
LoadCredential = mapAttrsToList (credential: path: "${credential}:${path}") config.credentials;
RuntimeDirectory = "django-apps/${name}";
StateDirectory = "django-apps/${name}";
UMask = "0027";
User = "dj-${name}";
WorkingDirectory = "/var/lib/django-apps/${name}";
environment =
let
mkValue = v: if builtins.isString v then v else builtins.toJSON v;
in
(mapAttrs' (key: value: nameValuePair "${config.env_prefix}_${key}" (mkValue value)) {
DATABASES =
if (config.dbType != "manual") then { default = mkDatabase name config.dbType; } else null;
STATIC_ROOT = "/var/lib/django-apps/${name}/${config.staticDirectory}";
MEDIA_ROOT = "/var/lib/django-apps/${name}/${config.mediaDirectory}";
ALLOWED_HOSTS = [ config.domain ];
})
// {
DJANGO_SETTINGS_MODULE = config.application.settingsModule;
}
// (mapAttrs (_: mkValue) config.environment);
path = config.extraPackages ++ [ config.djangoEnv ];
after = [ "network.target" ] ++ (optional (config.dbType == "postgresql") "postgresql.service");
in
{
"dj-${name}" = {
inherit after environment path;
preStart = ''
if [ ! -f .initialized ]; then
# The previous initialization might have failed, so restart from the beginning
rm -rf source
# We need to download the application source and run the migrations first
${lib.getExe pkgs.git} clone --single-branch --branch ${config.branch} ${config.source} source
(cd source && python ${config.managePath} migrate --no-input && python ${config.managePath} collectstatic --no-input)
touch .initialized
fi
# Create the necessary directory with the correct user/group
mkdir -p ${config.mediaDirectory} ${config.staticDirectory}
'';
requires = [ "dj-${name}.socket" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
inherit
Group
LoadCredential
RuntimeDirectory
StateDirectory
User
UMask
WorkingDirectory
;
DynamicUser = true;
ExecStart = escapeSystemdExecArgs (
if (config.application.type == "daphne") then
[
(getExe' config.djangoEnv "daphne")
"-u"
"/run/django-apps/${name}.sock"
"${config.application.module}.asgi:${config.application.channelLayer}"
]
else
(
[
(getExe' config.djangoEnv "gunicorn")
"--workers"
config.application.workers
"--bind"
"unix:/run/django-apps/${name}.sock"
"--pythonpath"
"source"
]
++ (optionals (config.application.type == "asgi") [
"--worker-class"
"uvicorn.workers.UvicornWorker"
])
++ [ "${config.application.module}.${config.application.type}" ]
)
);
ExecReload = "${getExe' pkgs.coreutils "kill"} -s HUP $MAINPID";
KillMode = "mixed";
Type = mkIf (config.application.type != "daphne") "notify";
};
};
"dj-${name}-update" = {
inherit environment path;
serviceConfig = {
inherit
Group
LoadCredential
StateDirectory
UMask
User
;
DynamicUser = true;
ExecStart = "${getExe config.updateScript}";
Type = "oneshot";
WorkingDirectory = "/var/lib/django-apps/${name}/source";
};
unitConfig = {
After = "dj-${name}.service";
Conflicts = "dj-${name}.service";
};
};
}
// (mapAttrs' (
serviceName: serviceContent:
nameValuePair "dj-${name}_${serviceName}" (
recursiveUpdate {
inherit after environment path;
partOf = [ "dj-${name}.service" ];
wantedBy = [ "multi-user.target" ];
upheldBy = [ "dj-${name}.service" ];
serviceConfig = {
inherit
Group
LoadCredential
RuntimeDirectory
StateDirectory
UMask
User
;
DynamicUser = true;
};
} serviceContent
)
) config.extraServices)
) cfg.sites);
};
};
}