From 5b5c57bb324aad4b990fd1fe6fe443556673731a Mon Sep 17 00:00:00 2001
From: sinavir <sinavir@sinavir.fr>
Date: Sun, 9 Jun 2024 14:39:18 +0200
Subject: [PATCH] init

---
 .gitignore                 |   5 +
 module.nix                 | 211 +++++++++++++++++++++++++++++++++++++
 npins/default.nix          |  80 ++++++++++++++
 npins/sources.json         |  11 ++
 test.nix                   |  86 +++++++++++++++
 test_assets/secret         |   1 +
 test_assets/urls.py        |   9 ++
 test_assets/views.py       |  16 +++
 utils/mkManagePy.nix       |  20 ++++
 utils/mkSettingsModule.nix |  70 ++++++++++++
 10 files changed, 509 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 module.nix
 create mode 100644 npins/default.nix
 create mode 100644 npins/sources.json
 create mode 100644 test.nix
 create mode 100644 test_assets/secret
 create mode 100644 test_assets/urls.py
 create mode 100644 test_assets/views.py
 create mode 100644 utils/mkManagePy.nix
 create mode 100644 utils/mkSettingsModule.nix

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..463db19
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+.nixos-test-history
+result
+result-*
diff --git a/module.nix b/module.nix
new file mode 100644
index 0000000..a5d18b0
--- /dev/null
+++ b/module.nix
@@ -0,0 +1,211 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  mkManagePy = pkgs.callPackage ./utils/mkManagePy.nix { };
+  mkStaticAssets =
+    { app, managePy }:
+    pkgs.runCommand "django-${app}-static" {} ''
+      mkdir -p "$out/static"
+      STATIC_ROOT="\"$out/static\"" \
+      DJANGO_SETTINGS_MODULE="${app}_settings.mock" \
+      ${lib.getExe managePy} collectstatic --noinput
+    '';
+
+  mkSettingsModule =
+    {
+      runtimeSettings,
+      settings,
+      secrets,
+      mainModule,
+      extraConfig,
+      p,
+    }:
+    p.callPackage ./utils/mkSettingsModule.nix {
+      inherit
+        runtimeSettings
+        settings
+        secrets
+        mainModule
+        extraConfig
+        ;
+    };
+
+  djangoAppModule = lib.types.submodule (
+    { config, name, ... }:
+    {
+      options = {
+        enable = lib.mkEnableOption (lib.mdDoc "Enable django application") // {
+          default = true;
+        };
+        src = lib.mkOption { type = lib.types.path; };
+        sourceRoot = lib.mkOption {
+          type = lib.types.str;
+          default = "";
+          example = "myproject/";
+          description = "Directory in the source directory to find the django app (must point to basedir of manage.py)";
+        };
+        mainModule = lib.mkOption {
+          type = lib.types.str;
+          description = "Name of the django application";
+          default = name;
+          defaultText = lib.literalMD "Defaults to attribute name in djangoAppModule.";
+        };
+        settings = lib.mkOption {
+          type = lib.types.submodule {
+            freeformType = with lib.types; attrsOf anything;
+            options = { };
+          };
+          description = ''
+            Settings to pass to django.
+          '';
+          example = {
+            DATABASES = {
+              "default" = {
+                "ENGINE" = "django.db.backends.sqlite3";
+                "NAME" = "/var/lib/django-myproject/db.sqlite3";
+              };
+            };
+          };
+        };
+        runtimeSettings = lib.mkOption {
+          type = with lib.types; attrsOf str;
+          default= {};
+          description = ''
+            Settings to pass to only at runtime.
+
+            Useful for settings that depends on the python package.
+          '';
+        };
+        secrets = lib.mkOption {
+          type = lib.types.attrsOf lib.types.path;
+          default = { };
+          example = {
+            SECRET_KEY = "/etc/django-secret.json";
+            DATABASES = "/etc/django-db-config.json";
+          };
+          description = ''
+            Secrets to pass to django through a file. The content of the file will be loaded as json
+          '';
+        };
+        extraConfig = lib.mkOption {
+          type = lib.types.lines;
+          default = "";
+          description = "Extra python code to append at the end of the production settings module.";
+        };
+        port = lib.mkOption {
+          type = lib.types.port;
+          default = 51666;
+          description = "Port for the gunicorn process";
+        };
+        processes = lib.mkOption {
+          type = lib.types.int;
+          default = 2;
+        };
+        threads = lib.mkOption {
+          type = lib.types.int;
+          default = 2;
+        };
+        staticAssets = lib.mkOption {
+          type = lib.types.path;
+          default = mkStaticAssets {
+            inherit (config) managePy;
+            app = name;
+          };
+          description = "Satic assets to be served directly by nginx. The default value should be good enough in most cases.";
+        };
+        manageFilePath = lib.mkOption {
+          type = lib.types.str;
+          default = "${config.sourceRoot}manage.py";
+          description = "Path relative to src pointing to manage.py file";
+        };
+        pythonPackage = lib.mkOption {
+          internal = true;
+          type = lib.types.path;
+          description = "Final python environment containing everything including the settings module";
+          default = pkgs.python3.withPackages (
+            p:
+            [
+              p.django
+              p.gunicorn
+              (mkSettingsModule {
+                inherit (config)
+                  secrets
+                  settings
+                  mainModule
+                  extraConfig
+                  ;
+                runtimeSettings = builtins.attrNames config.runtimeSettings;
+                inherit p;
+              })
+            ]
+            ++ config.extraPackages p
+          );
+        };
+        extraPackages = lib.mkOption {
+          type = lib.types.functionTo (lib.types.listOf lib.types.package);
+          default = p: [ ];
+          defaultText = lib.literalExpression "p: []";
+          description = ''
+            Extra Python packages available to django app. The
+            value must be a function which receives the attrset defined
+            in {var}`python3Packages` as the sole argument.
+          '';
+        };
+        managePy = lib.mkOption {
+          internal = true;
+          type = lib.types.package;
+          default = mkManagePy {
+            inherit (config) pythonPackage manageFilePath src;
+            app = name;
+          };
+          description = "Manage py script";
+        };
+      };
+      config = {
+        runtimeSettings.STATIC_ROOT = "";#config.staticAssets;
+      };
+    }
+  );
+in
+{
+  options = {
+    services.django = lib.mkOption {
+      type = lib.types.attrsOf djangoAppModule;
+      description = "Attribute set of djanfo app modules";
+    };
+  };
+  config.systemd.services = lib.mapAttrs' (
+    app: cfg:
+    lib.nameValuePair "django-${app}" (
+      lib.mkIf cfg.enable {
+        description = "${app} django service";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        wants = [ "network.target" ];
+        serviceConfig = rec {
+          Type =  "notify";
+          #NotifyAllow = "exec";
+          DynamicUser = true;
+
+          LoadCredential = lib.mapAttrsToList (k: v: "${k}:${v}") cfg.secrets;
+          StateDirectory = "django-${app}";
+        };
+        environment = {
+          DJANGO_SETTINGS_MODULE = "${cfg.mainModule}_settings.prod";
+        } // (lib.mapAttrs (_: v: builtins.toJSON v) cfg.runtimeSettings);
+        script = ''
+          ${lib.getExe cfg.managePy} migrate
+          exec ${cfg.pythonPackage}/bin/gunicorn ${cfg.mainModule}.wsgi \
+              --pythonpath ${cfg.src}/${cfg.sourceRoot} \
+              -b 127.0.0.1:${builtins.toString cfg.port} \
+              --workers=${builtins.toString cfg.processes} \
+              --threads=${builtins.toString cfg.threads}
+        '';
+      }
+    )
+  ) config.services.django;
+}
diff --git a/npins/default.nix b/npins/default.nix
new file mode 100644
index 0000000..5e7d086
--- /dev/null
+++ b/npins/default.nix
@@ -0,0 +1,80 @@
+# Generated by npins. Do not modify; will be overwritten regularly
+let
+  data = builtins.fromJSON (builtins.readFile ./sources.json);
+  version = data.version;
+
+  mkSource =
+    spec:
+    assert spec ? type;
+    let
+      path =
+        if spec.type == "Git" then
+          mkGitSource spec
+        else if spec.type == "GitRelease" then
+          mkGitSource spec
+        else if spec.type == "PyPi" then
+          mkPyPiSource spec
+        else if spec.type == "Channel" then
+          mkChannelSource spec
+        else
+          builtins.throw "Unknown source type ${spec.type}";
+    in
+    spec // { outPath = path; };
+
+  mkGitSource =
+    {
+      repository,
+      revision,
+      url ? null,
+      hash,
+      branch ? null,
+      ...
+    }:
+    assert repository ? type;
+    # At the moment, either it is a plain git repository (which has an url), or it is a GitHub/GitLab repository
+    # In the latter case, there we will always be an url to the tarball
+    if url != null then
+      (builtins.fetchTarball {
+        inherit url;
+        sha256 = hash; # FIXME: check nix version & use SRI hashes
+      })
+    else
+      assert repository.type == "Git";
+      let
+        urlToName =
+          url: rev:
+          let
+            matched = builtins.match "^.*/([^/]*)(\\.git)?$" repository.url;
+
+            short = builtins.substring 0 7 rev;
+
+            appendShort = if (builtins.match "[a-f0-9]*" rev) != null then "-${short}" else "";
+          in
+          "${if matched == null then "source" else builtins.head matched}${appendShort}";
+        name = urlToName repository.url revision;
+      in
+      builtins.fetchGit {
+        url = repository.url;
+        rev = revision;
+        inherit name;
+        # hash = hash;
+      };
+
+  mkPyPiSource =
+    { url, hash, ... }:
+    builtins.fetchurl {
+      inherit url;
+      sha256 = hash;
+    };
+
+  mkChannelSource =
+    { url, hash, ... }:
+    builtins.fetchTarball {
+      inherit url;
+      sha256 = hash;
+    };
+in
+if version == 3 then
+  builtins.mapAttrs (_: mkSource) data.pins
+else
+  throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`"
diff --git a/npins/sources.json b/npins/sources.json
new file mode 100644
index 0000000..e7774c0
--- /dev/null
+++ b/npins/sources.json
@@ -0,0 +1,11 @@
+{
+  "pins": {
+    "nixpkgs": {
+      "type": "Channel",
+      "name": "nixpkgs-unstable",
+      "url": "https://releases.nixos.org/nixpkgs/nixpkgs-24.11pre635739.31f409910124/nixexprs.tar.xz",
+      "hash": "03k8b8yawqcbz75n0fv8bfa7capm3shnd72z06plw5cp8wn9ymrm"
+    }
+  },
+  "version": 3
+}
\ No newline at end of file
diff --git a/test.nix b/test.nix
new file mode 100644
index 0000000..44dbcb9
--- /dev/null
+++ b/test.nix
@@ -0,0 +1,86 @@
+let
+  sources = import ./npins;
+  inherit (sources) nixpkgs;
+  pkgs = import nixpkgs { };
+
+  django = pkgs.python3.withPackages (p: [ p.django ]);
+  settings = {
+    SMOKE_TEST = "Hello settings";
+    JSON_SMOKE_TEST = {
+      "hello" = "json";
+    };
+
+    DATABASES = {
+      "default" = {
+        "ENGINE" = "django.db.backends.sqlite3";
+        "NAME" = "/var/lib/django-smoketest/db.sqlite3";
+      };
+    };
+  };
+
+  secretFile = "${./test_assets/secret}";
+
+  secrets = {
+    SECRETS_SMOKE_TEST = secretFile;
+  };
+
+  project = pkgs.runCommandNoCC "smoketest" { } ''
+    mkdir -p $out
+    ${django}/bin/django-admin startproject smoketest $out
+    cp ${./test_assets/views.py} $out/smoketest/views.py
+    cp ${./test_assets/urls.py} $out/smoketest/urls.py
+  '';
+in
+pkgs.testers.runNixOSTest (
+  { lib, ... }:
+  {
+    name = "djangonix smoke test";
+    nodes.machine =
+      { pkgs, config, ... }:
+      {
+        imports = [ ./module.nix ];
+
+        services.django.smoketest = {
+          src = project;
+          inherit settings secrets;
+          port = 8000;
+        };
+        services.nginx = {
+          enable = true;
+          recommendedProxySettings = true;
+          virtualHosts.netbox = {
+            default = true;
+            locations."/".proxyPass = "http://localhost:${toString config.services.django.smoketest.port}";
+            locations."/static/".alias = config.services.django.smoketest.staticAssets;
+          };
+        };
+      };
+    testScript = ''
+      import sys
+      import time
+      import json
+      start_all()
+      machine.wait_for_unit("django-smoketest.service")
+      machine.wait_for_unit("nginx.service")
+      time.sleep(1)
+      with subtest("Test settings"):
+          status, out = machine.execute("curl http://127.0.0.1/test_settings/")
+          print(status, repr(out))
+          if status != 0 or out != "${settings.SMOKE_TEST}":
+              sys.exit(1)
+
+      with subtest("Test secrets"):
+          status, out = machine.execute("curl http://127.0.0.1/test_secrets/")
+          print(status, repr(out))
+          if status != 0 or out != json.load(open("${secretFile}")):
+              sys.exit(1)
+
+      with subtest("Test json"):
+          status, out = machine.execute("curl http://127.0.0.1:8000/test_json/")
+          print(status, repr(out))
+          if status != 0 or json.loads(out) != json.loads('${builtins.toJSON settings.JSON_SMOKE_TEST}'):
+              sys.exit(1)
+
+    '';
+  }
+)
diff --git a/test_assets/secret b/test_assets/secret
new file mode 100644
index 0000000..754b578
--- /dev/null
+++ b/test_assets/secret
@@ -0,0 +1 @@
+"secret"
diff --git a/test_assets/urls.py b/test_assets/urls.py
new file mode 100644
index 0000000..dbea2e5
--- /dev/null
+++ b/test_assets/urls.py
@@ -0,0 +1,9 @@
+from django.urls import path
+
+from .views import test_json, test_secrets, test_settings
+
+urlpatterns = [
+    path("test_settings/", test_settings),
+    path("test_secrets/", test_secrets),
+    path("test_json/", test_json),
+]
diff --git a/test_assets/views.py b/test_assets/views.py
new file mode 100644
index 0000000..44217f1
--- /dev/null
+++ b/test_assets/views.py
@@ -0,0 +1,16 @@
+import json
+
+from django.conf import settings
+from django.http import HttpResponse
+
+
+def test_settings(request):
+    return HttpResponse(settings.SMOKE_TEST, content_type="text/plain")
+
+
+def test_secrets(request):
+    return HttpResponse(settings.SECRETS_SMOKE_TEST, content_type="text/plain")
+
+
+def test_json(request):
+    return HttpResponse(json.dumps(settings.JSON_SMOKE_TEST), content_type="application/json")
diff --git a/utils/mkManagePy.nix b/utils/mkManagePy.nix
new file mode 100644
index 0000000..7bf4528
--- /dev/null
+++ b/utils/mkManagePy.nix
@@ -0,0 +1,20 @@
+{
+  pkgs,
+  lib,
+  writeShellApplication,
+}:
+{
+  app,
+  pythonPackage,
+  manageFilePath,
+  src,
+}:
+writeShellApplication {
+  name = "manage-${app}";
+  runtimeInputs = [ pythonPackage ];
+  text = ''
+    PYTHON_PATH=${src} \
+    DJANGO_SETTINGS_MODULE=''${DJANGO_SETTINGS_MODULE:=${app}_settings.prod} \
+    python ${src}/${manageFilePath} "$@"
+  '';
+}
diff --git a/utils/mkSettingsModule.nix b/utils/mkSettingsModule.nix
new file mode 100644
index 0000000..d5afba8
--- /dev/null
+++ b/utils/mkSettingsModule.nix
@@ -0,0 +1,70 @@
+{
+  lib,
+  buildPythonPackage,
+  hatchling,
+  loadcredential,
+  runtimeSettings,
+  settings,
+  secrets,
+  mainModule,
+  extraConfig,
+}:
+let
+  secretsImport = lib.mapAttrsToList (k: _: "${k} = credentials.get_json(\"${k}\")") secrets;
+  runtimeImport = builtins.map (k: "${k} = credentials.get_json(\"${k}\")") runtimeSettings;
+  settingsImport = lib.mapAttrsToList (k: v: "${k} = json.loads('${builtins.toJSON v}')") settings;
+  prod = ''
+    from loadcredential import Credentials
+    from .mock import *
+
+    credentials = Credentials()
+
+    ${lib.concatLines secretsImport}
+
+    ${extraConfig}
+  '';
+  mock = ''
+    import json
+    from ${mainModule}.settings import *
+    from loadcredential import Credentials
+
+    credentials = Credentials()
+
+    ${lib.concatLines settingsImport}
+
+    ${lib.concatLines runtimeImport}
+  '';
+in
+buildPythonPackage {
+  name = "${mainModule}-settings";
+  unpackPhase = ''
+    cat > pyproject.toml << EOF
+    [project]
+    name = "${mainModule}-settings"
+    version="0.0.1"
+    dependencies = [
+      "loadcredential"
+    ]
+
+    [build-system]
+    build-backend = "hatchling.build"
+    requires = ["hatchling"]
+    EOF
+
+    mkdir -p src/${mainModule}_settings
+
+    touch src/${mainModule}_settings/__init__.py
+
+    cat > src/${mainModule}_settings/mock.py << EOF
+    ${mock}
+    EOF
+
+    cat > src/${mainModule}_settings/prod.py << EOF
+    ${prod}
+    EOF
+
+  '';
+  pyproject = true;
+  dependencies = [ loadcredential ];
+  build-system = [ hatchling ];
+}