feat(workflows/eval): Use nix runners

.forgejo/workflows/eval.yaml

@@ -6,17 +6,10 @@ on:
 
 jobs:
   build_configs:
-    runs-on: debian-latest
+    runs-on: nix
     steps:
-      - name: Install applications
-        run: apt-get update && apt-get install sudo
-
       - uses: actions/checkout@v3
 
-      - uses: https://github.com/cachix/install-nix-action@v22
-        with:
-          nix_path: nixpkgs=channel:nixos-unstable
-
       - name: Build the nodes
         run: |
           # Enter the shell

machines/storage01/_configuration.nix

@@ -19,6 +19,8 @@ lib.extra.mkConfig {
   extraConfig = {
     dgn-hardware.useZfs = true;
 
+    dgn-runners.enable = true;
+
     services.netbird.enable = true;
   };
 

machines/storage01/gitea-actions-runner.nix

@@ -42,12 +42,4 @@ in {
       ipv6_enabled = true;
     };
   };
-
-  virtualisation.containers.storage.settings = {
-    storage = {
-      driver = "overlay";
-      graphroot = "/data/slow/containers/storage";
-      runroot = "/run/containers/storage";
-    };
-  };
 }

modules/default.nix

@@ -43,6 +43,7 @@
     "dgn-fail2ban"
     "dgn-hardware"
     "dgn-network"
+    "dgn-runners"
     "dgn-secrets"
     "dgn-ssh"
     "dgn-web"

modules/dgn-runners/default.nix

@@ -0,0 +1,331 @@
+/* Copyright 2023 Clan contributers, Tom-Hubrecht
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to
+   deal in the Software without restriction, including without limitation the
+   rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+   sell copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in
+   all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+   DEALINGS IN THE SOFTWARE.
+*/
+
+{ config, pkgs, lib, ... }:
+
+let
+  inherit (lib) mkEnableOption mkOption types;
+
+  storeDeps = pkgs.runCommand "store-deps" { } ''
+    mkdir -p $out/bin
+    for dir in ${builtins.toString cfg.dependencies}; do
+      for bin in "$dir"/bin/*; do
+        ln -s "$bin" "$out/bin/$(basename "$bin")"
+      done
+    done
+
+    # Add SSL CA certs
+    mkdir -p $out/etc/ssl/certs
+    cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt
+  '';
+
+  cfg = config.dgn-runners;
+
+  storePaths = "nix-store --query -R ${builtins.toString cfg.dependencies}";
+in {
+  options.dgn-runners = {
+    enable = mkEnableOption "Forgero Actions Runners.";
+
+    nbRunners = mkOption {
+      type = types.ints.positive;
+      default = 4;
+    };
+
+    storePath = mkOption {
+      type = types.str;
+      default = "/data/slow/nix";
+    };
+
+    dependencies = mkOption { type = with types; listOf package; };
+  };
+
+  config = lib.mkIf cfg.enable {
+    dgn-runners.dependencies = [
+      pkgs.coreutils
+      pkgs.findutils
+      pkgs.gnugrep
+      pkgs.gawk
+      pkgs.git
+      pkgs.nix
+      pkgs.bash
+      pkgs.jq
+      pkgs.nodejs
+      pkgs.npins
+      pkgs.colmena
+      pkgs.tea
+    ];
+
+    # everything here has no dependencies on the store
+    systemd.services = {
+      forgejo-nix-store = {
+        wantedBy = [ "multi-user.target" ];
+        path = [ pkgs.curl pkgs.nix ];
+        script = ''
+          dest=${cfg.storePath}
+
+          echo "[+] Performing an installation of Nix at $dest."
+
+          if ! [ -e "$dest" ]; then
+            echo "[+] Directory $dest does not exist, creating it."
+            mkdir -m 0755 $dest
+          fi
+
+          mkdir -p "$dest/store" "$dest/var/nix"
+
+          # Copy the required store paths
+          for i in $(${storePaths} | sed -e 's|/nix/store/||'); do
+            i_tmp="$dest/store/$i.$$"
+            if [ -e "$i_tmp" ]; then
+                rm -rf "$i_tmp"
+            fi
+            if ! [ -e "$dest/store/$i" ]; then
+                cp -RPp "/nix/store/$i" "$i_tmp"
+                chmod -R a-w "$i_tmp"
+                chmod +w "$i_tmp"
+                mv "$i_tmp" "$dest/store/$i"
+                chmod -w "$dest/store/$i"
+            fi
+          done
+
+          # Dump the corresponding db
+          nix-store --dump-db $(${storePaths}) | sed -e "s|/nix|$dest|" > .storeinfo
+
+          # Switch to the new store
+          export NIX_ROOT="$dest"
+          export NIX_STORE_DIR="$NIX_ROOT/store"
+          export NIX_DATA_DIR="$NIX_ROOT/share"
+          export NIX_STATE_DIR="$NIX_ROOT/var/nix"
+
+          nix-store --load-db < .storeinfo
+
+          # Setup the permissions to nixuser
+          chown -R nixuser:nixuser ${cfg.storePath}
+        '';
+
+        serviceConfig = {
+          RuntimeDirectory = "forgejo-store-nix";
+          WorkingDirectory = "/run/forgejo-store-nix";
+          Type = "oneshot";
+          RemainAfterExit = true;
+        };
+      };
+
+      forgejo-runner-nix-image = {
+        wantedBy = [ "multi-user.target" ];
+        after = [ "podman.service" "forgejo-nix-store.service" ];
+        requires = [ "podman.service" "forgejo-nix-store.service" ];
+        path = [
+          config.virtualisation.podman.package
+          pkgs.gnutar
+          pkgs.shadow
+          pkgs.getent
+        ];
+
+        # we also include etc here because the cleanup job also wants the nixuser to be present
+        script = ''
+          set -eux -o pipefail
+
+          mkdir -p etc/nix
+
+          # Create an unpriveleged user that we can use also without the run-as-user.sh script
+          touch etc/passwd etc/group
+          groupid=$(cut -d: -f3 < <(getent group nixuser))
+          userid=$(cut -d: -f3 < <(getent passwd nixuser))
+          groupadd --prefix $(pwd) --gid "$groupid" nixuser
+          emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.'
+          useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser
+
+          cat <<NIX_CONFIG > etc/nix/nix.conf
+          accept-flake-config = true
+          experimental-features = nix-command flakes
+          NIX_CONFIG
+
+          cat <<NSSWITCH > etc/nsswitch.conf
+          passwd:    files mymachines systemd
+          group:     files mymachines systemd
+          shadow:    files
+
+          hosts:     files mymachines dns myhostname
+          networks:  files
+
+          ethers:    files
+          services:  files
+          protocols: files
+          rpc:       files
+          NSSWITCH
+
+          # list the content as it will be imported into the container
+          tar -cv . | tar -tvf -
+          tar -cv . | podman import - forgejo-runner-nix
+        '';
+
+        serviceConfig = {
+          RuntimeDirectory = "forgejo-runner-nix-image";
+          WorkingDirectory = "/run/forgejo-runner-nix-image";
+          Type = "oneshot";
+          RemainAfterExit = true;
+        };
+      };
+    } // lib.genAttrs
+      (builtins.genList (n: "forgejo-runner-nix${builtins.toString n}")
+        cfg.nbRunners) (name: {
+          after = [ "forgejo-runner-nix-image.service" ];
+          requires = [ "forgejo-runner-nix-image.service" ];
+
+          # TODO: systemd confinment
+          serviceConfig = {
+            # Hardening (may overlap with DynamicUser=)
+            # The following options are only for optimizing output of systemd-analyze
+            AmbientCapabilities = "";
+            CapabilityBoundingSet = "";
+            # ProtectClock= adds DeviceAllow=char-rtc r
+            DeviceAllow = "";
+            NoNewPrivileges = true;
+            PrivateDevices = true;
+            PrivateMounts = true;
+            PrivateTmp = true;
+            PrivateUsers = true;
+            ProtectClock = true;
+            ProtectControlGroups = true;
+            ProtectHome = true;
+            ProtectHostname = true;
+            ProtectKernelLogs = true;
+            ProtectKernelModules = true;
+            ProtectKernelTunables = true;
+            ProtectSystem = "strict";
+            RemoveIPC = true;
+            RestrictNamespaces = true;
+            RestrictRealtime = true;
+            RestrictSUIDSGID = true;
+            UMask = "0066";
+            ProtectProc = "invisible";
+            SystemCallFilter = [
+              "~@clock"
+              "~@cpu-emulation"
+              "~@module"
+              "~@mount"
+              "~@obsolete"
+              "~@raw-io"
+              "~@reboot"
+              "~@swap"
+              # needed by go?
+              #"~@resources"
+              "~@privileged"
+              "~capset"
+              "~setdomainname"
+              "~sethostname"
+            ];
+            RestrictAddressFamilies =
+              [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
+
+            # Needs network access
+            PrivateNetwork = false;
+            # Cannot be true due to Node
+            MemoryDenyWriteExecute = false;
+
+            # The more restrictive "pid" option makes `nix` commands in CI emit
+            # "GC Warning: Couldn't read /proc/stat"
+            # You may want to set this to "pid" if not using `nix` commands
+            ProcSubset = "all";
+            # Coverage programs for compiled code such as `cargo-tarpaulin` disable
+            # ASLR (address space layout randomization) which requires the
+            # `personality` syscall
+            # You may want to set this to `true` if not using coverage tooling on
+            # compiled code
+            LockPersonality = false;
+
+            # Note that this has some interactions with the User setting; so you may
+            # want to consult the systemd docs if using both.
+            DynamicUser = true;
+          };
+        });
+
+    users = {
+      users.nixuser = {
+        group = "nixuser";
+        description = "Used for running nix ci jobs";
+        home = "/var/empty";
+        isSystemUser = true;
+      };
+      groups.nixuser = { };
+    };
+
+    virtualisation = {
+      podman = {
+        enable = true;
+
+        defaultNetwork.settings = {
+          dns_enable = true;
+          ipv6_enabled = true;
+        };
+
+        extraPackages = [ pkgs.zfs_2_1 ];
+      };
+
+      containers.storage.settings = {
+        storage = {
+          driver = "overlay";
+          graphroot = "/data/slow/containers/storage";
+          runroot = "/run/containers/storage";
+        };
+      };
+    };
+
+    dgn-secrets.sources = [ ./. ];
+
+    services.gitea-actions-runner.instances = lib.genAttrs
+      (builtins.genList (n: "runner${builtins.toString n}") cfg.nbRunners)
+      (name: {
+        enable = true;
+        name = "nix-runner";
+        # take the git root url from the forgejo config
+        # only possible if you've also configured your forgejo though the same nix config
+        # otherwise you need to set it manually
+        url = "https://git.dgnum.eu";
+        # use your favourite nix secret manager to get a path for this
+        tokenFile = config.age.secrets."dgn_runners-token_file".path;
+
+        labels = [ "nix:docker://forgejo-runner-nix" ];
+        settings = {
+          container = {
+            options = builtins.toString [
+              "-e NIX_BUILD_SHELL=/bin/bash"
+              "-e PAGER=cat"
+              "-e PATH=/bin"
+              "--device /dev/kvm"
+              "-e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
+              "-v ${cfg.storePath}:/nix"
+              "-v ${storeDeps}/bin:/bin"
+              "-v ${storeDeps}/etc/ssl:/etc/ssl"
+              "--cpus=4"
+              "--device=/dev/kvm"
+              "--user nixuser"
+            ];
+
+            # the default network that also respects our dns server settings
+            network = "host";
+            valid_volumes =
+              [ cfg.storePath "${storeDeps}/bin" "${storeDeps}/etc/ssl" ];
+          };
+        };
+      });
+  };
+}

modules/dgn-runners/dgn_runners-token_file

@@ -0,0 +1,26 @@
+age-encryption.org/v1
+-> ssh-ed25519 rHotTw Y+aFwzGPr7mQUUfN2UHgvUdvLHYFJ5hsDMPZdYTVNQI
+yvJQ7dhxQ5hjxIItNMlZsnsPrVYB7QchFDE5Hi6zmek
+-> ssh-ed25519 jIXfPA Chc4vkshYdpc1zx9eLjssl+kGU2SJUeVlRgbo1n7+FI
+a+YJDGndBbmpkZOYwdezZKNYdyMdGW3geasVlmuqtTc
+-> ssh-ed25519 QlRB9Q Q5NBD14wdPQ3pEuzN7HByVkEsXluMyj2aKxCYqkViAw
+AeGzeWAUFift6LQcsOjXv8/JEDSKyUqlxctA6p0Hx+k
+-> ssh-ed25519 r+nK/Q Fe1nO9Z1iUODgfnaXfrcSRJO9IwzbuHF6pCyQgufNEo
+0TzwgI0NvoIHEa0mlrREgpYLZTk7SHj5dnCrJgLfyd4
+-> ssh-rsa krWCLQ
+dCOwZM/pdz8Iy0U2YdXChaUHOcyLyqabwnNrkYEV3tzs4d9aUp21Kb/jbTuW009a
+cNuJa/sOb2ORsfA+fh2lKvjwF7zfpli9bhheWnXFr8c07trfr9N/JRwaHEqJ8c3u
+x9hv8V5I0JaF76VaYY7sMImkyN65RXL3UNSlVsFfweuYrRAIpkq1yrad6+y+Aldy
+bnQWUGOQI2DpPOJeeCIECVNUhi0XOqFhMmP2UG5kf9cWLyY/7XYOzJa0SrvCqY/0
+udZjeGW20ZCa/NxHk4jq6Irlt9UI9OG24ti+kPxG31UcydVmFZhAMpzdzPHufk2F
+/RB+/k6g1O9Pr5GSUGmhvg
+-> ssh-ed25519 /vwQcQ SC5Y6TkLhhzwk7KgJ2njEUKLhVNSWsPqxLxY6HyemSU
+z/7whFueyYe1AWRgLc8Id3JlgZoQWQm/H2IRTpTXM5o
+-> ssh-ed25519 0R97PA cteDe++6zrpQvgnhtN/O7rqgjInRjZzEhEfmzjnI0HU
+gUmoNF92IHaXGDiV3BUlQB8O3sI5dJEbnoVsYKjJqBY
+-> ssh-ed25519 JGx7Ng mdk80U/UDMrzovesBvCJtjBEDDgvd+6gTBOA5qkwdQg
+UdUNgyA5FFIK+JNLhZFdqtiwpXkMxD/ZSU+EDMPyGeM
+-> cA-grease
+U5RhIlwzvTftwV0glkG/5w
+--- Bn6gp63oOQINTJnX71K6OQ1NryhemeHYPXi4zhDSM5c
+%„yöŽh_b4ö@j<>”øѵ—ÇïÇ…åõ“WdÚã׆dTï0Öœ$åAÌê®7ÝfF­¢`µ÷™…\	N~;É÷°NŽÜG¾Ì

modules/dgn-runners/secrets.nix

@@ -0,0 +1,4 @@
+let
+  lib = import ../../lib { };
+  publicKeys = builtins.concatMap lib.getNodeKeys [ "storage01" ];
+in lib.setDefault { inherit publicKeys; } [ "dgn_runners-token_file" ]