From df8831301f50669cffe355fca18202cdd5ede587 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Wed, 10 Jan 2024 13:59:11 +0100 Subject: [PATCH] feat(storage01): Switch to nix-modules for forgejo nix runners --- machines/storage01/_configuration.nix | 4 +- machines/storage01/forgejo-runners.nix | 65 ++++ machines/storage01/gitea-actions-runner.nix | 45 --- .../secrets/forgejo_runners-token_file | 0 machines/storage01/secrets/secrets.nix | 1 + modules/default.nix | 2 +- modules/dgn-runners/default.nix | 336 ------------------ modules/dgn-runners/secrets.nix | 4 - 8 files changed, 68 insertions(+), 389 deletions(-) create mode 100644 machines/storage01/forgejo-runners.nix delete mode 100644 machines/storage01/gitea-actions-runner.nix rename modules/dgn-runners/dgn_runners-token_file => machines/storage01/secrets/forgejo_runners-token_file (100%) delete mode 100644 modules/dgn-runners/default.nix delete mode 100644 modules/dgn-runners/secrets.nix diff --git a/machines/storage01/_configuration.nix b/machines/storage01/_configuration.nix index aedf297..89df797 100644 --- a/machines/storage01/_configuration.nix +++ b/machines/storage01/_configuration.nix @@ -11,8 +11,8 @@ lib.extra.mkConfig { # List of services to enable "atticd" "forgejo" + "forgejo-runners" "garage" - "gitea-actions-runner" "netbird" "peertube" ]; @@ -22,8 +22,6 @@ lib.extra.mkConfig { dgn-hardware.useZfs = true; - dgn-runners.enable = true; - services.netbird.enable = true; }; diff --git a/machines/storage01/forgejo-runners.nix b/machines/storage01/forgejo-runners.nix new file mode 100644 index 0000000..93ae147 --- /dev/null +++ b/machines/storage01/forgejo-runners.nix @@ -0,0 +1,65 @@ +{ config, pkgs, ... }: + +let + url = "https://git.dgnum.eu"; + + mkRunner = { labels, name, token }: { + enable = true; + + inherit name labels token url; + + settings.container = { + network = "host"; + options = "--cpus=4"; + }; + }; + +in { + services.forgejo-nix-runners = { + enable = true; + + inherit url; + + storePath = "/data/slow/nix"; + tokenFile = config.age.secrets."forgejo_runners-token_file".path; + + dependencies = [ pkgs.colmena pkgs.npins pkgs.tea ]; + + containerOptions = [ "--cpus=4" ]; + }; + + services.gitea-actions-runner.instances = builtins.mapAttrs (_: mkRunner) { + runner01 = { + token = "qT9nZXKgLcb3fWOj7VTj3S58raiCWwF0weuIIKlY"; + name = "storage01 [debian]"; + + labels = [ "debian-latest:docker://node:20-bookworm" ]; + }; + + runner02 = { + token = "m0rUDBxxkMb6xLTmPFwIieHswJFROqPFByPlT3V9"; + name = "storage01 [debian]"; + + labels = [ "debian-latest:docker://node:20-bookworm" ]; + }; + }; + + virtualisation = { + podman = { + enable = true; + + defaultNetwork.settings = { + dns_enable = true; + ipv6_enabled = true; + }; + }; + + containers.storage.settings = { + storage = { + driver = "overlay"; + graphroot = "/data/slow/containers/storage"; + runroot = "/run/containers/storage"; + }; + }; + }; +} diff --git a/machines/storage01/gitea-actions-runner.nix b/machines/storage01/gitea-actions-runner.nix deleted file mode 100644 index 8776949..0000000 --- a/machines/storage01/gitea-actions-runner.nix +++ /dev/null @@ -1,45 +0,0 @@ -{ pkgs, ... }: - -let - url = "https://git.dgnum.eu"; - - mkRunner = { labels, name, token }: { - enable = true; - - inherit name labels token url; - - settings.container = { - network = "host"; - options = "--cpus=4"; - }; - }; -in { - services.gitea-actions-runner = { - package = pkgs.forgejo-actions-runner; - - instances = builtins.mapAttrs (_: mkRunner) { - runner01 = { - token = "qT9nZXKgLcb3fWOj7VTj3S58raiCWwF0weuIIKlY"; - name = "storage01 [debian]"; - - labels = [ "debian-latest:docker://node:20-bookworm" ]; - }; - - runner02 = { - token = "m0rUDBxxkMb6xLTmPFwIieHswJFROqPFByPlT3V9"; - name = "storage01 [debian]"; - - labels = [ "debian-latest:docker://node:20-bookworm" ]; - }; - }; - }; - - virtualisation.podman = { - enable = true; - - defaultNetwork.settings = { - dns_enable = true; - ipv6_enabled = true; - }; - }; -} diff --git a/modules/dgn-runners/dgn_runners-token_file b/machines/storage01/secrets/forgejo_runners-token_file similarity index 100% rename from modules/dgn-runners/dgn_runners-token_file rename to machines/storage01/secrets/forgejo_runners-token_file diff --git a/machines/storage01/secrets/secrets.nix b/machines/storage01/secrets/secrets.nix index c7683b9..cf6edba 100644 --- a/machines/storage01/secrets/secrets.nix +++ b/machines/storage01/secrets/secrets.nix @@ -5,6 +5,7 @@ let in lib.setDefault { inherit publicKeys; } [ "atticd-credentials_file" "forgejo-database_password_file" + "forgejo_runners-token_file" "garage-environment_file" "netbird-auth_client_secret_file" "peertube-secrets_file" diff --git a/modules/default.nix b/modules/default.nix index f61d961..e3f64f7 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -43,7 +43,6 @@ "dgn-fail2ban" "dgn-hardware" "dgn-network" - "dgn-runners" "dgn-ssh" "dgn-web" "dgn-vm-variant" @@ -52,5 +51,6 @@ "${sources.attic}/nixos/atticd.nix" ] ++ ((import sources.nix-modules { inherit lib; }).importModules [ "age-secrets" + "services/forgejo-nix-runners" ]); } diff --git a/modules/dgn-runners/default.nix b/modules/dgn-runners/default.nix deleted file mode 100644 index 27f6f6e..0000000 --- a/modules/dgn-runners/default.nix +++ /dev/null @@ -1,336 +0,0 @@ -/* Copyright 2023 Clan contributers, Tom-Hubrecht - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to - deal in the Software without restriction, including without limitation the - rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. -*/ - -{ config, pkgs, lib, ... }: - -let - inherit (lib) mkEnableOption mkOption types; - - storeDeps = pkgs.runCommand "store-deps" { } '' - mkdir -p $out/bin - for dir in ${builtins.toString cfg.dependencies}; do - for bin in "$dir"/bin/*; do - ln -s "$bin" "$out/bin/$(basename "$bin")" - done - done - - # Add SSL CA certs - mkdir -p $out/etc/ssl/certs - cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt - ''; - - cfg = config.dgn-runners; - - storePaths = "nix-store --query -R ${builtins.toString (cfg.dependencies ++ [ pkgs.path ])}"; -in { - options.dgn-runners = { - enable = mkEnableOption "Forgero Actions Runners."; - - nbRunners = mkOption { - type = types.ints.positive; - default = 4; - }; - - storePath = mkOption { - type = types.str; - default = "/data/slow/nix"; - }; - - dependencies = mkOption { type = with types; listOf package; }; - }; - - config = lib.mkIf cfg.enable { - dgn-runners.dependencies = [ - pkgs.bash - pkgs.colmena - pkgs.coreutils - pkgs.curl - pkgs.findutils - pkgs.gawk - pkgs.git - pkgs.gnugrep - pkgs.jq - pkgs.nix - pkgs.nodejs - pkgs.npins - pkgs.openssh - pkgs.tea - ]; - - # everything here has no dependencies on the store - systemd.services = { - forgejo-nix-store = { - wantedBy = [ "multi-user.target" ]; - path = [ pkgs.curl pkgs.nix ]; - script = '' - dest=${cfg.storePath} - - echo "[+] Performing an installation of Nix at $dest." - - if ! [ -e "$dest" ]; then - echo "[+] Directory $dest does not exist, creating it." - mkdir -m 0755 $dest - fi - - mkdir -p "$dest/store" "$dest/var/nix" - - # Copy the required store paths - for i in $(${storePaths} | sed -e 's|/nix/store/||'); do - i_tmp="$dest/store/$i.$$" - if [ -e "$i_tmp" ]; then - rm -rf "$i_tmp" - fi - if ! [ -e "$dest/store/$i" ]; then - cp -RPp "/nix/store/$i" "$i_tmp" - chmod -R a-w "$i_tmp" - chmod +w "$i_tmp" - mv "$i_tmp" "$dest/store/$i" - chmod -w "$dest/store/$i" - fi - done - - # Dump the corresponding db - nix-store --dump-db $(${storePaths}) | sed -e "s|/nix|$dest|" > .storeinfo - - # Switch to the new store - export NIX_ROOT="$dest" - export NIX_STORE_DIR="$NIX_ROOT/store" - export NIX_DATA_DIR="$NIX_ROOT/share" - export NIX_STATE_DIR="$NIX_ROOT/var/nix" - - nix-store --load-db < .storeinfo - - # Setup the permissions to nixuser - chown -R nixuser:nixuser ${cfg.storePath} - ''; - - serviceConfig = { - RuntimeDirectory = "forgejo-store-nix"; - WorkingDirectory = "/run/forgejo-store-nix"; - Type = "oneshot"; - RemainAfterExit = true; - }; - }; - - forgejo-runner-nix-image = { - wantedBy = [ "multi-user.target" ]; - after = [ "podman.service" "forgejo-nix-store.service" ]; - requires = [ "podman.service" "forgejo-nix-store.service" ]; - path = [ - config.virtualisation.podman.package - pkgs.gnutar - pkgs.shadow - pkgs.getent - ]; - - # we also include etc here because the cleanup job also wants the nixuser to be present - script = '' - set -eux -o pipefail - - mkdir -p etc/nix - - # Create an unpriveleged user that we can use also without the run-as-user.sh script - touch etc/passwd etc/group - groupid=$(cut -d: -f3 < <(getent group nixuser)) - userid=$(cut -d: -f3 < <(getent passwd nixuser)) - groupadd --prefix $(pwd) --gid "$groupid" nixuser - emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.' - useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser - - cat < etc/nix/nix.conf - accept-flake-config = true - experimental-features = nix-command flakes - NIX_CONFIG - - cat < etc/nsswitch.conf - passwd: files mymachines systemd - group: files mymachines systemd - shadow: files - - hosts: files mymachines dns myhostname - networks: files - - ethers: files - services: files - protocols: files - rpc: files - NSSWITCH - - # Link usr/bin/env - mkdir -p usr/bin - ln -s /bin/env usr/bin/env - - # list the content as it will be imported into the container - tar -cv . | tar -tvf - - tar -cv . | podman import - forgejo-runner-nix - ''; - - serviceConfig = { - RuntimeDirectory = "forgejo-runner-nix-image"; - WorkingDirectory = "/run/forgejo-runner-nix-image"; - Type = "oneshot"; - RemainAfterExit = true; - }; - }; - } // lib.genAttrs - (builtins.genList (n: "forgejo-runner-nix${builtins.toString n}") - cfg.nbRunners) (name: { - after = [ "forgejo-runner-nix-image.service" ]; - requires = [ "forgejo-runner-nix-image.service" ]; - - # TODO: systemd confinment - serviceConfig = { - # Hardening (may overlap with DynamicUser=) - # The following options are only for optimizing output of systemd-analyze - AmbientCapabilities = ""; - CapabilityBoundingSet = ""; - # ProtectClock= adds DeviceAllow=char-rtc r - DeviceAllow = ""; - NoNewPrivileges = true; - PrivateDevices = true; - PrivateMounts = true; - PrivateTmp = true; - PrivateUsers = true; - ProtectClock = true; - ProtectControlGroups = true; - ProtectHome = true; - ProtectHostname = true; - ProtectKernelLogs = true; - ProtectKernelModules = true; - ProtectKernelTunables = true; - ProtectSystem = "strict"; - RemoveIPC = true; - RestrictNamespaces = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - UMask = "0066"; - ProtectProc = "invisible"; - SystemCallFilter = [ - "~@clock" - "~@cpu-emulation" - "~@module" - "~@mount" - "~@obsolete" - "~@raw-io" - "~@reboot" - "~@swap" - # needed by go? - #"~@resources" - "~@privileged" - "~capset" - "~setdomainname" - "~sethostname" - ]; - RestrictAddressFamilies = - [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ]; - - # Needs network access - PrivateNetwork = false; - # Cannot be true due to Node - MemoryDenyWriteExecute = false; - - # The more restrictive "pid" option makes `nix` commands in CI emit - # "GC Warning: Couldn't read /proc/stat" - # You may want to set this to "pid" if not using `nix` commands - ProcSubset = "all"; - # Coverage programs for compiled code such as `cargo-tarpaulin` disable - # ASLR (address space layout randomization) which requires the - # `personality` syscall - # You may want to set this to `true` if not using coverage tooling on - # compiled code - LockPersonality = false; - - # Note that this has some interactions with the User setting; so you may - # want to consult the systemd docs if using both. - DynamicUser = true; - }; - }); - - users = { - users.nixuser = { - group = "nixuser"; - description = "Used for running nix ci jobs"; - home = "/var/empty"; - isSystemUser = true; - }; - groups.nixuser = { }; - }; - - virtualisation = { - podman = { - enable = true; - - defaultNetwork.settings = { - dns_enable = true; - ipv6_enabled = true; - }; - }; - - containers.storage.settings = { - storage = { - driver = "overlay"; - graphroot = "/data/slow/containers/storage"; - runroot = "/run/containers/storage"; - }; - }; - }; - - age-secrets.sources = [ ./. ]; - - services.gitea-actions-runner.instances = lib.genAttrs - (builtins.genList (n: "runner${builtins.toString n}") cfg.nbRunners) - (name: { - enable = true; - name = "nix-runner"; - # take the git root url from the forgejo config - # only possible if you've also configured your forgejo though the same nix config - # otherwise you need to set it manually - url = "https://git.dgnum.eu"; - # use your favourite nix secret manager to get a path for this - tokenFile = config.age.secrets."dgn_runners-token_file".path; - - labels = [ "nix:docker://forgejo-runner-nix" ]; - settings = { - container = { - options = builtins.toString [ - "-e NIX_BUILD_SHELL=/bin/bash" - "-e PAGER=cat" - "-e PATH=/bin" - "-e NIX_PATH=nixpkgs=${builtins.toString pkgs.path}" - "--device /dev/kvm" - "-e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" - "-v ${cfg.storePath}:/nix" - "-v ${storeDeps}/bin:/bin" - "-v ${storeDeps}/etc/ssl:/etc/ssl" - "--cpus=4" - "--device=/dev/kvm" - "--user nixuser" - ]; - - # the default network that also respects our dns server settings - network = "host"; - valid_volumes = - [ cfg.storePath "${storeDeps}/bin" "${storeDeps}/etc/ssl" ]; - }; - }; - }); - }; -} diff --git a/modules/dgn-runners/secrets.nix b/modules/dgn-runners/secrets.nix deleted file mode 100644 index 9cd8fef..0000000 --- a/modules/dgn-runners/secrets.nix +++ /dev/null @@ -1,4 +0,0 @@ -let - lib = import ../../lib { }; - publicKeys = builtins.concatMap lib.getNodeKeys [ "storage01" ]; -in lib.setDefault { inherit publicKeys; } [ "dgn_runners-token_file" ]