From 8cef4a594a471f072b8f7c30b31db02eba1e5240 Mon Sep 17 00:00:00 2001 From: Tom Hubrecht Date: Fri, 15 Dec 2023 20:10:09 +0100 Subject: [PATCH] feat(dgn-runners): Init module for runners with a store --- modules/default.nix | 1 + modules/dgn-runners/default.nix | 331 +++++++++++++++++++++ modules/dgn-runners/dgn_runners-token_file | 26 ++ modules/dgn-runners/secrets.nix | 4 + 4 files changed, 362 insertions(+) create mode 100644 modules/dgn-runners/default.nix create mode 100644 modules/dgn-runners/dgn_runners-token_file create mode 100644 modules/dgn-runners/secrets.nix diff --git a/modules/default.nix b/modules/default.nix index 76b6998..17ace8b 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -43,6 +43,7 @@ "dgn-fail2ban" "dgn-hardware" "dgn-network" + "dgn-runners" "dgn-secrets" "dgn-ssh" "dgn-web" diff --git a/modules/dgn-runners/default.nix b/modules/dgn-runners/default.nix new file mode 100644 index 0000000..7fbc35d --- /dev/null +++ b/modules/dgn-runners/default.nix @@ -0,0 +1,331 @@ +/* Copyright 2023 Clan contributers, Tom-Hubrecht + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to + deal in the Software without restriction, including without limitation the + rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +*/ + +{ config, pkgs, lib, ... }: + +let + inherit (lib) mkEnableOption mkOption types; + + storeDeps = pkgs.runCommand "store-deps" { } '' + mkdir -p $out/bin + for dir in ${builtins.toString cfg.dependencies}; do + for bin in "$dir"/bin/*; do + ln -s "$bin" "$out/bin/$(basename "$bin")" + done + done + + # Add SSL CA certs + mkdir -p $out/etc/ssl/certs + cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt + ''; + + cfg = config.dgn-runners; + + storePaths = "nix-store --query -R ${builtins.toString cfg.dependencies}"; +in { + options.dgn-runners = { + enable = mkEnableOption "Forgero Actions Runners."; + + nbRunners = mkOption { + type = types.ints.positive; + default = 4; + }; + + storePath = mkOption { + type = types.str; + default = "/data/slow/nix"; + }; + + dependencies = mkOption { type = with types; listOf package; }; + }; + + config = lib.mkIf cfg.enable { + dgn-runners.dependencies = [ + pkgs.coreutils + pkgs.findutils + pkgs.gnugrep + pkgs.gawk + pkgs.git + pkgs.nix + pkgs.bash + pkgs.jq + pkgs.nodejs + pkgs.npins + pkgs.colmena + pkgs.tea + ]; + + # everything here has no dependencies on the store + systemd.services = { + forgejo-nix-store = { + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.curl pkgs.nix ]; + script = '' + dest=${cfg.storePath} + + echo "[+] Performing an installation of Nix at $dest." + + if ! [ -e "$dest" ]; then + echo "[+] Directory $dest does not exist, creating it." + mkdir -m 0755 $dest + fi + + mkdir -p "$dest/store" "$dest/var/nix" + + # Copy the required store paths + for i in $(${storePaths} | sed -e 's|/nix/store/||'); do + i_tmp="$dest/store/$i.$$" + if [ -e "$i_tmp" ]; then + rm -rf "$i_tmp" + fi + if ! [ -e "$dest/store/$i" ]; then + cp -RPp "/nix/store/$i" "$i_tmp" + chmod -R a-w "$i_tmp" + chmod +w "$i_tmp" + mv "$i_tmp" "$dest/store/$i" + chmod -w "$dest/store/$i" + fi + done + + # Dump the corresponding db + nix-store --dump-db $(${storePaths}) | sed -e "s|/nix|$dest|" > .storeinfo + + # Switch to the new store + export NIX_ROOT="$dest" + export NIX_STORE_DIR="$NIX_ROOT/store" + export NIX_DATA_DIR="$NIX_ROOT/share" + export NIX_STATE_DIR="$NIX_ROOT/var/nix" + + nix-store --load-db < .storeinfo + + # Setup the permissions to nixuser + chown -R nixuser:nixuser ${cfg.storePath} + ''; + + serviceConfig = { + RuntimeDirectory = "forgejo-store-nix"; + WorkingDirectory = "/run/forgejo-store-nix"; + Type = "oneshot"; + RemainAfterExit = true; + }; + }; + + forgejo-runner-nix-image = { + wantedBy = [ "multi-user.target" ]; + after = [ "podman.service" "forgejo-nix-store.service" ]; + requires = [ "podman.service" "forgejo-nix-store.service" ]; + path = [ + config.virtualisation.podman.package + pkgs.gnutar + pkgs.shadow + pkgs.getent + ]; + + # we also include etc here because the cleanup job also wants the nixuser to be present + script = '' + set -eux -o pipefail + + mkdir -p etc/nix + + # Create an unpriveleged user that we can use also without the run-as-user.sh script + touch etc/passwd etc/group + groupid=$(cut -d: -f3 < <(getent group nixuser)) + userid=$(cut -d: -f3 < <(getent passwd nixuser)) + groupadd --prefix $(pwd) --gid "$groupid" nixuser + emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.' + useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser + + cat < etc/nix/nix.conf + accept-flake-config = true + experimental-features = nix-command flakes + NIX_CONFIG + + cat < etc/nsswitch.conf + passwd: files mymachines systemd + group: files mymachines systemd + shadow: files + + hosts: files mymachines dns myhostname + networks: files + + ethers: files + services: files + protocols: files + rpc: files + NSSWITCH + + # list the content as it will be imported into the container + tar -cv . | tar -tvf - + tar -cv . | podman import - forgejo-runner-nix + ''; + + serviceConfig = { + RuntimeDirectory = "forgejo-runner-nix-image"; + WorkingDirectory = "/run/forgejo-runner-nix-image"; + Type = "oneshot"; + RemainAfterExit = true; + }; + }; + } // lib.genAttrs + (builtins.genList (n: "forgejo-runner-nix${builtins.toString n}") + cfg.nbRunners) (name: { + after = [ "forgejo-runner-nix-image.service" ]; + requires = [ "forgejo-runner-nix-image.service" ]; + + # TODO: systemd confinment + serviceConfig = { + # Hardening (may overlap with DynamicUser=) + # The following options are only for optimizing output of systemd-analyze + AmbientCapabilities = ""; + CapabilityBoundingSet = ""; + # ProtectClock= adds DeviceAllow=char-rtc r + DeviceAllow = ""; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateMounts = true; + PrivateTmp = true; + PrivateUsers = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + UMask = "0066"; + ProtectProc = "invisible"; + SystemCallFilter = [ + "~@clock" + "~@cpu-emulation" + "~@module" + "~@mount" + "~@obsolete" + "~@raw-io" + "~@reboot" + "~@swap" + # needed by go? + #"~@resources" + "~@privileged" + "~capset" + "~setdomainname" + "~sethostname" + ]; + RestrictAddressFamilies = + [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ]; + + # Needs network access + PrivateNetwork = false; + # Cannot be true due to Node + MemoryDenyWriteExecute = false; + + # The more restrictive "pid" option makes `nix` commands in CI emit + # "GC Warning: Couldn't read /proc/stat" + # You may want to set this to "pid" if not using `nix` commands + ProcSubset = "all"; + # Coverage programs for compiled code such as `cargo-tarpaulin` disable + # ASLR (address space layout randomization) which requires the + # `personality` syscall + # You may want to set this to `true` if not using coverage tooling on + # compiled code + LockPersonality = false; + + # Note that this has some interactions with the User setting; so you may + # want to consult the systemd docs if using both. + DynamicUser = true; + }; + }); + + users = { + users.nixuser = { + group = "nixuser"; + description = "Used for running nix ci jobs"; + home = "/var/empty"; + isSystemUser = true; + }; + groups.nixuser = { }; + }; + + virtualisation = { + podman = { + enable = true; + + defaultNetwork.settings = { + dns_enable = true; + ipv6_enabled = true; + }; + + extraPackages = [ pkgs.zfs_2_1 ]; + }; + + containers.storage.settings = { + storage = { + driver = "overlay"; + graphroot = "/data/slow/containers/storage"; + runroot = "/run/containers/storage"; + }; + }; + }; + + dgn-secrets.sources = [ ./. ]; + + services.gitea-actions-runner.instances = lib.genAttrs + (builtins.genList (n: "runner${builtins.toString n}") cfg.nbRunners) + (name: { + enable = true; + name = "nix-runner"; + # take the git root url from the forgejo config + # only possible if you've also configured your forgejo though the same nix config + # otherwise you need to set it manually + url = "https://git.dgnum.eu"; + # use your favourite nix secret manager to get a path for this + tokenFile = config.age.secrets."dgn_runners-token_file".path; + + labels = [ "nix:docker://forgejo-runner-nix" ]; + settings = { + container = { + options = builtins.toString [ + "-e NIX_BUILD_SHELL=/bin/bash" + "-e PAGER=cat" + "-e PATH=/bin" + "--device /dev/kvm" + "-e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + "-v ${cfg.storePath}:/nix" + "-v ${storeDeps}/bin:/bin" + "-v ${storeDeps}/etc/ssl:/etc/ssl" + "--cpus=4" + "--device=/dev/kvm" + "--user nixuser" + ]; + + # the default network that also respects our dns server settings + network = "host"; + valid_volumes = + [ cfg.storePath "${storeDeps}/bin" "${storeDeps}/etc/ssl" ]; + }; + }; + }); + }; +} diff --git a/modules/dgn-runners/dgn_runners-token_file b/modules/dgn-runners/dgn_runners-token_file new file mode 100644 index 0000000..253ff2a --- /dev/null +++ b/modules/dgn-runners/dgn_runners-token_file @@ -0,0 +1,26 @@ +age-encryption.org/v1 +-> ssh-ed25519 rHotTw Y+aFwzGPr7mQUUfN2UHgvUdvLHYFJ5hsDMPZdYTVNQI +yvJQ7dhxQ5hjxIItNMlZsnsPrVYB7QchFDE5Hi6zmek +-> ssh-ed25519 jIXfPA Chc4vkshYdpc1zx9eLjssl+kGU2SJUeVlRgbo1n7+FI +a+YJDGndBbmpkZOYwdezZKNYdyMdGW3geasVlmuqtTc +-> ssh-ed25519 QlRB9Q Q5NBD14wdPQ3pEuzN7HByVkEsXluMyj2aKxCYqkViAw +AeGzeWAUFift6LQcsOjXv8/JEDSKyUqlxctA6p0Hx+k +-> ssh-ed25519 r+nK/Q Fe1nO9Z1iUODgfnaXfrcSRJO9IwzbuHF6pCyQgufNEo +0TzwgI0NvoIHEa0mlrREgpYLZTk7SHj5dnCrJgLfyd4 +-> ssh-rsa krWCLQ +dCOwZM/pdz8Iy0U2YdXChaUHOcyLyqabwnNrkYEV3tzs4d9aUp21Kb/jbTuW009a +cNuJa/sOb2ORsfA+fh2lKvjwF7zfpli9bhheWnXFr8c07trfr9N/JRwaHEqJ8c3u +x9hv8V5I0JaF76VaYY7sMImkyN65RXL3UNSlVsFfweuYrRAIpkq1yrad6+y+Aldy +bnQWUGOQI2DpPOJeeCIECVNUhi0XOqFhMmP2UG5kf9cWLyY/7XYOzJa0SrvCqY/0 +udZjeGW20ZCa/NxHk4jq6Irlt9UI9OG24ti+kPxG31UcydVmFZhAMpzdzPHufk2F +/RB+/k6g1O9Pr5GSUGmhvg +-> ssh-ed25519 /vwQcQ SC5Y6TkLhhzwk7KgJ2njEUKLhVNSWsPqxLxY6HyemSU +z/7whFueyYe1AWRgLc8Id3JlgZoQWQm/H2IRTpTXM5o +-> ssh-ed25519 0R97PA cteDe++6zrpQvgnhtN/O7rqgjInRjZzEhEfmzjnI0HU +gUmoNF92IHaXGDiV3BUlQB8O3sI5dJEbnoVsYKjJqBY +-> ssh-ed25519 JGx7Ng mdk80U/UDMrzovesBvCJtjBEDDgvd+6gTBOA5qkwdQg +UdUNgyA5FFIK+JNLhZFdqtiwpXkMxD/ZSU+EDMPyGeM +-> cA-grease +U5RhIlwzvTftwV0glkG/5w +--- Bn6gp63oOQINTJnX71K6OQ1NryhemeHYPXi4zhDSM5c +%yh_b4@jѵDžWd׆dT0֜$A7fF`\ N~;N G \ No newline at end of file diff --git a/modules/dgn-runners/secrets.nix b/modules/dgn-runners/secrets.nix new file mode 100644 index 0000000..9cd8fef --- /dev/null +++ b/modules/dgn-runners/secrets.nix @@ -0,0 +1,4 @@ +let + lib = import ../../lib { }; + publicKeys = builtins.concatMap lib.getNodeKeys [ "storage01" ]; +in lib.setDefault { inherit publicKeys; } [ "dgn_runners-token_file" ]