feat(storage01): Switch to nix-modules for forgejo nix runners
This commit is contained in:
parent
8ddd33d88d
commit
df8831301f
8 changed files with 68 additions and 389 deletions
|
|
@ -11,8 +11,8 @@ lib.extra.mkConfig {
|
|||
# List of services to enable
|
||||
"atticd"
|
||||
"forgejo"
|
||||
"forgejo-runners"
|
||||
"garage"
|
||||
"gitea-actions-runner"
|
||||
"netbird"
|
||||
"peertube"
|
||||
];
|
||||
|
|
@ -22,8 +22,6 @@ lib.extra.mkConfig {
|
|||
|
||||
dgn-hardware.useZfs = true;
|
||||
|
||||
dgn-runners.enable = true;
|
||||
|
||||
services.netbird.enable = true;
|
||||
};
|
||||
|
||||
|
|
|
|||
65
machines/storage01/forgejo-runners.nix
Normal file
65
machines/storage01/forgejo-runners.nix
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
{ config, pkgs, ... }:
|
||||
|
||||
let
|
||||
url = "https://git.dgnum.eu";
|
||||
|
||||
mkRunner = { labels, name, token }: {
|
||||
enable = true;
|
||||
|
||||
inherit name labels token url;
|
||||
|
||||
settings.container = {
|
||||
network = "host";
|
||||
options = "--cpus=4";
|
||||
};
|
||||
};
|
||||
|
||||
in {
|
||||
services.forgejo-nix-runners = {
|
||||
enable = true;
|
||||
|
||||
inherit url;
|
||||
|
||||
storePath = "/data/slow/nix";
|
||||
tokenFile = config.age.secrets."forgejo_runners-token_file".path;
|
||||
|
||||
dependencies = [ pkgs.colmena pkgs.npins pkgs.tea ];
|
||||
|
||||
containerOptions = [ "--cpus=4" ];
|
||||
};
|
||||
|
||||
services.gitea-actions-runner.instances = builtins.mapAttrs (_: mkRunner) {
|
||||
runner01 = {
|
||||
token = "qT9nZXKgLcb3fWOj7VTj3S58raiCWwF0weuIIKlY";
|
||||
name = "storage01 [debian]";
|
||||
|
||||
labels = [ "debian-latest:docker://node:20-bookworm" ];
|
||||
};
|
||||
|
||||
runner02 = {
|
||||
token = "m0rUDBxxkMb6xLTmPFwIieHswJFROqPFByPlT3V9";
|
||||
name = "storage01 [debian]";
|
||||
|
||||
labels = [ "debian-latest:docker://node:20-bookworm" ];
|
||||
};
|
||||
};
|
||||
|
||||
virtualisation = {
|
||||
podman = {
|
||||
enable = true;
|
||||
|
||||
defaultNetwork.settings = {
|
||||
dns_enable = true;
|
||||
ipv6_enabled = true;
|
||||
};
|
||||
};
|
||||
|
||||
containers.storage.settings = {
|
||||
storage = {
|
||||
driver = "overlay";
|
||||
graphroot = "/data/slow/containers/storage";
|
||||
runroot = "/run/containers/storage";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
{ pkgs, ... }:
|
||||
|
||||
let
|
||||
url = "https://git.dgnum.eu";
|
||||
|
||||
mkRunner = { labels, name, token }: {
|
||||
enable = true;
|
||||
|
||||
inherit name labels token url;
|
||||
|
||||
settings.container = {
|
||||
network = "host";
|
||||
options = "--cpus=4";
|
||||
};
|
||||
};
|
||||
in {
|
||||
services.gitea-actions-runner = {
|
||||
package = pkgs.forgejo-actions-runner;
|
||||
|
||||
instances = builtins.mapAttrs (_: mkRunner) {
|
||||
runner01 = {
|
||||
token = "qT9nZXKgLcb3fWOj7VTj3S58raiCWwF0weuIIKlY";
|
||||
name = "storage01 [debian]";
|
||||
|
||||
labels = [ "debian-latest:docker://node:20-bookworm" ];
|
||||
};
|
||||
|
||||
runner02 = {
|
||||
token = "m0rUDBxxkMb6xLTmPFwIieHswJFROqPFByPlT3V9";
|
||||
name = "storage01 [debian]";
|
||||
|
||||
labels = [ "debian-latest:docker://node:20-bookworm" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
virtualisation.podman = {
|
||||
enable = true;
|
||||
|
||||
defaultNetwork.settings = {
|
||||
dns_enable = true;
|
||||
ipv6_enabled = true;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ let
|
|||
in lib.setDefault { inherit publicKeys; } [
|
||||
"atticd-credentials_file"
|
||||
"forgejo-database_password_file"
|
||||
"forgejo_runners-token_file"
|
||||
"garage-environment_file"
|
||||
"netbird-auth_client_secret_file"
|
||||
"peertube-secrets_file"
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@
|
|||
"dgn-fail2ban"
|
||||
"dgn-hardware"
|
||||
"dgn-network"
|
||||
"dgn-runners"
|
||||
"dgn-ssh"
|
||||
"dgn-web"
|
||||
"dgn-vm-variant"
|
||||
|
|
@ -52,5 +51,6 @@
|
|||
"${sources.attic}/nixos/atticd.nix"
|
||||
] ++ ((import sources.nix-modules { inherit lib; }).importModules [
|
||||
"age-secrets"
|
||||
"services/forgejo-nix-runners"
|
||||
]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,336 +0,0 @@
|
|||
/* Copyright 2023 Clan contributers, Tom-Hubrecht
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to
|
||||
deal in the Software without restriction, including without limitation the
|
||||
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib) mkEnableOption mkOption types;
|
||||
|
||||
storeDeps = pkgs.runCommand "store-deps" { } ''
|
||||
mkdir -p $out/bin
|
||||
for dir in ${builtins.toString cfg.dependencies}; do
|
||||
for bin in "$dir"/bin/*; do
|
||||
ln -s "$bin" "$out/bin/$(basename "$bin")"
|
||||
done
|
||||
done
|
||||
|
||||
# Add SSL CA certs
|
||||
mkdir -p $out/etc/ssl/certs
|
||||
cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt
|
||||
'';
|
||||
|
||||
cfg = config.dgn-runners;
|
||||
|
||||
storePaths = "nix-store --query -R ${builtins.toString (cfg.dependencies ++ [ pkgs.path ])}";
|
||||
in {
|
||||
options.dgn-runners = {
|
||||
enable = mkEnableOption "Forgero Actions Runners.";
|
||||
|
||||
nbRunners = mkOption {
|
||||
type = types.ints.positive;
|
||||
default = 4;
|
||||
};
|
||||
|
||||
storePath = mkOption {
|
||||
type = types.str;
|
||||
default = "/data/slow/nix";
|
||||
};
|
||||
|
||||
dependencies = mkOption { type = with types; listOf package; };
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
dgn-runners.dependencies = [
|
||||
pkgs.bash
|
||||
pkgs.colmena
|
||||
pkgs.coreutils
|
||||
pkgs.curl
|
||||
pkgs.findutils
|
||||
pkgs.gawk
|
||||
pkgs.git
|
||||
pkgs.gnugrep
|
||||
pkgs.jq
|
||||
pkgs.nix
|
||||
pkgs.nodejs
|
||||
pkgs.npins
|
||||
pkgs.openssh
|
||||
pkgs.tea
|
||||
];
|
||||
|
||||
# everything here has no dependencies on the store
|
||||
systemd.services = {
|
||||
forgejo-nix-store = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [ pkgs.curl pkgs.nix ];
|
||||
script = ''
|
||||
dest=${cfg.storePath}
|
||||
|
||||
echo "[+] Performing an installation of Nix at $dest."
|
||||
|
||||
if ! [ -e "$dest" ]; then
|
||||
echo "[+] Directory $dest does not exist, creating it."
|
||||
mkdir -m 0755 $dest
|
||||
fi
|
||||
|
||||
mkdir -p "$dest/store" "$dest/var/nix"
|
||||
|
||||
# Copy the required store paths
|
||||
for i in $(${storePaths} | sed -e 's|/nix/store/||'); do
|
||||
i_tmp="$dest/store/$i.$$"
|
||||
if [ -e "$i_tmp" ]; then
|
||||
rm -rf "$i_tmp"
|
||||
fi
|
||||
if ! [ -e "$dest/store/$i" ]; then
|
||||
cp -RPp "/nix/store/$i" "$i_tmp"
|
||||
chmod -R a-w "$i_tmp"
|
||||
chmod +w "$i_tmp"
|
||||
mv "$i_tmp" "$dest/store/$i"
|
||||
chmod -w "$dest/store/$i"
|
||||
fi
|
||||
done
|
||||
|
||||
# Dump the corresponding db
|
||||
nix-store --dump-db $(${storePaths}) | sed -e "s|/nix|$dest|" > .storeinfo
|
||||
|
||||
# Switch to the new store
|
||||
export NIX_ROOT="$dest"
|
||||
export NIX_STORE_DIR="$NIX_ROOT/store"
|
||||
export NIX_DATA_DIR="$NIX_ROOT/share"
|
||||
export NIX_STATE_DIR="$NIX_ROOT/var/nix"
|
||||
|
||||
nix-store --load-db < .storeinfo
|
||||
|
||||
# Setup the permissions to nixuser
|
||||
chown -R nixuser:nixuser ${cfg.storePath}
|
||||
'';
|
||||
|
||||
serviceConfig = {
|
||||
RuntimeDirectory = "forgejo-store-nix";
|
||||
WorkingDirectory = "/run/forgejo-store-nix";
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
};
|
||||
|
||||
forgejo-runner-nix-image = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "podman.service" "forgejo-nix-store.service" ];
|
||||
requires = [ "podman.service" "forgejo-nix-store.service" ];
|
||||
path = [
|
||||
config.virtualisation.podman.package
|
||||
pkgs.gnutar
|
||||
pkgs.shadow
|
||||
pkgs.getent
|
||||
];
|
||||
|
||||
# we also include etc here because the cleanup job also wants the nixuser to be present
|
||||
script = ''
|
||||
set -eux -o pipefail
|
||||
|
||||
mkdir -p etc/nix
|
||||
|
||||
# Create an unpriveleged user that we can use also without the run-as-user.sh script
|
||||
touch etc/passwd etc/group
|
||||
groupid=$(cut -d: -f3 < <(getent group nixuser))
|
||||
userid=$(cut -d: -f3 < <(getent passwd nixuser))
|
||||
groupadd --prefix $(pwd) --gid "$groupid" nixuser
|
||||
emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.'
|
||||
useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser
|
||||
|
||||
cat <<NIX_CONFIG > etc/nix/nix.conf
|
||||
accept-flake-config = true
|
||||
experimental-features = nix-command flakes
|
||||
NIX_CONFIG
|
||||
|
||||
cat <<NSSWITCH > etc/nsswitch.conf
|
||||
passwd: files mymachines systemd
|
||||
group: files mymachines systemd
|
||||
shadow: files
|
||||
|
||||
hosts: files mymachines dns myhostname
|
||||
networks: files
|
||||
|
||||
ethers: files
|
||||
services: files
|
||||
protocols: files
|
||||
rpc: files
|
||||
NSSWITCH
|
||||
|
||||
# Link usr/bin/env
|
||||
mkdir -p usr/bin
|
||||
ln -s /bin/env usr/bin/env
|
||||
|
||||
# list the content as it will be imported into the container
|
||||
tar -cv . | tar -tvf -
|
||||
tar -cv . | podman import - forgejo-runner-nix
|
||||
'';
|
||||
|
||||
serviceConfig = {
|
||||
RuntimeDirectory = "forgejo-runner-nix-image";
|
||||
WorkingDirectory = "/run/forgejo-runner-nix-image";
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
};
|
||||
} // lib.genAttrs
|
||||
(builtins.genList (n: "forgejo-runner-nix${builtins.toString n}")
|
||||
cfg.nbRunners) (name: {
|
||||
after = [ "forgejo-runner-nix-image.service" ];
|
||||
requires = [ "forgejo-runner-nix-image.service" ];
|
||||
|
||||
# TODO: systemd confinment
|
||||
serviceConfig = {
|
||||
# Hardening (may overlap with DynamicUser=)
|
||||
# The following options are only for optimizing output of systemd-analyze
|
||||
AmbientCapabilities = "";
|
||||
CapabilityBoundingSet = "";
|
||||
# ProtectClock= adds DeviceAllow=char-rtc r
|
||||
DeviceAllow = "";
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateMounts = true;
|
||||
PrivateTmp = true;
|
||||
PrivateUsers = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHome = true;
|
||||
ProtectHostname = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectSystem = "strict";
|
||||
RemoveIPC = true;
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
UMask = "0066";
|
||||
ProtectProc = "invisible";
|
||||
SystemCallFilter = [
|
||||
"~@clock"
|
||||
"~@cpu-emulation"
|
||||
"~@module"
|
||||
"~@mount"
|
||||
"~@obsolete"
|
||||
"~@raw-io"
|
||||
"~@reboot"
|
||||
"~@swap"
|
||||
# needed by go?
|
||||
#"~@resources"
|
||||
"~@privileged"
|
||||
"~capset"
|
||||
"~setdomainname"
|
||||
"~sethostname"
|
||||
];
|
||||
RestrictAddressFamilies =
|
||||
[ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
|
||||
|
||||
# Needs network access
|
||||
PrivateNetwork = false;
|
||||
# Cannot be true due to Node
|
||||
MemoryDenyWriteExecute = false;
|
||||
|
||||
# The more restrictive "pid" option makes `nix` commands in CI emit
|
||||
# "GC Warning: Couldn't read /proc/stat"
|
||||
# You may want to set this to "pid" if not using `nix` commands
|
||||
ProcSubset = "all";
|
||||
# Coverage programs for compiled code such as `cargo-tarpaulin` disable
|
||||
# ASLR (address space layout randomization) which requires the
|
||||
# `personality` syscall
|
||||
# You may want to set this to `true` if not using coverage tooling on
|
||||
# compiled code
|
||||
LockPersonality = false;
|
||||
|
||||
# Note that this has some interactions with the User setting; so you may
|
||||
# want to consult the systemd docs if using both.
|
||||
DynamicUser = true;
|
||||
};
|
||||
});
|
||||
|
||||
users = {
|
||||
users.nixuser = {
|
||||
group = "nixuser";
|
||||
description = "Used for running nix ci jobs";
|
||||
home = "/var/empty";
|
||||
isSystemUser = true;
|
||||
};
|
||||
groups.nixuser = { };
|
||||
};
|
||||
|
||||
virtualisation = {
|
||||
podman = {
|
||||
enable = true;
|
||||
|
||||
defaultNetwork.settings = {
|
||||
dns_enable = true;
|
||||
ipv6_enabled = true;
|
||||
};
|
||||
};
|
||||
|
||||
containers.storage.settings = {
|
||||
storage = {
|
||||
driver = "overlay";
|
||||
graphroot = "/data/slow/containers/storage";
|
||||
runroot = "/run/containers/storage";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
age-secrets.sources = [ ./. ];
|
||||
|
||||
services.gitea-actions-runner.instances = lib.genAttrs
|
||||
(builtins.genList (n: "runner${builtins.toString n}") cfg.nbRunners)
|
||||
(name: {
|
||||
enable = true;
|
||||
name = "nix-runner";
|
||||
# take the git root url from the forgejo config
|
||||
# only possible if you've also configured your forgejo though the same nix config
|
||||
# otherwise you need to set it manually
|
||||
url = "https://git.dgnum.eu";
|
||||
# use your favourite nix secret manager to get a path for this
|
||||
tokenFile = config.age.secrets."dgn_runners-token_file".path;
|
||||
|
||||
labels = [ "nix:docker://forgejo-runner-nix" ];
|
||||
settings = {
|
||||
container = {
|
||||
options = builtins.toString [
|
||||
"-e NIX_BUILD_SHELL=/bin/bash"
|
||||
"-e PAGER=cat"
|
||||
"-e PATH=/bin"
|
||||
"-e NIX_PATH=nixpkgs=${builtins.toString pkgs.path}"
|
||||
"--device /dev/kvm"
|
||||
"-e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
|
||||
"-v ${cfg.storePath}:/nix"
|
||||
"-v ${storeDeps}/bin:/bin"
|
||||
"-v ${storeDeps}/etc/ssl:/etc/ssl"
|
||||
"--cpus=4"
|
||||
"--device=/dev/kvm"
|
||||
"--user nixuser"
|
||||
];
|
||||
|
||||
# the default network that also respects our dns server settings
|
||||
network = "host";
|
||||
valid_volumes =
|
||||
[ cfg.storePath "${storeDeps}/bin" "${storeDeps}/etc/ssl" ];
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
let
|
||||
lib = import ../../lib { };
|
||||
publicKeys = builtins.concatMap lib.getNodeKeys [ "storage01" ];
|
||||
in lib.setDefault { inherit publicKeys; } [ "dgn_runners-token_file" ]
|
||||
Loading…
Reference in a new issue