forked from DGNum/infrastructure
feat(storage01): Switch to nix-modules for forgejo nix runners
This commit is contained in:
parent
8ddd33d88d
commit
df8831301f
8 changed files with 68 additions and 389 deletions
|
@ -11,8 +11,8 @@ lib.extra.mkConfig {
|
||||||
# List of services to enable
|
# List of services to enable
|
||||||
"atticd"
|
"atticd"
|
||||||
"forgejo"
|
"forgejo"
|
||||||
|
"forgejo-runners"
|
||||||
"garage"
|
"garage"
|
||||||
"gitea-actions-runner"
|
|
||||||
"netbird"
|
"netbird"
|
||||||
"peertube"
|
"peertube"
|
||||||
];
|
];
|
||||||
|
@ -22,8 +22,6 @@ lib.extra.mkConfig {
|
||||||
|
|
||||||
dgn-hardware.useZfs = true;
|
dgn-hardware.useZfs = true;
|
||||||
|
|
||||||
dgn-runners.enable = true;
|
|
||||||
|
|
||||||
services.netbird.enable = true;
|
services.netbird.enable = true;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
65
machines/storage01/forgejo-runners.nix
Normal file
65
machines/storage01/forgejo-runners.nix
Normal file
|
@ -0,0 +1,65 @@
|
||||||
|
{ config, pkgs, ... }:
|
||||||
|
|
||||||
|
let
|
||||||
|
url = "https://git.dgnum.eu";
|
||||||
|
|
||||||
|
mkRunner = { labels, name, token }: {
|
||||||
|
enable = true;
|
||||||
|
|
||||||
|
inherit name labels token url;
|
||||||
|
|
||||||
|
settings.container = {
|
||||||
|
network = "host";
|
||||||
|
options = "--cpus=4";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
in {
|
||||||
|
services.forgejo-nix-runners = {
|
||||||
|
enable = true;
|
||||||
|
|
||||||
|
inherit url;
|
||||||
|
|
||||||
|
storePath = "/data/slow/nix";
|
||||||
|
tokenFile = config.age.secrets."forgejo_runners-token_file".path;
|
||||||
|
|
||||||
|
dependencies = [ pkgs.colmena pkgs.npins pkgs.tea ];
|
||||||
|
|
||||||
|
containerOptions = [ "--cpus=4" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
services.gitea-actions-runner.instances = builtins.mapAttrs (_: mkRunner) {
|
||||||
|
runner01 = {
|
||||||
|
token = "qT9nZXKgLcb3fWOj7VTj3S58raiCWwF0weuIIKlY";
|
||||||
|
name = "storage01 [debian]";
|
||||||
|
|
||||||
|
labels = [ "debian-latest:docker://node:20-bookworm" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
runner02 = {
|
||||||
|
token = "m0rUDBxxkMb6xLTmPFwIieHswJFROqPFByPlT3V9";
|
||||||
|
name = "storage01 [debian]";
|
||||||
|
|
||||||
|
labels = [ "debian-latest:docker://node:20-bookworm" ];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
virtualisation = {
|
||||||
|
podman = {
|
||||||
|
enable = true;
|
||||||
|
|
||||||
|
defaultNetwork.settings = {
|
||||||
|
dns_enable = true;
|
||||||
|
ipv6_enabled = true;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
containers.storage.settings = {
|
||||||
|
storage = {
|
||||||
|
driver = "overlay";
|
||||||
|
graphroot = "/data/slow/containers/storage";
|
||||||
|
runroot = "/run/containers/storage";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
|
@ -1,45 +0,0 @@
|
||||||
{ pkgs, ... }:
|
|
||||||
|
|
||||||
let
|
|
||||||
url = "https://git.dgnum.eu";
|
|
||||||
|
|
||||||
mkRunner = { labels, name, token }: {
|
|
||||||
enable = true;
|
|
||||||
|
|
||||||
inherit name labels token url;
|
|
||||||
|
|
||||||
settings.container = {
|
|
||||||
network = "host";
|
|
||||||
options = "--cpus=4";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
in {
|
|
||||||
services.gitea-actions-runner = {
|
|
||||||
package = pkgs.forgejo-actions-runner;
|
|
||||||
|
|
||||||
instances = builtins.mapAttrs (_: mkRunner) {
|
|
||||||
runner01 = {
|
|
||||||
token = "qT9nZXKgLcb3fWOj7VTj3S58raiCWwF0weuIIKlY";
|
|
||||||
name = "storage01 [debian]";
|
|
||||||
|
|
||||||
labels = [ "debian-latest:docker://node:20-bookworm" ];
|
|
||||||
};
|
|
||||||
|
|
||||||
runner02 = {
|
|
||||||
token = "m0rUDBxxkMb6xLTmPFwIieHswJFROqPFByPlT3V9";
|
|
||||||
name = "storage01 [debian]";
|
|
||||||
|
|
||||||
labels = [ "debian-latest:docker://node:20-bookworm" ];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
virtualisation.podman = {
|
|
||||||
enable = true;
|
|
||||||
|
|
||||||
defaultNetwork.settings = {
|
|
||||||
dns_enable = true;
|
|
||||||
ipv6_enabled = true;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
|
@ -5,6 +5,7 @@ let
|
||||||
in lib.setDefault { inherit publicKeys; } [
|
in lib.setDefault { inherit publicKeys; } [
|
||||||
"atticd-credentials_file"
|
"atticd-credentials_file"
|
||||||
"forgejo-database_password_file"
|
"forgejo-database_password_file"
|
||||||
|
"forgejo_runners-token_file"
|
||||||
"garage-environment_file"
|
"garage-environment_file"
|
||||||
"netbird-auth_client_secret_file"
|
"netbird-auth_client_secret_file"
|
||||||
"peertube-secrets_file"
|
"peertube-secrets_file"
|
||||||
|
|
|
@ -43,7 +43,6 @@
|
||||||
"dgn-fail2ban"
|
"dgn-fail2ban"
|
||||||
"dgn-hardware"
|
"dgn-hardware"
|
||||||
"dgn-network"
|
"dgn-network"
|
||||||
"dgn-runners"
|
|
||||||
"dgn-ssh"
|
"dgn-ssh"
|
||||||
"dgn-web"
|
"dgn-web"
|
||||||
"dgn-vm-variant"
|
"dgn-vm-variant"
|
||||||
|
@ -52,5 +51,6 @@
|
||||||
"${sources.attic}/nixos/atticd.nix"
|
"${sources.attic}/nixos/atticd.nix"
|
||||||
] ++ ((import sources.nix-modules { inherit lib; }).importModules [
|
] ++ ((import sources.nix-modules { inherit lib; }).importModules [
|
||||||
"age-secrets"
|
"age-secrets"
|
||||||
|
"services/forgejo-nix-runners"
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,336 +0,0 @@
|
||||||
/* Copyright 2023 Clan contributers, Tom-Hubrecht
|
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
||||||
of this software and associated documentation files (the "Software"), to
|
|
||||||
deal in the Software without restriction, including without limitation the
|
|
||||||
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
|
||||||
sell copies of the Software, and to permit persons to whom the Software is
|
|
||||||
furnished to do so, subject to the following conditions:
|
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in
|
|
||||||
all copies or substantial portions of the Software.
|
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
||||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
||||||
DEALINGS IN THE SOFTWARE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
{ config, pkgs, lib, ... }:
|
|
||||||
|
|
||||||
let
|
|
||||||
inherit (lib) mkEnableOption mkOption types;
|
|
||||||
|
|
||||||
storeDeps = pkgs.runCommand "store-deps" { } ''
|
|
||||||
mkdir -p $out/bin
|
|
||||||
for dir in ${builtins.toString cfg.dependencies}; do
|
|
||||||
for bin in "$dir"/bin/*; do
|
|
||||||
ln -s "$bin" "$out/bin/$(basename "$bin")"
|
|
||||||
done
|
|
||||||
done
|
|
||||||
|
|
||||||
# Add SSL CA certs
|
|
||||||
mkdir -p $out/etc/ssl/certs
|
|
||||||
cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt
|
|
||||||
'';
|
|
||||||
|
|
||||||
cfg = config.dgn-runners;
|
|
||||||
|
|
||||||
storePaths = "nix-store --query -R ${builtins.toString (cfg.dependencies ++ [ pkgs.path ])}";
|
|
||||||
in {
|
|
||||||
options.dgn-runners = {
|
|
||||||
enable = mkEnableOption "Forgero Actions Runners.";
|
|
||||||
|
|
||||||
nbRunners = mkOption {
|
|
||||||
type = types.ints.positive;
|
|
||||||
default = 4;
|
|
||||||
};
|
|
||||||
|
|
||||||
storePath = mkOption {
|
|
||||||
type = types.str;
|
|
||||||
default = "/data/slow/nix";
|
|
||||||
};
|
|
||||||
|
|
||||||
dependencies = mkOption { type = with types; listOf package; };
|
|
||||||
};
|
|
||||||
|
|
||||||
config = lib.mkIf cfg.enable {
|
|
||||||
dgn-runners.dependencies = [
|
|
||||||
pkgs.bash
|
|
||||||
pkgs.colmena
|
|
||||||
pkgs.coreutils
|
|
||||||
pkgs.curl
|
|
||||||
pkgs.findutils
|
|
||||||
pkgs.gawk
|
|
||||||
pkgs.git
|
|
||||||
pkgs.gnugrep
|
|
||||||
pkgs.jq
|
|
||||||
pkgs.nix
|
|
||||||
pkgs.nodejs
|
|
||||||
pkgs.npins
|
|
||||||
pkgs.openssh
|
|
||||||
pkgs.tea
|
|
||||||
];
|
|
||||||
|
|
||||||
# everything here has no dependencies on the store
|
|
||||||
systemd.services = {
|
|
||||||
forgejo-nix-store = {
|
|
||||||
wantedBy = [ "multi-user.target" ];
|
|
||||||
path = [ pkgs.curl pkgs.nix ];
|
|
||||||
script = ''
|
|
||||||
dest=${cfg.storePath}
|
|
||||||
|
|
||||||
echo "[+] Performing an installation of Nix at $dest."
|
|
||||||
|
|
||||||
if ! [ -e "$dest" ]; then
|
|
||||||
echo "[+] Directory $dest does not exist, creating it."
|
|
||||||
mkdir -m 0755 $dest
|
|
||||||
fi
|
|
||||||
|
|
||||||
mkdir -p "$dest/store" "$dest/var/nix"
|
|
||||||
|
|
||||||
# Copy the required store paths
|
|
||||||
for i in $(${storePaths} | sed -e 's|/nix/store/||'); do
|
|
||||||
i_tmp="$dest/store/$i.$$"
|
|
||||||
if [ -e "$i_tmp" ]; then
|
|
||||||
rm -rf "$i_tmp"
|
|
||||||
fi
|
|
||||||
if ! [ -e "$dest/store/$i" ]; then
|
|
||||||
cp -RPp "/nix/store/$i" "$i_tmp"
|
|
||||||
chmod -R a-w "$i_tmp"
|
|
||||||
chmod +w "$i_tmp"
|
|
||||||
mv "$i_tmp" "$dest/store/$i"
|
|
||||||
chmod -w "$dest/store/$i"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
# Dump the corresponding db
|
|
||||||
nix-store --dump-db $(${storePaths}) | sed -e "s|/nix|$dest|" > .storeinfo
|
|
||||||
|
|
||||||
# Switch to the new store
|
|
||||||
export NIX_ROOT="$dest"
|
|
||||||
export NIX_STORE_DIR="$NIX_ROOT/store"
|
|
||||||
export NIX_DATA_DIR="$NIX_ROOT/share"
|
|
||||||
export NIX_STATE_DIR="$NIX_ROOT/var/nix"
|
|
||||||
|
|
||||||
nix-store --load-db < .storeinfo
|
|
||||||
|
|
||||||
# Setup the permissions to nixuser
|
|
||||||
chown -R nixuser:nixuser ${cfg.storePath}
|
|
||||||
'';
|
|
||||||
|
|
||||||
serviceConfig = {
|
|
||||||
RuntimeDirectory = "forgejo-store-nix";
|
|
||||||
WorkingDirectory = "/run/forgejo-store-nix";
|
|
||||||
Type = "oneshot";
|
|
||||||
RemainAfterExit = true;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
forgejo-runner-nix-image = {
|
|
||||||
wantedBy = [ "multi-user.target" ];
|
|
||||||
after = [ "podman.service" "forgejo-nix-store.service" ];
|
|
||||||
requires = [ "podman.service" "forgejo-nix-store.service" ];
|
|
||||||
path = [
|
|
||||||
config.virtualisation.podman.package
|
|
||||||
pkgs.gnutar
|
|
||||||
pkgs.shadow
|
|
||||||
pkgs.getent
|
|
||||||
];
|
|
||||||
|
|
||||||
# we also include etc here because the cleanup job also wants the nixuser to be present
|
|
||||||
script = ''
|
|
||||||
set -eux -o pipefail
|
|
||||||
|
|
||||||
mkdir -p etc/nix
|
|
||||||
|
|
||||||
# Create an unpriveleged user that we can use also without the run-as-user.sh script
|
|
||||||
touch etc/passwd etc/group
|
|
||||||
groupid=$(cut -d: -f3 < <(getent group nixuser))
|
|
||||||
userid=$(cut -d: -f3 < <(getent passwd nixuser))
|
|
||||||
groupadd --prefix $(pwd) --gid "$groupid" nixuser
|
|
||||||
emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.'
|
|
||||||
useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser
|
|
||||||
|
|
||||||
cat <<NIX_CONFIG > etc/nix/nix.conf
|
|
||||||
accept-flake-config = true
|
|
||||||
experimental-features = nix-command flakes
|
|
||||||
NIX_CONFIG
|
|
||||||
|
|
||||||
cat <<NSSWITCH > etc/nsswitch.conf
|
|
||||||
passwd: files mymachines systemd
|
|
||||||
group: files mymachines systemd
|
|
||||||
shadow: files
|
|
||||||
|
|
||||||
hosts: files mymachines dns myhostname
|
|
||||||
networks: files
|
|
||||||
|
|
||||||
ethers: files
|
|
||||||
services: files
|
|
||||||
protocols: files
|
|
||||||
rpc: files
|
|
||||||
NSSWITCH
|
|
||||||
|
|
||||||
# Link usr/bin/env
|
|
||||||
mkdir -p usr/bin
|
|
||||||
ln -s /bin/env usr/bin/env
|
|
||||||
|
|
||||||
# list the content as it will be imported into the container
|
|
||||||
tar -cv . | tar -tvf -
|
|
||||||
tar -cv . | podman import - forgejo-runner-nix
|
|
||||||
'';
|
|
||||||
|
|
||||||
serviceConfig = {
|
|
||||||
RuntimeDirectory = "forgejo-runner-nix-image";
|
|
||||||
WorkingDirectory = "/run/forgejo-runner-nix-image";
|
|
||||||
Type = "oneshot";
|
|
||||||
RemainAfterExit = true;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
} // lib.genAttrs
|
|
||||||
(builtins.genList (n: "forgejo-runner-nix${builtins.toString n}")
|
|
||||||
cfg.nbRunners) (name: {
|
|
||||||
after = [ "forgejo-runner-nix-image.service" ];
|
|
||||||
requires = [ "forgejo-runner-nix-image.service" ];
|
|
||||||
|
|
||||||
# TODO: systemd confinment
|
|
||||||
serviceConfig = {
|
|
||||||
# Hardening (may overlap with DynamicUser=)
|
|
||||||
# The following options are only for optimizing output of systemd-analyze
|
|
||||||
AmbientCapabilities = "";
|
|
||||||
CapabilityBoundingSet = "";
|
|
||||||
# ProtectClock= adds DeviceAllow=char-rtc r
|
|
||||||
DeviceAllow = "";
|
|
||||||
NoNewPrivileges = true;
|
|
||||||
PrivateDevices = true;
|
|
||||||
PrivateMounts = true;
|
|
||||||
PrivateTmp = true;
|
|
||||||
PrivateUsers = true;
|
|
||||||
ProtectClock = true;
|
|
||||||
ProtectControlGroups = true;
|
|
||||||
ProtectHome = true;
|
|
||||||
ProtectHostname = true;
|
|
||||||
ProtectKernelLogs = true;
|
|
||||||
ProtectKernelModules = true;
|
|
||||||
ProtectKernelTunables = true;
|
|
||||||
ProtectSystem = "strict";
|
|
||||||
RemoveIPC = true;
|
|
||||||
RestrictNamespaces = true;
|
|
||||||
RestrictRealtime = true;
|
|
||||||
RestrictSUIDSGID = true;
|
|
||||||
UMask = "0066";
|
|
||||||
ProtectProc = "invisible";
|
|
||||||
SystemCallFilter = [
|
|
||||||
"~@clock"
|
|
||||||
"~@cpu-emulation"
|
|
||||||
"~@module"
|
|
||||||
"~@mount"
|
|
||||||
"~@obsolete"
|
|
||||||
"~@raw-io"
|
|
||||||
"~@reboot"
|
|
||||||
"~@swap"
|
|
||||||
# needed by go?
|
|
||||||
#"~@resources"
|
|
||||||
"~@privileged"
|
|
||||||
"~capset"
|
|
||||||
"~setdomainname"
|
|
||||||
"~sethostname"
|
|
||||||
];
|
|
||||||
RestrictAddressFamilies =
|
|
||||||
[ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
|
|
||||||
|
|
||||||
# Needs network access
|
|
||||||
PrivateNetwork = false;
|
|
||||||
# Cannot be true due to Node
|
|
||||||
MemoryDenyWriteExecute = false;
|
|
||||||
|
|
||||||
# The more restrictive "pid" option makes `nix` commands in CI emit
|
|
||||||
# "GC Warning: Couldn't read /proc/stat"
|
|
||||||
# You may want to set this to "pid" if not using `nix` commands
|
|
||||||
ProcSubset = "all";
|
|
||||||
# Coverage programs for compiled code such as `cargo-tarpaulin` disable
|
|
||||||
# ASLR (address space layout randomization) which requires the
|
|
||||||
# `personality` syscall
|
|
||||||
# You may want to set this to `true` if not using coverage tooling on
|
|
||||||
# compiled code
|
|
||||||
LockPersonality = false;
|
|
||||||
|
|
||||||
# Note that this has some interactions with the User setting; so you may
|
|
||||||
# want to consult the systemd docs if using both.
|
|
||||||
DynamicUser = true;
|
|
||||||
};
|
|
||||||
});
|
|
||||||
|
|
||||||
users = {
|
|
||||||
users.nixuser = {
|
|
||||||
group = "nixuser";
|
|
||||||
description = "Used for running nix ci jobs";
|
|
||||||
home = "/var/empty";
|
|
||||||
isSystemUser = true;
|
|
||||||
};
|
|
||||||
groups.nixuser = { };
|
|
||||||
};
|
|
||||||
|
|
||||||
virtualisation = {
|
|
||||||
podman = {
|
|
||||||
enable = true;
|
|
||||||
|
|
||||||
defaultNetwork.settings = {
|
|
||||||
dns_enable = true;
|
|
||||||
ipv6_enabled = true;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
containers.storage.settings = {
|
|
||||||
storage = {
|
|
||||||
driver = "overlay";
|
|
||||||
graphroot = "/data/slow/containers/storage";
|
|
||||||
runroot = "/run/containers/storage";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
age-secrets.sources = [ ./. ];
|
|
||||||
|
|
||||||
services.gitea-actions-runner.instances = lib.genAttrs
|
|
||||||
(builtins.genList (n: "runner${builtins.toString n}") cfg.nbRunners)
|
|
||||||
(name: {
|
|
||||||
enable = true;
|
|
||||||
name = "nix-runner";
|
|
||||||
# take the git root url from the forgejo config
|
|
||||||
# only possible if you've also configured your forgejo though the same nix config
|
|
||||||
# otherwise you need to set it manually
|
|
||||||
url = "https://git.dgnum.eu";
|
|
||||||
# use your favourite nix secret manager to get a path for this
|
|
||||||
tokenFile = config.age.secrets."dgn_runners-token_file".path;
|
|
||||||
|
|
||||||
labels = [ "nix:docker://forgejo-runner-nix" ];
|
|
||||||
settings = {
|
|
||||||
container = {
|
|
||||||
options = builtins.toString [
|
|
||||||
"-e NIX_BUILD_SHELL=/bin/bash"
|
|
||||||
"-e PAGER=cat"
|
|
||||||
"-e PATH=/bin"
|
|
||||||
"-e NIX_PATH=nixpkgs=${builtins.toString pkgs.path}"
|
|
||||||
"--device /dev/kvm"
|
|
||||||
"-e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
|
|
||||||
"-v ${cfg.storePath}:/nix"
|
|
||||||
"-v ${storeDeps}/bin:/bin"
|
|
||||||
"-v ${storeDeps}/etc/ssl:/etc/ssl"
|
|
||||||
"--cpus=4"
|
|
||||||
"--device=/dev/kvm"
|
|
||||||
"--user nixuser"
|
|
||||||
];
|
|
||||||
|
|
||||||
# the default network that also respects our dns server settings
|
|
||||||
network = "host";
|
|
||||||
valid_volumes =
|
|
||||||
[ cfg.storePath "${storeDeps}/bin" "${storeDeps}/etc/ssl" ];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
});
|
|
||||||
};
|
|
||||||
}
|
|
|
@ -1,4 +0,0 @@
|
||||||
let
|
|
||||||
lib = import ../../lib { };
|
|
||||||
publicKeys = builtins.concatMap lib.getNodeKeys [ "storage01" ];
|
|
||||||
in lib.setDefault { inherit publicKeys; } [ "dgn_runners-token_file" ]
|
|
Loading…
Reference in a new issue