feat(dgn-runners): Init module for runners with a store
This commit is contained in:
Tom Hubrecht
parent
351ea9c90a
commit
8cef4a594a
4 changed files with 362 additions and 0 deletions
|
|
@ -43,6 +43,7 @@
|
|||
"dgn-fail2ban"
|
||||
"dgn-hardware"
|
||||
"dgn-network"
|
||||
"dgn-runners"
|
||||
"dgn-secrets"
|
||||
"dgn-ssh"
|
||||
"dgn-web"
|
||||
|
|
|
|||
331
modules/dgn-runners/default.nix
Normal file
331
modules/dgn-runners/default.nix
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
/* Copyright 2023 Clan contributers, Tom-Hubrecht
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to
|
||||
deal in the Software without restriction, including without limitation the
|
||||
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
sell copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
*/
|
||||
|
||||
{ config, pkgs, lib, ... }:
|
||||
|
||||
let
|
||||
inherit (lib) mkEnableOption mkOption types;
|
||||
|
||||
storeDeps = pkgs.runCommand "store-deps" { } ''
|
||||
mkdir -p $out/bin
|
||||
for dir in ${builtins.toString cfg.dependencies}; do
|
||||
for bin in "$dir"/bin/*; do
|
||||
ln -s "$bin" "$out/bin/$(basename "$bin")"
|
||||
done
|
||||
done
|
||||
|
||||
# Add SSL CA certs
|
||||
mkdir -p $out/etc/ssl/certs
|
||||
cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt
|
||||
'';
|
||||
|
||||
cfg = config.dgn-runners;
|
||||
|
||||
storePaths = "nix-store --query -R ${builtins.toString cfg.dependencies}";
|
||||
in {
|
||||
options.dgn-runners = {
|
||||
enable = mkEnableOption "Forgero Actions Runners.";
|
||||
|
||||
nbRunners = mkOption {
|
||||
type = types.ints.positive;
|
||||
default = 4;
|
||||
};
|
||||
|
||||
storePath = mkOption {
|
||||
type = types.str;
|
||||
default = "/data/slow/nix";
|
||||
};
|
||||
|
||||
dependencies = mkOption { type = with types; listOf package; };
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
dgn-runners.dependencies = [
|
||||
pkgs.coreutils
|
||||
pkgs.findutils
|
||||
pkgs.gnugrep
|
||||
pkgs.gawk
|
||||
pkgs.git
|
||||
pkgs.nix
|
||||
pkgs.bash
|
||||
pkgs.jq
|
||||
pkgs.nodejs
|
||||
pkgs.npins
|
||||
pkgs.colmena
|
||||
pkgs.tea
|
||||
];
|
||||
|
||||
# everything here has no dependencies on the store
|
||||
systemd.services = {
|
||||
forgejo-nix-store = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
path = [ pkgs.curl pkgs.nix ];
|
||||
script = ''
|
||||
dest=${cfg.storePath}
|
||||
|
||||
echo "[+] Performing an installation of Nix at $dest."
|
||||
|
||||
if ! [ -e "$dest" ]; then
|
||||
echo "[+] Directory $dest does not exist, creating it."
|
||||
mkdir -m 0755 $dest
|
||||
fi
|
||||
|
||||
mkdir -p "$dest/store" "$dest/var/nix"
|
||||
|
||||
# Copy the required store paths
|
||||
for i in $(${storePaths} | sed -e 's|/nix/store/||'); do
|
||||
i_tmp="$dest/store/$i.$$"
|
||||
if [ -e "$i_tmp" ]; then
|
||||
rm -rf "$i_tmp"
|
||||
fi
|
||||
if ! [ -e "$dest/store/$i" ]; then
|
||||
cp -RPp "/nix/store/$i" "$i_tmp"
|
||||
chmod -R a-w "$i_tmp"
|
||||
chmod +w "$i_tmp"
|
||||
mv "$i_tmp" "$dest/store/$i"
|
||||
chmod -w "$dest/store/$i"
|
||||
fi
|
||||
done
|
||||
|
||||
# Dump the corresponding db
|
||||
nix-store --dump-db $(${storePaths}) | sed -e "s|/nix|$dest|" > .storeinfo
|
||||
|
||||
# Switch to the new store
|
||||
export NIX_ROOT="$dest"
|
||||
export NIX_STORE_DIR="$NIX_ROOT/store"
|
||||
export NIX_DATA_DIR="$NIX_ROOT/share"
|
||||
export NIX_STATE_DIR="$NIX_ROOT/var/nix"
|
||||
|
||||
nix-store --load-db < .storeinfo
|
||||
|
||||
# Setup the permissions to nixuser
|
||||
chown -R nixuser:nixuser ${cfg.storePath}
|
||||
'';
|
||||
|
||||
serviceConfig = {
|
||||
RuntimeDirectory = "forgejo-store-nix";
|
||||
WorkingDirectory = "/run/forgejo-store-nix";
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
};
|
||||
|
||||
forgejo-runner-nix-image = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "podman.service" "forgejo-nix-store.service" ];
|
||||
requires = [ "podman.service" "forgejo-nix-store.service" ];
|
||||
path = [
|
||||
config.virtualisation.podman.package
|
||||
pkgs.gnutar
|
||||
pkgs.shadow
|
||||
pkgs.getent
|
||||
];
|
||||
|
||||
# we also include etc here because the cleanup job also wants the nixuser to be present
|
||||
script = ''
|
||||
set -eux -o pipefail
|
||||
|
||||
mkdir -p etc/nix
|
||||
|
||||
# Create an unpriveleged user that we can use also without the run-as-user.sh script
|
||||
touch etc/passwd etc/group
|
||||
groupid=$(cut -d: -f3 < <(getent group nixuser))
|
||||
userid=$(cut -d: -f3 < <(getent passwd nixuser))
|
||||
groupadd --prefix $(pwd) --gid "$groupid" nixuser
|
||||
emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.'
|
||||
useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser
|
||||
|
||||
cat <<NIX_CONFIG > etc/nix/nix.conf
|
||||
accept-flake-config = true
|
||||
experimental-features = nix-command flakes
|
||||
NIX_CONFIG
|
||||
|
||||
cat <<NSSWITCH > etc/nsswitch.conf
|
||||
passwd: files mymachines systemd
|
||||
group: files mymachines systemd
|
||||
shadow: files
|
||||
|
||||
hosts: files mymachines dns myhostname
|
||||
networks: files
|
||||
|
||||
ethers: files
|
||||
services: files
|
||||
protocols: files
|
||||
rpc: files
|
||||
NSSWITCH
|
||||
|
||||
# list the content as it will be imported into the container
|
||||
tar -cv . | tar -tvf -
|
||||
tar -cv . | podman import - forgejo-runner-nix
|
||||
'';
|
||||
|
||||
serviceConfig = {
|
||||
RuntimeDirectory = "forgejo-runner-nix-image";
|
||||
WorkingDirectory = "/run/forgejo-runner-nix-image";
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
};
|
||||
} // lib.genAttrs
|
||||
(builtins.genList (n: "forgejo-runner-nix${builtins.toString n}")
|
||||
cfg.nbRunners) (name: {
|
||||
after = [ "forgejo-runner-nix-image.service" ];
|
||||
requires = [ "forgejo-runner-nix-image.service" ];
|
||||
|
||||
# TODO: systemd confinment
|
||||
serviceConfig = {
|
||||
# Hardening (may overlap with DynamicUser=)
|
||||
# The following options are only for optimizing output of systemd-analyze
|
||||
AmbientCapabilities = "";
|
||||
CapabilityBoundingSet = "";
|
||||
# ProtectClock= adds DeviceAllow=char-rtc r
|
||||
DeviceAllow = "";
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateMounts = true;
|
||||
PrivateTmp = true;
|
||||
PrivateUsers = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHome = true;
|
||||
ProtectHostname = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectSystem = "strict";
|
||||
RemoveIPC = true;
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
UMask = "0066";
|
||||
ProtectProc = "invisible";
|
||||
SystemCallFilter = [
|
||||
"~@clock"
|
||||
"~@cpu-emulation"
|
||||
"~@module"
|
||||
"~@mount"
|
||||
"~@obsolete"
|
||||
"~@raw-io"
|
||||
"~@reboot"
|
||||
"~@swap"
|
||||
# needed by go?
|
||||
#"~@resources"
|
||||
"~@privileged"
|
||||
"~capset"
|
||||
"~setdomainname"
|
||||
"~sethostname"
|
||||
];
|
||||
RestrictAddressFamilies =
|
||||
[ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
|
||||
|
||||
# Needs network access
|
||||
PrivateNetwork = false;
|
||||
# Cannot be true due to Node
|
||||
MemoryDenyWriteExecute = false;
|
||||
|
||||
# The more restrictive "pid" option makes `nix` commands in CI emit
|
||||
# "GC Warning: Couldn't read /proc/stat"
|
||||
# You may want to set this to "pid" if not using `nix` commands
|
||||
ProcSubset = "all";
|
||||
# Coverage programs for compiled code such as `cargo-tarpaulin` disable
|
||||
# ASLR (address space layout randomization) which requires the
|
||||
# `personality` syscall
|
||||
# You may want to set this to `true` if not using coverage tooling on
|
||||
# compiled code
|
||||
LockPersonality = false;
|
||||
|
||||
# Note that this has some interactions with the User setting; so you may
|
||||
# want to consult the systemd docs if using both.
|
||||
DynamicUser = true;
|
||||
};
|
||||
});
|
||||
|
||||
users = {
|
||||
users.nixuser = {
|
||||
group = "nixuser";
|
||||
description = "Used for running nix ci jobs";
|
||||
home = "/var/empty";
|
||||
isSystemUser = true;
|
||||
};
|
||||
groups.nixuser = { };
|
||||
};
|
||||
|
||||
virtualisation = {
|
||||
podman = {
|
||||
enable = true;
|
||||
|
||||
defaultNetwork.settings = {
|
||||
dns_enable = true;
|
||||
ipv6_enabled = true;
|
||||
};
|
||||
|
||||
extraPackages = [ pkgs.zfs_2_1 ];
|
||||
};
|
||||
|
||||
containers.storage.settings = {
|
||||
storage = {
|
||||
driver = "overlay";
|
||||
graphroot = "/data/slow/containers/storage";
|
||||
runroot = "/run/containers/storage";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
dgn-secrets.sources = [ ./. ];
|
||||
|
||||
services.gitea-actions-runner.instances = lib.genAttrs
|
||||
(builtins.genList (n: "runner${builtins.toString n}") cfg.nbRunners)
|
||||
(name: {
|
||||
enable = true;
|
||||
name = "nix-runner";
|
||||
# take the git root url from the forgejo config
|
||||
# only possible if you've also configured your forgejo though the same nix config
|
||||
# otherwise you need to set it manually
|
||||
url = "https://git.dgnum.eu";
|
||||
# use your favourite nix secret manager to get a path for this
|
||||
tokenFile = config.age.secrets."dgn_runners-token_file".path;
|
||||
|
||||
labels = [ "nix:docker://forgejo-runner-nix" ];
|
||||
settings = {
|
||||
container = {
|
||||
options = builtins.toString [
|
||||
"-e NIX_BUILD_SHELL=/bin/bash"
|
||||
"-e PAGER=cat"
|
||||
"-e PATH=/bin"
|
||||
"--device /dev/kvm"
|
||||
"-e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
|
||||
"-v ${cfg.storePath}:/nix"
|
||||
"-v ${storeDeps}/bin:/bin"
|
||||
"-v ${storeDeps}/etc/ssl:/etc/ssl"
|
||||
"--cpus=4"
|
||||
"--device=/dev/kvm"
|
||||
"--user nixuser"
|
||||
];
|
||||
|
||||
# the default network that also respects our dns server settings
|
||||
network = "host";
|
||||
valid_volumes =
|
||||
[ cfg.storePath "${storeDeps}/bin" "${storeDeps}/etc/ssl" ];
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
}
|
||||
26
modules/dgn-runners/dgn_runners-token_file
Normal file
26
modules/dgn-runners/dgn_runners-token_file
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
age-encryption.org/v1
|
||||
-> ssh-ed25519 rHotTw Y+aFwzGPr7mQUUfN2UHgvUdvLHYFJ5hsDMPZdYTVNQI
|
||||
yvJQ7dhxQ5hjxIItNMlZsnsPrVYB7QchFDE5Hi6zmek
|
||||
-> ssh-ed25519 jIXfPA Chc4vkshYdpc1zx9eLjssl+kGU2SJUeVlRgbo1n7+FI
|
||||
a+YJDGndBbmpkZOYwdezZKNYdyMdGW3geasVlmuqtTc
|
||||
-> ssh-ed25519 QlRB9Q Q5NBD14wdPQ3pEuzN7HByVkEsXluMyj2aKxCYqkViAw
|
||||
AeGzeWAUFift6LQcsOjXv8/JEDSKyUqlxctA6p0Hx+k
|
||||
-> ssh-ed25519 r+nK/Q Fe1nO9Z1iUODgfnaXfrcSRJO9IwzbuHF6pCyQgufNEo
|
||||
0TzwgI0NvoIHEa0mlrREgpYLZTk7SHj5dnCrJgLfyd4
|
||||
-> ssh-rsa krWCLQ
|
||||
dCOwZM/pdz8Iy0U2YdXChaUHOcyLyqabwnNrkYEV3tzs4d9aUp21Kb/jbTuW009a
|
||||
cNuJa/sOb2ORsfA+fh2lKvjwF7zfpli9bhheWnXFr8c07trfr9N/JRwaHEqJ8c3u
|
||||
x9hv8V5I0JaF76VaYY7sMImkyN65RXL3UNSlVsFfweuYrRAIpkq1yrad6+y+Aldy
|
||||
bnQWUGOQI2DpPOJeeCIECVNUhi0XOqFhMmP2UG5kf9cWLyY/7XYOzJa0SrvCqY/0
|
||||
udZjeGW20ZCa/NxHk4jq6Irlt9UI9OG24ti+kPxG31UcydVmFZhAMpzdzPHufk2F
|
||||
/RB+/k6g1O9Pr5GSUGmhvg
|
||||
-> ssh-ed25519 /vwQcQ SC5Y6TkLhhzwk7KgJ2njEUKLhVNSWsPqxLxY6HyemSU
|
||||
z/7whFueyYe1AWRgLc8Id3JlgZoQWQm/H2IRTpTXM5o
|
||||
-> ssh-ed25519 0R97PA cteDe++6zrpQvgnhtN/O7rqgjInRjZzEhEfmzjnI0HU
|
||||
gUmoNF92IHaXGDiV3BUlQB8O3sI5dJEbnoVsYKjJqBY
|
||||
-> ssh-ed25519 JGx7Ng mdk80U/UDMrzovesBvCJtjBEDDgvd+6gTBOA5qkwdQg
|
||||
UdUNgyA5FFIK+JNLhZFdqtiwpXkMxD/ZSU+EDMPyGeM
|
||||
-> cA-grease
|
||||
U5RhIlwzvTftwV0glkG/5w
|
||||
--- Bn6gp63oOQINTJnX71K6OQ1NryhemeHYPXi4zhDSM5c
|
||||
%„yöŽh_b4ö@j<>”ø ѵ—ÇïÇ…åõ“WdÚã׆dTï0Öœ$åAÌê®7ÝfF¢`µ÷™… \ N~;É÷°NŽÜ G¾Ì
|
||||
4
modules/dgn-runners/secrets.nix
Normal file
4
modules/dgn-runners/secrets.nix
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
let
|
||||
lib = import ../../lib { };
|
||||
publicKeys = builtins.concatMap lib.getNodeKeys [ "storage01" ];
|
||||
in lib.setDefault { inherit publicKeys; } [ "dgn_runners-token_file" ]
|
||||
Loading…
Reference in a new issue