From dba014ed03f739728843c09a928796e7e381b95b Mon Sep 17 00:00:00 2001 From: Elias Coppens Date: Wed, 8 Jan 2025 00:41:01 +0100 Subject: [PATCH] feat(build01): Init --- .forgejo/workflows/eval-nodes.yaml | 11 +++ keys/default.nix | 1 + machines/nixos/build01/_configuration.nix | 23 ++++++ .../nixos/build01/_hardware-configuration.nix | 59 ++++++++++++++ machines/nixos/build01/nix-builder.nix | 78 +++++++++++++++++++ machines/nixos/build01/secrets/secrets.nix | 7 ++ machines/nixos/storage01/_configuration.nix | 6 ++ machines/nixos/storage01/forgejo-runners.nix | 39 +--------- meta/network.nix | 19 +++++ meta/nodes/nixos.nix | 19 +++++ meta/organization.nix | 4 + modules/nixos/default.nix | 1 + modules/nixos/dgn-forgejo-runners.nix | 78 +++++++++++++++++++ 13 files changed, 307 insertions(+), 38 deletions(-) create mode 100644 machines/nixos/build01/_configuration.nix create mode 100644 machines/nixos/build01/_hardware-configuration.nix create mode 100644 machines/nixos/build01/nix-builder.nix create mode 100644 machines/nixos/build01/secrets/secrets.nix create mode 100644 modules/nixos/dgn-forgejo-runners.nix diff --git a/.forgejo/workflows/eval-nodes.yaml b/.forgejo/workflows/eval-nodes.yaml index 452517e..b39bb0f 100644 --- a/.forgejo/workflows/eval-nodes.yaml +++ b/.forgejo/workflows/eval-nodes.yaml @@ -21,6 +21,17 @@ jobs: STORE_USER: admin name: Build and cache bridge01 run: nix-shell -A eval-nodes --run cache-node + build01: + runs-on: nix + steps: + - uses: actions/checkout@v3 + - env: + BUILD_NODE: build01 + STORE_ENDPOINT: https://tvix-store.dgnum.eu/infra-signing/ + STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }} + STORE_USER: admin + name: Build and cache build01 + run: nix-shell -A eval-nodes --run cache-node compute01: runs-on: nix steps: diff --git a/keys/default.nix b/keys/default.nix index 1a38900..5a1b797 100644 --- a/keys/default.nix +++ b/keys/default.nix @@ -20,6 +20,7 @@ rec { _keys = { # SSH keys of the nodes bridge01 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP5bS3iBXz8wycBnTvI5Qi79WLu0h4IVv/EOdKYbP5y7" ]; + build01 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIYJcEMQpOyKInqtd2/brnSQuzwgv6fNPlTSQx9tcvPu" ]; compute01 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE/YluSVS+4h3oV8CIUj0OmquyJXju8aEQy0Jz210vTu" ]; geo01 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEl6Pubbau+usQkemymoSKrTBbrX8JU5m5qpZbhNx8p4" ]; geo02 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFNXaCS0/Nsu5npqQk1TP6wMHCVIOaj4pblp2tIg6Ket" ]; diff --git a/machines/nixos/build01/_configuration.nix b/machines/nixos/build01/_configuration.nix new file mode 100644 index 0000000..9d269da --- /dev/null +++ b/machines/nixos/build01/_configuration.nix @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: 2025 Elias Coppens +# +# SPDX-License-Identifier: EUPL-1.2 + +{ lib, ... }: + +lib.extra.mkConfig { + enabledModules = [ + "dgn-forgejo-runners" + ]; + + enabledServices = [ + "nix-builder" + ]; + + extraConfig = { + dgn-forgejo-runners.nbRunners = 16; + + services.netbird.enable = true; + }; + + root = ./.; +} diff --git a/machines/nixos/build01/_hardware-configuration.nix b/machines/nixos/build01/_hardware-configuration.nix new file mode 100644 index 0000000..8a7c867 --- /dev/null +++ b/machines/nixos/build01/_hardware-configuration.nix @@ -0,0 +1,59 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ + config, + lib, + modulesPath, + ... +}: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ]; + + boot = { + initrd = { + availableKernelModules = [ + "xhci_pci" + "nvme" + "megaraid_sas" + "ehci_pci" + "ahci" + "usbhid" + "sd_mod" + ]; + kernelModules = [ "dm-snapshot" ]; + }; + kernelModules = [ "kvm-amd" ]; + extraModulePackages = [ ]; + }; + + fileSystems = { + "/" = { + device = "/dev/disk/by-uuid/fed99278-0916-4d9c-b974-c7125d3557b3"; + fsType = "xfs"; + }; + + "/data" = { + device = "/dev/disk/by-uuid/69b62f16-7db1-4720-a115-fd3b8dafe123"; + fsType = "xfs"; + }; + + "/boot" = { + device = "/dev/disk/by-uuid/1372-46EA"; + fsType = "vfat"; + options = [ + "fmask=0022" + "dmask=0022" + ]; + }; + }; + + swapDevices = [ + { device = "/dev/disk/by-uuid/34b9e0ab-c579-4293-849c-78f5093cf35a"; } + ]; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; +} diff --git a/machines/nixos/build01/nix-builder.nix b/machines/nixos/build01/nix-builder.nix new file mode 100644 index 0000000..071d241 --- /dev/null +++ b/machines/nixos/build01/nix-builder.nix @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: 2025 Elias Coppens +# +# SPDX-License-Identifier: EUPL-1.2 + +{ pkgs, lib, ... }: +let + org = import ../../../meta/organization.nix; + keys = (import ../../../keys/default.nix)._keys; +in +{ + config = { + users.users = builtins.listToAttrs ( + builtins.map (u: { + name = u; + value = { + isNormalUser = true; + home = "/home/${u}"; + openssh.authorizedKeys.keys = keys.${u}; + }; + }) org.groups.nix-builder + ); + + security.pam.loginLimits = [ + { + domain = "*"; + item = "nofile"; + type = "-"; + value = "20480"; + } + ]; + + systemd.services.nix-daemon.serviceConfig = { + MemoryAccounting = true; + MemoryMax = "450G"; + MemoryHigh = "440G"; + MemorySwapMax = "2G"; + ManagedOOMSwap = "kill"; + ManagedOOMMemoryPressure = "kill"; + MemoryPressureWatch = "on"; + }; + + nix = { + gc = { + automatic = true; + dates = lib.mkForce "*:45"; + options = lib.mkForce ''--max-freed "$((128 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"''; + + randomizedDelaySec = "1800"; + }; + + nrBuildUsers = 128; + + settings = { + keep-outputs = false; + keep-derivations = false; + use-cgroups = true; + http-connections = 0; + auto-allocate-uids = true; + cores = 0; + max-jobs = 8; # Do not build more than 2 derivations at once in the event, both of them are too big, yes this is stupid, fix it in Nix. + fsync-metadata = true; + system-features = [ + "benchmark" + "big-parallel" + "kvm" + "nixos-test" + ]; + experimental-features = [ + "auto-allocate-uids" + # "ca-derivations" this feature is really extremely broken. + "cgroups" + "fetch-closure" + "impure-derivations" + ]; + }; + }; + }; +} diff --git a/machines/nixos/build01/secrets/secrets.nix b/machines/nixos/build01/secrets/secrets.nix new file mode 100644 index 0000000..f200ecc --- /dev/null +++ b/machines/nixos/build01/secrets/secrets.nix @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2025 La Délégation Générale Numérique +# +# SPDX-License-Identifier: EUPL-1.2 + +(import ../../../../keys).mkSecrets [ "build01" ] [ + +] diff --git a/machines/nixos/storage01/_configuration.nix b/machines/nixos/storage01/_configuration.nix index 3c7cb32..5458120 100644 --- a/machines/nixos/storage01/_configuration.nix +++ b/machines/nixos/storage01/_configuration.nix @@ -9,6 +9,7 @@ lib.extra.mkConfig { # List of modules to enable "dgn-backups" "dgn-web" + "dgn-forgejo-runners" ]; enabledServices = [ @@ -27,6 +28,11 @@ lib.extra.mkConfig { extraConfig = { dgn-hardware.useZfs = true; + dgn-forgejo-runners = { + nbRunners = 6; + baseDataDir = "/data/slow"; + }; + services.netbird.enable = true; }; diff --git a/machines/nixos/storage01/forgejo-runners.nix b/machines/nixos/storage01/forgejo-runners.nix index fef5e6a..37b5a38 100644 --- a/machines/nixos/storage01/forgejo-runners.nix +++ b/machines/nixos/storage01/forgejo-runners.nix @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: EUPL-1.2 -{ config, pkgs, ... }: +_: let url = "https://git.dgnum.eu"; @@ -30,24 +30,6 @@ let }; in { - services.forgejo-nix-runners = { - enable = true; - - inherit url; - - storePath = "/data/slow"; - tokenFile = config.age.secrets."forgejo_runners-token_file".path; - - dependencies = [ - pkgs.npins - pkgs.tea - ]; - - containerOptions = [ "--cpus=4" ]; - - nbRunners = 6; - }; - services.gitea-actions-runner.instances = builtins.mapAttrs (_: mkRunner) { runner01 = { token = "qT9nZXKgLcb3fWOj7VTj3S58raiCWwF0weuIIKlY"; @@ -63,23 +45,4 @@ in labels = [ "debian-latest:docker://node:20-bookworm" ]; }; }; - - virtualisation = { - podman = { - enable = true; - - defaultNetwork.settings = { - dns_enable = true; - ipv6_enabled = true; - }; - }; - - containers.storage.settings = { - storage = { - driver = "overlay"; - graphroot = "/data/slow/containers/storage"; - runroot = "/run/containers/storage"; - }; - }; - }; } diff --git a/meta/network.nix b/meta/network.nix index 6980e44..d3eb943 100644 --- a/meta/network.nix +++ b/meta/network.nix @@ -13,6 +13,25 @@ netbirdIp = null; }; + build01 = { + interfaces = { + enp35s0f0np0 = { + ipv4 = [ + { + address = "10.0.254.21"; + prefixLength = 24; + } + ]; + + gateways = [ "10.0.254.1" ]; + enableDefaultDNS = true; + }; + }; + + hostId = "adb676ce"; + netbirdIp = "100.80.21.38"; + }; + compute01 = { interfaces = { eno1 = { diff --git a/meta/nodes/nixos.nix b/meta/nodes/nixos.nix index dea4288..a7eadde 100644 --- a/meta/nodes/nixos.nix +++ b/meta/nodes/nixos.nix @@ -49,6 +49,25 @@ }; }; + build01 = { + site = "pot01"; + + hashedPassword = "$y$j9T$n83qOn1OkQhFwQe50tPM11$jZ1tvgqMTcp4HLGEfJmTMsf0NnRUYQkzco9vibWTpU2"; + + stateVersion = "24.11"; + + nixpkgs = { + version = "24.11"; + system = "nixos"; + }; + + admins = [ "ecoppens" ]; + + deployment = { + targetHost = "build01.dgnum"; + }; + }; + compute01 = { site = "pav01"; diff --git a/meta/organization.nix b/meta/organization.nix index af2247b..46c6813 100644 --- a/meta/organization.nix +++ b/meta/organization.nix @@ -95,6 +95,10 @@ "catvayor" "ecoppens" ]; + + nix-builder = [ + "ecoppens" + ]; }; external = { diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix index cc1e316..4f23d74 100644 --- a/modules/nixos/default.nix +++ b/modules/nixos/default.nix @@ -21,6 +21,7 @@ "dgn-console" "dgn-chatops" "dgn-firewall" + "dgn-forgejo-runners" "dgn-hardware" "dgn-netbox-agent" "dgn-network" diff --git a/modules/nixos/dgn-forgejo-runners.nix b/modules/nixos/dgn-forgejo-runners.nix new file mode 100644 index 0000000..1ec251c --- /dev/null +++ b/modules/nixos/dgn-forgejo-runners.nix @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: 2024 Tom Hubrecht +# SPDX-FileCopyrightText: 2025 Elias Coppens +# +# SPDX-License-Identifier: EUPL-1.2 + +{ + config, + pkgs, + lib, + ... +}: + +let + url = "https://git.dgnum.eu"; + + inherit (lib) + mkEnableOption + mkOption + mkIf + types + ; + + cfg = config.dgn-forgejo-runners; +in +{ + options.dgn-forgejo-runners = { + enable = mkEnableOption "forgejo runners"; + + nbRunners = mkOption { + type = types.int; + }; + + baseDataDir = mkOption { + type = types.str; + default = "/data"; + }; + }; + + config = mkIf cfg.enable { + + services.forgejo-nix-runners = { + enable = true; + + inherit url; + + storePath = cfg.baseDataDir; + tokenFile = config.age.secrets."forgejo_runners-token_file".path; + + dependencies = [ + pkgs.npins + pkgs.tea + ]; + + containerOptions = [ "--cpus=4" ]; + + nbRunners = 6; + }; + + virtualisation = { + podman = { + enable = true; + + defaultNetwork.settings = { + dns_enable = true; + ipv6_enabled = true; + }; + }; + + containers.storage.settings = { + storage = { + driver = "overlay"; + graphroot = "${cfg.baseDataDir}/containers/storage"; + runroot = "/run/containers/storage"; + }; + }; + }; + }; +}