feat(build01): Init

.forgejo/workflows/eval-nodes.yaml

@@ -21,6 +21,17 @@ jobs:
         STORE_USER: admin
       name: Build and cache bridge01
       run: nix-shell -A eval-nodes --run cache-node
+  build01:
+    runs-on: nix
+    steps:
+    - uses: actions/checkout@v3
+    - env:
+        BUILD_NODE: build01
+        STORE_ENDPOINT: https://tvix-store.dgnum.eu/infra-signing/
+        STORE_PASSWORD: ${{ secrets.STORE_PASSWORD }}
+        STORE_USER: admin
+      name: Build and cache build01
+      run: nix-shell -A eval-nodes --run cache-node
   compute01:
     runs-on: nix
     steps:

keys/default.nix

@@ -20,6 +20,7 @@ rec {
   _keys = {
     # SSH keys of the nodes
     bridge01 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP5bS3iBXz8wycBnTvI5Qi79WLu0h4IVv/EOdKYbP5y7" ];
+    build01 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIYJcEMQpOyKInqtd2/brnSQuzwgv6fNPlTSQx9tcvPu" ];
     compute01 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE/YluSVS+4h3oV8CIUj0OmquyJXju8aEQy0Jz210vTu" ];
     geo01 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEl6Pubbau+usQkemymoSKrTBbrX8JU5m5qpZbhNx8p4" ];
     geo02 = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFNXaCS0/Nsu5npqQk1TP6wMHCVIOaj4pblp2tIg6Ket" ];

machines/nixos/build01/_configuration.nix

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: 2025 Elias Coppens <elias@dgnum.eu>
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+{ lib, ... }:
+
+lib.extra.mkConfig {
+  enabledModules = [
+    "forgejo-runners"
+  ];
+
+  enabledServices = [
+    "nix-builder"
+  ];
+
+  extraConfig = {
+    dgn-forgejo-runners.nbRunners = 16;
+
+    services.netbird.enable = true;
+  };
+
+  root = ./.;
+}

machines/nixos/build01/_hardware-configuration.nix

@@ -0,0 +1,59 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  boot = {
+    initrd = {
+      availableKernelModules = [
+        "xhci_pci"
+        "nvme"
+        "megaraid_sas"
+        "ehci_pci"
+        "ahci"
+        "usbhid"
+        "sd_mod"
+      ];
+      kernelModules = [ "dm-snapshot" ];
+    };
+    kernelModules = [ "kvm-amd" ];
+    extraModulePackages = [ ];
+  };
+
+  fileSystems = {
+    "/" = {
+      device = "/dev/disk/by-uuid/fed99278-0916-4d9c-b974-c7125d3557b3";
+      fsType = "xfs";
+    };
+
+    "/data" = {
+      device = "/dev/disk/by-uuid/69b62f16-7db1-4720-a115-fd3b8dafe123";
+      fsType = "xfs";
+    };
+
+    "/boot" = {
+      device = "/dev/disk/by-uuid/1372-46EA";
+      fsType = "vfat";
+      options = [
+        "fmask=0022"
+        "dmask=0022"
+      ];
+    };
+  };
+
+  swapDevices = [
+    { device = "/dev/disk/by-uuid/34b9e0ab-c579-4293-849c-78f5093cf35a"; }
+  ];
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

machines/nixos/build01/nix-builder.nix

@@ -0,0 +1,78 @@
+# SPDX-FileCopyrightText: 2025 Elias Coppens <elias@dgnum.eu>
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+{ pkgs, lib, ... }:
+let
+  org = import ../../../meta/organization.nix;
+  keys = (import ../../../keys/default.nix)._keys;
+in
+{
+  config = {
+    users.users = builtins.listToAttrs (
+      builtins.map (u: {
+        name = u;
+        value = {
+          isNormalUser = true;
+          home = "/home/${u}";
+          openssh.authorizedKeys.keys = keys.${u};
+        };
+      }) org.groups.nix-builder
+    );
+
+    security.pam.loginLimits = [
+      {
+        domain = "*";
+        item = "nofile";
+        type = "-";
+        value = "20480";
+      }
+    ];
+
+    systemd.services.nix-daemon.serviceConfig = {
+      MemoryAccounting = true;
+      MemoryMax = "450G";
+      MemoryHigh = "440G";
+      MemorySwapMax = "2G";
+      ManagedOOMSwap = "kill";
+      ManagedOOMMemoryPressure = "kill";
+      MemoryPressureWatch = "on";
+    };
+
+    nix = {
+      gc = {
+        automatic = true;
+        dates = lib.mkForce "*:45";
+        options = lib.mkForce ''--max-freed "$((128 * 1024**3 - 1024 * $(df -P -k /nix/store | tail -n 1 | ${pkgs.gawk}/bin/awk '{ print $4 }')))"'';
+
+        randomizedDelaySec = "1800";
+      };
+
+      nrBuildUsers = 128;
+
+      settings = {
+        keep-outputs = false;
+        keep-derivations = false;
+        use-cgroups = true;
+        http-connections = 0;
+        auto-allocate-uids = true;
+        cores = 0;
+        max-jobs = 8; # Do not build more than 2 derivations at once in the event, both of them are too big, yes this is stupid, fix it in Nix.
+        fsync-metadata = true;
+        system-features = [
+          "benchmark"
+          "big-parallel"
+          "kvm"
+          "nixos-test"
+        ];
+        experimental-features = [
+          "auto-allocate-uids"
+          # "ca-derivations" this feature is really extremely broken.
+          "cgroups"
+          "fetch-closure"
+          "impure-derivations"
+        ];
+      };
+    };
+  };
+}

machines/nixos/build01/secrets/secrets.nix

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2025 La Délégation Générale Numérique <contact@dgnum.eu>
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+(import ../../../../keys).mkSecrets [ "build01" ] [
+
+]

machines/nixos/storage01/_configuration.nix

@@ -9,13 +9,13 @@ lib.extra.mkConfig {
     # List of modules to enable
     "dgn-backups"
     "dgn-web"
+    "dgn-forgejo-runners"
   ];
 
   enabledServices = [
     # List of services to enable
     "tvix-cache"
     "forgejo"
-    "forgejo-runners"
     "garage"
     "influxdb"
     "netbird"
@@ -27,6 +27,12 @@ lib.extra.mkConfig {
   extraConfig = {
     dgn-hardware.useZfs = true;
 
+    dgn-forgejo-runners = {
+      nbRunners = 6;
+      baseDataDir = "/data/slow";
+      tokens = [ "qT9nZXKgLcb3fWOj7VTj3S58raiCWwF0weuIIKlY" "m0rUDBxxkMb6xLTmPFwIieHswJFROqPFByPlT3V9" ];
+    };
+
     services.netbird.enable = true;
   };
 

meta/network.nix

@@ -13,6 +13,25 @@
     netbirdIp = null;
   };
 
+  build01 = {
+    interfaces = {
+      enp35s0f0np0 = {
+        ipv4 = [
+          {
+            address = "10.0.254.21";
+            prefixLength = 24;
+          }
+        ];
+
+        gateways = [ "10.0.254.1" ];
+        enableDefaultDNS = true;
+      };
+    };
+
+    hostId = "adb676ce";
+    netbirdIp = "100.80.21.38";
+  };
+
   compute01 = {
     interfaces = {
       eno1 = {

meta/nodes/nixos.nix

@@ -49,6 +49,25 @@
     };
   };
 
+  build01 = {
+    site = "pot01";
+
+    hashedPassword = "$y$j9T$n83qOn1OkQhFwQe50tPM11$jZ1tvgqMTcp4HLGEfJmTMsf0NnRUYQkzco9vibWTpU2";
+
+    stateVersion = "24.11";
+
+    nixpkgs = {
+      version = "24.11";
+      system = "nixos";
+    };
+
+    admins = [ "ecoppens" ];
+
+    deployment = {
+      targetHost = "build01.dgnum";
+    };
+  };
+
   compute01 = {
     site = "pav01";
 

meta/organization.nix

@@ -95,6 +95,10 @@
       "catvayor"
       "ecoppens"
     ];
+
+    nix-builder = [
+      "ecoppens"
+    ];
   };
 
   external = {

modules/nixos/default.nix

@@ -21,6 +21,7 @@
       "dgn-console"
       "dgn-chatops"
       "dgn-firewall"
+      "dgn-forgejo-runners"
       "dgn-hardware"
       "dgn-netbox-agent"
       "dgn-network"

modules/nixos/dgn-forgejo-runners.nix

@@ -1,12 +1,15 @@
 # SPDX-FileCopyrightText: 2024 Tom Hubrecht <tom.hubrecht@dgnum.eu>
+# SPDX-FileCopyrightText: 2025 Elias Coppens <elias@dgnum.eu>
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-{ config, pkgs, ... }:
+{ config, pkgs, lib, name, ... }:
 
 let
   url = "https://git.dgnum.eu";
 
+  inherit (lib) mkEnableOption mkOption mkIf types;
+
   mkRunner =
     {
       labels,
@@ -28,14 +31,36 @@ let
         options = "--cpus=4";
       };
     };
+
+  cfg = config.dgn-forgejo-runners;
 in
 {
+  options.dgn-forgejo-runners = {
+    enable = mkEnableOption "forgejo runners";
+
+    nbRunners = mkOption {
+      type = types.int;
+    };
+
+    baseDataDir = mkOption {
+      type = types.str;
+      default = "/data";
+    };
+
+    tokens = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+    };
+  };
+
+  config = mkIf cfg.enable {
+
   services.forgejo-nix-runners = {
     enable = true;
 
     inherit url;
 
-    storePath = "/data/slow";
+    storePath = cfg.baseDataDir;
     tokenFile = config.age.secrets."forgejo_runners-token_file".path;
 
     dependencies = [
@@ -48,21 +73,14 @@ in
     nbRunners = 6;
   };
 
-  services.gitea-actions-runner.instances = builtins.mapAttrs (_: mkRunner) {
-    runner01 = {
-      token = "qT9nZXKgLcb3fWOj7VTj3S58raiCWwF0weuIIKlY";
-      name = "storage01 [debian]";
-
+  services.gitea-actions-runner.instances = builtins.mapAttrs (_: mkRunner) (builtins.listToAttrs (lib.imap (i: token: {
+    name = "runner${lib.strings.fixedWidthNumber 2 i}";
+    value = {
+      inherit token;
+      name = "${name} [debian]";
       labels = [ "debian-latest:docker://node:20-bookworm" ];
     };
-
-    runner02 = {
-      token = "m0rUDBxxkMb6xLTmPFwIieHswJFROqPFByPlT3V9";
-      name = "storage01 [debian]";
-
-      labels = [ "debian-latest:docker://node:20-bookworm" ];
-    };
-  };
+  }) cfg.tokens));
 
   virtualisation = {
     podman = {
@@ -77,9 +95,10 @@ in
     containers.storage.settings = {
       storage = {
         driver = "overlay";
-        graphroot = "/data/slow/containers/storage";
+        graphroot = "${cfg.baseDataDir}/containers/storage";
         runroot = "/run/containers/storage";
       };
     };
   };
+};
 }