From 344cc26d8ef35f04737c9457a9f29b63b65b1db7 Mon Sep 17 00:00:00 2001 From: Ryan Lahfa Date: Sat, 7 Dec 2024 13:03:35 +0100 Subject: [PATCH] feat(meta/*): support generalized deployments Our colmena knows how to deal with families of system types, e.g. NixOS systems or Liminix-based systems. This is a step 1 towards supporting our APs in our infrastructure. Signed-off-by: Ryan Lahfa --- default.nix | 5 +- hive.nix | 109 +++++++++++++------- lib/nix-lib/default.nix | 1 + lib/nix-lib/nixpkgs.nix | 4 +- machines/compute01/signal-irc-bridge.nix | 4 +- machines/compute01/stirling-pdf/default.nix | 2 +- machines/storage01/forgejo.nix | 2 +- machines/storage01/netbird.nix | 2 +- machines/web01/netbox.nix | 2 +- meta/nixpkgs.nix | 29 ++++-- meta/nodes.nix | 16 +-- meta/options.nix | 24 +++-- npins/sources.json | 11 ++ 13 files changed, 145 insertions(+), 66 deletions(-) diff --git a/default.nix b/default.nix index fb1fa57..c70bcce 100644 --- a/default.nix +++ b/default.nix @@ -37,7 +37,6 @@ { sources ? import ./npins, pkgs ? import sources.nixpkgs { }, - nix-pkgs ? import sources.nix-pkgs { inherit pkgs; }, }: let @@ -112,7 +111,9 @@ in })) pkgs.npins - (pkgs.callPackage ./lib/colmena { inherit (nix-pkgs) colmena; }) + (pkgs.callPackage ./lib/colmena { + colmena = pkgs.callPackage "${sources.colmena}/package.nix" { }; + }) (pkgs.callPackage "${sources.agenix}/pkgs/agenix.nix" { }) (pkgs.callPackage "${sources.lon}/nix/packages/lon.nix" { }) ] ++ (builtins.attrValues scripts); diff --git a/hive.nix b/hive.nix index ec85d59..6567a24 100644 --- a/hive.nix +++ b/hive.nix @@ -6,6 +6,7 @@ let .applyPatches' sources'; nix-lib = import ./lib/nix-lib; + inherit (nix-lib) warn; patch = import ./lib/nix-patches { patchFile = ./patches; }; @@ -15,15 +16,23 @@ let mkNode = node: { # Import the base configuration for each node imports = [ ./machines/${node}/_configuration.nix ]; + + deployment.systemType = "nixos"; }; nixpkgs' = import ./meta/nixpkgs.nix; - # All supported nixpkgs versions, instanciated - nixpkgs = nix-lib.mapSingleFuse mkNixpkgs nixpkgs'.supported; + # All supported nixpkgs versions × systems, instanciated + nixpkgs = nix-lib.mapSingleFuse ( + s: nix-lib.mapSingleFuse (mkSystemNixpkgs s) nixpkgs'.versions.supported + ) nixpkgs'.systems.supported; # Get the configured nixos version for the node, # defaulting to the one defined in meta/nixpkgs - version = node: nodes'.${node}.nixpkgs or nixpkgs'.default; + version = node: nodes'.${node}.nixpkgs.version or nixpkgs'.versions.default; + systemType = + node: + nodes'.${node}.nixpkgs.system + or (warn "${node}: Not specifying the `deployment.systemType` is deprecated!" "nixos"); # Builds a patched version of nixpkgs, only as the source mkNixpkgs' = @@ -33,17 +42,42 @@ let name = "nixos-${v}"; }; + # Build up the nixpkgs configuration for Liminix embedded systems + mkLiminixConfig = + system: _: + (import "${sources.liminix}/devices/${system}").system + // { + overlays = [ (import "${sources.liminix}/overlay.nix") ]; + config = { + allowUnsupportedSystem = true; # mipsel + permittedInsecurePackages = [ + "python-2.7.18.8" # Python < 3.x is needed for kernel backports. + ]; + }; + }; + + # Build up the arguments to instantiate a nixpkgs given a system and a version. + mkNixpkgsConfig = + system: version: + if system == "nixos" then + { } + else if system == "zyxel-nwa50ax" then + (mkLiminixConfig system version) + else + (throw "Unknown system: ${system} for nixpkgs configuration instantiation"); + # Instanciates the required nixpkgs version - mkNixpkgs = version: import (mkNixpkgs' version) { }; + mkSystemNixpkgs = system: version: import (mkNixpkgs' version) (mkNixpkgsConfig system version); ### # Function to create arguments based on the node # mkArgs = node: rec { - lib = nixpkgs.${version node}.lib // { + lib = nixpkgs.${systemType node}.${version node}.lib // { extra = nix-lib; }; + sourcePkgs = nixpkgs.${systemType node}.${version node}; meta = (import ./meta) lib; nodeMeta = meta.nodes.${node}; @@ -52,7 +86,7 @@ in { meta = { - nodeNixpkgs = nix-lib.mapSingleFuse (n: nixpkgs.${version n}) nodes; + nodeNixpkgs = nix-lib.mapSingleFuse (n: nixpkgs.${systemType n}.${version n}) nodes; specialArgs = { inherit nixpkgs sources; @@ -63,40 +97,45 @@ in nodeSpecialArgs = nix-lib.mapSingleFuse mkArgs nodes; }; - defaults = - { name, nodeMeta, ... }: - { - # Import the default modules - imports = [ - ./modules - (import "${sources.lix-module}/module.nix" { inherit (sources) lix; }) - ]; + registry = { + nixos = { + evalConfig = args: import "${args.specialArgs.sourcePkgs.path}/nixos/lib/eval-config.nix" args; + defaults = + { name, nodeMeta, ... }: + { + # Import the default modules + imports = [ + ./modules + (import "${sources.lix-module}/module.nix" { inherit (sources) lix; }) + ]; - # Include default secrets - age-secrets.sources = [ ./machines/${name}/secrets ]; + # Include default secrets + age-secrets.sources = [ ./machines/${name}/secrets ]; - # Deployment config is specified in meta.nodes.${node}.deployment - inherit (nodeMeta) deployment; + # Deployment config is specified in meta.nodes.${node}.deployment + inherit (nodeMeta) deployment; - nix = { - # Set NIX_PATH to the patched version of nixpkgs - nixPath = [ "nixpkgs=${mkNixpkgs' (version name)}" ]; - optimise.automatic = true; + nix = { + # Set NIX_PATH to the patched version of nixpkgs + nixPath = [ "nixpkgs=${mkNixpkgs' (version name)}" ]; + optimise.automatic = true; - gc = { - automatic = true; - dates = "weekly"; - options = "--delete-older-than 7d"; + gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 7d"; + }; + }; + + # Allow unfree packages + nixpkgs.config.allowUnfree = true; + + # Use the stateVersion declared in the metadata + system = { + inherit (nodeMeta) stateVersion; + }; }; - }; - - # Allow unfree packages - nixpkgs.config.allowUnfree = true; - - # Use the stateVersion declared in the metadata - system = { - inherit (nodeMeta) stateVersion; - }; }; + }; } // (nix-lib.mapSingleFuse mkNode nodes) diff --git a/lib/nix-lib/default.nix b/lib/nix-lib/default.nix index b4d480a..6b40b68 100644 --- a/lib/nix-lib/default.nix +++ b/lib/nix-lib/default.nix @@ -42,6 +42,7 @@ rec { recursiveUpdate splitString unique + warn ; /* diff --git a/lib/nix-lib/nixpkgs.nix b/lib/nix-lib/nixpkgs.nix index ff04b2e..f2670e2 100644 --- a/lib/nix-lib/nixpkgs.nix +++ b/lib/nix-lib/nixpkgs.nix @@ -453,10 +453,10 @@ rec { # `builtins.warn` requires a string message, so we enforce that in our implementation, so that callers aren't accidentally incompatible with newer Nix versions. assert builtins.isString msg; if mustAbort then - builtins.trace "␛[1;31mevaluation warning:␛[0m ${msg}" ( + builtins.trace "evaluation warning: ${msg}" ( abort "NIX_ABORT_ON_WARN=true; warnings are treated as unrecoverable errors." ) else - builtins.trace "␛[1;35mevaluation warning:␛[0m ${msg}" v + builtins.trace "evaluation warning: ${msg}" v ); } diff --git a/machines/compute01/signal-irc-bridge.nix b/machines/compute01/signal-irc-bridge.nix index 5d68179..1ec094e 100644 --- a/machines/compute01/signal-irc-bridge.nix +++ b/machines/compute01/signal-irc-bridge.nix @@ -9,7 +9,9 @@ services.signal-irc-bridge = { enable = true; - package = nixpkgs.unstable.callPackage (sources.signal-irc-bridge.outPath + "/package.nix") { }; + package = nixpkgs.nixos.unstable.callPackage ( + sources.signal-irc-bridge.outPath + "/package.nix" + ) { }; configFile = config.age.secrets."signal-irc-bridge-config".path; }; diff --git a/machines/compute01/stirling-pdf/default.nix b/machines/compute01/stirling-pdf/default.nix index ae9cf27..0904422 100644 --- a/machines/compute01/stirling-pdf/default.nix +++ b/machines/compute01/stirling-pdf/default.nix @@ -20,7 +20,7 @@ in services.stirling-pdf = { enable = true; - package = nixpkgs.unstable.stirling-pdf.overrideAttrs (old: { + package = nixpkgs.nixos.unstable.stirling-pdf.overrideAttrs (old: { patches = (old.patches or [ ]) ++ [ (builtins.fetchurl "https://git.dgnum.eu/DGNum/Stirling-PDF/commit/${dgn-id}.patch") ]; diff --git a/machines/storage01/forgejo.nix b/machines/storage01/forgejo.nix index 61b9566..772412b 100644 --- a/machines/storage01/forgejo.nix +++ b/machines/storage01/forgejo.nix @@ -10,7 +10,7 @@ in enable = true; user = "git"; - package = nixpkgs.unstable.forgejo; + package = nixpkgs.nixos.unstable.forgejo; stateDir = "/var/lib/git"; database = { diff --git a/machines/storage01/netbird.nix b/machines/storage01/netbird.nix index 3ec6666..cf01894 100644 --- a/machines/storage01/netbird.nix +++ b/machines/storage01/netbird.nix @@ -15,7 +15,7 @@ in netbird.server = { enable = true; - package = nixpkgs.unstable.netbird; + package = nixpkgs.nixos.unstable.netbird; inherit domain; diff --git a/machines/web01/netbox.nix b/machines/web01/netbox.nix index 7d6f239..14d6493 100644 --- a/machines/web01/netbox.nix +++ b/machines/web01/netbox.nix @@ -13,7 +13,7 @@ in services = { netbox = { enable = true; - package = nixpkgs.unstable.netbox_4_1; + package = nixpkgs.nixos.unstable.netbox_4_1; secretKeyFile = "/dev/null"; listenAddress = "127.0.0.1"; plugins = p: [ p.netbox-qrcode ]; diff --git a/meta/nixpkgs.nix b/meta/nixpkgs.nix index 317cdd3..08fe650 100644 --- a/meta/nixpkgs.nix +++ b/meta/nixpkgs.nix @@ -1,11 +1,24 @@ { - # Default version of nixpkgs to use - default = "24.05"; + versions = { + # Default version of nixpkgs to use + default = "24.05"; - # Supported nixpkgs versions - supported = [ - "unstable" - "23.11" - "24.05" - ]; + # Supported nixpkgs versions + supported = [ + "unstable" + "23.11" + "24.05" + ]; + }; + + systems = { + # Default system for our deployments + default = "nixos"; + + # Supported system types + supported = [ + "zyxel-nwa50ax" + "nixos" + ]; + }; } diff --git a/meta/nodes.nix b/meta/nodes.nix index fc1cf35..0973322 100644 --- a/meta/nodes.nix +++ b/meta/nodes.nix @@ -44,7 +44,7 @@ stateVersion = "23.05"; nix-modules = [ "services/stirling-pdf" ]; - nixpkgs = "24.05"; + nixpkgs.version = "24.05"; }; geo01 = { @@ -54,7 +54,7 @@ hashedPassword = "$y$j9T$2XmDpJu.QLhV57yYCh5Lf1$LK.X0HKB02Q0Ujvhj5nIofW2IRrIAL/Uxnvl9AXM1L8"; stateVersion = "24.05"; - nixpkgs = "24.05"; + nixpkgs.version = "24.05"; }; geo02 = { @@ -64,7 +64,7 @@ hashedPassword = "$y$j9T$Q4fbMpSm9beWu4DPNAR9t0$dx/1pH4GPY72LpS5ZiECXAZFDdxwmIywztsX.qo2VVA"; stateVersion = "24.05"; - nixpkgs = "24.05"; + nixpkgs.version = "24.05"; }; rescue01 = { @@ -84,7 +84,7 @@ hashedPassword = "$y$j9T$tvRu1EJ9MwDSvEm0ogwe70$bKSw6nNteN0L3NOy2Yix7KlIvO/oROQmQ.Ynq002Fg8"; stateVersion = "23.11"; - nixpkgs = "24.05"; + nixpkgs.version = "24.05"; nix-modules = [ "services/forgejo-nix-runners" @@ -99,7 +99,7 @@ hashedPassword = "$y$j9T$5osXVNxCDxu3jIndcyh7G.$UrjiDRpMu3W59tKHLGNdLWllZh.4p8IM4sBS5SrNrN1"; stateVersion = "23.11"; - nixpkgs = "24.05"; + nixpkgs.version = "24.05"; adminGroups = [ "fai" ]; }; @@ -114,7 +114,7 @@ stateVersion = "23.05"; vm-cluster = "Hyperviseur NPS"; - nixpkgs = "24.05"; + nixpkgs.version = "24.05"; }; web02 = { @@ -123,7 +123,7 @@ hashedPassword = "$y$j9T$p42UVNy78PykkQOjPwXNJ/$B/zCUOrHXVSFGUY63wnViMiSmU2vCWsiX0y62qqgNQ5"; stateVersion = "24.05"; - nixpkgs = "24.05"; + nixpkgs.version = "24.05"; vm-cluster = "Hyperviseur NPS"; }; @@ -133,7 +133,7 @@ hashedPassword = "$y$j9T$Un/tcX5SPKNXG.sy/BcTa.$kyNHELjb1GAOWnauJfcjyVi5tacWcuEBKflZDCUC6x4"; stateVersion = "24.05"; - nixpkgs = "unstable"; + nixpkgs.version = "unstable"; vm-cluster = "Hyperviseur NPS"; }; } diff --git a/meta/options.nix b/meta/options.nix index 538ebf9..53b51cf 100644 --- a/meta/options.nix +++ b/meta/options.nix @@ -11,6 +11,7 @@ let inherit (lib.types) attrs attrsOf + enum ints listOf nullOr @@ -35,6 +36,7 @@ let }; org = config.organization; + nixpkgs = import ./nixpkgs.nix; in { @@ -137,12 +139,22 @@ in ''; }; - nixpkgs = mkOption { - type = str; - inherit (import ./nixpkgs.nix) default; - description = '' - Version of nixpkgs to use. - ''; + nixpkgs = { + version = mkOption { + type = enum nixpkgs.versions.supported; + inherit (nixpkgs.versions) default; + description = '' + Version of nixpkgs to use. + ''; + }; + system = mkOption { + type = enum nixpkgs.systems.supported; + inherit (nixpkgs.systems) default; + description = '' + Type of system for this node, will impact how it is evaluated and deployed. + ''; + example = "zyxel-nwa50ax"; + }; }; nix-modules = mkOption { diff --git a/npins/sources.json b/npins/sources.json index 5e300a4..089db58 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -38,6 +38,17 @@ "url": null, "hash": "09z5l5yh4zm0mf9hb3xc18gjk2dgv3l1icywrsxax00y1i1zlvna" }, + "colmena": { + "type": "Git", + "repository": { + "type": "Git", + "url": "https://git.dgnum.eu/DGNum/colmena" + }, + "branch": "main", + "revision": "71b1b660f2cda2e34e134d0028cafbd56bb22008", + "url": null, + "hash": "0pkzs70j1qch2238mbpl9aspzbkh0kgab25vnpr37nb75lmq81ij" + }, "dgsi": { "type": "Git", "repository": {