chore: Refactor meta to a module architecture

Get rid of the weird half nix half module stuff.

README.md

@@ -98,7 +98,7 @@ The general metadata is declared in `meta/nodes.nix`, the main values to declare
 Create the directory `secrets` in the configuration folder, and add a `secrets.nix` file containing :
 
 ```nix
-(import ../../../keys).mkSecrets [ "host02" ] [
+(import ../../../keys.nix).mkSecrets [ "host02" ] [
   # List of secrets for host02
 ]
 ```

default.nix

@@ -4,8 +4,8 @@
 # SPDX-License-Identifier: EUPL-1.2
 
 {
-  sources ? import ./npins,
-  pkgs ? import sources.nixpkgs { },
+  sources ? import ./sources.nix,
+  pkgs ? sources.bootstrapNixpkgs,
 }:
 
 let

hive.nix

@@ -4,44 +4,33 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
+# TODO: change comments to ### \n # [text] \n #
+
 let
-  sources' = import ./npins;
+  ### Init some tooling
 
-  # Patch sources directly
-  sources = builtins.mapAttrs (patch.base { pkgs = import sources'.nixos-unstable { }; })
-    .applyPatches' sources';
+  sources = import ./sources.nix;
+
+  inherit (sources) bootstrapNixpkgs;
+
+  lib = bootstrapNixpkgs.lib.extend (_: _: {
+    extra = import ./lib/nix-lib; 
+  });
+
+  nix-lib = lib.extra; # TODO: Assess perf penalty of fixed point
+  # TODO same but using bootstrapnixpkgs for nix-lib instanciation
+  # original statement: import ./lib/nix-lib;
 
-  nix-lib = import ./lib/nix-lib;
   inherit (nix-lib) mapSingleFuse;
 
-  patch = import ./lib/nix-patches { patchFile = ./patches; };
+  ### Let's build meta
+  meta = (import ./meta) lib;
 
-  nodes' = import ./meta/nodes;
-  nodes = builtins.attrNames nodes';
+  nodes = builtins.attrNames meta.nodes;
 
-  mkNode = node: {
-    deployment.systemType = system node;
-  };
+  ### Nixpkgs instanciation
 
   nixpkgs' = import ./meta/nixpkgs.nix;
-  # All supported nixpkgs versions × systems, instanciated
-  nixpkgs = mapSingleFuse (s: mapSingleFuse (mkSystemNixpkgs s) nixpkgs'.versions) nixpkgs'.systems;
-
-  # Get the configured nixos version for the node,
-  # defaulting to the one defined in meta/nixpkgs
-  version = node: nodes'.${node}.nixpkgs.version;
-  system = node: nodes'.${node}.nixpkgs.system;
-  category = node: nixpkgs'.categories.${system node};
-
-  nodePkgs = node: nixpkgs.${system node}.${version node};
-
-  # Builds a patched version of nixpkgs, only as the source
-  mkNixpkgs' =
-    v:
-    patch.mkNixpkgsSrc rec {
-      src = sources'.${name};
-      name = "nixos-${v}";
-    };
 
   # Build up the nixpkgs configuration for Liminix embedded systems
   mkLiminixConfig =
@@ -57,20 +46,34 @@ let
       };
     };
 
+
   # Build up the arguments to instantiate a nixpkgs given a system and a version.
   mkNixpkgsConfig =
     system:
     {
-      nixos = _: { };
+      nixos = _: { }; # TODO: add nix-pkgs overlay here
       zyxel-nwa50ax = mkLiminixConfig system;
       netconf = _: { };
     }
     .${system} or (throw "Unknown system: ${system} for nixpkgs configuration instantiation");
 
   # Instanciates the required nixpkgs version
-  mkSystemNixpkgs = system: version: import (mkNixpkgs' version) (mkNixpkgsConfig system version);
+  mkSystemNixpkgs = system: version:
+    import sources."nixos-${version}" (mkNixpkgsConfig system version);
 
-  ###
+  # All supported nixpkgs versions × systems, instanciated
+  nixpkgs = mapSingleFuse (s: mapSingleFuse (mkSystemNixpkgs s) nixpkgs'.versions) nixpkgs'.systems;
+
+
+  # Get the configured nixos version for the node,
+  # defaulting to the one defined in meta/nixpkgs
+  version = node: meta.nodes.${node}.nixpkgs.version;
+  system = node: meta.nodes.${node}.nixpkgs.system;
+  category = node: nixpkgs'.categories.${system node};
+
+  nodePkgs = node: nixpkgs.${system node}.${version node};
+
+  ##########
   # Function to create arguments based on the node 
   #
   mkArgs = node: rec {
@@ -79,11 +82,19 @@ let
     };
 
     sourcePkgs = nodePkgs node;
-    meta = (import ./meta) lib;
+    inherit meta;
 
     nodeMeta = meta.nodes.${node};
     nodePath = "machines/${category node}/${node}";
   };
+
+  ##########
+  # Module for each node (quite empty since almost everything is in the default module)
+  #
+  mkNode = node: {
+    deployment.systemType = system node;
+  };
+
 in
 
 {
@@ -94,7 +105,7 @@ in
     specialArgs = {
       inherit nixpkgs sources;
 
-      dgn-keys = import ./keys;
+      dgn-keys = import ./lib/keys { inherit meta lib; }; 
     };
 
     nodeSpecialArgs = mapSingleFuse mkArgs nodes;
@@ -218,5 +229,6 @@ in
         };
     };
   };
+
 }
 // (mapSingleFuse mkNode nodes)

iso/configuration.nix

@@ -5,9 +5,9 @@
 { lib, pkgs, ... }:
 
 let
-  dgn-keys = import ../keys;
+  dgn-keys = import ../keys.nix;
 
-  dgn-members = (import ../meta lib).organization.groups.root;
+  dgn-members = (import ../meta lib).config.organization.groups.root;
 in
 
 {

keys.nix

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2024 Tom Hubrecht <tom.hubrecht@dgnum.eu>
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+let
+  _sources = import ../npins;
+
+  inherit (import _sources.nixpkgs { }) lib;
+
+  meta = (import ../meta lib).config;
+
+in
+  import ./lib/keys { inherit meta lib; }

lib/keys/default.nix

@@ -1,20 +1,15 @@
+# SPDX-FileCopyrightText: 2024 Ryan Lahfa <ryan.lahfa@dgnum.eu>
 # SPDX-FileCopyrightText: 2024 Tom Hubrecht <tom.hubrecht@dgnum.eu>
+# SPDX-FileContributor: Maurice Debray <maurice.debray@dgnum.eu>
 #
 # SPDX-License-Identifier: EUPL-1.2
 
+{ meta, lib }:
 let
-  _sources = import ../npins;
-
-  inherit (import _sources.nixpkgs { }) lib;
-
-  meta = import ../meta lib;
-
-  inherit (import ../lib/nix-lib) setDefault unique;
+  inherit (import ../nix-lib) setDefault unique;
 
   getAttr = lib.flip builtins.getAttr;
-
 in
-
 rec {
 
   _memberKeys = builtins.mapAttrs (_: v: v.sshKeys) meta.organization.members;
@@ -42,7 +37,10 @@ rec {
 
   mkSecrets = nodes: setDefault { publicKeys = unique (builtins.concatMap getNodeKeys' nodes); };
 
-  machineKeysBySystem = system:
+  machineKeysBySystem =
+    system:
     rootKeys
-    ++ (getNodeKeys (builtins.attrNames (lib.filterAttrs (_: v: v.nixpkgs.system == system) meta.nodes)));
+    ++ (getNodeKeys (
+      builtins.attrNames (lib.filterAttrs (_: v: v.nixpkgs.system == system) meta.nodes)
+    ));
 }

machines/nixos/bridge01/secrets/secrets.nix

@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "bridge01" ] [
+(import ../../../../keys.nix).mkSecrets [ "bridge01" ] [
   # List of secrets for bridge01
 ]

machines/nixos/compute01/kanidm/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../../keys).mkSecrets [ "compute01" ] [
+(import ../../../../../keys.nix).mkSecrets [ "compute01" ] [
   "kanidm-password_admin"
   "kanidm-password_idm_admin"
 ]

machines/nixos/compute01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "compute01" ] [
+(import ../../../../keys.nix).mkSecrets [ "compute01" ] [
   # List of secrets for compute01
   "arkheon-env_file"
   "bupstash-put_key"

machines/nixos/geo01/secrets/secrets.nix

@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "geo01" ] [
+(import ../../../../keys.nix).mkSecrets [ "geo01" ] [
   # List of secrets for geo01
 ]

machines/nixos/geo02/secrets/secrets.nix

@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "geo02" ] [
+(import ../../../../keys.nix).mkSecrets [ "geo02" ] [
   # List of secrets for geo02
 ]

machines/nixos/hypervisor01/secrets/secrets.nix

@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifer: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "hypervisor01" ] [
+(import ../../../../keys.nix).mkSecrets [ "hypervisor01" ] [
 
 ]

machines/nixos/hypervisor02/secrets/secrets.nix

@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifer: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "hypervisor02" ] [
+(import ../../../../keys.nix).mkSecrets [ "hypervisor02" ] [
 
 ]

machines/nixos/hypervisor03/secrets/secrets.nix

@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifer: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "hypervisor03" ] [
+(import ../../../../keys.nix).mkSecrets [ "hypervisor03" ] [
 
 ]

machines/nixos/rescue01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "rescue01" ] [
+(import ../../../../keys.nix).mkSecrets [ "rescue01" ] [
   # List of secrets for rescue01
   "stateless-uptime-kuma-password"
 ]

machines/nixos/storage01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "storage01" ] [
+(import ../../../../keys.nix).mkSecrets [ "storage01" ] [
   # List of secrets for storage01
   "bupstash-put_key"
   "forgejo-mailer_password_file"

machines/nixos/tower01/secrets/secrets.nix

@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifer: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "tower01" ] [
+(import ../../../../keys.nix).mkSecrets [ "tower01" ] [
 
 ]

machines/nixos/vault01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "vault01" ] [
+(import ../../../../keys.nix).mkSecrets [ "vault01" ] [
   # List of secrets for vault01
   "radius-auth_token_file"
   "radius-ca_pem_file"

machines/nixos/web01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "web01" ] [
+(import ../../../../keys.nix).mkSecrets [ "web01" ] [
   # List of secrets for web01
   "acme-certs_secret"
   "bupstash-put_key"

machines/nixos/web02/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "web02" ] [
+(import ../../../../keys.nix).mkSecrets [ "web02" ] [
   # List of secrets for web02
   "cas_eleves-secret_key_file"
   "kadenios-secret_key_file"

machines/nixos/web03/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ "web03" ] [
+(import ../../../../keys.nix).mkSecrets [ "web03" ] [
   # List of secrets for web03
   "bupstash-put_key"
   "dj_annuaire-secret_key_file"

meta/default.nix

@@ -12,11 +12,9 @@ lib:
 (lib.evalModules {
   modules = [
     ./options.nix
-    {
-      network = import ./network.nix;
-      nodes = import ./nodes;
-      organization = import ./organization.nix;
-    }
+    ./network.nix
+    ./nodes
+    ./organization.nix
   ];
   class = "dgnumMeta";
 }).config

meta/network.nix

@@ -6,6 +6,7 @@
 # SPDX-License-Identifier: EUPL-1.2
 
 {
+  network = {
     bridge01 = {
       hostId = "f57f3ba0";
 
@@ -287,4 +288,5 @@
       hostId = "8afc7749";
       netbirdIp = "100.80.157.46";
     };
+  };
 }

meta/nodes/default.nix

@@ -2,9 +2,10 @@
 # SPDX-FileContributor: Ryan Lahfa <ryan.lahfa@dgnum.eu>
 #
 # SPDX-License-Identifier: EUPL-1.2
-
-builtins.foldl' (nodes: path: nodes // import path) { } [
+{
+  imports = [
     ./liminix.nix
     ./nixos.nix
     ./netconf.nix
-]
+  ];
+}

meta/nodes/liminix.nix

@@ -16,6 +16,7 @@
 # }
 
 {
+  nodes = {
     ap01 = {
       site = "unknown";
       adminGroups = [ "fai" ];
@@ -29,4 +30,5 @@
         version = "24.05";
       };
     };
+  };
 }

meta/nodes/netconf.nix

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 {
+  nodes = {
     netcore02 = {
       site = "hyp01";
 
@@ -46,4 +47,5 @@
     #      system = "netconf";
     #    };
     #  };
+  };
 }

meta/nodes/nixos.nix

@@ -26,6 +26,7 @@
   - luj01 -> VM de Luj
 */
 {
+  nodes = {
     bridge01 = {
       site = "hyp01";
 
@@ -290,4 +291,5 @@
         system = "nixos";
       };
     };
+  };
 }

meta/organization.nix

@@ -13,6 +13,7 @@
 */
 
 {
+  organization = {
     members = {
       agroudiev = {
         name = "Antoine Groudiev";
@@ -163,4 +164,5 @@
       # Videos DGNum
       peertube.admins = [ "thubrecht" ];
     };
+  };
 }

meta/verify.nix

@@ -6,8 +6,8 @@
 # Nix expression to check if meta module is evaluating correctly.
 # To do so run `nix-build ./verify.nix`
 let
-  sources = import ../npins;
-  pkgs = import sources.nixpkgs { };
+  sources = import ../sources.nix;
+  pkgs = sources.bootstrapNixpkgs;
 
   dns = import sources."dns.nix" { inherit pkgs; };
 in

modules/nixos/dgn-access-control.nix

@@ -58,7 +58,7 @@ in
     users.users = builtins.mapAttrs (
       username: members:
       {
-        openssh.authorizedKeys.keys = dgn-keys.getKeys members;
+        openssh.authorizedKeys.keys = dgn-keys.getMemberKeys members;
       }
       // optionalAttrs (username == "root") { inherit (nodeMeta) hashedPassword; }
     ) cfg.users;

modules/nixos/dgn-backups/default.nix

@@ -114,7 +114,7 @@ in
         access = [
           {
             repo = "default";
-            keys = dgn-keys.getKeys [
+            keys = dgn-keys.getNodeKeys [
               "compute01"
               "storage01"
               "vault01"
@@ -131,7 +131,7 @@ in
     };
 
     programs.ssh.knownHosts =
-      lib.extra.mapFuse (host: { "${host}.dgnum".publicKey = builtins.head (dgn-keys.getNodeKeys host); })
+      lib.extra.mapFuse (host: { "${host}.dgnum".publicKey = builtins.head dgn-keys._nodeKeys.${host}; })
         [
           "compute01"
           "geo01"

modules/nixos/dgn-backups/keys/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets [ ] [
+(import ../../../../keys.nix).mkSecrets [ ] [
   "compute01.key"
   "storage01.key"
   "web01.key"

modules/nixos/dgn-netbox-agent/secrets/secrets.nix

@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-{ netbox-agent.publicKeys = (import ../../../../keys).machineKeysBySystem "nixos"; }
+{ netbox-agent.publicKeys = (import ../../../../keys.nix).machineKeysBySystem "nixos"; }

modules/nixos/dgn-notify/secrets.nix

@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-{ mail.publicKeys = (import ../../../keys).machineKeysBySystem "nixos"; }
+{ mail.publicKeys = (import ../../../keys.nix).machineKeysBySystem "nixos"; }

modules/nixos/dgn-records/secrets.nix

@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-{ __arkheon-token_file.publicKeys = (import ../../../keys).machineKeysBySystem "nixos"; }
+{ __arkheon-token_file.publicKeys = (import ../../../keys.nix).machineKeysBySystem "nixos"; }

sources.nix

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: 2024 Ryan Lahfa <ryan.lahfa@dgnum.eu>
+# SPDX-FileCopyrightText: 2024 Tom Hubrecht <tom.hubrecht@dgnum.eu>
+# SPDX-FileContributor: Maurice Debray <maurice.debray@dgnum.eu>
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+let
+  sources' = import ./npins;
+
+  bootstrapNixpkgs = import sources'.nixpkgs { };
+
+  patch = (import ./lib/nix-patches { patchFile = ./patches; }).base {
+    pkgs = bootstrapNixpkgs;
+  };
+
+  sources = builtins.mapAttrs (
+    k: src:
+    patch.applyPatches {
+      inherit src;
+      name = k;
+    }
+  ) sources';
+in sources // { bootstrapNixpkgs = bootstrapNixpkgs; unpatchedSources = sources; }

workflows/eval-nodes.nix

@@ -7,7 +7,7 @@
 let
   inherit (lib) attrNames genAttrs;
 
-  nodes = attrNames (import ../meta/nodes);
+  nodes = attrNames (import ../meta lib).nodes;
 in
 
 {