chore: Refactor meta to a module architecture

Get rid of the weird half nix half module stuff.

README.md

@@ -98,7 +98,7 @@ The general metadata is declared in `meta/nodes.nix`, the main values to declare
 Create the directory `secrets` in the configuration folder, and add a `secrets.nix` file containing :
 
 ```nix
-(import ../../../keys).mkSecrets [ "host02" ] [
+(import ../../../keys.nix).mkSecrets [ "host02" ] [
   # List of secrets for host02
 ]
 ```

default.nix

@@ -4,8 +4,8 @@
 # SPDX-License-Identifier: EUPL-1.2
 
 {
-  sources ? import ./npins,
-  pkgs ? import sources.nixos-unstable { },
+  sources ? import ./sources.nix,
+  pkgs ? sources.bootstrapNixpkgs,
 }:
 
 let

hive.nix

@@ -4,45 +4,33 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
+# TODO: change comments to ### \n # [text] \n #
+
 let
-  sources' = import ./npins;
+  ### Init some tooling
 
-  # Patch sources directly
-  sources =
-    builtins.mapAttrs (patch.base { pkgs = import sources'.nixos-unstable { }; }).applyPatches'
-      sources';
+  sources = import ./sources.nix;
+
+  inherit (sources) bootstrapNixpkgs;
+
+  lib = bootstrapNixpkgs.lib.extend (_: _: {
+    extra = import ./lib/nix-lib; 
+  });
+
+  nix-lib = lib.extra; # TODO: Assess perf penalty of fixed point
+  # TODO same but using bootstrapnixpkgs for nix-lib instanciation
+  # original statement: import ./lib/nix-lib;
 
-  nix-lib = import ./lib/nix-lib;
   inherit (nix-lib) mapSingleFuse;
 
-  patch = import ./lib/nix-patches { patchFile = ./patches; };
+  ### Let's build meta
+  meta = (import ./meta) lib;
 
-  nodes' = import ./meta/nodes;
-  nodes = builtins.attrNames nodes';
+  nodes = builtins.attrNames meta.nodes;
 
-  mkNode = node: {
-    deployment.systemType = system node;
-  };
+  ### Nixpkgs instanciation
 
   nixpkgs' = import ./meta/nixpkgs.nix;
-  # All supported nixpkgs versions × systems, instanciated
-  nixpkgs = mapSingleFuse (s: mapSingleFuse (mkSystemNixpkgs s) nixpkgs'.versions) nixpkgs'.systems;
-
-  # Get the configured nixos version for the node,
-  # defaulting to the one defined in meta/nixpkgs
-  version = node: nodes'.${node}.nixpkgs.version;
-  system = node: nodes'.${node}.nixpkgs.system;
-  category = node: nixpkgs'.categories.${system node};
-
-  nodePkgs = node: nixpkgs.${system node}.${version node};
-
-  # Builds a patched version of nixpkgs, only as the source
-  mkNixpkgs' =
-    v:
-    patch.mkNixpkgsSrc rec {
-      src = sources'.${name};
-      name = "nixos-${v}";
-    };
 
   # Build up the nixpkgs configuration for Liminix embedded systems
   mkLiminixConfig =
@@ -58,20 +46,34 @@ let
       };
     };
 
+
   # Build up the arguments to instantiate a nixpkgs given a system and a version.
   mkNixpkgsConfig =
     system:
     {
-      nixos = _: { };
+      nixos = _: { }; # TODO: add nix-pkgs overlay here
       zyxel-nwa50ax = mkLiminixConfig system;
       netconf = _: { };
     }
     .${system} or (throw "Unknown system: ${system} for nixpkgs configuration instantiation");
 
   # Instanciates the required nixpkgs version
-  mkSystemNixpkgs = system: version: import (mkNixpkgs' version) (mkNixpkgsConfig system version);
+  mkSystemNixpkgs = system: version:
+    import sources."nixos-${version}" (mkNixpkgsConfig system version);
 
-  ###
+  # All supported nixpkgs versions × systems, instanciated
+  nixpkgs = mapSingleFuse (s: mapSingleFuse (mkSystemNixpkgs s) nixpkgs'.versions) nixpkgs'.systems;
+
+
+  # Get the configured nixos version for the node,
+  # defaulting to the one defined in meta/nixpkgs
+  version = node: meta.nodes.${node}.nixpkgs.version;
+  system = node: meta.nodes.${node}.nixpkgs.system;
+  category = node: nixpkgs'.categories.${system node};
+
+  nodePkgs = node: nixpkgs.${system node}.${version node};
+
+  ##########
   # Function to create arguments based on the node 
   #
   mkArgs = node: rec {
@@ -80,11 +82,19 @@ let
     };
 
     sourcePkgs = nodePkgs node;
-    meta = (import ./meta) lib;
+    inherit meta;
 
     nodeMeta = meta.nodes.${node};
     nodePath = "machines/${category node}/${node}";
   };
+
+  ##########
+  # Module for each node (quite empty since almost everything is in the default module)
+  #
+  mkNode = node: {
+    deployment.systemType = system node;
+  };
+
 in
 
 {
@@ -95,7 +105,7 @@ in
     specialArgs = {
       inherit nixpkgs sources;
 
-      dgn-keys = import ./keys;
+      dgn-keys = import ./lib/keys { inherit meta lib; }; 
     };
 
     nodeSpecialArgs = mapSingleFuse mkArgs nodes;
@@ -219,5 +229,6 @@ in
         };
     };
   };
+
 }
 // (mapSingleFuse mkNode nodes)

iso/configuration.nix

@@ -5,9 +5,9 @@
 { lib, pkgs, ... }:
 
 let
-  dgn-keys = import ../keys;
+  dgn-keys = import ../keys.nix;
 
-  dgn-members = (import ../meta lib).organization.groups.root;
+  dgn-members = (import ../meta lib).config.organization.groups.root;
 in
 
 {

keys.nix

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2024 Tom Hubrecht <tom.hubrecht@dgnum.eu>
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+let
+  _sources = import ../npins;
+
+  inherit (import _sources.nixpkgs { }) lib;
+
+  meta = (import ../meta lib).config;
+
+in
+  import ./lib/keys { inherit meta lib; }

lib/keys/default.nix

@@ -1,20 +1,15 @@
+# SPDX-FileCopyrightText: 2024 Ryan Lahfa <ryan.lahfa@dgnum.eu>
 # SPDX-FileCopyrightText: 2024 Tom Hubrecht <tom.hubrecht@dgnum.eu>
+# SPDX-FileContributor: Maurice Debray <maurice.debray@dgnum.eu>
 #
 # SPDX-License-Identifier: EUPL-1.2
 
+{ meta, lib }:
 let
-  _sources = import ../npins;
-
-  inherit (import _sources.nixos-unstable { }) lib;
-
-  meta = import ../meta lib;
-
-  inherit (import ../lib/nix-lib) setDefault unique;
+  inherit (import ../nix-lib) setDefault unique;
 
   getAttr = lib.flip builtins.getAttr;
-
 in
-
 rec {
   _memberKeys = builtins.mapAttrs (_: v: v.sshKeys) meta.organization.members;
   _nodeKeys = builtins.mapAttrs (_: v: v.sshKeys) meta.nodes;
@@ -41,7 +36,10 @@ rec {
 
   mkSecrets = nodes: setDefault { publicKeys = unique (builtins.concatMap getNodeKeys' nodes); };
 
-  machineKeysBySystem = system:
+  machineKeysBySystem =
+    system:
     rootKeys
-    ++ (getNodeKeys (builtins.attrNames (lib.filterAttrs (_: v: v.nixpkgs.system == system) meta.nodes)));
+    ++ (getNodeKeys (
+      builtins.attrNames (lib.filterAttrs (_: v: v.nixpkgs.system == system) meta.nodes)
+    ));
 }

machines/nixos/bridge01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "bridge01" ]
   [
     # List of secrets for bridge01

machines/nixos/compute01/kanidm/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../../keys).mkSecrets
+(import ../../../../../keys.nix).mkSecrets
   [ "compute01" ]
   [
     "kanidm-password_admin"

machines/nixos/compute01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "compute01" ]
   [
     # List of secrets for compute01

machines/nixos/geo01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "geo01" ]
   [
     # List of secrets for geo01

machines/nixos/geo02/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "geo02" ]
   [
     # List of secrets for geo02

machines/nixos/hypervisor01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifer: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "hypervisor01" ]
   [
 

machines/nixos/hypervisor02/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifer: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "hypervisor02" ]
   [
 

machines/nixos/hypervisor03/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifer: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "hypervisor03" ]
   [
 

machines/nixos/rescue01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "rescue01" ]
   [
     # List of secrets for rescue01

machines/nixos/storage01/secrets/secrets.nix

@@ -2,13 +2,14 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "storage01" ]
   [
     # List of secrets for storage01
     "bupstash-put_key"
     "forgejo-mailer_password_file"
     "forgejo_runners-token_file"
+    "garage-environment_file"
     "influxdb2-grafana_token_file"
     "influxdb2-initial_password_file"
     "influxdb2-initial_token_file"

machines/nixos/tower01/secrets/secrets.nix

@@ -2,7 +2,8 @@
 #
 # SPDX-License-Identifer: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "tower01" ]
   [
+
   ]

machines/nixos/vault01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "vault01" ]
   [
     # List of secrets for vault01

machines/nixos/web01/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "web01" ]
   [
     # List of secrets for web01

machines/nixos/web02/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "web02" ]
   [
     # List of secrets for web02

machines/nixos/web03/secrets/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ "web03" ]
   [
     # List of secrets for web03

meta/default.nix

@@ -12,11 +12,9 @@ lib:
 (lib.evalModules {
   modules = [
     ./options.nix
-    {
-      network = import ./network.nix;
-      nodes = import ./nodes;
-      organization = import ./organization.nix;
-    }
+    ./network.nix
+    ./nodes
+    ./organization.nix
   ];
   class = "dgnumMeta";
 }).config

meta/network.nix

@@ -6,6 +6,7 @@
 # SPDX-License-Identifier: EUPL-1.2
 
 {
+  network = {
     bridge01 = {
       hostId = "f57f3ba0";
 
@@ -306,4 +307,5 @@
       hostId = "8afc7749";
       netbirdIp = "100.80.157.46";
     };
+  };
 }

meta/nodes/default.nix

@@ -2,9 +2,10 @@
 # SPDX-FileContributor: Ryan Lahfa <ryan.lahfa@dgnum.eu>
 #
 # SPDX-License-Identifier: EUPL-1.2
-
-builtins.foldl' (nodes: path: nodes // import path) { } [
+{
+  imports = [
     ./liminix.nix
     ./nixos.nix
     ./netconf.nix
-]
+  ];
+}

meta/nodes/liminix.nix

@@ -16,6 +16,7 @@
 # }
 
 {
+  nodes = {
     ap01 = {
       site = "unknown";
       adminGroups = [ "fai" ];
@@ -29,4 +30,5 @@
         version = "24.05";
       };
     };
+  };
 }

meta/nodes/netconf.nix

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 {
+  nodes = {
     netcore02 = {
       site = "hyp01";
 
@@ -46,4 +47,5 @@
     #      system = "netconf";
     #    };
     #  };
+  };
 }

meta/nodes/nixos.nix

@@ -26,6 +26,7 @@
   - luj01 -> VM de Luj
 */
 {
+  nodes = {
     bridge01 = {
       site = "hyp01";
 
@@ -74,7 +75,6 @@
 
     compute01 = {
       site = "pav01";
-
       sshKeys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE/YluSVS+4h3oV8CIUj0OmquyJXju8aEQy0Jz210vTu" ];
 
       hashedPassword = "$y$j9T$2nxZHq84G7fWvWMEaGavE/$0ADnmD9qMpXJJ.rWWH9086EakvZ3wAg0mSxZYugOf3C";
@@ -203,6 +203,9 @@
         version = "24.11";
         system = "nixos";
       };
+      nix-modules = [
+        "services/netbird/server.nix"
+      ];
     };
 
     storage01 = {
@@ -220,6 +223,7 @@
       };
 
       nix-modules = [
+        "services/forgejo-nix-runners"
         "services/netbird/server.nix"
       ];
     };
@@ -240,13 +244,6 @@
         system = "nixos";
       };
 
-    deployment = {
-      sshOptions = [
-        "-J"
-        "root@vault01.hyp01.infra.dgnum.eu"
-      ];
-    };
-
       admins = [ "ecoppens" ];
     };
 
@@ -317,4 +314,5 @@
         system = "nixos";
       };
     };
+  };
 }

meta/organization.nix

@@ -13,6 +13,7 @@
 */
 
 {
+  organization = {
     members = {
       agroudiev = {
         name = "Antoine Groudiev";
@@ -133,7 +134,7 @@
         "catvayor"
         "ecoppens"
         "mdebray"
-      "raito"
+        "raiot"
         "thubrecht"
       ];
     };
@@ -171,4 +172,5 @@
       # Videos DGNum
       peertube.admins = [ "thubrecht" ];
     };
+  };
 }

meta/verify.nix

@@ -6,8 +6,8 @@
 # Nix expression to check if meta module is evaluating correctly.
 # To do so run `nix-build ./verify.nix`
 let
-  sources = import ../npins;
-  pkgs = import sources.nixos-unstable { };
+  sources = import ../sources.nix;
+  pkgs = sources.bootstrapNixpkgs;
 
   dns = import sources."dns.nix" { inherit pkgs; };
 in

modules/nixos/dgn-backups/keys/secrets.nix

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-(import ../../../../keys).mkSecrets
+(import ../../../../keys.nix).mkSecrets
   [ ]
   [
     "compute01.key"

modules/nixos/dgn-netbox-agent/secrets/secrets.nix

@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-{ netbox-agent.publicKeys = (import ../../../../keys).machineKeysBySystem "nixos"; }
+{ netbox-agent.publicKeys = (import ../../../../keys.nix).machineKeysBySystem "nixos"; }

modules/nixos/dgn-notify/secrets.nix

@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-{ mail.publicKeys = (import ../../../keys).machineKeysBySystem "nixos"; }
+{ mail.publicKeys = (import ../../../keys.nix).machineKeysBySystem "nixos"; }

modules/nixos/dgn-records/secrets.nix

@@ -2,4 +2,4 @@
 #
 # SPDX-License-Identifier: EUPL-1.2
 
-{ __arkheon-token_file.publicKeys = (import ../../../keys).machineKeysBySystem "nixos"; }
+{ __arkheon-token_file.publicKeys = (import ../../../keys.nix).machineKeysBySystem "nixos"; }

sources.nix

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: 2024 Ryan Lahfa <ryan.lahfa@dgnum.eu>
+# SPDX-FileCopyrightText: 2024 Tom Hubrecht <tom.hubrecht@dgnum.eu>
+# SPDX-FileContributor: Maurice Debray <maurice.debray@dgnum.eu>
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+let
+  sources' = import ./npins;
+
+  bootstrapNixpkgs = import sources'.nixos-unstable { };
+
+  patch = (import ./lib/nix-patches { patchFile = ./patches; }).base {
+    pkgs = bootstrapNixpkgs;
+  };
+
+  sources = builtins.mapAttrs (
+    k: src:
+    patch.applyPatches {
+      inherit src;
+      name = k;
+    }
+  ) sources';
+in sources // { bootstrapNixpkgs = bootstrapNixpkgs; unpatchedSources = sources; }

workflows/eval-nodes.nix

@@ -7,7 +7,7 @@
 let
   inherit (lib) attrNames genAttrs;
 
-  nodes = attrNames (import ../meta/nodes);
+  nodes = attrNames (import ../meta lib).nodes;
 in
 
 {