diff --git a/default.nix b/default.nix index 2020818..78b2a09 100644 --- a/default.nix +++ b/default.nix @@ -1,3 +1 @@ -let - flake = import ./flake-compat.nix; -in flake.defaultNix.default +(import ./flake-compat.nix).defaultNix.default diff --git a/flake-compat.nix b/flake-compat.nix index 96341ad..bfd5eea 100644 --- a/flake-compat.nix +++ b/flake-compat.nix @@ -1,9 +1,7 @@ let lock = builtins.fromJSON (builtins.readFile ./flake.lock); - flakeCompat = import (fetchTarball { - url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; - sha256 = lock.nodes.flake-compat.locked.narHash; - }); -in flakeCompat { - src = ./.; -} +in +import (builtins.fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; +}) { src = ./.; } diff --git a/flake.nix b/flake.nix index c686301..504a1ec 100644 --- a/flake.nix +++ b/flake.nix @@ -18,119 +18,167 @@ }; }; - outputs = { - self, - nixpkgs, - stable, - flake-utils, - nix-github-actions, - ... - } @ inputs: let - supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; - colmenaOptions = import ./src/nix/hive/options.nix; - colmenaModules = import ./src/nix/hive/modules.nix; - in flake-utils.lib.eachSystem supportedSystems (system: let - pkgs = import nixpkgs { - inherit system; - overlays = [ + outputs = + { + self, + nixpkgs, + stable, + flake-utils, + nix-github-actions, + ... + }@inputs: + let + supportedSystems = [ + "x86_64-linux" + "i686-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" ]; - }; - in rec { - # We still maintain the expression in a Nixpkgs-acceptable form - defaultPackage = self.packages.${system}.colmena; - packages = rec { - colmena = pkgs.callPackage ./package.nix { }; + in + flake-utils.lib.eachSystem supportedSystems ( + system: + let + pkgs = import nixpkgs { inherit system; }; + in + rec { + # We still maintain the expression in a Nixpkgs-acceptable form + defaultPackage = self.packages.${system}.colmena; + packages = rec { + colmena = pkgs.callPackage ./package.nix { }; - # Full user manual - manual = let - suppressModuleArgsDocs = { lib, ... }: { - options = { - _module.args = lib.mkOption { - internal = true; + # Full user manual + manual = + let + suppressModuleArgsDocs = + { lib, ... }: + { + options = { + _module.args = lib.mkOption { + internal = true; + }; + }; + }; + colmena = self.packages.${system}.colmena; + deploymentOptionsMd = + (pkgs.nixosOptionsDoc { + inherit + (pkgs.lib.evalModules { + modules = [ + ./src/nix/hive/options/deployment.nix + suppressModuleArgsDocs + ]; + specialArgs = { + name = "nixos"; + nodes = { }; + }; + }) + options + ; + }).optionsCommonMark; + metaOptionsMd = + (pkgs.nixosOptionsDoc { + inherit + (pkgs.lib.evalModules { + modules = [ + ./src/nix/hive/options/meta.nix + suppressModuleArgsDocs + ]; + }) + options + ; + }).optionsCommonMark; + in + pkgs.callPackage ./manual { + inherit colmena deploymentOptionsMd metaOptionsMd; }; - }; + + # User manual without the CLI reference + manualFast = manual.override { colmena = null; }; + + # User manual with the version treated as stable + manualForceStable = manual.override { unstable = false; }; }; - colmena = self.packages.${system}.colmena; - deploymentOptionsMd = (pkgs.nixosOptionsDoc { - inherit (pkgs.lib.evalModules { - modules = [ colmenaOptions.deploymentOptions suppressModuleArgsDocs]; - specialArgs = { name = "nixos"; nodes = {}; }; - }) options; - }).optionsCommonMark; - metaOptionsMd = (pkgs.nixosOptionsDoc { - inherit (pkgs.lib.evalModules { - modules = [ colmenaOptions.metaOptions suppressModuleArgsDocs]; - }) options; - }).optionsCommonMark; - in pkgs.callPackage ./manual { - inherit colmena deploymentOptionsMd metaOptionsMd; + + defaultApp = self.apps.${system}.colmena; + apps.default = self.apps.${system}.colmena; + apps.colmena = { + type = "app"; + program = pkgs.lib.getExe defaultPackage; + }; + + devShell = pkgs.mkShell { + RUST_SRC_PATH = pkgs.rustPlatform.rustLibSrc; + NIX_PATH = "nixpkgs=${pkgs.path}"; + + inputsFrom = [ + defaultPackage + packages.manualFast + ]; + packages = with pkgs; [ + bashInteractive + editorconfig-checker + clippy + rust-analyzer + cargo-outdated + cargo-audit + rustfmt + python3 + python3Packages.flake8 + ]; + }; + checks = + let + inputsOverlay = final: prev: { + _inputs = inputs; + }; + in + if pkgs.stdenv.isLinux then + import ./integration-tests { + pkgs = import nixpkgs { + inherit system; + overlays = [ + self.overlays.default + inputsOverlay + ]; + }; + pkgsStable = import stable { + inherit system; + overlays = [ + self.overlays.default + inputsOverlay + ]; + }; + } + else + { }; + } + ) + // { + overlay = self.overlays.default; + overlays.default = final: prev: { + colmena = final.callPackage ./package.nix { }; + }; + nixosModules = { + deploymentOptions = import ./src/nix/hive/options/deployment.nix; + metaOptions = import ./src/nix/hive/options/meta.nix; + + keyChownModule = import ./src/nix/hive/modules/key-chown.nix; + keyServiceModule = import ./src/nix/hive/modules/key-service.nix; + assertionModule = import ./src/nix/hive/modules/assertions.nix; }; - # User manual without the CLI reference - manualFast = manual.override { colmena = null; }; + lib.makeHive = + rawHive: + import ./src/nix/hive/eval.nix { + inherit rawHive; + hermetic = true; + }; - # User manual with the version treated as stable - manualForceStable = manual.override { unstable = false; }; - }; - - defaultApp = self.apps.${system}.colmena; - apps.default = self.apps.${system}.colmena; - apps.colmena = { - type = "app"; - program = "${defaultPackage}/bin/colmena"; - }; - - devShell = pkgs.mkShell { - RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library"; - NIX_PATH = "nixpkgs=${pkgs.path}"; - - inputsFrom = [ defaultPackage packages.manualFast ]; - packages = with pkgs; [ - bashInteractive - editorconfig-checker - clippy rust-analyzer cargo-outdated cargo-audit rustfmt - python3 python3Packages.flake8 - ]; - }; - checks = let - inputsOverlay = final: prev: { - _inputs = inputs; - }; - in if pkgs.stdenv.isLinux then import ./integration-tests { - pkgs = import nixpkgs { - inherit system; - overlays = [ - self.overlays.default - inputsOverlay - ]; - }; - pkgsStable = import stable { - inherit system; - overlays = [ - self.overlays.default - inputsOverlay - ]; - }; - } else {}; - }) // { - overlay = self.overlays.default; - overlays.default = final: prev: { - colmena = final.callPackage ./package.nix { }; - }; - nixosModules = { - inherit (colmenaOptions) deploymentOptions metaOptions; - inherit (colmenaModules) keyChownModule keyServiceModule assertionModule; - }; - - lib.makeHive = rawHive: import ./src/nix/hive/eval.nix { - inherit rawHive colmenaOptions colmenaModules; - hermetic = true; - }; - - githubActions = nix-github-actions.lib.mkGithubMatrix { - checks = { - inherit (self.checks) x86_64-linux; + githubActions = nix-github-actions.lib.mkGithubMatrix { + checks = { + inherit (self.checks) x86_64-linux; + }; }; }; - }; } diff --git a/integration-tests/allow-apply-all/default.nix b/integration-tests/allow-apply-all/default.nix index 5afd020..b070d60 100644 --- a/integration-tests/allow-apply-all/default.nix +++ b/integration-tests/allow-apply-all/default.nix @@ -4,7 +4,8 @@ let tools = pkgs.callPackage ../tools.nix { targets = [ "alpha" ]; }; -in tools.runTest { +in +tools.runTest { name = "colmena-allow-apply-all"; colmena.test = { diff --git a/integration-tests/allow-apply-all/hive.nix b/integration-tests/allow-apply-all/hive.nix index 6c5a5b0..cfd74ce 100644 --- a/integration-tests/allow-apply-all/hive.nix +++ b/integration-tests/allow-apply-all/hive.nix @@ -1,9 +1,10 @@ let tools = import ./tools.nix { insideVm = true; - targets = ["alpha"]; + targets = [ "alpha" ]; }; -in { +in +{ meta = { nixpkgs = tools.pkgs; allowApplyAll = false; diff --git a/integration-tests/apply-local/default.nix b/integration-tests/apply-local/default.nix index a5345e6..52eca87 100644 --- a/integration-tests/apply-local/default.nix +++ b/integration-tests/apply-local/default.nix @@ -2,7 +2,7 @@ let tools = pkgs.callPackage ../tools.nix { - targets = []; + targets = [ ]; prebuiltTarget = "deployer"; extraDeployerConfig = { users.users.colmena = { @@ -12,7 +12,8 @@ let security.sudo.wheelNeedsPassword = false; }; }; -in tools.runTest { +in +tools.runTest { name = "colmena-apply-local"; colmena.test = { diff --git a/integration-tests/apply-local/hive.nix b/integration-tests/apply-local/hive.nix index fbc2676..3cc0f45 100644 --- a/integration-tests/apply-local/hive.nix +++ b/integration-tests/apply-local/hive.nix @@ -1,26 +1,29 @@ let tools = import ./tools.nix { insideVm = true; - targets = []; + targets = [ ]; prebuiltTarget = "deployer"; }; -in { +in +{ meta = { nixpkgs = tools.pkgs; }; - deployer = { lib, ... }: { - imports = [ - (tools.getStandaloneConfigFor "deployer") - ]; + deployer = + { lib, ... }: + { + imports = [ + (tools.getStandaloneConfigFor "deployer") + ]; - deployment = { - allowLocalDeployment = true; + deployment = { + allowLocalDeployment = true; + }; + + environment.etc."deployment".text = "SUCCESS"; + + # /run/keys/key-text + deployment.keys."key-text".text = "SECRET"; }; - - environment.etc."deployment".text = "SUCCESS"; - - # /run/keys/key-text - deployment.keys."key-text".text = "SECRET"; - }; } diff --git a/integration-tests/apply/default.nix b/integration-tests/apply/default.nix index 53ff97d..cc3c067 100644 --- a/integration-tests/apply/default.nix +++ b/integration-tests/apply/default.nix @@ -1,17 +1,21 @@ -{ pkgs -, evaluator ? "chunked" +{ + pkgs, + evaluator ? "chunked", }: let - tools = pkgs.callPackage ../tools.nix {}; -in tools.runTest { + tools = pkgs.callPackage ../tools.nix { }; +in +tools.runTest { name = "colmena-apply-${evaluator}"; colmena.test = { bundle = ./.; - testScript = '' - colmena = "${tools.colmenaExec}" - evaluator = "${evaluator}" - '' + builtins.readFile ./test-script.py; + testScript = + '' + colmena = "${tools.colmenaExec}" + evaluator = "${evaluator}" + '' + + builtins.readFile ./test-script.py; }; } diff --git a/integration-tests/apply/hive.nix b/integration-tests/apply/hive.nix index bd23f0b..0959bd5 100644 --- a/integration-tests/apply/hive.nix +++ b/integration-tests/apply/hive.nix @@ -1,13 +1,16 @@ let tools = import ./tools.nix { insideVm = true; }; - testPkg = let - text = builtins.trace "must appear during evaluation" '' - echo "must appear during build" - mkdir -p $out - ''; - in tools.pkgs.runCommand "test-package" {} text; -in { + testPkg = + let + text = builtins.trace "must appear during evaluation" '' + echo "must appear during build" + mkdir -p $out + ''; + in + tools.pkgs.runCommand "test-package" { } text; +in +{ meta = { nixpkgs = tools.pkgs; }; @@ -20,7 +23,7 @@ in { isSystemUser = true; group = "testgroup"; }; - users.groups.testgroup = {}; + users.groups.testgroup = { }; # /run/keys/custom-name deployment.keys.original-name = { @@ -72,29 +75,33 @@ in { }; }; - alpha = { lib, ... }: { - imports = [ - (tools.getStandaloneConfigFor "alpha") - ]; + alpha = + { lib, ... }: + { + imports = [ + (tools.getStandaloneConfigFor "alpha") + ]; - environment.systemPackages = [ testPkg ]; + environment.systemPackages = [ testPkg ]; - documentation.nixos.enable = lib.mkForce true; + documentation.nixos.enable = lib.mkForce true; - system.activationScripts.colmena-test.text = '' - echo "must appear during activation" - ''; - }; + system.activationScripts.colmena-test.text = '' + echo "must appear during activation" + ''; + }; deployer = tools.getStandaloneConfigFor "deployer"; beta = tools.getStandaloneConfigFor "beta"; gamma = tools.getStandaloneConfigFor "gamma"; - "gamma.tld" = { lib, ... }: { - imports = [ - (tools.getStandaloneConfigFor "gamma") - ]; + "gamma.tld" = + { lib, ... }: + { + imports = [ + (tools.getStandaloneConfigFor "gamma") + ]; - deployment.tags = lib.mkForce []; - }; + deployment.tags = lib.mkForce [ ]; + }; } diff --git a/integration-tests/build-on-target/default.nix b/integration-tests/build-on-target/default.nix index f52f441..0374286 100644 --- a/integration-tests/build-on-target/default.nix +++ b/integration-tests/build-on-target/default.nix @@ -2,10 +2,15 @@ let tools = pkgs.callPackage ../tools.nix { - deployers = [ "deployer" "alpha" "beta" ]; - targets = []; + deployers = [ + "deployer" + "alpha" + "beta" + ]; + targets = [ ]; }; -in tools.runTest { +in +tools.runTest { name = "colmena-build-on-target"; colmena.test = { diff --git a/integration-tests/build-on-target/hive.nix b/integration-tests/build-on-target/hive.nix index 3fefa43..2a44ce4 100644 --- a/integration-tests/build-on-target/hive.nix +++ b/integration-tests/build-on-target/hive.nix @@ -1,10 +1,15 @@ let tools = import ./tools.nix { insideVm = true; - deployers = [ "deployer" "alpha" "beta" ]; - targets = []; + deployers = [ + "deployer" + "alpha" + "beta" + ]; + targets = [ ]; }; -in { +in +{ meta = { nixpkgs = tools.pkgs; }; diff --git a/integration-tests/default.nix b/integration-tests/default.nix index a9453a2..6fb8be6 100644 --- a/integration-tests/default.nix +++ b/integration-tests/default.nix @@ -1,10 +1,14 @@ -{ pkgs ? import ./nixpkgs.nix -, pkgsStable ? import ./nixpkgs-stable.nix +{ + pkgs ? import ./nixpkgs.nix, + pkgsStable ? import ./nixpkgs-stable.nix, }: { apply = import ./apply { inherit pkgs; }; - apply-streaming = import ./apply { inherit pkgs; evaluator = "streaming"; }; + apply-streaming = import ./apply { + inherit pkgs; + evaluator = "streaming"; + }; apply-local = import ./apply-local { inherit pkgs; }; build-on-target = import ./build-on-target { inherit pkgs; }; exec = import ./exec { inherit pkgs; }; diff --git a/integration-tests/exec/default.nix b/integration-tests/exec/default.nix index 23e1823..e0e1e6e 100644 --- a/integration-tests/exec/default.nix +++ b/integration-tests/exec/default.nix @@ -1,8 +1,9 @@ { pkgs }: let - tools = pkgs.callPackage ../tools.nix {}; -in tools.runTest { + tools = pkgs.callPackage ../tools.nix { }; +in +tools.runTest { name = "colmena-exec"; colmena.test = { diff --git a/integration-tests/exec/hive.nix b/integration-tests/exec/hive.nix index c704da8..76d18b0 100644 --- a/integration-tests/exec/hive.nix +++ b/integration-tests/exec/hive.nix @@ -1,6 +1,7 @@ let tools = import ./tools.nix { insideVm = true; }; -in { +in +{ meta = { nixpkgs = tools.pkgs; }; diff --git a/integration-tests/flakes/default.nix b/integration-tests/flakes/default.nix index c8b6935..fa04d89 100644 --- a/integration-tests/flakes/default.nix +++ b/integration-tests/flakes/default.nix @@ -1,7 +1,8 @@ -{ pkgs -, evaluator ? "chunked" -, extraApplyFlags ? "" -, pure ? true +{ + pkgs, + evaluator ? "chunked", + extraApplyFlags ? "", + pure ? true, }: let @@ -11,58 +12,58 @@ let targets = [ "alpha" ]; }; - applyFlags = "--evaluator ${evaluator} ${extraApplyFlags}" - + lib.optionalString (!pure) "--impure"; + applyFlags = "--evaluator ${evaluator} ${extraApplyFlags}" + lib.optionalString (!pure) "--impure"; # From integration-tests/nixpkgs.nix colmenaFlakeInputs = pkgs._inputs; -in tools.runTest { - name = "colmena-flakes-${evaluator}" - + lib.optionalString (!pure) "-impure"; +in +tools.runTest { + name = "colmena-flakes-${evaluator}" + lib.optionalString (!pure) "-impure"; nodes.deployer = { - virtualisation.additionalPaths = - lib.mapAttrsToList (k: v: v.outPath) colmenaFlakeInputs; + virtualisation.additionalPaths = lib.mapAttrsToList (k: v: v.outPath) colmenaFlakeInputs; }; colmena.test = { bundle = ./.; - testScript = '' - import re + testScript = + '' + import re - deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix") - deployer.succeed("sed -i 's @colmena@ path:${tools.colmena.src} g' /tmp/bundle/flake.nix") + deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix") + deployer.succeed("sed -i 's @colmena@ path:${tools.colmena.src} g' /tmp/bundle/flake.nix") - with subtest("Lock flake dependencies"): - deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock") + with subtest("Lock flake dependencies"): + deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock") - with subtest("Deploy with a plain flake without git"): - deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}") - alpha.succeed("grep FIRST /etc/deployment") + with subtest("Deploy with a plain flake without git"): + deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}") + alpha.succeed("grep FIRST /etc/deployment") - with subtest("Deploy with a git flake"): - deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/probe.nix") + with subtest("Deploy with a git flake"): + deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/probe.nix") - # don't put probe.nix in source control - should fail - deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix") - logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}") - assert re.search(r"probe.nix.*(No such file or directory|does not exist)", logs), "Expected error message not found in log" + # don't put probe.nix in source control - should fail + deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix") + logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}") + assert re.search(r"probe.nix.*(No such file or directory|does not exist)", logs), "Expected error message not found in log" - # now it should succeed - deployer.succeed("cd /tmp/bundle && git add probe.nix") - deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}") - alpha.succeed("grep SECOND /etc/deployment") + # now it should succeed + deployer.succeed("cd /tmp/bundle && git add probe.nix") + deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}") + alpha.succeed("grep SECOND /etc/deployment") - '' + lib.optionalString pure '' - with subtest("Check that impure expressions are forbidden"): - deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix") - logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}") - assert re.search(r"access to absolute path.*forbidden in pure (eval|evaluation) mode", logs), "Expected error message not found in log" + '' + + lib.optionalString pure '' + with subtest("Check that impure expressions are forbidden"): + deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix") + logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}") + assert re.search(r"access to absolute path.*forbidden in pure (eval|evaluation) mode", logs), "Expected error message not found in log" - with subtest("Check that impure expressions can be allowed with --impure"): - deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags} --impure") - alpha.succeed("grep deployer /etc/deployment") - ''; + with subtest("Check that impure expressions can be allowed with --impure"): + deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags} --impure") + alpha.succeed("grep deployer /etc/deployment") + ''; }; } diff --git a/integration-tests/flakes/flake.nix b/integration-tests/flakes/flake.nix index 3346774..17e1157 100644 --- a/integration-tests/flakes/flake.nix +++ b/integration-tests/flakes/flake.nix @@ -6,12 +6,19 @@ colmena.url = "@colmena@"; }; - outputs = { self, nixpkgs, colmena }: let - pkgs = import nixpkgs { - system = "x86_64-linux"; + outputs = + { + self, + nixpkgs, + colmena, + }: + let + pkgs = import nixpkgs { + system = "x86_64-linux"; + }; + in + { + colmena = import ./hive.nix { inherit pkgs; }; + colmenaHive = colmena.lib.makeHive self.outputs.colmena; }; - in { - colmena = import ./hive.nix { inherit pkgs; }; - colmenaHive = colmena.lib.makeHive self.outputs.colmena; - }; } diff --git a/integration-tests/flakes/hive.nix b/integration-tests/flakes/hive.nix index c2cefbe..14ec27c 100644 --- a/integration-tests/flakes/hive.nix +++ b/integration-tests/flakes/hive.nix @@ -6,7 +6,8 @@ let insideVm = true; targets = [ "alpha" ]; }; -in { +in +{ meta = { nixpkgs = tools.pkgs; }; diff --git a/integration-tests/nixpkgs-stable.nix b/integration-tests/nixpkgs-stable.nix index d0f3057..65b0e0d 100644 --- a/integration-tests/nixpkgs-stable.nix +++ b/integration-tests/nixpkgs-stable.nix @@ -1,6 +1,7 @@ let flake = (import ../flake-compat.nix).defaultNix; -in import flake.inputs.stable.outPath { +in +import flake.inputs.stable.outPath { overlays = [ flake.overlay diff --git a/integration-tests/nixpkgs.nix b/integration-tests/nixpkgs.nix index 9dc1c7d..0c65cbd 100644 --- a/integration-tests/nixpkgs.nix +++ b/integration-tests/nixpkgs.nix @@ -1,6 +1,7 @@ let flake = (import ../flake-compat.nix).defaultNix; -in import flake.inputs.nixpkgs.outPath { +in +import flake.inputs.nixpkgs.outPath { overlays = [ flake.overlay diff --git a/integration-tests/parallel/default.nix b/integration-tests/parallel/default.nix index 70434b1..d4d929d 100644 --- a/integration-tests/parallel/default.nix +++ b/integration-tests/parallel/default.nix @@ -1,8 +1,9 @@ { pkgs }: let - tools = pkgs.callPackage ../tools.nix {}; -in tools.runTest { + tools = pkgs.callPackage ../tools.nix { }; +in +tools.runTest { name = "colmena-parallel"; colmena.test = { diff --git a/integration-tests/parallel/hive.nix b/integration-tests/parallel/hive.nix index 1c2674e..8f5d4d4 100644 --- a/integration-tests/parallel/hive.nix +++ b/integration-tests/parallel/hive.nix @@ -1,6 +1,7 @@ let tools = import ./tools.nix { insideVm = true; }; -in { +in +{ meta = { nixpkgs = tools.pkgs; }; diff --git a/integration-tests/tools.nix b/integration-tests/tools.nix index 22a395f..4019399 100644 --- a/integration-tests/tools.nix +++ b/integration-tests/tools.nix @@ -5,14 +5,19 @@ # # TODO: Modularize most of this -{ insideVm ? false -, deployers ? [ "deployer" ] # Nodes configured as deployers (with Colmena and pre-built system closure) -, targets ? [ "alpha" "beta" "gamma" ] # Nodes configured as targets (minimal config) -, extraDeployerConfig ? {} # Extra config on the deployer -, prebuiltTarget ? "alpha" # Target node to prebuild system closure for, or null +{ + insideVm ? false, + deployers ? [ "deployer" ], # Nodes configured as deployers (with Colmena and pre-built system closure) + targets ? [ + "alpha" + "beta" + "gamma" + ], # Nodes configured as targets (minimal config) + extraDeployerConfig ? { }, # Extra config on the deployer + prebuiltTarget ? "alpha", # Target node to prebuild system closure for, or null -, pkgs ? if insideVm then import {} else throw "Must specify pkgs" -, colmena ? if !insideVm then pkgs.colmena else throw "Cannot eval inside VM" + pkgs ? if insideVm then import { } else throw "Must specify pkgs", + colmena ? if !insideVm then pkgs.colmena else throw "Cannot eval inside VM", }: with builtins; @@ -28,94 +33,106 @@ let sshKeys = import (pkgs.path + "/nixos/tests/ssh-keys.nix") pkgs; nixosLib = import (pkgs.path + "/nixos/lib") { }; - inputClosureOf = pkg: pkgs.runCommand "full-closure" { - refs = pkgs.writeReferencesToFile pkg.drvPath; - } '' - touch $out + inputClosureOf = + pkg: + pkgs.runCommand "full-closure" + { + refs = pkgs.writeReferencesToFile pkg.drvPath; + } + '' + touch $out - while read ref; do - case $ref in - *.drv) - cat $ref >>$out - ;; - esac - done <$refs - ''; + while read ref; do + case $ref in + *.drv) + cat $ref >>$out + ;; + esac + done <$refs + ''; ## The modular NixOS test framework with Colmena additions - colmenaTestModule = { lib, config, ... }: let - cfg = config.colmena.test; + colmenaTestModule = + { lib, config, ... }: + let + cfg = config.colmena.test; - targetList = "[${concatStringsSep ", " targets}]"; - bundle = pkgs.stdenv.mkDerivation { - name = "${config.name}-bundle"; - dontUnpack = true; - dontInstall = true; - buildPhase = '' - cp -r ${cfg.bundle} $out - chmod u+w $out - cp ${./tools.nix} $out/tools.nix - ''; - }; - in { - options = { - colmena.test = { - bundle = lib.mkOption { - description = '' - Path to a directory to copy into the deployer as /tmp/bundle. - ''; - type = lib.types.path; - }; + targetList = "[${concatStringsSep ", " targets}]"; + bundle = pkgs.stdenv.mkDerivation { + name = "${config.name}-bundle"; + dontUnpack = true; + dontInstall = true; + buildPhase = '' + cp -r ${cfg.bundle} $out + chmod u+w $out + cp ${./tools.nix} $out/tools.nix + ''; + }; + in + { + options = { + colmena.test = { + bundle = lib.mkOption { + description = '' + Path to a directory to copy into the deployer as /tmp/bundle. + ''; + type = lib.types.path; + }; - testScript = lib.mkOption { - description = '' - The test script. + testScript = lib.mkOption { + description = '' + The test script. - The Colmena test framework will prepend initialization - statements to the actual test script. - ''; - type = lib.types.str; + The Colmena test framework will prepend initialization + statements to the actual test script. + ''; + type = lib.types.str; + }; }; }; + config = { + testScript = + '' + start_all() + '' + + lib.optionalString (prebuiltTarget != null) '' + deployer.succeed("nix-store -qR ${prebuiltSystem}") + '' + + '' + deployer.succeed("nix-store -qR ${pkgs.path}") + deployer.succeed("ln -sf ${pkgs.path} /nixpkgs") + deployer.succeed("mkdir -p /root/.ssh && touch /root/.ssh/id_rsa && chmod 600 /root/.ssh/id_rsa && cat ${sshKeys.snakeOilPrivateKey} > /root/.ssh/id_rsa") + + ${lib.optionalString (length targets != 0) '' + for node in ${targetList}: + node.wait_for_unit("sshd.service") + deployer.wait_until_succeeds(f"ssh -o StrictHostKeyChecking=accept-new {node.name} true", timeout=30) + ''} + + deployer.succeed("cp --no-preserve=mode -r ${bundle} /tmp/bundle && chmod u+w /tmp/bundle") + + orig_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n")) + def get_new_store_paths(): + cur_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n")) + new_store_paths = cur_store_paths.difference(orig_store_paths) + deployer.log(f"{len(new_store_paths)} store paths were created") + + l = list(map(lambda n: f"/nix/store/{n}", new_store_paths)) + return l + + ${cfg.testScript} + ''; + }; }; - config = { - testScript = '' - start_all() - '' + lib.optionalString (prebuiltTarget != null) '' - deployer.succeed("nix-store -qR ${prebuiltSystem}") - '' + '' - deployer.succeed("nix-store -qR ${pkgs.path}") - deployer.succeed("ln -sf ${pkgs.path} /nixpkgs") - deployer.succeed("mkdir -p /root/.ssh && touch /root/.ssh/id_rsa && chmod 600 /root/.ssh/id_rsa && cat ${sshKeys.snakeOilPrivateKey} > /root/.ssh/id_rsa") - - ${lib.optionalString (length targets != 0) '' - for node in ${targetList}: - node.wait_for_unit("sshd.service") - deployer.wait_until_succeeds(f"ssh -o StrictHostKeyChecking=accept-new {node.name} true", timeout=30) - ''} - - deployer.succeed("cp --no-preserve=mode -r ${bundle} /tmp/bundle && chmod u+w /tmp/bundle") - - orig_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n")) - def get_new_store_paths(): - cur_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n")) - new_store_paths = cur_store_paths.difference(orig_store_paths) - deployer.log(f"{len(new_store_paths)} store paths were created") - - l = list(map(lambda n: f"/nix/store/{n}", new_store_paths)) - return l - - ${cfg.testScript} - ''; + evalTest = + module: + nixosLib.evalTest { + imports = [ + module + colmenaTestModule + { hostPkgs = pkgs; } + ]; }; - }; - evalTest = module: nixosLib.evalTest { - imports = [ - module - colmenaTestModule - { hostPkgs = pkgs; } - ]; - }; ## Common setup @@ -124,103 +141,137 @@ let # We include the input closure of a prebuilt system profile # so it can build system profiles for the targets without # network access. - deployerConfig = { pkgs, lib, config, ... }: { - imports = [ - extraDeployerConfig - ]; - - nix.registry = lib.mkIf (pkgs ? _inputs) { - nixpkgs.flake = pkgs._inputs.nixpkgs; - }; - - nix.nixPath = [ - "nixpkgs=${pkgs.path}" - ]; - - nix.settings.substituters = lib.mkForce []; - - virtualisation = { - memorySize = 6144; - writableStore = true; - additionalPaths = [ - "${pkgs.path}" - ] ++ lib.optionals (prebuiltTarget != null) [ - prebuiltSystem - (inputClosureOf prebuiltSystem) + deployerConfig = + { + pkgs, + lib, + config, + ... + }: + { + imports = [ + extraDeployerConfig ]; + + nix.registry = lib.mkIf (pkgs ? _inputs) { + nixpkgs.flake = pkgs._inputs.nixpkgs; + }; + + nix.nixPath = [ + "nixpkgs=${pkgs.path}" + ]; + + nix.settings.substituters = lib.mkForce [ ]; + + virtualisation = { + memorySize = 6144; + writableStore = true; + additionalPaths = + [ + "${pkgs.path}" + ] + ++ lib.optionals (prebuiltTarget != null) [ + prebuiltSystem + (inputClosureOf prebuiltSystem) + ]; + }; + + services.openssh.enable = true; + users.users.root.openssh.authorizedKeys.keys = [ + sshKeys.snakeOilPublicKey + ]; + + environment.systemPackages = with pkgs; [ + git # for git flake tests + inotify-tools # for key services build + + # HACK: copy stderr to both stdout and stderr + # (the test framework only captures stdout, and only stderr appears on screen during the build) + (writeShellScriptBin "run-copy-stderr" '' + exec "$@" 2> >(tee /dev/stderr) + '') + ]; + + # Re-enable switch-to-configuration + system.switch.enable = true; }; - services.openssh.enable = true; - users.users.root.openssh.authorizedKeys.keys = [ - sshKeys.snakeOilPublicKey - ]; - - environment.systemPackages = with pkgs; [ - git # for git flake tests - inotify-tools # for key services build - - # HACK: copy stderr to both stdout and stderr - # (the test framework only captures stdout, and only stderr appears on screen during the build) - (writeShellScriptBin "run-copy-stderr" '' - exec "$@" 2> >(tee /dev/stderr) - '') - ]; - - # Re-enable switch-to-configuration - system.switch.enable = true; - }; - # Setup for target nodes # # Kept as minimal as possible. - targetConfig = { lib, ... }: { - nix.settings.substituters = lib.mkForce []; + targetConfig = + { lib, ... }: + { + nix.settings.substituters = lib.mkForce [ ]; - documentation.nixos.enable = lib.mkOverride 60 true; + documentation.nixos.enable = lib.mkOverride 60 true; - services.openssh.enable = true; - users.users.root.openssh.authorizedKeys.keys = [ - sshKeys.snakeOilPublicKey - ]; - virtualisation.writableStore = true; + services.openssh.enable = true; + users.users.root.openssh.authorizedKeys.keys = [ + sshKeys.snakeOilPublicKey + ]; + virtualisation.writableStore = true; - # Re-enable switch-to-configuration - system.switch.enable = true; - }; + # Re-enable switch-to-configuration + system.switch.enable = true; + }; - nodes = let - deployerNodes = map (name: lib.nameValuePair name deployerConfig) deployers; - targetNodes = map (name: lib.nameValuePair name targetConfig) targets; - in listToAttrs (deployerNodes ++ targetNodes); + nodes = + let + deployerNodes = map (name: lib.nameValuePair name deployerConfig) deployers; + targetNodes = map (name: lib.nameValuePair name targetConfig) targets; + in + listToAttrs (deployerNodes ++ targetNodes); # A "shallow" re-evaluation of the test for use from Colmena - standaloneTest = evalTest ({ ... }: { - inherit nodes; - }); + standaloneTest = evalTest ( + { ... }: + { + inherit nodes; + } + ); prebuiltSystem = standaloneTest.config.nodes.${prebuiltTarget}.system.build.toplevel; - getStandaloneConfigFor = node: { lib, config, ... }: { - imports = [ - (pkgs.path + "/nixos/lib/testing/nixos-test-base.nix") - (if elem node deployers then deployerConfig else targetConfig) - standaloneTest.config.nodes.${node}.system.build.networkConfig - ]; + getStandaloneConfigFor = + node: + { lib, config, ... }: + { + imports = [ + (pkgs.path + "/nixos/lib/testing/nixos-test-base.nix") + (if elem node deployers then deployerConfig else targetConfig) + standaloneTest.config.nodes.${node}.system.build.networkConfig + ]; - documentation.nixos.enable = lib.mkOverride 55 false; - boot.loader.grub.enable = false; - system.nixos.revision = lib.mkForce "constant-nixos-revision"; + documentation.nixos.enable = lib.mkOverride 55 false; + boot.loader.grub.enable = false; + system.nixos.revision = lib.mkForce "constant-nixos-revision"; - nix.nixPath = lib.mkForce [ "nixpkgs=/nixpkgs" ]; + nix.nixPath = lib.mkForce [ "nixpkgs=/nixpkgs" ]; - deployment.tags = lib.optional (config.networking.hostName != "deployer") "target"; - }; -in { - inherit pkgs nodes colmena colmenaExec - getStandaloneConfigFor inputClosureOf; + deployment.tags = lib.optional (config.networking.hostName != "deployer") "target"; + }; +in +{ + inherit + pkgs + nodes + colmena + colmenaExec + getStandaloneConfigFor + inputClosureOf + ; - runTest = module: (evalTest ({ config, ... }: { - imports = [ module { inherit nodes; } ]; - result = config.test; - })).config.result; + runTest = + module: + (evalTest ( + { config, ... }: + { + imports = [ + module + { inherit nodes; } + ]; + result = config.test; + } + )).config.result; } diff --git a/manual/colorized-help.nix b/manual/colorized-help.nix index c59639b..e37c827 100644 --- a/manual/colorized-help.nix +++ b/manual/colorized-help.nix @@ -1,33 +1,52 @@ -{ lib, stdenv, runCommand, colmena, ansi2html }: +{ + lib, + stdenv, + runCommand, + colmena, + ansi2html, +}: with builtins; let - subcommands = [ - null - "apply" - ] - ++ lib.optional stdenv.isLinux "apply-local" - ++ [ - "build" - "upload-keys" - "eval" - "exec" - "nix-info" - "repl" - ]; - renderHelp = subcommand: let - fullCommand = if subcommand == null then "colmena" else "colmena ${subcommand}"; - in '' - ( - echo '## `${fullCommand}`' - echo -n '
'
-        TERM=xterm-256color CLICOLOR_FORCE=1 ${fullCommand} --help | ansi2html -p
-        echo '
' - )>>$out - ''; -in runCommand "colmena-colorized-help" { - nativeBuildInputs = [ colmena ansi2html ]; -} ('' - ansi2html -H > $out -'' + concatStringsSep "\n" (map renderHelp subcommands)) + subcommands = + [ + null + "apply" + ] + ++ lib.optional stdenv.isLinux "apply-local" + ++ [ + "build" + "upload-keys" + "eval" + "exec" + "nix-info" + "repl" + ]; + renderHelp = + subcommand: + let + fullCommand = if subcommand == null then "colmena" else "colmena ${subcommand}"; + in + '' + ( + echo '## `${fullCommand}`' + echo -n '
'
+          TERM=xterm-256color CLICOLOR_FORCE=1 ${fullCommand} --help | ansi2html -p
+          echo '
' + )>>$out + ''; +in +runCommand "colmena-colorized-help" + { + nativeBuildInputs = [ + colmena + ansi2html + ]; + } + ( + '' + ansi2html -H > $out + '' + + concatStringsSep "\n" (map renderHelp subcommands) + ) diff --git a/manual/default.nix b/manual/default.nix index 45199b2..d4bba4c 100644 --- a/manual/default.nix +++ b/manual/default.nix @@ -1,23 +1,33 @@ -{ lib, stdenv, nix-gitignore, mdbook, mdbook-linkcheck, python3, callPackage, writeScript -, deploymentOptionsMd ? null -, metaOptionsMd ? null -, colmena ? null +{ + lib, + stdenv, + nix-gitignore, + mdbook, + mdbook-linkcheck, + python3, + callPackage, + writeScript, + deploymentOptionsMd ? null, + metaOptionsMd ? null, + colmena ? null, -# Full version -, version ? if colmena != null then colmena.version else "unstable" + # Full version + version ? if colmena != null then colmena.version else "unstable", -# Whether this build is unstable -, unstable ? version == "unstable" || lib.hasInfix "-" version + # Whether this build is unstable + unstable ? version == "unstable" || lib.hasInfix "-" version, }: let apiVersion = builtins.concatStringsSep "." (lib.take 2 (lib.splitString "." version)); - colorizedHelp = let - help = callPackage ./colorized-help.nix { - inherit colmena; - }; - in if colmena != null then help else null; + colorizedHelp = + let + help = callPackage ./colorized-help.nix { + inherit colmena; + }; + in + if colmena != null then help else null; redirectTemplate = lib.escapeShellArg '' @@ -33,16 +43,29 @@ let ''; -in stdenv.mkDerivation { - inherit version deploymentOptionsMd metaOptionsMd colorizedHelp; +in +stdenv.mkDerivation { + inherit + version + deploymentOptionsMd + metaOptionsMd + colorizedHelp + ; pname = "colmena-manual" + (if unstable then "-unstable" else ""); - src = nix-gitignore.gitignoreSource [] ./.; + src = nix-gitignore.gitignoreSource [ ] ./.; - nativeBuildInputs = [ mdbook mdbook-linkcheck python3 ]; + nativeBuildInputs = [ + mdbook + mdbook-linkcheck + python3 + ]; - outputs = [ "out" "redirectFarm" ]; + outputs = [ + "out" + "redirectFarm" + ]; COLMENA_VERSION = version; COLMENA_UNSTABLE = unstable; diff --git a/package.nix b/package.nix index c0379ec..1eeef81 100644 --- a/package.nix +++ b/package.nix @@ -1,9 +1,10 @@ -{ lib -, stdenv -, rustPlatform -, nix-gitignore -, installShellFiles -, nix-eval-jobs +{ + lib, + stdenv, + rustPlatform, + nix-gitignore, + installShellFiles, + nix-eval-jobs, }: rustPlatform.buildRustPackage rec { @@ -20,7 +21,7 @@ rustPlatform.buildRustPackage rec { buildInputs = [ nix-eval-jobs ]; - NIX_EVAL_JOBS = "${nix-eval-jobs}/bin/nix-eval-jobs"; + env.NIX_EVAL_JOBS = lib.getExe nix-eval-jobs; preBuild = '' if [[ -z "$NIX_EVAL_JOBS" ]]; then diff --git a/shell.nix b/shell.nix index 9044128..2cb468c 100644 --- a/shell.nix +++ b/shell.nix @@ -1,3 +1 @@ -let - flake = import ./flake-compat.nix; -in flake.shellNix +(import ./flake-compat.nix).shellNix diff --git a/src/nix/hive/eval.nix b/src/nix/hive/eval.nix index 26f17e5..735ad08 100644 --- a/src/nix/hive/eval.nix +++ b/src/nix/hive/eval.nix @@ -1,197 +1,269 @@ -{ rawHive ? null # Colmena Hive attrset -, rawFlake ? null # Nix Flake attrset with `outputs.colmena` -, hermetic ? rawFlake != null # Whether we are allowed to use -, colmenaOptions ? import ./options.nix -, colmenaModules ? import ./modules.nix +{ + rawHive ? null, # Colmena Hive attrset + rawFlake ? null, # Nix Flake attrset with `outputs.colmena` + hermetic ? rawFlake != null, # Whether we are allowed to use }: -with builtins; let defaultHive = { # Will be set in defaultHiveMeta - meta = {}; + meta = { }; # Like in NixOps, there is a special host named `defaults` # containing configurations that will be applied to all # hosts. - defaults = {}; + defaults = { }; }; + uncheckedHive = + let + flakeToHive = + rawFlake: + if rawFlake.outputs ? colmena then + rawFlake.outputs.colmena + else + throw "Flake must define outputs.colmena."; - uncheckedHive = let - flakeToHive = rawFlake: - if rawFlake.outputs ? colmena then rawFlake.outputs.colmena else throw "Flake must define outputs.colmena."; - - rawToHive = rawHive: - if typeOf rawHive == "lambda" || rawHive ? __functor then rawHive {} - else if typeOf rawHive == "set" then rawHive - else throw "The config must evaluate to an attribute set."; - in - if rawHive != null then rawToHive rawHive - else if rawFlake != null then flakeToHive rawFlake - else throw "Either a plain Hive attribute set or a Nix Flake attribute set must be specified."; + rawToHive = + rawHive: + if (builtins.isFunction rawHive) || rawHive ? __functor then + rawHive { } + else if (builtins.isAttrs rawHive) then + rawHive + else + throw "The config must evaluate to an attribute set."; + in + if rawHive != null then + rawToHive rawHive + else if rawFlake != null then + flakeToHive rawFlake + else + throw "Either a plain Hive attribute set or a Nix Flake attribute set must be specified."; uncheckedUserMeta = if uncheckedHive ? meta && uncheckedHive ? network then throw "Only one of `network` and `meta` may be specified. `meta` should be used as `network` is for NixOps compatibility." - else if uncheckedHive ? meta then uncheckedHive.meta - else if uncheckedHive ? network then uncheckedHive.network - else {}; + else if uncheckedHive ? meta then + uncheckedHive.meta + else if uncheckedHive ? network then + uncheckedHive.network + else + { }; - uncheckedRegistries = if uncheckedHive ? registry then uncheckedHive.registry else {}; + uncheckedRegistries = if uncheckedHive ? registry then uncheckedHive.registry else { }; # The final hive will always have the meta key instead of network. - hive = let - userMeta = (lib.modules.evalModules { - modules = [ colmenaOptions.metaOptions uncheckedUserMeta ]; - }).config; + hive = + let + userMeta = + (lib.modules.evalModules { + modules = [ + ./options/meta.nix + uncheckedUserMeta + ]; + }).config; - registry = (lib.modules.evalModules { - modules = [ colmenaOptions.registryOptions { registry = uncheckedRegistries; } ]; - }).config.registry; + registry = + (lib.modules.evalModules { + modules = [ + ./options/registry.nix + { registry = uncheckedRegistries; } + ]; + }).config.registry; - mergedHive = - assert lib.assertMsg (!(uncheckedHive ? __schema)) '' - You cannot pass in an already-evaluated Hive into the evaluator. + mergedHive = + assert lib.assertMsg (!(uncheckedHive ? __schema)) '' + You cannot pass in an already-evaluated Hive into the evaluator. - Hint: Use the `colmenaHive` output instead of `colmena`. - ''; - removeAttrs (defaultHive // uncheckedHive) [ "meta" "network" "registry" ]; + Hint: Use the `colmenaHive` output instead of `colmena`. + ''; + removeAttrs (defaultHive // uncheckedHive) [ + "meta" + "network" + "registry" + ]; - meta = { - meta = - if !hermetic && userMeta.nixpkgs == null - then userMeta // { nixpkgs = ; } - else userMeta; - }; - in mergedHive // meta // { inherit registry; }; + meta = { + meta = + if !hermetic && userMeta.nixpkgs == null then userMeta // { nixpkgs = ; } else userMeta; + }; + in + mergedHive // meta // { inherit registry; }; - configsFor = node: let - nodeConfig = hive.${node}; - in - assert lib.assertMsg (!elem node reservedNames) "\"${node}\" is a reserved name and cannot be used as the name of a node"; - if typeOf nodeConfig == "list" then nodeConfig - else [ nodeConfig ]; + configsFor = + node: + let + nodeConfig = hive.${node}; + in + assert lib.assertMsg ( + !builtins.elem node reservedNames + ) "\"${node}\" is a reserved name and cannot be used as the name of a node"; + if (builtins.isList nodeConfig) then nodeConfig else [ nodeConfig ]; - mkNixpkgs = configName: pkgConf: let - uninitializedError = typ: '' - Passing ${typ} as ${configName} is no longer accepted with Flakes. - Please initialize Nixpkgs like the following: + mkNixpkgs = + configName: pkgConf: + let + uninitializedError = typ: '' + Passing ${typ} as ${configName} is no longer accepted with Flakes. + Please initialize Nixpkgs like the following: - { - # ... - outputs = { nixpkgs, ... }: { - colmena = { - ${configName} = import nixpkgs { - system = "x86_64-linux"; # Set your desired system here - overlays = []; + { + # ... + outputs = { nixpkgs, ... }: { + colmena = { + ${configName} = import nixpkgs { + system = "x86_64-linux"; # Set your desired system here + overlays = []; + }; }; }; - }; - } - ''; - in - if typeOf pkgConf == "path" || (typeOf pkgConf == "set" && pkgConf ? outPath) then - if hermetic then throw (uninitializedError "a path to Nixpkgs") + } + ''; + in + if (builtins.isPath pkgConf) || ((builtins.isAttrs pkgConf) && pkgConf ? outPath) then + if hermetic then + throw (uninitializedError "a path to Nixpkgs") # The referenced file might return an initialized Nixpkgs attribute set directly - else mkNixpkgs configName (import pkgConf) - else if typeOf pkgConf == "lambda" then - if hermetic then throw (uninitializedError "a Nixpkgs lambda") - else pkgConf { overlays = []; } - else if typeOf pkgConf == "set" then - if pkgConf ? outputs then throw (uninitializedError "an uninitialized Nixpkgs input") - else pkgConf - else throw '' - ${configName} must be one of: + else + mkNixpkgs configName (import pkgConf) + else if (builtins.isFunction pkgConf) then + if hermetic then throw (uninitializedError "a Nixpkgs lambda") else pkgConf { overlays = [ ]; } + else if (builtins.isAttrs pkgConf) then + if pkgConf ? outputs then throw (uninitializedError "an uninitialized Nixpkgs input") else pkgConf + else + throw '' + ${configName} must be one of: - - A path to Nixpkgs (e.g., ) - - A Nixpkgs lambda (e.g., import ) - - A Nixpkgs attribute set - ''; + - A path to Nixpkgs (e.g., ) + - A Nixpkgs lambda (e.g., import ) + - A Nixpkgs attribute set + ''; - nixpkgs = let - # Can't rely on the module system yet - nixpkgsConf = - if uncheckedUserMeta ? nixpkgs then uncheckedUserMeta.nixpkgs - else if hermetic then throw "meta.nixpkgs must be specified in hermetic mode." - else ; - in mkNixpkgs "meta.nixpkgs" nixpkgsConf; + nixpkgs = + let + # Can't rely on the module system yet + nixpkgsConf = + if uncheckedUserMeta ? nixpkgs then + uncheckedUserMeta.nixpkgs + else if hermetic then + throw "meta.nixpkgs must be specified in hermetic mode." + else + ; + in + mkNixpkgs "meta.nixpkgs" nixpkgsConf; lib = nixpkgs.lib; - reservedNames = [ "defaults" "network" "meta" "registry" ]; + reservedNames = [ + "defaults" + "network" + "meta" + "registry" + ]; - evalNode = name: configs: - # Some help on error messages. - assert (lib.assertMsg (lib.hasAttrByPath [ "deployment" "systemType" ] hive.${name}) - "${name} does not have a deployment system type!"); - assert (lib.assertMsg (builtins.typeOf hive.registry == "set")) - "The hive's registry is not a set, but of type '${builtins.typeOf hive.registry}'"; - assert (lib.assertMsg (lib.hasAttr hive.${name}.deployment.systemType hive.registry) - "${builtins.toJSON (hive.${name}.deployment.systemType)} does not exist in the registry of systems!"); - let - # We cannot use `configs` because we need to access to the raw configuration fragment. - inherit (hive.registry.${hive.${name}.deployment.systemType}) evalConfig; - npkgs = - if hasAttr name hive.meta.nodeNixpkgs - then mkNixpkgs "meta.nodeNixpkgs.${name}" hive.meta.nodeNixpkgs.${name} - else nixpkgs; + evalNode = + name: configs: + # Some help on error messages. + assert ( + lib.assertMsg (lib.hasAttrByPath [ + "deployment" + "systemType" + ] hive.${name}) "${name} does not have a deployment system type!" + ); + assert (lib.assertMsg ( + builtins.isAttrs hive.registry + )) "The hive's registry is not a set, but of type '${builtins.typeOf hive.registry}'"; + assert ( + lib.assertMsg (lib.hasAttr hive.${name}.deployment.systemType hive.registry) + "${builtins.toJSON (hive.${name}.deployment.systemType)} does not exist in the registry of systems!" + ); + let + # We cannot use `configs` because we need to access to the raw configuration fragment. + inherit (hive.registry.${hive.${name}.deployment.systemType}) evalConfig; + npkgs = + if builtins.hasAttr name hive.meta.nodeNixpkgs then + mkNixpkgs "meta.nodeNixpkgs.${name}" hive.meta.nodeNixpkgs.${name} + else + nixpkgs; - # Here we need to merge the configurations in meta.nixpkgs - # and in machine config. - nixpkgsModule = { config, lib, ... }: let - hasTypedConfig = lib.versionAtLeast lib.version "22.11pre"; - in { - nixpkgs.overlays = lib.mkBefore npkgs.overlays; - nixpkgs.config = if hasTypedConfig then lib.mkBefore npkgs.config else lib.mkOptionDefault npkgs.config; + # Here we need to merge the configurations in meta.nixpkgs + # and in machine config. + nixpkgsModule = + { config, lib, ... }: + let + hasTypedConfig = lib.versionAtLeast lib.version "22.11pre"; + in + { + nixpkgs.overlays = lib.mkBefore npkgs.overlays; + nixpkgs.config = + if hasTypedConfig then lib.mkBefore npkgs.config else lib.mkOptionDefault npkgs.config; - warnings = let - # Before 22.11, most config keys were untyped thus the merging - # was broken. Let's warn the user if not all config attributes - # set in meta.nixpkgs are overridden. - metaKeys = attrNames npkgs.config; - nodeKeys = [ "doCheckByDefault" "warnings" "allowAliases" ] ++ (attrNames config.nixpkgs.config); - remainingKeys = filter (k: ! elem k nodeKeys) metaKeys; - in - lib.optional (!hasTypedConfig && length remainingKeys != 0) - "The following Nixpkgs configuration keys set in meta.nixpkgs will be ignored: ${toString remainingKeys}"; - } // lib.optionalAttrs (builtins.hasAttr "localSystem" npkgs || builtins.hasAttr "crossSystem" npkgs) { - nixpkgs.localSystem = lib.mkBefore npkgs.localSystem; - nixpkgs.crossSystem = lib.mkBefore npkgs.crossSystem; - }; - in evalConfig { - # This doesn't exist for `evalModules` the generic way. - # inherit (npkgs) system; + warnings = + let + # Before 22.11, most config keys were untyped thus the merging + # was broken. Let's warn the user if not all config attributes + # set in meta.nixpkgs are overridden. + metaKeys = builtins.attrNames npkgs.config; + nodeKeys = [ + "doCheckByDefault" + "warnings" + "allowAliases" + ] ++ (builtins.attrNames config.nixpkgs.config); + remainingKeys = builtins.filter (k: !builtins.elem k nodeKeys) metaKeys; + in + lib.optional (!hasTypedConfig && builtins.length remainingKeys != 0) + "The following Nixpkgs configuration keys set in meta.nixpkgs will be ignored: ${toString remainingKeys}"; + } + // + lib.optionalAttrs (builtins.hasAttr "localSystem" npkgs || builtins.hasAttr "crossSystem" npkgs) + { + nixpkgs.localSystem = lib.mkBefore npkgs.localSystem; + nixpkgs.crossSystem = lib.mkBefore npkgs.crossSystem; + }; + in + evalConfig { + # This doesn't exist for `evalModules` the generic way. + # inherit (npkgs) system; - modules = [ - nixpkgsModule - colmenaModules.assertionModule - colmenaOptions.deploymentOptions - (hive.registry.${hive.${name}.deployment.systemType}.defaults or hive.defaults) - ] ++ configs; - specialArgs = { - inherit name; - nodes = uncheckedNodes; - } // hive.meta.specialArgs // (hive.meta.nodeSpecialArgs.${name} or {}); - }; + modules = [ + nixpkgsModule + ./modules/assertions.nix + ./options/deployment.nix + (hive.registry.${hive.${name}.deployment.systemType}.defaults or hive.defaults) + ] ++ configs; + specialArgs = + { + inherit name; + nodes = uncheckedNodes; + } + // hive.meta.specialArgs + // (hive.meta.nodeSpecialArgs.${name} or { }); + }; - nodeNames = filter (name: ! elem name reservedNames) (attrNames hive); + nodeNames = builtins.filter (name: !builtins.elem name reservedNames) (builtins.attrNames hive); # Used as the `nodes` argument in modules. We skip recursive type checking # for performance. - uncheckedNodes = listToAttrs (map (name: let - configs = [ + uncheckedNodes = builtins.listToAttrs ( + builtins.map ( + name: + let + configs = [ + { + _module.check = false; + } + ] ++ configsFor name; + in { - _module.check = false; + inherit name; + value = evalNode name configs; } - ] ++ configsFor name; - in { - inherit name; - value = evalNode name configs; - }) nodeNames); + ) nodeNames + ); # Add required config Key here since we don't want to eval nixpkgs metaConfigKeys = [ - "name" "description" + "name" + "description" "machinesFile" "allowApplyAll" ]; @@ -200,20 +272,35 @@ let "supportsDeployment" ]; -in rec { +in +rec { # Exported attributes __schema = "v0.20241006"; - nodes = listToAttrs (map (name: { inherit name; value = evalNode name (configsFor name); }) nodeNames); - toplevel = lib.mapAttrs (_: v: v.config.system.build.toplevel) nodes; - deploymentConfig = lib.mapAttrs (_: v: v.config.deployment) nodes; - deploymentConfigSelected = names: lib.filterAttrs (name: _: elem name names) deploymentConfig; - evalSelected = names: lib.filterAttrs (name: _: elem name names) toplevel; - evalSelectedDrvPaths = names: lib.mapAttrs (_: v: v.drvPath) (evalSelected names); - metaConfig = lib.filterAttrs (n: v: elem n metaConfigKeys) hive.meta; + nodes = builtins.listToAttrs ( + builtins.map (name: { + inherit name; + value = evalNode name (configsFor name); + }) nodeNames + ); + toplevel = lib.mapAttrs (_: v: v.config.system.build.toplevel) nodes; + deploymentConfig = lib.mapAttrs (_: v: v.config.deployment) nodes; + deploymentConfigSelected = + names: lib.filterAttrs (name: _: builtins.elem name names) deploymentConfig; + evalSelected = names: lib.filterAttrs (name: _: builtins.elem name names) toplevel; + evalSelectedDrvPaths = names: lib.mapAttrs (_: v: v.drvPath) (evalSelected names); + metaConfig = lib.filterAttrs (n: v: builtins.elem n metaConfigKeys) hive.meta; # We cannot perform a `metaConfigKeys`-style simple check here # because registry is arbitrarily deep and may evaluate nixpkgs indirectly. - registryConfig = lib.mapAttrs (systemTypeName: systemType: - lib.filterAttrs (n: v: elem n serializableSystemTypeConfigKeys) systemType) hive.registry; - introspect = f: f { inherit lib; pkgs = nixpkgs; inherit nodes; }; + registryConfig = lib.mapAttrs ( + systemTypeName: systemType: + lib.filterAttrs (n: v: builtins.elem n serializableSystemTypeConfigKeys) systemType + ) hive.registry; + introspect = + f: + f { + inherit lib; + pkgs = nixpkgs; + inherit nodes; + }; } diff --git a/src/nix/hive/flake.nix b/src/nix/hive/flake.nix index f40d528..0191634 100644 --- a/src/nix/hive/flake.nix +++ b/src/nix/hive/flake.nix @@ -5,39 +5,43 @@ hive.url = "%hive%"; }; - outputs = { self, hive }: { - processFlake = let - compatibleSchema = "v0.20241006"; + outputs = + { hive, ... }: + { + processFlake = + let + compatibleSchema = "v0.20241006"; - # Evaluates a raw hive. - # - # This uses the `colmena` output. - evalHive = rawFlake: import ./eval.nix { - inherit rawFlake; - hermetic = true; - colmenaOptions = import ./options.nix; - colmenaModules = import ./modules.nix; - }; + # Evaluates a raw hive. + # + # This uses the `colmena` output. + evalHive = + rawFlake: + import ./eval.nix { + inherit rawFlake; + hermetic = true; + }; - # Uses an already-evaluated hive. - # - # This uses the `colmenaHive` output. - checkPreparedHive = hiveOutput: - if !(hiveOutput ? __schema) then - throw '' - The colmenaHive output does not contain a valid evaluated hive. + # Uses an already-evaluated hive. + # + # This uses the `colmenaHive` output. + checkPreparedHive = + hiveOutput: + if !(hiveOutput ? __schema) then + throw '' + The colmenaHive output does not contain a valid evaluated hive. - Hint: Use `colmena.lib.makeHive`. - '' - else if hiveOutput.__schema != compatibleSchema then - throw '' - The colmenaHive output (schema ${hiveOutput.__schema}) isn't compatible with this version of Colmena. + Hint: Use `colmena.lib.makeHive`. + '' + else if hiveOutput.__schema != compatibleSchema then + throw '' + The colmenaHive output (schema ${hiveOutput.__schema}) isn't compatible with this version of Colmena. - Hint: Use the same version of Colmena as in the Flake input. - '' - else hiveOutput; - in - if hive.outputs ? colmenaHive then checkPreparedHive hive.outputs.colmenaHive - else evalHive hive; - }; + Hint: Use the same version of Colmena as in the Flake input. + '' + else + hiveOutput; + in + if hive.outputs ? colmenaHive then checkPreparedHive hive.outputs.colmenaHive else evalHive hive; + }; } diff --git a/src/nix/hive/modules.nix b/src/nix/hive/modules.nix deleted file mode 100644 index 2dbe1a8..0000000 --- a/src/nix/hive/modules.nix +++ /dev/null @@ -1,109 +0,0 @@ -with builtins; { - assertionModule = { config, lib, ... }: { - assertions = lib.mapAttrsToList (key: opts: let - nonNulls = l: filter (x: x != null) l; - in { - assertion = length (nonNulls [opts.text opts.keyCommand opts.keyFile]) == 1; - message = - let prefix = "${name}.deployment.keys.${key}"; - in "Exactly one of `${prefix}.text`, `${prefix}.keyCommand` and `${prefix}.keyFile` must be set."; - }) config.deployment.keys; - }; - - # Change the ownership of all keys uploaded pre-activation - # - # This is built as part of the system profile. - # We must be careful not to access `text` / `keyCommand` / `keyFile` here - keyChownModule = { lib, config, ... }: let - preActivationKeys = lib.filterAttrs (name: key: key.uploadAt == "pre-activation") config.deployment.keys; - scriptDeps = if config.system.activationScripts ? groups then [ "groups" ] else [ "users" ]; - - commands = lib.mapAttrsToList (name: key: let - keyPath = "${key.destDir}/${name}"; - in '' - if [ -f "${keyPath}" ]; then - if ! chown ${key.user}:${key.group} "${keyPath}"; then - # Error should be visible in stderr - failed=1 - fi - else - >&2 echo "Key ${keyPath} does not exist. Skipping chown." - fi - '') preActivationKeys; - - script = lib.stringAfter scriptDeps '' - # This script is injected by Colmena to change the ownerships - # of keys (`deployment.keys`) deployed before system activation. - - >&2 echo "setting up key ownerships..." - - # We set the ownership of as many keys as possible before failing - failed= - - ${concatStringsSep "\n" commands} - - if [ -n "$failed" ]; then - >&2 echo "Failed to set the ownership of some keys." - - # The activation script has a trap to handle failed - # commands and print out various debug information. - # Let's trigger that instead of `exit 1`. - false - fi - ''; - in { - system.activationScripts.colmena-chown-keys = lib.mkIf (length commands != 0) script; - }; - - # Create "${name}-key" services for NixOps compatibility - # - # This is built as part of the system profile. - # We must be careful not to access `text` / `keyCommand` / `keyFile` here - # - # Sadly, path units don't automatically deactivate the bound units when - # the key files are deleted, so we use inotifywait in the services' scripts. - # - # - keyServiceModule = { pkgs, lib, config, ... }: { - systemd.paths = lib.mapAttrs' (name: val: { - name = "${name}-key"; - value = { - wantedBy = [ "paths.target" ]; - pathConfig = { - PathExists = val.path; - }; - }; - }) config.deployment.keys; - - systemd.services = lib.mapAttrs' (name: val: { - name = "${name}-key"; - value = { - enable = true; - serviceConfig = { - TimeoutStartSec = "infinity"; - Restart = "always"; - RestartSec = "100ms"; - }; - path = [ pkgs.inotify-tools ]; - preStart = '' - (while read f; do if [ "$f" = "${val.name}" ]; then break; fi; done \ - < <(inotifywait -qm --format '%f' -e create,move ${val.destDir}) ) & - if [[ -e "${val.path}" ]]; then - echo 'flapped down' - kill %1 - exit 0 - fi - wait %1 - ''; - script = '' - inotifywait -qq -e delete_self "${val.path}" & - if [[ ! -e "${val.path}" ]]; then - echo 'flapped up' - exit 0 - fi - wait %1 - ''; - }; - }) config.deployment.keys; - }; -} diff --git a/src/nix/hive/modules/assertions.nix b/src/nix/hive/modules/assertions.nix new file mode 100644 index 0000000..b543b83 --- /dev/null +++ b/src/nix/hive/modules/assertions.nix @@ -0,0 +1,28 @@ +{ + config, + lib, + name, + ... +}: + +let + inherit (lib) filter mapAttrsToList; +in + +{ + assertions = mapAttrsToList (key: opts: { + assertion = + builtins.length ( + filter (x: x != null) [ + opts.text + opts.keyCommand + opts.keyFile + ] + ) == 1; + message = + let + prefix = "${name}.deployment.keys.${key}"; + in + "Exactly one of `${prefix}.text`, `${prefix}.keyCommand` and `${prefix}.keyFile` must be set."; + }) config.deployment.keys; +} diff --git a/src/nix/hive/modules/key-chown.nix b/src/nix/hive/modules/key-chown.nix new file mode 100644 index 0000000..6704d94 --- /dev/null +++ b/src/nix/hive/modules/key-chown.nix @@ -0,0 +1,64 @@ +# Change the ownership of all keys uploaded pre-activation +# +# This is built as part of the system profile. +# We must be careful not to access `text` / `keyCommand` / `keyFile` here + +{ config, lib, ... }: + +let + inherit (lib) + concatStringsSep + filterAttrs + mapAttrsToList + mkIf + stringAfter + ; + + preActivationKeys = filterAttrs ( + _: { uploadAt, ... }: uploadAt == "pre-activation" + ) config.deployment.keys; + + scriptDeps = if config.system.activationScripts ? groups then [ "groups" ] else [ "users" ]; + + commands = mapAttrsToList ( + name: key: + let + keyPath = "${key.destDir}/${name}"; + in + '' + if [ -f "${keyPath}" ]; then + if ! chown ${key.user}:${key.group} "${keyPath}"; then + # Error should be visible in stderr + failed=1 + fi + else + >&2 echo "Key ${keyPath} does not exist. Skipping chown." + fi + '' + ) preActivationKeys; + + script = stringAfter scriptDeps '' + # This script is injected by Colmena to change the ownerships + # of keys (`deployment.keys`) deployed before system activation. + + >&2 echo "setting up key ownerships..." + + # We set the ownership of as many keys as possible before failing + failed= + + ${concatStringsSep "\n" commands} + + if [ -n "$failed" ]; then + >&2 echo "Failed to set the ownership of some keys." + + # The activation script has a trap to handle failed + # commands and print out various debug information. + # Let's trigger that instead of `exit 1`. + false + fi + ''; +in + +{ + system.activationScripts.colmena-chown-keys = mkIf (commands != [ ]) script; +} diff --git a/src/nix/hive/modules/key-service.nix b/src/nix/hive/modules/key-service.nix new file mode 100644 index 0000000..6907db6 --- /dev/null +++ b/src/nix/hive/modules/key-service.nix @@ -0,0 +1,61 @@ +# Create "${name}-key" services for NixOps compatibility +# +# This is built as part of the system profile. +# We must be careful not to access `text` / `keyCommand` / `keyFile` here +# +# Sadly, path units don't automatically deactivate the bound units when +# the key files are deleted, so we use inotifywait in the services' scripts. +# +# + +{ + config, + lib, + pkgs, + ... +}: + +let + inherit (lib) mapAttrs' nameValuePair; +in + +{ + systemd.paths = mapAttrs' ( + name: value: + nameValuePair "${name}-key" { + wantedBy = [ "paths.target" ]; + pathConfig.PathExists = value.path; + } + ) config.deployment.keys; + + systemd.services = mapAttrs' ( + name: value: + nameValuePair "${name}-key" { + enable = true; + serviceConfig = { + TimeoutStartSec = "infinity"; + Restart = "always"; + RestartSec = "100ms"; + }; + path = [ pkgs.inotify-tools ]; + preStart = '' + (while read f; do if [ "$f" = "${value.name}" ]; then break; fi; done \ + < <(inotifywait -qm --format '%f' -e create,move ${value.destDir}) ) & + if [[ -e "${value.path}" ]]; then + echo 'flapped down' + kill %1 + exit 0 + fi + wait %1 + ''; + script = '' + inotifywait -qq -e delete_self "${value.path}" & + if [[ ! -e "${value.path}" ]]; then + echo 'flapped up' + exit 0 + fi + wait %1 + ''; + } + ) config.deployment.keys; +} diff --git a/src/nix/hive/options.nix b/src/nix/hive/options.nix deleted file mode 100644 index 04108ff..0000000 --- a/src/nix/hive/options.nix +++ /dev/null @@ -1,368 +0,0 @@ -with builtins; rec { - keyType = { lib, name, config, ... }: let - inherit (lib) types; - in { - options = { - name = lib.mkOption { - description = '' - File name of the key. - ''; - default = name; - type = types.str; - }; - text = lib.mkOption { - description = '' - Content of the key. - One of `text`, `keyCommand` and `keyFile` must be set. - ''; - default = null; - type = types.nullOr types.str; - }; - keyFile = lib.mkOption { - description = '' - Path of the local file to read the key from. - One of `text`, `keyCommand` and `keyFile` must be set. - ''; - default = null; - apply = value: if value == null then null else toString value; - type = types.nullOr types.path; - }; - keyCommand = lib.mkOption { - description = '' - Command to run to generate the key. - One of `text`, `keyCommand` and `keyFile` must be set. - ''; - default = null; - type = let - nonEmptyList = types.addCheck (types.listOf types.str) (l: length l > 0); - in types.nullOr nonEmptyList; - }; - destDir = lib.mkOption { - description = '' - Destination directory on the host. - ''; - default = "/run/keys"; - type = types.path; - }; - path = lib.mkOption { - description = '' - Full path to the destination. - ''; - default = "${config.destDir}/${config.name}"; - type = types.path; - internal = true; - }; - user = lib.mkOption { - description = '' - The group that will own the file. - ''; - default = "root"; - type = types.str; - }; - group = lib.mkOption { - description = '' - The group that will own the file. - ''; - default = "root"; - type = types.str; - }; - permissions = lib.mkOption { - description = '' - Permissions to set for the file. - ''; - default = "0600"; - type = types.str; - }; - uploadAt = lib.mkOption { - description = '' - When to upload the keys. - - - pre-activation (default): Upload the keys before activating the new system profile. - - post-activation: Upload the keys after successfully activating the new system profile. - - For `colmena upload-keys`, all keys are uploaded at the same time regardless of the configuration here. - ''; - default = "pre-activation"; - type = types.enum [ "pre-activation" "post-activation" ]; - }; - }; - }; - - # Colmena-specific options - # - # Largely compatible with NixOps/Morph. - deploymentOptions = { name, lib, ... }: let - inherit (lib) types; - in { - options = { - deployment = { - systemType = lib.mkOption { - description = mdDoc '' - System type used for this node, e.g. NixOS. - ''; - default = "nixos"; - # TODO: enum among all registered systems? - type = types.str; - }; - targetHost = lib.mkOption { - description = '' - The target SSH node for deployment. - - By default, the node's attribute name will be used. - If set to null, only local deployment will be supported. - ''; - type = types.nullOr types.str; - default = name; - }; - targetPort = lib.mkOption { - description = '' - The target SSH port for deployment. - - By default, the port is the standard port (22) or taken - from your ssh_config. - ''; - type = types.nullOr types.ints.unsigned; - default = null; - }; - targetUser = lib.mkOption { - description = '' - The user to use to log into the remote node. If set to null, the - target user will not be specified in SSH invocations. - ''; - type = types.nullOr types.str; - default = "root"; - }; - allowLocalDeployment = lib.mkOption { - description = '' - Allow the configuration to be applied locally on the host running - Colmena. - - For local deployment to work, all of the following must be true: - - The node must be running NixOS. - - The node must have deployment.allowLocalDeployment set to true. - - The node's networking.hostName must match the hostname. - - To apply the configurations locally, run `colmena apply-local`. - You can also set deployment.targetHost to null if the nost is not - accessible over SSH (only local deployment will be possible). - ''; - type = types.bool; - default = false; - }; - buildOnTarget = lib.mkOption { - description = '' - Whether to build the system profiles on the target node itself. - - When enabled, Colmena will copy the derivation to the target - node and initiate the build there. This avoids copying back the - build results involved with the native distributed build - feature. Furthermore, the `build` goal will be equivalent to - the `push` goal. Since builds happen on the target node, the - results are automatically "pushed" and won't exist in the local - Nix store. - - You can temporarily override per-node settings by passing - `--build-on-target` (enable for all nodes) or - `--no-build-on-target` (disable for all nodes) on the command - line. - ''; - type = types.bool; - default = false; - }; - tags = lib.mkOption { - description = '' - A list of tags for the node. - - Can be used to select a group of nodes for deployment. - ''; - type = types.listOf types.str; - default = []; - }; - keys = lib.mkOption { - description = '' - A set of secrets to be deployed to the node. - - Secrets are transferred to the node out-of-band and - never ends up in the Nix store. - ''; - type = types.attrsOf (types.submodule keyType); - default = {}; - }; - replaceUnknownProfiles = lib.mkOption { - description = '' - Allow a configuration to be applied to a host running a profile we - have no knowledge of. By setting this option to false, you reduce - the likelyhood of rolling back changes made via another Colmena user. - - Unknown profiles are usually the result of either: - - The node had a profile applied, locally or by another Colmena. - - The host running Colmena garbage-collecting the profile. - - To force profile replacement on all targeted nodes during apply, - use the flag `--force-replace-unknown-profiles`. - ''; - type = types.bool; - default = true; - }; - privilegeEscalationCommand = lib.mkOption { - description = '' - Command to use to elevate privileges when activating the new profiles on SSH hosts. - - This is used on SSH hosts when `deployment.targetUser` is not `root`. - The user must be allowed to use the command non-interactively. - ''; - type = types.listOf types.str; - default = [ "sudo" "-H" "--" ]; - }; - sshOptions = lib.mkOption { - description = '' - Extra SSH options to pass to the SSH command. - ''; - type = types.listOf types.str; - default = []; - }; - }; - }; - }; - # Options for a registered system type - systemTypeOptions = { name, lib, ... }: let - inherit (lib) types; - mdDoc = lib.mdDoc or lib.id; - in - { - options = { - evalConfig = lib.mkOption { - description = mdDoc '' - Evaluation function which share the same interface as `nixos/lib/eval-config.nix` - which can be tailored to your own usecases or to target another type of system, - e.g. nix-darwin. - ''; - type = types.functionTo types.unspecified; - }; - supportsDeployment = lib.mkOption { - description = mdDoc '' - Whether this system type supports deployment or not. - - If it supports deployment, it needs to have appropriate activation code, - refer to how to write custom activators. - ''; - default = name == "nixos"; - defaultText = "If a NixOS system, then true, otherwise false by default"; - }; - defaults = lib.mkOption { - description = mdDoc '' - Default configuration for that system type. - ''; - type = types.functionTo types.unspecified; - default = _: {}; - }; - }; - }; - registryOptions = { lib, ... }: let - inherit (lib) types; - mdDoc = lib.mdDoc or lib.id; - in - { - options.registry = lib.mkOption { - description = mdDoc '' - A registry of all system types. - ''; - type = types.attrsOf (types.submodule systemTypeOptions); - }; - }; - # Hive-wide options - metaOptions = { lib, ... }: let - inherit (lib) types; - in { - options = { - name = lib.mkOption { - description = '' - The name of the configuration. - ''; - type = types.str; - default = "hive"; - }; - description = lib.mkOption { - description = '' - A short description for the configuration. - ''; - type = types.str; - default = "A Colmena Hive"; - }; - nixpkgs = lib.mkOption { - description = '' - The pinned Nixpkgs package set. Accepts one of the following: - - - A path to a Nixpkgs checkout - - The Nixpkgs lambda (e.g., import ) - - An initialized Nixpkgs attribute set - - This option must be specified when using Flakes. - ''; - type = types.unspecified; - default = null; - }; - nodeNixpkgs = lib.mkOption { - description = '' - Node-specific Nixpkgs pins. - ''; - type = types.attrsOf types.unspecified; - default = {}; - }; - nodeSpecialArgs = lib.mkOption { - description = '' - Node-specific special args. - ''; - type = types.attrsOf types.unspecified; - default = {}; - }; - machinesFile = lib.mkOption { - description = '' - Use the machines listed in this file when building this hive configuration. - - If your Colmena host has nix configured to allow for remote builds - (for nix-daemon, your user being included in trusted-users) - you can set a machines file that will be passed to the underlying - nix-store command during derivation realization as a builders option. - For example, if you support multiple orginizations each with their own - build machine(s) you can ensure that builds only take place on your - local machine and/or the machines specified in this file. - - See https://nixos.org/manual/nix/stable/advanced-topics/distributed-builds - for the machine specification format. - - This option is ignored when builds are initiated on the remote nodes - themselves via `deployment.buildOnTarget` or `--build-on-target`. To - still use the Nix distributed build functionality, configure the - builders on the target nodes with `nix.buildMachines`. - ''; - default = null; - apply = value: if value == null then null else toString value; - type = types.nullOr types.path; - }; - specialArgs = lib.mkOption { - description = '' - A set of special arguments to be passed to NixOS modules. - - This will be merged into the `specialArgs` used to evaluate - the NixOS configurations. - ''; - default = {}; - type = types.attrsOf types.unspecified; - }; - allowApplyAll = lib.mkOption { - description = '' - Whether to allow deployments without a node filter set. - - If set to false, a node filter must be specified with `--on` when - deploying. - - It helps prevent accidental deployments to the entire cluster - when tags are used (e.g., `@production` and `@staging`). - ''; - default = true; - type = types.bool; - }; - }; - }; -} diff --git a/src/nix/hive/options/deployment.nix b/src/nix/hive/options/deployment.nix new file mode 100644 index 0000000..6717fc2 --- /dev/null +++ b/src/nix/hive/options/deployment.nix @@ -0,0 +1,151 @@ +{ lib, name, ... }: + +let + inherit (lib) mkOption; + + inherit (lib.types) + attrsOf + bool + ints + listOf + nullOr + str + submodule + ; +in + +{ + options = { + deployment = { + systemType = mkOption { + description = '' + System type used for this node, e.g. NixOS. + ''; + default = "nixos"; + # TODO: enum among all registered systems? + type = str; + }; + targetHost = mkOption { + description = '' + The target SSH node for deployment. + + By default, the node's attribute name will be used. + If set to null, only local deployment will be supported. + ''; + type = nullOr str; + default = name; + }; + targetPort = mkOption { + description = '' + The target SSH port for deployment. + + By default, the port is the standard port (22) or taken + from your ssh_config. + ''; + type = nullOr ints.unsigned; + default = null; + }; + targetUser = mkOption { + description = '' + The user to use to log into the remote node. If set to null, the + target user will not be specified in SSH invocations. + ''; + type = nullOr str; + default = "root"; + }; + allowLocalDeployment = mkOption { + description = '' + Allow the configuration to be applied locally on the host running + Colmena. + + For local deployment to work, all of the following must be true: + - The node must be running NixOS. + - The node must have deployment.allowLocalDeployment set to true. + - The node's networking.hostName must match the hostname. + + To apply the configurations locally, run `colmena apply-local`. + You can also set deployment.targetHost to null if the nost is not + accessible over SSH (only local deployment will be possible). + ''; + type = bool; + default = false; + }; + buildOnTarget = mkOption { + description = '' + Whether to build the system profiles on the target node itself. + + When enabled, Colmena will copy the derivation to the target + node and initiate the build there. This avoids copying back the + build results involved with the native distributed build + feature. Furthermore, the `build` goal will be equivalent to + the `push` goal. Since builds happen on the target node, the + results are automatically "pushed" and won't exist in the local + Nix store. + + You can temporarily override per-node settings by passing + `--build-on-target` (enable for all nodes) or + `--no-build-on-target` (disable for all nodes) on the command + line. + ''; + type = bool; + default = false; + }; + tags = mkOption { + description = '' + A list of tags for the node. + + Can be used to select a group of nodes for deployment. + ''; + type = listOf str; + default = [ ]; + }; + keys = mkOption { + description = '' + A set of secrets to be deployed to the node. + + Secrets are transferred to the node out-of-band and + never ends up in the Nix store. + ''; + type = attrsOf (submodule (import ./key.nix)); + default = { }; + }; + replaceUnknownProfiles = mkOption { + description = '' + Allow a configuration to be applied to a host running a profile we + have no knowledge of. By setting this option to false, you reduce + the likelyhood of rolling back changes made via another Colmena user. + + Unknown profiles are usually the result of either: + - The node had a profile applied, locally or by another Colmena. + - The host running Colmena garbage-collecting the profile. + + To force profile replacement on all targeted nodes during apply, + use the flag `--force-replace-unknown-profiles`. + ''; + type = bool; + default = true; + }; + privilegeEscalationCommand = mkOption { + description = '' + Command to use to elevate privileges when activating the new profiles on SSH hosts. + + This is used on SSH hosts when `deployment.targetUser` is not `root`. + The user must be allowed to use the command non-interactively. + ''; + type = listOf str; + default = [ + "sudo" + "-H" + "--" + ]; + }; + sshOptions = mkOption { + description = '' + Extra SSH options to pass to the SSH command. + ''; + type = listOf str; + default = [ ]; + }; + }; + }; +} diff --git a/src/nix/hive/options/key.nix b/src/nix/hive/options/key.nix new file mode 100644 index 0000000..a6f56d2 --- /dev/null +++ b/src/nix/hive/options/key.nix @@ -0,0 +1,120 @@ +{ + config, + lib, + name, + ... +}: + +let + inherit (lib) mkOption; + + inherit (lib.types) + addCheck + enum + listOf + nullOr + path + str + ; +in + +{ + options = { + name = mkOption { + description = '' + File name of the key. + ''; + default = name; + type = str; + }; + + text = mkOption { + description = '' + Content of the key. + One of `text`, `keyCommand` and `keyFile` must be set. + ''; + default = null; + type = nullOr str; + }; + + keyFile = mkOption { + description = '' + Path of the local file to read the key from. + One of `text`, `keyCommand` and `keyFile` must be set. + ''; + default = null; + apply = value: if value == null then null else builtins.toString value; + type = nullOr path; + }; + + keyCommand = mkOption { + description = '' + Command to run to generate the key. + One of `text`, `keyCommand` and `keyFile` must be set. + ''; + default = null; + type = + let + nonEmptyList = addCheck (listOf str) (l: builtins.length l > 0); + in + nullOr nonEmptyList; + }; + + destDir = mkOption { + description = '' + Destination directory on the host. + ''; + default = "/run/keys"; + type = path; + }; + + path = mkOption { + description = '' + Full path to the destination. + ''; + default = "${config.destDir}/${config.name}"; + type = path; + internal = true; + }; + + user = mkOption { + description = '' + The group that will own the file. + ''; + default = "root"; + type = str; + }; + + group = mkOption { + description = '' + The group that will own the file. + ''; + default = "root"; + type = str; + }; + + permissions = mkOption { + description = '' + Permissions to set for the file. + ''; + default = "0600"; + type = str; + }; + + uploadAt = mkOption { + description = '' + When to upload the keys. + + - pre-activation (default): Upload the keys before activating the new system profile. + - post-activation: Upload the keys after successfully activating the new system profile. + + For `colmena upload-keys`, all keys are uploaded at the same time regardless of the configuration here. + ''; + default = "pre-activation"; + type = enum [ + "pre-activation" + "post-activation" + ]; + }; + }; +} diff --git a/src/nix/hive/options/meta.nix b/src/nix/hive/options/meta.nix new file mode 100644 index 0000000..c0c2441 --- /dev/null +++ b/src/nix/hive/options/meta.nix @@ -0,0 +1,114 @@ +{ lib, ... }: + +let + inherit (lib) mkOption; + + inherit (lib.types) + attrsOf + bool + nullOr + path + str + unspecified + ; +in + +{ + options = { + name = mkOption { + description = '' + The name of the configuration. + ''; + type = str; + default = "hive"; + }; + + description = mkOption { + description = '' + A short description for the configuration. + ''; + type = str; + default = "A Colmena Hive"; + }; + + nixpkgs = mkOption { + description = '' + The pinned Nixpkgs package set. Accepts one of the following: + + - A path to a Nixpkgs checkout + - The Nixpkgs lambda (e.g., import ) + - An initialized Nixpkgs attribute set + + This option must be specified when using Flakes. + ''; + type = unspecified; + default = null; + }; + + nodeNixpkgs = mkOption { + description = '' + Node-specific Nixpkgs pins. + ''; + type = attrsOf unspecified; + default = { }; + }; + + nodeSpecialArgs = mkOption { + description = '' + Node-specific special args. + ''; + type = attrsOf unspecified; + default = { }; + }; + + machinesFile = mkOption { + description = '' + Use the machines listed in this file when building this hive configuration. + + If your Colmena host has nix configured to allow for remote builds + (for nix-daemon, your user being included in trusted-users) + you can set a machines file that will be passed to the underlying + nix-store command during derivation realization as a builders option. + For example, if you support multiple orginizations each with their own + build machine(s) you can ensure that builds only take place on your + local machine and/or the machines specified in this file. + + See https://nixos.org/manual/nix/stable/advanced-topics/distributed-builds + for the machine specification format. + + This option is ignored when builds are initiated on the remote nodes + themselves via `deployment.buildOnTarget` or `--build-on-target`. To + still use the Nix distributed build functionality, configure the + builders on the target nodes with `nix.buildMachines`. + ''; + default = null; + apply = value: if value == null then null else builtins.toString value; + type = nullOr path; + }; + + specialArgs = mkOption { + description = '' + A set of special arguments to be passed to NixOS modules. + + This will be merged into the `specialArgs` used to evaluate + the NixOS configurations. + ''; + default = { }; + type = attrsOf unspecified; + }; + + allowApplyAll = mkOption { + description = '' + Whether to allow deployments without a node filter set. + + If set to false, a node filter must be specified with `--on` when + deploying. + + It helps prevent accidental deployments to the entire cluster + when tags are used (e.g., `@production` and `@staging`). + ''; + default = true; + type = bool; + }; + }; +} diff --git a/src/nix/hive/options/registry.nix b/src/nix/hive/options/registry.nix new file mode 100644 index 0000000..9881ca4 --- /dev/null +++ b/src/nix/hive/options/registry.nix @@ -0,0 +1,54 @@ +{ lib, ... }: + +let + inherit (lib) mkOption; + + inherit (lib.types) + attrsOf + functionTo + submodule + unspecified + ; +in + +{ + options.registry = mkOption { + description = '' + A registry of all system types. + ''; + type = attrsOf ( + submodule ( + { name, ... }: + { + options = { + evalConfig = mkOption { + description = '' + Evaluation function which share the same interface as `nixos/lib/eval-config.nix` + which can be tailored to your own usecases or to target another type of system, + e.g. nix-darwin. + ''; + type = functionTo unspecified; + }; + supportsDeployment = mkOption { + description = '' + Whether this system type supports deployment or not. + + If it supports deployment, it needs to have appropriate activation code, + refer to how to write custom activators. + ''; + default = name == "nixos"; + defaultText = "If a NixOS system, then true, otherwise false by default"; + }; + defaults = mkOption { + description = '' + Default configuration for that system type. + ''; + type = functionTo unspecified; + default = _: { }; + }; + }; + } + ) + ); + }; +} diff --git a/src/nix/hive/tests/makehive-flake/flake.nix b/src/nix/hive/tests/makehive-flake/flake.nix index 11c7f62..0b38070 100644 --- a/src/nix/hive/tests/makehive-flake/flake.nix +++ b/src/nix/hive/tests/makehive-flake/flake.nix @@ -3,27 +3,36 @@ nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; colmena.url = "git+file://@repoPath@"; }; - outputs = { nixpkgs, colmena, ... }: { - colmenaHive = colmena.lib.makeHive { - meta = { - nixpkgs = import nixpkgs { - system = "x86_64-linux"; + outputs = + { nixpkgs, colmena, ... }: + { + colmenaHive = colmena.lib.makeHive { + meta = { + nixpkgs = import nixpkgs { + system = "x86_64-linux"; + }; }; - }; - host-a = { name, nodes, pkgs, ... }: { - boot.isContainer = true; - time.timeZone = nodes.host-b.config.time.timeZone; - }; - host-b = { - deployment = { - targetHost = "somehost.tld"; - targetPort = 1234; - targetUser = "luser"; + host-a = + { + name, + nodes, + pkgs, + ... + }: + { + boot.isContainer = true; + time.timeZone = nodes.host-b.config.time.timeZone; + }; + host-b = { + deployment = { + targetHost = "somehost.tld"; + targetPort = 1234; + targetUser = "luser"; + }; + boot.isContainer = true; + time.timeZone = "America/Los_Angeles"; }; - boot.isContainer = true; - time.timeZone = "America/Los_Angeles"; }; }; - }; } diff --git a/src/nix/hive/tests/simple-flake/flake.nix b/src/nix/hive/tests/simple-flake/flake.nix index 91bdc34..9f84ad7 100644 --- a/src/nix/hive/tests/simple-flake/flake.nix +++ b/src/nix/hive/tests/simple-flake/flake.nix @@ -2,27 +2,36 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; }; - outputs = { nixpkgs, ... }: { - colmena = { - meta = { - nixpkgs = import nixpkgs { - system = "x86_64-linux"; + outputs = + { nixpkgs, ... }: + { + colmena = { + meta = { + nixpkgs = import nixpkgs { + system = "x86_64-linux"; + }; }; - }; - host-a = { name, nodes, pkgs, ... }: { - boot.isContainer = true; - time.timeZone = nodes.host-b.config.time.timeZone; - }; - host-b = { - deployment = { - targetHost = "somehost.tld"; - targetPort = 1234; - targetUser = "luser"; + host-a = + { + name, + nodes, + pkgs, + ... + }: + { + boot.isContainer = true; + time.timeZone = nodes.host-b.config.time.timeZone; + }; + host-b = { + deployment = { + targetHost = "somehost.tld"; + targetPort = 1234; + targetUser = "luser"; + }; + boot.isContainer = true; + time.timeZone = "America/Los_Angeles"; }; - boot.isContainer = true; - time.timeZone = "America/Los_Angeles"; }; }; - }; }