DGNum/colmena
10
0
Fork 2
Code Issues 6 Pull requests 3 Projects Releases Packages Wiki Activity

Misc cleanup of nix files #15

Open
thubrecht wants to merge 5 commits from cleanup into main
pull from: cleanup
merge into: DGNum:main
Conversation 0 Commits 5 Files changed 39 +1562 -1156
39 changed files with 1562 additions and 1156 deletions
Download patch file Download diff file Expand all files Collapse all files

4
default.nix
View file

@ -1,3 +1 @@
let (import ./flake-compat.nix).defaultNix.default
flake = import ./flake-compat.nix;
in flake.defaultNix.default

12
flake-compat.nix
View file

@ -1,9 +1,7 @@
let let
lock = builtins.fromJSON (builtins.readFile ./flake.lock); lock = builtins.fromJSON (builtins.readFile ./flake.lock);
flakeCompat = import (fetchTarball { in
url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; import (builtins.fetchTarball {
sha256 = lock.nodes.flake-compat.locked.narHash; url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
}); sha256 = lock.nodes.flake-compat.locked.narHash;
in flakeCompat { }) { src = ./.; }
src = ./.;
}

260
flake.nix
View file

@ -18,119 +18,167 @@
}; };
}; };
outputs = { outputs =
self, {
nixpkgs, self,
stable, nixpkgs,
flake-utils, stable,
nix-github-actions, flake-utils,
... nix-github-actions,
} @ inputs: let ...
supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; }@inputs:
colmenaOptions = import ./src/nix/hive/options.nix; let
colmenaModules = import ./src/nix/hive/modules.nix; supportedSystems = [
in flake-utils.lib.eachSystem supportedSystems (system: let "x86_64-linux"
pkgs = import nixpkgs { "i686-linux"
inherit system; "aarch64-linux"
overlays = [ "x86_64-darwin"
"aarch64-darwin"
]; ];
}; in
in rec { flake-utils.lib.eachSystem supportedSystems (
# We still maintain the expression in a Nixpkgs-acceptable form system:
defaultPackage = self.packages.${system}.colmena; let
packages = rec { pkgs = import nixpkgs { inherit system; };
colmena = pkgs.callPackage ./package.nix { }; in
rec {
# We still maintain the expression in a Nixpkgs-acceptable form
defaultPackage = self.packages.${system}.colmena;
packages = rec {
colmena = pkgs.callPackage ./package.nix { };
# Full user manual # Full user manual
manual = let manual =
suppressModuleArgsDocs = { lib, ... }: { let
options = { suppressModuleArgsDocs =
_module.args = lib.mkOption { { lib, ... }:
internal = true; {
options = {
_module.args = lib.mkOption {
internal = true;
};
};
};
colmena = self.packages.${system}.colmena;
deploymentOptionsMd =
(pkgs.nixosOptionsDoc {
inherit
(pkgs.lib.evalModules {
modules = [
./src/nix/hive/options/deployment.nix
suppressModuleArgsDocs
];
specialArgs = {
name = "nixos";
nodes = { };
};
})
options
;
}).optionsCommonMark;
metaOptionsMd =
(pkgs.nixosOptionsDoc {
inherit
(pkgs.lib.evalModules {
modules = [
./src/nix/hive/options/meta.nix
suppressModuleArgsDocs
];
})
options
;
}).optionsCommonMark;
in
pkgs.callPackage ./manual {
inherit colmena deploymentOptionsMd metaOptionsMd;
}; };
};
# User manual without the CLI reference
manualFast = manual.override { colmena = null; };
# User manual with the version treated as stable
manualForceStable = manual.override { unstable = false; };
}; };
colmena = self.packages.${system}.colmena;
deploymentOptionsMd = (pkgs.nixosOptionsDoc { defaultApp = self.apps.${system}.colmena;
inherit (pkgs.lib.evalModules { apps.default = self.apps.${system}.colmena;
modules = [ colmenaOptions.deploymentOptions suppressModuleArgsDocs]; apps.colmena = {
specialArgs = { name = "nixos"; nodes = {}; }; type = "app";
}) options; program = pkgs.lib.getExe defaultPackage;
}).optionsCommonMark; };
metaOptionsMd = (pkgs.nixosOptionsDoc {
inherit (pkgs.lib.evalModules { devShell = pkgs.mkShell {
modules = [ colmenaOptions.metaOptions suppressModuleArgsDocs]; RUST_SRC_PATH = pkgs.rustPlatform.rustLibSrc;
}) options; NIX_PATH = "nixpkgs=${pkgs.path}";
}).optionsCommonMark;
in pkgs.callPackage ./manual { inputsFrom = [
inherit colmena deploymentOptionsMd metaOptionsMd; defaultPackage
packages.manualFast
];
packages = with pkgs; [
bashInteractive
editorconfig-checker
clippy
rust-analyzer
cargo-outdated
cargo-audit
rustfmt
python3
python3Packages.flake8
];
};
checks =
let
inputsOverlay = final: prev: {
_inputs = inputs;
};
in
if pkgs.stdenv.isLinux then
import ./integration-tests {
pkgs = import nixpkgs {
inherit system;
overlays = [
self.overlays.default
inputsOverlay
];
};
pkgsStable = import stable {
inherit system;
overlays = [
self.overlays.default
inputsOverlay
];
};
}
else
{ };
}
)
// {
overlay = self.overlays.default;
overlays.default = final: prev: {
colmena = final.callPackage ./package.nix { };
};
nixosModules = {
deploymentOptions = import ./src/nix/hive/options/deployment.nix;
metaOptions = import ./src/nix/hive/options/meta.nix;
keyChownModule = import ./src/nix/hive/modules/key-chown.nix;
keyServiceModule = import ./src/nix/hive/modules/key-service.nix;
assertionModule = import ./src/nix/hive/modules/assertions.nix;
}; };
# User manual without the CLI reference lib.makeHive =
manualFast = manual.override { colmena = null; }; rawHive:
import ./src/nix/hive/eval.nix {
inherit rawHive;
hermetic = true;
};
# User manual with the version treated as stable githubActions = nix-github-actions.lib.mkGithubMatrix {
manualForceStable = manual.override { unstable = false; }; checks = {
}; inherit (self.checks) x86_64-linux;
};
defaultApp = self.apps.${system}.colmena;
apps.default = self.apps.${system}.colmena;
apps.colmena = {
type = "app";
program = "${defaultPackage}/bin/colmena";
};
devShell = pkgs.mkShell {
RUST_SRC_PATH = "${pkgs.rustPlatform.rustcSrc}/library";
NIX_PATH = "nixpkgs=${pkgs.path}";
inputsFrom = [ defaultPackage packages.manualFast ];
packages = with pkgs; [
bashInteractive
editorconfig-checker
clippy rust-analyzer cargo-outdated cargo-audit rustfmt
python3 python3Packages.flake8
];
};
checks = let
inputsOverlay = final: prev: {
_inputs = inputs;
};
in if pkgs.stdenv.isLinux then import ./integration-tests {
pkgs = import nixpkgs {
inherit system;
overlays = [
self.overlays.default
inputsOverlay
];
};
pkgsStable = import stable {
inherit system;
overlays = [
self.overlays.default
inputsOverlay
];
};
} else {};
}) // {
overlay = self.overlays.default;
overlays.default = final: prev: {
colmena = final.callPackage ./package.nix { };
};
nixosModules = {
inherit (colmenaOptions) deploymentOptions metaOptions;
inherit (colmenaModules) keyChownModule keyServiceModule assertionModule;
};
lib.makeHive = rawHive: import ./src/nix/hive/eval.nix {
inherit rawHive colmenaOptions colmenaModules;
hermetic = true;
};
githubActions = nix-github-actions.lib.mkGithubMatrix {
checks = {
inherit (self.checks) x86_64-linux;
}; };
}; };
};
} }

3
integration-tests/allow-apply-all/default.nix
View file

@ -4,7 +4,8 @@ let
tools = pkgs.callPackage ../tools.nix { tools = pkgs.callPackage ../tools.nix {
targets = [ "alpha" ]; targets = [ "alpha" ];
}; };
in tools.runTest { in
tools.runTest {
name = "colmena-allow-apply-all"; name = "colmena-allow-apply-all";
colmena.test = { colmena.test = {

5
integration-tests/allow-apply-all/hive.nix
View file

@ -1,9 +1,10 @@
let let
tools = import ./tools.nix { tools = import ./tools.nix {
insideVm = true; insideVm = true;
targets = ["alpha"]; targets = [ "alpha" ];
}; };
in { in
{
meta = { meta = {
nixpkgs = tools.pkgs; nixpkgs = tools.pkgs;
allowApplyAll = false; allowApplyAll = false;

5
integration-tests/apply-local/default.nix
View file

@ -2,7 +2,7 @@
let let
tools = pkgs.callPackage ../tools.nix { tools = pkgs.callPackage ../tools.nix {
targets = []; targets = [ ];
prebuiltTarget = "deployer"; prebuiltTarget = "deployer";
extraDeployerConfig = { extraDeployerConfig = {
users.users.colmena = { users.users.colmena = {
@ -12,7 +12,8 @@ let
security.sudo.wheelNeedsPassword = false; security.sudo.wheelNeedsPassword = false;
}; };
}; };
in tools.runTest { in
tools.runTest {
name = "colmena-apply-local"; name = "colmena-apply-local";
colmena.test = { colmena.test = {

31
integration-tests/apply-local/hive.nix
View file

@ -1,26 +1,29 @@
let let
tools = import ./tools.nix { tools = import ./tools.nix {
insideVm = true; insideVm = true;
targets = []; targets = [ ];
prebuiltTarget = "deployer"; prebuiltTarget = "deployer";
}; };
in { in
{
meta = { meta = {
nixpkgs = tools.pkgs; nixpkgs = tools.pkgs;
}; };
deployer = { lib, ... }: { deployer =
imports = [ { lib, ... }:
(tools.getStandaloneConfigFor "deployer") {
]; imports = [
(tools.getStandaloneConfigFor "deployer")
];
deployment = { deployment = {
allowLocalDeployment = true; allowLocalDeployment = true;
};
environment.etc."deployment".text = "SUCCESS";
# /run/keys/key-text
deployment.keys."key-text".text = "SECRET";
}; };
environment.etc."deployment".text = "SUCCESS";
# /run/keys/key-text
deployment.keys."key-text".text = "SECRET";
};
} }

20
integration-tests/apply/default.nix
View file

@ -1,17 +1,21 @@
{ pkgs {
, evaluator ? "chunked" pkgs,
evaluator ? "chunked",
}: }:
let let
tools = pkgs.callPackage ../tools.nix {}; tools = pkgs.callPackage ../tools.nix { };
in tools.runTest { in
tools.runTest {
name = "colmena-apply-${evaluator}"; name = "colmena-apply-${evaluator}";
colmena.test = { colmena.test = {
bundle = ./.; bundle = ./.;
testScript = '' testScript =
colmena = "${tools.colmenaExec}" ''
evaluator = "${evaluator}" colmena = "${tools.colmenaExec}"
'' + builtins.readFile ./test-script.py; evaluator = "${evaluator}"
''
+ builtins.readFile ./test-script.py;
}; };
} }

55
integration-tests/apply/hive.nix
View file

@ -1,13 +1,16 @@
let let
tools = import ./tools.nix { insideVm = true; }; tools = import ./tools.nix { insideVm = true; };
testPkg = let testPkg =
text = builtins.trace "must appear during evaluation" '' let
echo "must appear during build" text = builtins.trace "must appear during evaluation" ''
mkdir -p $out echo "must appear during build"
''; mkdir -p $out
in tools.pkgs.runCommand "test-package" {} text; '';
in { in
tools.pkgs.runCommand "test-package" { } text;
in
{
meta = { meta = {
nixpkgs = tools.pkgs; nixpkgs = tools.pkgs;
}; };
@ -20,7 +23,7 @@ in {
isSystemUser = true; isSystemUser = true;
group = "testgroup"; group = "testgroup";
}; };
users.groups.testgroup = {}; users.groups.testgroup = { };
# /run/keys/custom-name # /run/keys/custom-name
deployment.keys.original-name = { deployment.keys.original-name = {
@ -72,29 +75,33 @@ in {
}; };
}; };
alpha = { lib, ... }: { alpha =
imports = [ { lib, ... }:
(tools.getStandaloneConfigFor "alpha") {
]; imports = [
(tools.getStandaloneConfigFor "alpha")
];
environment.systemPackages = [ testPkg ]; environment.systemPackages = [ testPkg ];
documentation.nixos.enable = lib.mkForce true; documentation.nixos.enable = lib.mkForce true;
system.activationScripts.colmena-test.text = '' system.activationScripts.colmena-test.text = ''
echo "must appear during activation" echo "must appear during activation"
''; '';
}; };
deployer = tools.getStandaloneConfigFor "deployer"; deployer = tools.getStandaloneConfigFor "deployer";
beta = tools.getStandaloneConfigFor "beta"; beta = tools.getStandaloneConfigFor "beta";
gamma = tools.getStandaloneConfigFor "gamma"; gamma = tools.getStandaloneConfigFor "gamma";
"gamma.tld" = { lib, ... }: { "gamma.tld" =
imports = [ { lib, ... }:
(tools.getStandaloneConfigFor "gamma") {
]; imports = [
(tools.getStandaloneConfigFor "gamma")
];
deployment.tags = lib.mkForce []; deployment.tags = lib.mkForce [ ];
}; };
} }

11
integration-tests/build-on-target/default.nix
View file

@ -2,10 +2,15 @@
let let
tools = pkgs.callPackage ../tools.nix { tools = pkgs.callPackage ../tools.nix {
deployers = [ "deployer" "alpha" "beta" ]; deployers = [
targets = []; "deployer"
"alpha"
"beta"
];
targets = [ ];
}; };
in tools.runTest { in
tools.runTest {
name = "colmena-build-on-target"; name = "colmena-build-on-target";
colmena.test = { colmena.test = {

11
integration-tests/build-on-target/hive.nix
View file

@ -1,10 +1,15 @@
let let
tools = import ./tools.nix { tools = import ./tools.nix {
insideVm = true; insideVm = true;
deployers = [ "deployer" "alpha" "beta" ]; deployers = [
targets = []; "deployer"
"alpha"
"beta"
];
targets = [ ];
}; };
in { in
{
meta = { meta = {
nixpkgs = tools.pkgs; nixpkgs = tools.pkgs;
}; };

10
integration-tests/default.nix
View file

@ -1,10 +1,14 @@
{ pkgs ? import ./nixpkgs.nix {
, pkgsStable ? import ./nixpkgs-stable.nix pkgs ? import ./nixpkgs.nix,
pkgsStable ? import ./nixpkgs-stable.nix,
}: }:
{ {
apply = import ./apply { inherit pkgs; }; apply = import ./apply { inherit pkgs; };
apply-streaming = import ./apply { inherit pkgs; evaluator = "streaming"; }; apply-streaming = import ./apply {
inherit pkgs;
evaluator = "streaming";
};
apply-local = import ./apply-local { inherit pkgs; }; apply-local = import ./apply-local { inherit pkgs; };
build-on-target = import ./build-on-target { inherit pkgs; }; build-on-target = import ./build-on-target { inherit pkgs; };
exec = import ./exec { inherit pkgs; }; exec = import ./exec { inherit pkgs; };

5
integration-tests/exec/default.nix
View file

@ -1,8 +1,9 @@
{ pkgs }: { pkgs }:
let let
tools = pkgs.callPackage ../tools.nix {}; tools = pkgs.callPackage ../tools.nix { };
in tools.runTest { in
tools.runTest {
name = "colmena-exec"; name = "colmena-exec";
colmena.test = { colmena.test = {

3
integration-tests/exec/hive.nix
View file

@ -1,6 +1,7 @@
let let
tools = import ./tools.nix { insideVm = true; }; tools = import ./tools.nix { insideVm = true; };
in { in
{
meta = { meta = {
nixpkgs = tools.pkgs; nixpkgs = tools.pkgs;
}; };

79
integration-tests/flakes/default.nix
View file

@ -1,7 +1,8 @@
{ pkgs {
, evaluator ? "chunked" pkgs,
, extraApplyFlags ? "" evaluator ? "chunked",
, pure ? true extraApplyFlags ? "",
pure ? true,
}: }:
let let
@ -11,58 +12,58 @@ let
targets = [ "alpha" ]; targets = [ "alpha" ];
}; };
applyFlags = "--evaluator ${evaluator} ${extraApplyFlags}" applyFlags = "--evaluator ${evaluator} ${extraApplyFlags}" + lib.optionalString (!pure) "--impure";
+ lib.optionalString (!pure) "--impure";
# From integration-tests/nixpkgs.nix # From integration-tests/nixpkgs.nix
colmenaFlakeInputs = pkgs._inputs; colmenaFlakeInputs = pkgs._inputs;
in tools.runTest { in
name = "colmena-flakes-${evaluator}" tools.runTest {
+ lib.optionalString (!pure) "-impure"; name = "colmena-flakes-${evaluator}" + lib.optionalString (!pure) "-impure";
nodes.deployer = { nodes.deployer = {
virtualisation.additionalPaths = virtualisation.additionalPaths = lib.mapAttrsToList (k: v: v.outPath) colmenaFlakeInputs;
lib.mapAttrsToList (k: v: v.outPath) colmenaFlakeInputs;
}; };
colmena.test = { colmena.test = {
bundle = ./.; bundle = ./.;
testScript = '' testScript =
import re ''
import re
deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix") deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix")
deployer.succeed("sed -i 's @colmena@ path:${tools.colmena.src} g' /tmp/bundle/flake.nix") deployer.succeed("sed -i 's @colmena@ path:${tools.colmena.src} g' /tmp/bundle/flake.nix")
with subtest("Lock flake dependencies"): with subtest("Lock flake dependencies"):
deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock") deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock")
with subtest("Deploy with a plain flake without git"): with subtest("Deploy with a plain flake without git"):
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}") deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}")
alpha.succeed("grep FIRST /etc/deployment") alpha.succeed("grep FIRST /etc/deployment")
with subtest("Deploy with a git flake"): with subtest("Deploy with a git flake"):
deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/probe.nix") deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/probe.nix")
# don't put probe.nix in source control - should fail # don't put probe.nix in source control - should fail
deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix") deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}") logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
assert re.search(r"probe.nix.*(No such file or directory|does not exist)", logs), "Expected error message not found in log" assert re.search(r"probe.nix.*(No such file or directory|does not exist)", logs), "Expected error message not found in log"
# now it should succeed # now it should succeed
deployer.succeed("cd /tmp/bundle && git add probe.nix") deployer.succeed("cd /tmp/bundle && git add probe.nix")
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}") deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags}")
alpha.succeed("grep SECOND /etc/deployment") alpha.succeed("grep SECOND /etc/deployment")
'' + lib.optionalString pure '' ''
with subtest("Check that impure expressions are forbidden"): + lib.optionalString pure ''
deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix") with subtest("Check that impure expressions are forbidden"):
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}") deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix")
assert re.search(r"access to absolute path.*forbidden in pure (eval|evaluation) mode", logs), "Expected error message not found in log" logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target ${applyFlags}")
assert re.search(r"access to absolute path.*forbidden in pure (eval|evaluation) mode", logs), "Expected error message not found in log"
with subtest("Check that impure expressions can be allowed with --impure"): with subtest("Check that impure expressions can be allowed with --impure"):
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags} --impure") deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target ${applyFlags} --impure")
alpha.succeed("grep deployer /etc/deployment") alpha.succeed("grep deployer /etc/deployment")
''; '';
}; };
} }

21
integration-tests/flakes/flake.nix
View file

@ -6,12 +6,19 @@
colmena.url = "@colmena@"; colmena.url = "@colmena@";
}; };
outputs = { self, nixpkgs, colmena }: let outputs =
pkgs = import nixpkgs { {
system = "x86_64-linux"; self,
nixpkgs,
colmena,
}:
let
pkgs = import nixpkgs {
system = "x86_64-linux";
};
in
{
colmena = import ./hive.nix { inherit pkgs; };
colmenaHive = colmena.lib.makeHive self.outputs.colmena;
}; };
in {
colmena = import ./hive.nix { inherit pkgs; };
colmenaHive = colmena.lib.makeHive self.outputs.colmena;
};
} }

3
integration-tests/flakes/hive.nix
View file

@ -6,7 +6,8 @@ let
insideVm = true; insideVm = true;
targets = [ "alpha" ]; targets = [ "alpha" ];
}; };
in { in
{
meta = { meta = {
nixpkgs = tools.pkgs; nixpkgs = tools.pkgs;
}; };

3
integration-tests/nixpkgs-stable.nix
View file

@ -1,6 +1,7 @@
let let
flake = (import ../flake-compat.nix).defaultNix; flake = (import ../flake-compat.nix).defaultNix;
in import flake.inputs.stable.outPath { in
import flake.inputs.stable.outPath {
overlays = [ overlays = [
flake.overlay flake.overlay

3
integration-tests/nixpkgs.nix
View file

@ -1,6 +1,7 @@
let let
flake = (import ../flake-compat.nix).defaultNix; flake = (import ../flake-compat.nix).defaultNix;
in import flake.inputs.nixpkgs.outPath { in
import flake.inputs.nixpkgs.outPath {
overlays = [ overlays = [
flake.overlay flake.overlay

5
integration-tests/parallel/default.nix
View file

@ -1,8 +1,9 @@
{ pkgs }: { pkgs }:
let let
tools = pkgs.callPackage ../tools.nix {}; tools = pkgs.callPackage ../tools.nix { };
in tools.runTest { in
tools.runTest {
name = "colmena-parallel"; name = "colmena-parallel";
colmena.test = { colmena.test = {

3
integration-tests/parallel/hive.nix
View file

@ -1,6 +1,7 @@
let let
tools = import ./tools.nix { insideVm = true; }; tools = import ./tools.nix { insideVm = true; };
in { in
{
meta = { meta = {
nixpkgs = tools.pkgs; nixpkgs = tools.pkgs;
}; };

381
integration-tests/tools.nix
View file

@ -5,14 +5,19 @@
# #
# TODO: Modularize most of this # TODO: Modularize most of this
{ insideVm ? false {
, deployers ? [ "deployer" ] # Nodes configured as deployers (with Colmena and pre-built system closure) insideVm ? false,
, targets ? [ "alpha" "beta" "gamma" ] # Nodes configured as targets (minimal config) deployers ? [ "deployer" ], # Nodes configured as deployers (with Colmena and pre-built system closure)
, extraDeployerConfig ? {} # Extra config on the deployer targets ? [
, prebuiltTarget ? "alpha" # Target node to prebuild system closure for, or null "alpha"
"beta"
"gamma"
], # Nodes configured as targets (minimal config)
extraDeployerConfig ? { }, # Extra config on the deployer
prebuiltTarget ? "alpha", # Target node to prebuild system closure for, or null
, pkgs ? if insideVm then import <nixpkgs> {} else throw "Must specify pkgs" pkgs ? if insideVm then import <nixpkgs> { } else throw "Must specify pkgs",
, colmena ? if !insideVm then pkgs.colmena else throw "Cannot eval inside VM" colmena ? if !insideVm then pkgs.colmena else throw "Cannot eval inside VM",
}: }:
with builtins; with builtins;
@ -28,94 +33,106 @@ let
sshKeys = import (pkgs.path + "/nixos/tests/ssh-keys.nix") pkgs; sshKeys = import (pkgs.path + "/nixos/tests/ssh-keys.nix") pkgs;
nixosLib = import (pkgs.path + "/nixos/lib") { }; nixosLib = import (pkgs.path + "/nixos/lib") { };
inputClosureOf = pkg: pkgs.runCommand "full-closure" { inputClosureOf =
refs = pkgs.writeReferencesToFile pkg.drvPath; pkg:
} '' pkgs.runCommand "full-closure"
touch $out {
refs = pkgs.writeReferencesToFile pkg.drvPath;
}
''
touch $out
while read ref; do while read ref; do
case $ref in case $ref in
*.drv) *.drv)
cat $ref >>$out cat $ref >>$out
;; ;;
esac esac
done <$refs done <$refs
''; '';
## The modular NixOS test framework with Colmena additions ## The modular NixOS test framework with Colmena additions
colmenaTestModule = { lib, config, ... }: let colmenaTestModule =
cfg = config.colmena.test; { lib, config, ... }:
let
cfg = config.colmena.test;
targetList = "[${concatStringsSep ", " targets}]"; targetList = "[${concatStringsSep ", " targets}]";
bundle = pkgs.stdenv.mkDerivation { bundle = pkgs.stdenv.mkDerivation {
name = "${config.name}-bundle"; name = "${config.name}-bundle";
dontUnpack = true; dontUnpack = true;
dontInstall = true; dontInstall = true;
buildPhase = '' buildPhase = ''
cp -r ${cfg.bundle} $out cp -r ${cfg.bundle} $out
chmod u+w $out chmod u+w $out
cp ${./tools.nix} $out/tools.nix cp ${./tools.nix} $out/tools.nix
''; '';
}; };
in { in
options = { {
colmena.test = { options = {
bundle = lib.mkOption { colmena.test = {
description = '' bundle = lib.mkOption {
Path to a directory to copy into the deployer as /tmp/bundle. description = ''
''; Path to a directory to copy into the deployer as /tmp/bundle.
type = lib.types.path; '';
}; type = lib.types.path;
};
testScript = lib.mkOption { testScript = lib.mkOption {
description = '' description = ''
The test script. The test script.
The Colmena test framework will prepend initialization The Colmena test framework will prepend initialization
statements to the actual test script. statements to the actual test script.
''; '';
type = lib.types.str; type = lib.types.str;
};
}; };
}; };
config = {
testScript =
''
start_all()
''
+ lib.optionalString (prebuiltTarget != null) ''
deployer.succeed("nix-store -qR ${prebuiltSystem}")
''
+ ''
deployer.succeed("nix-store -qR ${pkgs.path}")
deployer.succeed("ln -sf ${pkgs.path} /nixpkgs")
deployer.succeed("mkdir -p /root/.ssh && touch /root/.ssh/id_rsa && chmod 600 /root/.ssh/id_rsa && cat ${sshKeys.snakeOilPrivateKey} > /root/.ssh/id_rsa")
${lib.optionalString (length targets != 0) ''
for node in ${targetList}:
node.wait_for_unit("sshd.service")
deployer.wait_until_succeeds(f"ssh -o StrictHostKeyChecking=accept-new {node.name} true", timeout=30)
''}
deployer.succeed("cp --no-preserve=mode -r ${bundle} /tmp/bundle && chmod u+w /tmp/bundle")
orig_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n"))
def get_new_store_paths():
cur_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n"))
new_store_paths = cur_store_paths.difference(orig_store_paths)
deployer.log(f"{len(new_store_paths)} store paths were created")
l = list(map(lambda n: f"/nix/store/{n}", new_store_paths))
return l
${cfg.testScript}
'';
};
}; };
config = { evalTest =
testScript = '' module:
start_all() nixosLib.evalTest {
'' + lib.optionalString (prebuiltTarget != null) '' imports = [
deployer.succeed("nix-store -qR ${prebuiltSystem}") module
'' + '' colmenaTestModule
deployer.succeed("nix-store -qR ${pkgs.path}") { hostPkgs = pkgs; }
deployer.succeed("ln -sf ${pkgs.path} /nixpkgs") ];
deployer.succeed("mkdir -p /root/.ssh && touch /root/.ssh/id_rsa && chmod 600 /root/.ssh/id_rsa && cat ${sshKeys.snakeOilPrivateKey} > /root/.ssh/id_rsa")
${lib.optionalString (length targets != 0) ''
for node in ${targetList}:
node.wait_for_unit("sshd.service")
deployer.wait_until_succeeds(f"ssh -o StrictHostKeyChecking=accept-new {node.name} true", timeout=30)
''}
deployer.succeed("cp --no-preserve=mode -r ${bundle} /tmp/bundle && chmod u+w /tmp/bundle")
orig_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n"))
def get_new_store_paths():
cur_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n"))
new_store_paths = cur_store_paths.difference(orig_store_paths)
deployer.log(f"{len(new_store_paths)} store paths were created")
l = list(map(lambda n: f"/nix/store/{n}", new_store_paths))
return l
${cfg.testScript}
'';
}; };
};
evalTest = module: nixosLib.evalTest {
imports = [
module
colmenaTestModule
{ hostPkgs = pkgs; }
];
};
## Common setup ## Common setup
@ -124,103 +141,137 @@ let
# We include the input closure of a prebuilt system profile # We include the input closure of a prebuilt system profile
# so it can build system profiles for the targets without # so it can build system profiles for the targets without
# network access. # network access.
deployerConfig = { pkgs, lib, config, ... }: { deployerConfig =
imports = [ {
extraDeployerConfig pkgs,
]; lib,
config,
nix.registry = lib.mkIf (pkgs ? _inputs) { ...
nixpkgs.flake = pkgs._inputs.nixpkgs; }:
}; {
imports = [
nix.nixPath = [ extraDeployerConfig
"nixpkgs=${pkgs.path}"
];
nix.settings.substituters = lib.mkForce [];
virtualisation = {
memorySize = 6144;
writableStore = true;
additionalPaths = [
"${pkgs.path}"
] ++ lib.optionals (prebuiltTarget != null) [
prebuiltSystem
(inputClosureOf prebuiltSystem)
]; ];
nix.registry = lib.mkIf (pkgs ? _inputs) {
nixpkgs.flake = pkgs._inputs.nixpkgs;
};
nix.nixPath = [
"nixpkgs=${pkgs.path}"
];
nix.settings.substituters = lib.mkForce [ ];
virtualisation = {
memorySize = 6144;
writableStore = true;
additionalPaths =
[
"${pkgs.path}"
]
++ lib.optionals (prebuiltTarget != null) [
prebuiltSystem
(inputClosureOf prebuiltSystem)
];
};
services.openssh.enable = true;
users.users.root.openssh.authorizedKeys.keys = [
sshKeys.snakeOilPublicKey
];
environment.systemPackages = with pkgs; [
git # for git flake tests
inotify-tools # for key services build
# HACK: copy stderr to both stdout and stderr
# (the test framework only captures stdout, and only stderr appears on screen during the build)
(writeShellScriptBin "run-copy-stderr" ''
exec "$@" 2> >(tee /dev/stderr)
'')
];
# Re-enable switch-to-configuration
system.switch.enable = true;
}; };
services.openssh.enable = true;
users.users.root.openssh.authorizedKeys.keys = [
sshKeys.snakeOilPublicKey
];
environment.systemPackages = with pkgs; [
git # for git flake tests
inotify-tools # for key services build
# HACK: copy stderr to both stdout and stderr
# (the test framework only captures stdout, and only stderr appears on screen during the build)
(writeShellScriptBin "run-copy-stderr" ''
exec "$@" 2> >(tee /dev/stderr)
'')
];
# Re-enable switch-to-configuration
system.switch.enable = true;
};
# Setup for target nodes # Setup for target nodes
# #
# Kept as minimal as possible. # Kept as minimal as possible.
targetConfig = { lib, ... }: { targetConfig =
nix.settings.substituters = lib.mkForce []; { lib, ... }:
{
nix.settings.substituters = lib.mkForce [ ];
documentation.nixos.enable = lib.mkOverride 60 true; documentation.nixos.enable = lib.mkOverride 60 true;
services.openssh.enable = true; services.openssh.enable = true;
users.users.root.openssh.authorizedKeys.keys = [ users.users.root.openssh.authorizedKeys.keys = [
sshKeys.snakeOilPublicKey sshKeys.snakeOilPublicKey
]; ];
virtualisation.writableStore = true; virtualisation.writableStore = true;
# Re-enable switch-to-configuration # Re-enable switch-to-configuration
system.switch.enable = true; system.switch.enable = true;
}; };
nodes = let nodes =
deployerNodes = map (name: lib.nameValuePair name deployerConfig) deployers; let
targetNodes = map (name: lib.nameValuePair name targetConfig) targets; deployerNodes = map (name: lib.nameValuePair name deployerConfig) deployers;
in listToAttrs (deployerNodes ++ targetNodes); targetNodes = map (name: lib.nameValuePair name targetConfig) targets;
in
listToAttrs (deployerNodes ++ targetNodes);
# A "shallow" re-evaluation of the test for use from Colmena # A "shallow" re-evaluation of the test for use from Colmena
standaloneTest = evalTest ({ ... }: { standaloneTest = evalTest (
inherit nodes; { ... }:
}); {
inherit nodes;
}
);
prebuiltSystem = standaloneTest.config.nodes.${prebuiltTarget}.system.build.toplevel; prebuiltSystem = standaloneTest.config.nodes.${prebuiltTarget}.system.build.toplevel;
getStandaloneConfigFor = node: { lib, config, ... }: { getStandaloneConfigFor =
imports = [ node:
(pkgs.path + "/nixos/lib/testing/nixos-test-base.nix") { lib, config, ... }:
(if elem node deployers then deployerConfig else targetConfig) {
standaloneTest.config.nodes.${node}.system.build.networkConfig imports = [
]; (pkgs.path + "/nixos/lib/testing/nixos-test-base.nix")
(if elem node deployers then deployerConfig else targetConfig)
standaloneTest.config.nodes.${node}.system.build.networkConfig
];
documentation.nixos.enable = lib.mkOverride 55 false; documentation.nixos.enable = lib.mkOverride 55 false;
boot.loader.grub.enable = false; boot.loader.grub.enable = false;
system.nixos.revision = lib.mkForce "constant-nixos-revision"; system.nixos.revision = lib.mkForce "constant-nixos-revision";
nix.nixPath = lib.mkForce [ "nixpkgs=/nixpkgs" ]; nix.nixPath = lib.mkForce [ "nixpkgs=/nixpkgs" ];
deployment.tags = lib.optional (config.networking.hostName != "deployer") "target"; deployment.tags = lib.optional (config.networking.hostName != "deployer") "target";
}; };
in { in
inherit pkgs nodes colmena colmenaExec {
getStandaloneConfigFor inputClosureOf; inherit
pkgs
nodes
colmena
colmenaExec
getStandaloneConfigFor
inputClosureOf
;
runTest = module: (evalTest ({ config, ... }: { runTest =
imports = [ module { inherit nodes; } ]; module:
result = config.test; (evalTest (
})).config.result; { config, ... }:
{
imports = [
module
{ inherit nodes; }
];
result = config.test;
}
)).config.result;
} }

77
manual/colorized-help.nix
View file

@ -1,33 +1,52 @@
{ lib, stdenv, runCommand, colmena, ansi2html }: {
lib,
stdenv,
runCommand,
colmena,
ansi2html,
}:
with builtins; with builtins;
let let
subcommands = [ subcommands =
null [
"apply" null
] "apply"
++ lib.optional stdenv.isLinux "apply-local" ]
++ [ ++ lib.optional stdenv.isLinux "apply-local"
"build" ++ [
"upload-keys" "build"
"eval" "upload-keys"
"exec" "eval"
"nix-info" "exec"
"repl" "nix-info"
]; "repl"
renderHelp = subcommand: let ];
fullCommand = if subcommand == null then "colmena" else "colmena ${subcommand}"; renderHelp =
in '' subcommand:
( let
echo '## `${fullCommand}`' fullCommand = if subcommand == null then "colmena" else "colmena ${subcommand}";
echo -n '<pre><div class="hljs">' in
TERM=xterm-256color CLICOLOR_FORCE=1 ${fullCommand} --help | ansi2html -p ''
echo '</div></pre>' (
)>>$out echo '## `${fullCommand}`'
''; echo -n '<pre><div class="hljs">'
in runCommand "colmena-colorized-help" { TERM=xterm-256color CLICOLOR_FORCE=1 ${fullCommand} --help | ansi2html -p
nativeBuildInputs = [ colmena ansi2html ]; echo '</div></pre>'
} ('' )>>$out
ansi2html -H > $out '';
'' + concatStringsSep "\n" (map renderHelp subcommands)) in
runCommand "colmena-colorized-help"
{
nativeBuildInputs = [
colmena
ansi2html
];
}
(
''
ansi2html -H > $out
''
+ concatStringsSep "\n" (map renderHelp subcommands)
)

59
manual/default.nix
View file

@ -1,23 +1,33 @@
{ lib, stdenv, nix-gitignore, mdbook, mdbook-linkcheck, python3, callPackage, writeScript {
, deploymentOptionsMd ? null lib,
, metaOptionsMd ? null stdenv,
, colmena ? null nix-gitignore,
mdbook,
mdbook-linkcheck,
python3,
callPackage,
writeScript,
deploymentOptionsMd ? null,
metaOptionsMd ? null,
colmena ? null,
# Full version # Full version
, version ? if colmena != null then colmena.version else "unstable" version ? if colmena != null then colmena.version else "unstable",
# Whether this build is unstable # Whether this build is unstable
, unstable ? version == "unstable" || lib.hasInfix "-" version unstable ? version == "unstable" || lib.hasInfix "-" version,
}: }:
let let
apiVersion = builtins.concatStringsSep "." (lib.take 2 (lib.splitString "." version)); apiVersion = builtins.concatStringsSep "." (lib.take 2 (lib.splitString "." version));
colorizedHelp = let colorizedHelp =
help = callPackage ./colorized-help.nix { let
inherit colmena; help = callPackage ./colorized-help.nix {
}; inherit colmena;
in if colmena != null then help else null; };
in
if colmena != null then help else null;
redirectTemplate = lib.escapeShellArg '' redirectTemplate = lib.escapeShellArg ''
<!doctype html> <!doctype html>
@ -33,16 +43,29 @@ let
</html> </html>
''; '';
in stdenv.mkDerivation { in
inherit version deploymentOptionsMd metaOptionsMd colorizedHelp; stdenv.mkDerivation {
inherit
version
deploymentOptionsMd
metaOptionsMd
colorizedHelp
;
pname = "colmena-manual" + (if unstable then "-unstable" else ""); pname = "colmena-manual" + (if unstable then "-unstable" else "");
src = nix-gitignore.gitignoreSource [] ./.; src = nix-gitignore.gitignoreSource [ ] ./.;
nativeBuildInputs = [ mdbook mdbook-linkcheck python3 ]; nativeBuildInputs = [
mdbook
mdbook-linkcheck
python3
];
outputs = [ "out" "redirectFarm" ]; outputs = [
"out"
"redirectFarm"
];
COLMENA_VERSION = version; COLMENA_VERSION = version;
COLMENA_UNSTABLE = unstable; COLMENA_UNSTABLE = unstable;

15
package.nix
View file

@ -1,9 +1,10 @@
{ lib {
, stdenv lib,
, rustPlatform stdenv,
, nix-gitignore rustPlatform,
, installShellFiles nix-gitignore,
, nix-eval-jobs installShellFiles,
nix-eval-jobs,
}: }:
rustPlatform.buildRustPackage rec { rustPlatform.buildRustPackage rec {
@ -20,7 +21,7 @@ rustPlatform.buildRustPackage rec {
buildInputs = [ nix-eval-jobs ]; buildInputs = [ nix-eval-jobs ];
NIX_EVAL_JOBS = "${nix-eval-jobs}/bin/nix-eval-jobs"; env.NIX_EVAL_JOBS = lib.getExe nix-eval-jobs;
preBuild = '' preBuild = ''
if [[ -z "$NIX_EVAL_JOBS" ]]; then if [[ -z "$NIX_EVAL_JOBS" ]]; then

4
shell.nix
View file

@ -1,3 +1 @@
let (import ./flake-compat.nix).shellNix
flake = import ./flake-compat.nix;
in flake.shellNix

405
src/nix/hive/eval.nix
View file

@ -1,197 +1,269 @@
{ rawHive ? null # Colmena Hive attrset {
, rawFlake ? null # Nix Flake attrset with `outputs.colmena` rawHive ? null, # Colmena Hive attrset
, hermetic ? rawFlake != null # Whether we are allowed to use <nixpkgs> rawFlake ? null, # Nix Flake attrset with `outputs.colmena`
, colmenaOptions ? import ./options.nix hermetic ? rawFlake != null, # Whether we are allowed to use <nixpkgs>
, colmenaModules ? import ./modules.nix
}: }:
with builtins;
let let
defaultHive = { defaultHive = {
# Will be set in defaultHiveMeta # Will be set in defaultHiveMeta
meta = {}; meta = { };
# Like in NixOps, there is a special host named `defaults` # Like in NixOps, there is a special host named `defaults`
# containing configurations that will be applied to all # containing configurations that will be applied to all
# hosts. # hosts.
defaults = {}; defaults = { };
}; };
uncheckedHive =
let
flakeToHive =
rawFlake:
if rawFlake.outputs ? colmena then
rawFlake.outputs.colmena
else
throw "Flake must define outputs.colmena.";
uncheckedHive = let rawToHive =
flakeToHive = rawFlake: rawHive:
if rawFlake.outputs ? colmena then rawFlake.outputs.colmena else throw "Flake must define outputs.colmena."; if (builtins.isFunction rawHive) || rawHive ? __functor then
rawHive { }
rawToHive = rawHive: else if (builtins.isAttrs rawHive) then
if typeOf rawHive == "lambda" || rawHive ? __functor then rawHive {} rawHive
else if typeOf rawHive == "set" then rawHive else
else throw "The config must evaluate to an attribute set."; throw "The config must evaluate to an attribute set.";
in in
if rawHive != null then rawToHive rawHive if rawHive != null then
else if rawFlake != null then flakeToHive rawFlake rawToHive rawHive
else throw "Either a plain Hive attribute set or a Nix Flake attribute set must be specified."; else if rawFlake != null then
flakeToHive rawFlake
else
throw "Either a plain Hive attribute set or a Nix Flake attribute set must be specified.";
uncheckedUserMeta = uncheckedUserMeta =
if uncheckedHive ? meta && uncheckedHive ? network then if uncheckedHive ? meta && uncheckedHive ? network then
throw "Only one of `network` and `meta` may be specified. `meta` should be used as `network` is for NixOps compatibility." throw "Only one of `network` and `meta` may be specified. `meta` should be used as `network` is for NixOps compatibility."
else if uncheckedHive ? meta then uncheckedHive.meta else if uncheckedHive ? meta then
else if uncheckedHive ? network then uncheckedHive.network uncheckedHive.meta
else {}; else if uncheckedHive ? network then
uncheckedHive.network
else
{ };
uncheckedRegistries = if uncheckedHive ? registry then uncheckedHive.registry else {}; uncheckedRegistries = if uncheckedHive ? registry then uncheckedHive.registry else { };
# The final hive will always have the meta key instead of network. # The final hive will always have the meta key instead of network.
hive = let hive =
userMeta = (lib.modules.evalModules { let
modules = [ colmenaOptions.metaOptions uncheckedUserMeta ]; userMeta =
}).config; (lib.modules.evalModules {
modules = [
./options/meta.nix
uncheckedUserMeta
];
}).config;
registry = (lib.modules.evalModules { registry =
modules = [ colmenaOptions.registryOptions { registry = uncheckedRegistries; } ]; (lib.modules.evalModules {
}).config.registry; modules = [
./options/registry.nix
{ registry = uncheckedRegistries; }
];
}).config.registry;
mergedHive = mergedHive =
assert lib.assertMsg (!(uncheckedHive ? __schema)) '' assert lib.assertMsg (!(uncheckedHive ? __schema)) ''
You cannot pass in an already-evaluated Hive into the evaluator. You cannot pass in an already-evaluated Hive into the evaluator.
Hint: Use the `colmenaHive` output instead of `colmena`. Hint: Use the `colmenaHive` output instead of `colmena`.
''; '';
removeAttrs (defaultHive // uncheckedHive) [ "meta" "network" "registry" ]; removeAttrs (defaultHive // uncheckedHive) [
"meta"
"network"
"registry"
];
meta = { meta = {
meta = meta =
if !hermetic && userMeta.nixpkgs == null if !hermetic && userMeta.nixpkgs == null then userMeta // { nixpkgs = <nixpkgs>; } else userMeta;
then userMeta // { nixpkgs = <nixpkgs>; } };
else userMeta; in
}; mergedHive // meta // { inherit registry; };
in mergedHive // meta // { inherit registry; };
configsFor = node: let configsFor =
nodeConfig = hive.${node}; node:
in let
assert lib.assertMsg (!elem node reservedNames) "\"${node}\" is a reserved name and cannot be used as the name of a node"; nodeConfig = hive.${node};
if typeOf nodeConfig == "list" then nodeConfig in
else [ nodeConfig ]; assert lib.assertMsg (
!builtins.elem node reservedNames
) "\"${node}\" is a reserved name and cannot be used as the name of a node";
if (builtins.isList nodeConfig) then nodeConfig else [ nodeConfig ];
mkNixpkgs = configName: pkgConf: let mkNixpkgs =
uninitializedError = typ: '' configName: pkgConf:
Passing ${typ} as ${configName} is no longer accepted with Flakes. let
Please initialize Nixpkgs like the following: uninitializedError = typ: ''
Passing ${typ} as ${configName} is no longer accepted with Flakes.
Please initialize Nixpkgs like the following:
{ {
# ... # ...
outputs = { nixpkgs, ... }: { outputs = { nixpkgs, ... }: {
colmena = { colmena = {
${configName} = import nixpkgs { ${configName} = import nixpkgs {
system = "x86_64-linux"; # Set your desired system here system = "x86_64-linux"; # Set your desired system here
overlays = []; overlays = [];
};
}; };
}; };
}; }
} '';
''; in
in if (builtins.isPath pkgConf) || ((builtins.isAttrs pkgConf) && pkgConf ? outPath) then
if typeOf pkgConf == "path" || (typeOf pkgConf == "set" && pkgConf ? outPath) then if hermetic then
if hermetic then throw (uninitializedError "a path to Nixpkgs") throw (uninitializedError "a path to Nixpkgs")
# The referenced file might return an initialized Nixpkgs attribute set directly # The referenced file might return an initialized Nixpkgs attribute set directly
else mkNixpkgs configName (import pkgConf) else
else if typeOf pkgConf == "lambda" then mkNixpkgs configName (import pkgConf)
if hermetic then throw (uninitializedError "a Nixpkgs lambda") else if (builtins.isFunction pkgConf) then
else pkgConf { overlays = []; } if hermetic then throw (uninitializedError "a Nixpkgs lambda") else pkgConf { overlays = [ ]; }
else if typeOf pkgConf == "set" then else if (builtins.isAttrs pkgConf) then
if pkgConf ? outputs then throw (uninitializedError "an uninitialized Nixpkgs input") if pkgConf ? outputs then throw (uninitializedError "an uninitialized Nixpkgs input") else pkgConf
else pkgConf else
else throw '' throw ''
${configName} must be one of: ${configName} must be one of:
- A path to Nixpkgs (e.g., <nixpkgs>) - A path to Nixpkgs (e.g., <nixpkgs>)
- A Nixpkgs lambda (e.g., import <nixpkgs>) - A Nixpkgs lambda (e.g., import <nixpkgs>)
- A Nixpkgs attribute set - A Nixpkgs attribute set
''; '';
nixpkgs = let nixpkgs =
# Can't rely on the module system yet let
nixpkgsConf = # Can't rely on the module system yet
if uncheckedUserMeta ? nixpkgs then uncheckedUserMeta.nixpkgs nixpkgsConf =
else if hermetic then throw "meta.nixpkgs must be specified in hermetic mode." if uncheckedUserMeta ? nixpkgs then
else <nixpkgs>; uncheckedUserMeta.nixpkgs
in mkNixpkgs "meta.nixpkgs" nixpkgsConf; else if hermetic then
throw "meta.nixpkgs must be specified in hermetic mode."
else
<nixpkgs>;
in
mkNixpkgs "meta.nixpkgs" nixpkgsConf;
lib = nixpkgs.lib; lib = nixpkgs.lib;
reservedNames = [ "defaults" "network" "meta" "registry" ]; reservedNames = [
"defaults"
"network"
"meta"
"registry"
];
evalNode = name: configs: evalNode =
# Some help on error messages. name: configs:
assert (lib.assertMsg (lib.hasAttrByPath [ "deployment" "systemType" ] hive.${name}) # Some help on error messages.
"${name} does not have a deployment system type!"); assert (
assert (lib.assertMsg (builtins.typeOf hive.registry == "set")) lib.assertMsg (lib.hasAttrByPath [
"The hive's registry is not a set, but of type '${builtins.typeOf hive.registry}'"; "deployment"
assert (lib.assertMsg (lib.hasAttr hive.${name}.deployment.systemType hive.registry) "systemType"
"${builtins.toJSON (hive.${name}.deployment.systemType)} does not exist in the registry of systems!"); ] hive.${name}) "${name} does not have a deployment system type!"
let );
# We cannot use `configs` because we need to access to the raw configuration fragment. assert (lib.assertMsg (
inherit (hive.registry.${hive.${name}.deployment.systemType}) evalConfig; builtins.isAttrs hive.registry
npkgs = )) "The hive's registry is not a set, but of type '${builtins.typeOf hive.registry}'";
if hasAttr name hive.meta.nodeNixpkgs assert (
then mkNixpkgs "meta.nodeNixpkgs.${name}" hive.meta.nodeNixpkgs.${name} lib.assertMsg (lib.hasAttr hive.${name}.deployment.systemType hive.registry)
else nixpkgs; "${builtins.toJSON (hive.${name}.deployment.systemType)} does not exist in the registry of systems!"
);
let
# We cannot use `configs` because we need to access to the raw configuration fragment.
inherit (hive.registry.${hive.${name}.deployment.systemType}) evalConfig;
npkgs =
if builtins.hasAttr name hive.meta.nodeNixpkgs then
mkNixpkgs "meta.nodeNixpkgs.${name}" hive.meta.nodeNixpkgs.${name}
else
nixpkgs;
# Here we need to merge the configurations in meta.nixpkgs # Here we need to merge the configurations in meta.nixpkgs
# and in machine config. # and in machine config.
nixpkgsModule = { config, lib, ... }: let nixpkgsModule =
hasTypedConfig = lib.versionAtLeast lib.version "22.11pre"; { config, lib, ... }:
in { let
nixpkgs.overlays = lib.mkBefore npkgs.overlays; hasTypedConfig = lib.versionAtLeast lib.version "22.11pre";
nixpkgs.config = if hasTypedConfig then lib.mkBefore npkgs.config else lib.mkOptionDefault npkgs.config; in
{
nixpkgs.overlays = lib.mkBefore npkgs.overlays;
nixpkgs.config =
if hasTypedConfig then lib.mkBefore npkgs.config else lib.mkOptionDefault npkgs.config;
warnings = let warnings =
# Before 22.11, most config keys were untyped thus the merging let
# was broken. Let's warn the user if not all config attributes # Before 22.11, most config keys were untyped thus the merging
# set in meta.nixpkgs are overridden. # was broken. Let's warn the user if not all config attributes
metaKeys = attrNames npkgs.config; # set in meta.nixpkgs are overridden.
nodeKeys = [ "doCheckByDefault" "warnings" "allowAliases" ] ++ (attrNames config.nixpkgs.config); metaKeys = builtins.attrNames npkgs.config;
remainingKeys = filter (k: ! elem k nodeKeys) metaKeys; nodeKeys = [
in "doCheckByDefault"
lib.optional (!hasTypedConfig && length remainingKeys != 0) "warnings"
"The following Nixpkgs configuration keys set in meta.nixpkgs will be ignored: ${toString remainingKeys}"; "allowAliases"
} // lib.optionalAttrs (builtins.hasAttr "localSystem" npkgs || builtins.hasAttr "crossSystem" npkgs) { ] ++ (builtins.attrNames config.nixpkgs.config);
nixpkgs.localSystem = lib.mkBefore npkgs.localSystem; remainingKeys = builtins.filter (k: !builtins.elem k nodeKeys) metaKeys;
nixpkgs.crossSystem = lib.mkBefore npkgs.crossSystem; in
}; lib.optional (!hasTypedConfig && builtins.length remainingKeys != 0)
in evalConfig { "The following Nixpkgs configuration keys set in meta.nixpkgs will be ignored: ${toString remainingKeys}";
# This doesn't exist for `evalModules` the generic way. }
# inherit (npkgs) system; //
lib.optionalAttrs (builtins.hasAttr "localSystem" npkgs || builtins.hasAttr "crossSystem" npkgs)
{
nixpkgs.localSystem = lib.mkBefore npkgs.localSystem;
nixpkgs.crossSystem = lib.mkBefore npkgs.crossSystem;
};
in
evalConfig {
# This doesn't exist for `evalModules` the generic way.
# inherit (npkgs) system;
modules = [ modules = [
nixpkgsModule nixpkgsModule
colmenaModules.assertionModule ./modules/assertions.nix
colmenaOptions.deploymentOptions ./options/deployment.nix
(hive.registry.${hive.${name}.deployment.systemType}.defaults or hive.defaults) (hive.registry.${hive.${name}.deployment.systemType}.defaults or hive.defaults)
] ++ configs; ] ++ configs;
specialArgs = { specialArgs =
inherit name; {
nodes = uncheckedNodes; inherit name;
} // hive.meta.specialArgs // (hive.meta.nodeSpecialArgs.${name} or {}); nodes = uncheckedNodes;
}; }
// hive.meta.specialArgs
// (hive.meta.nodeSpecialArgs.${name} or { });
};
nodeNames = filter (name: ! elem name reservedNames) (attrNames hive); nodeNames = builtins.filter (name: !builtins.elem name reservedNames) (builtins.attrNames hive);
# Used as the `nodes` argument in modules. We skip recursive type checking # Used as the `nodes` argument in modules. We skip recursive type checking
# for performance. # for performance.
uncheckedNodes = listToAttrs (map (name: let uncheckedNodes = builtins.listToAttrs (
configs = [ builtins.map (
name:
let
configs = [
{
_module.check = false;
}
] ++ configsFor name;
in
{ {
_module.check = false; inherit name;
value = evalNode name configs;
} }
] ++ configsFor name; ) nodeNames
in { );
inherit name;
value = evalNode name configs;
}) nodeNames);
# Add required config Key here since we don't want to eval nixpkgs # Add required config Key here since we don't want to eval nixpkgs
metaConfigKeys = [ metaConfigKeys = [
"name" "description" "name"
"description"
"machinesFile" "machinesFile"
"allowApplyAll" "allowApplyAll"
]; ];
@ -200,20 +272,35 @@ let
"supportsDeployment" "supportsDeployment"
]; ];
in rec { in
rec {
# Exported attributes # Exported attributes
__schema = "v0.20241006"; __schema = "v0.20241006";
nodes = listToAttrs (map (name: { inherit name; value = evalNode name (configsFor name); }) nodeNames); nodes = builtins.listToAttrs (
toplevel = lib.mapAttrs (_: v: v.config.system.build.toplevel) nodes; builtins.map (name: {
deploymentConfig = lib.mapAttrs (_: v: v.config.deployment) nodes; inherit name;
deploymentConfigSelected = names: lib.filterAttrs (name: _: elem name names) deploymentConfig; value = evalNode name (configsFor name);
evalSelected = names: lib.filterAttrs (name: _: elem name names) toplevel; }) nodeNames
evalSelectedDrvPaths = names: lib.mapAttrs (_: v: v.drvPath) (evalSelected names); );
metaConfig = lib.filterAttrs (n: v: elem n metaConfigKeys) hive.meta; toplevel = lib.mapAttrs (_: v: v.config.system.build.toplevel) nodes;
deploymentConfig = lib.mapAttrs (_: v: v.config.deployment) nodes;
deploymentConfigSelected =
names: lib.filterAttrs (name: _: builtins.elem name names) deploymentConfig;
evalSelected = names: lib.filterAttrs (name: _: builtins.elem name names) toplevel;
evalSelectedDrvPaths = names: lib.mapAttrs (_: v: v.drvPath) (evalSelected names);
metaConfig = lib.filterAttrs (n: v: builtins.elem n metaConfigKeys) hive.meta;
# We cannot perform a `metaConfigKeys`-style simple check here # We cannot perform a `metaConfigKeys`-style simple check here
# because registry is arbitrarily deep and may evaluate nixpkgs indirectly. # because registry is arbitrarily deep and may evaluate nixpkgs indirectly.
registryConfig = lib.mapAttrs (systemTypeName: systemType: registryConfig = lib.mapAttrs (
lib.filterAttrs (n: v: elem n serializableSystemTypeConfigKeys) systemType) hive.registry; systemTypeName: systemType:
introspect = f: f { inherit lib; pkgs = nixpkgs; inherit nodes; }; lib.filterAttrs (n: v: builtins.elem n serializableSystemTypeConfigKeys) systemType
) hive.registry;
introspect =
f:
f {
inherit lib;
pkgs = nixpkgs;
inherit nodes;
};
} }

66
src/nix/hive/flake.nix
View file

@ -5,39 +5,43 @@
hive.url = "%hive%"; hive.url = "%hive%";
}; };
outputs = { self, hive }: { outputs =
processFlake = let { hive, ... }:
compatibleSchema = "v0.20241006"; {
processFlake =
let
compatibleSchema = "v0.20241006";
# Evaluates a raw hive. # Evaluates a raw hive.
# #
# This uses the `colmena` output. # This uses the `colmena` output.
evalHive = rawFlake: import ./eval.nix { evalHive =
inherit rawFlake; rawFlake:
hermetic = true; import ./eval.nix {
colmenaOptions = import ./options.nix; inherit rawFlake;
colmenaModules = import ./modules.nix; hermetic = true;
}; };
# Uses an already-evaluated hive. # Uses an already-evaluated hive.
# #
# This uses the `colmenaHive` output. # This uses the `colmenaHive` output.
checkPreparedHive = hiveOutput: checkPreparedHive =
if !(hiveOutput ? __schema) then hiveOutput:
throw '' if !(hiveOutput ? __schema) then
The colmenaHive output does not contain a valid evaluated hive. throw ''
The colmenaHive output does not contain a valid evaluated hive.
Hint: Use `colmena.lib.makeHive`. Hint: Use `colmena.lib.makeHive`.
'' ''
else if hiveOutput.__schema != compatibleSchema then else if hiveOutput.__schema != compatibleSchema then
throw '' throw ''
The colmenaHive output (schema ${hiveOutput.__schema}) isn't compatible with this version of Colmena. The colmenaHive output (schema ${hiveOutput.__schema}) isn't compatible with this version of Colmena.
Hint: Use the same version of Colmena as in the Flake input. Hint: Use the same version of Colmena as in the Flake input.
'' ''
else hiveOutput; else
in hiveOutput;
if hive.outputs ? colmenaHive then checkPreparedHive hive.outputs.colmenaHive in
else evalHive hive; if hive.outputs ? colmenaHive then checkPreparedHive hive.outputs.colmenaHive else evalHive hive;
}; };
} }

109
src/nix/hive/modules.nix
View file

@ -1,109 +0,0 @@
with builtins; {
assertionModule = { config, lib, ... }: {
assertions = lib.mapAttrsToList (key: opts: let
nonNulls = l: filter (x: x != null) l;
in {
assertion = length (nonNulls [opts.text opts.keyCommand opts.keyFile]) == 1;
message =
let prefix = "${name}.deployment.keys.${key}";
in "Exactly one of `${prefix}.text`, `${prefix}.keyCommand` and `${prefix}.keyFile` must be set.";
}) config.deployment.keys;
};
# Change the ownership of all keys uploaded pre-activation
#
# This is built as part of the system profile.
# We must be careful not to access `text` / `keyCommand` / `keyFile` here
keyChownModule = { lib, config, ... }: let
preActivationKeys = lib.filterAttrs (name: key: key.uploadAt == "pre-activation") config.deployment.keys;
scriptDeps = if config.system.activationScripts ? groups then [ "groups" ] else [ "users" ];
commands = lib.mapAttrsToList (name: key: let
keyPath = "${key.destDir}/${name}";
in ''
if [ -f "${keyPath}" ]; then
if ! chown ${key.user}:${key.group} "${keyPath}"; then
# Error should be visible in stderr
failed=1
fi
else
>&2 echo "Key ${keyPath} does not exist. Skipping chown."
fi
'') preActivationKeys;
script = lib.stringAfter scriptDeps ''
# This script is injected by Colmena to change the ownerships
# of keys (`deployment.keys`) deployed before system activation.
>&2 echo "setting up key ownerships..."
# We set the ownership of as many keys as possible before failing
failed=
${concatStringsSep "\n" commands}
if [ -n "$failed" ]; then
>&2 echo "Failed to set the ownership of some keys."
# The activation script has a trap to handle failed
# commands and print out various debug information.
# Let's trigger that instead of `exit 1`.
false
fi
'';
in {
system.activationScripts.colmena-chown-keys = lib.mkIf (length commands != 0) script;
};
# Create "${name}-key" services for NixOps compatibility
#
# This is built as part of the system profile.
# We must be careful not to access `text` / `keyCommand` / `keyFile` here
#
# Sadly, path units don't automatically deactivate the bound units when
# the key files are deleted, so we use inotifywait in the services' scripts.
#
# <https://github.com/systemd/systemd/issues/3642>
keyServiceModule = { pkgs, lib, config, ... }: {
systemd.paths = lib.mapAttrs' (name: val: {
name = "${name}-key";
value = {
wantedBy = [ "paths.target" ];
pathConfig = {
PathExists = val.path;
};
};
}) config.deployment.keys;
systemd.services = lib.mapAttrs' (name: val: {
name = "${name}-key";
value = {
enable = true;
serviceConfig = {
TimeoutStartSec = "infinity";
Restart = "always";
RestartSec = "100ms";
};
path = [ pkgs.inotify-tools ];
preStart = ''
(while read f; do if [ "$f" = "${val.name}" ]; then break; fi; done \
< <(inotifywait -qm --format '%f' -e create,move ${val.destDir}) ) &
if [[ -e "${val.path}" ]]; then
echo 'flapped down'
kill %1
exit 0
fi
wait %1
'';
script = ''
inotifywait -qq -e delete_self "${val.path}" &
if [[ ! -e "${val.path}" ]]; then
echo 'flapped up'
exit 0
fi
wait %1
'';
};
}) config.deployment.keys;
};
}

28
src/nix/hive/modules/assertions.nix Normal file
View file

@ -0,0 +1,28 @@
{
config,
lib,
name,
...
}:
let
inherit (lib) filter mapAttrsToList;
in
{
assertions = mapAttrsToList (key: opts: {
assertion =
builtins.length (
filter (x: x != null) [
opts.text
opts.keyCommand
opts.keyFile
]
) == 1;
message =
let
prefix = "${name}.deployment.keys.${key}";
in
"Exactly one of `${prefix}.text`, `${prefix}.keyCommand` and `${prefix}.keyFile` must be set.";
}) config.deployment.keys;
}

64
src/nix/hive/modules/key-chown.nix Normal file
View file

@ -0,0 +1,64 @@
# Change the ownership of all keys uploaded pre-activation
#
# This is built as part of the system profile.
# We must be careful not to access `text` / `keyCommand` / `keyFile` here
{ config, lib, ... }:
let
inherit (lib)
concatStringsSep
filterAttrs
mapAttrsToList
mkIf
stringAfter
;
preActivationKeys = filterAttrs (
_: { uploadAt, ... }: uploadAt == "pre-activation"
) config.deployment.keys;
scriptDeps = if config.system.activationScripts ? groups then [ "groups" ] else [ "users" ];
commands = mapAttrsToList (
name: key:
let
keyPath = "${key.destDir}/${name}";
in
''
if [ -f "${keyPath}" ]; then
if ! chown ${key.user}:${key.group} "${keyPath}"; then
# Error should be visible in stderr
failed=1
fi
else
>&2 echo "Key ${keyPath} does not exist. Skipping chown."
fi
''
) preActivationKeys;
script = stringAfter scriptDeps ''
# This script is injected by Colmena to change the ownerships
# of keys (`deployment.keys`) deployed before system activation.
>&2 echo "setting up key ownerships..."
# We set the ownership of as many keys as possible before failing
failed=
${concatStringsSep "\n" commands}
if [ -n "$failed" ]; then
>&2 echo "Failed to set the ownership of some keys."
# The activation script has a trap to handle failed
# commands and print out various debug information.
# Let's trigger that instead of `exit 1`.
false
fi
'';
in
{
system.activationScripts.colmena-chown-keys = mkIf (commands != [ ]) script;
}

61
src/nix/hive/modules/key-service.nix Normal file
View file

@ -0,0 +1,61 @@
# Create "${name}-key" services for NixOps compatibility
#
# This is built as part of the system profile.
# We must be careful not to access `text` / `keyCommand` / `keyFile` here
#
# Sadly, path units don't automatically deactivate the bound units when
# the key files are deleted, so we use inotifywait in the services' scripts.
#
# <https://github.com/systemd/systemd/issues/3642>
{
config,
lib,
pkgs,
...
}:
let
inherit (lib) mapAttrs' nameValuePair;
in
{
systemd.paths = mapAttrs' (
name: value:
nameValuePair "${name}-key" {
wantedBy = [ "paths.target" ];
pathConfig.PathExists = value.path;
}
) config.deployment.keys;
systemd.services = mapAttrs' (
name: value:
nameValuePair "${name}-key" {
enable = true;
serviceConfig = {
TimeoutStartSec = "infinity";
Restart = "always";
RestartSec = "100ms";
};
path = [ pkgs.inotify-tools ];
preStart = ''
(while read f; do if [ "$f" = "${value.name}" ]; then break; fi; done \
< <(inotifywait -qm --format '%f' -e create,move ${value.destDir}) ) &
if [[ -e "${value.path}" ]]; then
echo 'flapped down'
kill %1
exit 0
fi
wait %1
'';
script = ''
inotifywait -qq -e delete_self "${value.path}" &
if [[ ! -e "${value.path}" ]]; then
echo 'flapped up'
exit 0
fi
wait %1
'';
}
) config.deployment.keys;
}

368
src/nix/hive/options.nix
View file

@ -1,368 +0,0 @@
with builtins; rec {
keyType = { lib, name, config, ... }: let
inherit (lib) types;
in {
options = {
name = lib.mkOption {
description = ''
File name of the key.
'';
default = name;
type = types.str;
};
text = lib.mkOption {
description = ''
Content of the key.
One of `text`, `keyCommand` and `keyFile` must be set.
'';
default = null;
type = types.nullOr types.str;
};
keyFile = lib.mkOption {
description = ''
Path of the local file to read the key from.
One of `text`, `keyCommand` and `keyFile` must be set.
'';
default = null;
apply = value: if value == null then null else toString value;
type = types.nullOr types.path;
};
keyCommand = lib.mkOption {
description = ''
Command to run to generate the key.
One of `text`, `keyCommand` and `keyFile` must be set.
'';
default = null;
type = let
nonEmptyList = types.addCheck (types.listOf types.str) (l: length l > 0);
in types.nullOr nonEmptyList;
};
destDir = lib.mkOption {
description = ''
Destination directory on the host.
'';
default = "/run/keys";
type = types.path;
};
path = lib.mkOption {
description = ''
Full path to the destination.
'';
default = "${config.destDir}/${config.name}";
type = types.path;
internal = true;
};
user = lib.mkOption {
description = ''
The group that will own the file.
'';
default = "root";
type = types.str;
};
group = lib.mkOption {
description = ''
The group that will own the file.
'';
default = "root";
type = types.str;
};
permissions = lib.mkOption {
description = ''
Permissions to set for the file.
'';
default = "0600";
type = types.str;
};
uploadAt = lib.mkOption {
description = ''
When to upload the keys.
- pre-activation (default): Upload the keys before activating the new system profile.
- post-activation: Upload the keys after successfully activating the new system profile.
For `colmena upload-keys`, all keys are uploaded at the same time regardless of the configuration here.
'';
default = "pre-activation";
type = types.enum [ "pre-activation" "post-activation" ];
};
};
};
# Colmena-specific options
#
# Largely compatible with NixOps/Morph.
deploymentOptions = { name, lib, ... }: let
inherit (lib) types;
in {
options = {
deployment = {
systemType = lib.mkOption {
description = mdDoc ''
System type used for this node, e.g. NixOS.
'';
default = "nixos";
# TODO: enum among all registered systems?
type = types.str;
};
targetHost = lib.mkOption {
description = ''
The target SSH node for deployment.
By default, the node's attribute name will be used.
If set to null, only local deployment will be supported.
'';
type = types.nullOr types.str;
default = name;
};
targetPort = lib.mkOption {
description = ''
The target SSH port for deployment.
By default, the port is the standard port (22) or taken
from your ssh_config.
'';
type = types.nullOr types.ints.unsigned;
default = null;
};
targetUser = lib.mkOption {
description = ''
The user to use to log into the remote node. If set to null, the
target user will not be specified in SSH invocations.
'';
type = types.nullOr types.str;
default = "root";
};
allowLocalDeployment = lib.mkOption {
description = ''
Allow the configuration to be applied locally on the host running
Colmena.
For local deployment to work, all of the following must be true:
- The node must be running NixOS.
- The node must have deployment.allowLocalDeployment set to true.
- The node's networking.hostName must match the hostname.
To apply the configurations locally, run `colmena apply-local`.
You can also set deployment.targetHost to null if the nost is not
accessible over SSH (only local deployment will be possible).
'';
type = types.bool;
default = false;
};
buildOnTarget = lib.mkOption {
description = ''
Whether to build the system profiles on the target node itself.
When enabled, Colmena will copy the derivation to the target
node and initiate the build there. This avoids copying back the
build results involved with the native distributed build
feature. Furthermore, the `build` goal will be equivalent to
the `push` goal. Since builds happen on the target node, the
results are automatically "pushed" and won't exist in the local
Nix store.
You can temporarily override per-node settings by passing
`--build-on-target` (enable for all nodes) or
`--no-build-on-target` (disable for all nodes) on the command
line.
'';
type = types.bool;
default = false;
};
tags = lib.mkOption {
description = ''
A list of tags for the node.
Can be used to select a group of nodes for deployment.
'';
type = types.listOf types.str;
default = [];
};
keys = lib.mkOption {
description = ''
A set of secrets to be deployed to the node.
Secrets are transferred to the node out-of-band and
never ends up in the Nix store.
'';
type = types.attrsOf (types.submodule keyType);
default = {};
};
replaceUnknownProfiles = lib.mkOption {
description = ''
Allow a configuration to be applied to a host running a profile we
have no knowledge of. By setting this option to false, you reduce
the likelyhood of rolling back changes made via another Colmena user.
Unknown profiles are usually the result of either:
- The node had a profile applied, locally or by another Colmena.
- The host running Colmena garbage-collecting the profile.
To force profile replacement on all targeted nodes during apply,
use the flag `--force-replace-unknown-profiles`.
'';
type = types.bool;
default = true;
};
privilegeEscalationCommand = lib.mkOption {
description = ''
Command to use to elevate privileges when activating the new profiles on SSH hosts.
This is used on SSH hosts when `deployment.targetUser` is not `root`.
The user must be allowed to use the command non-interactively.
'';
type = types.listOf types.str;
default = [ "sudo" "-H" "--" ];
};
sshOptions = lib.mkOption {
description = ''
Extra SSH options to pass to the SSH command.
'';
type = types.listOf types.str;
default = [];
};
};
};
};
# Options for a registered system type
systemTypeOptions = { name, lib, ... }: let
inherit (lib) types;
mdDoc = lib.mdDoc or lib.id;
in
{
options = {
evalConfig = lib.mkOption {
description = mdDoc ''
Evaluation function which share the same interface as `nixos/lib/eval-config.nix`
which can be tailored to your own usecases or to target another type of system,
e.g. nix-darwin.
'';
type = types.functionTo types.unspecified;
};
supportsDeployment = lib.mkOption {
description = mdDoc ''
Whether this system type supports deployment or not.
If it supports deployment, it needs to have appropriate activation code,
refer to how to write custom activators.
'';
default = name == "nixos";
defaultText = "If a NixOS system, then true, otherwise false by default";
};
defaults = lib.mkOption {
description = mdDoc ''
Default configuration for that system type.
'';
type = types.functionTo types.unspecified;
default = _: {};
};
};
};
registryOptions = { lib, ... }: let
inherit (lib) types;
mdDoc = lib.mdDoc or lib.id;
in
{
options.registry = lib.mkOption {
description = mdDoc ''
A registry of all system types.
'';
type = types.attrsOf (types.submodule systemTypeOptions);
};
};
# Hive-wide options
metaOptions = { lib, ... }: let
inherit (lib) types;
in {
options = {
name = lib.mkOption {
description = ''
The name of the configuration.
'';
type = types.str;
default = "hive";
};
description = lib.mkOption {
description = ''
A short description for the configuration.
'';
type = types.str;
default = "A Colmena Hive";
};
nixpkgs = lib.mkOption {
description = ''
The pinned Nixpkgs package set. Accepts one of the following:
- A path to a Nixpkgs checkout
- The Nixpkgs lambda (e.g., import <nixpkgs>)
- An initialized Nixpkgs attribute set
This option must be specified when using Flakes.
'';
type = types.unspecified;
default = null;
};
nodeNixpkgs = lib.mkOption {
description = ''
Node-specific Nixpkgs pins.
'';
type = types.attrsOf types.unspecified;
default = {};
};
nodeSpecialArgs = lib.mkOption {
description = ''
Node-specific special args.
'';
type = types.attrsOf types.unspecified;
default = {};
};
machinesFile = lib.mkOption {
description = ''
Use the machines listed in this file when building this hive configuration.
If your Colmena host has nix configured to allow for remote builds
(for nix-daemon, your user being included in trusted-users)
you can set a machines file that will be passed to the underlying
nix-store command during derivation realization as a builders option.
For example, if you support multiple orginizations each with their own
build machine(s) you can ensure that builds only take place on your
local machine and/or the machines specified in this file.
See https://nixos.org/manual/nix/stable/advanced-topics/distributed-builds
for the machine specification format.
This option is ignored when builds are initiated on the remote nodes
themselves via `deployment.buildOnTarget` or `--build-on-target`. To
still use the Nix distributed build functionality, configure the
builders on the target nodes with `nix.buildMachines`.
'';
default = null;
apply = value: if value == null then null else toString value;
type = types.nullOr types.path;
};
specialArgs = lib.mkOption {
description = ''
A set of special arguments to be passed to NixOS modules.
This will be merged into the `specialArgs` used to evaluate
the NixOS configurations.
'';
default = {};
type = types.attrsOf types.unspecified;
};
allowApplyAll = lib.mkOption {
description = ''
Whether to allow deployments without a node filter set.
If set to false, a node filter must be specified with `--on` when
deploying.
It helps prevent accidental deployments to the entire cluster
when tags are used (e.g., `@production` and `@staging`).
'';
default = true;
type = types.bool;
};
};
};
}

151
src/nix/hive/options/deployment.nix Normal file
View file

@ -0,0 +1,151 @@
{ lib, name, ... }:
let
inherit (lib) mkOption;
inherit (lib.types)
attrsOf
bool
ints
listOf
nullOr
str
submodule
;
in
{
options = {
deployment = {
systemType = mkOption {
description = ''
System type used for this node, e.g. NixOS.
'';
default = "nixos";
# TODO: enum among all registered systems?
type = str;
};
targetHost = mkOption {
description = ''
The target SSH node for deployment.
By default, the node's attribute name will be used.
If set to null, only local deployment will be supported.
'';
type = nullOr str;
default = name;
};
targetPort = mkOption {
description = ''
The target SSH port for deployment.
By default, the port is the standard port (22) or taken
from your ssh_config.
'';
type = nullOr ints.unsigned;
default = null;
};
targetUser = mkOption {
description = ''
The user to use to log into the remote node. If set to null, the
target user will not be specified in SSH invocations.
'';
type = nullOr str;
default = "root";
};
allowLocalDeployment = mkOption {
description = ''
Allow the configuration to be applied locally on the host running
Colmena.
For local deployment to work, all of the following must be true:
- The node must be running NixOS.
- The node must have deployment.allowLocalDeployment set to true.
- The node's networking.hostName must match the hostname.
To apply the configurations locally, run `colmena apply-local`.
You can also set deployment.targetHost to null if the nost is not
accessible over SSH (only local deployment will be possible).
'';
type = bool;
default = false;
};
buildOnTarget = mkOption {
description = ''
Whether to build the system profiles on the target node itself.
When enabled, Colmena will copy the derivation to the target
node and initiate the build there. This avoids copying back the
build results involved with the native distributed build
feature. Furthermore, the `build` goal will be equivalent to
the `push` goal. Since builds happen on the target node, the
results are automatically "pushed" and won't exist in the local
Nix store.
You can temporarily override per-node settings by passing
`--build-on-target` (enable for all nodes) or
`--no-build-on-target` (disable for all nodes) on the command
line.
'';
type = bool;
default = false;
};
tags = mkOption {
description = ''
A list of tags for the node.
Can be used to select a group of nodes for deployment.
'';
type = listOf str;
default = [ ];
};
keys = mkOption {
description = ''
A set of secrets to be deployed to the node.
Secrets are transferred to the node out-of-band and
never ends up in the Nix store.
'';
type = attrsOf (submodule (import ./key.nix));
default = { };
};
replaceUnknownProfiles = mkOption {
description = ''
Allow a configuration to be applied to a host running a profile we
have no knowledge of. By setting this option to false, you reduce
the likelyhood of rolling back changes made via another Colmena user.
Unknown profiles are usually the result of either:
- The node had a profile applied, locally or by another Colmena.
- The host running Colmena garbage-collecting the profile.
To force profile replacement on all targeted nodes during apply,
use the flag `--force-replace-unknown-profiles`.
'';
type = bool;
default = true;
};
privilegeEscalationCommand = mkOption {
description = ''
Command to use to elevate privileges when activating the new profiles on SSH hosts.
This is used on SSH hosts when `deployment.targetUser` is not `root`.
The user must be allowed to use the command non-interactively.
'';
type = listOf str;
default = [
"sudo"
"-H"
"--"
];
};
sshOptions = mkOption {
description = ''
Extra SSH options to pass to the SSH command.
'';
type = listOf str;
default = [ ];
};
};
};
}

120
src/nix/hive/options/key.nix Normal file
View file

@ -0,0 +1,120 @@
{
config,
lib,
name,
...
}:
let
inherit (lib) mkOption;
inherit (lib.types)
addCheck
enum
listOf
nullOr
path
str
;
in
{
options = {
name = mkOption {
description = ''
File name of the key.
'';
default = name;
type = str;
};
text = mkOption {
description = ''
Content of the key.
One of `text`, `keyCommand` and `keyFile` must be set.
'';
default = null;
type = nullOr str;
};
keyFile = mkOption {
description = ''
Path of the local file to read the key from.
One of `text`, `keyCommand` and `keyFile` must be set.
'';
default = null;
apply = value: if value == null then null else builtins.toString value;
type = nullOr path;
};
keyCommand = mkOption {
description = ''
Command to run to generate the key.
One of `text`, `keyCommand` and `keyFile` must be set.
'';
default = null;
type =
let
nonEmptyList = addCheck (listOf str) (l: builtins.length l > 0);
in
nullOr nonEmptyList;
};
destDir = mkOption {
description = ''
Destination directory on the host.
'';
default = "/run/keys";
type = path;
};
path = mkOption {
description = ''
Full path to the destination.
'';
default = "${config.destDir}/${config.name}";
type = path;
internal = true;
};
user = mkOption {
description = ''
The group that will own the file.
'';
default = "root";
type = str;
};
group = mkOption {
description = ''
The group that will own the file.
'';
default = "root";
type = str;
};
permissions = mkOption {
description = ''
Permissions to set for the file.
'';
default = "0600";
type = str;
};
uploadAt = mkOption {
description = ''
When to upload the keys.
- pre-activation (default): Upload the keys before activating the new system profile.
- post-activation: Upload the keys after successfully activating the new system profile.
For `colmena upload-keys`, all keys are uploaded at the same time regardless of the configuration here.
'';
default = "pre-activation";
type = enum [
"pre-activation"
"post-activation"
];
};
};
}

114
src/nix/hive/options/meta.nix Normal file
View file

@ -0,0 +1,114 @@
{ lib, ... }:
let
inherit (lib) mkOption;
inherit (lib.types)
attrsOf
bool
nullOr
path
str
unspecified
;
in
{
options = {
name = mkOption {
description = ''
The name of the configuration.
'';
type = str;
default = "hive";
};
description = mkOption {
description = ''
A short description for the configuration.
'';
type = str;
default = "A Colmena Hive";
};
nixpkgs = mkOption {
description = ''
The pinned Nixpkgs package set. Accepts one of the following:
- A path to a Nixpkgs checkout
- The Nixpkgs lambda (e.g., import <nixpkgs>)
- An initialized Nixpkgs attribute set
This option must be specified when using Flakes.
'';
type = unspecified;
default = null;
};
nodeNixpkgs = mkOption {
description = ''
Node-specific Nixpkgs pins.
'';
type = attrsOf unspecified;
default = { };
};
nodeSpecialArgs = mkOption {
description = ''
Node-specific special args.
'';
type = attrsOf unspecified;
default = { };
};
machinesFile = mkOption {
description = ''
Use the machines listed in this file when building this hive configuration.
If your Colmena host has nix configured to allow for remote builds
(for nix-daemon, your user being included in trusted-users)
you can set a machines file that will be passed to the underlying
nix-store command during derivation realization as a builders option.
For example, if you support multiple orginizations each with their own
build machine(s) you can ensure that builds only take place on your
local machine and/or the machines specified in this file.
See https://nixos.org/manual/nix/stable/advanced-topics/distributed-builds
for the machine specification format.
This option is ignored when builds are initiated on the remote nodes
themselves via `deployment.buildOnTarget` or `--build-on-target`. To
still use the Nix distributed build functionality, configure the
builders on the target nodes with `nix.buildMachines`.
'';
default = null;
apply = value: if value == null then null else builtins.toString value;
type = nullOr path;
};
specialArgs = mkOption {
description = ''
A set of special arguments to be passed to NixOS modules.
This will be merged into the `specialArgs` used to evaluate
the NixOS configurations.
'';
default = { };
type = attrsOf unspecified;
};
allowApplyAll = mkOption {
description = ''
Whether to allow deployments without a node filter set.
If set to false, a node filter must be specified with `--on` when
deploying.
It helps prevent accidental deployments to the entire cluster
when tags are used (e.g., `@production` and `@staging`).
'';
default = true;
type = bool;
};
};
}

54
src/nix/hive/options/registry.nix Normal file
View file

@ -0,0 +1,54 @@
{ lib, ... }:
let
inherit (lib) mkOption;
inherit (lib.types)
attrsOf
functionTo
submodule
unspecified
;
in
{
options.registry = mkOption {
description = ''
A registry of all system types.
'';
type = attrsOf (
submodule (
{ name, ... }:
{
options = {
evalConfig = mkOption {
description = ''
Evaluation function which share the same interface as `nixos/lib/eval-config.nix`
which can be tailored to your own usecases or to target another type of system,
e.g. nix-darwin.
'';
type = functionTo unspecified;
};
supportsDeployment = mkOption {
description = ''
Whether this system type supports deployment or not.
If it supports deployment, it needs to have appropriate activation code,
refer to how to write custom activators.
'';
default = name == "nixos";
defaultText = "If a NixOS system, then true, otherwise false by default";
};
defaults = mkOption {
description = ''
Default configuration for that system type.
'';
type = functionTo unspecified;
default = _: { };
};
};
}
)
);
};
}

45
src/nix/hive/tests/makehive-flake/flake.nix
View file

@ -3,27 +3,36 @@
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
colmena.url = "git+file://@repoPath@"; colmena.url = "git+file://@repoPath@";
}; };
outputs = { nixpkgs, colmena, ... }: { outputs =
colmenaHive = colmena.lib.makeHive { { nixpkgs, colmena, ... }:
meta = { {
nixpkgs = import nixpkgs { colmenaHive = colmena.lib.makeHive {
system = "x86_64-linux"; meta = {
nixpkgs = import nixpkgs {
system = "x86_64-linux";
};
}; };
};
host-a = { name, nodes, pkgs, ... }: { host-a =
boot.isContainer = true; {
time.timeZone = nodes.host-b.config.time.timeZone; name,
}; nodes,
host-b = { pkgs,
deployment = { ...
targetHost = "somehost.tld"; }:
targetPort = 1234; {
targetUser = "luser"; boot.isContainer = true;
time.timeZone = nodes.host-b.config.time.timeZone;
};
host-b = {
deployment = {
targetHost = "somehost.tld";
targetPort = 1234;
targetUser = "luser";
};
boot.isContainer = true;
time.timeZone = "America/Los_Angeles";
}; };
boot.isContainer = true;
time.timeZone = "America/Los_Angeles";
}; };
}; };
};
} }

45
src/nix/hive/tests/simple-flake/flake.nix
View file

@ -2,27 +2,36 @@
inputs = { inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
}; };
outputs = { nixpkgs, ... }: { outputs =
colmena = { { nixpkgs, ... }:
meta = { {
nixpkgs = import nixpkgs { colmena = {
system = "x86_64-linux"; meta = {
nixpkgs = import nixpkgs {
system = "x86_64-linux";
};
}; };
};
host-a = { name, nodes, pkgs, ... }: { host-a =
boot.isContainer = true; {
time.timeZone = nodes.host-b.config.time.timeZone; name,
}; nodes,
host-b = { pkgs,
deployment = { ...
targetHost = "somehost.tld"; }:
targetPort = 1234; {
targetUser = "luser"; boot.isContainer = true;
time.timeZone = nodes.host-b.config.time.timeZone;
};
host-b = {
deployment = {
targetHost = "somehost.tld";
targetPort = 1234;
targetUser = "luser";
};
boot.isContainer = true;
time.timeZone = "America/Los_Angeles";
}; };
boot.isContainer = true;
time.timeZone = "America/Los_Angeles";
}; };
}; };
};
} }