integration-tests: Start migration to modular test framework

Still need to migrate most logic in tools.nix to modules.
This commit is contained in:
Zhaofeng Li 2022-12-01 01:57:56 -07:00
parent 92f0f155d4
commit 8b87f0de02
8 changed files with 280 additions and 190 deletions

View file

@ -4,16 +4,18 @@ let
tools = pkgs.callPackage ../tools.nix {
targets = [ "alpha" ];
};
in tools.makeTest {
in tools.runTest {
name = "colmena-allow-apply-all";
bundle = ./.;
colmena.test = {
bundle = ./.;
testScript = ''
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply")
testScript = ''
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply")
assert "No node filter" in logs
assert "No node filter" in logs
deployer.succeed("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target")
'';
deployer.succeed("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target")
'';
};
}

View file

@ -12,14 +12,16 @@ let
security.sudo.wheelNeedsPassword = false;
};
};
in tools.makeTest {
in tools.runTest {
name = "colmena-apply-local";
bundle = ./.;
colmena.test = {
bundle = ./.;
testScript = ''
deployer.succeed("cd /tmp/bundle && sudo -u colmena ${tools.colmenaExec} apply-local --sudo")
deployer.succeed("grep SUCCESS /etc/deployment")
deployer.succeed("grep SECRET /run/keys/key-text")
'';
testScript = ''
deployer.succeed("cd /tmp/bundle && sudo -u colmena ${tools.colmenaExec} apply-local --sudo")
deployer.succeed("grep SUCCESS /etc/deployment")
deployer.succeed("grep SECRET /run/keys/key-text")
'';
};
}

View file

@ -4,13 +4,14 @@
let
tools = pkgs.callPackage ../tools.nix {};
in tools.makeTest {
in tools.runTest {
name = "colmena-apply-${evaluator}";
bundle = ./.;
testScript = ''
colmena = "${tools.colmenaExec}"
evaluator = "${evaluator}"
'' + builtins.readFile ./test-script.py;
colmena.test = {
bundle = ./.;
testScript = ''
colmena = "${tools.colmenaExec}"
evaluator = "${evaluator}"
'' + builtins.readFile ./test-script.py;
};
}

View file

@ -5,31 +5,33 @@ let
deployers = [ "deployer" "alpha" "beta" ];
targets = [];
};
in tools.makeTest {
in tools.runTest {
name = "colmena-build-on-target";
bundle = ./.;
colmena.test = {
bundle = ./.;
testScript = ''
# The actual build will be initiated on alpha
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on alpha")
testScript = ''
# The actual build will be initiated on alpha
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on alpha")
with subtest("Check that the new configurations are indeed applied"):
alpha.succeed("grep SUCCESS /etc/deployment")
with subtest("Check that the new configurations are indeed applied"):
alpha.succeed("grep SUCCESS /etc/deployment")
alpha_profile = alpha.succeed("readlink /run/current-system")
alpha_profile = alpha.succeed("readlink /run/current-system")
with subtest("Check that the built profile is not on the deployer"):
deployer.fail(f"nix-store -qR {alpha_profile}")
with subtest("Check that the built profile is not on the deployer"):
deployer.fail(f"nix-store -qR {alpha_profile}")
with subtest("Check that we can override per-node settings and build locally"):
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} build --on alpha --no-build-on-target")
deployer.succeed(f"nix-store -qR {alpha_profile}")
with subtest("Check that we can override per-node settings and build locally"):
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} build --on alpha --no-build-on-target")
deployer.succeed(f"nix-store -qR {alpha_profile}")
with subtest("Check that we can override per-node settings and build remotely"):
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on beta --build-on-target")
beta.succeed("grep SUCCESS /etc/deployment")
profile = beta.succeed("readlink /run/current-system")
deployer.fail(f"nix-store -qR {profile}")
'';
with subtest("Check that we can override per-node settings and build remotely"):
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on beta --build-on-target")
beta.succeed("grep SUCCESS /etc/deployment")
profile = beta.succeed("readlink /run/current-system")
deployer.fail(f"nix-store -qR {profile}")
'';
};
}

View file

@ -2,16 +2,18 @@
let
tools = pkgs.callPackage ../tools.nix {};
in tools.makeTest {
in tools.runTest {
name = "colmena-exec";
bundle = ./.;
colmena.test = {
bundle = ./.;
testScript = ''
logs = deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} exec --on @target -- echo output from '$(hostname)' 2>&1")
testScript = ''
logs = deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} exec --on @target -- echo output from '$(hostname)' 2>&1")
assert "output from alpha" in logs
assert "output from beta" in logs
assert "output from gamma" in logs
'';
assert "output from alpha" in logs
assert "output from beta" in logs
assert "output from gamma" in logs
'';
};
}

View file

@ -6,43 +6,45 @@ let
tools = pkgs.callPackage ../tools.nix {
targets = [ "alpha" ];
};
in tools.makeTest {
in tools.runTest {
name = "colmena-flakes-${evaluator}";
bundle = ./.;
colmena.test = {
bundle = ./.;
testScript = ''
import re
testScript = ''
import re
deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix")
deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix")
with subtest("Lock flake dependencies"):
deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock")
with subtest("Lock flake dependencies"):
deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock")
with subtest("Deploy with a plain flake without git"):
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
alpha.succeed("grep FIRST /etc/deployment")
with subtest("Deploy with a plain flake without git"):
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
alpha.succeed("grep FIRST /etc/deployment")
with subtest("Deploy with a git flake"):
deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/probe.nix")
with subtest("Deploy with a git flake"):
deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/probe.nix")
# don't put probe.nix in source control - should fail
deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
assert re.search(r"probe.nix.*No such file or directory", logs)
# don't put probe.nix in source control - should fail
deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
assert re.search(r"probe.nix.*No such file or directory", logs)
# now it should succeed
deployer.succeed("cd /tmp/bundle && git add probe.nix")
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
alpha.succeed("grep SECOND /etc/deployment")
# now it should succeed
deployer.succeed("cd /tmp/bundle && git add probe.nix")
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
alpha.succeed("grep SECOND /etc/deployment")
with subtest("Check that impure expressions are forbidden"):
deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix")
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
assert re.search(r"access to absolute path.*forbidden in pure eval mode", logs)
with subtest("Check that impure expressions are forbidden"):
deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix")
logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
assert re.search(r"access to absolute path.*forbidden in pure eval mode", logs)
with subtest("Check that impure expressions can be allowed with --impure"):
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator} --impure")
alpha.succeed("grep deployer /etc/deployment")
'';
with subtest("Check that impure expressions can be allowed with --impure"):
deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator} --impure")
alpha.succeed("grep deployer /etc/deployment")
'';
};
}

View file

@ -5,25 +5,27 @@ let
in tools.makeTest {
name = "colmena-parallel";
bundle = ./.;
colmena.test = {
bundle = ./.;
testScript = ''
deployer.succeed("cd /tmp/bundle &&" \
"${tools.colmenaExec} apply push --eval-node-limit 4 --on @target")
testScript = ''
deployer.succeed("cd /tmp/bundle &&" \
"${tools.colmenaExec} apply push --eval-node-limit 4 --on @target")
logs = deployer.succeed("cd /tmp/bundle &&" \
"run-copy-stderr ${tools.colmenaExec} apply switch --eval-node-limit 4 --parallel 4 --on @target")
logs = deployer.succeed("cd /tmp/bundle &&" \
"run-copy-stderr ${tools.colmenaExec} apply switch --eval-node-limit 4 --parallel 4 --on @target")
for node in [alpha, beta, gamma]:
node.succeed("grep SUCCESS /etc/deployment")
for node in [alpha, beta, gamma]:
node.succeed("grep SUCCESS /etc/deployment")
with subtest("Check that activation is correctly parallelized"):
timestamps = list(map(lambda l: int(l.strip().split("---")[1]) / 1000000,
filter(lambda l: "Activation triggered" in l, logs.split("\n"))))
with subtest("Check that activation is correctly parallelized"):
timestamps = list(map(lambda l: int(l.strip().split("---")[1]) / 1000000,
filter(lambda l: "Activation triggered" in l, logs.split("\n"))))
delay = max(timestamps) - min(timestamps)
deployer.log(f"Time between activations: {delay}ms")
delay = max(timestamps) - min(timestamps)
deployer.log(f"Time between activations: {delay}ms")
assert delay < 2000
'';
assert delay < 2000
'';
};
}

View file

@ -3,9 +3,7 @@
# By default, we have four nodes: deployer, alpha, beta, gamma.
# deployer is where colmena will run.
#
# `nixos/lib/build-vms.nix` will generate NixOS configurations
# for each node, and we need to include those configurations
# in our Colmena setup as well.
# TODO: Modularize most of this
{ insideVm ? false
, deployers ? [ "deployer" ] # Nodes configured as deployers (with Colmena and pre-built system closure)
@ -26,103 +24,9 @@ let
colmenaExec = "${colmena}/bin/colmena";
## Utilities
sshKeys = import (pkgs.path + "/nixos/tests/ssh-keys.nix") pkgs;
buildVms = import (pkgs.path + "/nixos/lib/build-vms.nix") {
inherit (pkgs) system pkgs lib;
};
# Common setup
nodes = let
# Setup for deployer nodes
#
# We include the input closure of a prebuilt system profile
# so it can build system profiles for the targets without
# network access.
deployerConfig = { pkgs, lib, config, ... }: {
imports = [
extraDeployerConfig
];
nix.registry = lib.mkIf (pkgs ? _inputs) {
nixpkgs.flake = pkgs._inputs.nixpkgs;
};
nix.nixPath = [
"nixpkgs=${pkgs.path}"
];
nix.binaryCaches = lib.mkForce [];
virtualisation = {
memorySize = 3072;
writableStore = true;
additionalPaths = [
"${pkgs.path}"
] ++ lib.optionals (prebuiltTarget != null) [
prebuiltSystem
(inputClosureOf prebuiltSystem)
];
};
services.openssh.enable = true;
users.users.root.openssh.authorizedKeys.keys = [
sshKeys.snakeOilPublicKey
];
environment.systemPackages = with pkgs; [
git # for git flake tests
inotify-tools # for key services build
# HACK: copy stderr to both stdout and stderr
# (the test framework only captures stdout, and only stderr appears on screen during the build)
(writeShellScriptBin "run-copy-stderr" ''
exec "$@" 2> >(tee /dev/stderr)
'')
];
};
# Setup for target nodes
#
# Kept as minimal as possible.
targetConfig = { lib, ... }: {
nix.binaryCaches = lib.mkForce [];
documentation.nixos.enable = lib.mkOverride 60 true;
services.openssh.enable = true;
users.users.root.openssh.authorizedKeys.keys = [
sshKeys.snakeOilPublicKey
];
virtualisation.writableStore = true;
};
deployerNodes = map (name: lib.nameValuePair name deployerConfig) deployers;
targetNodes = map (name: lib.nameValuePair name targetConfig) targets;
in listToAttrs (deployerNodes ++ targetNodes);
prebuiltSystem = let
all = buildVms.buildVirtualNetwork nodes;
in all.${prebuiltTarget}.config.system.build.toplevel;
# Utilities
getStandaloneConfigFor = node: let
configsWithIp = buildVms.assignIPAddresses nodes;
in { modulesPath, lib, config, ... }: {
imports = configsWithIp.${node} ++ [
(modulesPath + "/virtualisation/qemu-vm.nix")
(modulesPath + "/testing/test-instrumentation.nix")
];
documentation.nixos.enable = lib.mkOverride 55 false;
boot.loader.grub.enable = false;
system.nixos.revision = lib.mkForce "constant-nixos-revision";
# otherwise the evaluation is unnecessarily slow in VM
virtualisation.additionalPaths = lib.mkForce [];
nix.nixPath = lib.mkForce [ "nixpkgs=/nixpkgs" ];
deployment.tags = lib.optional (config.networking.hostName != "deployer") "target";
};
nixosLib = import (pkgs.path + "/nixos/lib") { };
inputClosureOf = pkg: pkgs.runCommand "full-closure" {
refs = pkgs.writeReferencesToFile pkg.drvPath;
@ -138,6 +42,174 @@ let
done <$refs
'';
## The modular NixOS test framework with Colmena additions
colmenaTestModule = { lib, config, ... }: let
cfg = config.colmena.test;
targetList = "[${concatStringsSep ", " targets}]";
bundle = pkgs.stdenv.mkDerivation {
name = "${config.name}-bundle";
dontUnpack = true;
dontInstall = true;
buildPhase = ''
cp -r ${cfg.bundle} $out
chmod u+w $out
cp ${./tools.nix} $out/tools.nix
'';
};
in {
options = {
colmena.test = {
bundle = lib.mkOption {
description = ''
Path to a directory to copy into the deployer as /tmp/bundle.
'';
type = lib.types.path;
};
testScript = lib.mkOption {
description = ''
The test script.
The Colmena test framework will prepend initialization
statements to the actual test script.
'';
type = lib.types.str;
};
};
};
config = {
testScript = ''
start_all()
'' + lib.optionalString (prebuiltTarget != null) ''
deployer.succeed("nix-store -qR ${prebuiltSystem}")
'' + ''
deployer.succeed("nix-store -qR ${pkgs.path}")
deployer.succeed("ln -sf ${pkgs.path} /nixpkgs")
deployer.succeed("mkdir -p /root/.ssh && touch /root/.ssh/id_rsa && chmod 600 /root/.ssh/id_rsa && cat ${sshKeys.snakeOilPrivateKey} > /root/.ssh/id_rsa")
${lib.optionalString (length targets != 0) ''
for node in ${targetList}:
node.wait_for_unit("sshd.service")
deployer.succeed(f"ssh -o StrictHostKeyChecking=accept-new {node.name} true", timeout=30)
''}
deployer.succeed("cp --no-preserve=mode -r ${bundle} /tmp/bundle && chmod u+w /tmp/bundle")
orig_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n"))
def get_new_store_paths():
cur_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n"))
new_store_paths = cur_store_paths.difference(orig_store_paths)
deployer.log(f"{len(new_store_paths)} store paths were created")
l = list(map(lambda n: f"/nix/store/{n}", new_store_paths))
return l
${cfg.testScript}
'';
};
};
evalTest = module: nixosLib.evalTest {
imports = [
module
colmenaTestModule
{ hostPkgs = pkgs; }
];
};
## Common setup
# Setup for deployer nodes
#
# We include the input closure of a prebuilt system profile
# so it can build system profiles for the targets without
# network access.
deployerConfig = { pkgs, lib, config, ... }: {
imports = [
extraDeployerConfig
];
nix.registry = lib.mkIf (pkgs ? _inputs) {
nixpkgs.flake = pkgs._inputs.nixpkgs;
};
nix.nixPath = [
"nixpkgs=${pkgs.path}"
];
nix.binaryCaches = lib.mkForce [];
virtualisation = {
memorySize = 3072;
writableStore = true;
additionalPaths = [
"${pkgs.path}"
] ++ lib.optionals (prebuiltTarget != null) [
prebuiltSystem
(inputClosureOf prebuiltSystem)
];
};
services.openssh.enable = true;
users.users.root.openssh.authorizedKeys.keys = [
sshKeys.snakeOilPublicKey
];
environment.systemPackages = with pkgs; [
git # for git flake tests
inotify-tools # for key services build
# HACK: copy stderr to both stdout and stderr
# (the test framework only captures stdout, and only stderr appears on screen during the build)
(writeShellScriptBin "run-copy-stderr" ''
exec "$@" 2> >(tee /dev/stderr)
'')
];
};
# Setup for target nodes
#
# Kept as minimal as possible.
targetConfig = { lib, ... }: {
nix.binaryCaches = lib.mkForce [];
documentation.nixos.enable = lib.mkOverride 60 true;
services.openssh.enable = true;
users.users.root.openssh.authorizedKeys.keys = [
sshKeys.snakeOilPublicKey
];
virtualisation.writableStore = true;
};
nodes = let
deployerNodes = map (name: lib.nameValuePair name deployerConfig) deployers;
targetNodes = map (name: lib.nameValuePair name targetConfig) targets;
in listToAttrs (deployerNodes ++ targetNodes);
# A "shallow" re-evaluation of the test for use from Colmena
standaloneTest = evalTest ({ ... }: {
inherit nodes;
});
prebuiltSystem = standaloneTest.config.nodes.${prebuiltTarget}.system.build.toplevel;
getStandaloneConfigFor = node: { lib, config, ... }: {
imports = [
(pkgs.path + "/nixos/lib/testing/nixos-test-base.nix")
(if elem node deployers then deployerConfig else targetConfig)
standaloneTest.config.nodes.${node}.system.build.networkConfig
];
documentation.nixos.enable = lib.mkOverride 55 false;
boot.loader.grub.enable = false;
system.nixos.revision = lib.mkForce "constant-nixos-revision";
nix.nixPath = lib.mkForce [ "nixpkgs=/nixpkgs" ];
deployment.tags = lib.optional (config.networking.hostName != "deployer") "target";
};
makeTest = test: let
customArgs = [ "bundle" ];
@ -190,4 +262,9 @@ let
in {
inherit pkgs nodes colmena colmenaExec
getStandaloneConfigFor inputClosureOf makeTest;
runTest = module: (evalTest ({ config, ... }: {
imports = [ module { inherit nodes; } ];
result = config.test;
})).config.result;
}