From 8b87f0de021183105ca67a4b17af119770ee95a4 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Thu, 1 Dec 2022 01:57:56 -0700 Subject: [PATCH] integration-tests: Start migration to modular test framework Still need to migrate most logic in tools.nix to modules. --- integration-tests/allow-apply-all/default.nix | 16 +- integration-tests/apply-local/default.nix | 16 +- integration-tests/apply/default.nix | 15 +- integration-tests/build-on-target/default.nix | 40 +-- integration-tests/exec/default.nix | 18 +- integration-tests/flakes/default.nix | 58 ++-- integration-tests/parallel/default.nix | 32 +- integration-tests/tools.nix | 275 +++++++++++------- 8 files changed, 280 insertions(+), 190 deletions(-) diff --git a/integration-tests/allow-apply-all/default.nix b/integration-tests/allow-apply-all/default.nix index a069c04..bed21a5 100644 --- a/integration-tests/allow-apply-all/default.nix +++ b/integration-tests/allow-apply-all/default.nix @@ -4,16 +4,18 @@ let tools = pkgs.callPackage ../tools.nix { targets = [ "alpha" ]; }; -in tools.makeTest { +in tools.runTest { name = "colmena-allow-apply-all"; - bundle = ./.; + colmena.test = { + bundle = ./.; - testScript = '' - logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply") + testScript = '' + logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply") - assert "No node filter" in logs + assert "No node filter" in logs - deployer.succeed("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target") - ''; + deployer.succeed("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target") + ''; + }; } diff --git a/integration-tests/apply-local/default.nix b/integration-tests/apply-local/default.nix index f0cc54a..c7ed099 100644 --- a/integration-tests/apply-local/default.nix +++ b/integration-tests/apply-local/default.nix @@ -12,14 +12,16 @@ let security.sudo.wheelNeedsPassword = false; }; }; -in tools.makeTest { +in tools.runTest { name = "colmena-apply-local"; - bundle = ./.; + colmena.test = { + bundle = ./.; - testScript = '' - deployer.succeed("cd /tmp/bundle && sudo -u colmena ${tools.colmenaExec} apply-local --sudo") - deployer.succeed("grep SUCCESS /etc/deployment") - deployer.succeed("grep SECRET /run/keys/key-text") - ''; + testScript = '' + deployer.succeed("cd /tmp/bundle && sudo -u colmena ${tools.colmenaExec} apply-local --sudo") + deployer.succeed("grep SUCCESS /etc/deployment") + deployer.succeed("grep SECRET /run/keys/key-text") + ''; + }; } diff --git a/integration-tests/apply/default.nix b/integration-tests/apply/default.nix index 6422a32..366e5cb 100644 --- a/integration-tests/apply/default.nix +++ b/integration-tests/apply/default.nix @@ -4,13 +4,14 @@ let tools = pkgs.callPackage ../tools.nix {}; -in tools.makeTest { +in tools.runTest { name = "colmena-apply-${evaluator}"; - bundle = ./.; - - testScript = '' - colmena = "${tools.colmenaExec}" - evaluator = "${evaluator}" - '' + builtins.readFile ./test-script.py; + colmena.test = { + bundle = ./.; + testScript = '' + colmena = "${tools.colmenaExec}" + evaluator = "${evaluator}" + '' + builtins.readFile ./test-script.py; + }; } diff --git a/integration-tests/build-on-target/default.nix b/integration-tests/build-on-target/default.nix index 2e7591e..0e29754 100644 --- a/integration-tests/build-on-target/default.nix +++ b/integration-tests/build-on-target/default.nix @@ -5,31 +5,33 @@ let deployers = [ "deployer" "alpha" "beta" ]; targets = []; }; -in tools.makeTest { +in tools.runTest { name = "colmena-build-on-target"; - bundle = ./.; + colmena.test = { + bundle = ./.; - testScript = '' - # The actual build will be initiated on alpha - deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on alpha") + testScript = '' + # The actual build will be initiated on alpha + deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on alpha") - with subtest("Check that the new configurations are indeed applied"): - alpha.succeed("grep SUCCESS /etc/deployment") + with subtest("Check that the new configurations are indeed applied"): + alpha.succeed("grep SUCCESS /etc/deployment") - alpha_profile = alpha.succeed("readlink /run/current-system") + alpha_profile = alpha.succeed("readlink /run/current-system") - with subtest("Check that the built profile is not on the deployer"): - deployer.fail(f"nix-store -qR {alpha_profile}") + with subtest("Check that the built profile is not on the deployer"): + deployer.fail(f"nix-store -qR {alpha_profile}") - with subtest("Check that we can override per-node settings and build locally"): - deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} build --on alpha --no-build-on-target") - deployer.succeed(f"nix-store -qR {alpha_profile}") + with subtest("Check that we can override per-node settings and build locally"): + deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} build --on alpha --no-build-on-target") + deployer.succeed(f"nix-store -qR {alpha_profile}") - with subtest("Check that we can override per-node settings and build remotely"): - deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on beta --build-on-target") - beta.succeed("grep SUCCESS /etc/deployment") - profile = beta.succeed("readlink /run/current-system") - deployer.fail(f"nix-store -qR {profile}") - ''; + with subtest("Check that we can override per-node settings and build remotely"): + deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on beta --build-on-target") + beta.succeed("grep SUCCESS /etc/deployment") + profile = beta.succeed("readlink /run/current-system") + deployer.fail(f"nix-store -qR {profile}") + ''; + }; } diff --git a/integration-tests/exec/default.nix b/integration-tests/exec/default.nix index 8020b23..99916c0 100644 --- a/integration-tests/exec/default.nix +++ b/integration-tests/exec/default.nix @@ -2,16 +2,18 @@ let tools = pkgs.callPackage ../tools.nix {}; -in tools.makeTest { +in tools.runTest { name = "colmena-exec"; - bundle = ./.; + colmena.test = { + bundle = ./.; - testScript = '' - logs = deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} exec --on @target -- echo output from '$(hostname)' 2>&1") + testScript = '' + logs = deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} exec --on @target -- echo output from '$(hostname)' 2>&1") - assert "output from alpha" in logs - assert "output from beta" in logs - assert "output from gamma" in logs - ''; + assert "output from alpha" in logs + assert "output from beta" in logs + assert "output from gamma" in logs + ''; + }; } diff --git a/integration-tests/flakes/default.nix b/integration-tests/flakes/default.nix index 3c4385d..74fb265 100644 --- a/integration-tests/flakes/default.nix +++ b/integration-tests/flakes/default.nix @@ -6,43 +6,45 @@ let tools = pkgs.callPackage ../tools.nix { targets = [ "alpha" ]; }; -in tools.makeTest { +in tools.runTest { name = "colmena-flakes-${evaluator}"; - bundle = ./.; + colmena.test = { + bundle = ./.; - testScript = '' - import re + testScript = '' + import re - deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix") + deployer.succeed("sed -i 's @nixpkgs@ path:${pkgs._inputs.nixpkgs.outPath}?narHash=${pkgs._inputs.nixpkgs.narHash} g' /tmp/bundle/flake.nix") - with subtest("Lock flake dependencies"): - deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock") + with subtest("Lock flake dependencies"): + deployer.succeed("cd /tmp/bundle && nix --extra-experimental-features \"nix-command flakes\" flake lock") - with subtest("Deploy with a plain flake without git"): - deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}") - alpha.succeed("grep FIRST /etc/deployment") + with subtest("Deploy with a plain flake without git"): + deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}") + alpha.succeed("grep FIRST /etc/deployment") - with subtest("Deploy with a git flake"): - deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/probe.nix") + with subtest("Deploy with a git flake"): + deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/probe.nix") - # don't put probe.nix in source control - should fail - deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix") - logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}") - assert re.search(r"probe.nix.*No such file or directory", logs) + # don't put probe.nix in source control - should fail + deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix") + logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}") + assert re.search(r"probe.nix.*No such file or directory", logs) - # now it should succeed - deployer.succeed("cd /tmp/bundle && git add probe.nix") - deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}") - alpha.succeed("grep SECOND /etc/deployment") + # now it should succeed + deployer.succeed("cd /tmp/bundle && git add probe.nix") + deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}") + alpha.succeed("grep SECOND /etc/deployment") - with subtest("Check that impure expressions are forbidden"): - deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix") - logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}") - assert re.search(r"access to absolute path.*forbidden in pure eval mode", logs) + with subtest("Check that impure expressions are forbidden"): + deployer.succeed("sed -i 's|SECOND|''${builtins.readFile /etc/hostname}|g' /tmp/bundle/probe.nix") + logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}") + assert re.search(r"access to absolute path.*forbidden in pure eval mode", logs) - with subtest("Check that impure expressions can be allowed with --impure"): - deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator} --impure") - alpha.succeed("grep deployer /etc/deployment") - ''; + with subtest("Check that impure expressions can be allowed with --impure"): + deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator} --impure") + alpha.succeed("grep deployer /etc/deployment") + ''; + }; } diff --git a/integration-tests/parallel/default.nix b/integration-tests/parallel/default.nix index 0d1a3c6..b3f9b11 100644 --- a/integration-tests/parallel/default.nix +++ b/integration-tests/parallel/default.nix @@ -5,25 +5,27 @@ let in tools.makeTest { name = "colmena-parallel"; - bundle = ./.; + colmena.test = { + bundle = ./.; - testScript = '' - deployer.succeed("cd /tmp/bundle &&" \ - "${tools.colmenaExec} apply push --eval-node-limit 4 --on @target") + testScript = '' + deployer.succeed("cd /tmp/bundle &&" \ + "${tools.colmenaExec} apply push --eval-node-limit 4 --on @target") - logs = deployer.succeed("cd /tmp/bundle &&" \ - "run-copy-stderr ${tools.colmenaExec} apply switch --eval-node-limit 4 --parallel 4 --on @target") + logs = deployer.succeed("cd /tmp/bundle &&" \ + "run-copy-stderr ${tools.colmenaExec} apply switch --eval-node-limit 4 --parallel 4 --on @target") - for node in [alpha, beta, gamma]: - node.succeed("grep SUCCESS /etc/deployment") + for node in [alpha, beta, gamma]: + node.succeed("grep SUCCESS /etc/deployment") - with subtest("Check that activation is correctly parallelized"): - timestamps = list(map(lambda l: int(l.strip().split("---")[1]) / 1000000, - filter(lambda l: "Activation triggered" in l, logs.split("\n")))) + with subtest("Check that activation is correctly parallelized"): + timestamps = list(map(lambda l: int(l.strip().split("---")[1]) / 1000000, + filter(lambda l: "Activation triggered" in l, logs.split("\n")))) - delay = max(timestamps) - min(timestamps) - deployer.log(f"Time between activations: {delay}ms") + delay = max(timestamps) - min(timestamps) + deployer.log(f"Time between activations: {delay}ms") - assert delay < 2000 - ''; + assert delay < 2000 + ''; + }; } diff --git a/integration-tests/tools.nix b/integration-tests/tools.nix index f643f8b..9b0cfea 100644 --- a/integration-tests/tools.nix +++ b/integration-tests/tools.nix @@ -3,9 +3,7 @@ # By default, we have four nodes: deployer, alpha, beta, gamma. # deployer is where colmena will run. # -# `nixos/lib/build-vms.nix` will generate NixOS configurations -# for each node, and we need to include those configurations -# in our Colmena setup as well. +# TODO: Modularize most of this { insideVm ? false , deployers ? [ "deployer" ] # Nodes configured as deployers (with Colmena and pre-built system closure) @@ -26,103 +24,9 @@ let colmenaExec = "${colmena}/bin/colmena"; + ## Utilities sshKeys = import (pkgs.path + "/nixos/tests/ssh-keys.nix") pkgs; - buildVms = import (pkgs.path + "/nixos/lib/build-vms.nix") { - inherit (pkgs) system pkgs lib; - }; - - # Common setup - nodes = let - # Setup for deployer nodes - # - # We include the input closure of a prebuilt system profile - # so it can build system profiles for the targets without - # network access. - deployerConfig = { pkgs, lib, config, ... }: { - imports = [ - extraDeployerConfig - ]; - - nix.registry = lib.mkIf (pkgs ? _inputs) { - nixpkgs.flake = pkgs._inputs.nixpkgs; - }; - - nix.nixPath = [ - "nixpkgs=${pkgs.path}" - ]; - - nix.binaryCaches = lib.mkForce []; - - virtualisation = { - memorySize = 3072; - writableStore = true; - additionalPaths = [ - "${pkgs.path}" - ] ++ lib.optionals (prebuiltTarget != null) [ - prebuiltSystem - (inputClosureOf prebuiltSystem) - ]; - }; - - services.openssh.enable = true; - users.users.root.openssh.authorizedKeys.keys = [ - sshKeys.snakeOilPublicKey - ]; - - environment.systemPackages = with pkgs; [ - git # for git flake tests - inotify-tools # for key services build - - # HACK: copy stderr to both stdout and stderr - # (the test framework only captures stdout, and only stderr appears on screen during the build) - (writeShellScriptBin "run-copy-stderr" '' - exec "$@" 2> >(tee /dev/stderr) - '') - ]; - }; - - # Setup for target nodes - # - # Kept as minimal as possible. - targetConfig = { lib, ... }: { - nix.binaryCaches = lib.mkForce []; - - documentation.nixos.enable = lib.mkOverride 60 true; - - services.openssh.enable = true; - users.users.root.openssh.authorizedKeys.keys = [ - sshKeys.snakeOilPublicKey - ]; - virtualisation.writableStore = true; - }; - - deployerNodes = map (name: lib.nameValuePair name deployerConfig) deployers; - targetNodes = map (name: lib.nameValuePair name targetConfig) targets; - in listToAttrs (deployerNodes ++ targetNodes); - - prebuiltSystem = let - all = buildVms.buildVirtualNetwork nodes; - in all.${prebuiltTarget}.config.system.build.toplevel; - - # Utilities - getStandaloneConfigFor = node: let - configsWithIp = buildVms.assignIPAddresses nodes; - in { modulesPath, lib, config, ... }: { - imports = configsWithIp.${node} ++ [ - (modulesPath + "/virtualisation/qemu-vm.nix") - (modulesPath + "/testing/test-instrumentation.nix") - ]; - - documentation.nixos.enable = lib.mkOverride 55 false; - boot.loader.grub.enable = false; - system.nixos.revision = lib.mkForce "constant-nixos-revision"; - - # otherwise the evaluation is unnecessarily slow in VM - virtualisation.additionalPaths = lib.mkForce []; - nix.nixPath = lib.mkForce [ "nixpkgs=/nixpkgs" ]; - - deployment.tags = lib.optional (config.networking.hostName != "deployer") "target"; - }; + nixosLib = import (pkgs.path + "/nixos/lib") { }; inputClosureOf = pkg: pkgs.runCommand "full-closure" { refs = pkgs.writeReferencesToFile pkg.drvPath; @@ -138,6 +42,174 @@ let done <$refs ''; + ## The modular NixOS test framework with Colmena additions + colmenaTestModule = { lib, config, ... }: let + cfg = config.colmena.test; + + targetList = "[${concatStringsSep ", " targets}]"; + bundle = pkgs.stdenv.mkDerivation { + name = "${config.name}-bundle"; + dontUnpack = true; + dontInstall = true; + buildPhase = '' + cp -r ${cfg.bundle} $out + chmod u+w $out + cp ${./tools.nix} $out/tools.nix + ''; + }; + in { + options = { + colmena.test = { + bundle = lib.mkOption { + description = '' + Path to a directory to copy into the deployer as /tmp/bundle. + ''; + type = lib.types.path; + }; + + testScript = lib.mkOption { + description = '' + The test script. + + The Colmena test framework will prepend initialization + statements to the actual test script. + ''; + type = lib.types.str; + }; + }; + }; + config = { + testScript = '' + start_all() + '' + lib.optionalString (prebuiltTarget != null) '' + deployer.succeed("nix-store -qR ${prebuiltSystem}") + '' + '' + deployer.succeed("nix-store -qR ${pkgs.path}") + deployer.succeed("ln -sf ${pkgs.path} /nixpkgs") + deployer.succeed("mkdir -p /root/.ssh && touch /root/.ssh/id_rsa && chmod 600 /root/.ssh/id_rsa && cat ${sshKeys.snakeOilPrivateKey} > /root/.ssh/id_rsa") + + ${lib.optionalString (length targets != 0) '' + for node in ${targetList}: + node.wait_for_unit("sshd.service") + deployer.succeed(f"ssh -o StrictHostKeyChecking=accept-new {node.name} true", timeout=30) + ''} + + deployer.succeed("cp --no-preserve=mode -r ${bundle} /tmp/bundle && chmod u+w /tmp/bundle") + + orig_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n")) + def get_new_store_paths(): + cur_store_paths = set(deployer.succeed("ls /nix/store").strip().split("\n")) + new_store_paths = cur_store_paths.difference(orig_store_paths) + deployer.log(f"{len(new_store_paths)} store paths were created") + + l = list(map(lambda n: f"/nix/store/{n}", new_store_paths)) + return l + + ${cfg.testScript} + ''; + }; + }; + evalTest = module: nixosLib.evalTest { + imports = [ + module + colmenaTestModule + { hostPkgs = pkgs; } + ]; + }; + + ## Common setup + + # Setup for deployer nodes + # + # We include the input closure of a prebuilt system profile + # so it can build system profiles for the targets without + # network access. + deployerConfig = { pkgs, lib, config, ... }: { + imports = [ + extraDeployerConfig + ]; + + nix.registry = lib.mkIf (pkgs ? _inputs) { + nixpkgs.flake = pkgs._inputs.nixpkgs; + }; + + nix.nixPath = [ + "nixpkgs=${pkgs.path}" + ]; + + nix.binaryCaches = lib.mkForce []; + + virtualisation = { + memorySize = 3072; + writableStore = true; + additionalPaths = [ + "${pkgs.path}" + ] ++ lib.optionals (prebuiltTarget != null) [ + prebuiltSystem + (inputClosureOf prebuiltSystem) + ]; + }; + + services.openssh.enable = true; + users.users.root.openssh.authorizedKeys.keys = [ + sshKeys.snakeOilPublicKey + ]; + + environment.systemPackages = with pkgs; [ + git # for git flake tests + inotify-tools # for key services build + + # HACK: copy stderr to both stdout and stderr + # (the test framework only captures stdout, and only stderr appears on screen during the build) + (writeShellScriptBin "run-copy-stderr" '' + exec "$@" 2> >(tee /dev/stderr) + '') + ]; + }; + + # Setup for target nodes + # + # Kept as minimal as possible. + targetConfig = { lib, ... }: { + nix.binaryCaches = lib.mkForce []; + + documentation.nixos.enable = lib.mkOverride 60 true; + + services.openssh.enable = true; + users.users.root.openssh.authorizedKeys.keys = [ + sshKeys.snakeOilPublicKey + ]; + virtualisation.writableStore = true; + }; + + nodes = let + deployerNodes = map (name: lib.nameValuePair name deployerConfig) deployers; + targetNodes = map (name: lib.nameValuePair name targetConfig) targets; + in listToAttrs (deployerNodes ++ targetNodes); + + # A "shallow" re-evaluation of the test for use from Colmena + standaloneTest = evalTest ({ ... }: { + inherit nodes; + }); + + prebuiltSystem = standaloneTest.config.nodes.${prebuiltTarget}.system.build.toplevel; + + getStandaloneConfigFor = node: { lib, config, ... }: { + imports = [ + (pkgs.path + "/nixos/lib/testing/nixos-test-base.nix") + (if elem node deployers then deployerConfig else targetConfig) + standaloneTest.config.nodes.${node}.system.build.networkConfig + ]; + + documentation.nixos.enable = lib.mkOverride 55 false; + boot.loader.grub.enable = false; + system.nixos.revision = lib.mkForce "constant-nixos-revision"; + + nix.nixPath = lib.mkForce [ "nixpkgs=/nixpkgs" ]; + + deployment.tags = lib.optional (config.networking.hostName != "deployer") "target"; + }; + makeTest = test: let customArgs = [ "bundle" ]; @@ -190,4 +262,9 @@ let in { inherit pkgs nodes colmena colmenaExec getStandaloneConfigFor inputClosureOf makeTest; + + runTest = module: (evalTest ({ config, ... }: { + imports = [ module { inherit nodes; } ]; + result = config.test; + })).config.result; }