From ba16f507221d90ce83785ad4b672bcde67b4f2e1 Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Sat, 4 Dec 2021 01:03:26 -0800 Subject: [PATCH] integration-tests: Cleanup, add parallelism test --- integration-tests/apply-fail/default.nix | 18 ---- integration-tests/apply/default.nix | 74 +--------------- integration-tests/apply/hive.nix | 25 +++--- integration-tests/apply/test-script.py | 88 +++++++++++++++++++ integration-tests/parallel/default.nix | 27 ++++++ .../{apply-fail => parallel}/hive.nix | 12 ++- integration-tests/tools.nix | 8 ++ 7 files changed, 144 insertions(+), 108 deletions(-) delete mode 100644 integration-tests/apply-fail/default.nix create mode 100644 integration-tests/apply/test-script.py create mode 100644 integration-tests/parallel/default.nix rename integration-tests/{apply-fail => parallel}/hive.nix (67%) diff --git a/integration-tests/apply-fail/default.nix b/integration-tests/apply-fail/default.nix deleted file mode 100644 index 9f3e5b1..0000000 --- a/integration-tests/apply-fail/default.nix +++ /dev/null @@ -1,18 +0,0 @@ -let - tools = import ../tools.nix {}; -in tools.makeTest { - name = "colmena-apply-fail"; - - bundle = ./.; - - testScript = '' - beta.block() - - # HACK: copy stderr to both stdout and stderr - # (stdout is what's returned, and only stderr appears on screen during the build) - logs = deployer.fail("cd /tmp/bundle && ${tools.colmenaExec} apply -v --eval-node-limit 4 --on @target 2> >(tee /dev/stderr)") - - alpha.succeed("grep SUCCESS /etc/deployment") - gamma.succeed("grep SUCCESS /etc/deployment") - ''; -} diff --git a/integration-tests/apply/default.nix b/integration-tests/apply/default.nix index 17b68b1..094e8f1 100644 --- a/integration-tests/apply/default.nix +++ b/integration-tests/apply/default.nix @@ -6,76 +6,6 @@ in tools.makeTest { bundle = ./.; testScript = '' - poison = " ".join(["this", "must", "not", "exist", "in", "nix", "store"]) - deployer.succeed(f"echo '{poison}' > /tmp/bundle/key-file") - deployer.succeed(f"sed -i 's|@poison@|{poison}|g' /tmp/bundle/hive.nix") - - # HACK: copy stderr to both stdout and stderr - # (stdout is what's returned, and only stderr appears on screen during the build) - logs = deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply -v --eval-node-limit 4 --on @target 2> >(tee /dev/stderr)") - - with subtest("Check that evaluation messages were logged correctly"): - assert "must appear during evaluation" in logs - - with subtest("Check that build messages were logged correctly"): - assert "must appear during build" in logs - - with subtest("Check that push messages were logged correctly"): - assert "copying path" in logs - - with subtest("Check that activation messages were logged correctly"): - assert "must appear during activation" in logs - - with subtest("Check that we can still connect to the target nodes"): - deployer.succeed("ssh alpha true") - deployer.succeed("ssh beta true") - deployer.succeed("ssh gamma true") - - with subtest("Check that the new configuration is indeed applied"): - alpha.succeed("grep SUCCESS /etc/deployment") - - with subtest("Check that key files have correct contents"): - contents = { - "/run/keys/key-text": poison, - "/tmp/another-key-dir/key-command": "deployer", - "/tmp/another-key-dir/key-file": poison, - "/tmp/another-key-dir/key-file-2": poison, - "/run/keys/pre-activation": "pre-activation key", - "/run/keys/post-activation": "post-activation key", - } - - for path, content in contents.items(): - alpha.succeed(f"grep '{content}' '{path}'") - - with subtest("Check that key files have correct permissions"): - alpha.succeed("getent passwd testuser") - alpha.succeed("getent group testgroup") - - permissions = { - "/run/keys/key-text": "600 root root", - "/tmp/another-key-dir/key-command": "600 root root", - "/tmp/another-key-dir/key-file": "600 root root", - "/tmp/another-key-dir/key-file-2": "600 root root", - "/run/keys/pre-activation": "640 testuser testgroup", - "/run/keys/post-activation": "600 testuser testgroup", - } - - for path, permission in permissions.items(): - alpha.succeed(f"if [[ \"{permission}\" != \"$(stat -c '%a %U %G' '{path}')\" ]]; then ls -lah '{path}'; exit 1; fi") - - with subtest("Check that key contents are not in the Nix store"): - new_store_paths = " ".join(get_new_store_paths()) - - ret, stdout = deployer.execute(f"grep -r '{poison}' {new_store_paths}") - - if ret != 1: - deployer.log("Forbidden text found in: " + stdout) - - assert ret == 1 - - with subtest("Check that our Nix store test is actually working"): - deployer.succeed(f"nix-build -E 'with import {{}}; writeText \"forbidden-text.txt\" \"{poison}\"'") - new_store_paths = " ".join(get_new_store_paths()) - deployer.succeed(f"grep -r '{poison}' {new_store_paths}") - ''; + colmena = "${tools.colmenaExec}" + '' + builtins.readFile ./test-script.py; } diff --git a/integration-tests/apply/hive.nix b/integration-tests/apply/hive.nix index 1509c0b..7204245 100644 --- a/integration-tests/apply/hive.nix +++ b/integration-tests/apply/hive.nix @@ -12,17 +12,8 @@ in { nixpkgs = tools.pkgs; }; - alpha = { lib, ... }: { - imports = [ - (tools.getStandaloneConfigFor "alpha") - ]; - - environment.systemPackages = [ testPkg ]; - environment.etc."deployment".text = "SUCCESS"; - - system.activationScripts.colmena-test.text = '' - echo "must appear during activation" - ''; + defaults = { + environment.etc."deployment".text = "FIRST"; # Will be created during activation users.users.testuser = { @@ -77,6 +68,18 @@ in { }; }; + alpha = { lib, ... }: { + imports = [ + (tools.getStandaloneConfigFor "alpha") + ]; + + environment.systemPackages = [ testPkg ]; + + system.activationScripts.colmena-test.text = '' + echo "must appear during activation" + ''; + }; + deployer = tools.getStandaloneConfigFor "deployer"; beta = tools.getStandaloneConfigFor "beta"; gamma = tools.getStandaloneConfigFor "gamma"; diff --git a/integration-tests/apply/test-script.py b/integration-tests/apply/test-script.py new file mode 100644 index 0000000..3eb85a6 --- /dev/null +++ b/integration-tests/apply/test-script.py @@ -0,0 +1,88 @@ +# Setup injected here + +poison = " ".join(["this", "must", "not", "appear", "in", "the", "nix", "store"]) +deployer.succeed(f"echo '{poison}' > /tmp/bundle/key-file") +deployer.succeed(f"sed -i 's|@poison@|{poison}|g' /tmp/bundle/hive.nix") + +targets = [alpha, beta, gamma] + +logs = deployer.succeed("cd /tmp/bundle &&" \ + f"run-copy-stderr {colmena} apply -v --eval-node-limit 4 --on @target") + +with subtest("Check that evaluation messages were logged correctly"): + assert "must appear during evaluation" in logs + +with subtest("Check that build messages were logged correctly"): + assert "must appear during build" in logs + +with subtest("Check that push messages were logged correctly"): + assert "copying path" in logs + +with subtest("Check that activation messages were logged correctly"): + assert "must appear during activation" in logs + +with subtest("Check that we can still connect to the target nodes"): + deployer.succeed("ssh alpha true") + deployer.succeed("ssh beta true") + deployer.succeed("ssh gamma true") + +with subtest("Check that the new configurations are indeed applied"): + for node in targets: + node.succeed("grep FIRST /etc/deployment") + +with subtest("Check that key files have correct contents"): + contents = { + "/run/keys/key-text": poison, + "/tmp/another-key-dir/key-command": "deployer", + "/tmp/another-key-dir/key-file": poison, + "/tmp/another-key-dir/key-file-2": poison, + "/run/keys/pre-activation": "pre-activation key", + "/run/keys/post-activation": "post-activation key", + } + + for node in targets: + for path, content in contents.items(): + node.succeed(f"grep '{content}' '{path}'") + +with subtest("Check that key files have correct permissions"): + permissions = { + "/run/keys/key-text": "600 root root", + "/tmp/another-key-dir/key-command": "600 root root", + "/tmp/another-key-dir/key-file": "600 root root", + "/tmp/another-key-dir/key-file-2": "600 root root", + "/run/keys/pre-activation": "640 testuser testgroup", + "/run/keys/post-activation": "600 testuser testgroup", + } + + for node in targets: + node.succeed("getent passwd testuser") + node.succeed("getent group testgroup") + + for path, permission in permissions.items(): + node.succeed(f"if [[ \"{permission}\" != \"$(stat -c '%a %U %G' '{path}')\" ]]; then ls -lah '{path}'; exit 1; fi") + +with subtest("Check that we can correctly deploy to remaining nodes despite failures"): + beta.systemctl("stop sshd") + + deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/hive.nix") + deployer.fail("cd /tmp/bundle &&" \ + f"{colmena} apply -v --eval-node-limit 4 --on @target") + + alpha.succeed("grep SECOND /etc/deployment") + beta.succeed("grep FIRST /etc/deployment") + gamma.succeed("grep SECOND /etc/deployment") + +with subtest("Check that key contents are not in the Nix store"): + new_store_paths = " ".join(get_new_store_paths()) + + ret, stdout = deployer.execute(f"grep -r '{poison}' {new_store_paths}") + + if ret != 1: + deployer.log("Forbidden text found in: " + stdout) + + assert ret == 1 + +with subtest("Check that our Nix store test is actually working"): + deployer.succeed(f"nix-build -E 'with import {{}}; writeText \"forbidden-text.txt\" \"{poison}\"'") + new_store_paths = " ".join(get_new_store_paths()) + deployer.succeed(f"grep -r '{poison}' {new_store_paths}") diff --git a/integration-tests/parallel/default.nix b/integration-tests/parallel/default.nix new file mode 100644 index 0000000..6d1b17a --- /dev/null +++ b/integration-tests/parallel/default.nix @@ -0,0 +1,27 @@ +let + tools = import ../tools.nix {}; +in tools.makeTest { + name = "colmena-parallel"; + + bundle = ./.; + + testScript = '' + deployer.succeed("cd /tmp/bundle &&" \ + "${tools.colmenaExec} apply push -v --eval-node-limit 4 --on @target") + + logs = deployer.succeed("cd /tmp/bundle &&" \ + "run-copy-stderr ${tools.colmenaExec} apply switch -v --eval-node-limit 4 --parallel 4 --on @target") + + for node in [alpha, beta, gamma]: + node.succeed("grep SUCCESS /etc/deployment") + + with subtest("Check that activation is correctly parallelized"): + timestamps = list(map(lambda l: int(l.strip().split("---")[1]) / 1000000, + filter(lambda l: "Activation triggered" in l, logs.split("\n")))) + + delay = max(timestamps) - min(timestamps) + deployer.log(f"Time between activations: {delay}ms") + + assert delay < 2000 + ''; +} diff --git a/integration-tests/apply-fail/hive.nix b/integration-tests/parallel/hive.nix similarity index 67% rename from integration-tests/apply-fail/hive.nix rename to integration-tests/parallel/hive.nix index f86a35f..1c2674e 100644 --- a/integration-tests/apply-fail/hive.nix +++ b/integration-tests/parallel/hive.nix @@ -1,12 +1,5 @@ let tools = import ./tools.nix { insideVm = true; }; - - testPkg = let - text = builtins.trace "must appear during evaluation" '' - echo "must appear during build" - mkdir -p $out - ''; - in tools.pkgs.runCommand "test-package" {} text; in { meta = { nixpkgs = tools.pkgs; @@ -14,6 +7,11 @@ in { defaults = { environment.etc."deployment".text = "SUCCESS"; + + system.activationScripts.activationDelay.text = '' + >&2 echo "Activation triggered --- $(date +%s%N)" + sleep 3 + ''; }; deployer = tools.getStandaloneConfigFor "deployer"; diff --git a/integration-tests/tools.nix b/integration-tests/tools.nix index 3340668..2464683 100644 --- a/integration-tests/tools.nix +++ b/integration-tests/tools.nix @@ -45,6 +45,14 @@ let (inputClosureOf prebuiltNode) ]; }; + + environment.systemPackages = [ + # HACK: copy stderr to both stdout and stderr + # (the test framework only captures stdout, and only stderr appears on screen during the build) + (pkgs.writeShellScriptBin "run-copy-stderr" '' + exec "$@" 2> >(tee /dev/stderr) + '') + ]; }; target = { lib, ... }: { nix.binaryCaches = lib.mkForce [];