integration-tests: Cleanup, add parallelism test

integration-tests/apply-fail/default.nix

@@ -1,18 +0,0 @@
-let
-  tools = import ../tools.nix {};
-in tools.makeTest {
-  name = "colmena-apply-fail";
-
-  bundle = ./.;
-
-  testScript = ''
-    beta.block()
-
-    # HACK: copy stderr to both stdout and stderr
-    # (stdout is what's returned, and only stderr appears on screen during the build)
-    logs = deployer.fail("cd /tmp/bundle && ${tools.colmenaExec} apply -v --eval-node-limit 4 --on @target 2> >(tee /dev/stderr)")
-
-    alpha.succeed("grep SUCCESS /etc/deployment")
-    gamma.succeed("grep SUCCESS /etc/deployment")
-  '';
-}

integration-tests/apply/default.nix

@@ -6,76 +6,6 @@ in tools.makeTest {
   bundle = ./.;
 
   testScript = ''
-    poison = " ".join(["this", "must", "not", "exist", "in", "nix", "store"])
+    colmena = "${tools.colmenaExec}"
-    deployer.succeed(f"echo '{poison}' > /tmp/bundle/key-file")
+  '' + builtins.readFile ./test-script.py;
-    deployer.succeed(f"sed -i 's|@poison@|{poison}|g' /tmp/bundle/hive.nix")
-
-    # HACK: copy stderr to both stdout and stderr
-    # (stdout is what's returned, and only stderr appears on screen during the build)
-    logs = deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply -v --eval-node-limit 4 --on @target 2> >(tee /dev/stderr)")
-
-    with subtest("Check that evaluation messages were logged correctly"):
-        assert "must appear during evaluation" in logs
-
-    with subtest("Check that build messages were logged correctly"):
-        assert "must appear during build" in logs
-
-    with subtest("Check that push messages were logged correctly"):
-        assert "copying path" in logs
-
-    with subtest("Check that activation messages were logged correctly"):
-        assert "must appear during activation" in logs
-
-    with subtest("Check that we can still connect to the target nodes"):
-        deployer.succeed("ssh alpha true")
-        deployer.succeed("ssh beta true")
-        deployer.succeed("ssh gamma true")
-
-    with subtest("Check that the new configuration is indeed applied"):
-        alpha.succeed("grep SUCCESS /etc/deployment")
-
-    with subtest("Check that key files have correct contents"):
-        contents = {
-            "/run/keys/key-text":               poison,
-            "/tmp/another-key-dir/key-command": "deployer",
-            "/tmp/another-key-dir/key-file":    poison,
-            "/tmp/another-key-dir/key-file-2":  poison,
-            "/run/keys/pre-activation":         "pre-activation key",
-            "/run/keys/post-activation":        "post-activation key",
-        }
-
-        for path, content in contents.items():
-            alpha.succeed(f"grep '{content}' '{path}'")
-
-    with subtest("Check that key files have correct permissions"):
-        alpha.succeed("getent passwd testuser")
-        alpha.succeed("getent group testgroup")
-
-        permissions = {
-            "/run/keys/key-text":               "600 root root",
-            "/tmp/another-key-dir/key-command": "600 root root",
-            "/tmp/another-key-dir/key-file":    "600 root root",
-            "/tmp/another-key-dir/key-file-2":  "600 root root",
-            "/run/keys/pre-activation":         "640 testuser testgroup",
-            "/run/keys/post-activation":        "600 testuser testgroup",
-        }
-
-        for path, permission in permissions.items():
-            alpha.succeed(f"if [[ \"{permission}\" != \"$(stat -c '%a %U %G' '{path}')\" ]]; then ls -lah '{path}'; exit 1; fi")
-
-    with subtest("Check that key contents are not in the Nix store"):
-        new_store_paths = " ".join(get_new_store_paths())
-
-        ret, stdout = deployer.execute(f"grep -r '{poison}' {new_store_paths}")
-
-        if ret != 1:
-            deployer.log("Forbidden text found in: " + stdout)
-
-        assert ret == 1
-
-    with subtest("Check that our Nix store test is actually working"):
-        deployer.succeed(f"nix-build -E 'with import <nixpkgs> {{}}; writeText \"forbidden-text.txt\" \"{poison}\"'")
-        new_store_paths = " ".join(get_new_store_paths())
-        deployer.succeed(f"grep -r '{poison}' {new_store_paths}")
-  '';
 }

integration-tests/apply/hive.nix

@@ -12,17 +12,8 @@ in {
     nixpkgs = tools.pkgs;
   };
 
-  alpha = { lib, ... }: {
+  defaults = {
-    imports = [
+    environment.etc."deployment".text = "FIRST";
-      (tools.getStandaloneConfigFor "alpha")
-    ];
-
-    environment.systemPackages = [ testPkg ];
-    environment.etc."deployment".text = "SUCCESS";
-
-    system.activationScripts.colmena-test.text = ''
-      echo "must appear during activation"
-    '';
 
     # Will be created during activation
     users.users.testuser = {
@@ -77,6 +68,18 @@ in {
     };
   };
 
+  alpha = { lib, ... }: {
+    imports = [
+      (tools.getStandaloneConfigFor "alpha")
+    ];
+
+    environment.systemPackages = [ testPkg ];
+
+    system.activationScripts.colmena-test.text = ''
+      echo "must appear during activation"
+    '';
+  };
+
   deployer = tools.getStandaloneConfigFor "deployer";
   beta = tools.getStandaloneConfigFor "beta";
   gamma = tools.getStandaloneConfigFor "gamma";

integration-tests/apply/test-script.py

@@ -0,0 +1,88 @@
+# Setup injected here
+
+poison = " ".join(["this", "must", "not", "appear", "in", "the", "nix", "store"])
+deployer.succeed(f"echo '{poison}' > /tmp/bundle/key-file")
+deployer.succeed(f"sed -i 's|@poison@|{poison}|g' /tmp/bundle/hive.nix")
+
+targets = [alpha, beta, gamma]
+
+logs = deployer.succeed("cd /tmp/bundle &&" \
+    f"run-copy-stderr {colmena} apply -v --eval-node-limit 4 --on @target")
+
+with subtest("Check that evaluation messages were logged correctly"):
+    assert "must appear during evaluation" in logs
+
+with subtest("Check that build messages were logged correctly"):
+    assert "must appear during build" in logs
+
+with subtest("Check that push messages were logged correctly"):
+    assert "copying path" in logs
+
+with subtest("Check that activation messages were logged correctly"):
+    assert "must appear during activation" in logs
+
+with subtest("Check that we can still connect to the target nodes"):
+    deployer.succeed("ssh alpha true")
+    deployer.succeed("ssh beta true")
+    deployer.succeed("ssh gamma true")
+
+with subtest("Check that the new configurations are indeed applied"):
+    for node in targets:
+        node.succeed("grep FIRST /etc/deployment")
+
+with subtest("Check that key files have correct contents"):
+    contents = {
+        "/run/keys/key-text":               poison,
+        "/tmp/another-key-dir/key-command": "deployer",
+        "/tmp/another-key-dir/key-file":    poison,
+        "/tmp/another-key-dir/key-file-2":  poison,
+        "/run/keys/pre-activation":         "pre-activation key",
+        "/run/keys/post-activation":        "post-activation key",
+    }
+
+    for node in targets:
+        for path, content in contents.items():
+            node.succeed(f"grep '{content}' '{path}'")
+
+with subtest("Check that key files have correct permissions"):
+    permissions = {
+        "/run/keys/key-text":               "600 root root",
+        "/tmp/another-key-dir/key-command": "600 root root",
+        "/tmp/another-key-dir/key-file":    "600 root root",
+        "/tmp/another-key-dir/key-file-2":  "600 root root",
+        "/run/keys/pre-activation":         "640 testuser testgroup",
+        "/run/keys/post-activation":        "600 testuser testgroup",
+    }
+
+    for node in targets:
+        node.succeed("getent passwd testuser")
+        node.succeed("getent group testgroup")
+
+        for path, permission in permissions.items():
+            node.succeed(f"if [[ \"{permission}\" != \"$(stat -c '%a %U %G' '{path}')\" ]]; then ls -lah '{path}'; exit 1; fi")
+
+with subtest("Check that we can correctly deploy to remaining nodes despite failures"):
+    beta.systemctl("stop sshd")
+
+    deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/hive.nix")
+    deployer.fail("cd /tmp/bundle &&" \
+        f"{colmena} apply -v --eval-node-limit 4 --on @target")
+
+    alpha.succeed("grep SECOND /etc/deployment")
+    beta.succeed("grep FIRST /etc/deployment")
+    gamma.succeed("grep SECOND /etc/deployment")
+
+with subtest("Check that key contents are not in the Nix store"):
+    new_store_paths = " ".join(get_new_store_paths())
+
+    ret, stdout = deployer.execute(f"grep -r '{poison}' {new_store_paths}")
+
+    if ret != 1:
+        deployer.log("Forbidden text found in: " + stdout)
+
+    assert ret == 1
+
+with subtest("Check that our Nix store test is actually working"):
+    deployer.succeed(f"nix-build -E 'with import <nixpkgs> {{}}; writeText \"forbidden-text.txt\" \"{poison}\"'")
+    new_store_paths = " ".join(get_new_store_paths())
+    deployer.succeed(f"grep -r '{poison}' {new_store_paths}")

integration-tests/parallel/default.nix

@@ -0,0 +1,27 @@
+let
+  tools = import ../tools.nix {};
+in tools.makeTest {
+  name = "colmena-parallel";
+
+  bundle = ./.;
+
+  testScript = ''
+    deployer.succeed("cd /tmp/bundle &&" \
+        "${tools.colmenaExec} apply push -v --eval-node-limit 4 --on @target")
+
+    logs = deployer.succeed("cd /tmp/bundle &&" \
+        "run-copy-stderr ${tools.colmenaExec} apply switch -v --eval-node-limit 4 --parallel 4 --on @target")
+
+    for node in [alpha, beta, gamma]:
+        node.succeed("grep SUCCESS /etc/deployment")
+
+    with subtest("Check that activation is correctly parallelized"):
+        timestamps = list(map(lambda l: int(l.strip().split("---")[1]) / 1000000,
+            filter(lambda l: "Activation triggered" in l, logs.split("\n"))))
+
+        delay = max(timestamps) - min(timestamps)
+        deployer.log(f"Time between activations: {delay}ms")
+
+        assert delay < 2000
+  '';
+}

integration-tests/parallel/hive.nix

@@ -1,12 +1,5 @@
 let
   tools = import ./tools.nix { insideVm = true; };
-
-  testPkg = let
-    text = builtins.trace "must appear during evaluation" ''
-      echo "must appear during build"
-      mkdir -p $out
-    '';
-  in tools.pkgs.runCommand "test-package" {} text;
 in {
   meta = {
     nixpkgs = tools.pkgs;
@@ -14,6 +7,11 @@ in {
 
   defaults = {
     environment.etc."deployment".text = "SUCCESS";
+
+    system.activationScripts.activationDelay.text = ''
+      >&2 echo "Activation triggered --- $(date +%s%N)"
+      sleep 3
+    '';
   };
 
   deployer = tools.getStandaloneConfigFor "deployer";

integration-tests/tools.nix

@@ -45,6 +45,14 @@ let
           (inputClosureOf prebuiltNode)
         ];
       };
+
+      environment.systemPackages = [
+        # HACK: copy stderr to both stdout and stderr
+        # (the test framework only captures stdout, and only stderr appears on screen during the build)
+        (pkgs.writeShellScriptBin "run-copy-stderr" ''
+          exec "$@" 2> >(tee /dev/stderr)
+        '')
+      ];
     };
     target = { lib, ... }: {
       nix.binaryCaches = lib.mkForce [];