integration-tests: More elaborate tests

integration-tests/apply-local/default.nix

@@ -1,7 +1,7 @@
 let
   tools = import ../tools.nix {};
 in tools.makeTest {
-  name = "colmena-exec";
+  name = "colmena-apply-local";
 
   bundle = ./.;
 

integration-tests/apply/default.nix

@@ -6,22 +6,76 @@ in tools.makeTest {
   bundle = ./.;
 
   testScript = ''
-    logs = deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply -v --on @target 2> >(tee /dev/stderr)")
+    poison = " ".join(["this", "must", "not", "exist", "in", "nix", "store"])
+    deployer.succeed(f"echo '{poison}' > /tmp/bundle/key-file")
+    deployer.succeed(f"sed -i 's|@poison@|{poison}|g' /tmp/bundle/hive.nix")
 
-    with subtest("Check whether build messages are logged correctly"):
+    # HACK: copy stderr to both stdout and stderr
+    # (stdout is what's returned, and only stderr appears on screen during the build)
+    logs = deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply -v --eval-node-limit 4 --on @target 2> >(tee /dev/stderr)")
+
+    with subtest("Check that evaluation messages were logged correctly"):
+        assert "must appear during evaluation" in logs
+
+    with subtest("Check that build messages were logged correctly"):
         assert "must appear during build" in logs
 
-    with subtest("Check whether push messages are logged correctly"):
+    with subtest("Check that push messages were logged correctly"):
         assert "copying path" in logs
 
-    with subtest("Check whether activation messages are logged correctly"):
+    with subtest("Check that activation messages were logged correctly"):
         assert "must appear during activation" in logs
 
-    alpha.succeed("grep SUCCESS /etc/deployment")
+    with subtest("Check that we can still connect to the target nodes"):
-    alpha.succeed("grep 'key content' /run/keys/example")
+        deployer.succeed("ssh alpha true")
+        deployer.succeed("ssh beta true")
+        deployer.succeed("ssh gamma true")
 
-    deployer.succeed("ssh alpha true")
+    with subtest("Check that the new configuration is indeed applied"):
-    deployer.succeed("ssh beta true")
+        alpha.succeed("grep SUCCESS /etc/deployment")
-    deployer.succeed("ssh gamma true")
+
+    with subtest("Check that key files have correct contents"):
+        contents = {
+            "/run/keys/key-text":               poison,
+            "/tmp/another-key-dir/key-command": "deployer",
+            "/tmp/another-key-dir/key-file":    poison,
+            "/tmp/another-key-dir/key-file-2":  poison,
+            "/run/keys/pre-activation":         "pre-activation key",
+            "/run/keys/post-activation":        "post-activation key",
+        }
+
+        for path, content in contents.items():
+            alpha.succeed(f"grep '{content}' '{path}'")
+
+    with subtest("Check that key files have correct permissions"):
+        alpha.succeed("getent passwd testuser")
+        alpha.succeed("getent group testgroup")
+
+        permissions = {
+            "/run/keys/key-text":               "600 root root",
+            "/tmp/another-key-dir/key-command": "600 root root",
+            "/tmp/another-key-dir/key-file":    "600 root root",
+            "/tmp/another-key-dir/key-file-2":  "600 root root",
+            "/run/keys/pre-activation":         "640 testuser testgroup",
+            "/run/keys/post-activation":        "600 testuser testgroup",
+        }
+
+        for path, permission in permissions.items():
+            alpha.succeed(f"if [[ \"{permission}\" != \"$(stat -c '%a %U %G' '{path}')\" ]]; then ls -lah '{path}'; exit 1; fi")
+
+    with subtest("Check that key contents are not in the Nix store"):
+        new_store_paths = " ".join(get_new_store_paths())
+
+        ret, stdout = deployer.execute(f"grep -r '{poison}' {new_store_paths}")
+
+        if ret != 1:
+            deployer.log("Forbidden text found in: " + stdout)
+
+        assert ret == 1
+
+    with subtest("Check that our Nix store test is actually working"):
+        deployer.succeed(f"nix-build -E 'with import <nixpkgs> {{}}; writeText \"forbidden-text.txt\" \"{poison}\"'")
+        new_store_paths = " ".join(get_new_store_paths())
+        deployer.succeed(f"grep -r '{poison}' {new_store_paths}")
   '';
 }

integration-tests/apply/hive.nix

@@ -1,10 +1,12 @@
 let
   tools = import ./tools.nix { insideVm = true; };
 
-  testPkg = tools.pkgs.runCommand "test-package" {} ''
+  testPkg = let
-    echo "must appear during build"
+    text = builtins.trace "must appear during evaluation" ''
-    mkdir -p $out
+      echo "must appear during build"
-  '';
+      mkdir -p $out
+    '';
+  in tools.pkgs.runCommand "test-package" {} text;
 in {
   meta = {
     nixpkgs = tools.pkgs;
@@ -22,9 +24,57 @@ in {
       echo "must appear during activation"
     '';
 
-    deployment.keys.example.text = ''
+    # Will be created during activation
-      key content
+    users.users.testuser = {
-    '';
+      isSystemUser = true;
+      group = "testgroup";
+    };
+    users.groups.testgroup = {};
+
+    # /run/keys/key-text
+    deployment.keys.key-text = {
+      text = ''
+        @poison@
+      '';
+    };
+
+    # /tmp/another-key-dir/key-command
+    deployment.keys.key-command = {
+      destDir = "/tmp/another-key-dir";
+      keyCommand = [ "hostname" ];
+    };
+
+    # /tmp/another-key-dir/key-file
+    deployment.keys.key-file = {
+      destDir = "/tmp/another-key-dir";
+      keyFile = "/tmp/bundle/key-file";
+    };
+
+    # /tmp/another-key-dir/key-file-2
+    deployment.keys.key-file-2 = {
+      destDir = "/tmp/another-key-dir";
+      keyFile = ./key-file;
+    };
+
+    # /run/keys/pre-activation
+    deployment.keys.pre-activation = {
+      text = "pre-activation key";
+      uploadAt = "pre-activation";
+
+      user = "testuser";
+      group = "testgroup";
+      permissions = "640";
+    };
+
+    # /run/keys/post-activation
+    deployment.keys.post-activation = {
+      text = "post-activation key";
+      uploadAt = "post-activation";
+
+      user = "testuser";
+      group = "testgroup";
+      permissions = "600";
+    };
   };
 
   deployer = tools.getStandaloneConfigFor "deployer";

integration-tests/tools.nix

@@ -37,7 +37,7 @@ let
       nix.binaryCaches = lib.mkForce [];
 
       virtualisation = {
-        memorySize = 1024;
+        memorySize = 2048;
         writableStore = true;
         additionalPaths = [
           "${pkgs.path}"
@@ -46,7 +46,9 @@ let
         ];
       };
     };
-    target = {
+    target = { lib, ... }: {
+      nix.binaryCaches = lib.mkForce [];
+
       services.openssh.enable = true;
       users.users.root.openssh.authorizedKeys.keys = [
         sshKeys.snakeOilPublicKey
@@ -112,6 +114,15 @@ let
       deployer.succeed("ssh -o StrictHostKeyChecking=accept-new alpha ls")
 
       deployer.succeed("cp --no-preserve=mode -r ${bundle} /tmp/bundle && chmod u+w /tmp/bundle")
+
+      orig_store_paths = set(deployer.succeed("ls /nix/store | sort").strip().split("\n"))
+      def get_new_store_paths():
+          cur_store_paths = set(deployer.succeed("ls /nix/store | sort").strip().split("\n"))
+          new_store_paths = cur_store_paths.difference(orig_store_paths)
+          deployer.log(f"{len(new_store_paths)} store paths were created")
+
+          l = list(map(lambda n: f"/nix/store/{n}", new_store_paths))
+          return l
     '' + test.testScript;
 
     bundle = pkgs.stdenv.mkDerivation {