integration-tests: Test streaming evaluator

integration-tests/apply/default.nix

@@ -1,13 +1,16 @@
-{ pkgs ? import ../nixpkgs.nix }:
+{ pkgs ? import ../nixpkgs.nix
+, evaluator ? "chunked"
+}:
 
 let
   tools = pkgs.callPackage ../tools.nix {};
 in tools.makeTest {
-  name = "colmena-apply";
+  name = "colmena-apply-${evaluator}";
 
   bundle = ./.;
 
   testScript = ''
     colmena = "${tools.colmenaExec}"
+    evaluator = "${evaluator}"
   '' + builtins.readFile ./test-script.py;
 }

integration-tests/apply/test-script.py

@@ -7,7 +7,7 @@ deployer.succeed(f"sed -i 's|@poison@|{poison}|g' /tmp/bundle/hive.nix")
 targets = [alpha, beta, gamma]
 
 logs = deployer.succeed("cd /tmp/bundle &&" \
-    f"run-copy-stderr {colmena} apply --eval-node-limit 4 --on @target --keep-result")
+    f"run-copy-stderr {colmena} apply --evaluator {evaluator} --eval-node-limit 4 --on @target --keep-result")
 
 with subtest("Check that evaluation messages were logged correctly"):
     assert "must appear during evaluation" in logs
@@ -73,7 +73,7 @@ with subtest("Check that we can correctly deploy to remaining nodes despite fail
 
     deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/hive.nix")
     deployer.fail("cd /tmp/bundle &&" \
-        f"{colmena} apply --eval-node-limit 4 --on @target")
+        f"{colmena} apply --evaluator {evaluator} --eval-node-limit 4 --on @target")
 
     alpha.succeed("grep SECOND /etc/deployment")
     beta.succeed("grep FIRST /etc/deployment")

integration-tests/default.nix

@@ -1,8 +1,10 @@
 {
   apply = import ./apply {};
+  apply-streaming = import ./apply { evaluator = "streaming"; };
   apply-local = import ./apply-local {};
   build-on-target = import ./build-on-target {};
   exec = import ./exec {};
   flakes = import ./flakes {};
+  flakes-streaming = import ./flakes { evaluator = "streaming"; };
   parallel = import ./parallel {};
 }

integration-tests/flakes/default.nix

@@ -1,11 +1,13 @@
-{ pkgs ? import ../nixpkgs.nix }:
+{ pkgs ? import ../nixpkgs.nix
+, evaluator ? "chunked"
+}:
 
 let
   tools = pkgs.callPackage ../tools.nix {
     targets = [ "alpha" ];
   };
 in tools.makeTest {
-  name = "colmena-flakes";
+  name = "colmena-flakes-${evaluator}";
 
   bundle = ./.;
 
@@ -17,7 +19,7 @@ in tools.makeTest {
         deployer.succeed("cd /tmp/bundle && nix --experimental-features \"nix-command flakes\" flake lock --impure")
 
     with subtest("Deploy with a plain flake without git"):
-        deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target")
+        deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
         alpha.succeed("grep FIRST /etc/deployment")
 
     with subtest("Deploy with a git flake"):
@@ -25,12 +27,12 @@ in tools.makeTest {
 
         # don't put probe.nix in source control - should fail
         deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
-        logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target")
+        logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
         assert re.search(r"probe.nix.*No such file or directory", logs)
 
         # now it should succeed
         deployer.succeed("cd /tmp/bundle && git add probe.nix")
-        deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target")
+        deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
         alpha.succeed("grep SECOND /etc/deployment")
   '';
 }

integration-tests/tools.nix

@@ -45,7 +45,7 @@ let
       nix.binaryCaches = lib.mkForce [];
 
       virtualisation = {
-        memorySize = 2048;
+        memorySize = 3072;
         writableStore = true;
         additionalPaths = [
           "${pkgs.path}"