From 57b3ccca8591659cfe28cf0727f3136ffb0594f7 Mon Sep 17 00:00:00 2001
From: Zhaofeng Li <hello@zhaofeng.li>
Date: Sat, 22 Jan 2022 17:50:53 -0800
Subject: [PATCH] integration-tests: Test streaming evaluator

---
 integration-tests/apply/default.nix    |  7 +++++--
 integration-tests/apply/test-script.py |  4 ++--
 integration-tests/default.nix          |  2 ++
 integration-tests/flakes/default.nix   | 12 +++++++-----
 integration-tests/tools.nix            |  2 +-
 5 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/integration-tests/apply/default.nix b/integration-tests/apply/default.nix
index 15b8d38..6422a32 100644
--- a/integration-tests/apply/default.nix
+++ b/integration-tests/apply/default.nix
@@ -1,13 +1,16 @@
-{ pkgs ? import ../nixpkgs.nix }:
+{ pkgs ? import ../nixpkgs.nix
+, evaluator ? "chunked"
+}:
 
 let
   tools = pkgs.callPackage ../tools.nix {};
 in tools.makeTest {
-  name = "colmena-apply";
+  name = "colmena-apply-${evaluator}";
 
   bundle = ./.;
 
   testScript = ''
     colmena = "${tools.colmenaExec}"
+    evaluator = "${evaluator}"
   '' + builtins.readFile ./test-script.py;
 }
diff --git a/integration-tests/apply/test-script.py b/integration-tests/apply/test-script.py
index 71233c0..24823d1 100644
--- a/integration-tests/apply/test-script.py
+++ b/integration-tests/apply/test-script.py
@@ -7,7 +7,7 @@ deployer.succeed(f"sed -i 's|@poison@|{poison}|g' /tmp/bundle/hive.nix")
 targets = [alpha, beta, gamma]
 
 logs = deployer.succeed("cd /tmp/bundle &&" \
-    f"run-copy-stderr {colmena} apply --eval-node-limit 4 --on @target --keep-result")
+    f"run-copy-stderr {colmena} apply --evaluator {evaluator} --eval-node-limit 4 --on @target --keep-result")
 
 with subtest("Check that evaluation messages were logged correctly"):
     assert "must appear during evaluation" in logs
@@ -73,7 +73,7 @@ with subtest("Check that we can correctly deploy to remaining nodes despite fail
 
     deployer.succeed("sed -i s/FIRST/SECOND/g /tmp/bundle/hive.nix")
     deployer.fail("cd /tmp/bundle &&" \
-        f"{colmena} apply --eval-node-limit 4 --on @target")
+        f"{colmena} apply --evaluator {evaluator} --eval-node-limit 4 --on @target")
 
     alpha.succeed("grep SECOND /etc/deployment")
     beta.succeed("grep FIRST /etc/deployment")
diff --git a/integration-tests/default.nix b/integration-tests/default.nix
index 3f1707e..1341f3c 100644
--- a/integration-tests/default.nix
+++ b/integration-tests/default.nix
@@ -1,8 +1,10 @@
 {
   apply = import ./apply {};
+  apply-streaming = import ./apply { evaluator = "streaming"; };
   apply-local = import ./apply-local {};
   build-on-target = import ./build-on-target {};
   exec = import ./exec {};
   flakes = import ./flakes {};
+  flakes-streaming = import ./flakes { evaluator = "streaming"; };
   parallel = import ./parallel {};
 }
diff --git a/integration-tests/flakes/default.nix b/integration-tests/flakes/default.nix
index c0ccc83..693b17b 100644
--- a/integration-tests/flakes/default.nix
+++ b/integration-tests/flakes/default.nix
@@ -1,11 +1,13 @@
-{ pkgs ? import ../nixpkgs.nix }:
+{ pkgs ? import ../nixpkgs.nix
+, evaluator ? "chunked"
+}:
 
 let
   tools = pkgs.callPackage ../tools.nix {
     targets = [ "alpha" ];
   };
 in tools.makeTest {
-  name = "colmena-flakes";
+  name = "colmena-flakes-${evaluator}";
 
   bundle = ./.;
 
@@ -17,7 +19,7 @@ in tools.makeTest {
         deployer.succeed("cd /tmp/bundle && nix --experimental-features \"nix-command flakes\" flake lock --impure")
 
     with subtest("Deploy with a plain flake without git"):
-        deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target")
+        deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
         alpha.succeed("grep FIRST /etc/deployment")
 
     with subtest("Deploy with a git flake"):
@@ -25,12 +27,12 @@ in tools.makeTest {
 
         # don't put probe.nix in source control - should fail
         deployer.succeed("cd /tmp/bundle && git init && git add flake.nix flake.lock hive.nix tools.nix")
-        logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target")
+        logs = deployer.fail("cd /tmp/bundle && run-copy-stderr ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
         assert re.search(r"probe.nix.*No such file or directory", logs)
 
         # now it should succeed
         deployer.succeed("cd /tmp/bundle && git add probe.nix")
-        deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target")
+        deployer.succeed("cd /tmp/bundle && ${tools.colmenaExec} apply --on @target --evaluator ${evaluator}")
         alpha.succeed("grep SECOND /etc/deployment")
   '';
 }
diff --git a/integration-tests/tools.nix b/integration-tests/tools.nix
index 92eb0b0..b1d2047 100644
--- a/integration-tests/tools.nix
+++ b/integration-tests/tools.nix
@@ -45,7 +45,7 @@ let
       nix.binaryCaches = lib.mkForce [];
 
       virtualisation = {
-        memorySize = 2048;
+        memorySize = 3072;
         writableStore = true;
         additionalPaths = [
           "${pkgs.path}"