From 0ab6494286e77abcdc0bba1c92386fc7e654ca12 Mon Sep 17 00:00:00 2001
From: sterni <sternenseemann@systemli.org>
Date: Sat, 27 May 2023 21:26:38 +0200
Subject: [PATCH] refactor(tvix/eval/nix_oracle): allow specifying eval
 strictness
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This will be useful for comparing thunking behavior to C++ Nix. I
considered adding this capability to the tvix_tests/nix_tests
infrastructure, but as it would require changing the test file naming
scheme to do it in a clean way, I've postponed it–it's nice that our
tests are compatible with C++ Nix's test suite.

Change-Id: I60bcdd98ed25140e716f0858f8dc28f21ab957aa
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8657
Autosubmit: sterni <sternenseemann@systemli.org>
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
---
 tvix/eval/tests/nix_oracle.rs | 38 ++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/tvix/eval/tests/nix_oracle.rs b/tvix/eval/tests/nix_oracle.rs
index b3ef08f87..5a5be56f7 100644
--- a/tvix/eval/tests/nix_oracle.rs
+++ b/tvix/eval/tests/nix_oracle.rs
@@ -11,11 +11,23 @@ fn nix_binary_path() -> PathBuf {
         .into()
 }
 
-fn nix_eval(expr: &str) -> String {
+#[derive(Clone, Copy)]
+enum Strictness {
+    Lazy,
+    Strict,
+}
+
+fn nix_eval(expr: &str, strictness: Strictness) -> String {
     let store_dir = TempDir::new("store-dir").unwrap();
 
+    let mut args = match strictness {
+        Strictness::Lazy => vec![],
+        Strictness::Strict => vec!["--strict"],
+    };
+    args.extend_from_slice(&["--eval", "-E"]);
+
     let output = Command::new(nix_binary_path())
-        .args(["--eval", "--strict", "-E"])
+        .args(&args[..])
         .arg(format!("({expr})"))
         .env(
             "NIX_REMOTE",
@@ -38,10 +50,10 @@ fn nix_eval(expr: &str) -> String {
 /// `NIX_INSTANTIATE_BINARY_PATH` env var to resolve the `nix-instantiate` binary) and tvix, and
 /// assert that the result is identical
 #[track_caller]
-fn compare_eval(expr: &str) {
-    let nix_result = nix_eval(expr);
+fn compare_eval(expr: &str, strictness: Strictness) {
+    let nix_result = nix_eval(expr, strictness);
     let mut eval = tvix_eval::Evaluation::new(expr, None);
-    eval.strict = true;
+    eval.strict = matches!(strictness, Strictness::Strict);
     eval.io_handle = Box::new(tvix_eval::StdIO);
 
     let tvix_result = eval
@@ -56,19 +68,25 @@ fn compare_eval(expr: &str) {
 /// Generate a suite of tests which call [`compare_eval`] on expressions, checking that nix and tvix
 /// return identical results.
 macro_rules! compare_eval_tests {
-    () => {};
-    ($(#[$meta:meta])* $test_name: ident($expr: expr); $($rest:tt)*) => {
+    ($strictness:expr, {}) => {};
+    ($strictness:expr, {$(#[$meta:meta])* $test_name: ident($expr: expr); $($rest:tt)*}) => {
         #[test]
         $(#[$meta])*
         fn $test_name() {
-            compare_eval($expr);
+            compare_eval($expr, $strictness);
         }
 
-        compare_eval_tests!($($rest)*);
+        compare_eval_tests!($strictness, { $($rest)* });
     }
 }
 
-compare_eval_tests! {
+macro_rules! compare_strict_eval_tests {
+    ($($tests:tt)*) => {
+        compare_eval_tests!(Strictness::Lazy, { $($tests)* });
+    }
+}
+
+compare_strict_eval_tests! {
     literal_int("1");
     add_ints("1 + 1");
     add_lists("[1 2] ++ [3 4]");