From 01bc04b1d25c09d53d8c4b953f62a5270f89fab9 Mon Sep 17 00:00:00 2001
From: Adam Joseph <adam@westernsemico.com>
Date: Wed, 12 Oct 2022 20:07:18 -0700
Subject: [PATCH] feat(tvix/eval): enable the XFAIL tests

This commit adds support for running the "expected failure" tests in
both the nix and tvix test suites.

I have disabled the eval-fail-blackhole.nix test because it gets
stuck running forever.

Signed-off-by: Adam Joseph <adam@westernsemico.com>
Change-Id: Iba75ce6c8f2becab3c834fcfdd9f4fdc5a4bdb9f
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6990
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
Reviewed-by: grfn <grfn@gws.fyi>
---
 tvix/eval/src/tests/mod.rs                    | 59 ++++++++++++++-----
 ...e.nix => eval-fail-blackhole.nix-disabled} |  0
 2 files changed, 45 insertions(+), 14 deletions(-)
 rename tvix/eval/src/tests/nix_tests/{eval-fail-blackhole.nix => eval-fail-blackhole.nix-disabled} (100%)

diff --git a/tvix/eval/src/tests/mod.rs b/tvix/eval/src/tests/mod.rs
index 03e6f9f02..a0980cc02 100644
--- a/tvix/eval/src/tests/mod.rs
+++ b/tvix/eval/src/tests/mod.rs
@@ -4,24 +4,40 @@ use pretty_assertions::assert_eq;
 
 use test_generator::test_resources;
 
-fn eval_okay_test(code_path: &str) {
+fn eval_test(code_path: &str, expect_success: bool) {
     let base = code_path
         .strip_suffix("nix")
         .expect("test files always end in .nix");
     let exp_path = format!("{}exp", base);
 
     let code = std::fs::read_to_string(code_path).expect("should be able to read test code");
-    let exp = std::fs::read_to_string(exp_path).expect("should be able to read test expectation");
 
-    let result = interpret(&code, Some(code_path.into()), Options::test_options())
-        .expect("evaluation of eval-okay test should succeed");
-    let result_str = format!("{}", result);
-
-    assert_eq!(
-        result_str,
-        exp.trim(),
-        "result value representation (left) must match expectation (right)"
-    );
+    match interpret(&code, Some(code_path.into()), Options::test_options()) {
+        Ok(result) => {
+            if !expect_success {
+                panic!(
+                    "test should have failed, but succeeded with output {}",
+                    result
+                );
+            }
+            let result_str = format!("{}", result);
+            let exp =
+                std::fs::read_to_string(exp_path).expect("should be able to read test expectation");
+            assert_eq!(
+                result_str,
+                exp.trim(),
+                "result value representation (left) must match expectation (right)"
+            );
+        }
+        Err(e) => {
+            if expect_success {
+                panic!(
+                    "evaluation of eval-okay test should succeed, but failed with {:?}",
+                    e
+                );
+            }
+        }
+    }
 }
 
 // identity-* tests contain Nix code snippets which should evaluate to
@@ -48,12 +64,27 @@ fn identity(code_path: &str) {
 // are guaranteed to be valid Nix code.
 #[test_resources("src/tests/tvix_tests/eval-okay-*.nix")]
 fn eval_okay(code_path: &str) {
-    eval_okay_test(code_path)
+    eval_test(code_path, true)
 }
 
-// eval-okay-* tests from the original Nix test suite.
+// eval-fail-* tests from the original Nix test suite.
 #[cfg(feature = "nix_tests")]
 #[test_resources("src/tests/nix_tests/eval-okay-*.nix")]
 fn nix_eval_okay(code_path: &str) {
-    eval_okay_test(code_path)
+    eval_test(code_path, true)
+}
+
+// eval-fail-* tests contain a snippet of Nix code, which is
+// expected to fail evaluation.  The exact type of failure
+// (assertion, parse error, etc) is not currently checked.
+#[test_resources("src/tests/tvix_tests/eval-fail-*.nix")]
+fn eval_fail(code_path: &str) {
+    eval_test(code_path, false)
+}
+
+// eval-fail-* tests from the original Nix test suite.
+#[cfg(feature = "nix_tests")]
+#[test_resources("src/tests/nix_tests/eval-fail-*.nix")]
+fn nix_eval_fail(code_path: &str) {
+    eval_test(code_path, false)
 }
diff --git a/tvix/eval/src/tests/nix_tests/eval-fail-blackhole.nix b/tvix/eval/src/tests/nix_tests/eval-fail-blackhole.nix-disabled
similarity index 100%
rename from tvix/eval/src/tests/nix_tests/eval-fail-blackhole.nix
rename to tvix/eval/src/tests/nix_tests/eval-fail-blackhole.nix-disabled