refactor(tvix/eval): move nix_tests to separate module
Gate this behind the nix_tests feature flag (which wasn't applied consistently). Also, at least right now all of these use StdIO internally, either by creating it on their own, or through the `eval_test` helper. Once we have some proper classification system for tests we can probably split this up further, but for now only run them if the impure feature is enabled. Change-Id: I3236cf09b3391585df99073367c4e4832c5e7898 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11723 Tested-by: BuildkiteCI Autosubmit: flokli <flokli@flokli.de> Reviewed-by: Connor Brewster <cbrewster@hey.com>
This commit is contained in:
Florian Klink
flokli
parent
543c103903
commit
2c628f6e1c
2 changed files with 207 additions and 198 deletions
|
|
@ -1,203 +1,6 @@
|
||||||
use crate::{value::Value, EvalIO};
|
|
||||||
use builtin_macros::builtins;
|
|
||||||
use pretty_assertions::assert_eq;
|
|
||||||
use rstest::rstest;
|
|
||||||
use std::path::PathBuf;
|
|
||||||
|
|
||||||
/// Module for one-off tests which do not follow the rest of the
|
/// Module for one-off tests which do not follow the rest of the
|
||||||
/// test layout.
|
/// test layout.
|
||||||
mod one_offs;
|
mod one_offs;
|
||||||
|
|
||||||
#[builtins]
|
|
||||||
mod mock_builtins {
|
|
||||||
//! Builtins which are required by language tests, but should not
|
|
||||||
//! actually exist in //tvix/eval.
|
|
||||||
use crate as tvix_eval;
|
|
||||||
use crate::generators::GenCo;
|
|
||||||
use crate::*;
|
|
||||||
use genawaiter::rc::Gen;
|
|
||||||
|
|
||||||
#[builtin("derivation")]
|
|
||||||
async fn builtin_derivation(co: GenCo, input: Value) -> Result<Value, ErrorKind> {
|
|
||||||
let input = input.to_attrs()?;
|
|
||||||
let attrs = input.update(NixAttrs::from_iter(
|
|
||||||
[
|
|
||||||
(
|
|
||||||
"outPath",
|
|
||||||
"/nix/store/00000000000000000000000000000000-mock",
|
|
||||||
),
|
|
||||||
(
|
|
||||||
"drvPath",
|
|
||||||
"/nix/store/00000000000000000000000000000000-mock.drv",
|
|
||||||
),
|
|
||||||
("type", "derivation"),
|
|
||||||
]
|
|
||||||
.into_iter(),
|
|
||||||
));
|
|
||||||
|
|
||||||
Ok(Value::Attrs(Box::new(attrs)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn eval_test(code_path: PathBuf, expect_success: bool) {
|
|
||||||
std::env::set_var("TEST_VAR", "foo"); // for eval-okay-getenv.nix
|
|
||||||
|
|
||||||
eprintln!("path: {}", code_path.display());
|
|
||||||
assert_eq!(
|
|
||||||
code_path.extension().unwrap(),
|
|
||||||
"nix",
|
|
||||||
"test files always end in .nix"
|
|
||||||
);
|
|
||||||
|
|
||||||
let code = std::fs::read_to_string(&code_path).expect("should be able to read test code");
|
|
||||||
|
|
||||||
let mut eval = crate::Evaluation::new_impure();
|
|
||||||
eval.strict = true;
|
|
||||||
eval.builtins.extend(mock_builtins::builtins());
|
|
||||||
|
|
||||||
let result = eval.evaluate(code, Some(code_path.clone()));
|
|
||||||
let failed = match result.value {
|
|
||||||
Some(Value::Catchable(_)) => true,
|
|
||||||
_ => !result.errors.is_empty(),
|
|
||||||
};
|
|
||||||
if expect_success && failed {
|
|
||||||
panic!(
|
|
||||||
"{}: evaluation of eval-okay test should succeed, but failed with {:?}",
|
|
||||||
code_path.display(),
|
|
||||||
result.errors,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if !expect_success && failed {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// !expect_success can also mean the output differs, so don't panic if the
|
|
||||||
// evaluation didn't fail.
|
|
||||||
|
|
||||||
let value = result.value.unwrap();
|
|
||||||
let result_str = value.to_string();
|
|
||||||
|
|
||||||
let exp_path = code_path.with_extension("exp");
|
|
||||||
if exp_path.exists() {
|
|
||||||
// If there's an .exp file provided alongside, compare it with the
|
|
||||||
// output of the NixValue .to_string() method.
|
|
||||||
let exp_str = std::fs::read_to_string(&exp_path).expect("unable to read .exp file");
|
|
||||||
|
|
||||||
if expect_success {
|
|
||||||
assert_eq!(
|
|
||||||
result_str,
|
|
||||||
exp_str.trim(),
|
|
||||||
"{}: result value representation (left) must match expectation (right)",
|
|
||||||
code_path.display()
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
assert_ne!(
|
|
||||||
result_str,
|
|
||||||
exp_str.trim(),
|
|
||||||
"{}: test passed unexpectedly! consider moving it out of notyetpassing",
|
|
||||||
code_path.display()
|
|
||||||
);
|
|
||||||
|
|
||||||
// Early return here, we don't compare .xml outputs if this is a !
|
|
||||||
// expect_success test.
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let exp_xml_path = code_path.with_extension("exp.xml");
|
|
||||||
if exp_xml_path.exists() {
|
|
||||||
// If there's an XML file provided alongside, compare it with the
|
|
||||||
// output produced when serializing the Value as XML.
|
|
||||||
let exp_xml_str = std::fs::read_to_string(exp_xml_path).expect("unable to read .xml file");
|
|
||||||
|
|
||||||
let mut xml_actual_buf = Vec::new();
|
|
||||||
crate::builtins::value_to_xml(&mut xml_actual_buf, &value).expect("value_to_xml failed");
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
String::from_utf8(xml_actual_buf).expect("to_xml produced invalid utf-8"),
|
|
||||||
exp_xml_str,
|
|
||||||
"{}: result value representation (left) must match expectation (right)",
|
|
||||||
code_path.display()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// identity-* tests contain Nix code snippets which should evaluate to
|
|
||||||
// themselves exactly (i.e. literals).
|
|
||||||
#[rstest]
|
|
||||||
fn identity(#[files("src/tests/tvix_tests/identity-*.nix")] code_path: PathBuf) {
|
|
||||||
let code = std::fs::read_to_string(code_path).expect("should be able to read test code");
|
|
||||||
|
|
||||||
let mut eval = crate::Evaluation::new(Box::new(crate::StdIO) as Box<dyn EvalIO>, false);
|
|
||||||
eval.strict = true;
|
|
||||||
|
|
||||||
let result = eval.evaluate(&code, None);
|
|
||||||
assert!(
|
|
||||||
result.errors.is_empty(),
|
|
||||||
"evaluation of identity test failed: {:?}",
|
|
||||||
result.errors
|
|
||||||
);
|
|
||||||
|
|
||||||
let result_str = result.value.unwrap().to_string();
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
result_str,
|
|
||||||
code.trim(),
|
|
||||||
"result value representation (left) must match expectation (right)"
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// eval-okay-* tests contain a snippet of Nix code, and an expectation
|
|
||||||
// of the produced string output of the evaluator.
|
|
||||||
//
|
|
||||||
// These evaluations are always supposed to succeed, i.e. all snippets
|
|
||||||
// are guaranteed to be valid Nix code.
|
|
||||||
#[rstest]
|
|
||||||
fn eval_okay(#[files("src/tests/tvix_tests/eval-okay-*.nix")] code_path: PathBuf) {
|
|
||||||
eval_test(code_path, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
// eval-okay-* tests from the original Nix test suite.
|
|
||||||
#[cfg(feature = "nix_tests")]
|
#[cfg(feature = "nix_tests")]
|
||||||
#[rstest]
|
mod nix_tests;
|
||||||
fn nix_eval_okay(#[files("src/tests/nix_tests/eval-okay-*.nix")] code_path: PathBuf) {
|
|
||||||
eval_test(code_path, true)
|
|
||||||
}
|
|
||||||
|
|
||||||
// eval-okay-* tests from the original Nix test suite which do not yet pass for tvix
|
|
||||||
//
|
|
||||||
// Eventually there will be none of these left, and this function
|
|
||||||
// will disappear :)
|
|
||||||
//
|
|
||||||
// Please don't submit failing tests unless they're in
|
|
||||||
// notyetpassing; this makes the test suite much more useful for
|
|
||||||
// regression testing, since there should always be zero non-ignored
|
|
||||||
// failing tests.
|
|
||||||
#[rstest]
|
|
||||||
fn nix_eval_okay_currently_failing(
|
|
||||||
#[files("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf,
|
|
||||||
) {
|
|
||||||
eval_test(code_path, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[rstest]
|
|
||||||
fn eval_okay_currently_failing(
|
|
||||||
#[files("src/tests/tvix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf,
|
|
||||||
) {
|
|
||||||
eval_test(code_path, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
// eval-fail-* tests contain a snippet of Nix code, which is
|
|
||||||
// expected to fail evaluation. The exact type of failure
|
|
||||||
// (assertion, parse error, etc) is not currently checked.
|
|
||||||
#[rstest]
|
|
||||||
fn eval_fail(#[files("src/tests/tvix_tests/eval-fail-*.nix")] code_path: PathBuf) {
|
|
||||||
eval_test(code_path, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
// eval-fail-* tests from the original Nix test suite.
|
|
||||||
#[cfg(feature = "nix_tests")]
|
|
||||||
#[rstest]
|
|
||||||
fn nix_eval_fail(#[files("src/tests/nix_tests/eval-fail-*.nix")] code_path: PathBuf) {
|
|
||||||
eval_test(code_path, false)
|
|
||||||
}
|
|
||||||
|
|
|
||||||
206
tvix/eval/src/tests/nix_tests.rs
Normal file
206
tvix/eval/src/tests/nix_tests.rs
Normal file
|
|
@ -0,0 +1,206 @@
|
||||||
|
use crate::value::Value;
|
||||||
|
use builtin_macros::builtins;
|
||||||
|
use pretty_assertions::assert_eq;
|
||||||
|
use rstest::rstest;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
#[builtins]
|
||||||
|
mod mock_builtins {
|
||||||
|
//! Builtins which are required by language tests, but should not
|
||||||
|
//! actually exist in //tvix/eval.
|
||||||
|
use crate as tvix_eval;
|
||||||
|
use crate::generators::GenCo;
|
||||||
|
use crate::*;
|
||||||
|
use genawaiter::rc::Gen;
|
||||||
|
|
||||||
|
#[builtin("derivation")]
|
||||||
|
async fn builtin_derivation(co: GenCo, input: Value) -> Result<Value, ErrorKind> {
|
||||||
|
let input = input.to_attrs()?;
|
||||||
|
let attrs = input.update(NixAttrs::from_iter(
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"outPath",
|
||||||
|
"/nix/store/00000000000000000000000000000000-mock",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"drvPath",
|
||||||
|
"/nix/store/00000000000000000000000000000000-mock.drv",
|
||||||
|
),
|
||||||
|
("type", "derivation"),
|
||||||
|
]
|
||||||
|
.into_iter(),
|
||||||
|
));
|
||||||
|
|
||||||
|
Ok(Value::Attrs(Box::new(attrs)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn eval_test(code_path: PathBuf, expect_success: bool) {
|
||||||
|
std::env::set_var("TEST_VAR", "foo"); // for eval-okay-getenv.nix
|
||||||
|
|
||||||
|
eprintln!("path: {}", code_path.display());
|
||||||
|
assert_eq!(
|
||||||
|
code_path.extension().unwrap(),
|
||||||
|
"nix",
|
||||||
|
"test files always end in .nix"
|
||||||
|
);
|
||||||
|
|
||||||
|
let code = std::fs::read_to_string(&code_path).expect("should be able to read test code");
|
||||||
|
|
||||||
|
let mut eval = crate::Evaluation::new_impure();
|
||||||
|
eval.strict = true;
|
||||||
|
eval.builtins.extend(mock_builtins::builtins());
|
||||||
|
|
||||||
|
let result = eval.evaluate(code, Some(code_path.clone()));
|
||||||
|
let failed = match result.value {
|
||||||
|
Some(Value::Catchable(_)) => true,
|
||||||
|
_ => !result.errors.is_empty(),
|
||||||
|
};
|
||||||
|
if expect_success && failed {
|
||||||
|
panic!(
|
||||||
|
"{}: evaluation of eval-okay test should succeed, but failed with {:?}",
|
||||||
|
code_path.display(),
|
||||||
|
result.errors,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if !expect_success && failed {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// !expect_success can also mean the output differs, so don't panic if the
|
||||||
|
// evaluation didn't fail.
|
||||||
|
|
||||||
|
let value = result.value.unwrap();
|
||||||
|
let result_str = value.to_string();
|
||||||
|
|
||||||
|
let exp_path = code_path.with_extension("exp");
|
||||||
|
if exp_path.exists() {
|
||||||
|
// If there's an .exp file provided alongside, compare it with the
|
||||||
|
// output of the NixValue .to_string() method.
|
||||||
|
let exp_str = std::fs::read_to_string(&exp_path).expect("unable to read .exp file");
|
||||||
|
|
||||||
|
if expect_success {
|
||||||
|
assert_eq!(
|
||||||
|
result_str,
|
||||||
|
exp_str.trim(),
|
||||||
|
"{}: result value representation (left) must match expectation (right)",
|
||||||
|
code_path.display()
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
assert_ne!(
|
||||||
|
result_str,
|
||||||
|
exp_str.trim(),
|
||||||
|
"{}: test passed unexpectedly! consider moving it out of notyetpassing",
|
||||||
|
code_path.display()
|
||||||
|
);
|
||||||
|
|
||||||
|
// Early return here, we don't compare .xml outputs if this is a !
|
||||||
|
// expect_success test.
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let exp_xml_path = code_path.with_extension("exp.xml");
|
||||||
|
if exp_xml_path.exists() {
|
||||||
|
// If there's an XML file provided alongside, compare it with the
|
||||||
|
// output produced when serializing the Value as XML.
|
||||||
|
let exp_xml_str = std::fs::read_to_string(exp_xml_path).expect("unable to read .xml file");
|
||||||
|
|
||||||
|
let mut xml_actual_buf = Vec::new();
|
||||||
|
crate::builtins::value_to_xml(&mut xml_actual_buf, &value).expect("value_to_xml failed");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
String::from_utf8(xml_actual_buf).expect("to_xml produced invalid utf-8"),
|
||||||
|
exp_xml_str,
|
||||||
|
"{}: result value representation (left) must match expectation (right)",
|
||||||
|
code_path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// identity-* tests contain Nix code snippets which should evaluate to
|
||||||
|
// themselves exactly (i.e. literals).
|
||||||
|
#[cfg(feature = "impure")]
|
||||||
|
#[rstest]
|
||||||
|
fn identity(#[files("src/tests/tvix_tests/identity-*.nix")] code_path: PathBuf) {
|
||||||
|
use crate::EvalIO;
|
||||||
|
|
||||||
|
let code = std::fs::read_to_string(code_path).expect("should be able to read test code");
|
||||||
|
|
||||||
|
let mut eval = crate::Evaluation::new(Box::new(crate::StdIO) as Box<dyn EvalIO>, false);
|
||||||
|
eval.strict = true;
|
||||||
|
|
||||||
|
let result = eval.evaluate(&code, None);
|
||||||
|
assert!(
|
||||||
|
result.errors.is_empty(),
|
||||||
|
"evaluation of identity test failed: {:?}",
|
||||||
|
result.errors
|
||||||
|
);
|
||||||
|
|
||||||
|
let result_str = result.value.unwrap().to_string();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
result_str,
|
||||||
|
code.trim(),
|
||||||
|
"result value representation (left) must match expectation (right)"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// eval-okay-* tests contain a snippet of Nix code, and an expectation
|
||||||
|
// of the produced string output of the evaluator.
|
||||||
|
//
|
||||||
|
// These evaluations are always supposed to succeed, i.e. all snippets
|
||||||
|
// are guaranteed to be valid Nix code.
|
||||||
|
#[cfg(feature = "impure")]
|
||||||
|
#[rstest]
|
||||||
|
fn eval_okay(#[files("src/tests/tvix_tests/eval-okay-*.nix")] code_path: PathBuf) {
|
||||||
|
eval_test(code_path, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// eval-okay-* tests from the original Nix test suite.
|
||||||
|
#[cfg(feature = "impure")]
|
||||||
|
#[rstest]
|
||||||
|
fn nix_eval_okay(#[files("src/tests/nix_tests/eval-okay-*.nix")] code_path: PathBuf) {
|
||||||
|
eval_test(code_path, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// eval-okay-* tests from the original Nix test suite which do not yet pass for tvix
|
||||||
|
//
|
||||||
|
// Eventually there will be none of these left, and this function
|
||||||
|
// will disappear :)
|
||||||
|
//
|
||||||
|
// Please don't submit failing tests unless they're in
|
||||||
|
// notyetpassing; this makes the test suite much more useful for
|
||||||
|
// regression testing, since there should always be zero non-ignored
|
||||||
|
// failing tests.
|
||||||
|
#[cfg(feature = "impure")]
|
||||||
|
#[rstest]
|
||||||
|
fn nix_eval_okay_currently_failing(
|
||||||
|
#[files("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf,
|
||||||
|
) {
|
||||||
|
eval_test(code_path, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "impure")]
|
||||||
|
#[rstest]
|
||||||
|
fn eval_okay_currently_failing(
|
||||||
|
#[files("src/tests/tvix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf,
|
||||||
|
) {
|
||||||
|
eval_test(code_path, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// eval-fail-* tests contain a snippet of Nix code, which is
|
||||||
|
// expected to fail evaluation. The exact type of failure
|
||||||
|
// (assertion, parse error, etc) is not currently checked.
|
||||||
|
#[cfg(feature = "impure")]
|
||||||
|
#[rstest]
|
||||||
|
fn eval_fail(#[files("src/tests/tvix_tests/eval-fail-*.nix")] code_path: PathBuf) {
|
||||||
|
eval_test(code_path, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// eval-fail-* tests from the original Nix test suite.
|
||||||
|
#[cfg(feature = "impure")]
|
||||||
|
#[rstest]
|
||||||
|
fn nix_eval_fail(#[files("src/tests/nix_tests/eval-fail-*.nix")] code_path: PathBuf) {
|
||||||
|
eval_test(code_path, false)
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue