mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/eval): move away from test_generator to rstest

Browse source 
`test-generator` has not been updated in the past 2 years.
`rstest` has not been updated in the past 5 months.

This is an improvement in the maintenance state… I guess?
We get also new features, it changes the name of the tests with numbers too.

Change-Id: I5376104c7704f525dba7524da78daa09867cc669
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10623
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
This commit is contained in:
Ryan Lahfa 2024-01-14 16:31:04 +01:00 committed by raitobezarius
parent 850a4bfc7b
commit bc8fb825c7
4 changed files with 209 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
tvix/Cargo.lock generated
View file

@ -954,6 +954,12 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
[[package]]
name = "futures-timer"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
[[package]]
name = "futures-util"
version = "0.3.30"
@ -2216,6 +2222,12 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c"
[[package]]
name = "relative-path"
version = "1.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc"
[[package]]
name = "reqwest"
version = "0.11.22"
@ -2295,6 +2307,35 @@ dependencies = [
"text-size",
]
[[package]]
name = "rstest"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199"
dependencies = [
"futures",
"futures-timer",
"rstest_macros",
"rustc_version",
]
[[package]]
name = "rstest_macros"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605"
dependencies = [
"cfg-if",
"glob",
"proc-macro2 1.0.75",
"quote 1.0.35",
"regex",
"relative-path",
"rustc_version",
"syn 2.0.48",
"unicode-ident",
]
[[package]]
name = "rustc-demangle"
version = "0.1.23"
@ -3382,12 +3423,12 @@ dependencies = [
"regex",
"rnix",
"rowan",
"rstest",
"serde",
"serde_json",
"smol_str",
"tabwriter",
"tempfile",
"test-generator",
"test-strategy",
"toml",
"tvix-eval-builtin-macros",

127
tvix/Cargo.nix
View file

@ -2791,6 +2791,20 @@ rec {
};
resolvedDefaultFeatures = [ "alloc" "std" ];
};
"futures-timer" = rec {
crateName = "futures-timer";
version = "3.0.2";
edition = "2018";
sha256 = "0b5v7lk9838ix6jdcrainsyrh7xrf24pwm61dp13907qkn806jz6";
authors = [
"Alex Crichton <alex@alexcrichton.com>"
];
features = {
"gloo-timers" = [ "dep:gloo-timers" ];
"send_wrapper" = [ "dep:send_wrapper" ];
"wasm-bindgen" = [ "gloo-timers" "send_wrapper" ];
};
};
"futures-util" = rec {
crateName = "futures-util";
version = "0.3.30";
@ -6586,6 +6600,19 @@ rec {
};
resolvedDefaultFeatures = [ "default" "std" "unicode" "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ];
};
"relative-path" = rec {
crateName = "relative-path";
version = "1.9.2";
edition = "2021";
sha256 = "1g0gc604zwm73gvpcicn8si25j9j5agqz50r0x1bkmgx6f7mi678";
authors = [
"John-John Tedro <udoprog@tedro.se>"
];
features = {
"serde" = [ "dep:serde" ];
};
resolvedDefaultFeatures = [ "default" ];
};
"reqwest" = rec {
crateName = "reqwest";
version = "0.11.22";
@ -6987,6 +7014,98 @@ rec {
"serde1" = [ "serde" "text-size/serde" ];
};
};
"rstest" = rec {
crateName = "rstest";
version = "0.18.2";
edition = "2021";
sha256 = "1681ncnlzhc8894idm3pqf40nndn4k4kcp0kpv29n68a7hpspvlp";
authors = [
"Michele d'Amico <michele.damico@gmail.com>"
];
dependencies = [
{
name = "futures";
packageId = "futures";
optional = true;
}
{
name = "futures-timer";
packageId = "futures-timer";
optional = true;
}
{
name = "rstest_macros";
packageId = "rstest_macros";
usesDefaultFeatures = false;
}
];
buildDependencies = [
{
name = "rustc_version";
packageId = "rustc_version";
}
];
features = {
"async-timeout" = [ "dep:futures" "dep:futures-timer" "rstest_macros/async-timeout" ];
"default" = [ "async-timeout" ];
};
resolvedDefaultFeatures = [ "async-timeout" "default" ];
};
"rstest_macros" = rec {
crateName = "rstest_macros";
version = "0.18.2";
edition = "2021";
sha256 = "01g6rg60snmscipc9xiili7nsn0v25sv64713gp99y2jg0jgha6l";
procMacro = true;
authors = [
"Michele d'Amico <michele.damico@gmail.com>"
];
dependencies = [
{
name = "cfg-if";
packageId = "cfg-if";
}
{
name = "glob";
packageId = "glob";
}
{
name = "proc-macro2";
packageId = "proc-macro2 1.0.75";
}
{
name = "quote";
packageId = "quote 1.0.35";
}
{
name = "regex";
packageId = "regex";
}
{
name = "relative-path";
packageId = "relative-path";
}
{
name = "syn";
packageId = "syn 2.0.48";
features = [ "full" "parsing" "extra-traits" "visit" "visit-mut" ];
}
{
name = "unicode-ident";
packageId = "unicode-ident";
}
];
buildDependencies = [
{
name = "rustc_version";
packageId = "rustc_version";
}
];
features = {
"default" = [ "async-timeout" ];
};
resolvedDefaultFeatures = [ "async-timeout" ];
};
"rustc-demangle" = rec {
crateName = "rustc-demangle";
version = "0.1.23";
@ -10689,12 +10808,12 @@ rec {
packageId = "pretty_assertions";
}
{
name = "tempfile";
packageId = "tempfile";
name = "rstest";
packageId = "rstest";
}
{
name = "test-generator";
packageId = "test-generator";
name = "tempfile";
packageId = "tempfile";
}
];
features = {

7
tvix/eval/Cargo.toml
View file

@ -35,14 +35,9 @@ xml-rs = "0.8.4"
criterion = "0.5"
itertools = "0.12.0"
pretty_assertions = "1.2.1"
rstest = "0.18.2"
tempfile = "3.3.0"
[dev-dependencies.test-generator]
# This fork of test-generator adds support for cargo workspaces, see
# also https://github.com/frehberg/test-generator/pull/14
git = "https://github.com/JamesGuthrie/test-generator.git"
rev = "82e799979980962aec1aa324ec6e0e4cad781f41"
[features]
default = ["impure", "arbitrary", "nix_tests"]

71
tvix/eval/src/tests/mod.rs
View file

@ -1,7 +1,8 @@
use crate::value::Value;
use builtin_macros::builtins;
use pretty_assertions::assert_eq;
use test_generator::test_resources;
use rstest::rstest;
use std::path::PathBuf;
/// Module for one-off tests which do not follow the rest of the
/// test layout.
@ -38,14 +39,17 @@ mod mock_builtins {
}
}
fn eval_test(code_path: &str, expect_success: bool) {
let base = code_path
.strip_suffix("nix")
.expect("test files always end in .nix");
let exp_path = format!("{}exp", base);
let exp_xml_path = std::path::PathBuf::from(format!("{}exp.xml", base));
fn eval_test(code_path: PathBuf, expect_success: bool) {
eprintln!("path: {}", code_path.display());
assert_eq!(
code_path.extension().unwrap(),
"nix",
"test files always end in .nix"
);
let exp_path = code_path.with_extension("exp");
let exp_xml_path = code_path.with_extension("exp.xml");
let code = std::fs::read_to_string(code_path).expect("should be able to read test code");
let code = std::fs::read_to_string(&code_path).expect("should be able to read test code");
if exp_xml_path.exists() {
// We can't test them at the moment because we don't have XML output yet.
@ -57,14 +61,15 @@ fn eval_test(code_path: &str, expect_success: bool) {
eval.strict = true;
eval.builtins.extend(mock_builtins::builtins());
let result = eval.evaluate(code, Some(code_path.into()));
let result = eval.evaluate(code, Some(code_path.clone()));
let failed = match result.value {
Some(Value::Catchable(_)) => true,
_ => !result.errors.is_empty(),
};
if expect_success && failed {
panic!(
"{code_path}: evaluation of eval-okay test should succeed, but failed with {:?}",
"{}: evaluation of eval-okay test should succeed, but failed with {:?}",
code_path.display(),
result.errors,
);
}
@ -81,20 +86,26 @@ fn eval_test(code_path: &str, expect_success: bool) {
assert_eq!(
result_str,
exp.trim(),
"{code_path}: result value representation (left) must match expectation (right)"
"{}: result value representation (left) must match expectation (right)",
code_path.display()
);
} else {
assert_ne!(
result_str,
exp.trim(),
"{code_path}: test passed unexpectedly! consider moving it out of notyetpassing"
"{}: test passed unexpectedly! consider moving it out of notyetpassing",
code_path.display()
);
}
} else if expect_success {
panic!("{code_path}: should be able to read test expectation");
panic!(
"{}: should be able to read test expectation",
code_path.display()
);
} else {
panic!(
"{code_path}: test should have failed, but succeeded with output {}",
"{}: test should have failed, but succeeded with output {}",
code_path.display(),
result_str
);
}
@ -102,8 +113,8 @@ fn eval_test(code_path: &str, expect_success: bool) {
// identity-* tests contain Nix code snippets which should evaluate to
// themselves exactly (i.e. literals).
#[test_resources("src/tests/tvix_tests/identity-*.nix")]
fn identity(code_path: &str) {
#[rstest]
fn identity(#[files("src/tests/tvix_tests/identity-*.nix")] code_path: PathBuf) {
let code = std::fs::read_to_string(code_path).expect("should be able to read test code");
let eval = crate::Evaluation {
@ -133,15 +144,15 @@ fn identity(code_path: &str) {
//
// These evaluations are always supposed to succeed, i.e. all snippets
// are guaranteed to be valid Nix code.
#[test_resources("src/tests/tvix_tests/eval-okay-*.nix")]
fn eval_okay(code_path: &str) {
#[rstest]
fn eval_okay(#[files("src/tests/tvix_tests/eval-okay-*.nix")] code_path: PathBuf) {
eval_test(code_path, true)
}
// eval-okay-* tests from the original Nix test suite.
#[cfg(feature = "nix_tests")]
#[test_resources("src/tests/nix_tests/eval-okay-*.nix")]
fn nix_eval_okay(code_path: &str) {
#[rstest]
fn nix_eval_okay(#[files("src/tests/nix_tests/eval-okay-*.nix")] code_path: PathBuf) {
eval_test(code_path, true)
}
@ -163,27 +174,31 @@ fn nix_eval_okay(code_path: &str) {
//
// https://github.com/frehberg/test-generator/pull/10
// https://github.com/frehberg/test-generator/pull/8
#[test_resources("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")]
fn nix_eval_okay_currently_failing(code_path: &str) {
#[rstest]
fn nix_eval_okay_currently_failing(
#[files("src/tests/nix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf,
) {
eval_test(code_path, false)
}
#[test_resources("src/tests/tvix_tests/notyetpassing/eval-okay-*.nix")]
fn eval_okay_currently_failing(code_path: &str) {
#[rstest]
fn eval_okay_currently_failing(
#[files("src/tests/tvix_tests/notyetpassing/eval-okay-*.nix")] code_path: PathBuf,
) {
eval_test(code_path, false)
}
// eval-fail-* tests contain a snippet of Nix code, which is
// expected to fail evaluation. The exact type of failure
// (assertion, parse error, etc) is not currently checked.
#[test_resources("src/tests/tvix_tests/eval-fail-*.nix")]
fn eval_fail(code_path: &str) {
#[rstest]
fn eval_fail(#[files("src/tests/tvix_tests/eval-fail-*.nix")] code_path: PathBuf) {
eval_test(code_path, false)
}
// eval-fail-* tests from the original Nix test suite.
#[cfg(feature = "nix_tests")]
#[test_resources("src/tests/nix_tests/eval-fail-*.nix")]
fn nix_eval_fail(code_path: &str) {
#[rstest]
fn nix_eval_fail(#[files("src/tests/nix_tests/eval-fail-*.nix")] code_path: PathBuf) {
eval_test(code_path, false)
}