From 61a7b97c3927f28ef588196bc2efbd8c6cceb8c4 Mon Sep 17 00:00:00 2001 From: Vincent Ambo Date: Sun, 19 Jul 2020 15:57:43 +0100 Subject: [PATCH] test(3p/nix): Enable output comparison for evaluator success tests Enables loading of the expected output of evaluator tests from the corresponding .exp files, and checks that the output matches. This again leaves some tests behind in the disabled folder, but we now have almost the entire suite up and running so I can get around to cleaning up the disabled ones. Other note: Some tests had XML output, despite not being related to XML testing at all - I'm not sure why they chose to do this, but have converted those test outputs to normal Nix instead. We have a separate test suite for JSON & XML serialisation already, which was contributed by andi-. Change-Id: Id7c42c836edfec4c22db9d893e35489f3e6dd559 Reviewed-on: https://cl.tvl.fyi/c/depot/+/1285 Tested-by: BuildkiteCI Reviewed-by: isomer Reviewed-by: glittershark --- third_party/nix/default.nix | 2 + .../nix/src/tests/lang/disabled/README.txt | 2 +- .../{ => disabled}/eval-okay-autoargs.exp | 0 .../{ => disabled}/eval-okay-autoargs.nix | 0 .../lang/{ => disabled}/eval-okay-sort.nix | 0 .../lang/{ => disabled}/eval-okay-toxml.nix | 0 .../lang/{ => disabled}/eval-okay-xml.nix | 0 .../nix/src/tests/lang/eval-okay-closure.exp | 1 + .../src/tests/lang/eval-okay-closure.exp.xml | 343 ------------------ .../nix/src/tests/lang/eval-okay-eq.exp | 1 + .../src/tests/lang/eval-okay-eq.exp.disabled | 1 - .../src/tests/lang/eval-okay-functionargs.exp | 1 + .../tests/lang/eval-okay-functionargs.exp.xml | 15 - .../nix/src/tests/lang/eval-okay-getenv.nix | 2 +- ...val-okay-xml.exp.xml => eval-okay-xml.exp} | 0 third_party/nix/src/tests/language-tests.cc | 33 +- 16 files changed, 37 insertions(+), 364 deletions(-) rename third_party/nix/src/tests/lang/{ => disabled}/eval-okay-autoargs.exp (100%) rename third_party/nix/src/tests/lang/{ => disabled}/eval-okay-autoargs.nix (100%) rename third_party/nix/src/tests/lang/{ => disabled}/eval-okay-sort.nix (100%) rename third_party/nix/src/tests/lang/{ => disabled}/eval-okay-toxml.nix (100%) rename third_party/nix/src/tests/lang/{ => disabled}/eval-okay-xml.nix (100%) create mode 100644 third_party/nix/src/tests/lang/eval-okay-closure.exp delete mode 100644 third_party/nix/src/tests/lang/eval-okay-closure.exp.xml create mode 100644 third_party/nix/src/tests/lang/eval-okay-eq.exp delete mode 100644 third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled create mode 100644 third_party/nix/src/tests/lang/eval-okay-functionargs.exp delete mode 100644 third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml rename third_party/nix/src/tests/lang/{eval-okay-xml.exp.xml => eval-okay-xml.exp} (100%) diff --git a/third_party/nix/default.nix b/third_party/nix/default.nix index 54e77ec50..cceb6b4f1 100644 --- a/third_party/nix/default.nix +++ b/third_party/nix/default.nix @@ -89,6 +89,7 @@ in pkgs.llvmPackages.libcxxStdenv.mkDerivation { installCheckPhase = '' export NIX_DATA_DIR=$out/share + export NIX_TEST_VAR=foo # this is required by a language test make test ''; @@ -124,6 +125,7 @@ in pkgs.llvmPackages.libcxxStdenv.mkDerivation { shellHook = '' export NIX_DATA_DIR="${toString depotPath}/third_party" + export NIX_TEST_VAR=foo ''; # TODO(tazjin): integration test setup? diff --git a/third_party/nix/src/tests/lang/disabled/README.txt b/third_party/nix/src/tests/lang/disabled/README.txt index 3dbcbf4f0..50225deb6 100644 --- a/third_party/nix/src/tests/lang/disabled/README.txt +++ b/third_party/nix/src/tests/lang/disabled/README.txt @@ -1,4 +1,4 @@ -These tests are disabeld primarily because the DummyStore used for +These tests are disabled primarily because the DummyStore used for tests does not interact with real files on disk at the moment, but the tests expect it to. diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.exp b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp similarity index 100% rename from third_party/nix/src/tests/lang/eval-okay-autoargs.exp rename to third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix similarity index 100% rename from third_party/nix/src/tests/lang/eval-okay-autoargs.nix rename to third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-sort.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix similarity index 100% rename from third_party/nix/src/tests/lang/eval-okay-sort.nix rename to third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-toxml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix similarity index 100% rename from third_party/nix/src/tests/lang/eval-okay-toxml.nix rename to third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix similarity index 100% rename from third_party/nix/src/tests/lang/eval-okay-xml.nix rename to third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp b/third_party/nix/src/tests/lang/eval-okay-closure.exp new file mode 100644 index 000000000..e7dbf9781 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-closure.exp @@ -0,0 +1 @@ +[ { foo = true; key = -13; } { foo = true; key = -12; } { foo = true; key = -11; } { foo = true; key = -9; } { foo = true; key = -8; } { foo = true; key = -7; } { foo = true; key = -5; } { foo = true; key = -4; } { foo = true; key = -3; } { key = -1; } { foo = true; key = 0; } { foo = true; key = 1; } { foo = true; key = 2; } { foo = true; key = 4; } { foo = true; key = 5; } { foo = true; key = 6; } { key = 8; } { foo = true; key = 9; } { foo = true; key = 10; } { foo = true; key = 13; } { foo = true; key = 14; } { foo = true; key = 15; } { key = 17; } { foo = true; key = 18; } { foo = true; key = 19; } { foo = true; key = 22; } { foo = true; key = 23; } { key = 26; } { foo = true; key = 27; } { foo = true; key = 28; } { foo = true; key = 31; } { foo = true; key = 32; } { key = 35; } { foo = true; key = 36; } { foo = true; key = 40; } { foo = true; key = 41; } { key = 44; } { foo = true; key = 45; } { foo = true; key = 49; } { key = 53; } { foo = true; key = 54; } { foo = true; key = 58; } { key = 62; } { foo = true; key = 67; } { key = 71; } { key = 80; } ] diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml b/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml deleted file mode 100644 index dffc03a99..000000000 --- a/third_party/nix/src/tests/lang/eval-okay-closure.exp.xml +++ /dev/null @@ -1,343 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp b/third_party/nix/src/tests/lang/eval-okay-eq.exp new file mode 100644 index 000000000..27ba77dda --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-eq.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled b/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled deleted file mode 100644 index 2015847b6..000000000 --- a/third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled +++ /dev/null @@ -1 +0,0 @@ -Bool(True) diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp new file mode 100644 index 000000000..c1c9f8ffa --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp @@ -0,0 +1 @@ +[ "stdenv" "fetchurl" "aterm-stdenv" "aterm-stdenv2" "libX11" "libXv" "mplayer-stdenv2.libXv-libX11" "mplayer-stdenv2.libXv-libX11_2" "nix-stdenv-aterm-stdenv" "nix-stdenv2-aterm2-stdenv2" ] diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml deleted file mode 100644 index 651f54c36..000000000 --- a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/third_party/nix/src/tests/lang/eval-okay-getenv.nix b/third_party/nix/src/tests/lang/eval-okay-getenv.nix index 4cfec5f55..ea8bb9f0a 100644 --- a/third_party/nix/src/tests/lang/eval-okay-getenv.nix +++ b/third_party/nix/src/tests/lang/eval-okay-getenv.nix @@ -1 +1 @@ -builtins.getEnv "TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla") +builtins.getEnv "NIX_TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla") diff --git a/third_party/nix/src/tests/lang/eval-okay-xml.exp.xml b/third_party/nix/src/tests/lang/eval-okay-xml.exp similarity index 100% rename from third_party/nix/src/tests/lang/eval-okay-xml.exp.xml rename to third_party/nix/src/tests/lang/eval-okay-xml.exp diff --git a/third_party/nix/src/tests/language-tests.cc b/third_party/nix/src/tests/language-tests.cc index daa2fa233..cac5aee58 100644 --- a/third_party/nix/src/tests/language-tests.cc +++ b/third_party/nix/src/tests/language-tests.cc @@ -23,9 +23,12 @@ #include #include +#include #include +#include #include #include +#include #include #include @@ -86,6 +89,20 @@ std::string TestNameFor( return name; } +// Load the expected output of a given test as a string. +std::string ExpectedOutputFor(absl::string_view stem) { + std::filesystem::path path( + absl::StrCat(NIX_SRC_DIR, "/src/tests/lang/", stem, ".exp")); + + EXPECT_TRUE(std::filesystem::exists(path)) + << stem << ": expected output file should exist"; + + std::ifstream input(path); + std::stringstream buffer; + buffer << input.rdbuf(); + return std::string(absl::StripTrailingAsciiWhitespace(buffer.str())); +} + } // namespace using nix::tests::DummyStore; @@ -189,7 +206,7 @@ INSTANTIATE_TEST_SUITE_P(Eval, EvalFailureTest, class EvalSuccessTest : public testing::TestWithParam {}; -// Test pattern for files that should fail to evaluate. +// Test pattern for files that should evaluate successfully. TEST_P(EvalSuccessTest, Fails) { std::shared_ptr store = std::make_shared(); EvalState state({}, ref(store)); @@ -200,8 +217,18 @@ TEST_P(EvalSuccessTest, Fails) { << path.stem().string() << ": should parse successfully"; Value result; - state.eval(expr, result); - state.forceValue(result); + + EXPECT_NO_THROW({ + state.eval(expr, result); + state.forceValueDeep(result); + }) << path.stem().string() + << ": should evaluate successfully"; + + auto expected = ExpectedOutputFor(path.stem().string()); + std::ostringstream value_str; + value_str << result; + + EXPECT_EQ(expected, value_str.str()) << "evaluator output should match"; } INSTANTIATE_TEST_SUITE_P(Eval, EvalSuccessTest,