mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test(3p/nix): Enable output comparison for evaluator success tests

Browse source 
Enables loading of the expected output of evaluator tests from the
corresponding .exp files, and checks that the output matches.

This again leaves some tests behind in the disabled folder, but we now
have almost the entire suite up and running so I can get around to
cleaning up the disabled ones.

Other note: Some tests had XML output, despite not being related to
XML testing at all - I'm not sure why they chose to do this, but have
converted those test outputs to normal Nix instead.

We have a separate test suite for JSON & XML serialisation already,
which was contributed by andi-.

Change-Id: Id7c42c836edfec4c22db9d893e35489f3e6dd559
Reviewed-on: https://cl.tvl.fyi/c/depot/+/1285
Tested-by: BuildkiteCI
Reviewed-by: isomer <isomer@tvl.fyi>
Reviewed-by: glittershark <grfn@gws.fyi>
This commit is contained in:
Vincent Ambo 2020-07-19 15:57:43 +01:00 committed by tazjin
parent 97e6499225
commit 61a7b97c39
16 changed files with 37 additions and 364 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
third_party/nix/default.nix vendored
View file

@ -89,6 +89,7 @@ in pkgs.llvmPackages.libcxxStdenv.mkDerivation {
installCheckPhase = ''
export NIX_DATA_DIR=$out/share
export NIX_TEST_VAR=foo # this is required by a language test
make test
'';
@ -124,6 +125,7 @@ in pkgs.llvmPackages.libcxxStdenv.mkDerivation {
shellHook = ''
export NIX_DATA_DIR="${toString depotPath}/third_party"
export NIX_TEST_VAR=foo
'';
# TODO(tazjin): integration test setup?

2
third_party/nix/src/tests/lang/disabled/README.txt vendored
View file

@ -1,4 +1,4 @@
These tests are disabeld primarily because the DummyStore used for
These tests are disabled primarily because the DummyStore used for
tests does not interact with real files on disk at the moment, but the
tests expect it to.

0
third_party/nix/src/tests/lang/eval-okay-autoargs.exp → third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.exp vendored
View file

0
third_party/nix/src/tests/lang/eval-okay-autoargs.nix → third_party/nix/src/tests/lang/disabled/eval-okay-autoargs.nix vendored
View file

0
third_party/nix/src/tests/lang/eval-okay-sort.nix → third_party/nix/src/tests/lang/disabled/eval-okay-sort.nix vendored
View file

0
third_party/nix/src/tests/lang/eval-okay-toxml.nix → third_party/nix/src/tests/lang/disabled/eval-okay-toxml.nix vendored
View file

0
third_party/nix/src/tests/lang/eval-okay-xml.nix → third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix vendored
View file

1
third_party/nix/src/tests/lang/eval-okay-closure.exp vendored Normal file
View file

@ -0,0 +1 @@
[ { foo = true; key = -13; } { foo = true; key = -12; } { foo = true; key = -11; } { foo = true; key = -9; } { foo = true; key = -8; } { foo = true; key = -7; } { foo = true; key = -5; } { foo = true; key = -4; } { foo = true; key = -3; } { key = -1; } { foo = true; key = 0; } { foo = true; key = 1; } { foo = true; key = 2; } { foo = true; key = 4; } { foo = true; key = 5; } { foo = true; key = 6; } { key = 8; } { foo = true; key = 9; } { foo = true; key = 10; } { foo = true; key = 13; } { foo = true; key = 14; } { foo = true; key = 15; } { key = 17; } { foo = true; key = 18; } { foo = true; key = 19; } { foo = true; key = 22; } { foo = true; key = 23; } { key = 26; } { foo = true; key = 27; } { foo = true; key = 28; } { foo = true; key = 31; } { foo = true; key = 32; } { key = 35; } { foo = true; key = 36; } { foo = true; key = 40; } { foo = true; key = 41; } { key = 44; } { foo = true; key = 45; } { foo = true; key = 49; } { key = 53; } { foo = true; key = 54; } { foo = true; key = 58; } { key = 62; } { foo = true; key = 67; } { key = 71; } { key = 80; } ]

343
third_party/nix/src/tests/lang/eval-okay-closure.exp.xml vendored
View file

@ -1,343 +0,0 @@
<?xml version='1.0' encoding='utf-8'?>
<expr>
<list>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="-13" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="-12" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="-11" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="-9" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="-8" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="-7" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="-5" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="-4" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="-3" />
</attr>
</attrs>
<attrs>
<attr name="key">
<int value="-1" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="0" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="1" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="2" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="4" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="5" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="6" />
</attr>
</attrs>
<attrs>
<attr name="key">
<int value="8" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="9" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="10" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="13" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="14" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="15" />
</attr>
</attrs>
<attrs>
<attr name="key">
<int value="17" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="18" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="19" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="22" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="23" />
</attr>
</attrs>
<attrs>
<attr name="key">
<int value="26" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="27" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="28" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="31" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="32" />
</attr>
</attrs>
<attrs>
<attr name="key">
<int value="35" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="36" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="40" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="41" />
</attr>
</attrs>
<attrs>
<attr name="key">
<int value="44" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="45" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="49" />
</attr>
</attrs>
<attrs>
<attr name="key">
<int value="53" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="54" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="58" />
</attr>
</attrs>
<attrs>
<attr name="key">
<int value="62" />
</attr>
</attrs>
<attrs>
<attr name="foo">
<bool value="true" />
</attr>
<attr name="key">
<int value="67" />
</attr>
</attrs>
<attrs>
<attr name="key">
<int value="71" />
</attr>
</attrs>
<attrs>
<attr name="key">
<int value="80" />
</attr>
</attrs>
</list>
</expr>

1
third_party/nix/src/tests/lang/eval-okay-eq.exp vendored Normal file
View file

@ -0,0 +1 @@
true

1
third_party/nix/src/tests/lang/eval-okay-eq.exp.disabled vendored
View file

@ -1 +0,0 @@
Bool(True)

1
third_party/nix/src/tests/lang/eval-okay-functionargs.exp vendored Normal file
View file

@ -0,0 +1 @@
[ "stdenv" "fetchurl" "aterm-stdenv" "aterm-stdenv2" "libX11" "libXv" "mplayer-stdenv2.libXv-libX11" "mplayer-stdenv2.libXv-libX11_2" "nix-stdenv-aterm-stdenv" "nix-stdenv2-aterm2-stdenv2" ]

15
third_party/nix/src/tests/lang/eval-okay-functionargs.exp.xml vendored
View file

@ -1,15 +0,0 @@
<?xml version='1.0' encoding='utf-8'?>
<expr>
<list>
<string value="stdenv" />
<string value="fetchurl" />
<string value="aterm-stdenv" />
<string value="aterm-stdenv2" />
<string value="libX11" />
<string value="libXv" />
<string value="mplayer-stdenv2.libXv-libX11" />
<string value="mplayer-stdenv2.libXv-libX11_2" />
<string value="nix-stdenv-aterm-stdenv" />
<string value="nix-stdenv2-aterm2-stdenv2" />
</list>
</expr>

2
third_party/nix/src/tests/lang/eval-okay-getenv.nix vendored
View file

@ -1 +1 @@
builtins.getEnv "TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla")
builtins.getEnv "NIX_TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla")

0
third_party/nix/src/tests/lang/eval-okay-xml.exp.xml → third_party/nix/src/tests/lang/eval-okay-xml.exp vendored
View file

33
third_party/nix/src/tests/language-tests.cc vendored
View file

@ -23,9 +23,12 @@
#include <algorithm>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <iterator>
#include <memory>
#include <optional>
#include <sstream>
#include <string>
#include <absl/strings/ascii.h>
@ -86,6 +89,20 @@ std::string TestNameFor(
return name;
}
// Load the expected output of a given test as a string.
std::string ExpectedOutputFor(absl::string_view stem) {
std::filesystem::path path(
absl::StrCat(NIX_SRC_DIR, "/src/tests/lang/", stem, ".exp"));
EXPECT_TRUE(std::filesystem::exists(path))
<< stem << ": expected output file should exist";
std::ifstream input(path);
std::stringstream buffer;
buffer << input.rdbuf();
return std::string(absl::StripTrailingAsciiWhitespace(buffer.str()));
}
} // namespace
using nix::tests::DummyStore;
@ -189,7 +206,7 @@ INSTANTIATE_TEST_SUITE_P(Eval, EvalFailureTest,
class EvalSuccessTest : public testing::TestWithParam<std::filesystem::path> {};
// Test pattern for files that should fail to evaluate.
// Test pattern for files that should evaluate successfully.
TEST_P(EvalSuccessTest, Fails) {
std::shared_ptr<Store> store = std::make_shared<DummyStore>();
EvalState state({}, ref<Store>(store));
@ -200,8 +217,18 @@ TEST_P(EvalSuccessTest, Fails) {
<< path.stem().string() << ": should parse successfully";
Value result;
state.eval(expr, result);
state.forceValue(result);
EXPECT_NO_THROW({
state.eval(expr, result);
state.forceValueDeep(result);
}) << path.stem().string()
<< ": should evaluate successfully";
auto expected = ExpectedOutputFor(path.stem().string());
std::ostringstream value_str;
value_str << result;
EXPECT_EQ(expected, value_str.str()) << "evaluator output should match";
}
INSTANTIATE_TEST_SUITE_P(Eval, EvalSuccessTest,