mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/cli/default.nix): make eval tests and benches derivations

Browse source 
This makes the eval tests and benchmarks standalone Nix derivations and
readTree targets:

```
nix-build -A tvix.cli.eval-nixpkgs-cross-hello-outpath
nix-build -A tvix.cli.benchmark-nixpkgs-attrnames
```

Even without doing any fetches, We need to set `SSL_CERT_FILE`, so
reqwest is able to load its CA roots.

Change-Id: Ib45282d01044165c7816391adbeeb26334f8e924
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11493
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Tested-by: BuildkiteCI
Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink 2024-04-20 20:56:40 +03:00 committed by clbot
parent f3ec974895
commit 4124f0e679
1 changed files with 45 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

94
tvix/cli/default.nix
View file

@ -1,23 +1,11 @@
{ depot, pkgs, lib, ... }: { depot, pkgs, lib, ... }:
(depot.tvix.crates.workspaceMembers.tvix-cli.build.override {
runTests = true;
}).overrideAttrs (finalAttrs: previousAttrs:
let let
mkNixpkgsEvalCheck = attrset: expectedPath: { tvix-cli = finalAttrs.finalPackage;
label = ":nix: evaluate nixpkgs.${attrset} in tvix";
needsOutput = true;
command = pkgs.writeShellScript "tvix-eval-${builtins.replaceStrings [".drv"] ["-drv"] attrset}" ''
TVIX_OUTPUT=$(result/bin/tvix -E '(import ${pkgs.path} {}).${attrset}')
EXPECTED='${/* the verbatim expected Tvix output: */ "=> \"${builtins.unsafeDiscardStringContext expectedPath}\" :: string"}'
echo "Tvix output: ''${TVIX_OUTPUT}"
if [ "$TVIX_OUTPUT" != "$EXPECTED" ]; then
echo "Correct would have been ''${EXPECTED}"
exit 1
fi
echo "Output was correct."
'';
};
benchmark-gnutime-format-string = benchmark-gnutime-format-string =
description: description:
@ -30,24 +18,16 @@ let
}; };
}); });
in
(depot.tvix.crates.workspaceMembers.tvix-cli.build.override {
runTests = true;
}).overrideAttrs (finalAttrs: previousAttrs:
let
tvix-cli = finalAttrs.finalPackage;
# You can run the benchmark with a simple `nix run`, like: # You can run the benchmark with a simple `nix run`, like:
# #
# nix run -f . tvix.cli.meta.ci.extraSteps.benchmark-nixpkgs-cross-hello-outpath # nix-build -A tvix.cli.meta.ci.extraSteps.benchmark-nixpkgs-cross-hello-outpath
# #
# TODO(amjoseph): store these results someplace more durable, like git trailers # TODO(amjoseph): store these results someplace more durable, like git trailers
# #
mkExprBenchmark = { expr, description }: mkExprBenchmark = { expr, description }:
let name = "tvix-cli-benchmark-${description}"; in let name = "tvix-cli-benchmark-${description}"; in
(pkgs.writeShellScriptBin name '' (pkgs.runCommand name { } ''
export SSL_CERT_FILE=${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt
${lib.escapeShellArgs [ ${lib.escapeShellArgs [
"${pkgs.time}/bin/time" "${pkgs.time}/bin/time"
"--format" "${benchmark-gnutime-format-string description}" "--format" "${benchmark-gnutime-format-string description}"
@ -55,15 +35,8 @@ let
"--no-warnings" "--no-warnings"
"-E" expr "-E" expr
]} ]}
'').overrideAttrs (finalAttrs: previousAttrs: { touch $out
passthru = (previousAttrs.passthru or { }) // { '');
ci = {
label = ":nix: benchmark ${description} in tvix";
needsOutput = true;
command = "${finalAttrs.finalPackage}/bin/${finalAttrs.meta.mainProgram}";
};
};
});
mkNixpkgsBenchmark = attrpath: mkNixpkgsBenchmark = attrpath:
mkExprBenchmark { mkExprBenchmark {
@ -71,6 +44,28 @@ let
expr = "(import ${pkgs.path} {}).${attrpath}"; expr = "(import ${pkgs.path} {}).${attrpath}";
}; };
# Constructs a Derivation invoking tvix-cli inside a build, ensures the
# calculated tvix output path matches what's passed in externally.
mkNixpkgsEvalTest = attrpath: expectedPath:
let
name = "tvix-eval-test-${builtins.replaceStrings [".drv"] ["-drv"] attrpath}";
in
(pkgs.runCommand name { } ''
export SSL_CERT_FILE=${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt
TVIX_OUTPUT=$(${tvix-cli}/bin/tvix -E '(import ${pkgs.path} {}).${attrpath}')
EXPECTED='${/* the verbatim expected Tvix output: */ "=> \"${builtins.unsafeDiscardStringContext expectedPath}\" :: string"}'
echo "Tvix output: ''${TVIX_OUTPUT}"
if [ "$TVIX_OUTPUT" != "$EXPECTED" ]; then
echo "Correct would have been ''${EXPECTED}"
exit 1
fi
echo "Output was correct."
touch $out
'');
benchmarks = { benchmarks = {
benchmark-hello = (mkNixpkgsBenchmark "hello.outPath"); benchmark-hello = (mkNixpkgsBenchmark "hello.outPath");
benchmark-cross-hello = (mkNixpkgsBenchmark "pkgsCross.aarch64-multiplatform.hello.outPath"); benchmark-cross-hello = (mkNixpkgsBenchmark "pkgsCross.aarch64-multiplatform.hello.outPath");
@ -79,21 +74,22 @@ let
# Example used for benchmarking LightSpan::Delayed in commit bf286a54bc2ac5eeb78c3d5c5ae66e9af24d74d4 # Example used for benchmarking LightSpan::Delayed in commit bf286a54bc2ac5eeb78c3d5c5ae66e9af24d74d4
benchmark-nixpkgs-attrnames = (mkExprBenchmark { expr = "builtins.length (builtins.attrNames (import ${pkgs.path} {}))"; description = "nixpkgs-attrnames"; }); benchmark-nixpkgs-attrnames = (mkExprBenchmark { expr = "builtins.length (builtins.attrNames (import ${pkgs.path} {}))"; description = "nixpkgs-attrnames"; });
}; };
evalTests = {
eval-nixpkgs-stdenv-drvpath = (mkNixpkgsEvalTest "stdenv.drvPath" pkgs.stdenv.drvPath);
eval-nixpkgs-stdenv-outpath = (mkNixpkgsEvalTest "stdenv.outPath" pkgs.stdenv.outPath);
eval-nixpkgs-hello-outpath = (mkNixpkgsEvalTest "hello.outPath" pkgs.hello.outPath);
eval-nixpkgs-firefox-outpath = (mkNixpkgsEvalTest "firefox.outPath" pkgs.firefox.outPath);
eval-nixpkgs-firefox-drvpath = (mkNixpkgsEvalTest "firefox.drvPath" pkgs.firefox.drvPath);
eval-nixpkgs-cross-stdenv-outpath = (mkNixpkgsEvalTest "pkgsCross.aarch64-multiplatform.stdenv.outPath" pkgs.pkgsCross.aarch64-multiplatform.stdenv.outPath);
eval-nixpkgs-cross-hello-outpath = (mkNixpkgsEvalTest "pkgsCross.aarch64-multiplatform.hello.outPath" pkgs.pkgsCross.aarch64-multiplatform.hello.outPath);
};
in in
{ {
meta = { meta = {
ci.extraSteps = { ci.targets = (builtins.attrNames benchmarks) ++ (builtins.attrNames evalTests);
eval-nixpkgs-stdenv-drvpath = (mkNixpkgsEvalCheck "stdenv.drvPath" pkgs.stdenv.drvPath);
eval-nixpkgs-stdenv-outpath = (mkNixpkgsEvalCheck "stdenv.outPath" pkgs.stdenv.outPath);
eval-nixpkgs-hello-outpath = (mkNixpkgsEvalCheck "hello.outPath" pkgs.hello.outPath);
eval-nixpkgs-firefox-outpath = (mkNixpkgsEvalCheck "firefox.outPath" pkgs.firefox.outPath);
eval-nixpkgs-firefox-drvpath = (mkNixpkgsEvalCheck "firefox.drvPath" pkgs.firefox.drvPath);
eval-nixpkgs-cross-stdenv-outpath = (mkNixpkgsEvalCheck "pkgsCross.aarch64-multiplatform.stdenv.outPath" pkgs.pkgsCross.aarch64-multiplatform.stdenv.outPath);
eval-nixpkgs-cross-hello-outpath = (mkNixpkgsEvalCheck "pkgsCross.aarch64-multiplatform.hello.outPath" pkgs.pkgsCross.aarch64-multiplatform.hello.outPath);
};
ci.targets = builtins.attrNames benchmarks;
}; };
# Expose benchmarks as standard CI targets. # Expose benchmarks and evalTests as standard CI targets.
passthru = benchmarks; passthru = benchmarks // evalTests;
}) })