mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/cli): add macrobenchmark

Browse source 
This commit adds a simple MVP benchmark, built on our nix
infrastructure instead of cargo.  It simply runs `tvix-eval` inside
of GNU time, and prints the three essential statistics in a short
JSON blob.

You can run the benchmark with a simple `nix run`, like:

  nix run -f . tvix.cli.benchmark-hello
  nix run -f . tvix.cli.benchmark-firefox
  nix run -f . tvix.cli.benchmark-cross-firefox

Currently these blobs are stored only in the CI logs, which I'm sure
get garbage-collected at some point.  We should be putting them in
the git trailers, but that can wait for a future CL.

I tried using `cargo bench` for this but found it incredibly
frustrating.  Maybe I'm doing it wrong.  It seems to be designed for
microbenchmarks only, and very hard to control.  It kept building
all sorts of unnecessary stuff (like the tests), and unlike
crate2nix it was doing all the builds on only a single machine
instead of using more than one machine.  Worse, for that single
machine it kept picking my laptop instead of my fast servers!  It
seems excessively cargo-flavored for such a straightforward task.

Benchmark: {"hello.outPath":{"kbytes":"244736","system":"0.36","user":"2.76"}}
Benchmark: {"firefox.outPath":{"kbytes":"1506736","system":"2.38","user":"32.01"}}
Benchmark: {"pkgsCross.aarch64-multiplatform.firefox.outPath":{"kbytes":"11334548","system":"10.70","user":"107.07"}}

Change-Id: I85bc046ec551360284d7ecfc81a03914f0085909
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10216
Autosubmit: Adam Joseph <adam@westernsemico.com>
Reviewed-by: grfn <grfn@gws.fyi>
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
This commit is contained in:
Adam Joseph 2023-12-08 20:15:31 -08:00 committed by clbot
parent 02c6621813
commit 0c15a09b15
1 changed files with 61 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
tvix/cli/default.nix
View file

@ -18,11 +18,67 @@ let
echo "Output was correct."
'';
};
benchmark-gnutime-format-string =
description:
"Benchmark: " +
(builtins.toJSON {
"${description}" = {
kbytes = "%M";
system = "%S";
user = "%U";
};
});
in
(depot.tvix.crates.workspaceMembers.tvix-cli.build.override {
runTests = true;
}).overrideAttrs (_: {
}).overrideAttrs (finalAttrs: previousAttrs:
let
tvix-cli = finalAttrs.finalPackage;
# You can run the benchmark with a simple `nix run`, like:
#
# nix run -f . tvix.cli.meta.ci.extraSteps.benchmark-nixpkgs-cross-hello-outpath
#
# TODO(amjoseph): store these results someplace more durable, like git trailers
#
mkExprBenchmark = { expr, description }:
let name = "tvix-cli-benchmark-${description}"; in
(pkgs.writeShellScriptBin name ''
${lib.escapeShellArgs [
"${pkgs.time}/bin/time"
"--format" "${benchmark-gnutime-format-string description}"
"${tvix-cli}/bin/tvix"
"--no-warnings"
"-E" expr
]}
'').overrideAttrs (finalAttrs: previousAttrs: {
passthru = (previousAttrs.passthru or { }) // {
ci = {
label = ":nix: benchmark ${description} in tvix";
needsOutput = true;
command = "${finalAttrs.finalPackage}/bin/${finalAttrs.meta.mainProgram}";
};
};
});
mkNixpkgsBenchmark = attrpath:
mkExprBenchmark {
description = builtins.replaceStrings [ ".drv" ] [ "-drv" ] attrpath;
expr = "(import ${pkgs.path} {}).${attrpath}";
};
benchmarks = {
benchmark-hello = (mkNixpkgsBenchmark "hello.outPath");
benchmark-cross-hello = (mkNixpkgsBenchmark "pkgsCross.aarch64-multiplatform.hello.outPath");
benchmark-firefox = (mkNixpkgsBenchmark "firefox.outPath");
benchmark-cross-firefox = (mkNixpkgsBenchmark "pkgsCross.aarch64-multiplatform.firefox.outPath");
};
in
{
meta = {
ci.extraSteps = {
eval-nixpkgs-stdenv-drvpath = (mkNixpkgsEvalCheck "stdenv.drvPath" pkgs.stdenv.drvPath);
@ -31,5 +87,9 @@ in
eval-nixpkgs-cross-stdenv-outpath = (mkNixpkgsEvalCheck "pkgsCross.aarch64-multiplatform.stdenv.outPath" pkgs.pkgsCross.aarch64-multiplatform.stdenv.outPath);
eval-nixpkgs-cross-hello-outpath = (mkNixpkgsEvalCheck "pkgsCross.aarch64-multiplatform.hello.outPath" pkgs.pkgsCross.aarch64-multiplatform.hello.outPath);
};
ci.targets = builtins.attrNames benchmarks;
};
# Expose benchmarks as standard CI targets.
passthru = benchmarks;
})