refactor(tvix/boot): use tvix-store copy

Instead of using an additional Nix build producing a directory structure
containing xz-compressed NAR files and NARInfos, and uploading these via
`curl` to NAR-bridge, which then decomposes them and uploads them to
tvix-store, have tvix-store interpret the exported reference graph
directly, and deal with the uploading on its own.

Change-Id: I44ee0b132944c53d11abb688d2d017f0cc9c3d97
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11398
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2024-04-12 20:27:15 +03:00 committed by clbot
parent a4f65ddba0
commit 820e98fa7f

View file

@ -32,14 +32,12 @@ let
pkgs.stdenv.mkDerivation { pkgs.stdenv.mkDerivation {
name = "run-vm"; name = "run-vm";
__structuredAttrs = true;
exportReferencesGraph.closure = [ path ];
nativeBuildInputs = [ nativeBuildInputs = [
depot.tvix.store depot.tvix.store
depot.tvix.boot.runVM depot.tvix.boot.runVM
] ++ lib.optionals isClosure [
depot.tvix.nar-bridge
pkgs.curl
pkgs.parallel
pkgs.xz.bin
]; ];
buildCommand = '' buildCommand = ''
touch $out touch $out
@ -67,34 +65,9 @@ let
echo "imported to $outpath" echo "imported to $outpath"
'' + lib.optionalString (isClosure) '' '' + lib.optionalString (isClosure) ''
echo "Starting nar-bridge" echo "Copying closure ${path}"
nar-bridge-http \ # This picks up the `closure` key in `$NIX_ATTRS_JSON_FILE` automatically.
--otlp=false \ tvix-store --otlp=false copy
--store-addr=unix://$PWD/tvix-store.sock \
--listen-addr=$PWD/nar-bridge.sock &
# Wait for the socket to be created.
while [ ! -e $PWD/nar-bridge.sock ]; do sleep 1; done
# Upload. We can't use nix copy --to http://…, as it wants access to the nix db.
# However, we can use mkBinaryCache to assemble .narinfo and .nar.xz to upload,
# and then drive a HTTP client ourselves.
to_upload=${pkgs.mkBinaryCache { rootPaths = [path];}}
# Upload all NAR files (with some parallelism).
# As mkBinaryCache produces them xz-compressed, unpack them on the fly.
# nar-bridge doesn't care about the path we upload *to*, but a
# subsequent .narinfo upload need to refer to its contents (by narhash).
echo -e "Uploading NARs "
ls -d $to_upload/nar/*.nar.xz | parallel 'xz -d < {} | curl -s -T - --unix-socket $PWD/nar-bridge.sock http://localhost:9000/nar/$(basename {} | cut -d "." -f 1).nar'
echo "Done."
# Upload all NARInfo files.
# FUTUREWORK: This doesn't upload them in order, and currently relies
# on PathInfoService not doing any checking.
# In the future, we might want to make this behaviour configurable,
# and disable checking here, to keep the logic simple.
ls -d $to_upload/*.narinfo | parallel 'curl -s -T - --unix-socket $PWD/nar-bridge.sock http://localhost:9000/$(basename {}) < {}'
'' + '' '' + ''
# Invoke a VM using tvix as the backing store, ensure the outpath appears in its listing. # Invoke a VM using tvix as the backing store, ensure the outpath appears in its listing.