From 28cd4b1a2f98759dc33390db78f328f20f2db515 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 9 Oct 2023 17:23:43 +0200 Subject: [PATCH] feat(tvix/nar-bridge): add ToNixNarInfo() Convenience function, moves all code converting from a PathInfo struct to to go-nix's NarInfo. Change-Id: Idf0dcc38675674563f2dfd3286a4a55fa2a24a82 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9593 Autosubmit: flokli Tested-by: BuildkiteCI Reviewed-by: Brian McGee --- tvix/default.nix | 2 +- tvix/nar-bridge/pkg/http/narinfo.go | 49 +++++++++++++++++++++++++ tvix/nar-bridge/pkg/http/narinfo_get.go | 38 ++----------------- tvix/store/protos/go.mod | 2 +- tvix/store/protos/go.sum | 4 +- 5 files changed, 56 insertions(+), 39 deletions(-) create mode 100644 tvix/nar-bridge/pkg/http/narinfo.go diff --git a/tvix/default.nix b/tvix/default.nix index 802ec9497..f8513ffac 100644 --- a/tvix/default.nix +++ b/tvix/default.nix @@ -108,7 +108,7 @@ in store-protos-go = pkgs.buildGoModule { name = "store-golang"; src = depot.third_party.gitignoreSource ./store/protos; - vendorHash = "sha256-L+mHTUYRZu8PSbD7LJ9QRuW1+ImIYbH9/SKgAoL9W8w="; + vendorHash = "sha256-injKuXHUvjyJraZOyDWoKD4NXIdQJS7VorpWyoNo1jk="; }; # Build the Rust documentation for publishing on docs.tvix.dev. diff --git a/tvix/nar-bridge/pkg/http/narinfo.go b/tvix/nar-bridge/pkg/http/narinfo.go new file mode 100644 index 000000000..5d963c34f --- /dev/null +++ b/tvix/nar-bridge/pkg/http/narinfo.go @@ -0,0 +1,49 @@ +package http + +import ( + "fmt" + + storev1pb "code.tvl.fyi/tvix/store/protos" + nixhash "github.com/nix-community/go-nix/pkg/hash" + "github.com/nix-community/go-nix/pkg/narinfo" + "github.com/nix-community/go-nix/pkg/narinfo/signature" + "github.com/nix-community/go-nix/pkg/nixbase32" +) + +// ToNixNarInfo converts the PathInfo to a narinfo.NarInfo. +func ToNixNarInfo(p *storev1pb.PathInfo) (*narinfo.NarInfo, error) { + // ensure the PathInfo is valid, and extract the StorePath from the node in + // there. + storePath, err := p.Validate() + if err != nil { + return nil, fmt.Errorf("failed to validate PathInfo: %w", err) + } + + // convert the signatures from storev1pb signatures to narinfo signatures + narinfoSignatures := make([]signature.Signature, len(p.GetNarinfo().GetSignatures())) + for i, pathInfoSignature := range p.GetNarinfo().GetSignatures() { + narinfoSignatures[i] = signature.Signature{ + Name: pathInfoSignature.GetName(), + Data: pathInfoSignature.GetData(), + } + } + + // produce nixhash for the narsha256. + narHash, err := nixhash.FromHashTypeAndDigest( + 0x12, // SHA2_256 + p.GetNarinfo().GetNarSha256(), + ) + if err != nil { + return nil, fmt.Errorf("invalid narsha256: %w", err) + } + + return &narinfo.NarInfo{ + StorePath: storePath.Absolute(), + URL: "nar/" + nixbase32.EncodeToString(narHash.Digest()) + ".nar", + Compression: "none", + NarHash: narHash, + NarSize: uint64(p.GetNarinfo().GetNarSize()), + References: p.GetNarinfo().GetReferenceNames(), + Signatures: narinfoSignatures, + }, nil +} diff --git a/tvix/nar-bridge/pkg/http/narinfo_get.go b/tvix/nar-bridge/pkg/http/narinfo_get.go index 93192c03e..8338d4367 100644 --- a/tvix/nar-bridge/pkg/http/narinfo_get.go +++ b/tvix/nar-bridge/pkg/http/narinfo_get.go @@ -8,18 +8,13 @@ import ( "io" "io/fs" "net/http" - "path" "strings" "sync" - castorev1pb "code.tvl.fyi/tvix/castore/protos" storev1pb "code.tvl.fyi/tvix/store/protos" "github.com/go-chi/chi/v5" nixhash "github.com/nix-community/go-nix/pkg/hash" - "github.com/nix-community/go-nix/pkg/narinfo" - "github.com/nix-community/go-nix/pkg/narinfo/signature" "github.com/nix-community/go-nix/pkg/nixbase32" - "github.com/nix-community/go-nix/pkg/storepath" log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -71,37 +66,10 @@ func renderNarinfo( return nil } - // convert the signatures from storev1pb signatures to narinfo signatures - narinfoSignatures := make([]signature.Signature, 0) - for _, pathInfoSignature := range pathInfo.Narinfo.Signatures { - narinfoSignatures = append(narinfoSignatures, signature.Signature{ - Name: pathInfoSignature.GetName(), - Data: pathInfoSignature.GetData(), - }) - } + // convert the PathInfo to NARInfo. + narInfo, err := ToNixNarInfo(pathInfo) - // extract the name of the node in the pathInfo structure, which will become the output path - var nodeName []byte - switch v := (pathInfo.GetNode().GetNode()).(type) { - case *castorev1pb.Node_File: - nodeName = v.File.GetName() - case *castorev1pb.Node_Symlink: - nodeName = v.Symlink.GetName() - case *castorev1pb.Node_Directory: - nodeName = v.Directory.GetName() - } - - narInfo := narinfo.NarInfo{ - StorePath: path.Join(storepath.StoreDir, string(nodeName)), - URL: "nar/" + nixbase32.EncodeToString(narHash.Digest()) + ".nar", - Compression: "none", // TODO: implement zstd compression - NarHash: narHash, - NarSize: uint64(pathInfo.Narinfo.NarSize), - References: pathInfo.Narinfo.GetReferenceNames(), - Signatures: narinfoSignatures, - } - - // render .narinfo from pathInfo + // Write it out to the client. _, err = io.Copy(w, strings.NewReader(narInfo.String())) if err != nil { return fmt.Errorf("unable to write narinfo to client: %w", err) diff --git a/tvix/store/protos/go.mod b/tvix/store/protos/go.mod index 5c6306f51..607f13cad 100644 --- a/tvix/store/protos/go.mod +++ b/tvix/store/protos/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( code.tvl.fyi/tvix/castore/protos v0.0.0-20230922125121-72355662d742 - github.com/nix-community/go-nix v0.0.0-20231005143722-b0f8b73c06df + github.com/nix-community/go-nix v0.0.0-20231009143713-ebca3299475b github.com/stretchr/testify v1.8.1 google.golang.org/grpc v1.51.0 google.golang.org/protobuf v1.28.1 diff --git a/tvix/store/protos/go.sum b/tvix/store/protos/go.sum index b5b92a74d..c19933144 100644 --- a/tvix/store/protos/go.sum +++ b/tvix/store/protos/go.sum @@ -30,8 +30,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/nix-community/go-nix v0.0.0-20231005143722-b0f8b73c06df h1:n4I26uXUST5vmdsDWPo9ikK57il4htQyhnsLWoHYFmY= -github.com/nix-community/go-nix v0.0.0-20231005143722-b0f8b73c06df/go.mod h1:hHM9UK2zOCjvmiLgeaW4LVbOW/vBaRWFJGzfi31/slQ= +github.com/nix-community/go-nix v0.0.0-20231009143713-ebca3299475b h1:AWEKOdDO3JnHApQDOmONEKLXbMCQJhYJJfJpiWB9VGI= +github.com/nix-community/go-nix v0.0.0-20231009143713-ebca3299475b/go.mod h1:hHM9UK2zOCjvmiLgeaW4LVbOW/vBaRWFJGzfi31/slQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=