mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/nar-bridge): update to new separated protos

Browse source 
Change-Id: I5eaadc837a4d3a7f635574437127a22de88f556b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9407
Tested-by: BuildkiteCI
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink 2023-09-22 16:38:10 +03:00 committed by flokli
parent e13c86d916
commit 732dc68727
16 changed files with 170 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
tvix/nar-bridge/cmd/nar_bridge/main.go
View file

@ -11,6 +11,7 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
"code.tvl.fyi/tvix/nar-bridge/pkg/server"
storev1pb "code.tvl.fyi/tvix/store/protos"
"github.com/sirupsen/logrus"
@ -47,8 +48,8 @@ func main() {
defer conn.Close()
s := server.New(
storev1pb.NewDirectoryServiceClient(conn),
storev1pb.NewBlobServiceClient(conn),
castorev1pb.NewDirectoryServiceClient(conn),
castorev1pb.NewBlobServiceClient(conn),
storev1pb.NewPathInfoServiceClient(conn),
cli.EnableAccessLog,
30,

2
tvix/nar-bridge/default.nix
View file

@ -6,5 +6,5 @@ pkgs.buildGoModule {
name = "nar-bridge";
src = depot.third_party.gitignoreSource ./.;
vendorHash = "sha256-xaNf/bnSuQpt1vadFmYt4NcpJQD3KmiYQ4SrdtiK33U=";
vendorHash = "sha256-sxyoVyHq/FJbNzVa6Xlnv0+/vxLxf0SAY4ZbvRySoFQ=";
}

3
tvix/nar-bridge/go.mod
View file

@ -1,7 +1,8 @@
module code.tvl.fyi/tvix/nar-bridge
require (
code.tvl.fyi/tvix/store/protos v0.0.0-20230918103330-20a5285e3e82
code.tvl.fyi/tvix/castore/protos v0.0.0-20230922125121-72355662d742
code.tvl.fyi/tvix/store/protos v0.0.0-20230922125121-4251671e09c1
github.com/alecthomas/kong v0.7.1
github.com/go-chi/chi v1.5.4
github.com/go-chi/chi/v5 v5.0.7

6
tvix/nar-bridge/go.sum
View file

@ -1,6 +1,8 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
code.tvl.fyi/tvix/store/protos v0.0.0-20230918103330-20a5285e3e82 h1:DffXv1J4znl2BT6bATBaR3lnb8P4cPr02fNOn9RuGYY=
code.tvl.fyi/tvix/store/protos v0.0.0-20230918103330-20a5285e3e82/go.mod h1:uOGlmwyEraDd9kzAWHdDeZdNgKrpOAmtWCdMeY+DVKs=
code.tvl.fyi/tvix/castore/protos v0.0.0-20230922125121-72355662d742 h1:x7LsxggggaN3acnCMNDO5LZLAV+A+rZ+R8TXzr+Lgsk=
code.tvl.fyi/tvix/castore/protos v0.0.0-20230922125121-72355662d742/go.mod h1:Ejhyvc0dJUWQMxtJxddfFuAF5N8IKIO94q5CP4czY8Y=
code.tvl.fyi/tvix/store/protos v0.0.0-20230922125121-4251671e09c1 h1:ttMO8cE4rVbqJlqLsoirePWEtXhT4MxrGvLMFyJ+UhA=
code.tvl.fyi/tvix/store/protos v0.0.0-20230922125121-4251671e09c1/go.mod h1:uOGlmwyEraDd9kzAWHdDeZdNgKrpOAmtWCdMeY+DVKs=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/alecthomas/assert/v2 v2.1.0 h1:tbredtNcQnoSd3QBhQWI7QZ3XHOVkw1Moklp2ojoH/0=
github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4=

39
tvix/nar-bridge/pkg/reader/reader.go
View file

@ -9,6 +9,7 @@ import (
"path"
"strings"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
storev1pb "code.tvl.fyi/tvix/store/protos"
"github.com/nix-community/go-nix/pkg/nar"
"lukechampine.com/blake3"
@ -21,7 +22,7 @@ type Reader struct {
// An item on the directories stack
type item struct {
path string
directory *storev1pb.Directory
directory *castorev1pb.Directory
}
func New(r io.Reader) *Reader {
@ -43,7 +44,7 @@ func (r *Reader) Import(
// callback function called with each regular file content
blobCb func(fileReader io.Reader) error,
// callback function called with each finalized directory node
directoryCb func(directory *storev1pb.Directory) error,
directoryCb func(directory *castorev1pb.Directory) error,
) (*storev1pb.PathInfo, error) {
// construct a NAR reader, by reading through hrSha256
@ -55,9 +56,9 @@ func (r *Reader) Import(
// If we store a symlink or regular file at the root, these are not nil.
// If they are nil, we instead have a stackDirectory.
var rootSymlink *storev1pb.SymlinkNode
var rootFile *storev1pb.FileNode
var stackDirectory *storev1pb.Directory
var rootSymlink *castorev1pb.SymlinkNode
var rootFile *castorev1pb.FileNode
var stackDirectory *castorev1pb.Directory
var stack = []item{}
@ -81,7 +82,7 @@ func (r *Reader) Import(
}
topOfStack := stack[len(stack)-1].directory
topOfStack.Directories = append(topOfStack.Directories, &storev1pb.DirectoryNode{
topOfStack.Directories = append(topOfStack.Directories, &castorev1pb.DirectoryNode{
Name: []byte(path.Base(toPop.path)),
Digest: dgst,
Size: toPop.directory.Size(),
@ -154,15 +155,15 @@ func (r *Reader) Import(
// assemble pathInfo with these and return.
pi := assemblePathInfo()
if rootFile != nil {
pi.Node = &storev1pb.Node{
Node: &storev1pb.Node_File{
pi.Node = &castorev1pb.Node{
Node: &castorev1pb.Node_File{
File: rootFile,
},
}
}
if rootSymlink != nil {
pi.Node = &storev1pb.Node{
Node: &storev1pb.Node_Symlink{
pi.Node = &castorev1pb.Node{
Node: &castorev1pb.Node_Symlink{
Symlink: rootSymlink,
},
}
@ -174,9 +175,9 @@ func (r *Reader) Import(
return nil, fmt.Errorf("unable to calculate root directory digest: %w", err)
}
pi.Node = &storev1pb.Node{
Node: &storev1pb.Node_Directory{
Directory: &storev1pb.DirectoryNode{
pi.Node = &castorev1pb.Node{
Node: &castorev1pb.Node_Directory{
Directory: &castorev1pb.DirectoryNode{
Name: []byte{},
Digest: dgst,
Size: stackDirectory.Size(),
@ -203,7 +204,7 @@ func (r *Reader) Import(
}
if hdr.Type == nar.TypeSymlink {
symlinkNode := &storev1pb.SymlinkNode{
symlinkNode := &castorev1pb.SymlinkNode{
Name: []byte(getBasename(hdr.Path)),
Target: []byte(hdr.LinkTarget),
}
@ -236,7 +237,7 @@ func (r *Reader) Import(
// read the blake3 hash
dgst := fileReader.Sum(nil)
fileNode := &storev1pb.FileNode{
fileNode := &castorev1pb.FileNode{
Name: []byte(getBasename(hdr.Path)),
Digest: dgst,
Size: uint32(hdr.Size),
@ -250,10 +251,10 @@ func (r *Reader) Import(
}
}
if hdr.Type == nar.TypeDirectory {
directory := &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
directory := &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
stack = append(stack, item{
directory: directory,

107
tvix/nar-bridge/pkg/reader/reader_test.go
View file

@ -7,6 +7,7 @@ import (
"os"
"testing"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
"code.tvl.fyi/tvix/nar-bridge/pkg/reader"
storev1pb "code.tvl.fyi/tvix/store/protos"
"github.com/google/go-cmp/cmp"
@ -20,7 +21,7 @@ func requireProtoEq(t *testing.T, expected interface{}, actual interface{}) {
}
}
func mustDigest(d *storev1pb.Directory) []byte {
func mustDigest(d *castorev1pb.Directory) []byte {
dgst, err := d.Digest()
if err != nil {
panic(err)
@ -38,16 +39,16 @@ func TestSymlink(t *testing.T) {
context.Background(),
func(fileReader io.Reader) error {
panic("no file contents expected!")
}, func(directory *storev1pb.Directory) error {
}, func(directory *castorev1pb.Directory) error {
panic("no directories expected!")
},
)
require.NoError(t, err)
expectedPathInfo := &storev1pb.PathInfo{
Node: &storev1pb.Node{
Node: &storev1pb.Node_Symlink{
Symlink: &storev1pb.SymlinkNode{
Node: &castorev1pb.Node{
Node: &castorev1pb.Node_Symlink{
Symlink: &castorev1pb.SymlinkNode{
Name: []byte(""),
Target: []byte("/nix/store/somewhereelse"),
},
@ -80,7 +81,7 @@ func TestRegular(t *testing.T) {
require.NoError(t, err, "reading fileReader should not error")
require.Equal(t, []byte{0x01}, contents, "contents read from fileReader should match expectations")
return nil
}, func(directory *storev1pb.Directory) error {
}, func(directory *castorev1pb.Directory) error {
panic("no directories expected!")
},
)
@ -94,9 +95,9 @@ func TestRegular(t *testing.T) {
}
expectedPathInfo := &storev1pb.PathInfo{
Node: &storev1pb.Node{
Node: &storev1pb.Node_File{
File: &storev1pb.FileNode{
Node: &castorev1pb.Node{
Node: &castorev1pb.Node_File{
File: &castorev1pb.FileNode{
Name: []byte(""),
Digest: BLAKE3_DIGEST_0X01,
Size: 1,
@ -124,16 +125,16 @@ func TestEmptyDirectory(t *testing.T) {
r := reader.New(f)
expectedDirectory := &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
expectedDirectory := &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
actualPathInfo, err := r.Import(
context.Background(),
func(fileReader io.Reader) error {
panic("no file contents expected!")
}, func(directory *storev1pb.Directory) error {
}, func(directory *castorev1pb.Directory) error {
requireProtoEq(t, expectedDirectory, directory)
return nil
},
@ -141,9 +142,9 @@ func TestEmptyDirectory(t *testing.T) {
require.NoError(t, err)
expectedPathInfo := &storev1pb.PathInfo{
Node: &storev1pb.Node{
Node: &storev1pb.Node_Directory{
Directory: &storev1pb.DirectoryNode{
Node: &castorev1pb.Node{
Node: &castorev1pb.Node_Directory{
Directory: &castorev1pb.DirectoryNode{
Name: []byte(""),
Digest: mustDigest(expectedDirectory),
Size: expectedDirectory.Size(),
@ -178,12 +179,12 @@ func TestFull(t *testing.T) {
"/share",
"/",
}
expectedDirectories := make(map[string]*storev1pb.Directory, len(expectedDirectoryPaths))
expectedDirectories := make(map[string]*castorev1pb.Directory, len(expectedDirectoryPaths))
// /bin is a leaf directory
expectedDirectories["/bin"] = &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{
expectedDirectories["/bin"] = &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{
{
Name: []byte("arp"),
Digest: []byte{
@ -257,7 +258,7 @@ func TestFull(t *testing.T) {
Executable: true,
},
},
Symlinks: []*storev1pb.SymlinkNode{
Symlinks: []*castorev1pb.SymlinkNode{
{
Name: []byte("dnsdomainname"),
Target: []byte("hostname"),
@ -279,9 +280,9 @@ func TestFull(t *testing.T) {
// /share/man/man1 is a leaf directory.
// The parser traversed over /sbin, but only added it to / which is still on the stack.
expectedDirectories["/share/man/man1"] = &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{
expectedDirectories["/share/man/man1"] = &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{
{
Name: []byte("dnsdomainname.1.gz"),
Digest: []byte{
@ -323,13 +324,13 @@ func TestFull(t *testing.T) {
Executable: false,
},
},
Symlinks: []*storev1pb.SymlinkNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
// /share/man/man5 is a leaf directory
expectedDirectories["/share/man/man5"] = &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{
expectedDirectories["/share/man/man5"] = &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{
{
Name: []byte("ethers.5.gz"),
Digest: []byte{
@ -339,13 +340,13 @@ func TestFull(t *testing.T) {
Executable: false,
},
},
Symlinks: []*storev1pb.SymlinkNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
// /share/man/man8 is a leaf directory
expectedDirectories["/share/man/man8"] = &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{
expectedDirectories["/share/man/man8"] = &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{
{
Name: []byte("arp.8.gz"),
Digest: []byte{
@ -411,12 +412,12 @@ func TestFull(t *testing.T) {
Executable: false,
},
},
Symlinks: []*storev1pb.SymlinkNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
// /share/man holds /share/man/man{1,5,8}.
expectedDirectories["/share/man"] = &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{
expectedDirectories["/share/man"] = &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{
{
Name: []byte("man1"),
Digest: mustDigest(expectedDirectories["/share/man/man1"]),
@ -433,26 +434,26 @@ func TestFull(t *testing.T) {
Size: expectedDirectories["/share/man/man8"].Size(),
},
},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
// /share holds /share/man.
expectedDirectories["/share"] = &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{
expectedDirectories["/share"] = &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{
{
Name: []byte("man"),
Digest: mustDigest(expectedDirectories["/share/man"]),
Size: expectedDirectories["/share/man"].Size(),
},
},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
// / holds /bin, /share, and a /sbin symlink.
expectedDirectories["/"] = &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{
expectedDirectories["/"] = &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{
{
Name: []byte("bin"),
Digest: mustDigest(expectedDirectories["/bin"]),
@ -464,8 +465,8 @@ func TestFull(t *testing.T) {
Size: expectedDirectories["/share"].Size(),
},
},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{
{
Name: []byte("sbin"),
Target: []byte("bin"),
@ -486,7 +487,7 @@ func TestFull(t *testing.T) {
// This also covers the case when the client doesn't read from the reader, and that the
// importer will take care of reading all the way to the end no matter what.
return nil
}, func(directory *storev1pb.Directory) error {
}, func(directory *castorev1pb.Directory) error {
// use actualDirectoryOrder to look up the Directory object we expect at this specific invocation.
currentDirectoryPath := expectedDirectoryPaths[numDirectoriesReceived]
@ -502,9 +503,9 @@ func TestFull(t *testing.T) {
require.NoError(t, err)
expectedPathInfo := &storev1pb.PathInfo{
Node: &storev1pb.Node{
Node: &storev1pb.Node_Directory{
Directory: &storev1pb.DirectoryNode{
Node: &castorev1pb.Node{
Node: &castorev1pb.Node_Directory{
Directory: &castorev1pb.DirectoryNode{
Name: []byte(""),
Digest: mustDigest(expectedDirectories["/"]),
Size: expectedDirectories["/"].Size(),
@ -540,7 +541,7 @@ func TestCallbackErrors(t *testing.T) {
context.Background(),
func(fileReader io.Reader) error {
return targetErr
}, func(directory *storev1pb.Directory) error {
}, func(directory *castorev1pb.Directory) error {
panic("no directories expected!")
},
)
@ -559,7 +560,7 @@ func TestCallbackErrors(t *testing.T) {
context.Background(),
func(fileReader io.Reader) error {
panic("no file contents expected!")
}, func(directory *storev1pb.Directory) error {
}, func(directory *castorev1pb.Directory) error {
return targetErr
},
)
@ -588,7 +589,7 @@ func TestPopDirectories(t *testing.T) {
_, err = r.Import(
context.Background(),
func(fileReader io.Reader) error { return nil },
func(directory *storev1pb.Directory) error {
func(directory *castorev1pb.Directory) error {
return directory.Validate()
},
)

6
tvix/nar-bridge/pkg/server/blob_upload.go
View file

@ -8,7 +8,7 @@ import (
"fmt"
"io"
storev1pb "code.tvl.fyi/tvix/store/protos"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
log "github.com/sirupsen/logrus"
)
@ -17,7 +17,7 @@ const chunkSize = 1024 * 1024
// this produces a callback function that can be used as blobCb for the
// reader.Import function call
func genBlobServiceWriteCb(ctx context.Context, blobServiceClient storev1pb.BlobServiceClient) func(io.Reader) error {
func genBlobServiceWriteCb(ctx context.Context, blobServiceClient castorev1pb.BlobServiceClient) func(io.Reader) error {
return func(blobReader io.Reader) error {
// Ensure the blobReader is buffered to at least the chunk size.
blobReader = bufio.NewReaderSize(blobReader, chunkSize)
@ -42,7 +42,7 @@ func genBlobServiceWriteCb(ctx context.Context, blobServiceClient storev1pb.Blob
blobSize += n
// send the blob chunk to the server. The err is only valid in the inner scope
if err := putter.Send(&storev1pb.BlobChunk{
if err := putter.Send(&castorev1pb.BlobChunk{
Data: chunk[:n],
}); err != nil {
return fmt.Errorf("sending blob chunk: %w", err)

12
tvix/nar-bridge/pkg/server/directory_upload.go
View file

@ -5,17 +5,17 @@ import (
"encoding/base64"
"fmt"
storev1pb "code.tvl.fyi/tvix/store/protos"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
log "github.com/sirupsen/logrus"
)
type DirectoriesUploader struct {
ctx context.Context
directoryServiceClient storev1pb.DirectoryServiceClient
directoryServicePutStream storev1pb.DirectoryService_PutClient
directoryServiceClient castorev1pb.DirectoryServiceClient
directoryServicePutStream castorev1pb.DirectoryService_PutClient
}
func NewDirectoriesUploader(ctx context.Context, directoryServiceClient storev1pb.DirectoryServiceClient) *DirectoriesUploader {
func NewDirectoriesUploader(ctx context.Context, directoryServiceClient castorev1pb.DirectoryServiceClient) *DirectoriesUploader {
return &DirectoriesUploader{
ctx: ctx,
directoryServiceClient: directoryServiceClient,
@ -23,7 +23,7 @@ func NewDirectoriesUploader(ctx context.Context, directoryServiceClient storev1p
}
}
func (du *DirectoriesUploader) Put(directory *storev1pb.Directory) error {
func (du *DirectoriesUploader) Put(directory *castorev1pb.Directory) error {
directoryDgst, err := directory.Digest()
if err != nil {
return fmt.Errorf("failed calculating directory digest: %w", err)
@ -50,7 +50,7 @@ func (du *DirectoriesUploader) Put(directory *storev1pb.Directory) error {
}
// Done is called whenever we're
func (du *DirectoriesUploader) Done() (*storev1pb.PutDirectoryResponse, error) {
func (du *DirectoriesUploader) Done() (*castorev1pb.PutDirectoryResponse, error) {
// only close once, and only if we opened.
if du.directoryServicePutStream == nil {
return nil, nil

15
tvix/nar-bridge/pkg/server/nar_get.go
View file

@ -12,6 +12,7 @@ import (
"net/http"
"sync"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
"code.tvl.fyi/tvix/nar-bridge/pkg/writer"
storev1pb "code.tvl.fyi/tvix/store/protos"
"github.com/go-chi/chi/v5"
@ -27,8 +28,8 @@ const (
func renderNar(
ctx context.Context,
log *log.Entry,
directoryServiceClient storev1pb.DirectoryServiceClient,
blobServiceClient storev1pb.BlobServiceClient,
directoryServiceClient castorev1pb.DirectoryServiceClient,
blobServiceClient castorev1pb.BlobServiceClient,
narHashToPathInfoMu *sync.Mutex,
narHashToPathInfo map[string]*storev1pb.PathInfo,
w io.Writer,
@ -50,15 +51,15 @@ func renderNar(
return nil
}
directories := make(map[string]*storev1pb.Directory)
directories := make(map[string]*castorev1pb.Directory)
// If the root node is a directory, ask the directory service for all directories
if pathInfoDirectory := pathInfo.GetNode().GetDirectory(); pathInfoDirectory != nil {
rootDirectoryDigest := pathInfoDirectory.GetDigest()
log = log.WithField("root_directory", base64.StdEncoding.EncodeToString(rootDirectoryDigest))
directoryStream, err := directoryServiceClient.Get(ctx, &storev1pb.GetDirectoryRequest{
ByWhat: &storev1pb.GetDirectoryRequest_Digest{
directoryStream, err := directoryServiceClient.Get(ctx, &castorev1pb.GetDirectoryRequest{
ByWhat: &castorev1pb.GetDirectoryRequest_Digest{
Digest: rootDirectoryDigest,
},
Recursive: true,
@ -96,7 +97,7 @@ func renderNar(
err := writer.Export(
w,
pathInfo,
func(directoryDigest []byte) (*storev1pb.Directory, error) {
func(directoryDigest []byte) (*castorev1pb.Directory, error) {
log.WithField("directory", base64.StdEncoding.EncodeToString(directoryDigest)).Debug("Get directory")
directoryRefStr := hex.EncodeToString(directoryDigest)
directory, found := directories[directoryRefStr]
@ -112,7 +113,7 @@ func renderNar(
},
func(blobDigest []byte) (io.ReadCloser, error) {
log.WithField("blob", base64.StdEncoding.EncodeToString(blobDigest)).Debug("Get blob")
resp, err := blobServiceClient.Read(ctx, &storev1pb.ReadBlobRequest{
resp, err := blobServiceClient.Read(ctx, &castorev1pb.ReadBlobRequest{
Digest: blobDigest,
})
if err != nil {

4
tvix/nar-bridge/pkg/server/nar_put.go
View file

@ -6,8 +6,8 @@ import (
"fmt"
"net/http"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
"code.tvl.fyi/tvix/nar-bridge/pkg/reader"
storev1pb "code.tvl.fyi/tvix/store/protos"
"github.com/go-chi/chi/v5"
nixhash "github.com/nix-community/go-nix/pkg/hash"
"github.com/nix-community/go-nix/pkg/nixbase32"
@ -44,7 +44,7 @@ func registerNarPut(s *Server) {
pathInfo, err := rd.Import(
ctx,
genBlobServiceWriteCb(ctx, s.blobServiceClient),
func(directory *storev1pb.Directory) error {
func(directory *castorev1pb.Directory) error {
return directoriesUploader.Put(directory)
},
)

7
tvix/nar-bridge/pkg/server/narinfo_get.go
View file

@ -12,6 +12,7 @@ import (
"strings"
"sync"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
storev1pb "code.tvl.fyi/tvix/store/protos"
"github.com/go-chi/chi/v5"
nixhash "github.com/nix-community/go-nix/pkg/hash"
@ -82,11 +83,11 @@ func renderNarinfo(
// extract the name of the node in the pathInfo structure, which will become the output path
var nodeName []byte
switch v := (pathInfo.GetNode().GetNode()).(type) {
case *storev1pb.Node_File:
case *castorev1pb.Node_File:
nodeName = v.File.GetName()
case *storev1pb.Node_Symlink:
case *castorev1pb.Node_Symlink:
nodeName = v.Symlink.GetName()
case *storev1pb.Node_Directory:
case *castorev1pb.Node_Directory:
nodeName = v.Directory.GetName()
}

25
tvix/nar-bridge/pkg/server/narinfo_put.go
View file

@ -4,6 +4,7 @@ import (
"net/http"
"path"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
storev1pb "code.tvl.fyi/tvix/store/protos"
"github.com/go-chi/chi/v5"
"github.com/nix-community/go-nix/pkg/narinfo"
@ -123,10 +124,10 @@ func registerNarinfoPut(s *Server) {
// We need to add the basename of the storepath from the .narinfo
// to the pathInfo to be sent.
switch v := (pathInfo.GetNode().GetNode()).(type) {
case *storev1pb.Node_File:
pathInfoToUpload.Node = &storev1pb.Node{
Node: &storev1pb.Node_File{
File: &storev1pb.FileNode{
case *castorev1pb.Node_File:
pathInfoToUpload.Node = &castorev1pb.Node{
Node: &castorev1pb.Node_File{
File: &castorev1pb.FileNode{
Name: []byte(path.Base(narInfo.StorePath)),
Digest: v.File.Digest,
Size: v.File.Size,
@ -134,19 +135,19 @@ func registerNarinfoPut(s *Server) {
},
},
}
case *storev1pb.Node_Symlink:
pathInfoToUpload.Node = &storev1pb.Node{
Node: &storev1pb.Node_Symlink{
Symlink: &storev1pb.SymlinkNode{
case *castorev1pb.Node_Symlink:
pathInfoToUpload.Node = &castorev1pb.Node{
Node: &castorev1pb.Node_Symlink{
Symlink: &castorev1pb.SymlinkNode{
Name: []byte(path.Base(narInfo.StorePath)),
Target: v.Symlink.Target,
},
},
}
case *storev1pb.Node_Directory:
pathInfoToUpload.Node = &storev1pb.Node{
Node: &storev1pb.Node_Directory{
Directory: &storev1pb.DirectoryNode{
case *castorev1pb.Node_Directory:
pathInfoToUpload.Node = &castorev1pb.Node{
Node: &castorev1pb.Node_Directory{
Directory: &castorev1pb.DirectoryNode{
Name: []byte(path.Base(narInfo.StorePath)),
Digest: v.Directory.Digest,
Size: v.Directory.Size,

9
tvix/nar-bridge/pkg/server/server.go
View file

@ -7,6 +7,7 @@ import (
"sync"
"time"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
storev1pb "code.tvl.fyi/tvix/store/protos"
"github.com/go-chi/chi/middleware"
"github.com/go-chi/chi/v5"
@ -17,8 +18,8 @@ type Server struct {
srv *http.Server
handler chi.Router
directoryServiceClient storev1pb.DirectoryServiceClient
blobServiceClient storev1pb.BlobServiceClient
directoryServiceClient castorev1pb.DirectoryServiceClient
blobServiceClient castorev1pb.BlobServiceClient
pathInfoServiceClient storev1pb.PathInfoServiceClient
// When uploading NAR files to a HTTP binary cache, the .nar
@ -32,8 +33,8 @@ type Server struct {
}
func New(
directoryServiceClient storev1pb.DirectoryServiceClient,
blobServiceClient storev1pb.BlobServiceClient,
directoryServiceClient castorev1pb.DirectoryServiceClient,
blobServiceClient castorev1pb.BlobServiceClient,
pathInfoServiceClient storev1pb.PathInfoServiceClient,
enableAccessLog bool,
priority int,

27
tvix/nar-bridge/pkg/writer/writer.go
View file

@ -5,11 +5,12 @@ import (
"io"
"path"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
storev1pb "code.tvl.fyi/tvix/store/protos"
"github.com/nix-community/go-nix/pkg/nar"
)
type DirectoryLookupFn func([]byte) (*storev1pb.Directory, error)
type DirectoryLookupFn func([]byte) (*castorev1pb.Directory, error)
type BlobLookupFn func([]byte) (io.ReadCloser, error)
// Export will traverse a given pathInfo structure, and write the contents
@ -38,13 +39,13 @@ func Export(
// and emit individual elements to the NAR writer, draining the Directory object.
// once it's empty, we can pop it off the stack.
var stackPaths = []string{}
var stackDirectories = []*storev1pb.Directory{}
var stackDirectories = []*castorev1pb.Directory{}
// peek at the pathInfo root and assemble the root node and write to writer
// in the case of a regular file, we retrieve and write the contents, close and exit
// in the case of a symlink, we write the symlink, close and exit
switch v := (pathInfo.GetNode().GetNode()).(type) {
case *storev1pb.Node_File:
case *castorev1pb.Node_File:
rootHeader.Type = nar.TypeRegular
rootHeader.Size = int64(v.File.GetSize())
rootHeader.Executable = v.File.GetExecutable()
@ -77,7 +78,7 @@ func Export(
return nil
case *storev1pb.Node_Symlink:
case *castorev1pb.Node_Symlink:
rootHeader.Type = nar.TypeSymlink
rootHeader.LinkTarget = string(v.Symlink.GetTarget())
err := narWriter.WriteHeader(rootHeader)
@ -91,7 +92,7 @@ func Export(
}
return nil
case *storev1pb.Node_Directory:
case *castorev1pb.Node_Directory:
// We have a directory at the root, look it up and put in on the stack.
directory, err := directoryLookupFn(v.Directory.Digest)
if err != nil {
@ -136,7 +137,7 @@ func Export(
}
switch n := (nextNode).(type) {
case *storev1pb.DirectoryNode:
case *castorev1pb.DirectoryNode:
err := narWriter.WriteHeader(&nar.Header{
Path: path.Join(topOfStackPath, string(n.GetName())),
Type: nar.TypeDirectory,
@ -153,7 +154,7 @@ func Export(
// add to stack
stackDirectories = append(stackDirectories, d)
stackPaths = append(stackPaths, path.Join(topOfStackPath, string(n.GetName())))
case *storev1pb.FileNode:
case *castorev1pb.FileNode:
err := narWriter.WriteHeader(&nar.Header{
Path: path.Join(topOfStackPath, string(n.GetName())),
Type: nar.TypeRegular,
@ -180,7 +181,7 @@ func Export(
if err != nil {
return fmt.Errorf("unable to close content reader: %w", err)
}
case *storev1pb.SymlinkNode:
case *castorev1pb.SymlinkNode:
err := narWriter.WriteHeader(&nar.Header{
Path: path.Join(topOfStackPath, string(n.GetName())),
Type: nar.TypeSymlink,
@ -198,15 +199,15 @@ func Export(
// drainNextNode will drain a directory message with one of its child nodes,
// whichever comes first alphabetically.
func drainNextNode(d *storev1pb.Directory) interface{} {
func drainNextNode(d *castorev1pb.Directory) interface{} {
switch v := (smallestNode(d)).(type) {
case *storev1pb.DirectoryNode:
case *castorev1pb.DirectoryNode:
d.Directories = d.Directories[1:]
return v
case *storev1pb.FileNode:
case *castorev1pb.FileNode:
d.Files = d.Files[1:]
return v
case *storev1pb.SymlinkNode:
case *castorev1pb.SymlinkNode:
d.Symlinks = d.Symlinks[1:]
return v
case nil:
@ -218,7 +219,7 @@ func drainNextNode(d *storev1pb.Directory) interface{} {
// smallestNode will return the node from a directory message,
// whichever comes first alphabetically.
func smallestNode(d *storev1pb.Directory) interface{} {
func smallestNode(d *castorev1pb.Directory) interface{} {
childDirectories := d.GetDirectories()
childFiles := d.GetFiles()
childSymlinks := d.GetSymlinks()

20
tvix/nar-bridge/pkg/writer/writer_pick_next_node_test.go
View file

@ -3,7 +3,7 @@ package writer
import (
"testing"
storev1pb "code.tvl.fyi/tvix/store/protos"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/testing/protocmp"
@ -17,18 +17,18 @@ func requireProtoEq(t *testing.T, expected interface{}, actual interface{}) {
func TestPopNextNode(t *testing.T) {
t.Run("empty directory", func(t *testing.T) {
d := &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
d := &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
n := drainNextNode(d)
require.Equal(t, nil, n)
})
t.Run("only directories", func(t *testing.T) {
ds := &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{{
ds := &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{{
Name: []byte("a"),
Digest: []byte{},
Size: 0,
@ -37,12 +37,12 @@ func TestPopNextNode(t *testing.T) {
Digest: []byte{},
Size: 0,
}},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
n := drainNextNode(ds)
requireProtoEq(t, &storev1pb.DirectoryNode{
requireProtoEq(t, &castorev1pb.DirectoryNode{
Name: []byte("a"),
Digest: []byte{},
Size: 0,

41
tvix/nar-bridge/pkg/writer/writer_test.go
View file

@ -8,6 +8,7 @@ import (
"os"
"testing"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
"code.tvl.fyi/tvix/nar-bridge/pkg/reader"
"code.tvl.fyi/tvix/nar-bridge/pkg/writer"
storev1pb "code.tvl.fyi/tvix/store/protos"
@ -15,7 +16,7 @@ import (
"lukechampine.com/blake3"
)
func mustDigest(d *storev1pb.Directory) []byte {
func mustDigest(d *castorev1pb.Directory) []byte {
dgst, err := d.Digest()
if err != nil {
panic(err)
@ -26,9 +27,9 @@ func mustDigest(d *storev1pb.Directory) []byte {
func TestSymlink(t *testing.T) {
pathInfo := &storev1pb.PathInfo{
Node: &storev1pb.Node{
Node: &storev1pb.Node_Symlink{
Symlink: &storev1pb.SymlinkNode{
Node: &castorev1pb.Node{
Node: &castorev1pb.Node_Symlink{
Symlink: &castorev1pb.SymlinkNode{
Name: []byte("doesntmatter"),
Target: []byte("/nix/store/somewhereelse"),
},
@ -38,7 +39,7 @@ func TestSymlink(t *testing.T) {
var buf bytes.Buffer
err := writer.Export(&buf, pathInfo, func([]byte) (*storev1pb.Directory, error) {
err := writer.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) {
panic("no directories expected")
}, func([]byte) (io.ReadCloser, error) {
panic("no files expected")
@ -65,9 +66,9 @@ func TestRegular(t *testing.T) {
}
pathInfo := &storev1pb.PathInfo{
Node: &storev1pb.Node{
Node: &storev1pb.Node_File{
File: &storev1pb.FileNode{
Node: &castorev1pb.Node{
Node: &castorev1pb.Node_File{
File: &castorev1pb.FileNode{
Name: []byte("doesntmatter"),
Digest: BLAKE3_DIGEST_0X01,
Size: 1,
@ -79,7 +80,7 @@ func TestRegular(t *testing.T) {
var buf bytes.Buffer
err := writer.Export(&buf, pathInfo, func([]byte) (*storev1pb.Directory, error) {
err := writer.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) {
panic("no directories expected")
}, func(blobRef []byte) (io.ReadCloser, error) {
if !bytes.Equal(blobRef, BLAKE3_DIGEST_0X01) {
@ -102,17 +103,17 @@ func TestRegular(t *testing.T) {
func TestEmptyDirectory(t *testing.T) {
// construct empty directory node this refers to
emptyDirectory := &storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
emptyDirectory := &castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
emptyDirectoryDigest := mustDigest(emptyDirectory)
pathInfo := &storev1pb.PathInfo{
Node: &storev1pb.Node{
Node: &storev1pb.Node_Directory{
Directory: &storev1pb.DirectoryNode{
Node: &castorev1pb.Node{
Node: &castorev1pb.Node_Directory{
Directory: &castorev1pb.DirectoryNode{
Name: []byte("doesntmatter"),
Digest: emptyDirectoryDigest,
Size: 0,
@ -123,7 +124,7 @@ func TestEmptyDirectory(t *testing.T) {
var buf bytes.Buffer
err := writer.Export(&buf, pathInfo, func(directoryRef []byte) (*storev1pb.Directory, error) {
err := writer.Export(&buf, pathInfo, func(directoryRef []byte) (*castorev1pb.Directory, error) {
if !bytes.Equal(directoryRef, emptyDirectoryDigest) {
panic("unexpected directoryRef")
}
@ -156,7 +157,7 @@ func TestFull(t *testing.T) {
require.NoError(t, err)
filesMap := make(map[string][]byte, 0)
directoriesMap := make(map[string]*storev1pb.Directory)
directoriesMap := make(map[string]*castorev1pb.Directory)
r := reader.New(bytes.NewBuffer(narContents))
pathInfo, err := r.Import(
@ -174,7 +175,7 @@ func TestFull(t *testing.T) {
return nil
},
func(directory *storev1pb.Directory) error {
func(directory *castorev1pb.Directory) error {
dgst := mustDigest(directory)
directoriesMap[hex.EncodeToString(dgst)] = directory
@ -190,7 +191,7 @@ func TestFull(t *testing.T) {
err = writer.Export(
&buf,
pathInfo,
func(directoryRef []byte) (*storev1pb.Directory, error) {
func(directoryRef []byte) (*castorev1pb.Directory, error) {
d, found := directoriesMap[hex.EncodeToString(directoryRef)]
if !found {
panic("directories not found")