0ecd10bf30
This provides a Nix HTTP Binary Cache interface in front of a tvix-store that's reachable via gRPC. TODOs: - remove import command, move serve up to toplevel. We have nix-copy- closure and tvix-store commands. - loop into CI. We should be able to fetch the protos as a third-party dependency. - Check if we can test nar-bridge slightly easier in an integration test. - Ensure we support connecting to unix sockets and grpc+http at least, using the same syntax as tvix-store. - Don't buffer the entire blob when rendering NAR Co-Authored-By: Connor Brewster <cbrewster@hey.com> Co-Authored-By: Márton Boros <martonboros@gmail.com> Co-Authored-By: Vo Minh Thu <noteed@gmail.com> Change-Id: I6064474e49dfe78cea67676957462d9f28658d4a Reviewed-on: https://cl.tvl.fyi/c/depot/+/9339 Tested-by: BuildkiteCI Reviewed-by: tazjin <tazjin@tvl.su>
48 lines
1.3 KiB
Go
48 lines
1.3 KiB
Go
package server
|
|
|
|
import (
|
|
storev1pb "code.tvl.fyi/tvix/store/protos"
|
|
"context"
|
|
"encoding/base64"
|
|
"fmt"
|
|
log "github.com/sirupsen/logrus"
|
|
"io"
|
|
)
|
|
|
|
// this returns a callback function that can be used as fileCb
|
|
// for the reader.Import function call
|
|
func genBlobServiceWriteCb(ctx context.Context, blobServiceClient storev1pb.BlobServiceClient) func(io.Reader) error {
|
|
return func(fileReader io.Reader) error {
|
|
// Read from fileReader into a buffer.
|
|
// We currently buffer all contents and send them to blobServiceClient at once,
|
|
// but that's about to change.
|
|
contents, err := io.ReadAll(fileReader)
|
|
if err != nil {
|
|
return fmt.Errorf("unable to read all contents from file reader: %w", err)
|
|
}
|
|
|
|
log := log.WithField("blob_size", len(contents))
|
|
|
|
log.Infof("about to upload blob")
|
|
|
|
putter, err := blobServiceClient.Put(ctx)
|
|
if err != nil {
|
|
// return error to the importer
|
|
return fmt.Errorf("error from blob service: %w", err)
|
|
}
|
|
err = putter.Send(&storev1pb.BlobChunk{
|
|
Data: contents,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("putting blob chunk: %w", err)
|
|
}
|
|
resp, err := putter.CloseAndRecv()
|
|
if err != nil {
|
|
return fmt.Errorf("close blob putter: %w", err)
|
|
}
|
|
|
|
log.WithField("digest", base64.StdEncoding.EncodeToString(resp.GetDigest())).Info("uploaded blob")
|
|
|
|
return nil
|
|
}
|
|
}
|