mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore(tvix/store/protos): add PathInfoService::CalculateNAR()

Browse source 
Expose the NAR calculation to a separate `CalculateNAR` method, which
responds with the NAR size and sha256 hash.

Contrary to what cl/7618 and cl/7620 initially did, don't add different
other request types.

In the CalculateNARResponse message, there's now some duplication in the
(optional) `narinfo` field of a PathInfo, but I'm not entirely sure if
we want to drop the fields from there yet.

Change-Id: Id797c56e17efedac115fbd43de9dfde9fa1db140
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7663
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2022-12-28 14:06:02 +01:00 committed by flokli
parent ce7be00994
commit d973c9772d
3 changed files with 223 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

130
tvix/store/protos/rpc_pathinfo.pb.go
View file

@ -97,6 +97,67 @@ type GetPathInfoRequest_ByOutputHash struct {
func (*GetPathInfoRequest_ByOutputHash) isGetPathInfoRequest_ByWhat() {}
// CalculateNARResponse is the response returned by the CalculateNAR request.
//
// It contains the size of the NAR representation (in bytes), and the sha56
// digest.
type CalculateNARResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// This size of the NAR file, in bytes.
NarSize uint32 `protobuf:"varint,1,opt,name=nar_size,json=narSize,proto3" json:"nar_size,omitempty"`
// The sha256 of the NAR file representation.
NarSha256 []byte `protobuf:"bytes,2,opt,name=nar_sha256,json=narSha256,proto3" json:"nar_sha256,omitempty"`
}
func (x *CalculateNARResponse) Reset() {
*x = CalculateNARResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *CalculateNARResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*CalculateNARResponse) ProtoMessage() {}
func (x *CalculateNARResponse) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use CalculateNARResponse.ProtoReflect.Descriptor instead.
func (*CalculateNARResponse) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_rpc_pathinfo_proto_rawDescGZIP(), []int{1}
}
func (x *CalculateNARResponse) GetNarSize() uint32 {
if x != nil {
return x.NarSize
}
return 0
}
func (x *CalculateNARResponse) GetNarSha256() []byte {
if x != nil {
return x.NarSha256
}
return nil
}
var File_tvix_store_protos_rpc_pathinfo_proto protoreflect.FileDescriptor
var file_tvix_store_protos_rpc_pathinfo_proto_rawDesc = []byte{
@ -110,19 +171,28 @@ var file_tvix_store_protos_rpc_pathinfo_proto_rawDesc = []byte{
0x0e, 0x62, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18,
0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0c, 0x62, 0x79, 0x4f, 0x75, 0x74, 0x70, 0x75,
0x74, 0x48, 0x61, 0x73, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x62, 0x79, 0x5f, 0x77, 0x68, 0x61, 0x74,
0x32, 0x8d, 0x01, 0x0a, 0x0f, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x65, 0x72,
0x76, 0x69, 0x63, 0x65, 0x12, 0x41, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x21, 0x2e, 0x74, 0x76,
0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50,
0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17,
0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50,
0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x17,
0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50,
0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f,
0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69,
0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
0x22, 0x50, 0x0a, 0x14, 0x43, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f,
0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53,
0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35,
0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32,
0x35, 0x36, 0x32, 0xd7, 0x01, 0x0a, 0x0f, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x53,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x41, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x21, 0x2e,
0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65,
0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31,
0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x03, 0x50, 0x75, 0x74,
0x12, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31,
0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78,
0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e,
0x66, 0x6f, 0x12, 0x48, 0x0a, 0x0c, 0x43, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e,
0x41, 0x52, 0x12, 0x13, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e,
0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x1a, 0x23, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74,
0x65, 0x4e, 0x41, 0x52, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x28, 0x5a, 0x26,
0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69,
0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@ -137,18 +207,22 @@ func file_tvix_store_protos_rpc_pathinfo_proto_rawDescGZIP() []byte {
return file_tvix_store_protos_rpc_pathinfo_proto_rawDescData
}
var file_tvix_store_protos_rpc_pathinfo_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
var file_tvix_store_protos_rpc_pathinfo_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_tvix_store_protos_rpc_pathinfo_proto_goTypes = []interface{}{
(*GetPathInfoRequest)(nil), // 0: tvix.store.v1.GetPathInfoRequest
(*PathInfo)(nil), // 1: tvix.store.v1.PathInfo
(*GetPathInfoRequest)(nil), // 0: tvix.store.v1.GetPathInfoRequest
(*CalculateNARResponse)(nil), // 1: tvix.store.v1.CalculateNARResponse
(*PathInfo)(nil), // 2: tvix.store.v1.PathInfo
(*Node)(nil), // 3: tvix.store.v1.Node
}
var file_tvix_store_protos_rpc_pathinfo_proto_depIdxs = []int32{
0, // 0: tvix.store.v1.PathInfoService.Get:input_type -> tvix.store.v1.GetPathInfoRequest
1, // 1: tvix.store.v1.PathInfoService.Put:input_type -> tvix.store.v1.PathInfo
1, // 2: tvix.store.v1.PathInfoService.Get:output_type -> tvix.store.v1.PathInfo
1, // 3: tvix.store.v1.PathInfoService.Put:output_type -> tvix.store.v1.PathInfo
2, // [2:4] is the sub-list for method output_type
0, // [0:2] is the sub-list for method input_type
2, // 1: tvix.store.v1.PathInfoService.Put:input_type -> tvix.store.v1.PathInfo
3, // 2: tvix.store.v1.PathInfoService.CalculateNAR:input_type -> tvix.store.v1.Node
2, // 3: tvix.store.v1.PathInfoService.Get:output_type -> tvix.store.v1.PathInfo
2, // 4: tvix.store.v1.PathInfoService.Put:output_type -> tvix.store.v1.PathInfo
1, // 5: tvix.store.v1.PathInfoService.CalculateNAR:output_type -> tvix.store.v1.CalculateNARResponse
3, // [3:6] is the sub-list for method output_type
0, // [0:3] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
@ -173,6 +247,18 @@ func file_tvix_store_protos_rpc_pathinfo_proto_init() {
return nil
}
}
file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*CalculateNARResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
file_tvix_store_protos_rpc_pathinfo_proto_msgTypes[0].OneofWrappers = []interface{}{
(*GetPathInfoRequest_ByOutputHash)(nil),
@ -183,7 +269,7 @@ func file_tvix_store_protos_rpc_pathinfo_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_tvix_store_protos_rpc_pathinfo_proto_rawDesc,
NumEnums: 0,
NumMessages: 1,
NumMessages: 2,
NumExtensions: 0,
NumServices: 1,
},

45
tvix/store/protos/rpc_pathinfo.proto
View file

@ -9,23 +9,42 @@ import "tvix/store/protos/pathinfo.proto";
option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
service PathInfoService {
// Get retrieves a PathInfo object, by using the lookup parameters in
// GetPathInfoRequest.
// If the PathInfo object contains a DirectoryNode, it needs to be looked
// up separately via the DirectoryService, which is purely
// content-addressed.
// Return a PathInfo message, identified by the decoded nixbase32 part
// of a Nix output path.
//
// To substitute /nix/store/xm35nga2g20mz5sm5l6n8v3bdm86yj83-
// cowsay-3.04 the bytes in the request would be
// nixbase32dec("xm35nga2g20mz5sm5l6n8v3bdm86yj83").
rpc Get(GetPathInfoRequest) returns (PathInfo);
// Put uploads a PathInfo object to the remote end. It MUST not return
// until the PathInfo object has been written on the the remote end.
//
// The remote end MAY check if a potential DirectoryNode has already been
// uploaded.
//
// Uploading clients SHOULD obviously not steer other machines to try to
// substitute before from the remote end before having finished uploading
// PathInfo, Directories and Blobs.
// The returned PathInfo object MAY contain additional narinfo signatures,
// but is otherwise left untouched.
rpc Put(PathInfo) returns (PathInfo);
// Calculate the NAR representation of the contents specified by the
// root_node. The calculation SHOULD be cached server-side for subsequent
// requests.
//
// All references (to blobs or Directory messages) MUST already exist in
// the store.
//
// The method can be used to produce a Nix fixed-output path, which
// contains the (compressed) sha256 of the NAR content representation in
// the root_node name (suffixed with the name).
//
// It can also be used to calculate arbitrary NAR hashes of output paths,
// in case a legacy Nix Binary Cache frontend is provided.
rpc CalculateNAR(Node) returns (CalculateNARResponse);
}
// GetPathInfoRequest describes the lookup parameters that can be used to
@ -38,7 +57,17 @@ message GetPathInfoRequest {
// /nix/store/xm35nga2g20mz5sm5l6n8v3bdm86yj83-cowsay-3.04
// this field would contain nixbase32dec("xm35nga2g20mz5sm5l6n8v3bdm86yj83").
bytes by_output_hash = 1;
// placeholder: by_drv and output name?
}
};
}
// CalculateNARResponse is the response returned by the CalculateNAR request.
//
// It contains the size of the NAR representation (in bytes), and the sha56
// digest.
message CalculateNARResponse {
// This size of the NAR file, in bytes.
uint32 nar_size = 1;
// The sha256 of the NAR file representation.
bytes nar_sha256 = 2;
}

88
tvix/store/protos/rpc_pathinfo_grpc.pb.go
View file

@ -22,22 +22,39 @@ const _ = grpc.SupportPackageIsVersion7
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
type PathInfoServiceClient interface {
// Get retrieves a PathInfo object, by using the lookup parameters in
// GetPathInfoRequest.
// If the PathInfo object contains a DirectoryNode, it needs to be looked
// up separately via the DirectoryService, which is purely
// content-addressed.
// Return a PathInfo message, identified by the decoded nixbase32 part
// of a Nix output path.
//
// To substitute /nix/store/xm35nga2g20mz5sm5l6n8v3bdm86yj83-
// cowsay-3.04 the bytes in the request would be
// nixbase32dec("xm35nga2g20mz5sm5l6n8v3bdm86yj83").
Get(ctx context.Context, in *GetPathInfoRequest, opts ...grpc.CallOption) (*PathInfo, error)
// Put uploads a PathInfo object to the remote end. It MUST not return
// until the PathInfo object has been written on the the remote end.
//
// The remote end MAY check if a potential DirectoryNode has already been
// uploaded.
//
// Uploading clients SHOULD obviously not steer other machines to try to
// substitute before from the remote end before having finished uploading
// PathInfo, Directories and Blobs.
// The returned PathInfo object MAY contain additional narinfo signatures,
// but is otherwise left untouched.
Put(ctx context.Context, in *PathInfo, opts ...grpc.CallOption) (*PathInfo, error)
// Calculate the NAR representation of the contents specified by the
// root_node. The calculation SHOULD be cached server-side for subsequent
// requests.
//
// All references (to blobs or Directory messages) MUST already exist in
// the store.
//
// The method can be used to produce a Nix fixed-output path, which
// contains the (compressed) sha256 of the NAR content representation in
// the root_node name (suffixed with the name).
//
// It can also be used to calculate arbitrary NAR hashes of output paths,
// in case a legacy Nix Binary Cache frontend is provided.
CalculateNAR(ctx context.Context, in *Node, opts ...grpc.CallOption) (*CalculateNARResponse, error)
}
type pathInfoServiceClient struct {
@ -66,26 +83,52 @@ func (c *pathInfoServiceClient) Put(ctx context.Context, in *PathInfo, opts ...g
return out, nil
}
func (c *pathInfoServiceClient) CalculateNAR(ctx context.Context, in *Node, opts ...grpc.CallOption) (*CalculateNARResponse, error) {
out := new(CalculateNARResponse)
err := c.cc.Invoke(ctx, "/tvix.store.v1.PathInfoService/CalculateNAR", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// PathInfoServiceServer is the server API for PathInfoService service.
// All implementations must embed UnimplementedPathInfoServiceServer
// for forward compatibility
type PathInfoServiceServer interface {
// Get retrieves a PathInfo object, by using the lookup parameters in
// GetPathInfoRequest.
// If the PathInfo object contains a DirectoryNode, it needs to be looked
// up separately via the DirectoryService, which is purely
// content-addressed.
// Return a PathInfo message, identified by the decoded nixbase32 part
// of a Nix output path.
//
// To substitute /nix/store/xm35nga2g20mz5sm5l6n8v3bdm86yj83-
// cowsay-3.04 the bytes in the request would be
// nixbase32dec("xm35nga2g20mz5sm5l6n8v3bdm86yj83").
Get(context.Context, *GetPathInfoRequest) (*PathInfo, error)
// Put uploads a PathInfo object to the remote end. It MUST not return
// until the PathInfo object has been written on the the remote end.
//
// The remote end MAY check if a potential DirectoryNode has already been
// uploaded.
//
// Uploading clients SHOULD obviously not steer other machines to try to
// substitute before from the remote end before having finished uploading
// PathInfo, Directories and Blobs.
// The returned PathInfo object MAY contain additional narinfo signatures,
// but is otherwise left untouched.
Put(context.Context, *PathInfo) (*PathInfo, error)
// Calculate the NAR representation of the contents specified by the
// root_node. The calculation SHOULD be cached server-side for subsequent
// requests.
//
// All references (to blobs or Directory messages) MUST already exist in
// the store.
//
// The method can be used to produce a Nix fixed-output path, which
// contains the (compressed) sha256 of the NAR content representation in
// the root_node name (suffixed with the name).
//
// It can also be used to calculate arbitrary NAR hashes of output paths,
// in case a legacy Nix Binary Cache frontend is provided.
CalculateNAR(context.Context, *Node) (*CalculateNARResponse, error)
mustEmbedUnimplementedPathInfoServiceServer()
}
@ -99,6 +142,9 @@ func (UnimplementedPathInfoServiceServer) Get(context.Context, *GetPathInfoReque
func (UnimplementedPathInfoServiceServer) Put(context.Context, *PathInfo) (*PathInfo, error) {
return nil, status.Errorf(codes.Unimplemented, "method Put not implemented")
}
func (UnimplementedPathInfoServiceServer) CalculateNAR(context.Context, *Node) (*CalculateNARResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method CalculateNAR not implemented")
}
func (UnimplementedPathInfoServiceServer) mustEmbedUnimplementedPathInfoServiceServer() {}
// UnsafePathInfoServiceServer may be embedded to opt out of forward compatibility for this service.
@ -148,6 +194,24 @@ func _PathInfoService_Put_Handler(srv interface{}, ctx context.Context, dec func
return interceptor(ctx, in, info, handler)
}
func _PathInfoService_CalculateNAR_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(Node)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(PathInfoServiceServer).CalculateNAR(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/tvix.store.v1.PathInfoService/CalculateNAR",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PathInfoServiceServer).CalculateNAR(ctx, req.(*Node))
}
return interceptor(ctx, in, info, handler)
}
// PathInfoService_ServiceDesc is the grpc.ServiceDesc for PathInfoService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@ -163,6 +227,10 @@ var PathInfoService_ServiceDesc = grpc.ServiceDesc{
MethodName: "Put",
Handler: _PathInfoService_Put_Handler,
},
{
MethodName: "CalculateNAR",
Handler: _PathInfoService_CalculateNAR_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "tvix/store/protos/rpc_pathinfo.proto",