chore(tvix/store/protof): buf format

Change-Id: I29b0fb62aa849ee01860dc7fe3dbba9b941ca172
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10238
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: Adam Joseph <adam@westernsemico.com>
This commit is contained in:
Florian Klink 2023-12-09 15:23:29 +02:00 committed by clbot
parent 459d9e106f
commit 3c0a9a949a
6 changed files with 158 additions and 158 deletions

View file

@ -249,11 +249,11 @@ type NARInfo struct {
NarSha256 []byte `protobuf:"bytes,2,opt,name=nar_sha256,json=narSha256,proto3" json:"nar_sha256,omitempty"` NarSha256 []byte `protobuf:"bytes,2,opt,name=nar_sha256,json=narSha256,proto3" json:"nar_sha256,omitempty"`
// The signatures in a .narinfo file. // The signatures in a .narinfo file.
Signatures []*NARInfo_Signature `protobuf:"bytes,3,rep,name=signatures,proto3" json:"signatures,omitempty"` Signatures []*NARInfo_Signature `protobuf:"bytes,3,rep,name=signatures,proto3" json:"signatures,omitempty"`
// A list of references. To validate .narinfo signatures, a fingerprint // A list of references. To validate .narinfo signatures, a fingerprint needs
// needs to be constructed. // to be constructed.
// This fingerprint doesn't just contain the hashes of the output paths of // This fingerprint doesn't just contain the hashes of the output paths of all
// all references (like PathInfo.references), but their whole (base)names, // references (like PathInfo.references), but their whole (base)names, so we
// so we need to keep them somewhere. // need to keep them somewhere.
ReferenceNames []string `protobuf:"bytes,4,rep,name=reference_names,json=referenceNames,proto3" json:"reference_names,omitempty"` ReferenceNames []string `protobuf:"bytes,4,rep,name=reference_names,json=referenceNames,proto3" json:"reference_names,omitempty"`
// The StorePath of the .drv file producing this output. // The StorePath of the .drv file producing this output.
// The .drv suffix is omitted in its `name` field. // The .drv suffix is omitted in its `name` field.
@ -278,8 +278,8 @@ type NARInfo struct {
// There are some restrictions on the possible combinations. // There are some restrictions on the possible combinations.
// For example, `text` and `fixed:recursive` always imply sha256. // For example, `text` and `fixed:recursive` always imply sha256.
// //
// We use an enum to encode the possible combinations, and optimize // We use an enum to encode the possible combinations, and optimize for the
// for the common case, `fixed:recursive`, identified as `NAR_SHA256`. // common case, `fixed:recursive`, identified as `NAR_SHA256`.
Ca *NARInfo_CA `protobuf:"bytes,6,opt,name=ca,proto3" json:"ca,omitempty"` Ca *NARInfo_CA `protobuf:"bytes,6,opt,name=ca,proto3" json:"ca,omitempty"`
} }

View file

@ -204,11 +204,11 @@ var file_tvix_store_protos_rpc_pathinfo_proto_rawDesc = []byte{
0x0a, 0x24, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x0a, 0x24, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x69, 0x6e, 0x66, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x69, 0x6e, 0x66, 0x6f,
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f,
0x72, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x20, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74,
0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x70, 0x61, 0x74, 0x68, 0x69, 0x6e, 0x66, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f,
0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x20, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73,
0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x70, 0x61, 0x74, 0x68,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x47, 0x0a, 0x12, 0x47, 0x65, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x47, 0x0a, 0x12, 0x47, 0x65,
0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x12, 0x26, 0x0a, 0x0e, 0x62, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, 0x12, 0x26, 0x0a, 0x0e, 0x62, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61,
0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0c, 0x62, 0x79, 0x4f, 0x75, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0c, 0x62, 0x79, 0x4f, 0x75,

View file

@ -45,22 +45,22 @@ type PathInfoServiceClient interface {
// Uploading clients SHOULD obviously not steer other machines to try to // Uploading clients SHOULD obviously not steer other machines to try to
// substitute before from the remote end before having finished uploading // substitute before from the remote end before having finished uploading
// PathInfo, Directories and Blobs. // PathInfo, Directories and Blobs.
// The returned PathInfo object MAY contain additional narinfo signatures, // The returned PathInfo object MAY contain additional narinfo signatures, but
// but is otherwise left untouched. // is otherwise left untouched.
Put(ctx context.Context, in *PathInfo, opts ...grpc.CallOption) (*PathInfo, error) Put(ctx context.Context, in *PathInfo, opts ...grpc.CallOption) (*PathInfo, error)
// Calculate the NAR representation of the contents specified by the // Calculate the NAR representation of the contents specified by the
// root_node. The calculation SHOULD be cached server-side for subsequent // root_node. The calculation SHOULD be cached server-side for subsequent
// requests. // requests.
// //
// All references (to blobs or Directory messages) MUST already exist in // All references (to blobs or Directory messages) MUST already exist in the
// the store. // store.
// //
// The method can be used to produce a Nix fixed-output path, which // The method can be used to produce a Nix fixed-output path, which contains
// contains the (compressed) sha256 of the NAR content representation in // the (compressed) sha256 of the NAR content representation in the root_node
// the root_node name (suffixed with the name). // name (suffixed with the name).
// //
// It can also be used to calculate arbitrary NAR hashes of output paths, // It can also be used to calculate arbitrary NAR hashes of output paths, in
// in case a legacy Nix Binary Cache frontend is provided. // case a legacy Nix Binary Cache frontend is provided.
CalculateNAR(ctx context.Context, in *castore_go.Node, opts ...grpc.CallOption) (*CalculateNARResponse, error) CalculateNAR(ctx context.Context, in *castore_go.Node, opts ...grpc.CallOption) (*CalculateNARResponse, error)
// Return a stream of PathInfo messages matching the criteria specified in // Return a stream of PathInfo messages matching the criteria specified in
// ListPathInfoRequest. // ListPathInfoRequest.
@ -150,22 +150,22 @@ type PathInfoServiceServer interface {
// Uploading clients SHOULD obviously not steer other machines to try to // Uploading clients SHOULD obviously not steer other machines to try to
// substitute before from the remote end before having finished uploading // substitute before from the remote end before having finished uploading
// PathInfo, Directories and Blobs. // PathInfo, Directories and Blobs.
// The returned PathInfo object MAY contain additional narinfo signatures, // The returned PathInfo object MAY contain additional narinfo signatures, but
// but is otherwise left untouched. // is otherwise left untouched.
Put(context.Context, *PathInfo) (*PathInfo, error) Put(context.Context, *PathInfo) (*PathInfo, error)
// Calculate the NAR representation of the contents specified by the // Calculate the NAR representation of the contents specified by the
// root_node. The calculation SHOULD be cached server-side for subsequent // root_node. The calculation SHOULD be cached server-side for subsequent
// requests. // requests.
// //
// All references (to blobs or Directory messages) MUST already exist in // All references (to blobs or Directory messages) MUST already exist in the
// the store. // store.
// //
// The method can be used to produce a Nix fixed-output path, which // The method can be used to produce a Nix fixed-output path, which contains
// contains the (compressed) sha256 of the NAR content representation in // the (compressed) sha256 of the NAR content representation in the root_node
// the root_node name (suffixed with the name). // name (suffixed with the name).
// //
// It can also be used to calculate arbitrary NAR hashes of output paths, // It can also be used to calculate arbitrary NAR hashes of output paths, in
// in case a legacy Nix Binary Cache frontend is provided. // case a legacy Nix Binary Cache frontend is provided.
CalculateNAR(context.Context, *castore_go.Node) (*CalculateNARResponse, error) CalculateNAR(context.Context, *castore_go.Node) (*CalculateNARResponse, error)
// Return a stream of PathInfo messages matching the criteria specified in // Return a stream of PathInfo messages matching the criteria specified in
// ListPathInfoRequest. // ListPathInfoRequest.

View file

@ -25,6 +25,7 @@
buildPhase = '' buildPhase = ''
export HOME=$TMPDIR export HOME=$TMPDIR
buf lint buf lint
buf format -d --exit-code
buf generate buf generate
mkdir -p $out mkdir -p $out

View file

@ -11,27 +11,27 @@ option go_package = "code.tvl.fyi/tvix/store-go;storev1";
// PathInfo shows information about a Nix Store Path. // PathInfo shows information about a Nix Store Path.
// That's a single element inside /nix/store. // That's a single element inside /nix/store.
message PathInfo { message PathInfo {
// The path can be a directory, file or symlink. // The path can be a directory, file or symlink.
tvix.castore.v1.Node node = 1; tvix.castore.v1.Node node = 1;
// List of references (output path hashes) // List of references (output path hashes)
// This really is the raw *bytes*, after decoding nixbase32, and not a // This really is the raw *bytes*, after decoding nixbase32, and not a
// base32-encoded string. // base32-encoded string.
repeated bytes references = 2; repeated bytes references = 2;
// see below. // see below.
NARInfo narinfo = 3; NARInfo narinfo = 3;
} }
// Represents a path in the Nix store (a direct child of STORE_DIR). // Represents a path in the Nix store (a direct child of STORE_DIR).
// It is commonly formatted by a nixbase32-encoding the digest, and // It is commonly formatted by a nixbase32-encoding the digest, and
// concatenating the name, separated by a `-`. // concatenating the name, separated by a `-`.
message StorePath { message StorePath {
// The string after digest and `-`. // The string after digest and `-`.
string name = 1; string name = 1;
// The digest (20 bytes). // The digest (20 bytes).
bytes digest = 2; bytes digest = 2;
} }
// Nix C++ uses NAR (Nix Archive) as a format to transfer store paths, // Nix C++ uses NAR (Nix Archive) as a format to transfer store paths,
@ -44,85 +44,85 @@ message StorePath {
// moving to another signature scheme is desired. // moving to another signature scheme is desired.
// Even then, it still makes sense to hold this data, for old clients. // Even then, it still makes sense to hold this data, for old clients.
message NARInfo { message NARInfo {
// This represents a (parsed) signature line in a .narinfo file. // This represents a (parsed) signature line in a .narinfo file.
message Signature { message Signature {
string name = 1; string name = 1;
bytes data = 2; bytes data = 2;
}; }
// This size of the NAR file, in bytes. // This size of the NAR file, in bytes.
uint64 nar_size = 1; uint64 nar_size = 1;
// The sha256 of the NAR file representation. // The sha256 of the NAR file representation.
bytes nar_sha256 = 2; bytes nar_sha256 = 2;
// The signatures in a .narinfo file. // The signatures in a .narinfo file.
repeated Signature signatures = 3; repeated Signature signatures = 3;
// A list of references. To validate .narinfo signatures, a fingerprint // A list of references. To validate .narinfo signatures, a fingerprint needs
// needs to be constructed. // to be constructed.
// This fingerprint doesn't just contain the hashes of the output paths of // This fingerprint doesn't just contain the hashes of the output paths of all
// all references (like PathInfo.references), but their whole (base)names, // references (like PathInfo.references), but their whole (base)names, so we
// so we need to keep them somewhere. // need to keep them somewhere.
repeated string reference_names = 4; repeated string reference_names = 4;
// The StorePath of the .drv file producing this output. // The StorePath of the .drv file producing this output.
// The .drv suffix is omitted in its `name` field. // The .drv suffix is omitted in its `name` field.
StorePath deriver = 5; StorePath deriver = 5;
// The CA field in the .narinfo. // The CA field in the .narinfo.
// Its textual representations seen in the wild are one of the following: // Its textual representations seen in the wild are one of the following:
// - `fixed:r:sha256:1gcky5hlf5vqfzpyhihydmm54grhc94mcs8w7xr8613qsqb1v2j6` // - `fixed:r:sha256:1gcky5hlf5vqfzpyhihydmm54grhc94mcs8w7xr8613qsqb1v2j6`
// fixed-output derivations using "recursive" `outputHashMode`. // fixed-output derivations using "recursive" `outputHashMode`.
// - `fixed:sha256:19xqkh72crbcba7flwxyi3n293vav6d7qkzkh2v4zfyi4iia8vj8 // - `fixed:sha256:19xqkh72crbcba7flwxyi3n293vav6d7qkzkh2v4zfyi4iia8vj8
// fixed-output derivations using "flat" `outputHashMode` // fixed-output derivations using "flat" `outputHashMode`
// - `text:sha256:19xqkh72crbcba7flwxyi3n293vav6d7qkzkh2v4zfyi4iia8vj8` // - `text:sha256:19xqkh72crbcba7flwxyi3n293vav6d7qkzkh2v4zfyi4iia8vj8`
// Text hashing, used for uploaded .drv files and outputs produced by // Text hashing, used for uploaded .drv files and outputs produced by
// builtins.toFile. // builtins.toFile.
// //
// Semantically, they can be split into the following components: // Semantically, they can be split into the following components:
// - "content address prefix". Currently, "fixed" and "text" are supported. // - "content address prefix". Currently, "fixed" and "text" are supported.
// - "hash mode". Currently, "flat" and "recursive" are supported. // - "hash mode". Currently, "flat" and "recursive" are supported.
// - "hash type". The underlying hash function used. // - "hash type". The underlying hash function used.
// Currently, sha1, md5, sha256, sha512. // Currently, sha1, md5, sha256, sha512.
// - "digest". The digest itself. // - "digest". The digest itself.
// //
// There are some restrictions on the possible combinations. // There are some restrictions on the possible combinations.
// For example, `text` and `fixed:recursive` always imply sha256. // For example, `text` and `fixed:recursive` always imply sha256.
// //
// We use an enum to encode the possible combinations, and optimize // We use an enum to encode the possible combinations, and optimize for the
// for the common case, `fixed:recursive`, identified as `NAR_SHA256`. // common case, `fixed:recursive`, identified as `NAR_SHA256`.
CA ca = 6; CA ca = 6;
message CA { message CA {
enum Hash { enum Hash {
// produced when uploading fixed-output store paths using NAR-based // produced when uploading fixed-output store paths using NAR-based
// hashing (`outputHashMode = "recursive"`). // hashing (`outputHashMode = "recursive"`).
NAR_SHA256 = 0; NAR_SHA256 = 0;
NAR_SHA1 = 1; NAR_SHA1 = 1;
NAR_SHA512 = 2; NAR_SHA512 = 2;
NAR_MD5 = 3; NAR_MD5 = 3;
// Produced when uploading .drv files or outputs produced by // Produced when uploading .drv files or outputs produced by
// builtins.toFile. // builtins.toFile.
// Produces equivalent digests as FLAT_SHA256, but is a separate // Produces equivalent digests as FLAT_SHA256, but is a separate
// hashing type in Nix, affecting output path calculation. // hashing type in Nix, affecting output path calculation.
TEXT_SHA256 = 4; TEXT_SHA256 = 4;
// Produced when using fixed-output derivations with // Produced when using fixed-output derivations with
// `outputHashMode = "flat"`. // `outputHashMode = "flat"`.
FLAT_SHA1 = 5; FLAT_SHA1 = 5;
FLAT_MD5 = 6; FLAT_MD5 = 6;
FLAT_SHA256 = 7; FLAT_SHA256 = 7;
FLAT_SHA512 = 8; FLAT_SHA512 = 8;
// TODO: what happens in Rust if we introduce a new enum kind here? // TODO: what happens in Rust if we introduce a new enum kind here?
}
// The hashing type used.
Hash type = 1;
// The digest, in raw bytes.
bytes digest = 2;
} }
// The hashing type used.
Hash type = 1;
// The digest, in raw bytes.
bytes digest = 2;
}
} }

View file

@ -4,74 +4,73 @@ syntax = "proto3";
package tvix.store.v1; package tvix.store.v1;
import "tvix/store/protos/pathinfo.proto";
import "tvix/castore/protos/castore.proto"; import "tvix/castore/protos/castore.proto";
import "tvix/store/protos/pathinfo.proto";
option go_package = "code.tvl.fyi/tvix/store-go;storev1"; option go_package = "code.tvl.fyi/tvix/store-go;storev1";
service PathInfoService { service PathInfoService {
// Return a PathInfo message matching the criteria specified in the // Return a PathInfo message matching the criteria specified in the
// GetPathInfoRequest message. // GetPathInfoRequest message.
rpc Get(GetPathInfoRequest) returns (PathInfo); rpc Get(GetPathInfoRequest) returns (PathInfo);
// Upload a PathInfo object to the remote end. It MUST not return until the // Upload a PathInfo object to the remote end. It MUST not return until the
// PathInfo object has been written on the the remote end. // PathInfo object has been written on the the remote end.
// //
// The remote end MAY check if a potential DirectoryNode has already been // The remote end MAY check if a potential DirectoryNode has already been
// uploaded. // uploaded.
// //
// Uploading clients SHOULD obviously not steer other machines to try to // Uploading clients SHOULD obviously not steer other machines to try to
// substitute before from the remote end before having finished uploading // substitute before from the remote end before having finished uploading
// PathInfo, Directories and Blobs. // PathInfo, Directories and Blobs.
// The returned PathInfo object MAY contain additional narinfo signatures, // The returned PathInfo object MAY contain additional narinfo signatures, but
// but is otherwise left untouched. // is otherwise left untouched.
rpc Put(PathInfo) returns (PathInfo); rpc Put(PathInfo) returns (PathInfo);
// Calculate the NAR representation of the contents specified by the
// root_node. The calculation SHOULD be cached server-side for subsequent
// requests.
//
// All references (to blobs or Directory messages) MUST already exist in the
// store.
//
// The method can be used to produce a Nix fixed-output path, which contains
// the (compressed) sha256 of the NAR content representation in the root_node
// name (suffixed with the name).
//
// It can also be used to calculate arbitrary NAR hashes of output paths, in
// case a legacy Nix Binary Cache frontend is provided.
rpc CalculateNAR(tvix.castore.v1.Node) returns (CalculateNARResponse);
// Calculate the NAR representation of the contents specified by the // Return a stream of PathInfo messages matching the criteria specified in
// root_node. The calculation SHOULD be cached server-side for subsequent // ListPathInfoRequest.
// requests. rpc List(ListPathInfoRequest) returns (stream PathInfo);
//
// All references (to blobs or Directory messages) MUST already exist in
// the store.
//
// The method can be used to produce a Nix fixed-output path, which
// contains the (compressed) sha256 of the NAR content representation in
// the root_node name (suffixed with the name).
//
// It can also be used to calculate arbitrary NAR hashes of output paths,
// in case a legacy Nix Binary Cache frontend is provided.
rpc CalculateNAR(tvix.castore.v1.Node) returns (CalculateNARResponse);
// Return a stream of PathInfo messages matching the criteria specified in
// ListPathInfoRequest.
rpc List(ListPathInfoRequest) returns (stream PathInfo);
} }
// The parameters that can be used to lookup a (single) PathInfo object. // The parameters that can be used to lookup a (single) PathInfo object.
// Currently, only a lookup by output hash is supported. // Currently, only a lookup by output hash is supported.
message GetPathInfoRequest { message GetPathInfoRequest {
oneof by_what { oneof by_what {
// The output hash of a nix path (20 bytes). // The output hash of a nix path (20 bytes).
// This is the nixbase32-decoded portion of a Nix output path, so to substitute // This is the nixbase32-decoded portion of a Nix output path, so to substitute
// /nix/store/xm35nga2g20mz5sm5l6n8v3bdm86yj83-cowsay-3.04 // /nix/store/xm35nga2g20mz5sm5l6n8v3bdm86yj83-cowsay-3.04
// this field would contain nixbase32dec("xm35nga2g20mz5sm5l6n8v3bdm86yj83"). // this field would contain nixbase32dec("xm35nga2g20mz5sm5l6n8v3bdm86yj83").
bytes by_output_hash = 1; bytes by_output_hash = 1;
}; }
} }
// The parameters that can be used to lookup (multiple) PathInfo objects. // The parameters that can be used to lookup (multiple) PathInfo objects.
// Currently no filtering is possible, all objects are returned. // Currently no filtering is possible, all objects are returned.
message ListPathInfoRequest { } message ListPathInfoRequest {}
// CalculateNARResponse is the response returned by the CalculateNAR request. // CalculateNARResponse is the response returned by the CalculateNAR request.
// //
// It contains the size of the NAR representation (in bytes), and the sha56 // It contains the size of the NAR representation (in bytes), and the sha56
// digest. // digest.
message CalculateNARResponse { message CalculateNARResponse {
// This size of the NAR file, in bytes. // This size of the NAR file, in bytes.
uint64 nar_size = 1; uint64 nar_size = 1;
// The sha256 of the NAR file representation. // The sha256 of the NAR file representation.
bytes nar_sha256 = 2; bytes nar_sha256 = 2;
} }