feat(tvix/store/protos): rename Get to Read, add Stat method
Stat exposes metadata about a given blob, such as more granular chunking, baos. It implicitly allows checking for existence too, as asking this for a non-existing Blob will return a Status::not_found grpc error. The previous version returned a Status::not_found error on the Get request too, but there was no chance to prevent the server from starting to stream (except sending an immediate cancellation). Being able to check whether something exists in a BlobStore helps to prevent from uploading in first place. The granular chunking bits are an optional optimization - if the BlobStore implements no more granular chunking, the Stat response can simply contain a single chunk. Read returns a stream of BlobChunk, which is just a stream of bytes - not necessarily using the chunking that's returned in the reply of a Stat() call. It can be used to read blobs or chunks. Change-Id: I4b6030ef184ace5484c84ca273b49d710433731d Reviewed-on: https://cl.tvl.fyi/c/depot/+/7652 Reviewed-by: tazjin <tazjin@tvl.su> Tested-by: BuildkiteCI
This commit is contained in:
parent
1c15154b83
commit
c4ee942b1c
3 changed files with 441 additions and 95 deletions
|
@ -23,17 +23,28 @@ const (
|
||||||
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
|
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
|
||||||
)
|
)
|
||||||
|
|
||||||
type GetBlobRequest struct {
|
type StatBlobRequest struct {
|
||||||
state protoimpl.MessageState
|
state protoimpl.MessageState
|
||||||
sizeCache protoimpl.SizeCache
|
sizeCache protoimpl.SizeCache
|
||||||
unknownFields protoimpl.UnknownFields
|
unknownFields protoimpl.UnknownFields
|
||||||
|
|
||||||
// The blake3 digest of the blob requested
|
// The blake3 digest of the blob requested
|
||||||
Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"`
|
Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"`
|
||||||
|
// Whether to include the chunks field
|
||||||
|
IncludeChunks bool `protobuf:"varint,2,opt,name=include_chunks,json=includeChunks,proto3" json:"include_chunks,omitempty"`
|
||||||
|
// Whether to include the inline_bao field, containing an (outboard) bao.
|
||||||
|
// The [bao](https://github.com/oconnor663/bao/blob/master/docs/spec.md)
|
||||||
|
// can be used to validate chunks end up hashing to the same root digest.
|
||||||
|
// These only really matter when only downloading parts of a blob. Some
|
||||||
|
// caution needs to be applied when validating chunks - the bao works with
|
||||||
|
// 1K leaf nodes, which might not align with the chunk sizes - this might
|
||||||
|
// imply a neighboring chunk might need to be (partially) fetched to
|
||||||
|
// validate the hash.
|
||||||
|
IncludeBao bool `protobuf:"varint,3,opt,name=include_bao,json=includeBao,proto3" json:"include_bao,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *GetBlobRequest) Reset() {
|
func (x *StatBlobRequest) Reset() {
|
||||||
*x = GetBlobRequest{}
|
*x = StatBlobRequest{}
|
||||||
if protoimpl.UnsafeEnabled {
|
if protoimpl.UnsafeEnabled {
|
||||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0]
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0]
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
|
@ -41,13 +52,13 @@ func (x *GetBlobRequest) Reset() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *GetBlobRequest) String() string {
|
func (x *StatBlobRequest) String() string {
|
||||||
return protoimpl.X.MessageStringOf(x)
|
return protoimpl.X.MessageStringOf(x)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*GetBlobRequest) ProtoMessage() {}
|
func (*StatBlobRequest) ProtoMessage() {}
|
||||||
|
|
||||||
func (x *GetBlobRequest) ProtoReflect() protoreflect.Message {
|
func (x *StatBlobRequest) ProtoReflect() protoreflect.Message {
|
||||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0]
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0]
|
||||||
if protoimpl.UnsafeEnabled && x != nil {
|
if protoimpl.UnsafeEnabled && x != nil {
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
|
@ -59,18 +70,189 @@ func (x *GetBlobRequest) ProtoReflect() protoreflect.Message {
|
||||||
return mi.MessageOf(x)
|
return mi.MessageOf(x)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Deprecated: Use GetBlobRequest.ProtoReflect.Descriptor instead.
|
// Deprecated: Use StatBlobRequest.ProtoReflect.Descriptor instead.
|
||||||
func (*GetBlobRequest) Descriptor() ([]byte, []int) {
|
func (*StatBlobRequest) Descriptor() ([]byte, []int) {
|
||||||
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{0}
|
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{0}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *GetBlobRequest) GetDigest() []byte {
|
func (x *StatBlobRequest) GetDigest() []byte {
|
||||||
if x != nil {
|
if x != nil {
|
||||||
return x.Digest
|
return x.Digest
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (x *StatBlobRequest) GetIncludeChunks() bool {
|
||||||
|
if x != nil {
|
||||||
|
return x.IncludeChunks
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *StatBlobRequest) GetIncludeBao() bool {
|
||||||
|
if x != nil {
|
||||||
|
return x.IncludeBao
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// BlobMeta provides more granular chunking information for the requested blob,
|
||||||
|
// and baos.
|
||||||
|
type BlobMeta struct {
|
||||||
|
state protoimpl.MessageState
|
||||||
|
sizeCache protoimpl.SizeCache
|
||||||
|
unknownFields protoimpl.UnknownFields
|
||||||
|
|
||||||
|
// This provides a list of chunks.
|
||||||
|
// Concatenating their contents would produce a blob with the digest that
|
||||||
|
// was specified in the request.
|
||||||
|
Chunks []*BlobMeta_ChunkMeta `protobuf:"bytes,1,rep,name=chunks,proto3" json:"chunks,omitempty"`
|
||||||
|
InlineBao []byte `protobuf:"bytes,2,opt,name=inline_bao,json=inlineBao,proto3" json:"inline_bao,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *BlobMeta) Reset() {
|
||||||
|
*x = BlobMeta{}
|
||||||
|
if protoimpl.UnsafeEnabled {
|
||||||
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1]
|
||||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
|
ms.StoreMessageInfo(mi)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *BlobMeta) String() string {
|
||||||
|
return protoimpl.X.MessageStringOf(x)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*BlobMeta) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (x *BlobMeta) ProtoReflect() protoreflect.Message {
|
||||||
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1]
|
||||||
|
if protoimpl.UnsafeEnabled && x != nil {
|
||||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
|
if ms.LoadMessageInfo() == nil {
|
||||||
|
ms.StoreMessageInfo(mi)
|
||||||
|
}
|
||||||
|
return ms
|
||||||
|
}
|
||||||
|
return mi.MessageOf(x)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deprecated: Use BlobMeta.ProtoReflect.Descriptor instead.
|
||||||
|
func (*BlobMeta) Descriptor() ([]byte, []int) {
|
||||||
|
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *BlobMeta) GetChunks() []*BlobMeta_ChunkMeta {
|
||||||
|
if x != nil {
|
||||||
|
return x.Chunks
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *BlobMeta) GetInlineBao() []byte {
|
||||||
|
if x != nil {
|
||||||
|
return x.InlineBao
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type ReadBlobRequest struct {
|
||||||
|
state protoimpl.MessageState
|
||||||
|
sizeCache protoimpl.SizeCache
|
||||||
|
unknownFields protoimpl.UnknownFields
|
||||||
|
|
||||||
|
// The blake3 digest of the blob requested
|
||||||
|
Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *ReadBlobRequest) Reset() {
|
||||||
|
*x = ReadBlobRequest{}
|
||||||
|
if protoimpl.UnsafeEnabled {
|
||||||
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2]
|
||||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
|
ms.StoreMessageInfo(mi)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *ReadBlobRequest) String() string {
|
||||||
|
return protoimpl.X.MessageStringOf(x)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*ReadBlobRequest) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (x *ReadBlobRequest) ProtoReflect() protoreflect.Message {
|
||||||
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2]
|
||||||
|
if protoimpl.UnsafeEnabled && x != nil {
|
||||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
|
if ms.LoadMessageInfo() == nil {
|
||||||
|
ms.StoreMessageInfo(mi)
|
||||||
|
}
|
||||||
|
return ms
|
||||||
|
}
|
||||||
|
return mi.MessageOf(x)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deprecated: Use ReadBlobRequest.ProtoReflect.Descriptor instead.
|
||||||
|
func (*ReadBlobRequest) Descriptor() ([]byte, []int) {
|
||||||
|
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{2}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *ReadBlobRequest) GetDigest() []byte {
|
||||||
|
if x != nil {
|
||||||
|
return x.Digest
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// This represents some bytes of a blob.
|
||||||
|
// Blobs are sent in smaller chunks to keep message sizes manageable.
|
||||||
|
type BlobChunk struct {
|
||||||
|
state protoimpl.MessageState
|
||||||
|
sizeCache protoimpl.SizeCache
|
||||||
|
unknownFields protoimpl.UnknownFields
|
||||||
|
|
||||||
|
Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *BlobChunk) Reset() {
|
||||||
|
*x = BlobChunk{}
|
||||||
|
if protoimpl.UnsafeEnabled {
|
||||||
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3]
|
||||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
|
ms.StoreMessageInfo(mi)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *BlobChunk) String() string {
|
||||||
|
return protoimpl.X.MessageStringOf(x)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (*BlobChunk) ProtoMessage() {}
|
||||||
|
|
||||||
|
func (x *BlobChunk) ProtoReflect() protoreflect.Message {
|
||||||
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3]
|
||||||
|
if protoimpl.UnsafeEnabled && x != nil {
|
||||||
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
|
if ms.LoadMessageInfo() == nil {
|
||||||
|
ms.StoreMessageInfo(mi)
|
||||||
|
}
|
||||||
|
return ms
|
||||||
|
}
|
||||||
|
return mi.MessageOf(x)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deprecated: Use BlobChunk.ProtoReflect.Descriptor instead.
|
||||||
|
func (*BlobChunk) Descriptor() ([]byte, []int) {
|
||||||
|
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{3}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (x *BlobChunk) GetData() []byte {
|
||||||
|
if x != nil {
|
||||||
|
return x.Data
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type PutBlobResponse struct {
|
type PutBlobResponse struct {
|
||||||
state protoimpl.MessageState
|
state protoimpl.MessageState
|
||||||
sizeCache protoimpl.SizeCache
|
sizeCache protoimpl.SizeCache
|
||||||
|
@ -83,7 +265,7 @@ type PutBlobResponse struct {
|
||||||
func (x *PutBlobResponse) Reset() {
|
func (x *PutBlobResponse) Reset() {
|
||||||
*x = PutBlobResponse{}
|
*x = PutBlobResponse{}
|
||||||
if protoimpl.UnsafeEnabled {
|
if protoimpl.UnsafeEnabled {
|
||||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1]
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4]
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
ms.StoreMessageInfo(mi)
|
ms.StoreMessageInfo(mi)
|
||||||
}
|
}
|
||||||
|
@ -96,7 +278,7 @@ func (x *PutBlobResponse) String() string {
|
||||||
func (*PutBlobResponse) ProtoMessage() {}
|
func (*PutBlobResponse) ProtoMessage() {}
|
||||||
|
|
||||||
func (x *PutBlobResponse) ProtoReflect() protoreflect.Message {
|
func (x *PutBlobResponse) ProtoReflect() protoreflect.Message {
|
||||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1]
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4]
|
||||||
if protoimpl.UnsafeEnabled && x != nil {
|
if protoimpl.UnsafeEnabled && x != nil {
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
if ms.LoadMessageInfo() == nil {
|
if ms.LoadMessageInfo() == nil {
|
||||||
|
@ -109,7 +291,7 @@ func (x *PutBlobResponse) ProtoReflect() protoreflect.Message {
|
||||||
|
|
||||||
// Deprecated: Use PutBlobResponse.ProtoReflect.Descriptor instead.
|
// Deprecated: Use PutBlobResponse.ProtoReflect.Descriptor instead.
|
||||||
func (*PutBlobResponse) Descriptor() ([]byte, []int) {
|
func (*PutBlobResponse) Descriptor() ([]byte, []int) {
|
||||||
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1}
|
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{4}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *PutBlobResponse) GetDigest() []byte {
|
func (x *PutBlobResponse) GetDigest() []byte {
|
||||||
|
@ -119,33 +301,32 @@ func (x *PutBlobResponse) GetDigest() []byte {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// This represents a part of a chunk.
|
type BlobMeta_ChunkMeta struct {
|
||||||
// Blobs are sent in smaller chunks to keep message sizes manageable.
|
|
||||||
type BlobChunk struct {
|
|
||||||
state protoimpl.MessageState
|
state protoimpl.MessageState
|
||||||
sizeCache protoimpl.SizeCache
|
sizeCache protoimpl.SizeCache
|
||||||
unknownFields protoimpl.UnknownFields
|
unknownFields protoimpl.UnknownFields
|
||||||
|
|
||||||
Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
|
Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"`
|
||||||
|
Size uint32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *BlobChunk) Reset() {
|
func (x *BlobMeta_ChunkMeta) Reset() {
|
||||||
*x = BlobChunk{}
|
*x = BlobMeta_ChunkMeta{}
|
||||||
if protoimpl.UnsafeEnabled {
|
if protoimpl.UnsafeEnabled {
|
||||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2]
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[5]
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
ms.StoreMessageInfo(mi)
|
ms.StoreMessageInfo(mi)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *BlobChunk) String() string {
|
func (x *BlobMeta_ChunkMeta) String() string {
|
||||||
return protoimpl.X.MessageStringOf(x)
|
return protoimpl.X.MessageStringOf(x)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*BlobChunk) ProtoMessage() {}
|
func (*BlobMeta_ChunkMeta) ProtoMessage() {}
|
||||||
|
|
||||||
func (x *BlobChunk) ProtoReflect() protoreflect.Message {
|
func (x *BlobMeta_ChunkMeta) ProtoReflect() protoreflect.Message {
|
||||||
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2]
|
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[5]
|
||||||
if protoimpl.UnsafeEnabled && x != nil {
|
if protoimpl.UnsafeEnabled && x != nil {
|
||||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||||
if ms.LoadMessageInfo() == nil {
|
if ms.LoadMessageInfo() == nil {
|
||||||
|
@ -156,45 +337,73 @@ func (x *BlobChunk) ProtoReflect() protoreflect.Message {
|
||||||
return mi.MessageOf(x)
|
return mi.MessageOf(x)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Deprecated: Use BlobChunk.ProtoReflect.Descriptor instead.
|
// Deprecated: Use BlobMeta_ChunkMeta.ProtoReflect.Descriptor instead.
|
||||||
func (*BlobChunk) Descriptor() ([]byte, []int) {
|
func (*BlobMeta_ChunkMeta) Descriptor() ([]byte, []int) {
|
||||||
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{2}
|
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1, 0}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *BlobChunk) GetData() []byte {
|
func (x *BlobMeta_ChunkMeta) GetDigest() []byte {
|
||||||
if x != nil {
|
if x != nil {
|
||||||
return x.Data
|
return x.Digest
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (x *BlobMeta_ChunkMeta) GetSize() uint32 {
|
||||||
|
if x != nil {
|
||||||
|
return x.Size
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
var File_tvix_store_protos_rpc_blobstore_proto protoreflect.FileDescriptor
|
var File_tvix_store_protos_rpc_blobstore_proto protoreflect.FileDescriptor
|
||||||
|
|
||||||
var file_tvix_store_protos_rpc_blobstore_proto_rawDesc = []byte{
|
var file_tvix_store_protos_rpc_blobstore_proto_rawDesc = []byte{
|
||||||
0x0a, 0x25, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
|
0x0a, 0x25, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
|
||||||
0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72,
|
0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72,
|
||||||
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74,
|
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74,
|
||||||
0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x28, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f,
|
0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x71, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x74, 0x42, 0x6c,
|
||||||
0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65,
|
0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67,
|
||||||
|
0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73,
|
||||||
|
0x74, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, 0x63, 0x68, 0x75,
|
||||||
|
0x6e, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x69, 0x6e, 0x63, 0x6c, 0x75,
|
||||||
|
0x64, 0x65, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x63, 0x6c,
|
||||||
|
0x75, 0x64, 0x65, 0x5f, 0x62, 0x61, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69,
|
||||||
|
0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x42, 0x61, 0x6f, 0x22, 0x9d, 0x01, 0x0a, 0x08, 0x42, 0x6c,
|
||||||
|
0x6f, 0x62, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x39, 0x0a, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b, 0x73,
|
||||||
|
0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74,
|
||||||
|
0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74, 0x61, 0x2e,
|
||||||
|
0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x06, 0x63, 0x68, 0x75, 0x6e, 0x6b,
|
||||||
|
0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x5f, 0x62, 0x61, 0x6f, 0x18,
|
||||||
|
0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x42, 0x61, 0x6f,
|
||||||
|
0x1a, 0x37, 0x0a, 0x09, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x16, 0x0a,
|
||||||
|
0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64,
|
||||||
|
0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20,
|
||||||
|
0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x29, 0x0a, 0x0f, 0x52, 0x65, 0x61,
|
||||||
|
0x64, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06,
|
||||||
|
0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69,
|
||||||
|
0x67, 0x65, 0x73, 0x74, 0x22, 0x1f, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e,
|
||||||
|
0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
|
||||||
|
0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62,
|
||||||
|
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65,
|
||||||
0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74,
|
0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74,
|
||||||
0x22, 0x29, 0x0a, 0x0f, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
0x32, 0xd5, 0x01, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
|
||||||
0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20,
|
0x12, 0x3f, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, 0x12, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e,
|
||||||
0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x1f, 0x0a, 0x09, 0x42,
|
0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x42, 0x6c, 0x6f,
|
||||||
0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61,
|
0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e,
|
||||||
0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x32, 0x92, 0x01, 0x0a,
|
0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74,
|
||||||
0x0b, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x03,
|
0x61, 0x12, 0x42, 0x0a, 0x04, 0x52, 0x65, 0x61, 0x64, 0x12, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78,
|
||||||
0x47, 0x65, 0x74, 0x12, 0x1d, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65,
|
0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x42, 0x6c,
|
||||||
0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65,
|
0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78,
|
||||||
0x73, 0x74, 0x1a, 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e,
|
0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68,
|
||||||
0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41,
|
0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x18, 0x2e, 0x74,
|
||||||
0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f,
|
0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f,
|
||||||
0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a,
|
0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74,
|
||||||
0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e,
|
0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65,
|
||||||
0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28,
|
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65,
|
||||||
0x01, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79,
|
0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74,
|
||||||
0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
|
0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65,
|
||||||
0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f,
|
0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||||
0x74, 0x6f, 0x33,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -209,22 +418,28 @@ func file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP() []byte {
|
||||||
return file_tvix_store_protos_rpc_blobstore_proto_rawDescData
|
return file_tvix_store_protos_rpc_blobstore_proto_rawDescData
|
||||||
}
|
}
|
||||||
|
|
||||||
var file_tvix_store_protos_rpc_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
|
var file_tvix_store_protos_rpc_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
|
||||||
var file_tvix_store_protos_rpc_blobstore_proto_goTypes = []interface{}{
|
var file_tvix_store_protos_rpc_blobstore_proto_goTypes = []interface{}{
|
||||||
(*GetBlobRequest)(nil), // 0: tvix.store.v1.GetBlobRequest
|
(*StatBlobRequest)(nil), // 0: tvix.store.v1.StatBlobRequest
|
||||||
(*PutBlobResponse)(nil), // 1: tvix.store.v1.PutBlobResponse
|
(*BlobMeta)(nil), // 1: tvix.store.v1.BlobMeta
|
||||||
(*BlobChunk)(nil), // 2: tvix.store.v1.BlobChunk
|
(*ReadBlobRequest)(nil), // 2: tvix.store.v1.ReadBlobRequest
|
||||||
|
(*BlobChunk)(nil), // 3: tvix.store.v1.BlobChunk
|
||||||
|
(*PutBlobResponse)(nil), // 4: tvix.store.v1.PutBlobResponse
|
||||||
|
(*BlobMeta_ChunkMeta)(nil), // 5: tvix.store.v1.BlobMeta.ChunkMeta
|
||||||
}
|
}
|
||||||
var file_tvix_store_protos_rpc_blobstore_proto_depIdxs = []int32{
|
var file_tvix_store_protos_rpc_blobstore_proto_depIdxs = []int32{
|
||||||
0, // 0: tvix.store.v1.BlobService.Get:input_type -> tvix.store.v1.GetBlobRequest
|
5, // 0: tvix.store.v1.BlobMeta.chunks:type_name -> tvix.store.v1.BlobMeta.ChunkMeta
|
||||||
2, // 1: tvix.store.v1.BlobService.Put:input_type -> tvix.store.v1.BlobChunk
|
0, // 1: tvix.store.v1.BlobService.Stat:input_type -> tvix.store.v1.StatBlobRequest
|
||||||
2, // 2: tvix.store.v1.BlobService.Get:output_type -> tvix.store.v1.BlobChunk
|
2, // 2: tvix.store.v1.BlobService.Read:input_type -> tvix.store.v1.ReadBlobRequest
|
||||||
1, // 3: tvix.store.v1.BlobService.Put:output_type -> tvix.store.v1.PutBlobResponse
|
3, // 3: tvix.store.v1.BlobService.Put:input_type -> tvix.store.v1.BlobChunk
|
||||||
2, // [2:4] is the sub-list for method output_type
|
1, // 4: tvix.store.v1.BlobService.Stat:output_type -> tvix.store.v1.BlobMeta
|
||||||
0, // [0:2] is the sub-list for method input_type
|
3, // 5: tvix.store.v1.BlobService.Read:output_type -> tvix.store.v1.BlobChunk
|
||||||
0, // [0:0] is the sub-list for extension type_name
|
4, // 6: tvix.store.v1.BlobService.Put:output_type -> tvix.store.v1.PutBlobResponse
|
||||||
0, // [0:0] is the sub-list for extension extendee
|
4, // [4:7] is the sub-list for method output_type
|
||||||
0, // [0:0] is the sub-list for field type_name
|
1, // [1:4] is the sub-list for method input_type
|
||||||
|
1, // [1:1] is the sub-list for extension type_name
|
||||||
|
1, // [1:1] is the sub-list for extension extendee
|
||||||
|
0, // [0:1] is the sub-list for field type_name
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() { file_tvix_store_protos_rpc_blobstore_proto_init() }
|
func init() { file_tvix_store_protos_rpc_blobstore_proto_init() }
|
||||||
|
@ -234,7 +449,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
|
||||||
}
|
}
|
||||||
if !protoimpl.UnsafeEnabled {
|
if !protoimpl.UnsafeEnabled {
|
||||||
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
|
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
|
||||||
switch v := v.(*GetBlobRequest); i {
|
switch v := v.(*StatBlobRequest); i {
|
||||||
case 0:
|
case 0:
|
||||||
return &v.state
|
return &v.state
|
||||||
case 1:
|
case 1:
|
||||||
|
@ -246,7 +461,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
|
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
|
||||||
switch v := v.(*PutBlobResponse); i {
|
switch v := v.(*BlobMeta); i {
|
||||||
case 0:
|
case 0:
|
||||||
return &v.state
|
return &v.state
|
||||||
case 1:
|
case 1:
|
||||||
|
@ -258,6 +473,18 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
|
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
|
||||||
|
switch v := v.(*ReadBlobRequest); i {
|
||||||
|
case 0:
|
||||||
|
return &v.state
|
||||||
|
case 1:
|
||||||
|
return &v.sizeCache
|
||||||
|
case 2:
|
||||||
|
return &v.unknownFields
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
|
||||||
switch v := v.(*BlobChunk); i {
|
switch v := v.(*BlobChunk); i {
|
||||||
case 0:
|
case 0:
|
||||||
return &v.state
|
return &v.state
|
||||||
|
@ -269,6 +496,30 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
|
||||||
|
switch v := v.(*PutBlobResponse); i {
|
||||||
|
case 0:
|
||||||
|
return &v.state
|
||||||
|
case 1:
|
||||||
|
return &v.sizeCache
|
||||||
|
case 2:
|
||||||
|
return &v.unknownFields
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
|
||||||
|
switch v := v.(*BlobMeta_ChunkMeta); i {
|
||||||
|
case 0:
|
||||||
|
return &v.state
|
||||||
|
case 1:
|
||||||
|
return &v.sizeCache
|
||||||
|
case 2:
|
||||||
|
return &v.unknownFields
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
type x struct{}
|
type x struct{}
|
||||||
out := protoimpl.TypeBuilder{
|
out := protoimpl.TypeBuilder{
|
||||||
|
@ -276,7 +527,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
|
||||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||||
RawDescriptor: file_tvix_store_protos_rpc_blobstore_proto_rawDesc,
|
RawDescriptor: file_tvix_store_protos_rpc_blobstore_proto_rawDesc,
|
||||||
NumEnums: 0,
|
NumEnums: 0,
|
||||||
NumMessages: 3,
|
NumMessages: 6,
|
||||||
NumExtensions: 0,
|
NumExtensions: 0,
|
||||||
NumServices: 1,
|
NumServices: 1,
|
||||||
},
|
},
|
||||||
|
|
|
@ -7,27 +7,67 @@ package tvix.store.v1;
|
||||||
option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
|
option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
|
||||||
|
|
||||||
service BlobService {
|
service BlobService {
|
||||||
rpc Get(GetBlobRequest) returns (stream BlobChunk);
|
// Stat exposes metadata about a given blob,
|
||||||
|
// such as more granular chunking, baos.
|
||||||
|
// It implicitly allows checking for existence too, as asking this for a
|
||||||
|
// non-existing Blob will return a Status::not_found grpc error.
|
||||||
|
// If there's no more granular chunking available, the response will simply
|
||||||
|
// contain a single chunk.
|
||||||
|
rpc Stat(StatBlobRequest) returns (BlobMeta);
|
||||||
|
|
||||||
|
// Read returns a stream of BlobChunk, which is just a stream of bytes - not necessarily
|
||||||
|
// using the chunking that's returned in the reply of a Stat() call.
|
||||||
|
rpc Read(ReadBlobRequest) returns (stream BlobChunk);
|
||||||
|
|
||||||
|
// Put uploads a Blob, by reading a stream of bytes.
|
||||||
rpc Put(stream BlobChunk) returns (PutBlobResponse);
|
rpc Put(stream BlobChunk) returns (PutBlobResponse);
|
||||||
|
|
||||||
// TODO(flokli): We can get fancy here, and add methods to retrieve
|
|
||||||
// [Bao](https://github.com/oconnor663/bao/blob/master/docs/spec.md), and
|
|
||||||
// then support range requests, but that's left for later.
|
|
||||||
}
|
}
|
||||||
|
|
||||||
message GetBlobRequest {
|
message StatBlobRequest {
|
||||||
// The blake3 digest of the blob requested
|
// The blake3 digest of the blob requested
|
||||||
bytes digest = 1;
|
bytes digest = 1;
|
||||||
|
|
||||||
|
// Whether to include the chunks field
|
||||||
|
bool include_chunks = 2;
|
||||||
|
// Whether to include the inline_bao field, containing an (outboard) bao.
|
||||||
|
// The [bao](https://github.com/oconnor663/bao/blob/master/docs/spec.md)
|
||||||
|
// can be used to validate chunks end up hashing to the same root digest.
|
||||||
|
// These only really matter when only downloading parts of a blob. Some
|
||||||
|
// caution needs to be applied when validating chunks - the bao works with
|
||||||
|
// 1K leaf nodes, which might not align with the chunk sizes - this might
|
||||||
|
// imply a neighboring chunk might need to be (partially) fetched to
|
||||||
|
// validate the hash.
|
||||||
|
bool include_bao = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
// BlobMeta provides more granular chunking information for the requested blob,
|
||||||
|
// and baos.
|
||||||
|
message BlobMeta {
|
||||||
|
// This provides a list of chunks.
|
||||||
|
// Concatenating their contents would produce a blob with the digest that
|
||||||
|
// was specified in the request.
|
||||||
|
repeated ChunkMeta chunks = 1;
|
||||||
|
|
||||||
|
message ChunkMeta {
|
||||||
|
bytes digest = 1;
|
||||||
|
uint32 size = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
bytes inline_bao = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message ReadBlobRequest {
|
||||||
|
// The blake3 digest of the blob requested
|
||||||
|
bytes digest = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// This represents some bytes of a blob.
|
||||||
|
// Blobs are sent in smaller chunks to keep message sizes manageable.
|
||||||
|
message BlobChunk {
|
||||||
|
bytes data = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
message PutBlobResponse {
|
message PutBlobResponse {
|
||||||
// The blake3 digest of the data that was sent.
|
// The blake3 digest of the data that was sent.
|
||||||
bytes digest = 1;
|
bytes digest = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// This represents a part of a chunk.
|
|
||||||
// Blobs are sent in smaller chunks to keep message sizes manageable.
|
|
||||||
message BlobChunk {
|
|
||||||
bytes data = 1;
|
|
||||||
}
|
|
||||||
|
|
|
@ -22,7 +22,17 @@ const _ = grpc.SupportPackageIsVersion7
|
||||||
//
|
//
|
||||||
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
||||||
type BlobServiceClient interface {
|
type BlobServiceClient interface {
|
||||||
Get(ctx context.Context, in *GetBlobRequest, opts ...grpc.CallOption) (BlobService_GetClient, error)
|
// Stat exposes metadata about a given blob,
|
||||||
|
// such as more granular chunking, baos.
|
||||||
|
// It implicitly allows checking for existence too, as asking this for a
|
||||||
|
// non-existing Blob will return a Status::not_found grpc error.
|
||||||
|
// If there's no more granular chunking available, the response will simply
|
||||||
|
// contain a single chunk.
|
||||||
|
Stat(ctx context.Context, in *StatBlobRequest, opts ...grpc.CallOption) (*BlobMeta, error)
|
||||||
|
// Read returns a stream of BlobChunk, which is just a stream of bytes - not necessarily
|
||||||
|
// using the chunking that's returned in the reply of a Stat() call.
|
||||||
|
Read(ctx context.Context, in *ReadBlobRequest, opts ...grpc.CallOption) (BlobService_ReadClient, error)
|
||||||
|
// Put uploads a Blob, by reading a stream of bytes.
|
||||||
Put(ctx context.Context, opts ...grpc.CallOption) (BlobService_PutClient, error)
|
Put(ctx context.Context, opts ...grpc.CallOption) (BlobService_PutClient, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -34,12 +44,21 @@ func NewBlobServiceClient(cc grpc.ClientConnInterface) BlobServiceClient {
|
||||||
return &blobServiceClient{cc}
|
return &blobServiceClient{cc}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *blobServiceClient) Get(ctx context.Context, in *GetBlobRequest, opts ...grpc.CallOption) (BlobService_GetClient, error) {
|
func (c *blobServiceClient) Stat(ctx context.Context, in *StatBlobRequest, opts ...grpc.CallOption) (*BlobMeta, error) {
|
||||||
stream, err := c.cc.NewStream(ctx, &BlobService_ServiceDesc.Streams[0], "/tvix.store.v1.BlobService/Get", opts...)
|
out := new(BlobMeta)
|
||||||
|
err := c.cc.Invoke(ctx, "/tvix.store.v1.BlobService/Stat", in, out, opts...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
x := &blobServiceGetClient{stream}
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *blobServiceClient) Read(ctx context.Context, in *ReadBlobRequest, opts ...grpc.CallOption) (BlobService_ReadClient, error) {
|
||||||
|
stream, err := c.cc.NewStream(ctx, &BlobService_ServiceDesc.Streams[0], "/tvix.store.v1.BlobService/Read", opts...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
x := &blobServiceReadClient{stream}
|
||||||
if err := x.ClientStream.SendMsg(in); err != nil {
|
if err := x.ClientStream.SendMsg(in); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -49,16 +68,16 @@ func (c *blobServiceClient) Get(ctx context.Context, in *GetBlobRequest, opts ..
|
||||||
return x, nil
|
return x, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type BlobService_GetClient interface {
|
type BlobService_ReadClient interface {
|
||||||
Recv() (*BlobChunk, error)
|
Recv() (*BlobChunk, error)
|
||||||
grpc.ClientStream
|
grpc.ClientStream
|
||||||
}
|
}
|
||||||
|
|
||||||
type blobServiceGetClient struct {
|
type blobServiceReadClient struct {
|
||||||
grpc.ClientStream
|
grpc.ClientStream
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *blobServiceGetClient) Recv() (*BlobChunk, error) {
|
func (x *blobServiceReadClient) Recv() (*BlobChunk, error) {
|
||||||
m := new(BlobChunk)
|
m := new(BlobChunk)
|
||||||
if err := x.ClientStream.RecvMsg(m); err != nil {
|
if err := x.ClientStream.RecvMsg(m); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -104,7 +123,17 @@ func (x *blobServicePutClient) CloseAndRecv() (*PutBlobResponse, error) {
|
||||||
// All implementations must embed UnimplementedBlobServiceServer
|
// All implementations must embed UnimplementedBlobServiceServer
|
||||||
// for forward compatibility
|
// for forward compatibility
|
||||||
type BlobServiceServer interface {
|
type BlobServiceServer interface {
|
||||||
Get(*GetBlobRequest, BlobService_GetServer) error
|
// Stat exposes metadata about a given blob,
|
||||||
|
// such as more granular chunking, baos.
|
||||||
|
// It implicitly allows checking for existence too, as asking this for a
|
||||||
|
// non-existing Blob will return a Status::not_found grpc error.
|
||||||
|
// If there's no more granular chunking available, the response will simply
|
||||||
|
// contain a single chunk.
|
||||||
|
Stat(context.Context, *StatBlobRequest) (*BlobMeta, error)
|
||||||
|
// Read returns a stream of BlobChunk, which is just a stream of bytes - not necessarily
|
||||||
|
// using the chunking that's returned in the reply of a Stat() call.
|
||||||
|
Read(*ReadBlobRequest, BlobService_ReadServer) error
|
||||||
|
// Put uploads a Blob, by reading a stream of bytes.
|
||||||
Put(BlobService_PutServer) error
|
Put(BlobService_PutServer) error
|
||||||
mustEmbedUnimplementedBlobServiceServer()
|
mustEmbedUnimplementedBlobServiceServer()
|
||||||
}
|
}
|
||||||
|
@ -113,8 +142,11 @@ type BlobServiceServer interface {
|
||||||
type UnimplementedBlobServiceServer struct {
|
type UnimplementedBlobServiceServer struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (UnimplementedBlobServiceServer) Get(*GetBlobRequest, BlobService_GetServer) error {
|
func (UnimplementedBlobServiceServer) Stat(context.Context, *StatBlobRequest) (*BlobMeta, error) {
|
||||||
return status.Errorf(codes.Unimplemented, "method Get not implemented")
|
return nil, status.Errorf(codes.Unimplemented, "method Stat not implemented")
|
||||||
|
}
|
||||||
|
func (UnimplementedBlobServiceServer) Read(*ReadBlobRequest, BlobService_ReadServer) error {
|
||||||
|
return status.Errorf(codes.Unimplemented, "method Read not implemented")
|
||||||
}
|
}
|
||||||
func (UnimplementedBlobServiceServer) Put(BlobService_PutServer) error {
|
func (UnimplementedBlobServiceServer) Put(BlobService_PutServer) error {
|
||||||
return status.Errorf(codes.Unimplemented, "method Put not implemented")
|
return status.Errorf(codes.Unimplemented, "method Put not implemented")
|
||||||
|
@ -132,24 +164,42 @@ func RegisterBlobServiceServer(s grpc.ServiceRegistrar, srv BlobServiceServer) {
|
||||||
s.RegisterService(&BlobService_ServiceDesc, srv)
|
s.RegisterService(&BlobService_ServiceDesc, srv)
|
||||||
}
|
}
|
||||||
|
|
||||||
func _BlobService_Get_Handler(srv interface{}, stream grpc.ServerStream) error {
|
func _BlobService_Stat_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||||
m := new(GetBlobRequest)
|
in := new(StatBlobRequest)
|
||||||
|
if err := dec(in); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if interceptor == nil {
|
||||||
|
return srv.(BlobServiceServer).Stat(ctx, in)
|
||||||
|
}
|
||||||
|
info := &grpc.UnaryServerInfo{
|
||||||
|
Server: srv,
|
||||||
|
FullMethod: "/tvix.store.v1.BlobService/Stat",
|
||||||
|
}
|
||||||
|
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||||
|
return srv.(BlobServiceServer).Stat(ctx, req.(*StatBlobRequest))
|
||||||
|
}
|
||||||
|
return interceptor(ctx, in, info, handler)
|
||||||
|
}
|
||||||
|
|
||||||
|
func _BlobService_Read_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||||
|
m := new(ReadBlobRequest)
|
||||||
if err := stream.RecvMsg(m); err != nil {
|
if err := stream.RecvMsg(m); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return srv.(BlobServiceServer).Get(m, &blobServiceGetServer{stream})
|
return srv.(BlobServiceServer).Read(m, &blobServiceReadServer{stream})
|
||||||
}
|
}
|
||||||
|
|
||||||
type BlobService_GetServer interface {
|
type BlobService_ReadServer interface {
|
||||||
Send(*BlobChunk) error
|
Send(*BlobChunk) error
|
||||||
grpc.ServerStream
|
grpc.ServerStream
|
||||||
}
|
}
|
||||||
|
|
||||||
type blobServiceGetServer struct {
|
type blobServiceReadServer struct {
|
||||||
grpc.ServerStream
|
grpc.ServerStream
|
||||||
}
|
}
|
||||||
|
|
||||||
func (x *blobServiceGetServer) Send(m *BlobChunk) error {
|
func (x *blobServiceReadServer) Send(m *BlobChunk) error {
|
||||||
return x.ServerStream.SendMsg(m)
|
return x.ServerStream.SendMsg(m)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -185,11 +235,16 @@ func (x *blobServicePutServer) Recv() (*BlobChunk, error) {
|
||||||
var BlobService_ServiceDesc = grpc.ServiceDesc{
|
var BlobService_ServiceDesc = grpc.ServiceDesc{
|
||||||
ServiceName: "tvix.store.v1.BlobService",
|
ServiceName: "tvix.store.v1.BlobService",
|
||||||
HandlerType: (*BlobServiceServer)(nil),
|
HandlerType: (*BlobServiceServer)(nil),
|
||||||
Methods: []grpc.MethodDesc{},
|
Methods: []grpc.MethodDesc{
|
||||||
|
{
|
||||||
|
MethodName: "Stat",
|
||||||
|
Handler: _BlobService_Stat_Handler,
|
||||||
|
},
|
||||||
|
},
|
||||||
Streams: []grpc.StreamDesc{
|
Streams: []grpc.StreamDesc{
|
||||||
{
|
{
|
||||||
StreamName: "Get",
|
StreamName: "Read",
|
||||||
Handler: _BlobService_Get_Handler,
|
Handler: _BlobService_Read_Handler,
|
||||||
ServerStreams: true,
|
ServerStreams: true,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in a new issue