87e1934f2d
Further emphasize Read() can be used to ask for blobs OR chunks, and that clients usually want to stat and then request (smaller) chunks, rather than reading whole blobs. Also clarify that the chunking used to send BlobChunks over has nothing to do with the chunk sizes communicated in a Stat() request. Change-Id: Ia615d190aae570611de2655b11342a14d0b75976 Reviewed-on: https://cl.tvl.fyi/c/depot/+/8028 Reviewed-by: tazjin <tazjin@tvl.su> Tested-by: BuildkiteCI
89 lines
3.1 KiB
Protocol Buffer
89 lines
3.1 KiB
Protocol Buffer
// SPDX-License-Identifier: MIT
|
|
// Copyright © 2022 The Tvix Authors
|
|
syntax = "proto3";
|
|
|
|
package tvix.store.v1;
|
|
|
|
option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
|
|
|
|
service BlobService {
|
|
// Stat exposes metadata about a given blob,
|
|
// such as more granular chunking, baos.
|
|
// It implicitly allows checking for existence too, as asking this for a
|
|
// non-existing Blob will return a Status::not_found grpc error.
|
|
// If there's no more granular chunking available, the response will simply
|
|
// contain a single chunk.
|
|
rpc Stat(StatBlobRequest) returns (BlobMeta);
|
|
|
|
// Read returns a stream of BlobChunk, which is just a stream of bytes with
|
|
// the digest specified in ReadBlobRequest.
|
|
//
|
|
// The server may decide on whatever chunking it may seem fit as a size for
|
|
// the individual BlobChunk sent in the response stream.
|
|
//
|
|
// It specifically is NOT necessarily using chunk sizes communicated in a
|
|
// previous Stat request.
|
|
//
|
|
// It's up to the specific store to decide on whether it allows Read on a
|
|
// Blob at all, or only on smaller chunks communicated in a Stat() call
|
|
// first.
|
|
//
|
|
// Clients are enouraged to Stat() first, and then only read the individual
|
|
// chunks they don't have yet.
|
|
rpc Read(ReadBlobRequest) returns (stream BlobChunk);
|
|
|
|
// Put uploads a Blob, by reading a stream of bytes.
|
|
//
|
|
// The way the data is chunked up in individual BlobChunk messages sent in
|
|
// the stream has no effect on how the server ends up chunking blobs up.
|
|
rpc Put(stream BlobChunk) returns (PutBlobResponse);
|
|
}
|
|
|
|
message StatBlobRequest {
|
|
// The blake3 digest of the blob requested
|
|
bytes digest = 1;
|
|
|
|
// Whether to include the chunks field
|
|
bool include_chunks = 2;
|
|
// Whether to include the inline_bao field, containing an (outboard) bao.
|
|
// The [bao](https://github.com/oconnor663/bao/blob/master/docs/spec.md)
|
|
// can be used to validate chunks end up hashing to the same root digest.
|
|
// These only really matter when only downloading parts of a blob. Some
|
|
// caution needs to be applied when validating chunks - the bao works with
|
|
// 1K leaf nodes, which might not align with the chunk sizes - this might
|
|
// imply a neighboring chunk might need to be (partially) fetched to
|
|
// validate the hash.
|
|
bool include_bao = 3;
|
|
}
|
|
|
|
// BlobMeta provides more granular chunking information for the requested blob,
|
|
// and baos.
|
|
message BlobMeta {
|
|
// This provides a list of chunks.
|
|
// Concatenating their contents would produce a blob with the digest that
|
|
// was specified in the request.
|
|
repeated ChunkMeta chunks = 1;
|
|
|
|
message ChunkMeta {
|
|
bytes digest = 1;
|
|
uint32 size = 2;
|
|
}
|
|
|
|
bytes inline_bao = 2;
|
|
}
|
|
|
|
message ReadBlobRequest {
|
|
// The blake3 digest of the blob or chunk requested
|
|
bytes digest = 1;
|
|
}
|
|
|
|
// This represents some bytes of a blob.
|
|
// Blobs are sent in smaller chunks to keep message sizes manageable.
|
|
message BlobChunk {
|
|
bytes data = 1;
|
|
}
|
|
|
|
message PutBlobResponse {
|
|
// The blake3 digest of the data that was sent.
|
|
bytes digest = 1;
|
|
}
|