mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix): move castore into tvix-castore crate

Browse source 
This splits the pure content-addressed layers from tvix-store into a
`castore` crate, and only leaves PathInfo related things, as well as the
CLI entrypoint in the tvix-store crate.

Notable changes:
 - `fixtures` and `utils` had to be moved out of the `test` cfg, so they
   can be imported from tvix-store.
 - Some ad-hoc fixtures in the test were moved to proper fixtures in the
   same step.
 - The protos are now created by a (more static) recipe in the protos/
   directory.

The (now two) golang targets are commented out, as it's not possible to
update them properly in the same CL. This will be done by a followup CL
once this is merged (and whitby deployed)

Bug: https://b.tvl.fyi/issues/301

Change-Id: I8d675d4bf1fb697eb7d479747c1b1e3635718107
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9370
Reviewed-by: tazjin <tazjin@tvl.su>
Reviewed-by: flokli <flokli@flokli.de>
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: Connor Brewster <cbrewster@hey.com>
This commit is contained in:
Florian Klink 2023-09-21 22:32:44 +03:00 committed by clbot
parent d8ef0cfb4a
commit 32f41458c0
89 changed files with 2308 additions and 1829 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
nix/bufCheck/default.nix
View file

@ -7,6 +7,7 @@ pkgs.writeShellScriptBin "ci-buf-check" ''
(cd $(git rev-parse --show-toplevel) && buf lint .)
# Run buf generate, and bail out if generated files are changed.
(cd $(git rev-parse --show-toplevel) && buf generate --path tvix/castore/protos)
(cd $(git rev-parse --show-toplevel) && buf generate --path tvix/store/protos)
# Check if any files have changed
if [[ -n "$(git status --porcelain -unormal)" ]]; then

4
ops/modules/www/code.tvl.fyi.nix
View file

@ -20,6 +20,10 @@
alias ${depot.tvix.docs.svg}/component-flow.svg;
}
location = /go-get/tvix/castore/protos {
alias ${pkgs.writeText "go-import-metadata.html" ''<html><meta name="go-import" content="code.tvl.fyi/tvix/castore/protos git https://code.tvl.fyi/depot.git:/tvix/castore/protos.git"></html>''};
}
location = /go-get/tvix/store/protos {
alias ${pkgs.writeText "go-import-metadata.html" ''<html><meta name="go-import" content="code.tvl.fyi/tvix/store/protos git https://code.tvl.fyi/depot.git:/tvix/store/protos.git"></html>''};
}

33
tvix/Cargo.lock generated
View file

@ -2730,6 +2730,37 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
[[package]]
name = "tvix-castore"
version = "0.1.0"
dependencies = [
"anyhow",
"async-stream",
"blake3",
"bytes",
"data-encoding",
"futures",
"lazy_static",
"pin-project-lite",
"prost",
"prost-build",
"sled",
"tempfile",
"test-case",
"thiserror",
"tokio",
"tokio-stream",
"tokio-util",
"tonic",
"tonic-build",
"tonic-mock",
"tonic-reflection",
"tower",
"tracing",
"url",
"walkdir",
]
[[package]]
name = "tvix-cli"
version = "0.1.0"
@ -2742,6 +2773,7 @@ dependencies = [
"thiserror",
"tokio",
"tracing",
"tvix-castore",
"tvix-eval",
"tvix-store",
"wu-manber",
@ -2832,6 +2864,7 @@ dependencies = [
"tower",
"tracing",
"tracing-subscriber",
"tvix-castore",
"url",
"walkdir",
]

147
tvix/Cargo.nix
View file

@ -53,6 +53,16 @@ rec {
# File a bug if you depend on any for non-debug work!
debug = internal.debugCrate { inherit packageId; };
};
"tvix-castore" = rec {
packageId = "tvix-castore";
build = internal.buildRustCrateWithFeatures {
packageId = "tvix-castore";
};
# Debug support which might change between releases.
# File a bug if you depend on any for non-debug work!
debug = internal.debugCrate { inherit packageId; };
};
"tvix-cli" = rec {
packageId = "tvix-cli";
build = internal.buildRustCrateWithFeatures {
@ -8115,6 +8125,135 @@ rec {
];
};
"tvix-castore" = rec {
crateName = "tvix-castore";
version = "0.1.0";
edition = "2021";
# We can't filter paths with references in Nix 2.4
# See https://github.com/NixOS/nix/issues/5410
src =
if (lib.versionOlder builtins.nixVersion "2.4pre20211007")
then lib.cleanSourceWith { filter = sourceFilter; src = ./castore; }
else ./castore;
dependencies = [
{
name = "anyhow";
packageId = "anyhow";
}
{
name = "async-stream";
packageId = "async-stream";
}
{
name = "blake3";
packageId = "blake3";
features = [ "rayon" "std" ];
}
{
name = "bytes";
packageId = "bytes";
}
{
name = "data-encoding";
packageId = "data-encoding";
}
{
name = "futures";
packageId = "futures";
}
{
name = "lazy_static";
packageId = "lazy_static";
}
{
name = "pin-project-lite";
packageId = "pin-project-lite";
}
{
name = "prost";
packageId = "prost";
}
{
name = "sled";
packageId = "sled";
features = [ "compression" ];
}
{
name = "thiserror";
packageId = "thiserror";
}
{
name = "tokio";
packageId = "tokio";
features = [ "fs" "net" "rt-multi-thread" "signal" ];
}
{
name = "tokio-stream";
packageId = "tokio-stream";
features = [ "fs" "net" ];
}
{
name = "tokio-util";
packageId = "tokio-util";
features = [ "io" "io-util" ];
}
{
name = "tonic";
packageId = "tonic";
}
{
name = "tonic-reflection";
packageId = "tonic-reflection";
optional = true;
}
{
name = "tower";
packageId = "tower";
}
{
name = "tracing";
packageId = "tracing";
}
{
name = "url";
packageId = "url";
}
{
name = "walkdir";
packageId = "walkdir";
}
];
buildDependencies = [
{
name = "prost-build";
packageId = "prost-build";
}
{
name = "tonic-build";
packageId = "tonic-build";
}
];
devDependencies = [
{
name = "tempfile";
packageId = "tempfile";
}
{
name = "test-case";
packageId = "test-case";
}
{
name = "tonic-mock";
packageId = "tonic-mock";
}
];
features = {
"default" = [ "reflection" ];
"reflection" = [ "tonic-reflection" ];
"tonic-reflection" = [ "dep:tonic-reflection" ];
};
resolvedDefaultFeatures = [ "default" "reflection" "tonic-reflection" ];
};
"tvix-cli" = rec {
crateName = "tvix-cli";
version = "0.1.0";
@ -8166,6 +8305,10 @@ rec {
name = "tracing";
packageId = "tracing";
}
{
name = "tvix-castore";
packageId = "tvix-castore";
}
{
name = "tvix-eval";
packageId = "tvix-eval";
@ -8509,6 +8652,10 @@ rec {
packageId = "tracing-subscriber";
features = [ "json" ];
}
{
name = "tvix-castore";
packageId = "tvix-castore";
}
{
name = "url";
packageId = "url";

1
tvix/Cargo.toml
View file

@ -19,6 +19,7 @@
resolver = "2"
members = [
"castore",
"cli",
"eval",
"eval/builtin-macros",

42
tvix/castore/Cargo.toml Normal file
View file

@ -0,0 +1,42 @@
[package]
name = "tvix-castore"
version = "0.1.0"
edition = "2021"
[dependencies]
anyhow = "1.0.68"
async-stream = "0.3.5"
blake3 = { version = "1.3.1", features = ["rayon", "std"] }
bytes = "1.4.0"
data-encoding = "2.3.3"
futures = "0.3.28"
lazy_static = "1.4.0"
pin-project-lite = "0.2.13"
prost = "0.11.2"
sled = { version = "0.34.7", features = ["compression"] }
thiserror = "1.0.38"
tokio-stream = { version = "0.1.14", features = ["fs", "net"] }
tokio-util = { version = "0.7.8", features = ["io", "io-util"] }
tokio = { version = "1.28.0", features = ["fs", "net", "rt-multi-thread", "signal"] }
tonic = "0.8.2"
tower = "0.4.13"
tracing = "0.1.37"
url = "2.4.0"
walkdir = "2.4.0"
[dependencies.tonic-reflection]
optional = true
version = "0.5.0"
[build-dependencies]
prost-build = "0.11.2"
tonic-build = "0.8.2"
[dev-dependencies]
test-case = "2.2.2"
tempfile = "3.3.0"
tonic-mock = { git = "https://github.com/brainrake/tonic-mock", branch = "bump-dependencies" }
[features]
default = ["reflection"]
reflection = ["tonic-reflection"]

38
tvix/castore/build.rs Normal file
View file

@ -0,0 +1,38 @@
use std::io::Result;
fn main() -> Result<()> {
#[allow(unused_mut)]
let mut builder = tonic_build::configure();
#[cfg(feature = "reflection")]
{
let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap());
let descriptor_path = out_dir.join("tvix.castore.v1.bin");
builder = builder.file_descriptor_set_path(descriptor_path);
};
// https://github.com/hyperium/tonic/issues/908
let mut config = prost_build::Config::new();
config.bytes(["."]);
builder
.build_server(true)
.build_client(true)
.compile_with_config(
config,
&[
"tvix/castore/protos/castore.proto",
"tvix/castore/protos/rpc_blobstore.proto",
"tvix/castore/protos/rpc_directory.proto",
],
// If we are in running `cargo build` manually, using `../..` works fine,
// but in case we run inside a nix build, we need to instead point PROTO_ROOT
// to a sparseTree containing that structure.
&[match std::env::var_os("PROTO_ROOT") {
Some(proto_root) => proto_root.to_str().unwrap().to_owned(),
None => "../..".to_string(),
}],
)?;
Ok(())
}

5
tvix/castore/default.nix Normal file
View file

@ -0,0 +1,5 @@
{ depot, pkgs, ... }:
depot.tvix.crates.workspaceMembers.tvix-castore.build.override {
runTests = true;
}

21
tvix/castore/protos/LICENSE Normal file
View file

@ -0,0 +1,21 @@
Copyright © The Tvix Authors
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
“Software”), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

2
tvix/store/protos/castore.go → tvix/castore/protos/castore.go
View file

@ -1,4 +1,4 @@
package storev1
package castorev1
import (
"bytes"

580
tvix/castore/protos/castore.pb.go Normal file
View file

@ -0,0 +1,580 @@
// SPDX-FileCopyrightText: edef <edef@unfathomable.blue>
// SPDX-License-Identifier: OSL-3.0 OR MIT OR Apache-2.0
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
// protoc (unknown)
// source: tvix/castore/protos/castore.proto
package castorev1
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
// A Directory can contain Directory, File or Symlink nodes.
// Each of these nodes have a name attribute, which is the basename in that directory
// and node type specific attributes.
// The name attribute:
// - MUST not contain slashes or null bytes
// - MUST not be '.' or '..'
// - MUST be unique across all three lists
//
// Elements in each list need to be lexicographically ordered by the name
// attribute.
type Directory struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Directories []*DirectoryNode `protobuf:"bytes,1,rep,name=directories,proto3" json:"directories,omitempty"`
Files []*FileNode `protobuf:"bytes,2,rep,name=files,proto3" json:"files,omitempty"`
Symlinks []*SymlinkNode `protobuf:"bytes,3,rep,name=symlinks,proto3" json:"symlinks,omitempty"`
}
func (x *Directory) Reset() {
*x = Directory{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_castore_protos_castore_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Directory) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Directory) ProtoMessage() {}
func (x *Directory) ProtoReflect() protoreflect.Message {
mi := &file_tvix_castore_protos_castore_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Directory.ProtoReflect.Descriptor instead.
func (*Directory) Descriptor() ([]byte, []int) {
return file_tvix_castore_protos_castore_proto_rawDescGZIP(), []int{0}
}
func (x *Directory) GetDirectories() []*DirectoryNode {
if x != nil {
return x.Directories
}
return nil
}
func (x *Directory) GetFiles() []*FileNode {
if x != nil {
return x.Files
}
return nil
}
func (x *Directory) GetSymlinks() []*SymlinkNode {
if x != nil {
return x.Symlinks
}
return nil
}
// A DirectoryNode represents a directory in a Directory.
type DirectoryNode struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// The (base)name of the directory
Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
// The blake3 hash of a Directory message, serialized in protobuf canonical form.
Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"`
// Number of child elements in the Directory referred to by `digest`.
// Calculated by summing up the numbers of `directories`, `files` and
// `symlinks`, and for each directory, its size field. Used for inode
// number calculation.
// This field is precisely as verifiable as any other Merkle tree edge.
// Resolve `digest`, and you can compute it incrementally. Resolve the
// entire tree, and you can fully compute it from scratch.
// A credulous implementation won't reject an excessive size, but this is
// harmless: you'll have some ordinals without nodes. Undersizing is
// obvious and easy to reject: you won't have an ordinal for some nodes.
Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
}
func (x *DirectoryNode) Reset() {
*x = DirectoryNode{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_castore_protos_castore_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DirectoryNode) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DirectoryNode) ProtoMessage() {}
func (x *DirectoryNode) ProtoReflect() protoreflect.Message {
mi := &file_tvix_castore_protos_castore_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DirectoryNode.ProtoReflect.Descriptor instead.
func (*DirectoryNode) Descriptor() ([]byte, []int) {
return file_tvix_castore_protos_castore_proto_rawDescGZIP(), []int{1}
}
func (x *DirectoryNode) GetName() []byte {
if x != nil {
return x.Name
}
return nil
}
func (x *DirectoryNode) GetDigest() []byte {
if x != nil {
return x.Digest
}
return nil
}
func (x *DirectoryNode) GetSize() uint32 {
if x != nil {
return x.Size
}
return 0
}
// A FileNode represents a regular or executable file in a Directory.
type FileNode struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// The (base)name of the file
Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
// The blake3 digest of the file contents
Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"`
// The file content size
Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
// Whether the file is executable
Executable bool `protobuf:"varint,4,opt,name=executable,proto3" json:"executable,omitempty"`
}
func (x *FileNode) Reset() {
*x = FileNode{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_castore_protos_castore_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *FileNode) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FileNode) ProtoMessage() {}
func (x *FileNode) ProtoReflect() protoreflect.Message {
mi := &file_tvix_castore_protos_castore_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FileNode.ProtoReflect.Descriptor instead.
func (*FileNode) Descriptor() ([]byte, []int) {
return file_tvix_castore_protos_castore_proto_rawDescGZIP(), []int{2}
}
func (x *FileNode) GetName() []byte {
if x != nil {
return x.Name
}
return nil
}
func (x *FileNode) GetDigest() []byte {
if x != nil {
return x.Digest
}
return nil
}
func (x *FileNode) GetSize() uint32 {
if x != nil {
return x.Size
}
return 0
}
func (x *FileNode) GetExecutable() bool {
if x != nil {
return x.Executable
}
return false
}
// A SymlinkNode represents a symbolic link in a Directory.
type SymlinkNode struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// The (base)name of the symlink
Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
// The target of the symlink.
Target []byte `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"`
}
func (x *SymlinkNode) Reset() {
*x = SymlinkNode{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_castore_protos_castore_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *SymlinkNode) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*SymlinkNode) ProtoMessage() {}
func (x *SymlinkNode) ProtoReflect() protoreflect.Message {
mi := &file_tvix_castore_protos_castore_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use SymlinkNode.ProtoReflect.Descriptor instead.
func (*SymlinkNode) Descriptor() ([]byte, []int) {
return file_tvix_castore_protos_castore_proto_rawDescGZIP(), []int{3}
}
func (x *SymlinkNode) GetName() []byte {
if x != nil {
return x.Name
}
return nil
}
func (x *SymlinkNode) GetTarget() []byte {
if x != nil {
return x.Target
}
return nil
}
// A Node is either a DirectoryNode, FileNode or SymlinkNode.
type Node struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Types that are assignable to Node:
//
// *Node_Directory
// *Node_File
// *Node_Symlink
Node isNode_Node `protobuf_oneof:"node"`
}
func (x *Node) Reset() {
*x = Node{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_castore_protos_castore_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Node) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Node) ProtoMessage() {}
func (x *Node) ProtoReflect() protoreflect.Message {
mi := &file_tvix_castore_protos_castore_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Node.ProtoReflect.Descriptor instead.
func (*Node) Descriptor() ([]byte, []int) {
return file_tvix_castore_protos_castore_proto_rawDescGZIP(), []int{4}
}
func (m *Node) GetNode() isNode_Node {
if m != nil {
return m.Node
}
return nil
}
func (x *Node) GetDirectory() *DirectoryNode {
if x, ok := x.GetNode().(*Node_Directory); ok {
return x.Directory
}
return nil
}
func (x *Node) GetFile() *FileNode {
if x, ok := x.GetNode().(*Node_File); ok {
return x.File
}
return nil
}
func (x *Node) GetSymlink() *SymlinkNode {
if x, ok := x.GetNode().(*Node_Symlink); ok {
return x.Symlink
}
return nil
}
type isNode_Node interface {
isNode_Node()
}
type Node_Directory struct {
Directory *DirectoryNode `protobuf:"bytes,1,opt,name=directory,proto3,oneof"`
}
type Node_File struct {
File *FileNode `protobuf:"bytes,2,opt,name=file,proto3,oneof"`
}
type Node_Symlink struct {
Symlink *SymlinkNode `protobuf:"bytes,3,opt,name=symlink,proto3,oneof"`
}
func (*Node_Directory) isNode_Node() {}
func (*Node_File) isNode_Node() {}
func (*Node_Symlink) isNode_Node() {}
var File_tvix_castore_protos_castore_proto protoreflect.FileDescriptor
var file_tvix_castore_protos_castore_proto_rawDesc = []byte{
0x0a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2e, 0x76, 0x31, 0x22, 0xb8, 0x01, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f,
0x72, 0x79, 0x12, 0x40, 0x0a, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65,
0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63,
0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f,
0x72, 0x69, 0x65, 0x73, 0x12, 0x2f, 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f,
0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05,
0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x08, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b,
0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63,
0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e,
0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x08, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22,
0x4f, 0x0a, 0x0d, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x4e, 0x6f, 0x64, 0x65,
0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04,
0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65,
0x22, 0x6a, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c,
0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65,
0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1e, 0x0a, 0x0a,
0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08,
0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x39, 0x0a, 0x0b,
0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e,
0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12,
0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52,
0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0xb9, 0x01, 0x0a, 0x04, 0x4e, 0x6f, 0x64, 0x65,
0x12, 0x3e, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x18, 0x01, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f,
0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x4e,
0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79,
0x12, 0x2f, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19,
0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31,
0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x04, 0x66, 0x69, 0x6c,
0x65, 0x12, 0x38, 0x0a, 0x07, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, 0x65,
0x48, 0x00, 0x52, 0x07, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x42, 0x06, 0x0a, 0x04, 0x6e,
0x6f, 0x64, 0x65, 0x42, 0x2c, 0x5a, 0x2a, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e,
0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65,
0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76,
0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_tvix_castore_protos_castore_proto_rawDescOnce sync.Once
file_tvix_castore_protos_castore_proto_rawDescData = file_tvix_castore_protos_castore_proto_rawDesc
)
func file_tvix_castore_protos_castore_proto_rawDescGZIP() []byte {
file_tvix_castore_protos_castore_proto_rawDescOnce.Do(func() {
file_tvix_castore_protos_castore_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_castore_protos_castore_proto_rawDescData)
})
return file_tvix_castore_protos_castore_proto_rawDescData
}
var file_tvix_castore_protos_castore_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
var file_tvix_castore_protos_castore_proto_goTypes = []interface{}{
(*Directory)(nil), // 0: tvix.castore.v1.Directory
(*DirectoryNode)(nil), // 1: tvix.castore.v1.DirectoryNode
(*FileNode)(nil), // 2: tvix.castore.v1.FileNode
(*SymlinkNode)(nil), // 3: tvix.castore.v1.SymlinkNode
(*Node)(nil), // 4: tvix.castore.v1.Node
}
var file_tvix_castore_protos_castore_proto_depIdxs = []int32{
1, // 0: tvix.castore.v1.Directory.directories:type_name -> tvix.castore.v1.DirectoryNode
2, // 1: tvix.castore.v1.Directory.files:type_name -> tvix.castore.v1.FileNode
3, // 2: tvix.castore.v1.Directory.symlinks:type_name -> tvix.castore.v1.SymlinkNode
1, // 3: tvix.castore.v1.Node.directory:type_name -> tvix.castore.v1.DirectoryNode
2, // 4: tvix.castore.v1.Node.file:type_name -> tvix.castore.v1.FileNode
3, // 5: tvix.castore.v1.Node.symlink:type_name -> tvix.castore.v1.SymlinkNode
6, // [6:6] is the sub-list for method output_type
6, // [6:6] is the sub-list for method input_type
6, // [6:6] is the sub-list for extension type_name
6, // [6:6] is the sub-list for extension extendee
0, // [0:6] is the sub-list for field type_name
}
func init() { file_tvix_castore_protos_castore_proto_init() }
func file_tvix_castore_protos_castore_proto_init() {
if File_tvix_castore_protos_castore_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_tvix_castore_protos_castore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Directory); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_castore_protos_castore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DirectoryNode); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_castore_protos_castore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*FileNode); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_castore_protos_castore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*SymlinkNode); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_castore_protos_castore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Node); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
file_tvix_castore_protos_castore_proto_msgTypes[4].OneofWrappers = []interface{}{
(*Node_Directory)(nil),
(*Node_File)(nil),
(*Node_Symlink)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_tvix_castore_protos_castore_proto_rawDesc,
NumEnums: 0,
NumMessages: 5,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_tvix_castore_protos_castore_proto_goTypes,
DependencyIndexes: file_tvix_castore_protos_castore_proto_depIdxs,
MessageInfos: file_tvix_castore_protos_castore_proto_msgTypes,
}.Build()
File_tvix_castore_protos_castore_proto = out.File
file_tvix_castore_protos_castore_proto_rawDesc = nil
file_tvix_castore_protos_castore_proto_goTypes = nil
file_tvix_castore_protos_castore_proto_depIdxs = nil
}

14
tvix/store/protos/castore.proto → tvix/castore/protos/castore.proto
View file

@ -3,9 +3,9 @@
syntax = "proto3";
package tvix.store.v1;
package tvix.castore.v1;
option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
option go_package = "code.tvl.fyi/tvix/castore/protos;castorev1";
// A Directory can contain Directory, File or Symlink nodes.
// Each of these nodes have a name attribute, which is the basename in that directory
@ -60,3 +60,13 @@ message SymlinkNode {
// The target of the symlink.
bytes target = 2;
}
// A Node is either a DirectoryNode, FileNode or SymlinkNode.
message Node {
oneof node {
DirectoryNode directory = 1;
FileNode file = 2;
SymlinkNode symlink = 3;
}
}

140
tvix/store/protos/castore_test.go → tvix/castore/protos/castore_test.go
View file

@ -1,9 +1,9 @@
package storev1_test
package castorev1_test
import (
"testing"
storev1pb "code.tvl.fyi/tvix/store/protos"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
"github.com/stretchr/testify/assert"
)
@ -17,63 +17,63 @@ var (
func TestDirectorySize(t *testing.T) {
t.Run("empty", func(t *testing.T) {
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.Equal(t, uint32(0), d.Size())
})
t.Run("containing single empty directory", func(t *testing.T) {
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{{
Name: []byte([]byte("foo")),
Digest: dummyDigest,
Size: 0,
}},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.Equal(t, uint32(1), d.Size())
})
t.Run("containing single non-empty directory", func(t *testing.T) {
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{{
Name: []byte("foo"),
Digest: dummyDigest,
Size: 4,
}},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.Equal(t, uint32(5), d.Size())
})
t.Run("containing single file", func(t *testing.T) {
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{{
Name: []byte("foo"),
Digest: dummyDigest,
Size: 42,
Executable: false,
}},
Symlinks: []*storev1pb.SymlinkNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.Equal(t, uint32(1), d.Size())
})
t.Run("containing single symlink", func(t *testing.T) {
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{{
Name: []byte("foo"),
Target: []byte("bar"),
}},
@ -84,10 +84,10 @@ func TestDirectorySize(t *testing.T) {
}
func TestDirectoryDigest(t *testing.T) {
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
dgst, err := d.Digest()
@ -101,10 +101,10 @@ func TestDirectoryDigest(t *testing.T) {
func TestDirectoryValidate(t *testing.T) {
t.Run("empty", func(t *testing.T) {
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.NoError(t, d.Validate())
@ -112,50 +112,50 @@ func TestDirectoryValidate(t *testing.T) {
t.Run("invalid names", func(t *testing.T) {
{
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{{
Name: []byte{},
Digest: dummyDigest,
Size: 42,
}},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.ErrorContains(t, d.Validate(), "invalid name")
}
{
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{{
Name: []byte("."),
Digest: dummyDigest,
Size: 42,
}},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.ErrorContains(t, d.Validate(), "invalid name")
}
{
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{{
Name: []byte(".."),
Digest: dummyDigest,
Size: 42,
Executable: false,
}},
Symlinks: []*storev1pb.SymlinkNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.ErrorContains(t, d.Validate(), "invalid name")
}
{
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{{
Name: []byte("\x00"),
Target: []byte("foo"),
}},
@ -164,10 +164,10 @@ func TestDirectoryValidate(t *testing.T) {
assert.ErrorContains(t, d.Validate(), "invalid name")
}
{
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{{
Name: []byte("foo/bar"),
Target: []byte("foo"),
}},
@ -178,14 +178,14 @@ func TestDirectoryValidate(t *testing.T) {
})
t.Run("invalid digest", func(t *testing.T) {
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{{
Name: []byte("foo"),
Digest: nil,
Size: 42,
}},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.ErrorContains(t, d.Validate(), "invalid digest length")
@ -194,8 +194,8 @@ func TestDirectoryValidate(t *testing.T) {
t.Run("sorting", func(t *testing.T) {
// "b" comes before "a", bad.
{
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{{
Name: []byte("b"),
Digest: dummyDigest,
Size: 42,
@ -204,35 +204,35 @@ func TestDirectoryValidate(t *testing.T) {
Digest: dummyDigest,
Size: 42,
}},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.ErrorContains(t, d.Validate(), "is not in sorted order")
}
// "a" exists twice, bad.
{
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{{
Name: []byte("a"),
Digest: dummyDigest,
Size: 42,
}},
Files: []*storev1pb.FileNode{{
Files: []*castorev1pb.FileNode{{
Name: []byte("a"),
Digest: dummyDigest,
Size: 42,
Executable: false,
}},
Symlinks: []*storev1pb.SymlinkNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.ErrorContains(t, d.Validate(), "duplicate name")
}
// "a" comes before "b", all good.
{
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{{
Name: []byte("a"),
Digest: dummyDigest,
Size: 42,
@ -241,16 +241,16 @@ func TestDirectoryValidate(t *testing.T) {
Digest: dummyDigest,
Size: 42,
}},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{},
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{},
}
assert.NoError(t, d.Validate(), "shouldn't error")
}
// [b, c] and [a] are both properly sorted.
{
d := storev1pb.Directory{
Directories: []*storev1pb.DirectoryNode{{
d := castorev1pb.Directory{
Directories: []*castorev1pb.DirectoryNode{{
Name: []byte("b"),
Digest: dummyDigest,
Size: 42,
@ -259,8 +259,8 @@ func TestDirectoryValidate(t *testing.T) {
Digest: dummyDigest,
Size: 42,
}},
Files: []*storev1pb.FileNode{},
Symlinks: []*storev1pb.SymlinkNode{{
Files: []*castorev1pb.FileNode{},
Symlinks: []*castorev1pb.SymlinkNode{{
Name: []byte("a"),
Target: []byte("foo"),
}},

19
tvix/castore/protos/go.mod Normal file
View file

@ -0,0 +1,19 @@
module code.tvl.fyi/tvix/castore/protos
go 1.19
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.8.1 // indirect
golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
golang.org/x/text v0.4.0 // indirect
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
google.golang.org/grpc v1.51.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
)

96
tvix/castore/protos/go.sum Normal file
View file

@ -0,0 +1,96 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=

170
tvix/store/protos/rpc_blobstore.pb.go → tvix/castore/protos/rpc_blobstore.pb.go
View file

@ -5,9 +5,9 @@
// versions:
// protoc-gen-go v1.31.0
// protoc (unknown)
// source: tvix/store/protos/rpc_blobstore.proto
// source: tvix/castore/protos/rpc_blobstore.proto
package storev1
package castorev1
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
@ -35,7 +35,7 @@ type StatBlobRequest struct {
func (x *StatBlobRequest) Reset() {
*x = StatBlobRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0]
mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -48,7 +48,7 @@ func (x *StatBlobRequest) String() string {
func (*StatBlobRequest) ProtoMessage() {}
func (x *StatBlobRequest) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0]
mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -61,7 +61,7 @@ func (x *StatBlobRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use StatBlobRequest.ProtoReflect.Descriptor instead.
func (*StatBlobRequest) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{0}
return file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP(), []int{0}
}
func (x *StatBlobRequest) GetDigest() []byte {
@ -80,7 +80,7 @@ type BlobMeta struct {
func (x *BlobMeta) Reset() {
*x = BlobMeta{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1]
mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -93,7 +93,7 @@ func (x *BlobMeta) String() string {
func (*BlobMeta) ProtoMessage() {}
func (x *BlobMeta) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1]
mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -106,7 +106,7 @@ func (x *BlobMeta) ProtoReflect() protoreflect.Message {
// Deprecated: Use BlobMeta.ProtoReflect.Descriptor instead.
func (*BlobMeta) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1}
return file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP(), []int{1}
}
type ReadBlobRequest struct {
@ -121,7 +121,7 @@ type ReadBlobRequest struct {
func (x *ReadBlobRequest) Reset() {
*x = ReadBlobRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2]
mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -134,7 +134,7 @@ func (x *ReadBlobRequest) String() string {
func (*ReadBlobRequest) ProtoMessage() {}
func (x *ReadBlobRequest) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2]
mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -147,7 +147,7 @@ func (x *ReadBlobRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use ReadBlobRequest.ProtoReflect.Descriptor instead.
func (*ReadBlobRequest) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{2}
return file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP(), []int{2}
}
func (x *ReadBlobRequest) GetDigest() []byte {
@ -170,7 +170,7 @@ type BlobChunk struct {
func (x *BlobChunk) Reset() {
*x = BlobChunk{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3]
mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -183,7 +183,7 @@ func (x *BlobChunk) String() string {
func (*BlobChunk) ProtoMessage() {}
func (x *BlobChunk) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3]
mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -196,7 +196,7 @@ func (x *BlobChunk) ProtoReflect() protoreflect.Message {
// Deprecated: Use BlobChunk.ProtoReflect.Descriptor instead.
func (*BlobChunk) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{3}
return file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP(), []int{3}
}
func (x *BlobChunk) GetData() []byte {
@ -218,7 +218,7 @@ type PutBlobResponse struct {
func (x *PutBlobResponse) Reset() {
*x = PutBlobResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4]
mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -231,7 +231,7 @@ func (x *PutBlobResponse) String() string {
func (*PutBlobResponse) ProtoMessage() {}
func (x *PutBlobResponse) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4]
mi := &file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[4]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -244,7 +244,7 @@ func (x *PutBlobResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use PutBlobResponse.ProtoReflect.Descriptor instead.
func (*PutBlobResponse) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP(), []int{4}
return file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP(), []int{4}
}
func (x *PutBlobResponse) GetDigest() []byte {
@ -254,69 +254,71 @@ func (x *PutBlobResponse) GetDigest() []byte {
return nil
}
var File_tvix_store_protos_rpc_blobstore_proto protoreflect.FileDescriptor
var File_tvix_castore_protos_rpc_blobstore_proto protoreflect.FileDescriptor
var file_tvix_store_protos_rpc_blobstore_proto_rawDesc = []byte{
0x0a, 0x25, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74,
0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x29, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x74, 0x42, 0x6c,
0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67,
0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73,
0x74, 0x22, 0x0a, 0x0a, 0x08, 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74, 0x61, 0x22, 0x29, 0x0a,
0x0f, 0x52, 0x65, 0x61, 0x64, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
var file_tvix_castore_protos_rpc_blobstore_proto_rawDesc = []byte{
0x0a, 0x27, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x62, 0x6c, 0x6f, 0x62, 0x73, 0x74,
0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x74, 0x76, 0x69, 0x78, 0x2e,
0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x22, 0x29, 0x0a, 0x0f, 0x53, 0x74,
0x61, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a,
0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64,
0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x0a, 0x0a, 0x08, 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74,
0x61, 0x22, 0x29, 0x0a, 0x0f, 0x52, 0x65, 0x61, 0x64, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x1f, 0x0a, 0x09,
0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74,
0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x29, 0x0a,
0x0f, 0x50, 0x75, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c,
0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x1f, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x62,
0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20,
0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x29, 0x0a, 0x0f, 0x50, 0x75, 0x74,
0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06,
0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69,
0x67, 0x65, 0x73, 0x74, 0x32, 0xd5, 0x01, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, 0x62, 0x53, 0x65, 0x72,
0x76, 0x69, 0x63, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, 0x12, 0x1e, 0x2e, 0x74,
0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61,
0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74,
0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f,
0x62, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x42, 0x0a, 0x04, 0x52, 0x65, 0x61, 0x64, 0x12, 0x1e, 0x2e,
0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65,
0x61, 0x64, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e,
0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c,
0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x41, 0x0a, 0x03, 0x50, 0x75, 0x74,
0x12, 0x18, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31,
0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, 0x1e, 0x2e, 0x74, 0x76, 0x69,
0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x42, 0x6c,
0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x28, 0x5a, 0x26,
0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x32, 0xe1, 0x01, 0x0a, 0x0b, 0x42, 0x6c, 0x6f,
0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74,
0x12, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e,
0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x42, 0x6c, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x19, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x46, 0x0a,
0x04, 0x52, 0x65, 0x61, 0x64, 0x12, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x42, 0x6c, 0x6f, 0x62,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63,
0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x43, 0x68,
0x75, 0x6e, 0x6b, 0x30, 0x01, 0x12, 0x45, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x1a, 0x2e, 0x74,
0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42,
0x6c, 0x6f, 0x62, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e,
0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x42, 0x6c,
0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x2c, 0x5a, 0x2a,
0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69,
0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73,
0x3b, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (
file_tvix_store_protos_rpc_blobstore_proto_rawDescOnce sync.Once
file_tvix_store_protos_rpc_blobstore_proto_rawDescData = file_tvix_store_protos_rpc_blobstore_proto_rawDesc
file_tvix_castore_protos_rpc_blobstore_proto_rawDescOnce sync.Once
file_tvix_castore_protos_rpc_blobstore_proto_rawDescData = file_tvix_castore_protos_rpc_blobstore_proto_rawDesc
)
func file_tvix_store_protos_rpc_blobstore_proto_rawDescGZIP() []byte {
file_tvix_store_protos_rpc_blobstore_proto_rawDescOnce.Do(func() {
file_tvix_store_protos_rpc_blobstore_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_store_protos_rpc_blobstore_proto_rawDescData)
func file_tvix_castore_protos_rpc_blobstore_proto_rawDescGZIP() []byte {
file_tvix_castore_protos_rpc_blobstore_proto_rawDescOnce.Do(func() {
file_tvix_castore_protos_rpc_blobstore_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_castore_protos_rpc_blobstore_proto_rawDescData)
})
return file_tvix_store_protos_rpc_blobstore_proto_rawDescData
return file_tvix_castore_protos_rpc_blobstore_proto_rawDescData
}
var file_tvix_store_protos_rpc_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
var file_tvix_store_protos_rpc_blobstore_proto_goTypes = []interface{}{
(*StatBlobRequest)(nil), // 0: tvix.store.v1.StatBlobRequest
(*BlobMeta)(nil), // 1: tvix.store.v1.BlobMeta
(*ReadBlobRequest)(nil), // 2: tvix.store.v1.ReadBlobRequest
(*BlobChunk)(nil), // 3: tvix.store.v1.BlobChunk
(*PutBlobResponse)(nil), // 4: tvix.store.v1.PutBlobResponse
var file_tvix_castore_protos_rpc_blobstore_proto_msgTypes = make([]protoimpl.MessageInfo, 5)
var file_tvix_castore_protos_rpc_blobstore_proto_goTypes = []interface{}{
(*StatBlobRequest)(nil), // 0: tvix.castore.v1.StatBlobRequest
(*BlobMeta)(nil), // 1: tvix.castore.v1.BlobMeta
(*ReadBlobRequest)(nil), // 2: tvix.castore.v1.ReadBlobRequest
(*BlobChunk)(nil), // 3: tvix.castore.v1.BlobChunk
(*PutBlobResponse)(nil), // 4: tvix.castore.v1.PutBlobResponse
}
var file_tvix_store_protos_rpc_blobstore_proto_depIdxs = []int32{
0, // 0: tvix.store.v1.BlobService.Stat:input_type -> tvix.store.v1.StatBlobRequest
2, // 1: tvix.store.v1.BlobService.Read:input_type -> tvix.store.v1.ReadBlobRequest
3, // 2: tvix.store.v1.BlobService.Put:input_type -> tvix.store.v1.BlobChunk
1, // 3: tvix.store.v1.BlobService.Stat:output_type -> tvix.store.v1.BlobMeta
3, // 4: tvix.store.v1.BlobService.Read:output_type -> tvix.store.v1.BlobChunk
4, // 5: tvix.store.v1.BlobService.Put:output_type -> tvix.store.v1.PutBlobResponse
var file_tvix_castore_protos_rpc_blobstore_proto_depIdxs = []int32{
0, // 0: tvix.castore.v1.BlobService.Stat:input_type -> tvix.castore.v1.StatBlobRequest
2, // 1: tvix.castore.v1.BlobService.Read:input_type -> tvix.castore.v1.ReadBlobRequest
3, // 2: tvix.castore.v1.BlobService.Put:input_type -> tvix.castore.v1.BlobChunk
1, // 3: tvix.castore.v1.BlobService.Stat:output_type -> tvix.castore.v1.BlobMeta
3, // 4: tvix.castore.v1.BlobService.Read:output_type -> tvix.castore.v1.BlobChunk
4, // 5: tvix.castore.v1.BlobService.Put:output_type -> tvix.castore.v1.PutBlobResponse
3, // [3:6] is the sub-list for method output_type
0, // [0:3] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
@ -324,13 +326,13 @@ var file_tvix_store_protos_rpc_blobstore_proto_depIdxs = []int32{
0, // [0:0] is the sub-list for field type_name
}
func init() { file_tvix_store_protos_rpc_blobstore_proto_init() }
func file_tvix_store_protos_rpc_blobstore_proto_init() {
if File_tvix_store_protos_rpc_blobstore_proto != nil {
func init() { file_tvix_castore_protos_rpc_blobstore_proto_init() }
func file_tvix_castore_protos_rpc_blobstore_proto_init() {
if File_tvix_castore_protos_rpc_blobstore_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*StatBlobRequest); i {
case 0:
return &v.state
@ -342,7 +344,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
return nil
}
}
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*BlobMeta); i {
case 0:
return &v.state
@ -354,7 +356,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
return nil
}
}
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*ReadBlobRequest); i {
case 0:
return &v.state
@ -366,7 +368,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
return nil
}
}
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*BlobChunk); i {
case 0:
return &v.state
@ -378,7 +380,7 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
return nil
}
}
file_tvix_store_protos_rpc_blobstore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
file_tvix_castore_protos_rpc_blobstore_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*PutBlobResponse); i {
case 0:
return &v.state
@ -395,18 +397,18 @@ func file_tvix_store_protos_rpc_blobstore_proto_init() {
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_tvix_store_protos_rpc_blobstore_proto_rawDesc,
RawDescriptor: file_tvix_castore_protos_rpc_blobstore_proto_rawDesc,
NumEnums: 0,
NumMessages: 5,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_tvix_store_protos_rpc_blobstore_proto_goTypes,
DependencyIndexes: file_tvix_store_protos_rpc_blobstore_proto_depIdxs,
MessageInfos: file_tvix_store_protos_rpc_blobstore_proto_msgTypes,
GoTypes: file_tvix_castore_protos_rpc_blobstore_proto_goTypes,
DependencyIndexes: file_tvix_castore_protos_rpc_blobstore_proto_depIdxs,
MessageInfos: file_tvix_castore_protos_rpc_blobstore_proto_msgTypes,
}.Build()
File_tvix_store_protos_rpc_blobstore_proto = out.File
file_tvix_store_protos_rpc_blobstore_proto_rawDesc = nil
file_tvix_store_protos_rpc_blobstore_proto_goTypes = nil
file_tvix_store_protos_rpc_blobstore_proto_depIdxs = nil
File_tvix_castore_protos_rpc_blobstore_proto = out.File
file_tvix_castore_protos_rpc_blobstore_proto_rawDesc = nil
file_tvix_castore_protos_rpc_blobstore_proto_goTypes = nil
file_tvix_castore_protos_rpc_blobstore_proto_depIdxs = nil
}

4
tvix/store/protos/rpc_blobstore.proto → tvix/castore/protos/rpc_blobstore.proto
View file

@ -2,9 +2,9 @@
// Copyright © 2022 The Tvix Authors
syntax = "proto3";
package tvix.store.v1;
package tvix.castore.v1;
option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
option go_package = "code.tvl.fyi/tvix/castore/protos;castorev1";
service BlobService {
// In the future, Stat will expose more metadata about a given blob,

14
tvix/store/protos/rpc_blobstore_grpc.pb.go → tvix/castore/protos/rpc_blobstore_grpc.pb.go
View file

@ -5,9 +5,9 @@
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc (unknown)
// source: tvix/store/protos/rpc_blobstore.proto
// source: tvix/castore/protos/rpc_blobstore.proto
package storev1
package castorev1
import (
context "context"
@ -22,9 +22,9 @@ import (
const _ = grpc.SupportPackageIsVersion7
const (
BlobService_Stat_FullMethodName = "/tvix.store.v1.BlobService/Stat"
BlobService_Read_FullMethodName = "/tvix.store.v1.BlobService/Read"
BlobService_Put_FullMethodName = "/tvix.store.v1.BlobService/Put"
BlobService_Stat_FullMethodName = "/tvix.castore.v1.BlobService/Stat"
BlobService_Read_FullMethodName = "/tvix.castore.v1.BlobService/Read"
BlobService_Put_FullMethodName = "/tvix.castore.v1.BlobService/Put"
)
// BlobServiceClient is the client API for BlobService service.
@ -250,7 +250,7 @@ func (x *blobServicePutServer) Recv() (*BlobChunk, error) {
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var BlobService_ServiceDesc = grpc.ServiceDesc{
ServiceName: "tvix.store.v1.BlobService",
ServiceName: "tvix.castore.v1.BlobService",
HandlerType: (*BlobServiceServer)(nil),
Methods: []grpc.MethodDesc{
{
@ -270,5 +270,5 @@ var BlobService_ServiceDesc = grpc.ServiceDesc{
ClientStreams: true,
},
},
Metadata: "tvix/store/protos/rpc_blobstore.proto",
Metadata: "tvix/castore/protos/rpc_blobstore.proto",
}

273
tvix/castore/protos/rpc_directory.pb.go Normal file
View file

@ -0,0 +1,273 @@
// SPDX-License-Identifier: MIT
// Copyright © 2022 The Tvix Authors
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
// protoc (unknown)
// source: tvix/castore/protos/rpc_directory.proto
package castorev1
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type GetDirectoryRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Types that are assignable to ByWhat:
//
// *GetDirectoryRequest_Digest
ByWhat isGetDirectoryRequest_ByWhat `protobuf_oneof:"by_what"`
// If set to true, recursively resolve all child Directory messages.
// Directory messages SHOULD be streamed in a recursive breadth-first walk,
// but other orders are also fine, as long as Directory messages are only
// sent after they are referred to from previously sent Directory messages.
Recursive bool `protobuf:"varint,2,opt,name=recursive,proto3" json:"recursive,omitempty"`
}
func (x *GetDirectoryRequest) Reset() {
*x = GetDirectoryRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_castore_protos_rpc_directory_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *GetDirectoryRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*GetDirectoryRequest) ProtoMessage() {}
func (x *GetDirectoryRequest) ProtoReflect() protoreflect.Message {
mi := &file_tvix_castore_protos_rpc_directory_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use GetDirectoryRequest.ProtoReflect.Descriptor instead.
func (*GetDirectoryRequest) Descriptor() ([]byte, []int) {
return file_tvix_castore_protos_rpc_directory_proto_rawDescGZIP(), []int{0}
}
func (m *GetDirectoryRequest) GetByWhat() isGetDirectoryRequest_ByWhat {
if m != nil {
return m.ByWhat
}
return nil
}
func (x *GetDirectoryRequest) GetDigest() []byte {
if x, ok := x.GetByWhat().(*GetDirectoryRequest_Digest); ok {
return x.Digest
}
return nil
}
func (x *GetDirectoryRequest) GetRecursive() bool {
if x != nil {
return x.Recursive
}
return false
}
type isGetDirectoryRequest_ByWhat interface {
isGetDirectoryRequest_ByWhat()
}
type GetDirectoryRequest_Digest struct {
// The blake3 hash of the (root) Directory message, serialized in
// protobuf canonical form.
// Keep in mind this can be a subtree of another root.
Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3,oneof"`
}
func (*GetDirectoryRequest_Digest) isGetDirectoryRequest_ByWhat() {}
type PutDirectoryResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
RootDigest []byte `protobuf:"bytes,1,opt,name=root_digest,json=rootDigest,proto3" json:"root_digest,omitempty"`
}
func (x *PutDirectoryResponse) Reset() {
*x = PutDirectoryResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_castore_protos_rpc_directory_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *PutDirectoryResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*PutDirectoryResponse) ProtoMessage() {}
func (x *PutDirectoryResponse) ProtoReflect() protoreflect.Message {
mi := &file_tvix_castore_protos_rpc_directory_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use PutDirectoryResponse.ProtoReflect.Descriptor instead.
func (*PutDirectoryResponse) Descriptor() ([]byte, []int) {
return file_tvix_castore_protos_rpc_directory_proto_rawDescGZIP(), []int{1}
}
func (x *PutDirectoryResponse) GetRootDigest() []byte {
if x != nil {
return x.RootDigest
}
return nil
}
var File_tvix_castore_protos_rpc_directory_proto protoreflect.FileDescriptor
var file_tvix_castore_protos_rpc_directory_proto_rawDesc = []byte{
0x0a, 0x27, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x74, 0x76, 0x69, 0x78, 0x2e,
0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78,
0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f,
0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x58, 0x0a,
0x13, 0x47, 0x65, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x1c,
0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
0x08, 0x52, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x42, 0x09, 0x0a, 0x07,
0x62, 0x79, 0x5f, 0x77, 0x68, 0x61, 0x74, 0x22, 0x37, 0x0a, 0x14, 0x50, 0x75, 0x74, 0x44, 0x69,
0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
0x1f, 0x0a, 0x0b, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x72, 0x6f, 0x6f, 0x74, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74,
0x32, 0xa9, 0x01, 0x0a, 0x10, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x53, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x49, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x24, 0x2e, 0x74,
0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47,
0x65, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x30, 0x01,
0x12, 0x4a, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x1a, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63,
0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x79, 0x1a, 0x25, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f,
0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f,
0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x2c, 0x5a, 0x2a,
0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69,
0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73,
0x3b, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x33,
}
var (
file_tvix_castore_protos_rpc_directory_proto_rawDescOnce sync.Once
file_tvix_castore_protos_rpc_directory_proto_rawDescData = file_tvix_castore_protos_rpc_directory_proto_rawDesc
)
func file_tvix_castore_protos_rpc_directory_proto_rawDescGZIP() []byte {
file_tvix_castore_protos_rpc_directory_proto_rawDescOnce.Do(func() {
file_tvix_castore_protos_rpc_directory_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_castore_protos_rpc_directory_proto_rawDescData)
})
return file_tvix_castore_protos_rpc_directory_proto_rawDescData
}
var file_tvix_castore_protos_rpc_directory_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_tvix_castore_protos_rpc_directory_proto_goTypes = []interface{}{
(*GetDirectoryRequest)(nil), // 0: tvix.castore.v1.GetDirectoryRequest
(*PutDirectoryResponse)(nil), // 1: tvix.castore.v1.PutDirectoryResponse
(*Directory)(nil), // 2: tvix.castore.v1.Directory
}
var file_tvix_castore_protos_rpc_directory_proto_depIdxs = []int32{
0, // 0: tvix.castore.v1.DirectoryService.Get:input_type -> tvix.castore.v1.GetDirectoryRequest
2, // 1: tvix.castore.v1.DirectoryService.Put:input_type -> tvix.castore.v1.Directory
2, // 2: tvix.castore.v1.DirectoryService.Get:output_type -> tvix.castore.v1.Directory
1, // 3: tvix.castore.v1.DirectoryService.Put:output_type -> tvix.castore.v1.PutDirectoryResponse
2, // [2:4] is the sub-list for method output_type
0, // [0:2] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_tvix_castore_protos_rpc_directory_proto_init() }
func file_tvix_castore_protos_rpc_directory_proto_init() {
if File_tvix_castore_protos_rpc_directory_proto != nil {
return
}
file_tvix_castore_protos_castore_proto_init()
if !protoimpl.UnsafeEnabled {
file_tvix_castore_protos_rpc_directory_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*GetDirectoryRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_castore_protos_rpc_directory_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*PutDirectoryResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
file_tvix_castore_protos_rpc_directory_proto_msgTypes[0].OneofWrappers = []interface{}{
(*GetDirectoryRequest_Digest)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_tvix_castore_protos_rpc_directory_proto_rawDesc,
NumEnums: 0,
NumMessages: 2,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_tvix_castore_protos_rpc_directory_proto_goTypes,
DependencyIndexes: file_tvix_castore_protos_rpc_directory_proto_depIdxs,
MessageInfos: file_tvix_castore_protos_rpc_directory_proto_msgTypes,
}.Build()
File_tvix_castore_protos_rpc_directory_proto = out.File
file_tvix_castore_protos_rpc_directory_proto_rawDesc = nil
file_tvix_castore_protos_rpc_directory_proto_goTypes = nil
file_tvix_castore_protos_rpc_directory_proto_depIdxs = nil
}

6
tvix/store/protos/rpc_directory.proto → tvix/castore/protos/rpc_directory.proto
View file

@ -2,11 +2,11 @@
// Copyright © 2022 The Tvix Authors
syntax = "proto3";
package tvix.store.v1;
package tvix.castore.v1;
import "tvix/store/protos/castore.proto";
import "tvix/castore/protos/castore.proto";
option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
option go_package = "code.tvl.fyi/tvix/castore/protos;castorev1";
service DirectoryService {
// Get retrieves a stream of Directory messages, by using the lookup

12
tvix/store/protos/rpc_directory_grpc.pb.go → tvix/castore/protos/rpc_directory_grpc.pb.go
View file

@ -5,9 +5,9 @@
// versions:
// - protoc-gen-go-grpc v1.3.0
// - protoc (unknown)
// source: tvix/store/protos/rpc_directory.proto
// source: tvix/castore/protos/rpc_directory.proto
package storev1
package castorev1
import (
context "context"
@ -22,8 +22,8 @@ import (
const _ = grpc.SupportPackageIsVersion7
const (
DirectoryService_Get_FullMethodName = "/tvix.store.v1.DirectoryService/Get"
DirectoryService_Put_FullMethodName = "/tvix.store.v1.DirectoryService/Put"
DirectoryService_Get_FullMethodName = "/tvix.castore.v1.DirectoryService/Get"
DirectoryService_Put_FullMethodName = "/tvix.castore.v1.DirectoryService/Put"
)
// DirectoryServiceClient is the client API for DirectoryService service.
@ -219,7 +219,7 @@ func (x *directoryServicePutServer) Recv() (*Directory, error) {
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var DirectoryService_ServiceDesc = grpc.ServiceDesc{
ServiceName: "tvix.store.v1.DirectoryService",
ServiceName: "tvix.castore.v1.DirectoryService",
HandlerType: (*DirectoryServiceServer)(nil),
Methods: []grpc.MethodDesc{},
Streams: []grpc.StreamDesc{
@ -234,5 +234,5 @@ var DirectoryService_ServiceDesc = grpc.ServiceDesc{
ClientStreams: true,
},
},
Metadata: "tvix/store/protos/rpc_directory.proto",
Metadata: "tvix/castore/protos/rpc_directory.proto",
}

0
tvix/store/src/blobservice/from_addr.rs → tvix/castore/src/blobservice/from_addr.rs
View file

2
tvix/store/src/blobservice/grpc.rs → tvix/castore/src/blobservice/grpc.rs
View file

@ -289,8 +289,8 @@ mod tests {
use tokio_stream::wrappers::UnixListenerStream;
use crate::blobservice::MemoryBlobService;
use crate::fixtures;
use crate::proto::GRPCBlobServiceWrapper;
use crate::tests::fixtures;
use super::BlobService;
use super::GRPCBlobService;

0
tvix/store/src/blobservice/memory.rs → tvix/castore/src/blobservice/memory.rs
View file

0
tvix/store/src/blobservice/mod.rs → tvix/castore/src/blobservice/mod.rs
View file

0
tvix/store/src/blobservice/naive_seeker.rs → tvix/castore/src/blobservice/naive_seeker.rs
View file

0
tvix/store/src/blobservice/sled.rs → tvix/castore/src/blobservice/sled.rs
View file

2
tvix/store/src/blobservice/tests.rs → tvix/castore/src/blobservice/tests.rs
View file

@ -9,7 +9,7 @@ use super::B3Digest;
use super::BlobService;
use super::MemoryBlobService;
use super::SledBlobService;
use crate::tests::fixtures;
use crate::fixtures;
// TODO: avoid having to define all different services we test against for all functions.
// maybe something like rstest can be used?

0
tvix/store/src/digests.rs → tvix/castore/src/digests.rs
View file

0
tvix/store/src/directoryservice/from_addr.rs → tvix/castore/src/directoryservice/from_addr.rs
View file

6
tvix/store/src/directoryservice/grpc.rs → tvix/castore/src/directoryservice/grpc.rs
View file

@ -348,12 +348,10 @@ mod tests {
use crate::{
directoryservice::DirectoryService,
fixtures::{DIRECTORY_A, DIRECTORY_B},
proto,
proto::{directory_service_server::DirectoryServiceServer, GRPCDirectoryServiceWrapper},
tests::{
fixtures::{DIRECTORY_A, DIRECTORY_B},
utils::gen_directory_service,
},
utils::gen_directory_service,
};
#[test]

0
tvix/store/src/directoryservice/memory.rs → tvix/castore/src/directoryservice/memory.rs
View file

0
tvix/store/src/directoryservice/mod.rs → tvix/castore/src/directoryservice/mod.rs
View file

0
tvix/store/src/directoryservice/sled.rs → tvix/castore/src/directoryservice/sled.rs
View file

6
tvix/store/src/directoryservice/traverse.rs → tvix/castore/src/directoryservice/traverse.rs
View file

@ -84,10 +84,8 @@ pub async fn descend_to(
mod tests {
use std::path::PathBuf;
use crate::tests::{
fixtures::{DIRECTORY_COMPLICATED, DIRECTORY_WITH_KEEP},
utils::gen_directory_service,
};
use crate::fixtures::{DIRECTORY_COMPLICATED, DIRECTORY_WITH_KEEP};
use crate::utils::gen_directory_service;
use super::descend_to;

0
tvix/store/src/directoryservice/utils.rs → tvix/castore/src/directoryservice/utils.rs
View file

0
tvix/store/src/errors.rs → tvix/castore/src/errors.rs
View file

95
tvix/castore/src/fixtures.rs Normal file
View file

@ -0,0 +1,95 @@
use crate::{
proto::{self, Directory, DirectoryNode, FileNode, SymlinkNode},
B3Digest,
};
use lazy_static::lazy_static;
pub const HELLOWORLD_BLOB_CONTENTS: &[u8] = b"Hello World!";
pub const EMPTY_BLOB_CONTENTS: &[u8] = b"";
lazy_static! {
pub static ref DUMMY_DIGEST: B3Digest = {
let u: &[u8; 32] = &[
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
];
u.into()
};
pub static ref DUMMY_DIGEST_2: B3Digest = {
let u: &[u8; 32] = &[
0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
];
u.into()
};
pub static ref DUMMY_DATA_1: bytes::Bytes = vec![0x01, 0x02, 0x03].into();
pub static ref DUMMY_DATA_2: bytes::Bytes = vec![0x04, 0x05].into();
pub static ref HELLOWORLD_BLOB_DIGEST: B3Digest =
blake3::hash(HELLOWORLD_BLOB_CONTENTS).as_bytes().into();
pub static ref EMPTY_BLOB_DIGEST: B3Digest =
blake3::hash(EMPTY_BLOB_CONTENTS).as_bytes().into();
// 2 bytes
pub static ref BLOB_A: bytes::Bytes = vec![0x00, 0x01].into();
pub static ref BLOB_A_DIGEST: B3Digest = blake3::hash(&BLOB_A).as_bytes().into();
// 1MB
pub static ref BLOB_B: bytes::Bytes = (0..255).collect::<Vec<u8>>().repeat(4 * 1024).into();
pub static ref BLOB_B_DIGEST: B3Digest = blake3::hash(&BLOB_B).as_bytes().into();
// Directories
pub static ref DIRECTORY_WITH_KEEP: proto::Directory = proto::Directory {
directories: vec![],
files: vec![FileNode {
name: b".keep".to_vec().into(),
digest: EMPTY_BLOB_DIGEST.clone().into(),
size: 0,
executable: false,
}],
symlinks: vec![],
};
pub static ref DIRECTORY_COMPLICATED: proto::Directory = proto::Directory {
directories: vec![DirectoryNode {
name: b"keep".to_vec().into(),
digest: DIRECTORY_WITH_KEEP.digest().into(),
size: DIRECTORY_WITH_KEEP.size(),
}],
files: vec![FileNode {
name: b".keep".to_vec().into(),
digest: EMPTY_BLOB_DIGEST.clone().into(),
size: 0,
executable: false,
}],
symlinks: vec![SymlinkNode {
name: b"aa".to_vec().into(),
target: b"/nix/store/somewhereelse".to_vec().into(),
}],
};
pub static ref DIRECTORY_A: Directory = Directory::default();
pub static ref DIRECTORY_B: Directory = Directory {
directories: vec![DirectoryNode {
name: b"a".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
}],
..Default::default()
};
pub static ref DIRECTORY_C: Directory = Directory {
directories: vec![
DirectoryNode {
name: b"a".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
},
DirectoryNode {
name: b"a'".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
}
],
..Default::default()
};
}

48
tvix/store/src/import.rs → tvix/castore/src/import.rs
View file

@ -1,6 +1,12 @@
use crate::blobservice::BlobService;
use crate::directoryservice::DirectoryPutter;
use crate::directoryservice::DirectoryService;
use crate::{directoryservice::DirectoryPutter, proto};
use crate::proto::node::Node;
use crate::proto::Directory;
use crate::proto::DirectoryNode;
use crate::proto::FileNode;
use crate::proto::SymlinkNode;
use crate::Error as CastoreError;
use std::os::unix::ffi::OsStrExt;
use std::sync::Arc;
use std::{
@ -15,7 +21,7 @@ use walkdir::WalkDir;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("failed to upload directory at {0}: {1}")]
UploadDirectoryError(PathBuf, crate::Error),
UploadDirectoryError(PathBuf, CastoreError),
#[error("invalid encoding encountered for entry {0:?}")]
InvalidEncoding(PathBuf),
@ -30,11 +36,11 @@ pub enum Error {
UnableToRead(PathBuf, std::io::Error),
}
impl From<super::Error> for Error {
fn from(value: super::Error) -> Self {
impl From<CastoreError> for Error {
fn from(value: CastoreError) -> Self {
match value {
crate::Error::InvalidRequest(_) => panic!("tvix bug"),
crate::Error::StorageError(_) => panic!("error"),
CastoreError::InvalidRequest(_) => panic!("tvix bug"),
CastoreError::StorageError(_) => panic!("error"),
}
}
}
@ -59,8 +65,8 @@ async fn process_entry(
blob_service: Arc<dyn BlobService>,
directory_putter: &mut Box<dyn DirectoryPutter>,
entry: &walkdir::DirEntry,
maybe_directory: Option<proto::Directory>,
) -> Result<proto::node::Node, Error> {
maybe_directory: Option<Directory>,
) -> Result<Node, Error> {
let file_type = entry.file_type();
if file_type.is_dir() {
@ -75,7 +81,7 @@ async fn process_entry(
.await
.map_err(|e| Error::UploadDirectoryError(entry.path().to_path_buf(), e))?;
return Ok(proto::node::Node::Directory(proto::DirectoryNode {
return Ok(Node::Directory(DirectoryNode {
name: entry.file_name().as_bytes().to_owned().into(),
digest: directory_digest.into(),
size: directory_size,
@ -90,7 +96,7 @@ async fn process_entry(
.to_owned()
.into();
return Ok(proto::node::Node::Symlink(proto::SymlinkNode {
return Ok(Node::Symlink(SymlinkNode {
name: entry.file_name().as_bytes().to_owned().into(),
target,
}));
@ -113,7 +119,7 @@ async fn process_entry(
let digest = writer.close().await?;
return Ok(proto::node::Node::File(proto::FileNode {
return Ok(Node::File(FileNode {
name: entry.file_name().as_bytes().to_vec().into(),
digest: digest.into(),
size: metadata.len() as u32,
@ -132,17 +138,17 @@ async fn process_entry(
/// It does not follow symlinks at the root, they will be ingested as actual
/// symlinks.
///
/// It's not interacting with a
/// [PathInfoService](crate::pathinfoservice::PathInfoService), it's up to the
/// caller to possibly register it somewhere (and potentially rename it based on
/// some naming scheme.
/// It's not interacting with a PathInfoService (from tvix-store), or anything
/// else giving it a "non-content-addressed name".
/// It's up to the caller to possibly register it somewhere (and potentially
/// rename it based on some naming scheme)
#[instrument(skip(blob_service, directory_service), fields(path=?p))]
pub async fn ingest_path<P: AsRef<Path> + Debug>(
blob_service: Arc<dyn BlobService>,
directory_service: Arc<dyn DirectoryService>,
p: P,
) -> Result<proto::node::Node, Error> {
let mut directories: HashMap<PathBuf, proto::Directory> = HashMap::default();
) -> Result<Node, Error> {
let mut directories: HashMap<PathBuf, Directory> = HashMap::default();
// TODO: pass this one instead?
let mut directory_putter = directory_service.put_multiple_start();
@ -157,7 +163,7 @@ pub async fn ingest_path<P: AsRef<Path> + Debug>(
// process_entry wants an Option<Directory> in case the entry points to a directory.
// make sure to provide it.
let maybe_directory: Option<proto::Directory> = {
let maybe_directory: Option<Directory> = {
if entry.file_type().is_dir() {
Some(
directories
@ -188,9 +194,9 @@ pub async fn ingest_path<P: AsRef<Path> + Debug>(
// record node in parent directory, creating a new [proto:Directory] if not there yet.
let parent_directory = directories.entry(parent_path).or_default();
match node {
proto::node::Node::Directory(e) => parent_directory.directories.push(e),
proto::node::Node::File(e) => parent_directory.files.push(e),
proto::node::Node::Symlink(e) => parent_directory.symlinks.push(e),
Node::Directory(e) => parent_directory.directories.push(e),
Node::File(e) => parent_directory.files.push(e),
Node::Symlink(e) => parent_directory.symlinks.push(e),
}
}
}

15
tvix/castore/src/lib.rs Normal file
View file

@ -0,0 +1,15 @@
mod digests;
mod errors;
pub mod blobservice;
pub mod directoryservice;
pub mod fixtures;
pub mod import;
pub mod proto;
pub mod utils;
pub use digests::B3Digest;
pub use errors::Error;
#[cfg(test)]
mod tests;

0
tvix/store/src/proto/grpc_blobservice_wrapper.rs → tvix/castore/src/proto/grpc_blobservice_wrapper.rs
View file

0
tvix/store/src/proto/grpc_directoryservice_wrapper.rs → tvix/castore/src/proto/grpc_directoryservice_wrapper.rs
View file

279
tvix/castore/src/proto/mod.rs Normal file
View file

@ -0,0 +1,279 @@
#![allow(clippy::derive_partial_eq_without_eq, non_snake_case)]
// https://github.com/hyperium/tonic/issues/1056
use data_encoding::BASE64;
use std::{collections::HashSet, iter::Peekable};
use thiserror::Error;
use prost::Message;
mod grpc_blobservice_wrapper;
mod grpc_directoryservice_wrapper;
pub use grpc_blobservice_wrapper::GRPCBlobServiceWrapper;
pub use grpc_directoryservice_wrapper::GRPCDirectoryServiceWrapper;
use crate::B3Digest;
tonic::include_proto!("tvix.castore.v1");
#[cfg(feature = "reflection")]
/// Compiled file descriptors for implementing [gRPC
/// reflection](https://github.com/grpc/grpc/blob/master/doc/server-reflection.md) with e.g.
/// [`tonic_reflection`](https://docs.rs/tonic-reflection).
pub const FILE_DESCRIPTOR_SET: &[u8] = tonic::include_file_descriptor_set!("tvix.castore.v1");
#[cfg(test)]
mod tests;
/// Errors that can occur during the validation of Directory messages.
#[derive(Debug, PartialEq, Eq, Error)]
pub enum ValidateDirectoryError {
/// Elements are not in sorted order
#[error("{} is not sorted", std::str::from_utf8(.0).unwrap_or(&BASE64.encode(.0)))]
WrongSorting(Vec<u8>),
/// Multiple elements with the same name encountered
#[error("{0:?} is a duplicate name")]
DuplicateName(Vec<u8>),
/// Invalid name encountered
#[error("Invalid name in {0:?}")]
InvalidName(Vec<u8>),
/// Invalid digest length encountered
#[error("Invalid Digest length: {0}")]
InvalidDigestLen(usize),
}
/// Checks a Node name for validity as an intermediate node, and returns an
/// error that's generated from the supplied constructor.
///
/// We disallow slashes, null bytes, '.', '..' and the empty string.
fn validate_node_name<E>(name: &[u8], err: fn(Vec<u8>) -> E) -> Result<(), E> {
if name.is_empty()
|| name == b".."
|| name == b"."
|| name.contains(&0x00)
|| name.contains(&b'/')
{
return Err(err(name.to_vec()));
}
Ok(())
}
/// NamedNode is implemented for [FileNode], [DirectoryNode] and [SymlinkNode]
/// and [node::Node], so we can ask all of them for the name easily.
pub trait NamedNode {
fn get_name(&self) -> &[u8];
}
impl NamedNode for &FileNode {
fn get_name(&self) -> &[u8] {
&self.name
}
}
impl NamedNode for &DirectoryNode {
fn get_name(&self) -> &[u8] {
&self.name
}
}
impl NamedNode for &SymlinkNode {
fn get_name(&self) -> &[u8] {
&self.name
}
}
impl NamedNode for node::Node {
fn get_name(&self) -> &[u8] {
match self {
node::Node::File(node_file) => &node_file.name,
node::Node::Directory(node_directory) => &node_directory.name,
node::Node::Symlink(node_symlink) => &node_symlink.name,
}
}
}
impl node::Node {
/// Returns the node with a new name.
pub fn rename(self, name: bytes::Bytes) -> Self {
match self {
node::Node::Directory(n) => node::Node::Directory(DirectoryNode { name, ..n }),
node::Node::File(n) => node::Node::File(FileNode { name, ..n }),
node::Node::Symlink(n) => node::Node::Symlink(SymlinkNode { name, ..n }),
}
}
}
/// Accepts a name, and a mutable reference to the previous name.
/// If the passed name is larger than the previous one, the reference is updated.
/// If it's not, an error is returned.
fn update_if_lt_prev<'n>(
prev_name: &mut &'n [u8],
name: &'n [u8],
) -> Result<(), ValidateDirectoryError> {
if *name < **prev_name {
return Err(ValidateDirectoryError::WrongSorting(name.to_vec()));
}
*prev_name = name;
Ok(())
}
/// Inserts the given name into a HashSet if it's not already in there.
/// If it is, an error is returned.
fn insert_once<'n>(
seen_names: &mut HashSet<&'n [u8]>,
name: &'n [u8],
) -> Result<(), ValidateDirectoryError> {
if seen_names.get(name).is_some() {
return Err(ValidateDirectoryError::DuplicateName(name.to_vec()));
}
seen_names.insert(name);
Ok(())
}
impl Directory {
/// The size of a directory is the number of all regular and symlink elements,
/// the number of directory elements, and their size fields.
pub fn size(&self) -> u32 {
self.files.len() as u32
+ self.symlinks.len() as u32
+ self
.directories
.iter()
.fold(0, |acc: u32, e| (acc + 1 + e.size))
}
/// Calculates the digest of a Directory, which is the blake3 hash of a
/// Directory protobuf message, serialized in protobuf canonical form.
pub fn digest(&self) -> B3Digest {
let mut hasher = blake3::Hasher::new();
hasher
.update(&self.encode_to_vec())
.finalize()
.as_bytes()
.into()
}
/// validate checks the directory for invalid data, such as:
/// - violations of name restrictions
/// - invalid digest lengths
/// - not properly sorted lists
/// - duplicate names in the three lists
pub fn validate(&self) -> Result<(), ValidateDirectoryError> {
let mut seen_names: HashSet<&[u8]> = HashSet::new();
let mut last_directory_name: &[u8] = b"";
let mut last_file_name: &[u8] = b"";
let mut last_symlink_name: &[u8] = b"";
// check directories
for directory_node in &self.directories {
validate_node_name(&directory_node.name, ValidateDirectoryError::InvalidName)?;
// ensure the digest has the appropriate size.
if TryInto::<B3Digest>::try_into(directory_node.digest.clone()).is_err() {
return Err(ValidateDirectoryError::InvalidDigestLen(
directory_node.digest.len(),
));
}
update_if_lt_prev(&mut last_directory_name, &directory_node.name)?;
insert_once(&mut seen_names, &directory_node.name)?;
}
// check files
for file_node in &self.files {
validate_node_name(&file_node.name, ValidateDirectoryError::InvalidName)?;
if TryInto::<B3Digest>::try_into(file_node.digest.clone()).is_err() {
return Err(ValidateDirectoryError::InvalidDigestLen(
file_node.digest.len(),
));
}
update_if_lt_prev(&mut last_file_name, &file_node.name)?;
insert_once(&mut seen_names, &file_node.name)?;
}
// check symlinks
for symlink_node in &self.symlinks {
validate_node_name(&symlink_node.name, ValidateDirectoryError::InvalidName)?;
update_if_lt_prev(&mut last_symlink_name, &symlink_node.name)?;
insert_once(&mut seen_names, &symlink_node.name)?;
}
Ok(())
}
/// Allows iterating over all three nodes ([DirectoryNode], [FileNode],
/// [SymlinkNode]) in an ordered fashion, as long as the individual lists
/// are sorted (which can be checked by the [Directory::validate]).
pub fn nodes(&self) -> DirectoryNodesIterator {
return DirectoryNodesIterator {
i_directories: self.directories.iter().peekable(),
i_files: self.files.iter().peekable(),
i_symlinks: self.symlinks.iter().peekable(),
};
}
}
/// Struct to hold the state of an iterator over all nodes of a Directory.
///
/// Internally, this keeps peekable Iterators over all three lists of a
/// Directory message.
pub struct DirectoryNodesIterator<'a> {
// directory: &Directory,
i_directories: Peekable<std::slice::Iter<'a, DirectoryNode>>,
i_files: Peekable<std::slice::Iter<'a, FileNode>>,
i_symlinks: Peekable<std::slice::Iter<'a, SymlinkNode>>,
}
/// looks at two elements implementing NamedNode, and returns true if "left
/// is smaller / comes first".
///
/// Some(_) is preferred over None.
fn left_name_lt_right<A: NamedNode, B: NamedNode>(left: Option<&A>, right: Option<&B>) -> bool {
match left {
// if left is None, right always wins
None => false,
Some(left_inner) => {
// left is Some.
match right {
// left is Some, right is None - left wins.
None => true,
Some(right_inner) => {
// both are Some - compare the name.
return left_inner.get_name() < right_inner.get_name();
}
}
}
}
}
impl Iterator for DirectoryNodesIterator<'_> {
type Item = node::Node;
// next returns the next node in the Directory.
// we peek at all three internal iterators, and pick the one with the
// smallest name, to ensure lexicographical ordering.
// The individual lists are already known to be sorted.
fn next(&mut self) -> Option<Self::Item> {
if left_name_lt_right(self.i_directories.peek(), self.i_files.peek()) {
// i_directories is still in the game, compare with symlinks
if left_name_lt_right(self.i_directories.peek(), self.i_symlinks.peek()) {
self.i_directories
.next()
.cloned()
.map(node::Node::Directory)
} else {
self.i_symlinks.next().cloned().map(node::Node::Symlink)
}
} else {
// i_files is still in the game, compare with symlinks
if left_name_lt_right(self.i_files.peek(), self.i_symlinks.peek()) {
self.i_files.next().cloned().map(node::Node::File)
} else {
self.i_symlinks.next().cloned().map(node::Node::Symlink)
}
}
}
}

0
tvix/store/src/proto/tests/directory.rs → tvix/castore/src/proto/tests/directory.rs
View file

0
tvix/store/src/proto/tests/directory_nodes_iterator.rs → tvix/castore/src/proto/tests/directory_nodes_iterator.rs
View file

4
tvix/store/src/proto/tests/grpc_blobservice.rs → tvix/castore/src/proto/tests/grpc_blobservice.rs
View file

@ -1,7 +1,7 @@
use crate::fixtures::{BLOB_A, BLOB_A_DIGEST};
use crate::proto::blob_service_server::BlobService as GRPCBlobService;
use crate::proto::{BlobChunk, GRPCBlobServiceWrapper, ReadBlobRequest, StatBlobRequest};
use crate::tests::fixtures::{BLOB_A, BLOB_A_DIGEST};
use crate::tests::utils::gen_blob_service;
use crate::utils::gen_blob_service;
use tokio_stream::StreamExt;
fn gen_grpc_blob_service() -> GRPCBlobServiceWrapper {

4
tvix/store/src/proto/tests/grpc_directoryservice.rs → tvix/castore/src/proto/tests/grpc_directoryservice.rs
View file

@ -1,9 +1,9 @@
use crate::fixtures::{DIRECTORY_A, DIRECTORY_B, DIRECTORY_C};
use crate::proto::directory_service_server::DirectoryService as GRPCDirectoryService;
use crate::proto::get_directory_request::ByWhat;
use crate::proto::{Directory, DirectoryNode, SymlinkNode};
use crate::proto::{GRPCDirectoryServiceWrapper, GetDirectoryRequest};
use crate::tests::fixtures::{DIRECTORY_A, DIRECTORY_B, DIRECTORY_C};
use crate::tests::utils::gen_directory_service;
use crate::utils::gen_directory_service;
use tokio_stream::StreamExt;
use tonic::Status;

4
tvix/castore/src/proto/tests/mod.rs Normal file
View file

@ -0,0 +1,4 @@
mod directory;
mod directory_nodes_iterator;
mod grpc_blobservice;
mod grpc_directoryservice;

11
tvix/store/src/tests/import.rs → tvix/castore/src/tests/import.rs
View file

@ -1,8 +1,7 @@
use super::utils::{gen_blob_service, gen_directory_service};
use crate::fixtures::*;
use crate::import::ingest_path;
use crate::proto;
use crate::tests::fixtures::DIRECTORY_COMPLICATED;
use crate::tests::fixtures::*;
use crate::utils::{gen_blob_service, gen_directory_service};
use tempfile::TempDir;
#[cfg(target_family = "unix")]
@ -29,7 +28,7 @@ async fn symlink() {
.expect("must succeed");
assert_eq!(
crate::proto::node::Node::Symlink(proto::SymlinkNode {
proto::node::Node::Symlink(proto::SymlinkNode {
name: "doesntmatter".into(),
target: "/nix/store/somewhereelse".into(),
}),
@ -54,7 +53,7 @@ async fn single_file() {
.expect("must succeed");
assert_eq!(
crate::proto::node::Node::File(proto::FileNode {
proto::node::Node::File(proto::FileNode {
name: "root".into(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
@ -94,7 +93,7 @@ async fn complicated() {
// ensure root_node matched expectations
assert_eq!(
crate::proto::node::Node::Directory(proto::DirectoryNode {
proto::node::Node::Directory(proto::DirectoryNode {
name: tmpdir
.path()
.file_name()

1
tvix/castore/src/tests/mod.rs Normal file
View file

@ -0,0 +1 @@
mod import;

19
tvix/castore/src/utils.rs Normal file
View file

@ -0,0 +1,19 @@
//! A crate containing constructors to provide instances of a BlobService and
//! DirectoryService.
//! Only used for testing purposes, but across crates.
//! Should be removed once we have a better concept of a "Service registry".
use std::sync::Arc;
use crate::{
blobservice::{BlobService, MemoryBlobService},
directoryservice::{DirectoryService, MemoryDirectoryService},
};
pub fn gen_blob_service() -> Arc<dyn BlobService> {
Arc::new(MemoryBlobService::default())
}
pub fn gen_directory_service() -> Arc<dyn DirectoryService> {
Arc::new(MemoryDirectoryService::default())
}

1
tvix/cli/Cargo.toml
View file

@ -9,6 +9,7 @@ path = "src/main.rs"
[dependencies]
nix-compat = { path = "../nix-compat" }
tvix-castore = { path = "../castore" }
tvix-store = { path = "../store", features = []}
tvix-eval = { path = "../eval" }
bytes = "1.4.0"

4
tvix/cli/src/main.rs
View file

@ -13,10 +13,10 @@ use std::{fs, path::PathBuf};
use clap::Parser;
use known_paths::KnownPaths;
use rustyline::{error::ReadlineError, Editor};
use tvix_castore::blobservice::MemoryBlobService;
use tvix_castore::directoryservice::MemoryDirectoryService;
use tvix_eval::observer::{DisassemblingObserver, TracingObserver};
use tvix_eval::Value;
use tvix_store::blobservice::MemoryBlobService;
use tvix_store::directoryservice::MemoryDirectoryService;
use tvix_store::pathinfoservice::MemoryPathInfoService;
use tvix_store_io::TvixStoreIO;

11
tvix/cli/src/tvix_store_io.rs
View file

@ -6,14 +6,17 @@ use tokio::io::AsyncReadExt;
use tracing::{error, instrument, warn};
use tvix_eval::{EvalIO, FileType, StdIO};
use tvix_store::{
use tvix_castore::{
blobservice::BlobService,
directoryservice::{self, DirectoryService},
import,
proto::{node::Node, NamedNode},
B3Digest,
};
use tvix_store::{
nar::calculate_size_and_sha256,
pathinfoservice::PathInfoService,
proto::{node::Node, NamedNode, NarInfo, PathInfo},
B3Digest,
proto::{NarInfo, PathInfo},
};
/// Implements [EvalIO], asking given [PathInfoService], [DirectoryService]
@ -330,7 +333,7 @@ async fn import_path_with_pathinfo(
// assemble the [PathInfo] object.
let path_info = PathInfo {
node: Some(tvix_store::proto::Node {
node: Some(tvix_castore::proto::Node {
node: Some(root_node),
}),
// There's no reference scanning on path contents ingested like this.

34
tvix/default.nix
View file

@ -36,8 +36,13 @@ let
nativeBuildInputs = protobufDep prev;
};
tvix-castore = prev: {
PROTO_ROOT = depot.tvix.proto;
nativeBuildInputs = protobufDep prev;
};
tvix-store = prev: {
PROTO_ROOT = depot.tvix.store.protos;
PROTO_ROOT = depot.tvix.proto;
nativeBuildInputs = protobufDep prev;
};
};
@ -103,20 +108,26 @@ in
] ++ iconvDarwinDep;
};
# Builds and tests the code in store/protos.
store-protos-go = pkgs.buildGoModule {
name = "store-golang";
src = depot.third_party.gitignoreSource ./store/protos;
# Builds and tests the code in castore/protos.
# castore-protos-go = pkgs.buildGoModule {
# name = "castore-golang";
# src = depot.third_party.gitignoreSource ./store/protos;
# vendorHash = "sha256-00000000000000000000000000000000000000000000";
# };
vendorHash = "sha256-7xfXBBU3xJz7ifjk7Owm/byTfCQ8oaZtqXzBKhLqo00=";
};
# Builds and tests the code in store/protos.
# store-protos-go = pkgs.buildGoModule {
# name = "store-golang";
# src = depot.third_party.gitignoreSource ./store/protos;
# vendorHash = "sha256-00000000000000000000000000000000000000000000";
# };
# Build the Rust documentation for publishing on docs.tvix.dev.
rust-docs = pkgs.stdenv.mkDerivation {
inherit cargoDeps;
name = "tvix-rust-docs";
src = depot.third_party.gitignoreSource ./.;
PROTO_ROOT = depot.tvix.store.protos;
PROTO_ROOT = depot.tvix.proto;
buildInputs = [
pkgs.fuse
@ -135,5 +146,10 @@ in
'';
};
meta.ci.targets = [ "store-protos-go" "shell" "rust-docs" ];
meta.ci.targets = [
# "castore-protos-go"
# "store-protos-go"
"shell"
"rust-docs"
];
}

22
tvix/proto/default.nix
View file

@ -1,9 +1,15 @@
# Build protocol buffer definitions to ensure that protos are valid in
# CI. Note that the output of this build target is not actually used
# anywhere, it just functions as a CI check for now.
{ pkgs, ... }:
# Target containing just the proto files used in tvix
pkgs.runCommand "tvix-cc-proto" { } ''
mkdir $out
${pkgs.protobuf}/bin/protoc -I ${./.} evaluator.proto --cpp_out=$out
''
{ depot, lib, ... }:
depot.nix.sparseTree {
name = "tvix-protos";
root = depot.path.origSrc;
paths = [
../castore/protos/castore.proto
../castore/protos/rpc_blobstore.proto
../castore/protos/rpc_directory.proto
../store/protos/pathinfo.proto
../store/protos/rpc_pathinfo.proto
];
}

13
tvix/store/Cargo.toml
View file

@ -7,28 +7,29 @@ edition = "2021"
anyhow = "1.0.68"
async-stream = "0.3.5"
blake3 = { version = "1.3.1", features = ["rayon", "std"] }
bytes = "1.4.0"
clap = { version = "4.0", features = ["derive", "env"] }
count-write = "0.1.0"
data-encoding = "2.3.3"
futures = "0.3.28"
lazy_static = "1.4.0"
nix-compat = { path = "../nix-compat" }
parking_lot = "0.12.1"
pin-project-lite = "0.2.13"
prost = "0.11.2"
sha2 = "0.10.6"
sled = { version = "0.34.7", features = ["compression"] }
thiserror = "1.0.38"
tokio-stream = { version = "0.1.14", features = ["fs"] }
tokio-util = { version = "0.7.8", features = ["io", "io-util"] }
tokio = { version = "1.28.0", features = ["fs", "net", "rt-multi-thread", "signal"] }
tonic = "0.8.2"
tower = "0.4.13"
tracing = "0.1.37"
tracing-subscriber = { version = "0.3.16", features = ["json"] }
walkdir = "2.4.0"
tokio-util = { version = "0.7.8", features = ["io", "io-util"] }
tower = "0.4.13"
futures = "0.3.28"
bytes = "1.4.0"
tvix-castore = { path = "../castore" }
url = "2.4.0"
pin-project-lite = "0.2.13"
walkdir = "2.4.0"
[dependencies.fuse-backend-rs]
optional = true

14
tvix/store/README.md
View file

@ -14,9 +14,12 @@ However, enough information is preserved to still be able to render NAR and
NARInfo when needed.
## More Information
Check the `protos/` subfolder for the definition of the exact RPC methods and
messages.
The store consists out of two different gRPC services, `tvix.castore.v1` for
the low-level content-addressed bits, and `tvix.store.v1` for the Nix and
`StorePath`-specific bits.
Check the `protos/` subfolder both here and in `castore` for the definition of
the exact RPC methods and messages.
## Interacting with the GRPC service manually
The shell environment in `//tvix` provides `evans`, which is an interactive
@ -37,15 +40,16 @@ $ evans --host localhost --port 8000 -r repl
more expressive universal gRPC client
tvix.store.v1@localhost:8000> service BlobService
localhost:8000> package tvix.castore.v1
tvix.castore.v1@localhost:8000> service BlobService
tvix.store.v1.BlobService@localhost:8000> call Put --bytes-from-file
tvix.castore.v1.BlobService@localhost:8000> call Put --bytes-from-file
data (TYPE_BYTES) => /run/current-system/system
{
"digest": "KOM3/IHEx7YfInAnlJpAElYezq0Sxn9fRz7xuClwNfA="
}
tvix.store.v1.BlobService@localhost:8000> call Get --bytes-as-base64
tvix.castore.v1.BlobService@localhost:8000> call Get --bytes-as-base64
digest (TYPE_BYTES) => KOM3/IHEx7YfInAnlJpAElYezq0Sxn9fRz7xuClwNfA=
{
"data": "eDg2XzY0LWxpbnV4"

4
tvix/store/build.rs
View file

@ -15,6 +15,7 @@ fn main() -> Result<()> {
// https://github.com/hyperium/tonic/issues/908
let mut config = prost_build::Config::new();
config.bytes(["."]);
config.extern_path(".tvix.castore.v1", "::tvix_castore::proto");
builder
.build_server(true)
@ -22,10 +23,7 @@ fn main() -> Result<()> {
.compile_with_config(
config,
&[
"tvix/store/protos/castore.proto",
"tvix/store/protos/pathinfo.proto",
"tvix/store/protos/rpc_blobstore.proto",
"tvix/store/protos/rpc_directory.proto",
"tvix/store/protos/rpc_pathinfo.proto",
],
// If we are in running `cargo build` manually, using `../..` works fine,

450
tvix/store/protos/castore.pb.go
View file

@ -1,450 +0,0 @@
// SPDX-FileCopyrightText: edef <edef@unfathomable.blue>
// SPDX-License-Identifier: OSL-3.0 OR MIT OR Apache-2.0
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
// protoc (unknown)
// source: tvix/store/protos/castore.proto
package storev1
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
// A Directory can contain Directory, File or Symlink nodes.
// Each of these nodes have a name attribute, which is the basename in that directory
// and node type specific attributes.
// The name attribute:
// - MUST not contain slashes or null bytes
// - MUST not be '.' or '..'
// - MUST be unique across all three lists
//
// Elements in each list need to be lexicographically ordered by the name
// attribute.
type Directory struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Directories []*DirectoryNode `protobuf:"bytes,1,rep,name=directories,proto3" json:"directories,omitempty"`
Files []*FileNode `protobuf:"bytes,2,rep,name=files,proto3" json:"files,omitempty"`
Symlinks []*SymlinkNode `protobuf:"bytes,3,rep,name=symlinks,proto3" json:"symlinks,omitempty"`
}
func (x *Directory) Reset() {
*x = Directory{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_castore_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Directory) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Directory) ProtoMessage() {}
func (x *Directory) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_castore_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Directory.ProtoReflect.Descriptor instead.
func (*Directory) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{0}
}
func (x *Directory) GetDirectories() []*DirectoryNode {
if x != nil {
return x.Directories
}
return nil
}
func (x *Directory) GetFiles() []*FileNode {
if x != nil {
return x.Files
}
return nil
}
func (x *Directory) GetSymlinks() []*SymlinkNode {
if x != nil {
return x.Symlinks
}
return nil
}
// A DirectoryNode represents a directory in a Directory.
type DirectoryNode struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// The (base)name of the directory
Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
// The blake3 hash of a Directory message, serialized in protobuf canonical form.
Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"`
// Number of child elements in the Directory referred to by `digest`.
// Calculated by summing up the numbers of `directories`, `files` and
// `symlinks`, and for each directory, its size field. Used for inode
// number calculation.
// This field is precisely as verifiable as any other Merkle tree edge.
// Resolve `digest`, and you can compute it incrementally. Resolve the
// entire tree, and you can fully compute it from scratch.
// A credulous implementation won't reject an excessive size, but this is
// harmless: you'll have some ordinals without nodes. Undersizing is
// obvious and easy to reject: you won't have an ordinal for some nodes.
Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
}
func (x *DirectoryNode) Reset() {
*x = DirectoryNode{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_castore_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *DirectoryNode) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*DirectoryNode) ProtoMessage() {}
func (x *DirectoryNode) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_castore_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use DirectoryNode.ProtoReflect.Descriptor instead.
func (*DirectoryNode) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{1}
}
func (x *DirectoryNode) GetName() []byte {
if x != nil {
return x.Name
}
return nil
}
func (x *DirectoryNode) GetDigest() []byte {
if x != nil {
return x.Digest
}
return nil
}
func (x *DirectoryNode) GetSize() uint32 {
if x != nil {
return x.Size
}
return 0
}
// A FileNode represents a regular or executable file in a Directory.
type FileNode struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// The (base)name of the file
Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
// The blake3 digest of the file contents
Digest []byte `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"`
// The file content size
Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
// Whether the file is executable
Executable bool `protobuf:"varint,4,opt,name=executable,proto3" json:"executable,omitempty"`
}
func (x *FileNode) Reset() {
*x = FileNode{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_castore_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *FileNode) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FileNode) ProtoMessage() {}
func (x *FileNode) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_castore_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FileNode.ProtoReflect.Descriptor instead.
func (*FileNode) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{2}
}
func (x *FileNode) GetName() []byte {
if x != nil {
return x.Name
}
return nil
}
func (x *FileNode) GetDigest() []byte {
if x != nil {
return x.Digest
}
return nil
}
func (x *FileNode) GetSize() uint32 {
if x != nil {
return x.Size
}
return 0
}
func (x *FileNode) GetExecutable() bool {
if x != nil {
return x.Executable
}
return false
}
// A SymlinkNode represents a symbolic link in a Directory.
type SymlinkNode struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// The (base)name of the symlink
Name []byte `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
// The target of the symlink.
Target []byte `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"`
}
func (x *SymlinkNode) Reset() {
*x = SymlinkNode{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_castore_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *SymlinkNode) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*SymlinkNode) ProtoMessage() {}
func (x *SymlinkNode) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_castore_proto_msgTypes[3]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use SymlinkNode.ProtoReflect.Descriptor instead.
func (*SymlinkNode) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_castore_proto_rawDescGZIP(), []int{3}
}
func (x *SymlinkNode) GetName() []byte {
if x != nil {
return x.Name
}
return nil
}
func (x *SymlinkNode) GetTarget() []byte {
if x != nil {
return x.Target
}
return nil
}
var File_tvix_store_protos_castore_proto protoreflect.FileDescriptor
var file_tvix_store_protos_castore_proto_rawDesc = []byte{
0x0a, 0x1f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74,
0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31,
0x22, 0xb2, 0x01, 0x0a, 0x09, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x3e,
0x0a, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20,
0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65,
0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x4e, 0x6f, 0x64,
0x65, 0x52, 0x0b, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x2d,
0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e,
0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69,
0x6c, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x36, 0x0a,
0x08, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32,
0x1a, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e,
0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x08, 0x73, 0x79, 0x6d,
0x6c, 0x69, 0x6e, 0x6b, 0x73, 0x22, 0x4f, 0x0a, 0x0d, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f,
0x72, 0x79, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69,
0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65,
0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d,
0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x6a, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f,
0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c,
0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74,
0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12,
0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69,
0x7a, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65,
0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62,
0x6c, 0x65, 0x22, 0x39, 0x0a, 0x0b, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64,
0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18,
0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x42, 0x28, 0x5a,
0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76,
0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b,
0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_tvix_store_protos_castore_proto_rawDescOnce sync.Once
file_tvix_store_protos_castore_proto_rawDescData = file_tvix_store_protos_castore_proto_rawDesc
)
func file_tvix_store_protos_castore_proto_rawDescGZIP() []byte {
file_tvix_store_protos_castore_proto_rawDescOnce.Do(func() {
file_tvix_store_protos_castore_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_store_protos_castore_proto_rawDescData)
})
return file_tvix_store_protos_castore_proto_rawDescData
}
var file_tvix_store_protos_castore_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
var file_tvix_store_protos_castore_proto_goTypes = []interface{}{
(*Directory)(nil), // 0: tvix.store.v1.Directory
(*DirectoryNode)(nil), // 1: tvix.store.v1.DirectoryNode
(*FileNode)(nil), // 2: tvix.store.v1.FileNode
(*SymlinkNode)(nil), // 3: tvix.store.v1.SymlinkNode
}
var file_tvix_store_protos_castore_proto_depIdxs = []int32{
1, // 0: tvix.store.v1.Directory.directories:type_name -> tvix.store.v1.DirectoryNode
2, // 1: tvix.store.v1.Directory.files:type_name -> tvix.store.v1.FileNode
3, // 2: tvix.store.v1.Directory.symlinks:type_name -> tvix.store.v1.SymlinkNode
3, // [3:3] is the sub-list for method output_type
3, // [3:3] is the sub-list for method input_type
3, // [3:3] is the sub-list for extension type_name
3, // [3:3] is the sub-list for extension extendee
0, // [0:3] is the sub-list for field type_name
}
func init() { file_tvix_store_protos_castore_proto_init() }
func file_tvix_store_protos_castore_proto_init() {
if File_tvix_store_protos_castore_proto != nil {
return
}
if !protoimpl.UnsafeEnabled {
file_tvix_store_protos_castore_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Directory); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_store_protos_castore_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*DirectoryNode); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_store_protos_castore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*FileNode); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_store_protos_castore_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*SymlinkNode); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_tvix_store_protos_castore_proto_rawDesc,
NumEnums: 0,
NumMessages: 4,
NumExtensions: 0,
NumServices: 0,
},
GoTypes: file_tvix_store_protos_castore_proto_goTypes,
DependencyIndexes: file_tvix_store_protos_castore_proto_depIdxs,
MessageInfos: file_tvix_store_protos_castore_proto_msgTypes,
}.Build()
File_tvix_store_protos_castore_proto = out.File
file_tvix_store_protos_castore_proto_rawDesc = nil
file_tvix_store_protos_castore_proto_goTypes = nil
file_tvix_store_protos_castore_proto_depIdxs = nil
}

16
tvix/store/protos/default.nix
View file

@ -1,16 +0,0 @@
# Target containing just the proto files.
{ depot, lib, ... }:
let
inherit (lib.strings) hasSuffix;
inherit (builtins) attrNames filter readDir;
protoFileNames = filter (hasSuffix ".proto") (attrNames (readDir ./.));
protoFiles = map (f: ./. + ("/" + f)) protoFileNames;
in
depot.nix.sparseTree {
name = "tvix-store-protos";
root = depot.path.origSrc;
paths = protoFiles;
}

233
tvix/store/protos/pathinfo.pb.go
View file

@ -10,6 +10,7 @@
package storev1
import (
protos "code.tvl.fyi/tvix/castore/protos"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
@ -31,7 +32,7 @@ type PathInfo struct {
unknownFields protoimpl.UnknownFields
// The path can be a directory, file or symlink.
Node *Node `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"`
Node *protos.Node `protobuf:"bytes,1,opt,name=node,proto3" json:"node,omitempty"`
// List of references (output path hashes)
// This really is the raw *bytes*, after decoding nixbase32, and not a
// base32-encoded string.
@ -72,7 +73,7 @@ func (*PathInfo) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{0}
}
func (x *PathInfo) GetNode() *Node {
func (x *PathInfo) GetNode() *protos.Node {
if x != nil {
return x.Node
}
@ -93,101 +94,6 @@ func (x *PathInfo) GetNarinfo() *NARInfo {
return nil
}
type Node struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Types that are assignable to Node:
//
// *Node_Directory
// *Node_File
// *Node_Symlink
Node isNode_Node `protobuf_oneof:"node"`
}
func (x *Node) Reset() {
*x = Node{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *Node) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Node) ProtoMessage() {}
func (x *Node) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Node.ProtoReflect.Descriptor instead.
func (*Node) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{1}
}
func (m *Node) GetNode() isNode_Node {
if m != nil {
return m.Node
}
return nil
}
func (x *Node) GetDirectory() *DirectoryNode {
if x, ok := x.GetNode().(*Node_Directory); ok {
return x.Directory
}
return nil
}
func (x *Node) GetFile() *FileNode {
if x, ok := x.GetNode().(*Node_File); ok {
return x.File
}
return nil
}
func (x *Node) GetSymlink() *SymlinkNode {
if x, ok := x.GetNode().(*Node_Symlink); ok {
return x.Symlink
}
return nil
}
type isNode_Node interface {
isNode_Node()
}
type Node_Directory struct {
Directory *DirectoryNode `protobuf:"bytes,1,opt,name=directory,proto3,oneof"`
}
type Node_File struct {
File *FileNode `protobuf:"bytes,2,opt,name=file,proto3,oneof"`
}
type Node_Symlink struct {
Symlink *SymlinkNode `protobuf:"bytes,3,opt,name=symlink,proto3,oneof"`
}
func (*Node_Directory) isNode_Node() {}
func (*Node_File) isNode_Node() {}
func (*Node_Symlink) isNode_Node() {}
// Nix C++ uses NAR (Nix Archive) as a format to transfer store paths,
// and stores metadata and signatures in NARInfo files.
// Store all these attributes in a separate message.
@ -219,7 +125,7 @@ type NARInfo struct {
func (x *NARInfo) Reset() {
*x = NARInfo{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2]
mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -232,7 +138,7 @@ func (x *NARInfo) String() string {
func (*NARInfo) ProtoMessage() {}
func (x *NARInfo) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2]
mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -245,7 +151,7 @@ func (x *NARInfo) ProtoReflect() protoreflect.Message {
// Deprecated: Use NARInfo.ProtoReflect.Descriptor instead.
func (*NARInfo) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{2}
return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{1}
}
func (x *NARInfo) GetNarSize() uint64 {
@ -289,7 +195,7 @@ type NARInfo_Signature struct {
func (x *NARInfo_Signature) Reset() {
*x = NARInfo_Signature{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[3]
mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@ -302,7 +208,7 @@ func (x *NARInfo_Signature) String() string {
func (*NARInfo_Signature) ProtoMessage() {}
func (x *NARInfo_Signature) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[3]
mi := &file_tvix_store_protos_pathinfo_proto_msgTypes[2]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@ -315,7 +221,7 @@ func (x *NARInfo_Signature) ProtoReflect() protoreflect.Message {
// Deprecated: Use NARInfo_Signature.ProtoReflect.Descriptor instead.
func (*NARInfo_Signature) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{2, 0}
return file_tvix_store_protos_pathinfo_proto_rawDescGZIP(), []int{1, 0}
}
func (x *NARInfo_Signature) GetName() string {
@ -338,46 +244,35 @@ var file_tvix_store_protos_pathinfo_proto_rawDesc = []byte{
0x0a, 0x20, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x73, 0x2f, 0x70, 0x61, 0x74, 0x68, 0x69, 0x6e, 0x66, 0x6f, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76,
0x31, 0x1a, 0x1f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x22, 0x85, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12,
0x27, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e,
0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f,
0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x66, 0x65,
0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0a, 0x72, 0x65,
0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x30, 0x0a, 0x07, 0x6e, 0x61, 0x72, 0x69,
0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x74, 0x76, 0x69, 0x78,
0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66,
0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0xb3, 0x01, 0x0a, 0x04, 0x4e,
0x6f, 0x64, 0x65, 0x12, 0x3c, 0x0a, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79,
0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74,
0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79,
0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x09, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72,
0x79, 0x12, 0x2d, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e,
0x46, 0x69, 0x6c, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65,
0x12, 0x36, 0x0a, 0x07, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x1a, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76,
0x31, 0x2e, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52,
0x07, 0x73, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x42, 0x06, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65,
0x22, 0xe3, 0x01, 0x0a, 0x07, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08,
0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07,
0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73,
0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72,
0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x12, 0x40, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74,
0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69,
0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e,
0x66, 0x6f, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69,
0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65,
0x72, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28,
0x09, 0x52, 0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65,
0x73, 0x1a, 0x33, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12,
0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61,
0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c,
0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74,
0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31,
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x31, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70,
0x72, 0x6f, 0x74, 0x6f, 0x22, 0x87, 0x01, 0x0a, 0x08, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66,
0x6f, 0x12, 0x29, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x15, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76,
0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a,
0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c,
0x52, 0x0a, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x73, 0x12, 0x30, 0x0a, 0x07,
0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e,
0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41,
0x52, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x69, 0x6e, 0x66, 0x6f, 0x22, 0xe3,
0x01, 0x0a, 0x07, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61,
0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61,
0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61,
0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68,
0x61, 0x32, 0x35, 0x36, 0x12, 0x40, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72,
0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e,
0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x41, 0x52, 0x49, 0x6e, 0x66, 0x6f,
0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x0a, 0x73, 0x69, 0x67, 0x6e,
0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65,
0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52,
0x0e, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a,
0x33, 0x0a, 0x09, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04,
0x64, 0x61, 0x74, 0x61, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c,
0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@ -392,28 +287,22 @@ func file_tvix_store_protos_pathinfo_proto_rawDescGZIP() []byte {
return file_tvix_store_protos_pathinfo_proto_rawDescData
}
var file_tvix_store_protos_pathinfo_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
var file_tvix_store_protos_pathinfo_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
var file_tvix_store_protos_pathinfo_proto_goTypes = []interface{}{
(*PathInfo)(nil), // 0: tvix.store.v1.PathInfo
(*Node)(nil), // 1: tvix.store.v1.Node
(*NARInfo)(nil), // 2: tvix.store.v1.NARInfo
(*NARInfo_Signature)(nil), // 3: tvix.store.v1.NARInfo.Signature
(*DirectoryNode)(nil), // 4: tvix.store.v1.DirectoryNode
(*FileNode)(nil), // 5: tvix.store.v1.FileNode
(*SymlinkNode)(nil), // 6: tvix.store.v1.SymlinkNode
(*NARInfo)(nil), // 1: tvix.store.v1.NARInfo
(*NARInfo_Signature)(nil), // 2: tvix.store.v1.NARInfo.Signature
(*protos.Node)(nil), // 3: tvix.castore.v1.Node
}
var file_tvix_store_protos_pathinfo_proto_depIdxs = []int32{
1, // 0: tvix.store.v1.PathInfo.node:type_name -> tvix.store.v1.Node
2, // 1: tvix.store.v1.PathInfo.narinfo:type_name -> tvix.store.v1.NARInfo
4, // 2: tvix.store.v1.Node.directory:type_name -> tvix.store.v1.DirectoryNode
5, // 3: tvix.store.v1.Node.file:type_name -> tvix.store.v1.FileNode
6, // 4: tvix.store.v1.Node.symlink:type_name -> tvix.store.v1.SymlinkNode
3, // 5: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature
6, // [6:6] is the sub-list for method output_type
6, // [6:6] is the sub-list for method input_type
6, // [6:6] is the sub-list for extension type_name
6, // [6:6] is the sub-list for extension extendee
0, // [0:6] is the sub-list for field type_name
3, // 0: tvix.store.v1.PathInfo.node:type_name -> tvix.castore.v1.Node
1, // 1: tvix.store.v1.PathInfo.narinfo:type_name -> tvix.store.v1.NARInfo
2, // 2: tvix.store.v1.NARInfo.signatures:type_name -> tvix.store.v1.NARInfo.Signature
3, // [3:3] is the sub-list for method output_type
3, // [3:3] is the sub-list for method input_type
3, // [3:3] is the sub-list for extension type_name
3, // [3:3] is the sub-list for extension extendee
0, // [0:3] is the sub-list for field type_name
}
func init() { file_tvix_store_protos_pathinfo_proto_init() }
@ -421,7 +310,6 @@ func file_tvix_store_protos_pathinfo_proto_init() {
if File_tvix_store_protos_pathinfo_proto != nil {
return
}
file_tvix_store_protos_castore_proto_init()
if !protoimpl.UnsafeEnabled {
file_tvix_store_protos_pathinfo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*PathInfo); i {
@ -436,18 +324,6 @@ func file_tvix_store_protos_pathinfo_proto_init() {
}
}
file_tvix_store_protos_pathinfo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*Node); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_store_protos_pathinfo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*NARInfo); i {
case 0:
return &v.state
@ -459,7 +335,7 @@ func file_tvix_store_protos_pathinfo_proto_init() {
return nil
}
}
file_tvix_store_protos_pathinfo_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
file_tvix_store_protos_pathinfo_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*NARInfo_Signature); i {
case 0:
return &v.state
@ -472,18 +348,13 @@ func file_tvix_store_protos_pathinfo_proto_init() {
}
}
}
file_tvix_store_protos_pathinfo_proto_msgTypes[1].OneofWrappers = []interface{}{
(*Node_Directory)(nil),
(*Node_File)(nil),
(*Node_Symlink)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_tvix_store_protos_pathinfo_proto_rawDesc,
NumEnums: 0,
NumMessages: 4,
NumMessages: 3,
NumExtensions: 0,
NumServices: 0,
},

12
tvix/store/protos/pathinfo.proto
View file

@ -4,7 +4,7 @@ syntax = "proto3";
package tvix.store.v1;
import "tvix/store/protos/castore.proto";
import "tvix/castore/protos/castore.proto";
option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
@ -12,7 +12,7 @@ option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
// That's a single element inside /nix/store.
message PathInfo {
// The path can be a directory, file or symlink.
Node node = 1;
tvix.castore.v1.Node node = 1;
// List of references (output path hashes)
// This really is the raw *bytes*, after decoding nixbase32, and not a
@ -23,14 +23,6 @@ message PathInfo {
NARInfo narinfo = 3;
}
message Node {
oneof node {
DirectoryNode directory = 1;
FileNode file = 2;
SymlinkNode symlink = 3;
}
}
// Nix C++ uses NAR (Nix Archive) as a format to transfer store paths,
// and stores metadata and signatures in NARInfo files.
// Store all these attributes in a separate message.

271
tvix/store/protos/rpc_directory.pb.go
View file

@ -1,271 +0,0 @@
// SPDX-License-Identifier: MIT
// Copyright © 2022 The Tvix Authors
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
// protoc (unknown)
// source: tvix/store/protos/rpc_directory.proto
package storev1
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
)
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
// Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
type GetDirectoryRequest struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
// Types that are assignable to ByWhat:
//
// *GetDirectoryRequest_Digest
ByWhat isGetDirectoryRequest_ByWhat `protobuf_oneof:"by_what"`
// If set to true, recursively resolve all child Directory messages.
// Directory messages SHOULD be streamed in a recursive breadth-first walk,
// but other orders are also fine, as long as Directory messages are only
// sent after they are referred to from previously sent Directory messages.
Recursive bool `protobuf:"varint,2,opt,name=recursive,proto3" json:"recursive,omitempty"`
}
func (x *GetDirectoryRequest) Reset() {
*x = GetDirectoryRequest{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_rpc_directory_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *GetDirectoryRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*GetDirectoryRequest) ProtoMessage() {}
func (x *GetDirectoryRequest) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_rpc_directory_proto_msgTypes[0]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use GetDirectoryRequest.ProtoReflect.Descriptor instead.
func (*GetDirectoryRequest) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_rpc_directory_proto_rawDescGZIP(), []int{0}
}
func (m *GetDirectoryRequest) GetByWhat() isGetDirectoryRequest_ByWhat {
if m != nil {
return m.ByWhat
}
return nil
}
func (x *GetDirectoryRequest) GetDigest() []byte {
if x, ok := x.GetByWhat().(*GetDirectoryRequest_Digest); ok {
return x.Digest
}
return nil
}
func (x *GetDirectoryRequest) GetRecursive() bool {
if x != nil {
return x.Recursive
}
return false
}
type isGetDirectoryRequest_ByWhat interface {
isGetDirectoryRequest_ByWhat()
}
type GetDirectoryRequest_Digest struct {
// The blake3 hash of the (root) Directory message, serialized in
// protobuf canonical form.
// Keep in mind this can be a subtree of another root.
Digest []byte `protobuf:"bytes,1,opt,name=digest,proto3,oneof"`
}
func (*GetDirectoryRequest_Digest) isGetDirectoryRequest_ByWhat() {}
type PutDirectoryResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
RootDigest []byte `protobuf:"bytes,1,opt,name=root_digest,json=rootDigest,proto3" json:"root_digest,omitempty"`
}
func (x *PutDirectoryResponse) Reset() {
*x = PutDirectoryResponse{}
if protoimpl.UnsafeEnabled {
mi := &file_tvix_store_protos_rpc_directory_proto_msgTypes[1]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
}
func (x *PutDirectoryResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*PutDirectoryResponse) ProtoMessage() {}
func (x *PutDirectoryResponse) ProtoReflect() protoreflect.Message {
mi := &file_tvix_store_protos_rpc_directory_proto_msgTypes[1]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use PutDirectoryResponse.ProtoReflect.Descriptor instead.
func (*PutDirectoryResponse) Descriptor() ([]byte, []int) {
return file_tvix_store_protos_rpc_directory_proto_rawDescGZIP(), []int{1}
}
func (x *PutDirectoryResponse) GetRootDigest() []byte {
if x != nil {
return x.RootDigest
}
return nil
}
var File_tvix_store_protos_rpc_directory_proto protoreflect.FileDescriptor
var file_tvix_store_protos_rpc_directory_proto_rawDesc = []byte{
0x0a, 0x25, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x5f, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72,
0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74,
0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x1f, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f,
0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x58, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x44, 0x69,
0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18,
0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00,
0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75,
0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x63,
0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x62, 0x79, 0x5f, 0x77, 0x68, 0x61,
0x74, 0x22, 0x37, 0x0a, 0x14, 0x50, 0x75, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72,
0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x6f, 0x6f,
0x74, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a,
0x72, 0x6f, 0x6f, 0x74, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x32, 0xa1, 0x01, 0x0a, 0x10, 0x44,
0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12,
0x45, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74,
0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x74, 0x76, 0x69,
0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63,
0x74, 0x6f, 0x72, 0x79, 0x30, 0x01, 0x12, 0x46, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x18, 0x2e,
0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69,
0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x1a, 0x23, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75, 0x74, 0x44, 0x69, 0x72, 0x65, 0x63,
0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x42, 0x28,
0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74,
0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73,
0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
file_tvix_store_protos_rpc_directory_proto_rawDescOnce sync.Once
file_tvix_store_protos_rpc_directory_proto_rawDescData = file_tvix_store_protos_rpc_directory_proto_rawDesc
)
func file_tvix_store_protos_rpc_directory_proto_rawDescGZIP() []byte {
file_tvix_store_protos_rpc_directory_proto_rawDescOnce.Do(func() {
file_tvix_store_protos_rpc_directory_proto_rawDescData = protoimpl.X.CompressGZIP(file_tvix_store_protos_rpc_directory_proto_rawDescData)
})
return file_tvix_store_protos_rpc_directory_proto_rawDescData
}
var file_tvix_store_protos_rpc_directory_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
var file_tvix_store_protos_rpc_directory_proto_goTypes = []interface{}{
(*GetDirectoryRequest)(nil), // 0: tvix.store.v1.GetDirectoryRequest
(*PutDirectoryResponse)(nil), // 1: tvix.store.v1.PutDirectoryResponse
(*Directory)(nil), // 2: tvix.store.v1.Directory
}
var file_tvix_store_protos_rpc_directory_proto_depIdxs = []int32{
0, // 0: tvix.store.v1.DirectoryService.Get:input_type -> tvix.store.v1.GetDirectoryRequest
2, // 1: tvix.store.v1.DirectoryService.Put:input_type -> tvix.store.v1.Directory
2, // 2: tvix.store.v1.DirectoryService.Get:output_type -> tvix.store.v1.Directory
1, // 3: tvix.store.v1.DirectoryService.Put:output_type -> tvix.store.v1.PutDirectoryResponse
2, // [2:4] is the sub-list for method output_type
0, // [0:2] is the sub-list for method input_type
0, // [0:0] is the sub-list for extension type_name
0, // [0:0] is the sub-list for extension extendee
0, // [0:0] is the sub-list for field type_name
}
func init() { file_tvix_store_protos_rpc_directory_proto_init() }
func file_tvix_store_protos_rpc_directory_proto_init() {
if File_tvix_store_protos_rpc_directory_proto != nil {
return
}
file_tvix_store_protos_castore_proto_init()
if !protoimpl.UnsafeEnabled {
file_tvix_store_protos_rpc_directory_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*GetDirectoryRequest); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
file_tvix_store_protos_rpc_directory_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*PutDirectoryResponse); i {
case 0:
return &v.state
case 1:
return &v.sizeCache
case 2:
return &v.unknownFields
default:
return nil
}
}
}
file_tvix_store_protos_rpc_directory_proto_msgTypes[0].OneofWrappers = []interface{}{
(*GetDirectoryRequest_Digest)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_tvix_store_protos_rpc_directory_proto_rawDesc,
NumEnums: 0,
NumMessages: 2,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_tvix_store_protos_rpc_directory_proto_goTypes,
DependencyIndexes: file_tvix_store_protos_rpc_directory_proto_depIdxs,
MessageInfos: file_tvix_store_protos_rpc_directory_proto_msgTypes,
}.Build()
File_tvix_store_protos_rpc_directory_proto = out.File
file_tvix_store_protos_rpc_directory_proto_rawDesc = nil
file_tvix_store_protos_rpc_directory_proto_goTypes = nil
file_tvix_store_protos_rpc_directory_proto_depIdxs = nil
}

74
tvix/store/protos/rpc_pathinfo.pb.go
View file

@ -10,6 +10,7 @@
package storev1
import (
protos "code.tvl.fyi/tvix/castore/protos"
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
@ -205,39 +206,42 @@ var file_tvix_store_protos_rpc_pathinfo_proto_rawDesc = []byte{
0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f,
0x72, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x20, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x70, 0x61, 0x74, 0x68, 0x69, 0x6e, 0x66,
0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x47, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x50, 0x61,
0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a,
0x0e, 0x62, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18,
0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0c, 0x62, 0x79, 0x4f, 0x75, 0x74, 0x70, 0x75,
0x74, 0x48, 0x61, 0x73, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x62, 0x79, 0x5f, 0x77, 0x68, 0x61, 0x74,
0x22, 0x15, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x50, 0x0a, 0x14, 0x43, 0x61, 0x6c, 0x63, 0x75,
0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61,
0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09,
0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x32, 0x9e, 0x02, 0x0a, 0x0f, 0x50, 0x61,
0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x41, 0x0a,
0x03, 0x47, 0x65, 0x74, 0x12, 0x21, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f,
0x12, 0x37, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f,
0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31,
0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x48, 0x0a, 0x0c, 0x43, 0x61, 0x6c,
0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x12, 0x13, 0x2e, 0x74, 0x76, 0x69, 0x78,
0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x1a, 0x23,
0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43,
0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x22, 0x2e, 0x74, 0x76,
0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74,
0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e,
0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, 0x42, 0x28, 0x5a, 0x26, 0x63, 0x6f,
0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f, 0x74, 0x76, 0x69, 0x78, 0x2f,
0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x3b, 0x73, 0x74, 0x6f,
0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x21, 0x74, 0x76, 0x69, 0x78, 0x2f, 0x63, 0x61,
0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x63, 0x61, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x47, 0x0a, 0x12, 0x47, 0x65,
0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x12, 0x26, 0x0a, 0x0e, 0x62, 0x79, 0x5f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x68, 0x61,
0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0c, 0x62, 0x79, 0x4f, 0x75,
0x74, 0x70, 0x75, 0x74, 0x48, 0x61, 0x73, 0x68, 0x42, 0x09, 0x0a, 0x07, 0x62, 0x79, 0x5f, 0x77,
0x68, 0x61, 0x74, 0x22, 0x15, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49,
0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x50, 0x0a, 0x14, 0x43, 0x61,
0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6e, 0x61, 0x72, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1d, 0x0a,
0x0a, 0x6e, 0x61, 0x72, 0x5f, 0x73, 0x68, 0x61, 0x32, 0x35, 0x36, 0x18, 0x02, 0x20, 0x01, 0x28,
0x0c, 0x52, 0x09, 0x6e, 0x61, 0x72, 0x53, 0x68, 0x61, 0x32, 0x35, 0x36, 0x32, 0xa0, 0x02, 0x0a,
0x0f, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
0x12, 0x41, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x21, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73,
0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49,
0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69,
0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49,
0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, 0x03, 0x50, 0x75, 0x74, 0x12, 0x17, 0x2e, 0x74, 0x76, 0x69,
0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49,
0x6e, 0x66, 0x6f, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65,
0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4a, 0x0a, 0x0c,
0x43, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52, 0x12, 0x15, 0x2e, 0x74,
0x76, 0x69, 0x78, 0x2e, 0x63, 0x61, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e,
0x6f, 0x64, 0x65, 0x1a, 0x23, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65,
0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x6c, 0x63, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x4e, 0x41, 0x52,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x04, 0x4c, 0x69, 0x73, 0x74,
0x12, 0x22, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31,
0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x74, 0x76, 0x69, 0x78, 0x2e, 0x73, 0x74, 0x6f, 0x72,
0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x74, 0x68, 0x49, 0x6e, 0x66, 0x6f, 0x30, 0x01, 0x42,
0x28, 0x5a, 0x26, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x74, 0x76, 0x6c, 0x2e, 0x66, 0x79, 0x69, 0x2f,
0x74, 0x76, 0x69, 0x78, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x73, 0x3b, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x33,
}
var (
@ -258,12 +262,12 @@ var file_tvix_store_protos_rpc_pathinfo_proto_goTypes = []interface{}{
(*ListPathInfoRequest)(nil), // 1: tvix.store.v1.ListPathInfoRequest
(*CalculateNARResponse)(nil), // 2: tvix.store.v1.CalculateNARResponse
(*PathInfo)(nil), // 3: tvix.store.v1.PathInfo
(*Node)(nil), // 4: tvix.store.v1.Node
(*protos.Node)(nil), // 4: tvix.castore.v1.Node
}
var file_tvix_store_protos_rpc_pathinfo_proto_depIdxs = []int32{
0, // 0: tvix.store.v1.PathInfoService.Get:input_type -> tvix.store.v1.GetPathInfoRequest
3, // 1: tvix.store.v1.PathInfoService.Put:input_type -> tvix.store.v1.PathInfo
4, // 2: tvix.store.v1.PathInfoService.CalculateNAR:input_type -> tvix.store.v1.Node
4, // 2: tvix.store.v1.PathInfoService.CalculateNAR:input_type -> tvix.castore.v1.Node
1, // 3: tvix.store.v1.PathInfoService.List:input_type -> tvix.store.v1.ListPathInfoRequest
3, // 4: tvix.store.v1.PathInfoService.Get:output_type -> tvix.store.v1.PathInfo
3, // 5: tvix.store.v1.PathInfoService.Put:output_type -> tvix.store.v1.PathInfo

3
tvix/store/protos/rpc_pathinfo.proto
View file

@ -5,6 +5,7 @@ syntax = "proto3";
package tvix.store.v1;
import "tvix/store/protos/pathinfo.proto";
import "tvix/castore/protos/castore.proto";
option go_package = "code.tvl.fyi/tvix/store/protos;storev1";
@ -40,7 +41,7 @@ service PathInfoService {
//
// It can also be used to calculate arbitrary NAR hashes of output paths,
// in case a legacy Nix Binary Cache frontend is provided.
rpc CalculateNAR(Node) returns (CalculateNARResponse);
rpc CalculateNAR(tvix.castore.v1.Node) returns (CalculateNARResponse);
// Return a stream of PathInfo messages matching the criteria specified in
// ListPathInfoRequest.

13
tvix/store/protos/rpc_pathinfo_grpc.pb.go
View file

@ -10,6 +10,7 @@
package storev1
import (
protos "code.tvl.fyi/tvix/castore/protos"
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
@ -60,7 +61,7 @@ type PathInfoServiceClient interface {
//
// It can also be used to calculate arbitrary NAR hashes of output paths,
// in case a legacy Nix Binary Cache frontend is provided.
CalculateNAR(ctx context.Context, in *Node, opts ...grpc.CallOption) (*CalculateNARResponse, error)
CalculateNAR(ctx context.Context, in *protos.Node, opts ...grpc.CallOption) (*CalculateNARResponse, error)
// Return a stream of PathInfo messages matching the criteria specified in
// ListPathInfoRequest.
List(ctx context.Context, in *ListPathInfoRequest, opts ...grpc.CallOption) (PathInfoService_ListClient, error)
@ -92,7 +93,7 @@ func (c *pathInfoServiceClient) Put(ctx context.Context, in *PathInfo, opts ...g
return out, nil
}
func (c *pathInfoServiceClient) CalculateNAR(ctx context.Context, in *Node, opts ...grpc.CallOption) (*CalculateNARResponse, error) {
func (c *pathInfoServiceClient) CalculateNAR(ctx context.Context, in *protos.Node, opts ...grpc.CallOption) (*CalculateNARResponse, error) {
out := new(CalculateNARResponse)
err := c.cc.Invoke(ctx, PathInfoService_CalculateNAR_FullMethodName, in, out, opts...)
if err != nil {
@ -165,7 +166,7 @@ type PathInfoServiceServer interface {
//
// It can also be used to calculate arbitrary NAR hashes of output paths,
// in case a legacy Nix Binary Cache frontend is provided.
CalculateNAR(context.Context, *Node) (*CalculateNARResponse, error)
CalculateNAR(context.Context, *protos.Node) (*CalculateNARResponse, error)
// Return a stream of PathInfo messages matching the criteria specified in
// ListPathInfoRequest.
List(*ListPathInfoRequest, PathInfoService_ListServer) error
@ -182,7 +183,7 @@ func (UnimplementedPathInfoServiceServer) Get(context.Context, *GetPathInfoReque
func (UnimplementedPathInfoServiceServer) Put(context.Context, *PathInfo) (*PathInfo, error) {
return nil, status.Errorf(codes.Unimplemented, "method Put not implemented")
}
func (UnimplementedPathInfoServiceServer) CalculateNAR(context.Context, *Node) (*CalculateNARResponse, error) {
func (UnimplementedPathInfoServiceServer) CalculateNAR(context.Context, *protos.Node) (*CalculateNARResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method CalculateNAR not implemented")
}
func (UnimplementedPathInfoServiceServer) List(*ListPathInfoRequest, PathInfoService_ListServer) error {
@ -238,7 +239,7 @@ func _PathInfoService_Put_Handler(srv interface{}, ctx context.Context, dec func
}
func _PathInfoService_CalculateNAR_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(Node)
in := new(protos.Node)
if err := dec(in); err != nil {
return nil, err
}
@ -250,7 +251,7 @@ func _PathInfoService_CalculateNAR_Handler(srv interface{}, ctx context.Context,
FullMethod: PathInfoService_CalculateNAR_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PathInfoServiceServer).CalculateNAR(ctx, req.(*Node))
return srv.(PathInfoServiceServer).CalculateNAR(ctx, req.(*protos.Node))
}
return interceptor(ctx, in, info, handler)
}

23
tvix/store/src/bin/tvix-store.rs
View file

@ -8,18 +8,18 @@ use std::path::Path;
use std::path::PathBuf;
use tokio::task::JoinHandle;
use tracing_subscriber::prelude::*;
use tvix_store::blobservice;
use tvix_store::directoryservice;
use tvix_store::import;
use tvix_castore::blobservice;
use tvix_castore::directoryservice;
use tvix_castore::import;
use tvix_castore::proto::blob_service_server::BlobServiceServer;
use tvix_castore::proto::directory_service_server::DirectoryServiceServer;
use tvix_castore::proto::node::Node;
use tvix_castore::proto::GRPCBlobServiceWrapper;
use tvix_castore::proto::GRPCDirectoryServiceWrapper;
use tvix_castore::proto::NamedNode;
use tvix_store::pathinfoservice;
use tvix_store::proto::blob_service_server::BlobServiceServer;
use tvix_store::proto::directory_service_server::DirectoryServiceServer;
use tvix_store::proto::node::Node;
use tvix_store::proto::path_info_service_server::PathInfoServiceServer;
use tvix_store::proto::GRPCBlobServiceWrapper;
use tvix_store::proto::GRPCDirectoryServiceWrapper;
use tvix_store::proto::GRPCPathInfoServiceWrapper;
use tvix_store::proto::NamedNode;
use tvix_store::proto::NarInfo;
use tvix_store::proto::PathInfo;
@ -29,6 +29,8 @@ use tvix_store::fs::TvixStoreFs;
#[cfg(feature = "fuse")]
use tvix_store::fs::fuse::FuseDaemon;
#[cfg(feature = "reflection")]
use tvix_castore::proto::FILE_DESCRIPTOR_SET as CASTORE_FILE_DESCRIPTOR_SET;
#[cfg(feature = "reflection")]
use tvix_store::proto::FILE_DESCRIPTOR_SET;
@ -185,6 +187,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
#[cfg(feature = "reflection")]
{
let reflection_svc = tonic_reflection::server::Builder::configure()
.register_encoded_file_descriptor_set(CASTORE_FILE_DESCRIPTOR_SET)
.register_encoded_file_descriptor_set(FILE_DESCRIPTOR_SET)
.build()?;
router = router.add_service(reflection_svc);
@ -248,7 +251,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// assemble the [crate::proto::PathInfo] object.
let path_info = PathInfo {
node: Some(tvix_store::proto::Node {
node: Some(tvix_castore::proto::Node {
node: Some(root_node),
}),
// There's no reference scanning on path contents ingested like this.

30
tvix/store/src/fs/inode_tracker.rs
View file

@ -1,8 +1,8 @@
use std::{collections::HashMap, sync::Arc};
use crate::{proto, B3Digest};
use super::inodes::{DirectoryInodeData, InodeData};
use tvix_castore::proto as castorepb;
use tvix_castore::B3Digest;
/// InodeTracker keeps track of inodes, stores data being these inodes and deals
/// with inode allocation.
@ -139,21 +139,21 @@ impl InodeTracker {
// Consume a list of children with zeroed inodes, and allocate (or fetch existing) inodes.
fn allocate_inodes_for_children(
&mut self,
children: Vec<(u64, proto::node::Node)>,
) -> Vec<(u64, proto::node::Node)> {
children: Vec<(u64, castorepb::node::Node)>,
) -> Vec<(u64, castorepb::node::Node)> {
// allocate new inodes for all children
let mut children_new: Vec<(u64, proto::node::Node)> = Vec::new();
let mut children_new: Vec<(u64, castorepb::node::Node)> = Vec::new();
for (child_ino, ref child_node) in children {
debug_assert_eq!(0, child_ino, "expected child inode to be 0");
let child_ino = match child_node {
proto::node::Node::Directory(directory_node) => {
castorepb::node::Node::Directory(directory_node) => {
// Try putting the sparse data in. If we already have a
// populated version, it'll not update it.
self.put(directory_node.into())
}
proto::node::Node::File(file_node) => self.put(file_node.into()),
proto::node::Node::Symlink(symlink_node) => self.put(symlink_node.into()),
castorepb::node::Node::File(file_node) => self.put(file_node.into()),
castorepb::node::Node::Symlink(symlink_node) => self.put(symlink_node.into()),
};
children_new.push((child_ino, child_node.clone()))
@ -198,8 +198,8 @@ impl InodeTracker {
#[cfg(test)]
mod tests {
use crate::fs::inodes::DirectoryInodeData;
use crate::proto;
use crate::tests::fixtures;
use tvix_castore::proto as castorepb;
use super::InodeData;
use super::InodeTracker;
@ -304,7 +304,7 @@ mod tests {
let (child_ino, child_node) = children.first().unwrap();
assert_ne!(dir_ino, *child_ino);
assert_eq!(
&proto::node::Node::File(
&castorepb::node::Node::File(
fixtures::DIRECTORY_WITH_KEEP.files.first().unwrap().clone()
),
child_node
@ -362,7 +362,9 @@ mod tests {
let (child_ino, child_node) = &children[0];
assert!(!seen_inodes.contains(child_ino));
assert_eq!(
&proto::node::Node::File(fixtures::DIRECTORY_COMPLICATED.files[0].clone()),
&castorepb::node::Node::File(
fixtures::DIRECTORY_COMPLICATED.files[0].clone()
),
child_node
);
seen_inodes.push(*child_ino);
@ -373,7 +375,7 @@ mod tests {
let (child_ino, child_node) = &children[1];
assert!(!seen_inodes.contains(child_ino));
assert_eq!(
&proto::node::Node::Symlink(
&castorepb::node::Node::Symlink(
fixtures::DIRECTORY_COMPLICATED.symlinks[0].clone()
),
child_node
@ -386,7 +388,7 @@ mod tests {
let (child_ino, child_node) = &children[2];
assert!(!seen_inodes.contains(child_ino));
assert_eq!(
&proto::node::Node::Directory(
&castorepb::node::Node::Directory(
fixtures::DIRECTORY_COMPLICATED.directories[0].clone()
),
child_node
@ -439,7 +441,7 @@ mod tests {
let (child_node_inode, child_node) = children.first().unwrap();
assert_ne!(dir_complicated_ino, *child_node_inode);
assert_eq!(
&proto::node::Node::File(
&castorepb::node::Node::File(
fixtures::DIRECTORY_WITH_KEEP.files.first().unwrap().clone()
),
child_node

42
tvix/store/src/fs/inodes.rs
View file

@ -1,6 +1,7 @@
//! This module contains all the data structures used to track information
//! about inodes, which present tvix-store nodes in a filesystem.
use crate::{proto, B3Digest};
use tvix_castore::proto as castorepb;
use tvix_castore::B3Digest;
#[derive(Clone, Debug)]
pub enum InodeData {
@ -10,33 +11,33 @@ pub enum InodeData {
}
/// This encodes the two different states of [InodeData::Directory].
/// Either the data still is sparse (we only saw a [proto::DirectoryNode], but
/// didn't fetch the [proto::Directory] struct yet,
/// or we processed a lookup and did fetch the data.
/// Either the data still is sparse (we only saw a [castorepb::DirectoryNode],
/// but didn't fetch the [castorepb::Directory] struct yet, or we processed a
/// lookup and did fetch the data.
#[derive(Clone, Debug)]
pub enum DirectoryInodeData {
Sparse(B3Digest, u32), // digest, size
Populated(B3Digest, Vec<(u64, proto::node::Node)>), // [(child_inode, node)]
Sparse(B3Digest, u32), // digest, size
Populated(B3Digest, Vec<(u64, castorepb::node::Node)>), // [(child_inode, node)]
}
impl From<&proto::node::Node> for InodeData {
fn from(value: &proto::node::Node) -> Self {
impl From<&castorepb::node::Node> for InodeData {
fn from(value: &castorepb::node::Node) -> Self {
match value {
proto::node::Node::Directory(directory_node) => directory_node.into(),
proto::node::Node::File(file_node) => file_node.into(),
proto::node::Node::Symlink(symlink_node) => symlink_node.into(),
castorepb::node::Node::Directory(directory_node) => directory_node.into(),
castorepb::node::Node::File(file_node) => file_node.into(),
castorepb::node::Node::Symlink(symlink_node) => symlink_node.into(),
}
}
}
impl From<&proto::SymlinkNode> for InodeData {
fn from(value: &proto::SymlinkNode) -> Self {
impl From<&castorepb::SymlinkNode> for InodeData {
fn from(value: &castorepb::SymlinkNode) -> Self {
InodeData::Symlink(value.target.clone())
}
}
impl From<&proto::FileNode> for InodeData {
fn from(value: &proto::FileNode) -> Self {
impl From<&castorepb::FileNode> for InodeData {
fn from(value: &castorepb::FileNode) -> Self {
InodeData::Regular(
value.digest.clone().try_into().unwrap(),
value.size,
@ -46,8 +47,8 @@ impl From<&proto::FileNode> for InodeData {
}
/// Converts a DirectoryNode to a sparsely populated InodeData::Directory.
impl From<&proto::DirectoryNode> for InodeData {
fn from(value: &proto::DirectoryNode) -> Self {
impl From<&castorepb::DirectoryNode> for InodeData {
fn from(value: &castorepb::DirectoryNode) -> Self {
InodeData::Directory(DirectoryInodeData::Sparse(
value.digest.clone().try_into().unwrap(),
value.size,
@ -57,11 +58,12 @@ impl From<&proto::DirectoryNode> for InodeData {
/// converts a proto::Directory to a InodeData::Directory(DirectoryInodeData::Populated(..)).
/// The inodes for each child are 0, because it's up to the InodeTracker to allocate them.
impl From<proto::Directory> for InodeData {
fn from(value: proto::Directory) -> Self {
impl From<castorepb::Directory> for InodeData {
fn from(value: castorepb::Directory) -> Self {
let digest = value.digest();
let children: Vec<(u64, proto::node::Node)> = value.nodes().map(|node| (0, node)).collect();
let children: Vec<(u64, castorepb::node::Node)> =
value.nodes().map(|node| (0, node)).collect();
InodeData::Directory(DirectoryInodeData::Populated(digest, children))
}

15
tvix/store/src/fs/mod.rs
View file

@ -8,13 +8,8 @@ pub mod fuse;
#[cfg(test)]
mod tests;
use crate::{
blobservice::{BlobReader, BlobService},
directoryservice::DirectoryService,
pathinfoservice::PathInfoService,
proto::{node::Node, NamedNode},
B3Digest, Error,
};
use crate::pathinfoservice::PathInfoService;
use fuse_backend_rs::api::filesystem::{Context, FileSystem, FsOptions, ROOT_ID};
use futures::StreamExt;
use nix_compat::store_path::StorePath;
@ -32,6 +27,12 @@ use tokio::{
sync::mpsc,
};
use tracing::{debug, info_span, warn};
use tvix_castore::{
blobservice::{BlobReader, BlobService},
directoryservice::DirectoryService,
proto::{node::Node, NamedNode},
B3Digest, Error,
};
use self::{
file_attr::{gen_file_attr, ROOT_FILE_ATTR},

44
tvix/store/src/fs/tests.rs
View file

@ -5,17 +5,17 @@ use std::path::Path;
use std::sync::Arc;
use tokio::{fs, io};
use tokio_stream::wrappers::ReadDirStream;
use tvix_castore::blobservice::BlobService;
use tvix_castore::directoryservice::DirectoryService;
use tempfile::TempDir;
use crate::blobservice::BlobService;
use crate::directoryservice::DirectoryService;
use crate::fs::{fuse::FuseDaemon, TvixStoreFs};
use crate::pathinfoservice::PathInfoService;
use crate::proto;
use crate::proto::{DirectoryNode, FileNode, PathInfo};
use crate::proto::PathInfo;
use crate::tests::fixtures;
use crate::tests::utils::{gen_blob_service, gen_directory_service, gen_pathinfo_service};
use tvix_castore::proto as castorepb;
const BLOB_A_NAME: &str = "00000000000000000000000000000000-test";
const BLOB_B_NAME: &str = "55555555555555555555555555555555-test";
@ -67,8 +67,8 @@ async fn populate_blob_a(
// Create a PathInfo for it
let path_info = PathInfo {
node: Some(proto::Node {
node: Some(proto::node::Node::File(FileNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::File(castorepb::FileNode {
name: BLOB_A_NAME.into(),
digest: fixtures::BLOB_A_DIGEST.clone().into(),
size: fixtures::BLOB_A.len() as u32,
@ -97,8 +97,8 @@ async fn populate_blob_b(
// Create a PathInfo for it
let path_info = PathInfo {
node: Some(proto::Node {
node: Some(proto::node::Node::File(FileNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::File(castorepb::FileNode {
name: BLOB_B_NAME.into(),
digest: fixtures::BLOB_B_DIGEST.clone().into(),
size: fixtures::BLOB_B.len() as u32,
@ -131,8 +131,8 @@ async fn populate_helloworld_blob(
// Create a PathInfo for it
let path_info = PathInfo {
node: Some(proto::Node {
node: Some(proto::node::Node::File(FileNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::File(castorepb::FileNode {
name: HELLOWORLD_BLOB_NAME.into(),
digest: fixtures::HELLOWORLD_BLOB_DIGEST.clone().into(),
size: fixtures::HELLOWORLD_BLOB_CONTENTS.len() as u32,
@ -154,8 +154,8 @@ async fn populate_symlink(
) {
// Create a PathInfo for it
let path_info = PathInfo {
node: Some(proto::Node {
node: Some(proto::node::Node::Symlink(proto::SymlinkNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::Symlink(castorepb::SymlinkNode {
name: SYMLINK_NAME.into(),
target: BLOB_A_NAME.into(),
})),
@ -177,8 +177,8 @@ async fn populate_symlink2(
) {
// Create a PathInfo for it
let path_info = PathInfo {
node: Some(proto::Node {
node: Some(proto::node::Node::Symlink(proto::SymlinkNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::Symlink(castorepb::SymlinkNode {
name: SYMLINK_NAME2.into(),
target: "/nix/store/somewhereelse".into(),
})),
@ -211,8 +211,8 @@ async fn populate_directory_with_keep(
// upload pathinfo
let path_info = PathInfo {
node: Some(proto::Node {
node: Some(proto::node::Node::Directory(DirectoryNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::Directory(castorepb::DirectoryNode {
name: DIRECTORY_WITH_KEEP_NAME.into(),
digest: fixtures::DIRECTORY_WITH_KEEP.digest().into(),
size: fixtures::DIRECTORY_WITH_KEEP.size(),
@ -235,8 +235,8 @@ async fn populate_pathinfo_without_directory(
) {
// upload pathinfo
let path_info = PathInfo {
node: Some(proto::Node {
node: Some(proto::node::Node::Directory(DirectoryNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::Directory(castorepb::DirectoryNode {
name: DIRECTORY_WITH_KEEP_NAME.into(),
digest: fixtures::DIRECTORY_WITH_KEEP.digest().into(),
size: fixtures::DIRECTORY_WITH_KEEP.size(),
@ -258,8 +258,8 @@ async fn populate_blob_a_without_blob(
) {
// Create a PathInfo for blob A
let path_info = PathInfo {
node: Some(proto::Node {
node: Some(proto::node::Node::File(FileNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::File(castorepb::FileNode {
name: BLOB_A_NAME.into(),
digest: fixtures::BLOB_A_DIGEST.clone().into(),
size: fixtures::BLOB_A.len() as u32,
@ -300,8 +300,8 @@ async fn populate_directory_complicated(
// upload pathinfo
let path_info = PathInfo {
node: Some(proto::Node {
node: Some(proto::node::Node::Directory(DirectoryNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::Directory(castorepb::DirectoryNode {
name: DIRECTORY_COMPLICATED_NAME.into(),
digest: fixtures::DIRECTORY_COMPLICATED.digest().into(),
size: fixtures::DIRECTORY_COMPLICATED.size(),

9
tvix/store/src/lib.rs
View file

@ -1,18 +1,9 @@
mod digests;
mod errors;
#[cfg(feature = "fs")]
pub mod fs;
pub mod blobservice;
pub mod directoryservice;
pub mod import;
pub mod nar;
pub mod pathinfoservice;
pub mod proto;
pub use digests::B3Digest;
pub use errors::Error;
#[cfg(test)]
mod tests;

7
tvix/store/src/nar/mod.rs
View file

@ -1,16 +1,15 @@
use crate::B3Digest;
use data_encoding::BASE64;
use thiserror::Error;
use tvix_castore::{B3Digest, Error};
mod renderer;
pub use renderer::calculate_size_and_sha256;
pub use renderer::write_nar;
/// Errors that can encounter while rendering NARs.
#[derive(Debug, Error)]
#[derive(Debug, thiserror::Error)]
pub enum RenderError {
#[error("failure talking to a backing store client: {0}")]
StoreError(crate::Error),
StoreError(Error),
#[error("unable to find directory {}, referred from {:?}", .0, .1)]
DirectoryNotFound(B3Digest, bytes::Bytes),

33
tvix/store/src/nar/renderer.rs
View file

@ -1,20 +1,21 @@
use super::RenderError;
use crate::{
blobservice::BlobService,
directoryservice::DirectoryService,
proto::{self, NamedNode},
};
use count_write::CountWrite;
use nix_compat::nar;
use sha2::{Digest, Sha256};
use std::{io, sync::Arc};
use tokio::{io::BufReader, task::spawn_blocking};
use tracing::warn;
use tvix_castore::{
blobservice::BlobService,
directoryservice::DirectoryService,
proto::{self as castorepb, NamedNode},
Error,
};
/// Invoke [write_nar], and return the size and sha256 digest of the produced
/// NAR output.
pub async fn calculate_size_and_sha256(
root_node: &proto::node::Node,
root_node: &castorepb::node::Node,
blob_service: Arc<dyn BlobService>,
directory_service: Arc<dyn DirectoryService>,
) -> Result<(u64, [u8; 32]), RenderError> {
@ -26,9 +27,9 @@ pub async fn calculate_size_and_sha256(
Ok((cw.count(), cw.into_inner().finalize().into()))
}
/// Accepts a [proto::node::Node] pointing to the root of a (store) path,
/// and uses the passed blob_service and directory_service to
/// perform the necessary lookups as it traverses the structure.
/// Accepts a [castorepb::node::Node] pointing to the root of a (store) path,
/// and uses the passed blob_service and directory_service to perform the
/// necessary lookups as it traverses the structure.
/// The contents in NAR serialization are writen to the passed [std::io::Write].
///
/// The writer is passed back in the return value. This is done because async Rust
@ -39,7 +40,7 @@ pub async fn calculate_size_and_sha256(
/// This will panic if called outside the context of a Tokio runtime.
pub async fn write_nar<W: std::io::Write + Send + 'static>(
mut w: W,
proto_root_node: &proto::node::Node,
proto_root_node: &castorepb::node::Node,
blob_service: Arc<dyn BlobService>,
directory_service: Arc<dyn DirectoryService>,
) -> Result<W, RenderError> {
@ -69,24 +70,24 @@ pub async fn write_nar<W: std::io::Write + Send + 'static>(
fn walk_node(
tokio_handle: tokio::runtime::Handle,
nar_node: nar::writer::Node,
proto_node: &proto::node::Node,
proto_node: &castorepb::node::Node,
blob_service: Arc<dyn BlobService>,
directory_service: Arc<dyn DirectoryService>,
) -> Result<(), RenderError> {
match proto_node {
proto::node::Node::Symlink(proto_symlink_node) => {
castorepb::node::Node::Symlink(proto_symlink_node) => {
nar_node
.symlink(&proto_symlink_node.target)
.map_err(RenderError::NARWriterError)?;
}
proto::node::Node::File(proto_file_node) => {
castorepb::node::Node::File(proto_file_node) => {
let digest = proto_file_node.digest.clone().try_into().map_err(|_e| {
warn!(
file_node = ?proto_file_node,
"invalid digest length in file node",
);
RenderError::StoreError(crate::Error::StorageError(
RenderError::StoreError(Error::StorageError(
"invalid digest len in file node".to_string(),
))
})?;
@ -110,13 +111,13 @@ fn walk_node(
)
.map_err(RenderError::NARWriterError)?;
}
proto::node::Node::Directory(proto_directory_node) => {
castorepb::node::Node::Directory(proto_directory_node) => {
let digest = proto_directory_node
.digest
.clone()
.try_into()
.map_err(|_e| {
RenderError::StoreError(crate::Error::StorageError(
RenderError::StoreError(Error::StorageError(
"invalid digest len in directory node".to_string(),
))
})?;

17
tvix/store/src/pathinfoservice/from_addr.rs
View file

@ -1,10 +1,9 @@
use std::sync::Arc;
use url::Url;
use crate::{blobservice::BlobService, directoryservice::DirectoryService};
use super::{GRPCPathInfoService, MemoryPathInfoService, PathInfoService, SledPathInfoService};
use std::sync::Arc;
use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService, Error};
use url::Url;
/// Constructs a new instance of a [PathInfoService] from an URI.
///
/// The following URIs are supported:
@ -26,9 +25,9 @@ pub fn from_addr(
uri: &str,
blob_service: Arc<dyn BlobService>,
directory_service: Arc<dyn DirectoryService>,
) -> Result<Arc<dyn PathInfoService>, crate::Error> {
let url = Url::parse(uri)
.map_err(|e| crate::Error::StorageError(format!("unable to parse url: {}", e)))?;
) -> Result<Arc<dyn PathInfoService>, Error> {
let url =
Url::parse(uri).map_err(|e| Error::StorageError(format!("unable to parse url: {}", e)))?;
Ok(if url.scheme() == "memory" {
Arc::new(MemoryPathInfoService::from_url(
@ -49,7 +48,7 @@ pub fn from_addr(
directory_service,
)?)
} else {
Err(crate::Error::StorageError(format!(
Err(Error::StorageError(format!(
"unknown scheme: {}",
url.scheme()
)))?

45
tvix/store/src/pathinfoservice/grpc.rs
View file

@ -1,14 +1,13 @@
use super::PathInfoService;
use crate::{
blobservice::BlobService,
directoryservice::DirectoryService,
proto::{self, ListPathInfoRequest},
};
use crate::proto::{self, ListPathInfoRequest, PathInfo};
use async_stream::try_stream;
use futures::Stream;
use std::{pin::Pin, sync::Arc};
use tokio::net::UnixStream;
use tonic::{async_trait, transport::Channel, Code};
use tvix_castore::{
blobservice::BlobService, directoryservice::DirectoryService, proto as castorepb, Error,
};
/// Connects to a (remote) tvix-store PathInfoService over gRPC.
#[derive(Clone)]
@ -40,16 +39,14 @@ impl PathInfoService for GRPCPathInfoService {
url: &url::Url,
_blob_service: Arc<dyn BlobService>,
_directory_service: Arc<dyn DirectoryService>,
) -> Result<Self, crate::Error> {
) -> Result<Self, tvix_castore::Error> {
// Start checking for the scheme to start with grpc+.
match url.scheme().strip_prefix("grpc+") {
None => Err(crate::Error::StorageError("invalid scheme".to_string())),
None => Err(Error::StorageError("invalid scheme".to_string())),
Some(rest) => {
if rest == "unix" {
if url.host_str().is_some() {
return Err(crate::Error::StorageError(
"host may not be set".to_string(),
));
return Err(Error::StorageError("host may not be set".to_string()));
}
let path = url.path().to_string();
let channel = tonic::transport::Endpoint::try_from("http://[::]:50051") // doesn't matter
@ -63,7 +60,7 @@ impl PathInfoService for GRPCPathInfoService {
} else {
// ensure path is empty, not supported with gRPC.
if !url.path().is_empty() {
return Err(crate::Error::StorageError(
return Err(tvix_castore::Error::StorageError(
"path may not be set".to_string(),
));
}
@ -89,7 +86,7 @@ impl PathInfoService for GRPCPathInfoService {
}
}
async fn get(&self, digest: [u8; 20]) -> Result<Option<proto::PathInfo>, crate::Error> {
async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> {
// Get a new handle to the gRPC client.
let mut grpc_client = self.grpc_client.clone();
@ -104,18 +101,18 @@ impl PathInfoService for GRPCPathInfoService {
match path_info {
Ok(path_info) => Ok(Some(path_info.into_inner())),
Err(e) if e.code() == Code::NotFound => Ok(None),
Err(e) => Err(crate::Error::StorageError(e.to_string())),
Err(e) => Err(Error::StorageError(e.to_string())),
}
}
async fn put(&self, path_info: proto::PathInfo) -> Result<proto::PathInfo, crate::Error> {
async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error> {
// Get a new handle to the gRPC client.
let mut grpc_client = self.grpc_client.clone();
let path_info = grpc_client
.put(path_info)
.await
.map_err(|e| crate::Error::StorageError(e.to_string()))?
.map_err(|e| Error::StorageError(e.to_string()))?
.into_inner();
Ok(path_info)
@ -123,36 +120,36 @@ impl PathInfoService for GRPCPathInfoService {
async fn calculate_nar(
&self,
root_node: &proto::node::Node,
) -> Result<(u64, [u8; 32]), crate::Error> {
root_node: &castorepb::node::Node,
) -> Result<(u64, [u8; 32]), Error> {
// Get a new handle to the gRPC client.
let mut grpc_client = self.grpc_client.clone();
let root_node = root_node.clone();
let path_info = grpc_client
.calculate_nar(proto::Node {
.calculate_nar(castorepb::Node {
node: Some(root_node),
})
.await
.map_err(|e| crate::Error::StorageError(e.to_string()))?
.map_err(|e| Error::StorageError(e.to_string()))?
.into_inner();
let nar_sha256: [u8; 32] = path_info
.nar_sha256
.to_vec()
.try_into()
.map_err(|_e| crate::Error::StorageError("invalid digest length".to_string()))?;
.map_err(|_e| Error::StorageError("invalid digest length".to_string()))?;
Ok((path_info.nar_size, nar_sha256))
}
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<proto::PathInfo, crate::Error>> + Send>> {
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<PathInfo, Error>> + Send>> {
let mut grpc_client = self.grpc_client.clone();
let stream = try_stream! {
let resp = grpc_client.list(ListPathInfoRequest::default()).await;
let mut stream = resp.map_err(|e| crate::Error::StorageError(e.to_string()))?.into_inner();
let mut stream = resp.map_err(|e| Error::StorageError(e.to_string()))?.into_inner();
loop {
match stream.message().await {
@ -160,7 +157,7 @@ impl PathInfoService for GRPCPathInfoService {
Some(pathinfo) => {
// validate the pathinfo
if let Err(e) = pathinfo.validate() {
Err(crate::Error::StorageError(format!(
Err(Error::StorageError(format!(
"pathinfo {:?} failed validation: {}",
pathinfo, e
)))?;
@ -171,7 +168,7 @@ impl PathInfoService for GRPCPathInfoService {
return;
},
},
Err(e) => Err(crate::Error::StorageError(e.to_string()))?,
Err(e) => Err(Error::StorageError(e.to_string()))?,
}
}
};

25
tvix/store/src/pathinfoservice/memory.rs
View file

@ -1,8 +1,5 @@
use super::PathInfoService;
use crate::{
blobservice::BlobService, directoryservice::DirectoryService, nar::calculate_size_and_sha256,
proto, Error,
};
use crate::{nar::calculate_size_and_sha256, proto::PathInfo};
use futures::{stream::iter, Stream};
use std::{
collections::HashMap,
@ -10,9 +7,12 @@ use std::{
sync::{Arc, RwLock},
};
use tonic::async_trait;
use tvix_castore::proto as castorepb;
use tvix_castore::Error;
use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService};
pub struct MemoryPathInfoService {
db: Arc<RwLock<HashMap<[u8; 20], proto::PathInfo>>>,
db: Arc<RwLock<HashMap<[u8; 20], PathInfo>>>,
blob_service: Arc<dyn BlobService>,
directory_service: Arc<dyn DirectoryService>,
@ -43,17 +43,17 @@ impl PathInfoService for MemoryPathInfoService {
directory_service: Arc<dyn DirectoryService>,
) -> Result<Self, Error> {
if url.scheme() != "memory" {
return Err(crate::Error::StorageError("invalid scheme".to_string()));
return Err(Error::StorageError("invalid scheme".to_string()));
}
if url.has_host() || !url.path().is_empty() {
return Err(crate::Error::StorageError("invalid url".to_string()));
return Err(Error::StorageError("invalid url".to_string()));
}
Ok(Self::new(blob_service, directory_service))
}
async fn get(&self, digest: [u8; 20]) -> Result<Option<proto::PathInfo>, Error> {
async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> {
let db = self.db.read().unwrap();
match db.get(&digest) {
@ -62,7 +62,7 @@ impl PathInfoService for MemoryPathInfoService {
}
}
async fn put(&self, path_info: proto::PathInfo) -> Result<proto::PathInfo, Error> {
async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error> {
// Call validate on the received PathInfo message.
match path_info.validate() {
Err(e) => Err(Error::InvalidRequest(format!(
@ -81,7 +81,10 @@ impl PathInfoService for MemoryPathInfoService {
}
}
async fn calculate_nar(&self, root_node: &proto::node::Node) -> Result<(u64, [u8; 32]), Error> {
async fn calculate_nar(
&self,
root_node: &castorepb::node::Node,
) -> Result<(u64, [u8; 32]), Error> {
calculate_size_and_sha256(
root_node,
self.blob_service.clone(),
@ -91,7 +94,7 @@ impl PathInfoService for MemoryPathInfoService {
.map_err(|e| Error::StorageError(e.to_string()))
}
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<proto::PathInfo, Error>> + Send>> {
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<PathInfo, Error>> + Send>> {
let db = self.db.read().unwrap();
// Copy all elements into a list.

19
tvix/store/src/pathinfoservice/mod.rs
View file

@ -8,10 +8,12 @@ use std::sync::Arc;
use futures::Stream;
use tonic::async_trait;
use tvix_castore::blobservice::BlobService;
use tvix_castore::directoryservice::DirectoryService;
use tvix_castore::proto as castorepb;
use tvix_castore::Error;
use crate::blobservice::BlobService;
use crate::directoryservice::DirectoryService;
use crate::{proto, Error};
use crate::proto::PathInfo;
pub use self::from_addr::from_addr;
pub use self::grpc::GRPCPathInfoService;
@ -34,16 +36,19 @@ pub trait PathInfoService: Send + Sync {
Self: Sized;
/// Retrieve a PathInfo message by the output digest.
async fn get(&self, digest: [u8; 20]) -> Result<Option<proto::PathInfo>, Error>;
async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error>;
/// Store a PathInfo message. Implementations MUST call validate and reject
/// invalid messages.
async fn put(&self, path_info: proto::PathInfo) -> Result<proto::PathInfo, Error>;
async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error>;
/// Return the nar size and nar sha256 digest for a given root node.
/// This can be used to calculate NAR-based output paths,
/// and implementations are encouraged to cache it.
async fn calculate_nar(&self, root_node: &proto::node::Node) -> Result<(u64, [u8; 32]), Error>;
async fn calculate_nar(
&self,
root_node: &castorepb::node::Node,
) -> Result<(u64, [u8; 32]), Error>;
/// Iterate over all PathInfo objects in the store.
/// Implementations can decide to disallow listing.
@ -52,5 +57,5 @@ pub trait PathInfoService: Send + Sync {
/// and the box allows different underlying stream implementations to be returned since
/// Rust doesn't support this as a generic in traits yet. This is the same thing that
/// [async_trait] generates, but for streams instead of futures.
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<proto::PathInfo, Error>> + Send>>;
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<PathInfo, Error>> + Send>>;
}

29
tvix/store/src/pathinfoservice/sled.rs
View file

@ -1,13 +1,13 @@
use super::PathInfoService;
use crate::{
blobservice::BlobService, directoryservice::DirectoryService, nar::calculate_size_and_sha256,
proto, Error,
};
use crate::nar::calculate_size_and_sha256;
use crate::proto::PathInfo;
use futures::{stream::iter, Stream};
use prost::Message;
use std::{path::PathBuf, pin::Pin, sync::Arc};
use tonic::async_trait;
use tracing::warn;
use tvix_castore::proto as castorepb;
use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService, Error};
/// SledPathInfoService stores PathInfo in a [sled](https://github.com/spacejam/sled).
///
@ -63,11 +63,11 @@ impl PathInfoService for SledPathInfoService {
directory_service: Arc<dyn DirectoryService>,
) -> Result<Self, Error> {
if url.scheme() != "sled" {
return Err(crate::Error::StorageError("invalid scheme".to_string()));
return Err(Error::StorageError("invalid scheme".to_string()));
}
if url.has_host() {
return Err(crate::Error::StorageError(format!(
return Err(Error::StorageError(format!(
"invalid host: {}",
url.host().unwrap()
)));
@ -78,7 +78,7 @@ impl PathInfoService for SledPathInfoService {
Self::new_temporary(blob_service, directory_service)
.map_err(|e| Error::StorageError(e.to_string()))
} else if url.path() == "/" {
Err(crate::Error::StorageError(
Err(Error::StorageError(
"cowardly refusing to open / with sled".to_string(),
))
} else {
@ -87,10 +87,10 @@ impl PathInfoService for SledPathInfoService {
}
}
async fn get(&self, digest: [u8; 20]) -> Result<Option<proto::PathInfo>, Error> {
async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> {
match self.db.get(digest) {
Ok(None) => Ok(None),
Ok(Some(data)) => match proto::PathInfo::decode(&*data) {
Ok(Some(data)) => match PathInfo::decode(&*data) {
Ok(path_info) => Ok(Some(path_info)),
Err(e) => {
warn!("failed to decode stored PathInfo: {}", e);
@ -110,7 +110,7 @@ impl PathInfoService for SledPathInfoService {
}
}
async fn put(&self, path_info: proto::PathInfo) -> Result<proto::PathInfo, Error> {
async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error> {
// Call validate on the received PathInfo message.
match path_info.validate() {
Err(e) => Err(Error::InvalidRequest(format!(
@ -131,7 +131,10 @@ impl PathInfoService for SledPathInfoService {
}
}
async fn calculate_nar(&self, root_node: &proto::node::Node) -> Result<(u64, [u8; 32]), Error> {
async fn calculate_nar(
&self,
root_node: &castorepb::node::Node,
) -> Result<(u64, [u8; 32]), Error> {
calculate_size_and_sha256(
root_node,
self.blob_service.clone(),
@ -141,11 +144,11 @@ impl PathInfoService for SledPathInfoService {
.map_err(|e| Error::StorageError(e.to_string()))
}
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<proto::PathInfo, Error>> + Send>> {
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<PathInfo, Error>> + Send>> {
Box::pin(iter(self.db.iter().values().map(|v| match v {
Ok(data) => {
// we retrieved some bytes
match proto::PathInfo::decode(&*data) {
match PathInfo::decode(&*data) {
Ok(path_info) => Ok(path_info),
Err(e) => {
warn!("failed to decode stored PathInfo: {}", e);

3
tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs
View file

@ -7,6 +7,7 @@ use tokio::task;
use tokio_stream::wrappers::ReceiverStream;
use tonic::{async_trait, Request, Response, Result, Status};
use tracing::{debug, instrument, warn};
use tvix_castore::proto as castorepb;
pub struct GRPCPathInfoServiceWrapper {
path_info_service: Arc<dyn PathInfoService>,
@ -67,7 +68,7 @@ impl proto::path_info_service_server::PathInfoService for GRPCPathInfoServiceWra
#[instrument(skip(self))]
async fn calculate_nar(
&self,
request: Request<proto::Node>,
request: Request<castorepb::Node>,
) -> Result<Response<proto::CalculateNarResponse>> {
match request.into_inner().node {
None => Err(Status::invalid_argument("no root node sent")),

293
tvix/store/src/proto/mod.rs
View file

@ -1,23 +1,13 @@
#![allow(clippy::derive_partial_eq_without_eq, non_snake_case)]
// https://github.com/hyperium/tonic/issues/1056
use data_encoding::BASE64;
use std::{collections::HashSet, iter::Peekable};
use thiserror::Error;
use prost::Message;
use nix_compat::store_path::{self, StorePath};
use thiserror::Error;
use tvix_castore::{proto as castorepb, B3Digest};
mod grpc_blobservice_wrapper;
mod grpc_directoryservice_wrapper;
mod grpc_pathinfoservice_wrapper;
pub use grpc_blobservice_wrapper::GRPCBlobServiceWrapper;
pub use grpc_directoryservice_wrapper::GRPCDirectoryServiceWrapper;
pub use grpc_pathinfoservice_wrapper::GRPCPathInfoServiceWrapper;
use crate::B3Digest;
tonic::include_proto!("tvix.store.v1");
#[cfg(feature = "reflection")]
@ -29,23 +19,6 @@ pub const FILE_DESCRIPTOR_SET: &[u8] = tonic::include_file_descriptor_set!("tvix
#[cfg(test)]
mod tests;
/// Errors that can occur during the validation of Directory messages.
#[derive(Debug, PartialEq, Eq, Error)]
pub enum ValidateDirectoryError {
/// Elements are not in sorted order
#[error("{} is not sorted", std::str::from_utf8(.0).unwrap_or(&BASE64.encode(.0)))]
WrongSorting(Vec<u8>),
/// Multiple elements with the same name encountered
#[error("{0:?} is a duplicate name")]
DuplicateName(Vec<u8>),
/// Invalid name encountered
#[error("Invalid name in {0:?}")]
InvalidName(Vec<u8>),
/// Invalid digest length encountered
#[error("Invalid Digest length: {0}")]
InvalidDigestLen(usize),
}
/// Errors that can occur during the validation of PathInfo messages.
#[derive(Debug, Error, PartialEq)]
pub enum ValidatePathInfoError {
@ -67,31 +40,6 @@ pub enum ValidatePathInfoError {
InconsistentNumberOfReferences(usize, usize),
}
/// Checks a Node name for validity as an intermediate node, and returns an
/// error that's generated from the supplied constructor.
///
/// We disallow slashes, null bytes, '.', '..' and the empty string.
fn validate_node_name<E>(name: &[u8], err: fn(Vec<u8>) -> E) -> Result<(), E> {
if name.is_empty()
|| name == b".."
|| name == b"."
|| name.contains(&0x00)
|| name.contains(&b'/')
{
return Err(err(name.to_vec()));
}
Ok(())
}
/// Checks a digest for validity.
/// Digests are 32 bytes long, as we store blake3 digests.
fn validate_digest<E>(digest: &bytes::Bytes, err: fn(usize) -> E) -> Result<(), E> {
if digest.len() != 32 {
return Err(err(digest.len()));
}
Ok(())
}
/// Parses a root node name.
///
/// On success, this returns the parsed [StorePath].
@ -129,16 +77,17 @@ impl PathInfo {
None => {
return Err(ValidatePathInfoError::NoNodePresent());
}
Some(Node { node }) => match node {
Some(castorepb::Node { node }) => match node {
None => {
return Err(ValidatePathInfoError::NoNodePresent());
}
Some(node::Node::Directory(directory_node)) => {
Some(castorepb::node::Node::Directory(directory_node)) => {
// ensure the digest has the appropriate size.
validate_digest(
&directory_node.digest,
ValidatePathInfoError::InvalidDigestLen,
)?;
if TryInto::<B3Digest>::try_into(directory_node.digest.clone()).is_err() {
return Err(ValidatePathInfoError::InvalidDigestLen(
directory_node.digest.len(),
));
}
// parse the name
parse_node_name_root(
@ -146,14 +95,18 @@ impl PathInfo {
ValidatePathInfoError::InvalidNodeName,
)?
}
Some(node::Node::File(file_node)) => {
Some(castorepb::node::Node::File(file_node)) => {
// ensure the digest has the appropriate size.
validate_digest(&file_node.digest, ValidatePathInfoError::InvalidDigestLen)?;
if TryInto::<B3Digest>::try_into(file_node.digest.clone()).is_err() {
return Err(ValidatePathInfoError::InvalidDigestLen(
file_node.digest.len(),
));
}
// parse the name
parse_node_name_root(&file_node.name, ValidatePathInfoError::InvalidNodeName)?
}
Some(node::Node::Symlink(symlink_node)) => {
Some(castorepb::node::Node::Symlink(symlink_node)) => {
// parse the name
parse_node_name_root(
&symlink_node.name,
@ -167,217 +120,3 @@ impl PathInfo {
Ok(root_nix_path)
}
}
/// NamedNode is implemented for [FileNode], [DirectoryNode] and [SymlinkNode]
/// and [node::Node], so we can ask all of them for the name easily.
pub trait NamedNode {
fn get_name(&self) -> &[u8];
}
impl NamedNode for &FileNode {
fn get_name(&self) -> &[u8] {
&self.name
}
}
impl NamedNode for &DirectoryNode {
fn get_name(&self) -> &[u8] {
&self.name
}
}
impl NamedNode for &SymlinkNode {
fn get_name(&self) -> &[u8] {
&self.name
}
}
impl NamedNode for node::Node {
fn get_name(&self) -> &[u8] {
match self {
node::Node::File(node_file) => &node_file.name,
node::Node::Directory(node_directory) => &node_directory.name,
node::Node::Symlink(node_symlink) => &node_symlink.name,
}
}
}
impl node::Node {
/// Returns the node with a new name.
pub fn rename(self, name: bytes::Bytes) -> Self {
match self {
node::Node::Directory(n) => node::Node::Directory(DirectoryNode { name, ..n }),
node::Node::File(n) => node::Node::File(FileNode { name, ..n }),
node::Node::Symlink(n) => node::Node::Symlink(SymlinkNode { name, ..n }),
}
}
}
/// Accepts a name, and a mutable reference to the previous name.
/// If the passed name is larger than the previous one, the reference is updated.
/// If it's not, an error is returned.
fn update_if_lt_prev<'n>(
prev_name: &mut &'n [u8],
name: &'n [u8],
) -> Result<(), ValidateDirectoryError> {
if *name < **prev_name {
return Err(ValidateDirectoryError::WrongSorting(name.to_vec()));
}
*prev_name = name;
Ok(())
}
/// Inserts the given name into a HashSet if it's not already in there.
/// If it is, an error is returned.
fn insert_once<'n>(
seen_names: &mut HashSet<&'n [u8]>,
name: &'n [u8],
) -> Result<(), ValidateDirectoryError> {
if seen_names.get(name).is_some() {
return Err(ValidateDirectoryError::DuplicateName(name.to_vec()));
}
seen_names.insert(name);
Ok(())
}
impl Directory {
/// The size of a directory is the number of all regular and symlink elements,
/// the number of directory elements, and their size fields.
pub fn size(&self) -> u32 {
self.files.len() as u32
+ self.symlinks.len() as u32
+ self
.directories
.iter()
.fold(0, |acc: u32, e| (acc + 1 + e.size))
}
/// Calculates the digest of a Directory, which is the blake3 hash of a
/// Directory protobuf message, serialized in protobuf canonical form.
pub fn digest(&self) -> B3Digest {
let mut hasher = blake3::Hasher::new();
hasher
.update(&self.encode_to_vec())
.finalize()
.as_bytes()
.into()
}
/// validate checks the directory for invalid data, such as:
/// - violations of name restrictions
/// - invalid digest lengths
/// - not properly sorted lists
/// - duplicate names in the three lists
pub fn validate(&self) -> Result<(), ValidateDirectoryError> {
let mut seen_names: HashSet<&[u8]> = HashSet::new();
let mut last_directory_name: &[u8] = b"";
let mut last_file_name: &[u8] = b"";
let mut last_symlink_name: &[u8] = b"";
// check directories
for directory_node in &self.directories {
validate_node_name(&directory_node.name, ValidateDirectoryError::InvalidName)?;
validate_digest(
&directory_node.digest,
ValidateDirectoryError::InvalidDigestLen,
)?;
update_if_lt_prev(&mut last_directory_name, &directory_node.name)?;
insert_once(&mut seen_names, &directory_node.name)?;
}
// check files
for file_node in &self.files {
validate_node_name(&file_node.name, ValidateDirectoryError::InvalidName)?;
validate_digest(&file_node.digest, ValidateDirectoryError::InvalidDigestLen)?;
update_if_lt_prev(&mut last_file_name, &file_node.name)?;
insert_once(&mut seen_names, &file_node.name)?;
}
// check symlinks
for symlink_node in &self.symlinks {
validate_node_name(&symlink_node.name, ValidateDirectoryError::InvalidName)?;
update_if_lt_prev(&mut last_symlink_name, &symlink_node.name)?;
insert_once(&mut seen_names, &symlink_node.name)?;
}
Ok(())
}
/// Allows iterating over all three nodes ([DirectoryNode], [FileNode],
/// [SymlinkNode]) in an ordered fashion, as long as the individual lists
/// are sorted (which can be checked by the [Directory::validate]).
pub fn nodes(&self) -> DirectoryNodesIterator {
return DirectoryNodesIterator {
i_directories: self.directories.iter().peekable(),
i_files: self.files.iter().peekable(),
i_symlinks: self.symlinks.iter().peekable(),
};
}
}
/// Struct to hold the state of an iterator over all nodes of a Directory.
///
/// Internally, this keeps peekable Iterators over all three lists of a
/// Directory message.
pub struct DirectoryNodesIterator<'a> {
// directory: &Directory,
i_directories: Peekable<std::slice::Iter<'a, DirectoryNode>>,
i_files: Peekable<std::slice::Iter<'a, FileNode>>,
i_symlinks: Peekable<std::slice::Iter<'a, SymlinkNode>>,
}
/// looks at two elements implementing NamedNode, and returns true if "left
/// is smaller / comes first".
///
/// Some(_) is preferred over None.
fn left_name_lt_right<A: NamedNode, B: NamedNode>(left: Option<&A>, right: Option<&B>) -> bool {
match left {
// if left is None, right always wins
None => false,
Some(left_inner) => {
// left is Some.
match right {
// left is Some, right is None - left wins.
None => true,
Some(right_inner) => {
// both are Some - compare the name.
return left_inner.get_name() < right_inner.get_name();
}
}
}
}
}
impl Iterator for DirectoryNodesIterator<'_> {
type Item = node::Node;
// next returns the next node in the Directory.
// we peek at all three internal iterators, and pick the one with the
// smallest name, to ensure lexicographical ordering.
// The individual lists are already known to be sorted.
fn next(&mut self) -> Option<Self::Item> {
if left_name_lt_right(self.i_directories.peek(), self.i_files.peek()) {
// i_directories is still in the game, compare with symlinks
if left_name_lt_right(self.i_directories.peek(), self.i_symlinks.peek()) {
self.i_directories
.next()
.cloned()
.map(node::Node::Directory)
} else {
self.i_symlinks.next().cloned().map(node::Node::Symlink)
}
} else {
// i_files is still in the game, compare with symlinks
if left_name_lt_right(self.i_files.peek(), self.i_symlinks.peek()) {
self.i_files.next().cloned().map(node::Node::File)
} else {
self.i_symlinks.next().cloned().map(node::Node::Symlink)
}
}
}
}

8
tvix/store/src/proto/tests/grpc_pathinfoservice.rs
View file

@ -1,9 +1,8 @@
use crate::proto::get_path_info_request::ByWhat::ByOutputHash;
use crate::proto::node::Node::Symlink;
use crate::proto::path_info_service_server::PathInfoService as GRPCPathInfoService;
use crate::proto::GRPCPathInfoServiceWrapper;
use crate::proto::GetPathInfoRequest;
use crate::proto::PathInfo;
use crate::proto::{GetPathInfoRequest, Node, SymlinkNode};
use crate::tests::fixtures::DUMMY_OUTPUT_HASH;
use crate::tests::utils::gen_blob_service;
use crate::tests::utils::gen_directory_service;
@ -11,6 +10,7 @@ use crate::tests::utils::gen_pathinfo_service;
use std::sync::Arc;
use tokio_stream::wrappers::ReceiverStream;
use tonic::Request;
use tvix_castore::proto as castorepb;
/// generates a GRPCPathInfoService out of blob, directory and pathinfo services.
///
@ -48,8 +48,8 @@ async fn put_get() {
let service = gen_grpc_service();
let path_info = PathInfo {
node: Some(Node {
node: Some(Symlink(SymlinkNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::Symlink(castorepb::SymlinkNode {
name: "00000000000000000000000000000000-foo".into(),
target: "doesntmatter".into(),
})),

4
tvix/store/src/proto/tests/mod.rs
View file

@ -1,6 +1,2 @@
mod directory;
mod directory_nodes_iterator;
mod grpc_blobservice;
mod grpc_directoryservice;
mod grpc_pathinfoservice;
mod pathinfo;

76
tvix/store/src/proto/tests/pathinfo.rs
View file

@ -1,31 +1,10 @@
use crate::proto::{self, Node, PathInfo, ValidatePathInfoError};
use crate::B3Digest;
use crate::proto::{NarInfo, PathInfo, ValidatePathInfoError};
use crate::tests::fixtures::*;
use bytes::Bytes;
use lazy_static::lazy_static;
use nix_compat::store_path::{self, StorePath};
use std::str::FromStr;
use test_case::test_case;
lazy_static! {
static ref DUMMY_DIGEST: B3Digest = {
let u: &[u8; 32] = &[
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
];
u.into()
};
static ref DUMMY_DIGEST_2: B3Digest = {
let u: &[u8; 32] = &[
0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
];
u.into()
};
}
const DUMMY_NAME: &str = "00000000000000000000000000000000-dummy";
use tvix_castore::proto as castorepb;
#[test_case(
None,
@ -33,12 +12,12 @@ const DUMMY_NAME: &str = "00000000000000000000000000000000-dummy";
"No node"
)]
#[test_case(
Some(Node { node: None }),
Some(castorepb::Node { node: None }),
Err(ValidatePathInfoError::NoNodePresent());
"No node 2"
)]
fn validate_no_node(
t_node: Option<proto::Node>,
t_node: Option<castorepb::Node>,
t_result: Result<StorePath, ValidatePathInfoError>,
) {
// construct the PathInfo object
@ -50,7 +29,7 @@ fn validate_no_node(
}
#[test_case(
proto::DirectoryNode {
castorepb::DirectoryNode {
name: DUMMY_NAME.into(),
digest: DUMMY_DIGEST.clone().into(),
size: 0,
@ -59,7 +38,7 @@ fn validate_no_node(
"ok"
)]
#[test_case(
proto::DirectoryNode {
castorepb::DirectoryNode {
name: DUMMY_NAME.into(),
digest: Bytes::new(),
size: 0,
@ -68,7 +47,7 @@ fn validate_no_node(
"invalid digest length"
)]
#[test_case(
proto::DirectoryNode {
castorepb::DirectoryNode {
name: "invalid".into(),
digest: DUMMY_DIGEST.clone().into(),
size: 0,
@ -80,13 +59,13 @@ fn validate_no_node(
"invalid node name"
)]
fn validate_directory(
t_directory_node: proto::DirectoryNode,
t_directory_node: castorepb::DirectoryNode,
t_result: Result<StorePath, ValidatePathInfoError>,
) {
// construct the PathInfo object
let p = PathInfo {
node: Some(Node {
node: Some(proto::node::Node::Directory(t_directory_node)),
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::Directory(t_directory_node)),
}),
..Default::default()
};
@ -94,7 +73,7 @@ fn validate_directory(
}
#[test_case(
proto::FileNode {
castorepb::FileNode {
name: DUMMY_NAME.into(),
digest: DUMMY_DIGEST.clone().into(),
size: 0,
@ -104,7 +83,7 @@ fn validate_directory(
"ok"
)]
#[test_case(
proto::FileNode {
castorepb::FileNode {
name: DUMMY_NAME.into(),
digest: Bytes::new(),
..Default::default()
@ -113,7 +92,7 @@ fn validate_directory(
"invalid digest length"
)]
#[test_case(
proto::FileNode {
castorepb::FileNode {
name: "invalid".into(),
digest: DUMMY_DIGEST.clone().into(),
..Default::default()
@ -124,11 +103,14 @@ fn validate_directory(
));
"invalid node name"
)]
fn validate_file(t_file_node: proto::FileNode, t_result: Result<StorePath, ValidatePathInfoError>) {
fn validate_file(
t_file_node: castorepb::FileNode,
t_result: Result<StorePath, ValidatePathInfoError>,
) {
// construct the PathInfo object
let p = PathInfo {
node: Some(Node {
node: Some(proto::node::Node::File(t_file_node)),
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::File(t_file_node)),
}),
..Default::default()
};
@ -136,7 +118,7 @@ fn validate_file(t_file_node: proto::FileNode, t_result: Result<StorePath, Valid
}
#[test_case(
proto::SymlinkNode {
castorepb::SymlinkNode {
name: DUMMY_NAME.into(),
..Default::default()
},
@ -144,7 +126,7 @@ fn validate_file(t_file_node: proto::FileNode, t_result: Result<StorePath, Valid
"ok"
)]
#[test_case(
proto::SymlinkNode {
castorepb::SymlinkNode {
name: "invalid".into(),
..Default::default()
},
@ -155,13 +137,13 @@ fn validate_file(t_file_node: proto::FileNode, t_result: Result<StorePath, Valid
"invalid node name"
)]
fn validate_symlink(
t_symlink_node: proto::SymlinkNode,
t_symlink_node: castorepb::SymlinkNode,
t_result: Result<StorePath, ValidatePathInfoError>,
) {
// construct the PathInfo object
let p = PathInfo {
node: Some(Node {
node: Some(proto::node::Node::Symlink(t_symlink_node)),
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::Symlink(t_symlink_node)),
}),
..Default::default()
};
@ -172,8 +154,8 @@ fn validate_symlink(
fn validate_references() {
// create a PathInfo without narinfo field.
let path_info = PathInfo {
node: Some(Node {
node: Some(proto::node::Node::Directory(proto::DirectoryNode {
node: Some(castorepb::Node {
node: Some(castorepb::node::Node::Directory(castorepb::DirectoryNode {
name: DUMMY_NAME.into(),
digest: DUMMY_DIGEST.clone().into(),
size: 0,
@ -186,7 +168,7 @@ fn validate_references() {
// create a PathInfo with a narinfo field, but an inconsistent set of references
let path_info_with_narinfo_missing_refs = PathInfo {
narinfo: Some(proto::NarInfo {
narinfo: Some(NarInfo {
nar_size: 0,
nar_sha256: DUMMY_DIGEST.clone().into(),
signatures: vec![],
@ -204,7 +186,7 @@ fn validate_references() {
// create a pathinfo with the correct number of references, should suceed
let path_info_with_narinfo = PathInfo {
narinfo: Some(proto::NarInfo {
narinfo: Some(NarInfo {
nar_size: 0,
nar_sha256: DUMMY_DIGEST.clone().into(),
signatures: vec![],

85
tvix/store/src/tests/fixtures.rs
View file

@ -1,90 +1,9 @@
use crate::{
proto::{self, Directory, DirectoryNode, FileNode, SymlinkNode},
B3Digest,
};
use lazy_static::lazy_static;
pub use tvix_castore::fixtures::*;
pub const HELLOWORLD_BLOB_CONTENTS: &[u8] = b"Hello World!";
pub const EMPTY_BLOB_CONTENTS: &[u8] = b"";
pub const DUMMY_NAME: &str = "00000000000000000000000000000000-dummy";
lazy_static! {
pub static ref DUMMY_DIGEST: B3Digest = {
let u: &[u8; 32] = &[
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
];
u.into()
};
pub static ref DUMMY_DATA_1: bytes::Bytes = vec![0x01, 0x02, 0x03].into();
pub static ref DUMMY_DATA_2: bytes::Bytes = vec![0x04, 0x05].into();
pub static ref HELLOWORLD_BLOB_DIGEST: B3Digest =
blake3::hash(HELLOWORLD_BLOB_CONTENTS).as_bytes().into();
pub static ref EMPTY_BLOB_DIGEST: B3Digest =
blake3::hash(EMPTY_BLOB_CONTENTS).as_bytes().into();
// 2 bytes
pub static ref BLOB_A: bytes::Bytes = vec![0x00, 0x01].into();
pub static ref BLOB_A_DIGEST: B3Digest = blake3::hash(&BLOB_A).as_bytes().into();
// 1MB
pub static ref BLOB_B: bytes::Bytes = (0..255).collect::<Vec<u8>>().repeat(4 * 1024).into();
pub static ref BLOB_B_DIGEST: B3Digest = blake3::hash(&BLOB_B).as_bytes().into();
// Directories
pub static ref DIRECTORY_WITH_KEEP: proto::Directory = proto::Directory {
directories: vec![],
files: vec![FileNode {
name: b".keep".to_vec().into(),
digest: EMPTY_BLOB_DIGEST.clone().into(),
size: 0,
executable: false,
}],
symlinks: vec![],
};
pub static ref DIRECTORY_COMPLICATED: proto::Directory = proto::Directory {
directories: vec![DirectoryNode {
name: b"keep".to_vec().into(),
digest: DIRECTORY_WITH_KEEP.digest().into(),
size: DIRECTORY_WITH_KEEP.size(),
}],
files: vec![FileNode {
name: b".keep".to_vec().into(),
digest: EMPTY_BLOB_DIGEST.clone().into(),
size: 0,
executable: false,
}],
symlinks: vec![SymlinkNode {
name: b"aa".to_vec().into(),
target: b"/nix/store/somewhereelse".to_vec().into(),
}],
};
pub static ref DIRECTORY_A: Directory = Directory::default();
pub static ref DIRECTORY_B: Directory = Directory {
directories: vec![DirectoryNode {
name: b"a".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
}],
..Default::default()
};
pub static ref DIRECTORY_C: Directory = Directory {
directories: vec![
DirectoryNode {
name: b"a".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
},
DirectoryNode {
name: b"a'".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
}
],
..Default::default()
};
// output hash
pub static ref DUMMY_OUTPUT_HASH: bytes::Bytes = vec![
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,

1
tvix/store/src/tests/mod.rs
View file

@ -1,4 +1,3 @@
pub mod fixtures;
mod import;
mod nar_renderer;
pub mod utils;

20
tvix/store/src/tests/nar_renderer.rs
View file

@ -1,12 +1,12 @@
use crate::nar::calculate_size_and_sha256;
use crate::nar::write_nar;
use crate::proto::DirectoryNode;
use crate::proto::FileNode;
use crate::proto::SymlinkNode;
use crate::tests::fixtures::*;
use crate::tests::utils::*;
use sha2::{Digest, Sha256};
use std::io;
use tvix_castore::proto::DirectoryNode;
use tvix_castore::proto::FileNode;
use tvix_castore::proto::{self as castorepb, SymlinkNode};
#[tokio::test]
async fn single_symlink() {
@ -14,7 +14,7 @@ async fn single_symlink() {
let buf = write_nar(
buf,
&crate::proto::node::Node::Symlink(SymlinkNode {
&castorepb::node::Node::Symlink(SymlinkNode {
name: "doesntmatter".into(),
target: "/nix/store/somewhereelse".into(),
}),
@ -35,7 +35,7 @@ async fn single_file_missing_blob() {
let e = write_nar(
buf,
&crate::proto::node::Node::File(FileNode {
&castorepb::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
@ -82,7 +82,7 @@ async fn single_file_wrong_blob_size() {
let e = write_nar(
buf,
&crate::proto::node::Node::File(FileNode {
&castorepb::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: 42, // <- note the wrong size here!
@ -109,7 +109,7 @@ async fn single_file_wrong_blob_size() {
let e = write_nar(
buf,
&crate::proto::node::Node::File(FileNode {
&castorepb::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: 2, // <- note the wrong size here!
@ -152,7 +152,7 @@ async fn single_file() {
let buf = write_nar(
buf,
&crate::proto::node::Node::File(FileNode {
&castorepb::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
@ -199,7 +199,7 @@ async fn test_complicated() {
let buf = write_nar(
buf,
&crate::proto::node::Node::Directory(DirectoryNode {
&castorepb::node::Node::Directory(DirectoryNode {
name: "doesntmatter".into(),
digest: DIRECTORY_COMPLICATED.digest().into(),
size: DIRECTORY_COMPLICATED.size(),
@ -216,7 +216,7 @@ async fn test_complicated() {
let bs = blob_service.clone();
let ds = directory_service.clone();
let (nar_size, nar_digest) = calculate_size_and_sha256(
&crate::proto::node::Node::Directory(DirectoryNode {
&castorepb::node::Node::Directory(DirectoryNode {
name: "doesntmatter".into(),
digest: DIRECTORY_COMPLICATED.digest().into(),
size: DIRECTORY_COMPLICATED.size(),

16
tvix/store/src/tests/utils.rs
View file

@ -1,18 +1,8 @@
use crate::pathinfoservice::{MemoryPathInfoService, PathInfoService};
use std::sync::Arc;
use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService};
use crate::{
blobservice::{BlobService, MemoryBlobService},
directoryservice::{DirectoryService, MemoryDirectoryService},
pathinfoservice::{MemoryPathInfoService, PathInfoService},
};
pub fn gen_blob_service() -> Arc<dyn BlobService> {
Arc::new(MemoryBlobService::default())
}
pub fn gen_directory_service() -> Arc<dyn DirectoryService> {
Arc::new(MemoryDirectoryService::default())
}
pub use tvix_castore::utils::*;
pub fn gen_pathinfo_service(
blob_service: Arc<dyn BlobService>,