mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/store/proto): use Bytes instead of Vec<u8>

Browse source 
Makes use of https://github.com/tokio-rs/prost/pull/341, which makes our
bytes field cheaper to clone.

It's a bit annoying to configure due to
https://github.com/hyperium/tonic/issues/908, but the workaround does
get the job done.

Change-Id: I25714600b041bb5432d3adf5859b151e72b12778
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8975
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink 2023-07-19 18:52:50 +03:00 committed by clbot
parent 7971d7d9ff
commit 432222f098
34 changed files with 216 additions and 164 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
corp/tvixbolt/Cargo.lock generated
View file

@ -52,6 +52,12 @@ version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
[[package]]
name = "bytes"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -790,6 +796,7 @@ dependencies = [
name = "tvix-eval"
version = "0.1.0"
dependencies = [
"bytes",
"codemap",
"codemap-diagnostic",
"dirs",

2
tvix/Cargo.lock generated
View file

@ -2697,6 +2697,7 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
name = "tvix-cli"
version = "0.1.0"
dependencies = [
"bytes",
"clap 4.2.7",
"data-encoding",
"dirs",
@ -2714,6 +2715,7 @@ dependencies = [
name = "tvix-eval"
version = "0.1.0"
dependencies = [
"bytes",
"codemap",
"codemap-diagnostic",
"criterion",

8
tvix/Cargo.nix
View file

@ -7931,6 +7931,10 @@ rec {
then lib.cleanSourceWith { filter = sourceFilter; src = ./cli; }
else ./cli;
dependencies = [
{
name = "bytes";
packageId = "bytes";
}
{
name = "clap";
packageId = "clap 4.2.7";
@ -7991,6 +7995,10 @@ rec {
else ./eval;
libName = "tvix_eval";
dependencies = [
{
name = "bytes";
packageId = "bytes";
}
{
name = "codemap";
packageId = "codemap";

1
tvix/cli/Cargo.toml
View file

@ -18,6 +18,7 @@ smol_str = "0.2.0"
ssri = "7.0.0"
data-encoding = "2.3.3"
thiserror = "1.0.38"
bytes = "1.4.0"
[dependencies.wu-manber]
git = "https://github.com/tvlfyi/wu-manber.git"

2
tvix/cli/src/nix_compat.rs
View file

@ -77,7 +77,7 @@ impl EvalIO for NixCompatIO {
self.underlying.read_to_string(path)
}
fn read_dir(&self, path: &Path) -> Result<Vec<(Vec<u8>, FileType)>, io::Error> {
fn read_dir(&self, path: &Path) -> Result<Vec<(bytes::Bytes, FileType)>, io::Error> {
self.underlying.read_dir(path)
}
}

2
tvix/cli/src/tvix_io.rs
View file

@ -72,7 +72,7 @@ impl<T: EvalIO> EvalIO for TvixIO<T> {
self.actual.read_to_string(path)
}
fn read_dir(&self, path: &Path) -> Result<Vec<(Vec<u8>, FileType)>, io::Error> {
fn read_dir(&self, path: &Path) -> Result<Vec<(bytes::Bytes, FileType)>, io::Error> {
self.actual.read_dir(path)
}
}

1
tvix/eval/Cargo.toml
View file

@ -10,6 +10,7 @@ name = "tvix_eval"
[dependencies]
builtin-macros = { path = "./builtin-macros", package = "tvix-eval-builtin-macros" }
bytes = "1.4.0"
codemap = "0.1.3"
codemap-diagnostic = "0.1.1"
dirs = "4.0.0"

6
tvix/eval/src/builtins/impure.rs
View file

@ -38,8 +38,10 @@ mod impure_builtins {
let dir = generators::request_read_dir(&co, path).await;
let res = dir.into_iter().map(|(name, ftype)| {
(
// TODO: propagate Vec<u8> into NixString.
NixString::from(String::from_utf8(name).expect("parsing file name as string")),
// TODO: propagate Vec<u8> or bytes::Bytes into NixString.
NixString::from(
String::from_utf8(name.to_vec()).expect("parsing file name as string"),
),
Value::String(
match ftype {
FileType::Directory => "directory",

8
tvix/eval/src/io.rs
View file

@ -42,7 +42,7 @@ pub trait EvalIO {
/// Read the directory at the specified path and return the names
/// of its entries associated with their [`FileType`].
fn read_dir(&self, path: &Path) -> Result<Vec<(Vec<u8>, FileType)>, io::Error>;
fn read_dir(&self, path: &Path) -> Result<Vec<(bytes::Bytes, FileType)>, io::Error>;
/// Import the given path. What this means depends on the
/// implementation, for example for a `std::io`-based
@ -76,7 +76,7 @@ impl EvalIO for StdIO {
std::fs::read_to_string(&path)
}
fn read_dir(&self, path: &Path) -> Result<Vec<(Vec<u8>, FileType)>, io::Error> {
fn read_dir(&self, path: &Path) -> Result<Vec<(bytes::Bytes, FileType)>, io::Error> {
let mut result = vec![];
for entry in path.read_dir()? {
@ -93,7 +93,7 @@ impl EvalIO for StdIO {
FileType::Unknown
};
result.push((entry.file_name().into_vec(), val))
result.push((entry.file_name().into_vec().into(), val))
}
Ok(result)
@ -125,7 +125,7 @@ impl EvalIO for DummyIO {
))
}
fn read_dir(&self, _: &Path) -> Result<Vec<(Vec<u8>, FileType)>, io::Error> {
fn read_dir(&self, _: &Path) -> Result<Vec<(bytes::Bytes, FileType)>, io::Error> {
Err(io::Error::new(
io::ErrorKind::Unsupported,
"I/O methods are not implemented in DummyIO",

4
tvix/eval/src/vm/generators.rs
View file

@ -186,7 +186,7 @@ pub enum VMResponse {
Path(PathBuf),
/// VM response with the contents of a directory.
Directory(Vec<(Vec<u8>, FileType)>),
Directory(Vec<(bytes::Bytes, FileType)>),
/// VM response with a span to use at the current point.
Span(LightSpan),
@ -735,7 +735,7 @@ pub(crate) async fn request_path_exists(co: &GenCo, path: PathBuf) -> Value {
}
}
pub(crate) async fn request_read_dir(co: &GenCo, path: PathBuf) -> Vec<(Vec<u8>, FileType)> {
pub(crate) async fn request_read_dir(co: &GenCo, path: PathBuf) -> Vec<(bytes::Bytes, FileType)> {
match co.yield_(VMRequest::ReadDir(path)).await {
VMResponse::Directory(dir) => dir,
msg => panic!(

40
tvix/store/build.rs
View file

@ -12,21 +12,29 @@ fn main() -> Result<()> {
builder = builder.file_descriptor_set_path(descriptor_path);
};
builder.build_server(true).build_client(true).compile(
&[
"tvix/store/protos/castore.proto",
"tvix/store/protos/pathinfo.proto",
"tvix/store/protos/rpc_blobstore.proto",
"tvix/store/protos/rpc_directory.proto",
"tvix/store/protos/rpc_pathinfo.proto",
],
// If we are in running `cargo build` manually, using `../..` works fine,
// but in case we run inside a nix build, we need to instead point PROTO_ROOT
// to a sparseTree containing that structure.
&[match std::env::var_os("PROTO_ROOT") {
Some(proto_root) => proto_root.to_str().unwrap().to_owned(),
None => "../..".to_string(),
}],
)?;
// https://github.com/hyperium/tonic/issues/908
let mut config = prost_build::Config::new();
config.bytes(["."]);
builder
.build_server(true)
.build_client(true)
.compile_with_config(
config,
&[
"tvix/store/protos/castore.proto",
"tvix/store/protos/pathinfo.proto",
"tvix/store/protos/rpc_blobstore.proto",
"tvix/store/protos/rpc_directory.proto",
"tvix/store/protos/rpc_pathinfo.proto",
],
// If we are in running `cargo build` manually, using `../..` works fine,
// but in case we run inside a nix build, we need to instead point PROTO_ROOT
// to a sparseTree containing that structure.
&[match std::env::var_os("PROTO_ROOT") {
Some(proto_root) => proto_root.to_str().unwrap().to_owned(),
None => "../..".to_string(),
}],
)?;
Ok(())
}

4
tvix/store/src/blobservice/grpc.rs
View file

@ -98,7 +98,7 @@ impl BlobService for GRPCBlobService {
self.tokio_handle.spawn(async move {
Ok(grpc_client
.stat(proto::StatBlobRequest {
digest: digest.to_vec(),
digest: digest.into(),
..Default::default()
})
.await?
@ -126,7 +126,7 @@ impl BlobService for GRPCBlobService {
self.tokio_handle.spawn(async move {
let stream = grpc_client
.read(proto::ReadBlobRequest {
digest: digest.to_vec(),
digest: digest.into(),
})
.await?
.into_inner();

11
tvix/store/src/directoryservice/grpc.rs
View file

@ -91,13 +91,12 @@ impl DirectoryService for GRPCDirectoryService {
fn get(&self, digest: &B3Digest) -> Result<Option<crate::proto::Directory>, crate::Error> {
// Get a new handle to the gRPC client, and copy the digest.
let mut grpc_client = self.grpc_client.clone();
let digest_as_vec = digest.to_vec();
let digest_cpy = digest.clone();
let task = self.tokio_handle.spawn(async move {
let mut s = grpc_client
.get(proto::GetDirectoryRequest {
recursive: false,
by_what: Some(ByWhat::Digest(digest_as_vec)),
by_what: Some(ByWhat::Digest(digest_cpy.into())),
})
.await?
.into_inner();
@ -160,13 +159,15 @@ impl DirectoryService for GRPCDirectoryService {
) -> Box<dyn Iterator<Item = Result<proto::Directory, Error>> + Send> {
let mut grpc_client = self.grpc_client.clone();
let root_directory_digest_as_vec = root_directory_digest.to_vec();
// clone so we can move it
let root_directory_digest_cpy = root_directory_digest.clone();
let task: tokio::task::JoinHandle<Result<Streaming<proto::Directory>, Status>> =
self.tokio_handle.spawn(async move {
let s = grpc_client
.get(proto::GetDirectoryRequest {
recursive: true,
by_what: Some(ByWhat::Digest(root_directory_digest_as_vec)),
by_what: Some(ByWhat::Digest(root_directory_digest_cpy.into())),
})
.await?
.into_inner();

2
tvix/store/src/directoryservice/traverse.rs
View file

@ -108,7 +108,7 @@ mod tests {
let node_directory_complicated =
crate::proto::node::Node::Directory(crate::proto::DirectoryNode {
name: "doesntmatter".into(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(),
digest: DIRECTORY_COMPLICATED.digest().into(),
size: DIRECTORY_COMPLICATED.size(),
});

4
tvix/store/src/fuse/inode_tracker.rs
View file

@ -13,7 +13,7 @@ pub struct InodeTracker {
blob_digest_to_inode: HashMap<B3Digest, u64>,
// lookup table for symlinks by their target
symlink_target_to_inode: HashMap<Vec<u8>, u64>,
symlink_target_to_inode: HashMap<bytes::Bytes, u64>,
// lookup table for directories by their B3Digest.
// Note the corresponding directory may not be present in data yet.
@ -171,7 +171,7 @@ impl InodeTracker {
self.blob_digest_to_inode.insert(digest.clone(), ino);
}
InodeData::Symlink(ref target) => {
self.symlink_target_to_inode.insert(target.to_vec(), ino);
self.symlink_target_to_inode.insert(target.clone(), ino);
}
InodeData::Directory(DirectoryInodeData::Sparse(ref digest, _size)) => {
self.directory_digest_to_inode.insert(digest.clone(), ino);

2
tvix/store/src/fuse/inodes.rs
View file

@ -5,7 +5,7 @@ use crate::{proto, B3Digest};
#[derive(Clone, Debug)]
pub enum InodeData {
Regular(B3Digest, u32, bool), // digest, size, executable
Symlink(Vec<u8>), // target
Symlink(bytes::Bytes), // target
Directory(DirectoryInodeData), // either [DirectoryInodeData:Sparse] or [DirectoryInodeData:Populated]
}

12
tvix/store/src/fuse/tests.rs
View file

@ -58,7 +58,7 @@ fn populate_blob_a(
node: Some(proto::Node {
node: Some(proto::node::Node::File(FileNode {
name: BLOB_A_NAME.into(),
digest: fixtures::BLOB_A_DIGEST.to_vec(),
digest: fixtures::BLOB_A_DIGEST.clone().into(),
size: fixtures::BLOB_A.len() as u32,
executable: false,
})),
@ -84,7 +84,7 @@ fn populate_blob_b(
node: Some(proto::Node {
node: Some(proto::node::Node::File(FileNode {
name: BLOB_B_NAME.into(),
digest: fixtures::BLOB_B_DIGEST.to_vec(),
digest: fixtures::BLOB_B_DIGEST.clone().into(),
size: fixtures::BLOB_B.len() as u32,
executable: false,
})),
@ -154,7 +154,7 @@ fn populate_directory_with_keep(
node: Some(proto::Node {
node: Some(proto::node::Node::Directory(DirectoryNode {
name: DIRECTORY_WITH_KEEP_NAME.into(),
digest: fixtures::DIRECTORY_WITH_KEEP.digest().to_vec(),
digest: fixtures::DIRECTORY_WITH_KEEP.digest().into(),
size: fixtures::DIRECTORY_WITH_KEEP.size(),
})),
}),
@ -175,7 +175,7 @@ fn populate_pathinfo_without_directory(
node: Some(proto::Node {
node: Some(proto::node::Node::Directory(DirectoryNode {
name: DIRECTORY_WITH_KEEP_NAME.into(),
digest: fixtures::DIRECTORY_WITH_KEEP.digest().to_vec(),
digest: fixtures::DIRECTORY_WITH_KEEP.digest().into(),
size: fixtures::DIRECTORY_WITH_KEEP.size(),
})),
}),
@ -195,7 +195,7 @@ fn populate_blob_a_without_blob(
node: Some(proto::Node {
node: Some(proto::node::Node::File(FileNode {
name: BLOB_A_NAME.into(),
digest: fixtures::BLOB_A_DIGEST.to_vec(),
digest: fixtures::BLOB_A_DIGEST.clone().into(),
size: fixtures::BLOB_A.len() as u32,
executable: false,
})),
@ -232,7 +232,7 @@ fn populate_directory_complicated(
node: Some(proto::Node {
node: Some(proto::node::Node::Directory(DirectoryNode {
name: DIRECTORY_COMPLICATED_NAME.into(),
digest: fixtures::DIRECTORY_COMPLICATED.digest().to_vec(),
digest: fixtures::DIRECTORY_COMPLICATED.digest().into(),
size: fixtures::DIRECTORY_COMPLICATED.size(),
})),
}),

35
tvix/store/src/import.rs
View file

@ -66,8 +66,6 @@ fn process_entry(
) -> Result<proto::node::Node, Error> {
let file_type = entry.file_type();
let entry_path: PathBuf = entry.path().to_path_buf();
if file_type.is_dir() {
let directory = maybe_directory
.expect("tvix bug: must be called with some directory in the case of directory");
@ -80,41 +78,45 @@ fn process_entry(
.map_err(|e| Error::UploadDirectoryError(entry.path().to_path_buf(), e))?;
return Ok(proto::node::Node::Directory(proto::DirectoryNode {
name: entry.file_name().as_bytes().to_vec(),
digest: directory_digest.to_vec(),
name: entry.file_name().as_bytes().to_owned().into(),
digest: directory_digest.into(),
size: directory_size,
}));
}
if file_type.is_symlink() {
let target = std::fs::read_link(&entry_path)
.map_err(|e| Error::UnableToStat(entry_path.clone(), e))?;
let target: bytes::Bytes = std::fs::read_link(entry.path())
.map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e))?
.as_os_str()
.as_bytes()
.to_owned()
.into();
return Ok(proto::node::Node::Symlink(proto::SymlinkNode {
name: entry.file_name().as_bytes().to_vec(),
target: target.as_os_str().as_bytes().to_vec(),
name: entry.file_name().as_bytes().to_owned().into(),
target,
}));
}
if file_type.is_file() {
let metadata = entry
.metadata()
.map_err(|e| Error::UnableToStat(entry_path.clone(), e.into()))?;
.map_err(|e| Error::UnableToStat(entry.path().to_path_buf(), e.into()))?;
let mut file = File::open(entry_path.clone())
.map_err(|e| Error::UnableToOpen(entry_path.clone(), e))?;
let mut file = File::open(entry.path())
.map_err(|e| Error::UnableToOpen(entry.path().to_path_buf(), e))?;
let mut writer = blob_service.open_write();
if let Err(e) = io::copy(&mut file, &mut writer) {
return Err(Error::UnableToRead(entry_path, e));
return Err(Error::UnableToRead(entry.path().to_path_buf(), e));
};
let digest = writer.close()?;
return Ok(proto::node::Node::File(proto::FileNode {
name: entry.file_name().as_bytes().to_vec(),
digest: digest.to_vec(),
name: entry.file_name().as_bytes().to_vec().into(),
digest: digest.into(),
size: metadata.len() as u32,
// If it's executable by the user, it'll become executable.
// This matches nix's dump() function behaviour.
@ -150,8 +152,9 @@ pub fn ingest_path<P: AsRef<Path> + Debug>(
.file_name()
.unwrap_or_default()
.as_bytes()
.to_vec(),
target: target.as_os_str().as_bytes().to_vec(),
.to_owned()
.into(),
target: target.as_os_str().as_bytes().to_vec().into(),
}));
}

6
tvix/store/src/nar/mod.rs
View file

@ -13,13 +13,13 @@ pub enum RenderError {
StoreError(crate::Error),
#[error("unable to find directory {}, referred from {:?}", .0, .1)]
DirectoryNotFound(B3Digest, Vec<u8>),
DirectoryNotFound(B3Digest, bytes::Bytes),
#[error("unable to find blob {}, referred from {:?}", BASE64.encode(.0), .1)]
BlobNotFound([u8; 32], Vec<u8>),
BlobNotFound([u8; 32], bytes::Bytes),
#[error("unexpected size in metadata for blob {}, referred from {:?} returned, expected {}, got {}", BASE64.encode(.0), .1, .2, .3)]
UnexpectedBlobMeta([u8; 32], Vec<u8>, u32, u32),
UnexpectedBlobMeta([u8; 32], bytes::Bytes, u32, u32),
#[error("failure using the NAR writer: {0}")]
NARWriterError(std::io::Error),

2
tvix/store/src/nar/renderer.rs
View file

@ -115,7 +115,7 @@ fn walk_node(
None => {
return Err(RenderError::DirectoryNotFound(
digest,
proto_directory_node.name.to_owned(),
proto_directory_node.name.clone(),
))
}
Some(proto_directory) => {

3
tvix/store/src/pathinfoservice/grpc.rs
View file

@ -97,7 +97,7 @@ impl PathInfoService for GRPCPathInfoService {
let path_info = grpc_client
.get(proto::GetPathInfoRequest {
by_what: Some(proto::get_path_info_request::ByWhat::ByOutputHash(
digest.to_vec(),
digest.to_vec().into(),
)),
})
.await?
@ -154,6 +154,7 @@ impl PathInfoService for GRPCPathInfoService {
let nar_sha256: [u8; 32] = resp
.nar_sha256
.to_vec()
.try_into()
.map_err(|_e| crate::Error::StorageError("invalid digest length".to_string()))?;

10
tvix/store/src/proto/grpc_blobservice_wrapper.rs
View file

@ -133,9 +133,7 @@ impl super::blob_service_server::BlobService for GRPCBlobServiceWrapper {
x: Result<bytes::Bytes, io::Error>,
) -> Result<super::BlobChunk, Status> {
match x {
Ok(bytes) => Ok(super::BlobChunk {
data: bytes.to_vec(),
}),
Ok(bytes) => Ok(super::BlobChunk { data: bytes }),
Err(e) => Err(Status::from(e)),
}
}
@ -156,7 +154,7 @@ impl super::blob_service_server::BlobService for GRPCBlobServiceWrapper {
let req_inner = request.into_inner();
let data_stream = req_inner.map(|x| {
x.map(|x| VecDeque::from(x.data))
x.map(|x| VecDeque::from(x.data.to_vec()))
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))
});
@ -182,7 +180,9 @@ impl super::blob_service_server::BlobService for GRPCBlobServiceWrapper {
})?
.to_vec();
Ok(super::PutBlobResponse { digest })
Ok(super::PutBlobResponse {
digest: digest.into(),
})
})
.await
.map_err(|_| Status::internal("failed to wait for task"))??;

2
tvix/store/src/proto/grpc_directoryservice_wrapper.rs
View file

@ -176,7 +176,7 @@ impl proto::directory_service_server::DirectoryService for GRPCDirectoryServiceW
match last_directory_dgst {
None => Err(Status::invalid_argument("no directories received")),
Some(last_directory_dgst) => Ok(Response::new(proto::PutDirectoryResponse {
root_digest: last_directory_dgst.to_vec(),
root_digest: last_directory_dgst.into(),
})),
}
}

9
tvix/store/src/proto/grpc_pathinfoservice_wrapper.rs
View file

@ -26,10 +26,11 @@ impl proto::path_info_service_server::PathInfoService for GRPCPathInfoServiceWra
) -> Result<Response<proto::PathInfo>> {
match request.into_inner().by_what {
None => Err(Status::unimplemented("by_what needs to be specified")),
Some(proto::get_path_info_request::ByWhat::ByOutputHash(digest)) => {
let digest: [u8; 20] = digest
Some(proto::get_path_info_request::ByWhat::ByOutputHash(output_digest)) => {
let digest: [u8; 20] = output_digest
.to_vec()
.try_into()
.map_err(|_e| Status::invalid_argument("invalid digest length"))?;
.map_err(|_e| Status::invalid_argument("invalid output digest length"))?;
match self.path_info_service.get(digest) {
Ok(None) => Err(Status::not_found("PathInfo not found")),
Ok(Some(path_info)) => Ok(Response::new(path_info)),
@ -72,7 +73,7 @@ impl proto::path_info_service_server::PathInfoService for GRPCPathInfoServiceWra
Ok(Response::new(proto::CalculateNarResponse {
nar_size,
nar_sha256: nar_sha256.to_vec(),
nar_sha256: nar_sha256.to_vec().into(),
}))
}
}

2
tvix/store/src/proto/mod.rs
View file

@ -86,7 +86,7 @@ fn validate_node_name<E>(name: &[u8], err: fn(Vec<u8>) -> E) -> Result<(), E> {
/// Checks a digest for validity.
/// Digests are 32 bytes long, as we store blake3 digests.
fn validate_digest<E>(digest: &Vec<u8>, err: fn(usize) -> E) -> Result<(), E> {
fn validate_digest<E>(digest: &bytes::Bytes, err: fn(usize) -> E) -> Result<(), E> {
if digest.len() != 32 {
return Err(err(digest.len()));
}

30
tvix/store/src/proto/tests/directory.rs
View file

@ -18,7 +18,7 @@ fn size() {
let d = Directory {
directories: vec![DirectoryNode {
name: "foo".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 0,
}],
..Default::default()
@ -29,7 +29,7 @@ fn size() {
let d = Directory {
directories: vec![DirectoryNode {
name: "foo".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 4,
}],
..Default::default()
@ -40,7 +40,7 @@ fn size() {
let d = Directory {
files: vec![FileNode {
name: "foo".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
executable: false,
}],
@ -88,7 +88,7 @@ fn validate_invalid_names() {
let d = Directory {
directories: vec![DirectoryNode {
name: "".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
}],
..Default::default()
@ -105,7 +105,7 @@ fn validate_invalid_names() {
let d = Directory {
directories: vec![DirectoryNode {
name: ".".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
}],
..Default::default()
@ -122,7 +122,7 @@ fn validate_invalid_names() {
let d = Directory {
files: vec![FileNode {
name: "..".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
executable: false,
}],
@ -174,7 +174,7 @@ fn validate_invalid_digest() {
let d = Directory {
directories: vec![DirectoryNode {
name: "foo".into(),
digest: vec![0x00, 0x42], // invalid length
digest: vec![0x00, 0x42].into(), // invalid length
size: 42,
}],
..Default::default()
@ -195,12 +195,12 @@ fn validate_sorting() {
directories: vec![
DirectoryNode {
name: "b".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
},
DirectoryNode {
name: "a".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
},
],
@ -220,12 +220,12 @@ fn validate_sorting() {
directories: vec![
DirectoryNode {
name: "a".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
},
DirectoryNode {
name: "a".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
},
],
@ -245,12 +245,12 @@ fn validate_sorting() {
directories: vec![
DirectoryNode {
name: "a".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
},
DirectoryNode {
name: "b".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
},
],
@ -266,12 +266,12 @@ fn validate_sorting() {
directories: vec![
DirectoryNode {
name: "b".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
},
DirectoryNode {
name: "c".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.to_vec().into(),
size: 42,
},
],

12
tvix/store/src/proto/tests/grpc_blobservice.rs
View file

@ -16,7 +16,7 @@ async fn not_found_read() {
let resp = service
.read(tonic::Request::new(ReadBlobRequest {
digest: BLOB_A_DIGEST.to_vec(),
digest: BLOB_A_DIGEST.clone().into(),
}))
.await;
@ -36,7 +36,7 @@ async fn not_found_stat() {
let resp = service
.stat(tonic::Request::new(StatBlobRequest {
digest: BLOB_A_DIGEST.to_vec(),
digest: BLOB_A_DIGEST.clone().into(),
..Default::default()
}))
.await
@ -54,7 +54,7 @@ async fn put_read_stat() {
// Send blob A.
let put_resp = service
.put(tonic_mock::streaming_request(vec![BlobChunk {
data: BLOB_A.clone(),
data: BLOB_A.clone().into(),
}]))
.await
.expect("must succeed")
@ -67,7 +67,7 @@ async fn put_read_stat() {
// expose it yet.
let _resp = service
.stat(tonic::Request::new(StatBlobRequest {
digest: BLOB_A_DIGEST.to_vec(),
digest: BLOB_A_DIGEST.clone().into(),
..Default::default()
}))
.await
@ -77,7 +77,7 @@ async fn put_read_stat() {
// Read the blob. It should return the same data.
let resp = service
.read(tonic::Request::new(ReadBlobRequest {
digest: BLOB_A_DIGEST.to_vec(),
digest: BLOB_A_DIGEST.clone().into(),
}))
.await;
@ -90,7 +90,7 @@ async fn put_read_stat() {
.expect("must be some")
.expect("must succeed");
assert_eq!(BLOB_A.to_vec(), item.data);
assert_eq!(BLOB_A.clone(), item.data);
// … and no more elements
assert!(rx.next().await.is_none());

12
tvix/store/src/proto/tests/grpc_directoryservice.rs
View file

@ -42,7 +42,7 @@ async fn not_found() {
let resp = service
.get(tonic::Request::new(GetDirectoryRequest {
by_what: Some(ByWhat::Digest(DIRECTORY_A.digest().to_vec())),
by_what: Some(ByWhat::Digest(DIRECTORY_A.digest().into())),
..Default::default()
}))
.await;
@ -80,7 +80,7 @@ async fn put_get() {
let items = get_directories(
&service,
GetDirectoryRequest {
by_what: Some(ByWhat::Digest(DIRECTORY_A.digest().to_vec())),
by_what: Some(ByWhat::Digest(DIRECTORY_A.digest().into())),
..Default::default()
},
)
@ -122,7 +122,7 @@ async fn put_get_multiple() {
&service,
GetDirectoryRequest {
recursive: false,
by_what: Some(ByWhat::Digest(DIRECTORY_B.digest().to_vec())),
by_what: Some(ByWhat::Digest(DIRECTORY_B.digest().into())),
},
)
.await
@ -136,7 +136,7 @@ async fn put_get_multiple() {
&service,
GetDirectoryRequest {
recursive: true,
by_what: Some(ByWhat::Digest(DIRECTORY_B.digest().to_vec())),
by_what: Some(ByWhat::Digest(DIRECTORY_B.digest().into())),
},
)
.await
@ -172,7 +172,7 @@ async fn put_get_dedup() {
&service,
GetDirectoryRequest {
recursive: true,
by_what: Some(ByWhat::Digest(DIRECTORY_C.digest().to_vec())),
by_what: Some(ByWhat::Digest(DIRECTORY_C.digest().into())),
},
)
.await
@ -215,7 +215,7 @@ async fn put_reject_wrong_size() {
let broken_parent_directory = Directory {
directories: vec![DirectoryNode {
name: "foo".into(),
digest: DIRECTORY_A.digest().to_vec(),
digest: DIRECTORY_A.digest().into(),
size: 42,
}],
..Default::default()

4
tvix/store/src/proto/tests/grpc_pathinfoservice.rs
View file

@ -32,7 +32,7 @@ async fn not_found() {
let resp = service
.get(Request::new(GetPathInfoRequest {
by_what: Some(ByOutputHash(DUMMY_OUTPUT_HASH.to_vec())),
by_what: Some(ByOutputHash(DUMMY_OUTPUT_HASH.clone())),
}))
.await;
@ -62,7 +62,7 @@ async fn put_get() {
let resp = service
.get(Request::new(GetPathInfoRequest {
by_what: Some(ByOutputHash(DUMMY_OUTPUT_HASH.to_vec())),
by_what: Some(ByOutputHash(DUMMY_OUTPUT_HASH.clone())),
}))
.await;

48
tvix/store/src/proto/tests/pathinfo.rs
View file

@ -1,20 +1,28 @@
use crate::proto::{self, Node, PathInfo, ValidatePathInfoError};
use crate::B3Digest;
use bytes::Bytes;
use lazy_static::lazy_static;
use nix_compat::store_path::{self, StorePath};
use std::str::FromStr;
use test_case::test_case;
lazy_static! {
static ref DUMMY_DIGEST: Vec<u8> = vec![
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
];
static ref DUMMY_DIGEST_2: Vec<u8> = vec![
0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
];
static ref DUMMY_DIGEST: B3Digest = {
let u: &[u8; 32] = &[
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
];
u.into()
};
static ref DUMMY_DIGEST_2: B3Digest = {
let u: &[u8; 32] = &[
0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
];
u.into()
};
}
const DUMMY_NAME: &str = "00000000000000000000000000000000-dummy";
@ -44,7 +52,7 @@ fn validate_no_node(
#[test_case(
proto::DirectoryNode {
name: DUMMY_NAME.into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.clone().into(),
size: 0,
},
Ok(StorePath::from_str(DUMMY_NAME).expect("must succeed"));
@ -53,7 +61,7 @@ fn validate_no_node(
#[test_case(
proto::DirectoryNode {
name: DUMMY_NAME.into(),
digest: vec![],
digest: Bytes::new(),
size: 0,
},
Err(ValidatePathInfoError::InvalidDigestLen(0));
@ -62,7 +70,7 @@ fn validate_no_node(
#[test_case(
proto::DirectoryNode {
name: "invalid".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.clone().into(),
size: 0,
},
Err(ValidatePathInfoError::InvalidNodeName(
@ -88,7 +96,7 @@ fn validate_directory(
#[test_case(
proto::FileNode {
name: DUMMY_NAME.into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.clone().into(),
size: 0,
executable: false,
},
@ -98,7 +106,7 @@ fn validate_directory(
#[test_case(
proto::FileNode {
name: DUMMY_NAME.into(),
digest: vec![],
digest: Bytes::new(),
..Default::default()
},
Err(ValidatePathInfoError::InvalidDigestLen(0));
@ -107,7 +115,7 @@ fn validate_directory(
#[test_case(
proto::FileNode {
name: "invalid".into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.clone().into(),
..Default::default()
},
Err(ValidatePathInfoError::InvalidNodeName(
@ -167,11 +175,11 @@ fn validate_references() {
node: Some(Node {
node: Some(proto::node::Node::Directory(proto::DirectoryNode {
name: DUMMY_NAME.into(),
digest: DUMMY_DIGEST.to_vec(),
digest: DUMMY_DIGEST.clone().into(),
size: 0,
})),
}),
references: vec![DUMMY_DIGEST_2.to_vec()],
references: vec![DUMMY_DIGEST_2.clone().into()],
narinfo: None,
};
assert!(path_info.validate().is_ok());
@ -180,7 +188,7 @@ fn validate_references() {
let path_info_with_narinfo_missing_refs = PathInfo {
narinfo: Some(proto::NarInfo {
nar_size: 0,
nar_sha256: DUMMY_DIGEST.to_vec(),
nar_sha256: DUMMY_DIGEST.clone().into(),
signatures: vec![],
reference_names: vec![],
}),
@ -198,7 +206,7 @@ fn validate_references() {
let path_info_with_narinfo = PathInfo {
narinfo: Some(proto::NarInfo {
nar_size: 0,
nar_sha256: DUMMY_DIGEST.to_vec(),
nar_sha256: DUMMY_DIGEST.clone().into(),
signatures: vec![],
reference_names: vec![format!("/nix/store/{}", DUMMY_NAME)],
}),

8
tvix/store/src/store_io.rs
View file

@ -129,7 +129,7 @@ impl TvixStoreIO {
// assemble a new root_node with a name that is derived from the nar hash.
let renamed_root_node = {
let name = output_path.to_string().into_bytes();
let name = output_path.to_string().into_bytes().into();
match root_node {
crate::proto::node::Node::Directory(n) => {
@ -153,7 +153,7 @@ impl TvixStoreIO {
references: vec![],
narinfo: Some(crate::proto::NarInfo {
nar_size,
nar_sha256: nar_sha256.to_vec(),
nar_sha256: nar_sha256.to_vec().into(),
signatures: vec![],
reference_names: vec![],
// TODO: narinfo for talosctl.src contains `CA: fixed:r:sha256:1x13j5hy75221bf6kz7cpgld9vgic6bqx07w5xjs4pxnksj6lxb6`
@ -264,7 +264,7 @@ impl EvalIO for TvixStoreIO {
}
#[instrument(skip(self), ret, err)]
fn read_dir(&self, path: &Path) -> Result<Vec<(Vec<u8>, FileType)>, io::Error> {
fn read_dir(&self, path: &Path) -> Result<Vec<(bytes::Bytes, FileType)>, io::Error> {
if let Ok((store_path, sub_path)) =
StorePath::from_absolute_path_full(&path.to_string_lossy())
{
@ -283,7 +283,7 @@ impl EvalIO for TvixStoreIO {
})?;
if let Some(directory) = self.directory_service.get(&digest)? {
let mut children: Vec<(Vec<u8>, FileType)> = Vec::new();
let mut children: Vec<(bytes::Bytes, FileType)> = Vec::new();
for node in directory.nodes() {
children.push(match node {
crate::proto::node::Node::Directory(e) => {

53
tvix/store/src/tests/fixtures.rs
View file

@ -8,13 +8,16 @@ pub const HELLOWORLD_BLOB_CONTENTS: &[u8] = b"Hello World!";
pub const EMPTY_BLOB_CONTENTS: &[u8] = b"";
lazy_static! {
pub static ref DUMMY_DIGEST: Vec<u8> = vec![
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
];
pub static ref DUMMY_DATA_1: Vec<u8> = vec![0x01, 0x02, 0x03];
pub static ref DUMMY_DATA_2: Vec<u8> = vec![0x04, 0x05];
pub static ref DUMMY_DIGEST: B3Digest = {
let u: &[u8; 32] = &[
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00,
];
u.into()
};
pub static ref DUMMY_DATA_1: bytes::Bytes = vec![0x01, 0x02, 0x03].into();
pub static ref DUMMY_DATA_2: bytes::Bytes = vec![0x04, 0x05].into();
pub static ref HELLOWORLD_BLOB_DIGEST: B3Digest =
blake3::hash(HELLOWORLD_BLOB_CONTENTS).as_bytes().into();
@ -22,19 +25,19 @@ lazy_static! {
blake3::hash(EMPTY_BLOB_CONTENTS).as_bytes().into();
// 2 bytes
pub static ref BLOB_A: Vec<u8> = vec![0x00, 0x01];
pub static ref BLOB_A: bytes::Bytes = vec![0x00, 0x01].into();
pub static ref BLOB_A_DIGEST: B3Digest = blake3::hash(&BLOB_A).as_bytes().into();
// 1MB
pub static ref BLOB_B: Vec<u8> = (0..255).collect::<Vec<u8>>().repeat(4 * 1024);
pub static ref BLOB_B: bytes::Bytes = (0..255).collect::<Vec<u8>>().repeat(4 * 1024).into();
pub static ref BLOB_B_DIGEST: B3Digest = blake3::hash(&BLOB_B).as_bytes().into();
// Directories
pub static ref DIRECTORY_WITH_KEEP: proto::Directory = proto::Directory {
directories: vec![],
files: vec![FileNode {
name: b".keep".to_vec(),
digest: EMPTY_BLOB_DIGEST.to_vec(),
name: b".keep".to_vec().into(),
digest: EMPTY_BLOB_DIGEST.clone().into(),
size: 0,
executable: false,
}],
@ -42,26 +45,26 @@ lazy_static! {
};
pub static ref DIRECTORY_COMPLICATED: proto::Directory = proto::Directory {
directories: vec![DirectoryNode {
name: b"keep".to_vec(),
digest: DIRECTORY_WITH_KEEP.digest().to_vec(),
name: b"keep".to_vec().into(),
digest: DIRECTORY_WITH_KEEP.digest().into(),
size: DIRECTORY_WITH_KEEP.size(),
}],
files: vec![FileNode {
name: b".keep".to_vec(),
digest: EMPTY_BLOB_DIGEST.to_vec(),
name: b".keep".to_vec().into(),
digest: EMPTY_BLOB_DIGEST.clone().into(),
size: 0,
executable: false,
}],
symlinks: vec![SymlinkNode {
name: b"aa".to_vec(),
target: b"/nix/store/somewhereelse".to_vec(),
name: b"aa".to_vec().into(),
target: b"/nix/store/somewhereelse".to_vec().into(),
}],
};
pub static ref DIRECTORY_A: Directory = Directory::default();
pub static ref DIRECTORY_B: Directory = Directory {
directories: vec![DirectoryNode {
name: b"a".to_vec(),
digest: DIRECTORY_A.digest().to_vec(),
name: b"a".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
}],
..Default::default()
@ -69,13 +72,13 @@ lazy_static! {
pub static ref DIRECTORY_C: Directory = Directory {
directories: vec![
DirectoryNode {
name: b"a".to_vec(),
digest: DIRECTORY_A.digest().to_vec(),
name: b"a".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
},
DirectoryNode {
name: b"a'".to_vec(),
digest: DIRECTORY_A.digest().to_vec(),
name: b"a'".to_vec().into(),
digest: DIRECTORY_A.digest().into(),
size: DIRECTORY_A.size(),
}
],
@ -83,10 +86,10 @@ lazy_static! {
};
// output hash
pub static ref DUMMY_OUTPUT_HASH: Vec<u8> = vec![
pub static ref DUMMY_OUTPUT_HASH: bytes::Bytes = vec![
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00
];
].into();
/// The NAR representation of a symlink pointing to `/nix/store/somewhereelse`
pub static ref NAR_CONTENTS_SYMLINK: Vec<u8> = vec![

12
tvix/store/src/tests/import.rs
View file

@ -54,7 +54,7 @@ fn single_file() {
assert_eq!(
crate::proto::node::Node::File(proto::FileNode {
name: "root".into(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
executable: false,
}),
@ -92,8 +92,14 @@ fn complicated() {
// ensure root_node matched expectations
assert_eq!(
crate::proto::node::Node::Directory(proto::DirectoryNode {
name: tmpdir.path().file_name().unwrap().as_bytes().to_vec(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(),
name: tmpdir
.path()
.file_name()
.unwrap()
.as_bytes()
.to_owned()
.into(),
digest: DIRECTORY_COMPLICATED.digest().into(),
size: DIRECTORY_COMPLICATED.size(),
}),
root_node,

16
tvix/store/src/tests/nar_renderer.rs
View file

@ -36,7 +36,7 @@ fn single_file_missing_blob() {
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
executable: false,
}),
@ -77,7 +77,7 @@ fn single_file_wrong_blob_size() {
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: 42, // <- note the wrong size here!
executable: false,
}),
@ -102,7 +102,7 @@ fn single_file_wrong_blob_size() {
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: 2, // <- note the wrong size here!
executable: false,
}),
@ -127,7 +127,7 @@ fn single_file() {
// insert blob into the store
let mut writer = blob_service.open_write();
io::copy(
&mut io::Cursor::new(HELLOWORLD_BLOB_CONTENTS.to_vec()),
&mut io::Cursor::new(HELLOWORLD_BLOB_CONTENTS.clone()),
&mut writer,
)
.unwrap();
@ -139,7 +139,7 @@ fn single_file() {
&mut buf,
&crate::proto::node::Node::File(FileNode {
name: "doesntmatter".into(),
digest: HELLOWORLD_BLOB_DIGEST.to_vec(),
digest: HELLOWORLD_BLOB_DIGEST.clone().into(),
size: HELLOWORLD_BLOB_CONTENTS.len() as u32,
executable: false,
}),
@ -160,7 +160,7 @@ fn test_complicated() {
// insert blob into the store
let mut writer = blob_service.open_write();
io::copy(
&mut io::Cursor::new(EMPTY_BLOB_CONTENTS.to_vec()),
&mut io::Cursor::new(EMPTY_BLOB_CONTENTS.clone()),
&mut writer,
)
.unwrap();
@ -177,7 +177,7 @@ fn test_complicated() {
&mut buf,
&crate::proto::node::Node::Directory(DirectoryNode {
name: "doesntmatter".into(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(),
digest: DIRECTORY_COMPLICATED.digest().clone().into(),
size: DIRECTORY_COMPLICATED.size(),
}),
blob_service.clone(),
@ -191,7 +191,7 @@ fn test_complicated() {
let (nar_size, nar_digest) = calculate_size_and_sha256(
&crate::proto::node::Node::Directory(DirectoryNode {
name: "doesntmatter".into(),
digest: DIRECTORY_COMPLICATED.digest().to_vec(),
digest: DIRECTORY_COMPLICATED.digest().clone().into(),
size: DIRECTORY_COMPLICATED.size(),
}),
blob_service,