mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/castore): drop {Directory,File,Symlink}Node

Browse source 
Add a `SymlinkTarget` type to represent validated symlink targets.
With this, no invalid states are representable, so we can make `Node` be
just an enum of all three kind of types, and allow access to these
fields directly.

Change-Id: I20bdd480c8d5e64a827649f303c97023b7e390f2
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12216
Reviewed-by: benjaminedwardwebb <benjaminedwardwebb@gmail.com>
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2024-08-16 02:24:12 +03:00 • committed by clbot
parent 49b173786c
commit 8ea7d2b60e
27 changed files with 555 additions and 461 deletions
Download patch file Download diff file Expand all files Collapse all files

8
tvix/glue/src/builtins/derivation.rs
View file

@ -179,7 +179,7 @@ pub(crate) mod derivation_builtins {
use nix_compat::nixhash::CAHash;
use nix_compat::store_path::{build_ca_path, hash_placeholder};
use sha2::Sha256;
use tvix_castore::{FileNode, Node};
use tvix_castore::Node;
use tvix_eval::generators::Gen;
use tvix_eval::{NixContext, NixContextElement, NixString};
use tvix_store::proto::{NarInfo, PathInfo};
@ -577,7 +577,11 @@ pub(crate) mod derivation_builtins {
})
.map_err(DerivationError::InvalidDerivation)?;
let root_node = Node::File(FileNode::new(blob_digest, blob_size, false));
let root_node = Node::File {
digest: blob_digest,
size: blob_size,
executable: false,
};
// calculate the nar hash
let (nar_size, nar_sha256) = state

8
tvix/glue/src/builtins/import.rs
View file

@ -3,7 +3,7 @@
use crate::builtins::errors::ImportError;
use std::path::Path;
use tvix_castore::import::ingest_entries;
use tvix_castore::{FileNode, Node};
use tvix_castore::Node;
use tvix_eval::{
builtin_macros::builtins,
generators::{self, GenCo},
@ -213,7 +213,11 @@ mod import_builtins {
.tokio_handle
.block_on(async { blob_writer.close().await })?;
let root_node = Node::File(FileNode::new(blob_digest, blob_size, false));
let root_node = Node::File {
digest: blob_digest,
size: blob_size,
executable: false,
};
let ca_hash = if recursive_ingestion {
let (_nar_size, nar_sha256) = state

14
tvix/glue/src/fetchers/mod.rs
View file

@ -10,7 +10,7 @@ use tokio::io::{AsyncBufRead, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader};
use tokio_util::io::{InspectReader, InspectWriter};
use tracing::{instrument, warn, Span};
use tracing_indicatif::span_ext::IndicatifSpanExt;
use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService, FileNode, Node};
use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService, Node};
use tvix_store::{nar::NarCalculationService, pathinfoservice::PathInfoService, proto::PathInfo};
use url::Url;
@ -327,7 +327,11 @@ where
// Construct and return the FileNode describing the downloaded contents.
Ok((
Node::File(FileNode::new(blob_writer.close().await?, blob_size, false)),
Node::File {
digest: blob_writer.close().await?,
size: blob_size,
executable: false,
},
CAHash::Flat(actual_hash),
blob_size,
))
@ -522,7 +526,11 @@ where
// Construct and return the FileNode describing the downloaded contents,
// make it executable.
let root_node = Node::File(FileNode::new(blob_digest, file_size, true));
let root_node = Node::File {
digest: blob_digest,
size: file_size,
executable: true,
};
Ok((root_node, CAHash::Nar(actual_hash), file_size))
}

8
tvix/glue/src/tvix_build.rs
View file

@ -200,7 +200,7 @@ mod test {
BuildRequest,
};
use tvix_castore::fixtures::DUMMY_DIGEST;
use tvix_castore::{DirectoryNode, Node};
use tvix_castore::Node;
use crate::tvix_build::NIX_ENVIRONMENT_VARS;
@ -209,8 +209,10 @@ mod test {
lazy_static! {
static ref INPUT_NODE_FOO_NAME: Bytes = "mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar".into();
static ref INPUT_NODE_FOO: Node =
Node::Directory(DirectoryNode::new(DUMMY_DIGEST.clone(), 42,));
static ref INPUT_NODE_FOO: Node = Node::Directory {
digest: DUMMY_DIGEST.clone(),
size: 42
};
}
#[test]

41
tvix/glue/src/tvix_store_io.rs
View file

@ -475,20 +475,16 @@ impl EvalIO for TvixStoreIO {
{
// depending on the node type, treat open differently
match node {
Node::Directory(_) => {
Node::Directory { .. } => {
// This would normally be a io::ErrorKind::IsADirectory (still unstable)
Err(io::Error::new(
io::ErrorKind::Unsupported,
format!("tried to open directory at {:?}", path),
))
}
Node::File(file_node) => {
Node::File { digest, .. } => {
self.tokio_handle.block_on(async {
let resp = self
.blob_service
.as_ref()
.open_read(file_node.digest())
.await?;
let resp = self.blob_service.as_ref().open_read(&digest).await?;
match resp {
Some(blob_reader) => {
// The VM Response needs a sync [std::io::Reader].
@ -497,18 +493,18 @@ impl EvalIO for TvixStoreIO {
}
None => {
error!(
blob.digest = %file_node.digest(),
blob.digest = %digest,
"blob not found",
);
Err(io::Error::new(
io::ErrorKind::NotFound,
format!("blob {} not found", &file_node.digest()),
format!("blob {} not found", &digest),
))
}
}
})
}
Node::Symlink(_symlink_node) => Err(io::Error::new(
Node::Symlink { .. } => Err(io::Error::new(
io::ErrorKind::Unsupported,
"open for symlinks is unsupported",
))?,
@ -534,9 +530,9 @@ impl EvalIO for TvixStoreIO {
.block_on(async { self.store_path_to_node(&store_path, &sub_path).await })?
{
match node {
Node::Directory(_) => Ok(FileType::Directory),
Node::File(_) => Ok(FileType::Regular),
Node::Symlink(_) => Ok(FileType::Symlink),
Node::Directory { .. } => Ok(FileType::Directory),
Node::File { .. } => Ok(FileType::Regular),
Node::Symlink { .. } => Ok(FileType::Symlink),
}
} else {
self.std_io.file_type(path)
@ -556,20 +552,19 @@ impl EvalIO for TvixStoreIO {
.block_on(async { self.store_path_to_node(&store_path, &sub_path).await })?
{
match node {
Node::Directory(directory_node) => {
Node::Directory { digest, .. } => {
// fetch the Directory itself.
let digest = directory_node.digest().clone();
if let Some(directory) = self.tokio_handle.block_on(async {
self.directory_service.as_ref().get(&digest).await
if let Some(directory) = self.tokio_handle.block_on({
let digest = digest.clone();
async move { self.directory_service.as_ref().get(&digest).await }
})? {
let mut children: Vec<(bytes::Bytes, FileType)> = Vec::new();
// TODO: into_nodes() to avoid cloning
for (name, node) in directory.nodes() {
children.push(match node {
Node::Directory(_) => (name.clone(), FileType::Directory),
Node::File(_) => (name.clone(), FileType::Regular),
Node::Symlink(_) => (name.clone(), FileType::Symlink),
Node::Directory { .. } => (name.clone(), FileType::Directory),
Node::File { .. } => (name.clone(), FileType::Regular),
Node::Symlink { .. } => (name.clone(), FileType::Symlink),
})
}
Ok(children)
@ -586,14 +581,14 @@ impl EvalIO for TvixStoreIO {
))?
}
}
Node::File(_file_node) => {
Node::File { .. } => {
// This would normally be a io::ErrorKind::NotADirectory (still unstable)
Err(io::Error::new(
io::ErrorKind::Unsupported,
"tried to readdir path {:?}, which is a file",
))?
}
Node::Symlink(_symlink_node) => Err(io::Error::new(
Node::Symlink { .. } => Err(io::Error::new(
io::ErrorKind::Unsupported,
"read_dir for symlinks is unsupported",
))?,