refactor(tvix/store): generalize PathInfo constructors

Instead of enforcing NAR SHA256 all the time, we generalize the
`PathInfo` constructor to take a `CAHash` argument which can drive
whether we are having a flat, NAR or text scheme.

With this, it is now possible to implement flat schemes in our
evaluation builtins, e.g. `builtins.path`.

Change-Id: I15bfee0ef4f0f428bfbd2f30c57c012cdcf6a976
Signed-off-by: Ryan Lahfa <tvl@lahfa.xyz>
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11286
Reviewed-by: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
This commit is contained in:
Ryan Lahfa 2024-03-29 00:43:56 +01:00 committed by raitobezarius
parent f2ca30774e
commit 14fe65a50b
3 changed files with 63 additions and 20 deletions

View file

@ -119,6 +119,7 @@ mod import_builtins {
use super::*; use super::*;
use nix_compat::nixhash::{CAHash, NixHash};
use tvix_eval::generators::Gen; use tvix_eval::generators::Gen;
use tvix_eval::{generators::GenCo, ErrorKind, Value}; use tvix_eval::{generators::GenCo, ErrorKind, Value};
@ -138,8 +139,19 @@ mod import_builtins {
Ok(state Ok(state
.tokio_handle .tokio_handle
.block_on(async { .block_on(async {
let (_, nar_sha256) = state
.path_info_service
.as_ref()
.calculate_nar(&root_node)
.await?;
state state
.register_node_in_path_info_service(name, &p, root_node) .register_node_in_path_info_service(
name,
&p,
CAHash::Nar(NixHash::Sha256(nar_sha256)),
root_node,
)
.await .await
}) })
.map_err(|err| ErrorKind::IO { .map_err(|err| ErrorKind::IO {

View file

@ -53,7 +53,8 @@ use crate::tvix_build::derivation_to_build_request;
pub struct TvixStoreIO { pub struct TvixStoreIO {
blob_service: Arc<dyn BlobService>, blob_service: Arc<dyn BlobService>,
directory_service: Arc<dyn DirectoryService>, directory_service: Arc<dyn DirectoryService>,
path_info_service: Arc<dyn PathInfoService>, // This is public so builtins can put PathInfos directly.
pub(crate) path_info_service: Arc<dyn PathInfoService>,
std_io: StdIO, std_io: StdIO,
#[allow(dead_code)] #[allow(dead_code)]
build_service: Arc<dyn BuildService>, build_service: Arc<dyn BuildService>,
@ -293,9 +294,13 @@ impl TvixStoreIO {
&self, &self,
name: &str, name: &str,
path: &Path, path: &Path,
ca: CAHash,
root_node: Node, root_node: Node,
) -> io::Result<(PathInfo, StorePath)> { ) -> io::Result<(PathInfo, StorePath)> {
// Ask the PathInfoService for the NAR size and sha256 // Ask the PathInfoService for the NAR size and sha256
// We always need it no matter what is the actual hash mode
// because the path info construct a narinfo which *always*
// require a SHA256 of the NAR representation and the NAR size.
let (nar_size, nar_sha256) = self let (nar_size, nar_sha256) = self
.path_info_service .path_info_service
.as_ref() .as_ref()
@ -303,20 +308,22 @@ impl TvixStoreIO {
.await?; .await?;
// Calculate the output path. This might still fail, as some names are illegal. // Calculate the output path. This might still fail, as some names are illegal.
let output_path = nix_compat::store_path::build_nar_based_store_path(&nar_sha256, name) let output_path =
.map_err(|_| { nix_compat::store_path::build_ca_path(name, &ca, Vec::<String>::new(), false).map_err(
|_| {
std::io::Error::new( std::io::Error::new(
std::io::ErrorKind::InvalidData, std::io::ErrorKind::InvalidData,
format!("invalid name: {}", name), format!("invalid name: {}", name),
) )
})?; },
)?;
// assemble a new root_node with a name that is derived from the nar hash. // assemble a new root_node with a name that is derived from the nar hash.
let root_node = root_node.rename(output_path.to_string().into_bytes().into()); let root_node = root_node.rename(output_path.to_string().into_bytes().into());
tvix_store::import::log_node(&root_node, path); tvix_store::import::log_node(&root_node, path);
let path_info = let path_info =
tvix_store::import::derive_nar_ca_path_info(nar_size, nar_sha256, root_node); tvix_store::import::derive_nar_ca_path_info(nar_size, nar_sha256, Some(ca), root_node);
Ok((path_info, output_path.to_owned())) Ok((path_info, output_path.to_owned()))
} }
@ -325,9 +332,10 @@ impl TvixStoreIO {
&self, &self,
name: &str, name: &str,
path: &Path, path: &Path,
ca: CAHash,
root_node: Node, root_node: Node,
) -> io::Result<StorePath> { ) -> io::Result<StorePath> {
let (path_info, output_path) = self.node_to_path_info(name, path, root_node).await?; let (path_info, output_path) = self.node_to_path_info(name, path, ca, root_node).await?;
let _path_info = self.path_info_service.as_ref().put(path_info).await?; let _path_info = self.path_info_service.as_ref().put(path_info).await?;
Ok(output_path) Ok(output_path)

View file

@ -4,13 +4,27 @@ use tvix_castore::{
blobservice::BlobService, directoryservice::DirectoryService, proto::node::Node, B3Digest, blobservice::BlobService, directoryservice::DirectoryService, proto::node::Node, B3Digest,
}; };
use nix_compat::store_path::{self, StorePath}; use nix_compat::{
nixhash::{CAHash, NixHash},
store_path::{self, StorePath},
};
use crate::{ use crate::{
pathinfoservice::PathInfoService, pathinfoservice::PathInfoService,
proto::{nar_info, NarInfo, PathInfo}, proto::{nar_info, NarInfo, PathInfo},
}; };
impl From<CAHash> for nar_info::Ca {
fn from(value: CAHash) -> Self {
let hash_type: nar_info::ca::Hash = (&value).into();
let digest: bytes::Bytes = value.hash().to_string().into();
nar_info::Ca {
r#type: hash_type.into(),
digest,
}
}
}
pub fn log_node(node: &Node, path: &Path) { pub fn log_node(node: &Node, path: &Path) {
match node { match node {
Node::Directory(directory_node) => { Node::Directory(directory_node) => {
@ -54,13 +68,20 @@ pub fn path_to_name(path: &Path) -> std::io::Result<&str> {
}) })
} }
/// Takes the NAR size, SHA-256 of the NAR representation and the root node. /// Takes the NAR size, SHA-256 of the NAR representation, the root node and optionally
/// Returns the path information object for a content addressed NAR-style (recursive) object. /// a CA hash information.
///
/// Returns the path information object for a NAR-style object.
/// ///
/// This [`PathInfo`] can be further filled for signatures, deriver or verified for the expected /// This [`PathInfo`] can be further filled for signatures, deriver or verified for the expected
/// hashes. /// hashes.
#[inline] #[inline]
pub fn derive_nar_ca_path_info(nar_size: u64, nar_sha256: [u8; 32], root_node: Node) -> PathInfo { pub fn derive_nar_ca_path_info(
nar_size: u64,
nar_sha256: [u8; 32],
ca: Option<CAHash>,
root_node: Node,
) -> PathInfo {
// assemble the [crate::proto::PathInfo] object. // assemble the [crate::proto::PathInfo] object.
PathInfo { PathInfo {
node: Some(tvix_castore::proto::Node { node: Some(tvix_castore::proto::Node {
@ -74,10 +95,7 @@ pub fn derive_nar_ca_path_info(nar_size: u64, nar_sha256: [u8; 32], root_node: N
signatures: vec![], signatures: vec![],
reference_names: vec![], reference_names: vec![],
deriver: None, deriver: None,
ca: Some(nar_info::Ca { ca: ca.map(|ca_hash| ca_hash.into()),
r#type: nar_info::ca::Hash::NarSha256.into(),
digest: nar_sha256.to_vec().into(),
}),
}), }),
} }
} }
@ -118,7 +136,12 @@ where
let root_node = root_node.rename(output_path.to_string().into_bytes().into()); let root_node = root_node.rename(output_path.to_string().into_bytes().into());
log_node(&root_node, path.as_ref()); log_node(&root_node, path.as_ref());
let path_info = derive_nar_ca_path_info(nar_size, nar_sha256, root_node); let path_info = derive_nar_ca_path_info(
nar_size,
nar_sha256,
Some(CAHash::Nar(NixHash::Sha256(nar_sha256))),
root_node,
);
// This new [`PathInfo`] that we get back from there might contain additional signatures or // This new [`PathInfo`] that we get back from there might contain additional signatures or
// information set by the service itself. In this function, we silently swallow it because // information set by the service itself. In this function, we silently swallow it because