refactor(tvix/store): generalize PathInfo
constructors
Instead of enforcing NAR SHA256 all the time, we generalize the `PathInfo` constructor to take a `CAHash` argument which can drive whether we are having a flat, NAR or text scheme. With this, it is now possible to implement flat schemes in our evaluation builtins, e.g. `builtins.path`. Change-Id: I15bfee0ef4f0f428bfbd2f30c57c012cdcf6a976 Signed-off-by: Ryan Lahfa <tvl@lahfa.xyz> Reviewed-on: https://cl.tvl.fyi/c/depot/+/11286 Reviewed-by: flokli <flokli@flokli.de> Tested-by: BuildkiteCI
This commit is contained in:
parent
f2ca30774e
commit
14fe65a50b
3 changed files with 63 additions and 20 deletions
|
@ -119,6 +119,7 @@ mod import_builtins {
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
use nix_compat::nixhash::{CAHash, NixHash};
|
||||||
use tvix_eval::generators::Gen;
|
use tvix_eval::generators::Gen;
|
||||||
use tvix_eval::{generators::GenCo, ErrorKind, Value};
|
use tvix_eval::{generators::GenCo, ErrorKind, Value};
|
||||||
|
|
||||||
|
@ -138,8 +139,19 @@ mod import_builtins {
|
||||||
Ok(state
|
Ok(state
|
||||||
.tokio_handle
|
.tokio_handle
|
||||||
.block_on(async {
|
.block_on(async {
|
||||||
|
let (_, nar_sha256) = state
|
||||||
|
.path_info_service
|
||||||
|
.as_ref()
|
||||||
|
.calculate_nar(&root_node)
|
||||||
|
.await?;
|
||||||
|
|
||||||
state
|
state
|
||||||
.register_node_in_path_info_service(name, &p, root_node)
|
.register_node_in_path_info_service(
|
||||||
|
name,
|
||||||
|
&p,
|
||||||
|
CAHash::Nar(NixHash::Sha256(nar_sha256)),
|
||||||
|
root_node,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
})
|
})
|
||||||
.map_err(|err| ErrorKind::IO {
|
.map_err(|err| ErrorKind::IO {
|
||||||
|
|
|
@ -53,7 +53,8 @@ use crate::tvix_build::derivation_to_build_request;
|
||||||
pub struct TvixStoreIO {
|
pub struct TvixStoreIO {
|
||||||
blob_service: Arc<dyn BlobService>,
|
blob_service: Arc<dyn BlobService>,
|
||||||
directory_service: Arc<dyn DirectoryService>,
|
directory_service: Arc<dyn DirectoryService>,
|
||||||
path_info_service: Arc<dyn PathInfoService>,
|
// This is public so builtins can put PathInfos directly.
|
||||||
|
pub(crate) path_info_service: Arc<dyn PathInfoService>,
|
||||||
std_io: StdIO,
|
std_io: StdIO,
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
build_service: Arc<dyn BuildService>,
|
build_service: Arc<dyn BuildService>,
|
||||||
|
@ -293,9 +294,13 @@ impl TvixStoreIO {
|
||||||
&self,
|
&self,
|
||||||
name: &str,
|
name: &str,
|
||||||
path: &Path,
|
path: &Path,
|
||||||
|
ca: CAHash,
|
||||||
root_node: Node,
|
root_node: Node,
|
||||||
) -> io::Result<(PathInfo, StorePath)> {
|
) -> io::Result<(PathInfo, StorePath)> {
|
||||||
// Ask the PathInfoService for the NAR size and sha256
|
// Ask the PathInfoService for the NAR size and sha256
|
||||||
|
// We always need it no matter what is the actual hash mode
|
||||||
|
// because the path info construct a narinfo which *always*
|
||||||
|
// require a SHA256 of the NAR representation and the NAR size.
|
||||||
let (nar_size, nar_sha256) = self
|
let (nar_size, nar_sha256) = self
|
||||||
.path_info_service
|
.path_info_service
|
||||||
.as_ref()
|
.as_ref()
|
||||||
|
@ -303,20 +308,22 @@ impl TvixStoreIO {
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Calculate the output path. This might still fail, as some names are illegal.
|
// Calculate the output path. This might still fail, as some names are illegal.
|
||||||
let output_path = nix_compat::store_path::build_nar_based_store_path(&nar_sha256, name)
|
let output_path =
|
||||||
.map_err(|_| {
|
nix_compat::store_path::build_ca_path(name, &ca, Vec::<String>::new(), false).map_err(
|
||||||
|
|_| {
|
||||||
std::io::Error::new(
|
std::io::Error::new(
|
||||||
std::io::ErrorKind::InvalidData,
|
std::io::ErrorKind::InvalidData,
|
||||||
format!("invalid name: {}", name),
|
format!("invalid name: {}", name),
|
||||||
)
|
)
|
||||||
})?;
|
},
|
||||||
|
)?;
|
||||||
|
|
||||||
// assemble a new root_node with a name that is derived from the nar hash.
|
// assemble a new root_node with a name that is derived from the nar hash.
|
||||||
let root_node = root_node.rename(output_path.to_string().into_bytes().into());
|
let root_node = root_node.rename(output_path.to_string().into_bytes().into());
|
||||||
tvix_store::import::log_node(&root_node, path);
|
tvix_store::import::log_node(&root_node, path);
|
||||||
|
|
||||||
let path_info =
|
let path_info =
|
||||||
tvix_store::import::derive_nar_ca_path_info(nar_size, nar_sha256, root_node);
|
tvix_store::import::derive_nar_ca_path_info(nar_size, nar_sha256, Some(ca), root_node);
|
||||||
|
|
||||||
Ok((path_info, output_path.to_owned()))
|
Ok((path_info, output_path.to_owned()))
|
||||||
}
|
}
|
||||||
|
@ -325,9 +332,10 @@ impl TvixStoreIO {
|
||||||
&self,
|
&self,
|
||||||
name: &str,
|
name: &str,
|
||||||
path: &Path,
|
path: &Path,
|
||||||
|
ca: CAHash,
|
||||||
root_node: Node,
|
root_node: Node,
|
||||||
) -> io::Result<StorePath> {
|
) -> io::Result<StorePath> {
|
||||||
let (path_info, output_path) = self.node_to_path_info(name, path, root_node).await?;
|
let (path_info, output_path) = self.node_to_path_info(name, path, ca, root_node).await?;
|
||||||
let _path_info = self.path_info_service.as_ref().put(path_info).await?;
|
let _path_info = self.path_info_service.as_ref().put(path_info).await?;
|
||||||
|
|
||||||
Ok(output_path)
|
Ok(output_path)
|
||||||
|
|
|
@ -4,13 +4,27 @@ use tvix_castore::{
|
||||||
blobservice::BlobService, directoryservice::DirectoryService, proto::node::Node, B3Digest,
|
blobservice::BlobService, directoryservice::DirectoryService, proto::node::Node, B3Digest,
|
||||||
};
|
};
|
||||||
|
|
||||||
use nix_compat::store_path::{self, StorePath};
|
use nix_compat::{
|
||||||
|
nixhash::{CAHash, NixHash},
|
||||||
|
store_path::{self, StorePath},
|
||||||
|
};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
pathinfoservice::PathInfoService,
|
pathinfoservice::PathInfoService,
|
||||||
proto::{nar_info, NarInfo, PathInfo},
|
proto::{nar_info, NarInfo, PathInfo},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
impl From<CAHash> for nar_info::Ca {
|
||||||
|
fn from(value: CAHash) -> Self {
|
||||||
|
let hash_type: nar_info::ca::Hash = (&value).into();
|
||||||
|
let digest: bytes::Bytes = value.hash().to_string().into();
|
||||||
|
nar_info::Ca {
|
||||||
|
r#type: hash_type.into(),
|
||||||
|
digest,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn log_node(node: &Node, path: &Path) {
|
pub fn log_node(node: &Node, path: &Path) {
|
||||||
match node {
|
match node {
|
||||||
Node::Directory(directory_node) => {
|
Node::Directory(directory_node) => {
|
||||||
|
@ -54,13 +68,20 @@ pub fn path_to_name(path: &Path) -> std::io::Result<&str> {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Takes the NAR size, SHA-256 of the NAR representation and the root node.
|
/// Takes the NAR size, SHA-256 of the NAR representation, the root node and optionally
|
||||||
/// Returns the path information object for a content addressed NAR-style (recursive) object.
|
/// a CA hash information.
|
||||||
|
///
|
||||||
|
/// Returns the path information object for a NAR-style object.
|
||||||
///
|
///
|
||||||
/// This [`PathInfo`] can be further filled for signatures, deriver or verified for the expected
|
/// This [`PathInfo`] can be further filled for signatures, deriver or verified for the expected
|
||||||
/// hashes.
|
/// hashes.
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn derive_nar_ca_path_info(nar_size: u64, nar_sha256: [u8; 32], root_node: Node) -> PathInfo {
|
pub fn derive_nar_ca_path_info(
|
||||||
|
nar_size: u64,
|
||||||
|
nar_sha256: [u8; 32],
|
||||||
|
ca: Option<CAHash>,
|
||||||
|
root_node: Node,
|
||||||
|
) -> PathInfo {
|
||||||
// assemble the [crate::proto::PathInfo] object.
|
// assemble the [crate::proto::PathInfo] object.
|
||||||
PathInfo {
|
PathInfo {
|
||||||
node: Some(tvix_castore::proto::Node {
|
node: Some(tvix_castore::proto::Node {
|
||||||
|
@ -74,10 +95,7 @@ pub fn derive_nar_ca_path_info(nar_size: u64, nar_sha256: [u8; 32], root_node: N
|
||||||
signatures: vec![],
|
signatures: vec![],
|
||||||
reference_names: vec![],
|
reference_names: vec![],
|
||||||
deriver: None,
|
deriver: None,
|
||||||
ca: Some(nar_info::Ca {
|
ca: ca.map(|ca_hash| ca_hash.into()),
|
||||||
r#type: nar_info::ca::Hash::NarSha256.into(),
|
|
||||||
digest: nar_sha256.to_vec().into(),
|
|
||||||
}),
|
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -118,7 +136,12 @@ where
|
||||||
let root_node = root_node.rename(output_path.to_string().into_bytes().into());
|
let root_node = root_node.rename(output_path.to_string().into_bytes().into());
|
||||||
log_node(&root_node, path.as_ref());
|
log_node(&root_node, path.as_ref());
|
||||||
|
|
||||||
let path_info = derive_nar_ca_path_info(nar_size, nar_sha256, root_node);
|
let path_info = derive_nar_ca_path_info(
|
||||||
|
nar_size,
|
||||||
|
nar_sha256,
|
||||||
|
Some(CAHash::Nar(NixHash::Sha256(nar_sha256))),
|
||||||
|
root_node,
|
||||||
|
);
|
||||||
|
|
||||||
// This new [`PathInfo`] that we get back from there might contain additional signatures or
|
// This new [`PathInfo`] that we get back from there might contain additional signatures or
|
||||||
// information set by the service itself. In this function, we silently swallow it because
|
// information set by the service itself. In this function, we silently swallow it because
|
||||||
|
|
Loading…
Reference in a new issue