From 14fe65a50b7bc1e31083c916d254043b0639d5aa Mon Sep 17 00:00:00 2001 From: Ryan Lahfa Date: Fri, 29 Mar 2024 00:43:56 +0100 Subject: [PATCH] refactor(tvix/store): generalize `PathInfo` constructors Instead of enforcing NAR SHA256 all the time, we generalize the `PathInfo` constructor to take a `CAHash` argument which can drive whether we are having a flat, NAR or text scheme. With this, it is now possible to implement flat schemes in our evaluation builtins, e.g. `builtins.path`. Change-Id: I15bfee0ef4f0f428bfbd2f30c57c012cdcf6a976 Signed-off-by: Ryan Lahfa Reviewed-on: https://cl.tvl.fyi/c/depot/+/11286 Reviewed-by: flokli Tested-by: BuildkiteCI --- tvix/glue/src/builtins/import.rs | 14 ++++++++++- tvix/glue/src/tvix_store_io.rs | 28 ++++++++++++++-------- tvix/store/src/import.rs | 41 +++++++++++++++++++++++++------- 3 files changed, 63 insertions(+), 20 deletions(-) diff --git a/tvix/glue/src/builtins/import.rs b/tvix/glue/src/builtins/import.rs index 88e483031..8c9efc679 100644 --- a/tvix/glue/src/builtins/import.rs +++ b/tvix/glue/src/builtins/import.rs @@ -119,6 +119,7 @@ mod import_builtins { use super::*; + use nix_compat::nixhash::{CAHash, NixHash}; use tvix_eval::generators::Gen; use tvix_eval::{generators::GenCo, ErrorKind, Value}; @@ -138,8 +139,19 @@ mod import_builtins { Ok(state .tokio_handle .block_on(async { + let (_, nar_sha256) = state + .path_info_service + .as_ref() + .calculate_nar(&root_node) + .await?; + state - .register_node_in_path_info_service(name, &p, root_node) + .register_node_in_path_info_service( + name, + &p, + CAHash::Nar(NixHash::Sha256(nar_sha256)), + root_node, + ) .await }) .map_err(|err| ErrorKind::IO { diff --git a/tvix/glue/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs index cbda365c2..bfdab0846 100644 --- a/tvix/glue/src/tvix_store_io.rs +++ b/tvix/glue/src/tvix_store_io.rs @@ -53,7 +53,8 @@ use crate::tvix_build::derivation_to_build_request; pub struct TvixStoreIO { blob_service: Arc, directory_service: Arc, - path_info_service: Arc, + // This is public so builtins can put PathInfos directly. + pub(crate) path_info_service: Arc, std_io: StdIO, #[allow(dead_code)] build_service: Arc, @@ -293,9 +294,13 @@ impl TvixStoreIO { &self, name: &str, path: &Path, + ca: CAHash, root_node: Node, ) -> io::Result<(PathInfo, StorePath)> { // Ask the PathInfoService for the NAR size and sha256 + // We always need it no matter what is the actual hash mode + // because the path info construct a narinfo which *always* + // require a SHA256 of the NAR representation and the NAR size. let (nar_size, nar_sha256) = self .path_info_service .as_ref() @@ -303,20 +308,22 @@ impl TvixStoreIO { .await?; // Calculate the output path. This might still fail, as some names are illegal. - let output_path = nix_compat::store_path::build_nar_based_store_path(&nar_sha256, name) - .map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("invalid name: {}", name), - ) - })?; + let output_path = + nix_compat::store_path::build_ca_path(name, &ca, Vec::::new(), false).map_err( + |_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid name: {}", name), + ) + }, + )?; // assemble a new root_node with a name that is derived from the nar hash. let root_node = root_node.rename(output_path.to_string().into_bytes().into()); tvix_store::import::log_node(&root_node, path); let path_info = - tvix_store::import::derive_nar_ca_path_info(nar_size, nar_sha256, root_node); + tvix_store::import::derive_nar_ca_path_info(nar_size, nar_sha256, Some(ca), root_node); Ok((path_info, output_path.to_owned())) } @@ -325,9 +332,10 @@ impl TvixStoreIO { &self, name: &str, path: &Path, + ca: CAHash, root_node: Node, ) -> io::Result { - let (path_info, output_path) = self.node_to_path_info(name, path, root_node).await?; + let (path_info, output_path) = self.node_to_path_info(name, path, ca, root_node).await?; let _path_info = self.path_info_service.as_ref().put(path_info).await?; Ok(output_path) diff --git a/tvix/store/src/import.rs b/tvix/store/src/import.rs index f2e7bdfe5..69f68d46a 100644 --- a/tvix/store/src/import.rs +++ b/tvix/store/src/import.rs @@ -4,13 +4,27 @@ use tvix_castore::{ blobservice::BlobService, directoryservice::DirectoryService, proto::node::Node, B3Digest, }; -use nix_compat::store_path::{self, StorePath}; +use nix_compat::{ + nixhash::{CAHash, NixHash}, + store_path::{self, StorePath}, +}; use crate::{ pathinfoservice::PathInfoService, proto::{nar_info, NarInfo, PathInfo}, }; +impl From for nar_info::Ca { + fn from(value: CAHash) -> Self { + let hash_type: nar_info::ca::Hash = (&value).into(); + let digest: bytes::Bytes = value.hash().to_string().into(); + nar_info::Ca { + r#type: hash_type.into(), + digest, + } + } +} + pub fn log_node(node: &Node, path: &Path) { match node { Node::Directory(directory_node) => { @@ -54,13 +68,20 @@ pub fn path_to_name(path: &Path) -> std::io::Result<&str> { }) } -/// Takes the NAR size, SHA-256 of the NAR representation and the root node. -/// Returns the path information object for a content addressed NAR-style (recursive) object. +/// Takes the NAR size, SHA-256 of the NAR representation, the root node and optionally +/// a CA hash information. +/// +/// Returns the path information object for a NAR-style object. /// /// This [`PathInfo`] can be further filled for signatures, deriver or verified for the expected /// hashes. #[inline] -pub fn derive_nar_ca_path_info(nar_size: u64, nar_sha256: [u8; 32], root_node: Node) -> PathInfo { +pub fn derive_nar_ca_path_info( + nar_size: u64, + nar_sha256: [u8; 32], + ca: Option, + root_node: Node, +) -> PathInfo { // assemble the [crate::proto::PathInfo] object. PathInfo { node: Some(tvix_castore::proto::Node { @@ -74,10 +95,7 @@ pub fn derive_nar_ca_path_info(nar_size: u64, nar_sha256: [u8; 32], root_node: N signatures: vec![], reference_names: vec![], deriver: None, - ca: Some(nar_info::Ca { - r#type: nar_info::ca::Hash::NarSha256.into(), - digest: nar_sha256.to_vec().into(), - }), + ca: ca.map(|ca_hash| ca_hash.into()), }), } } @@ -118,7 +136,12 @@ where let root_node = root_node.rename(output_path.to_string().into_bytes().into()); log_node(&root_node, path.as_ref()); - let path_info = derive_nar_ca_path_info(nar_size, nar_sha256, root_node); + let path_info = derive_nar_ca_path_info( + nar_size, + nar_sha256, + Some(CAHash::Nar(NixHash::Sha256(nar_sha256))), + root_node, + ); // This new [`PathInfo`] that we get back from there might contain additional signatures or // information set by the service itself. In this function, we silently swallow it because