refactor(tvix/store): use strictly typed PathInfo struct
This switches the PathInfoService trait from using the proto-derived PathInfo struct to a more restrictive struct, and updates all implementations to use it. It removes a lot of the previous conversion and checks, as invalid states became nonrepresentable, and validations are expressed on the type level. PathInfoService implementations consuming protobuf need to convert and do the verification internally, and can only return the strongly typed variant. The nix_compat::narinfo::NarInfo conversions for the proto PathInfo are removed, we only keep a version showing a NarInfo representation for the strong struct. Converting back to a PathInfo requires the root node now, but is otherwise trivial, so left to the users. Co-Authored-By: Florian Klink <flokli@flokli.de> Change-Id: I6fdfdb44063efebb44a8f0097b6b81a828717e03 Reviewed-on: https://cl.tvl.fyi/c/depot/+/12588 Reviewed-by: flokli <flokli@flokli.de> Tested-by: BuildkiteCI
This commit is contained in:
Marijan Petričević
parent
b4ccaac7ad
commit
e8040ec61f
26 changed files with 726 additions and 1042 deletions
|
|
@ -182,7 +182,7 @@ pub(crate) mod derivation_builtins {
|
|||
use tvix_castore::Node;
|
||||
use tvix_eval::generators::Gen;
|
||||
use tvix_eval::{NixContext, NixContextElement, NixString};
|
||||
use tvix_store::proto::{NarInfo, PathInfo};
|
||||
use tvix_store::pathinfoservice::PathInfo;
|
||||
|
||||
#[builtin("placeholder")]
|
||||
async fn builtin_placeholder(co: GenCo, input: Value) -> Result<Value, ErrorKind> {
|
||||
|
|
@ -568,15 +568,6 @@ pub(crate) mod derivation_builtins {
|
|||
let blob_digest = blob_writer.close().await?;
|
||||
let ca_hash = CAHash::Text(Sha256::digest(&content).into());
|
||||
|
||||
let store_path: StorePathRef =
|
||||
build_ca_path(name.to_str()?, &ca_hash, content.iter_ctx_plain(), false)
|
||||
.map_err(|_e| {
|
||||
nix_compat::derivation::DerivationError::InvalidOutputName(
|
||||
name.to_str_lossy().into_owned(),
|
||||
)
|
||||
})
|
||||
.map_err(DerivationError::InvalidDerivation)?;
|
||||
|
||||
let root_node = Node::File {
|
||||
digest: blob_digest,
|
||||
size: blob_size,
|
||||
|
|
@ -590,41 +581,38 @@ pub(crate) mod derivation_builtins {
|
|||
.await
|
||||
.map_err(|e| ErrorKind::TvixError(Rc::new(e)))?;
|
||||
|
||||
// assemble references from plain context.
|
||||
let reference_paths: Vec<StorePathRef> = content
|
||||
.iter_ctx_plain()
|
||||
.map(|elem| StorePathRef::from_absolute_path(elem.as_bytes()))
|
||||
.collect::<Result<_, _>>()
|
||||
.map_err(|e| ErrorKind::TvixError(Rc::new(e)))?;
|
||||
|
||||
// persist via pathinfo service.
|
||||
state
|
||||
.path_info_service
|
||||
.put(PathInfo {
|
||||
node: Some(tvix_castore::proto::Node::from_name_and_node(
|
||||
store_path.to_string().into(),
|
||||
root_node,
|
||||
)),
|
||||
references: reference_paths
|
||||
.iter()
|
||||
.map(|x| bytes::Bytes::copy_from_slice(x.digest()))
|
||||
.collect(),
|
||||
narinfo: Some(NarInfo {
|
||||
nar_size,
|
||||
nar_sha256: nar_sha256.to_vec().into(),
|
||||
signatures: vec![],
|
||||
reference_names: reference_paths
|
||||
.into_iter()
|
||||
.map(|x| x.to_string())
|
||||
.collect(),
|
||||
deriver: None,
|
||||
ca: Some(ca_hash.into()),
|
||||
}),
|
||||
store_path: build_ca_path(
|
||||
name.to_str()?,
|
||||
&ca_hash,
|
||||
content.iter_ctx_plain(),
|
||||
false,
|
||||
)
|
||||
.map_err(|_e| {
|
||||
nix_compat::derivation::DerivationError::InvalidOutputName(
|
||||
name.to_str_lossy().into_owned(),
|
||||
)
|
||||
})
|
||||
.map_err(DerivationError::InvalidDerivation)?,
|
||||
node: root_node,
|
||||
// assemble references from plain context.
|
||||
references: content
|
||||
.iter_ctx_plain()
|
||||
.map(|elem| StorePath::from_absolute_path(elem.as_bytes()))
|
||||
.collect::<Result<_, _>>()
|
||||
.map_err(|e| ErrorKind::TvixError(Rc::new(e)))?,
|
||||
nar_size,
|
||||
nar_sha256,
|
||||
signatures: vec![],
|
||||
deriver: None,
|
||||
ca: Some(ca_hash),
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ErrorKind::TvixError(Rc::new(e)))?;
|
||||
|
||||
Ok::<_, ErrorKind>(store_path)
|
||||
.map_err(|e| ErrorKind::TvixError(Rc::new(e)))
|
||||
.map(|path_info| path_info.store_path)
|
||||
})?;
|
||||
|
||||
let abs_path = store_path.to_absolute_path();
|
||||
|
|
|
|||
|
|
@ -11,7 +11,10 @@ use tokio_util::io::{InspectReader, InspectWriter};
|
|||
use tracing::{instrument, warn, Span};
|
||||
use tracing_indicatif::span_ext::IndicatifSpanExt;
|
||||
use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService, Node};
|
||||
use tvix_store::{nar::NarCalculationService, pathinfoservice::PathInfoService, proto::PathInfo};
|
||||
use tvix_store::{
|
||||
nar::NarCalculationService,
|
||||
pathinfoservice::{PathInfo, PathInfoService},
|
||||
};
|
||||
use url::Url;
|
||||
|
||||
use crate::builtins::FetcherError;
|
||||
|
|
@ -571,19 +574,14 @@ where
|
|||
|
||||
// Construct the PathInfo and persist it.
|
||||
let path_info = PathInfo {
|
||||
node: Some(tvix_castore::proto::Node::from_name_and_node(
|
||||
store_path.to_string().into(),
|
||||
node.clone(),
|
||||
)),
|
||||
store_path: store_path.to_owned(),
|
||||
node: node.clone(),
|
||||
references: vec![],
|
||||
narinfo: Some(tvix_store::proto::NarInfo {
|
||||
nar_size,
|
||||
nar_sha256: nar_sha256.to_vec().into(),
|
||||
signatures: vec![],
|
||||
reference_names: vec![],
|
||||
deriver: None,
|
||||
ca: Some(ca_hash.into()),
|
||||
}),
|
||||
nar_size,
|
||||
nar_sha256,
|
||||
signatures: vec![],
|
||||
deriver: None,
|
||||
ca: Some(ca_hash),
|
||||
};
|
||||
|
||||
self.path_info_service
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ use tvix_castore::{
|
|||
directoryservice::{self, DirectoryService},
|
||||
Node,
|
||||
};
|
||||
use tvix_store::{pathinfoservice::PathInfoService, proto::PathInfo};
|
||||
use tvix_store::pathinfoservice::{PathInfo, PathInfoService};
|
||||
|
||||
use crate::fetchers::Fetcher;
|
||||
use crate::known_paths::KnownPaths;
|
||||
|
|
@ -119,23 +119,8 @@ impl TvixStoreIO {
|
|||
.get(*store_path.digest())
|
||||
.await?
|
||||
{
|
||||
// if we have a PathInfo, we know there will be a root_node (due to validation)
|
||||
// TODO: use stricter typed BuildRequest here.
|
||||
Some(path_info) => {
|
||||
let (name, node) = path_info
|
||||
.node
|
||||
.expect("no node")
|
||||
.into_name_and_node()
|
||||
.expect("invalid node");
|
||||
|
||||
assert_eq!(
|
||||
store_path.to_string().as_bytes(),
|
||||
name.as_ref(),
|
||||
"returned node basename must match requested store path"
|
||||
);
|
||||
|
||||
node
|
||||
}
|
||||
Some(path_info) => path_info.node,
|
||||
// If there's no PathInfo found, this normally means we have to
|
||||
// trigger the build (and insert into PathInfoService, after
|
||||
// reference scanning).
|
||||
|
|
@ -336,47 +321,37 @@ impl TvixStoreIO {
|
|||
|
||||
// assemble the PathInfo to persist
|
||||
let path_info = PathInfo {
|
||||
node: Some(tvix_castore::proto::Node::from_name_and_node(
|
||||
drv_output
|
||||
.1
|
||||
.path
|
||||
.as_ref()
|
||||
.ok_or(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"missing output store path",
|
||||
))?
|
||||
.to_string()
|
||||
.into(),
|
||||
output_node,
|
||||
)),
|
||||
store_path: drv_output
|
||||
.1
|
||||
.path
|
||||
.as_ref()
|
||||
.ok_or(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"Tvix bug: missing output store path",
|
||||
))?
|
||||
.to_owned(),
|
||||
node: output_node,
|
||||
references: output_needles
|
||||
.iter()
|
||||
.map(|path| Bytes::from(path.digest().as_slice().to_vec()))
|
||||
.map(|s| (**s).to_owned())
|
||||
.collect(),
|
||||
narinfo: Some(tvix_store::proto::NarInfo {
|
||||
nar_size,
|
||||
nar_sha256: Bytes::from(nar_sha256.to_vec()),
|
||||
signatures: vec![],
|
||||
reference_names: output_needles
|
||||
.iter()
|
||||
.map(|path| path.to_string())
|
||||
.collect(),
|
||||
deriver: Some(tvix_store::proto::StorePath {
|
||||
name: drv_path
|
||||
nar_size,
|
||||
nar_sha256,
|
||||
signatures: vec![],
|
||||
deriver: Some(
|
||||
StorePath::from_name_and_digest_fixed(
|
||||
drv_path
|
||||
.name()
|
||||
.strip_suffix(".drv")
|
||||
.expect("missing .drv suffix")
|
||||
.to_string(),
|
||||
digest: drv_path.digest().to_vec().into(),
|
||||
}),
|
||||
ca: drv.fod_digest().map(
|
||||
|fod_digest| -> tvix_store::proto::nar_info::Ca {
|
||||
(&CAHash::Nar(nix_compat::nixhash::NixHash::Sha256(
|
||||
fod_digest,
|
||||
)))
|
||||
.into()
|
||||
},
|
||||
.expect("missing .drv suffix"),
|
||||
*drv_path.digest(),
|
||||
)
|
||||
.expect(
|
||||
"Tvix bug: StorePath without .drv suffix must be valid",
|
||||
),
|
||||
),
|
||||
ca: drv.fod_digest().map(|fod_digest| {
|
||||
CAHash::Nar(nix_compat::nixhash::NixHash::Sha256(fod_digest))
|
||||
}),
|
||||
};
|
||||
|
||||
|
|
@ -421,8 +396,7 @@ impl TvixStoreIO {
|
|||
) -> io::Result<(PathInfo, NixHash, StorePathRef<'a>)> {
|
||||
// Ask the PathInfoService for the NAR size and sha256
|
||||
// We always need it no matter what is the actual hash mode
|
||||
// because the path info construct a narinfo which *always*
|
||||
// require a SHA256 of the NAR representation and the NAR size.
|
||||
// because the [PathInfo] needs to contain nar_{sha256,size}.
|
||||
let (nar_size, nar_sha256) = self
|
||||
.nar_calculation_service
|
||||
.as_ref()
|
||||
|
|
@ -431,7 +405,7 @@ impl TvixStoreIO {
|
|||
|
||||
// Calculate the output path. This might still fail, as some names are illegal.
|
||||
let output_path =
|
||||
nix_compat::store_path::build_ca_path(name, ca, Vec::<String>::new(), false).map_err(
|
||||
nix_compat::store_path::build_ca_path(name, ca, Vec::<&str>::new(), false).map_err(
|
||||
|_| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
|
|
@ -446,8 +420,8 @@ impl TvixStoreIO {
|
|||
let path_info = tvix_store::import::derive_nar_ca_path_info(
|
||||
nar_size,
|
||||
nar_sha256,
|
||||
Some(ca),
|
||||
output_path.to_string().into(),
|
||||
Some(ca.clone()),
|
||||
output_path.to_owned(),
|
||||
root_node,
|
||||
);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue