feat(tvix/glue): implement builtins.toFile

We currently only had a dummy implementation that didn't actually
persist the files in the store(s).

Copy the contents to the BlobService, and do the output path calculation
as part of the upload.
Use the plain context elements to construct the references.

Change-Id: Ibdaf7a645ddc31e847faa4b87a79f2f95116a7ab
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11879
Reviewed-by: Ilan Joselevich <personal@ilanjoselevich.com>
Reviewed-by: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2024-06-25 23:44:42 +03:00 committed by flokli
parent 888028b674
commit b757897e97
4 changed files with 94 additions and 9 deletions

View file

@ -168,15 +168,23 @@ fn handle_fixed_output(
#[builtins(state = "Rc<TvixStoreIO>")] #[builtins(state = "Rc<TvixStoreIO>")]
pub(crate) mod derivation_builtins { pub(crate) mod derivation_builtins {
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::io::Cursor;
use crate::builtins::utils::{select_string, strong_importing_coerce_to_string}; use crate::builtins::utils::{select_string, strong_importing_coerce_to_string};
use crate::fetchurl::fetchurl_derivation_to_fetch; use crate::fetchurl::fetchurl_derivation_to_fetch;
use super::*; use super::*;
use bstr::ByteSlice; use bstr::ByteSlice;
use nix_compat::store_path::hash_placeholder; use md5::Digest;
use nix_compat::nixhash::CAHash;
use nix_compat::store_path::{build_ca_path, hash_placeholder};
use sha2::Sha256;
use tvix_castore::proto as castorepb;
use tvix_castore::proto::node::Node;
use tvix_castore::proto::FileNode;
use tvix_eval::generators::Gen; use tvix_eval::generators::Gen;
use tvix_eval::{NixContext, NixContextElement, NixString}; use tvix_eval::{NixContext, NixContextElement, NixString};
use tvix_store::proto::{NarInfo, PathInfo};
#[builtin("placeholder")] #[builtin("placeholder")]
async fn builtin_placeholder(co: GenCo, input: Value) -> Result<Value, ErrorKind> { async fn builtin_placeholder(co: GenCo, input: Value) -> Result<Value, ErrorKind> {
@ -525,7 +533,12 @@ pub(crate) mod derivation_builtins {
} }
#[builtin("toFile")] #[builtin("toFile")]
async fn builtin_to_file(co: GenCo, name: Value, content: Value) -> Result<Value, ErrorKind> { async fn builtin_to_file(
state: Rc<TvixStoreIO>,
co: GenCo,
name: Value,
content: Value,
) -> Result<Value, ErrorKind> {
if name.is_catchable() { if name.is_catchable() {
return Ok(name); return Ok(name);
} }
@ -545,20 +558,77 @@ pub(crate) mod derivation_builtins {
return Err(ErrorKind::UnexpectedContext); return Err(ErrorKind::UnexpectedContext);
} }
let path = let store_path = state.tokio_handle.block_on(async {
nix_compat::store_path::build_text_path(name.to_str()?, &content, content.iter_plain()) // upload contents to the blobservice and create a root node
let mut blob_writer = state.blob_service.open_write().await;
let mut r = Cursor::new(&content);
let blob_size = tokio::io::copy(&mut r, &mut blob_writer).await?;
let blob_digest = blob_writer.close().await?;
let ca_hash = CAHash::Text(Sha256::digest(&content).into());
let store_path = build_ca_path(name.to_str()?, &ca_hash, content.iter_plain(), false)
.map_err(|_e| { .map_err(|_e| {
nix_compat::derivation::DerivationError::InvalidOutputName( nix_compat::derivation::DerivationError::InvalidOutputName(
name.to_str_lossy().into_owned(), name.to_str_lossy().into_owned(),
) )
}) })
.map_err(DerivationError::InvalidDerivation)? .map_err(DerivationError::InvalidDerivation)?;
.to_absolute_path();
let context: NixContext = NixContextElement::Plain(path.clone()).into(); let root_node = Node::File(FileNode {
name: store_path.to_string().into(),
digest: blob_digest.into(),
size: blob_size,
executable: false,
});
// TODO: actually persist the file in the store at that path ... // calculate the nar hash
let (nar_size, nar_sha256) = state
.nar_calculation_service
.calculate_nar(&root_node)
.await
.map_err(|e| ErrorKind::TvixError(Rc::new(e)))?;
Ok(Value::from(NixString::new_context_from(context, path))) // assemble references from plain context.
let reference_paths: Vec<StorePathRef> = content
.iter_plain()
.map(|elem| StorePathRef::from_absolute_path(elem.as_bytes()))
.collect::<Result<_, _>>()
.map_err(|e| ErrorKind::TvixError(Rc::new(e)))?;
// persist via pathinfo service.
state
.path_info_service
.put(PathInfo {
node: Some(castorepb::Node {
node: Some(root_node),
}),
references: reference_paths
.iter()
.map(|x| bytes::Bytes::copy_from_slice(x.digest()))
.collect(),
narinfo: Some(NarInfo {
nar_size,
nar_sha256: nar_sha256.to_vec().into(),
signatures: vec![],
reference_names: reference_paths
.into_iter()
.map(|x| x.to_string())
.collect(),
deriver: None,
ca: Some(ca_hash.into()),
}),
})
.await
.map_err(|e| ErrorKind::TvixError(Rc::new(e)))?;
Ok::<_, ErrorKind>(store_path)
})?;
let abs_path = store_path.to_absolute_path();
let context: NixContext = NixContextElement::Plain(abs_path.clone()).into();
Ok(Value::from(NixString::new_context_from(context, abs_path)))
} }
} }

View file

@ -0,0 +1,3 @@
# in 'toFile': the file 'foo' cannot refer to derivation outputs, at (string):1:1
builtins.toFile "foo" "${(builtins.derivation {name = "foo"; builder = ":"; system = ":";})}"

View file

@ -0,0 +1 @@
[ "/nix/store/vxjiwkjkn7x4079qvh1jkl5pn05j2aw0-foo" "/nix/store/i7liwr52956m86kxp7dgbcwsk56r27v6-foo" "/nix/store/yw8k7ixk1zvb113p4y0bl3ahjxd5h9sr-foo" { "/nix/store/yw8k7ixk1zvb113p4y0bl3ahjxd5h9sr-foo" = { path = true; }; } ]

View file

@ -0,0 +1,11 @@
let
noContext = (builtins.toFile "foo" "bar");
someContext = (builtins.toFile "foo" "bar${noContext}");
moreContext = (builtins.toFile "foo" "bar${someContext}");
in
[
noContext
someContext
moreContext
(builtins.getContext moreContext)
]