mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/build): add derivation_to_build_request

Browse source 
This function converts from a nix_compat::derivation::Derivation to
a BuildRequest.

In addition to the Derivation itself, it needs two lookup functions to
map input paths to their castore nodes.

Change-Id: I0332982f0bc7933a5fda137fe39d5a850639d929
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10236
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
This commit is contained in:
Florian Klink 2023-12-09 12:53:17 +02:00 committed by clbot
parent 92bd69aef2
commit 8486f87e3c
7 changed files with 272 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
tvix/Cargo.lock generated
View file

@ -3054,6 +3054,7 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
name = "tvix-build"
version = "0.1.0"
dependencies = [
"bytes",
"prost",
"prost-build",
"tonic",
@ -3161,11 +3162,13 @@ name = "tvix-glue"
version = "0.1.0"
dependencies = [
"bytes",
"lazy_static",
"nix-compat",
"test-case",
"thiserror",
"tokio",
"tracing",
"tvix-build",
"tvix-castore",
"tvix-eval",
"tvix-store",

13
tvix/Cargo.nix
View file

@ -9369,6 +9369,10 @@ rec {
then lib.cleanSourceWith { filter = sourceFilter; src = ./build; }
else ./build;
dependencies = [
{
name = "bytes";
packageId = "bytes";
}
{
name = "prost";
packageId = "prost";
@ -9818,6 +9822,11 @@ rec {
name = "tracing";
packageId = "tracing";
}
{
name = "tvix-build";
packageId = "tvix-build";
usesDefaultFeatures = false;
}
{
name = "tvix-castore";
packageId = "tvix-castore";
@ -9837,6 +9846,10 @@ rec {
}
];
devDependencies = [
{
name = "lazy_static";
packageId = "lazy_static";
}
{
name = "test-case";
packageId = "test-case";

1
tvix/build/Cargo.toml
View file

@ -4,6 +4,7 @@ version = "0.1.0"
edition = "2021"
[dependencies]
bytes = "1.4.0"
prost = "0.12.1"
tonic = "0.10.2"
tvix-castore = { path = "../castore" }

2
tvix/glue/Cargo.toml
View file

@ -5,6 +5,7 @@ edition = "2021"
[dependencies]
nix-compat = { path = "../nix-compat" }
tvix-build = { path = "../build", default-features = false, features = []}
tvix-eval = { path = "../eval" }
tvix-castore = { path = "../castore" }
tvix-store = { path = "../store", default-features = false, features = []}
@ -17,4 +18,5 @@ thiserror = "1.0.38"
git = "https://github.com/tvlfyi/wu-manber.git"
[dev-dependencies]
lazy_static = "1.4.0"
test-case = "2.2.2"

1
tvix/glue/src/lib.rs
View file

@ -1,6 +1,7 @@
pub mod builtins;
pub mod known_paths;
pub mod refscan;
pub mod tvix_build;
pub mod tvix_io;
pub mod tvix_store_io;

1
tvix/glue/src/tests/ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv Normal file
View file

@ -0,0 +1 @@
Derive([("out","/nix/store/fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo","","")],[("/nix/store/ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv",["out"])],[],":",":",[],[("bar","/nix/store/mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar"),("builder",":"),("name","foo"),("out","/nix/store/fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo"),("system",":")])

251
tvix/glue/src/tvix_build.rs Normal file
View file

@ -0,0 +1,251 @@
//! This module contains glue code translating from
//! [nix_compat::derivation::Derivation] to [tvix_build::proto::BuildRequest].
use std::collections::BTreeMap;
use bytes::Bytes;
use nix_compat::{derivation::Derivation, store_path::StorePathRef};
use tvix_build::proto::{
build_request::{BuildConstraints, EnvVar},
BuildRequest,
};
use tvix_castore::proto::{NamedNode, Node};
/// These are the environment variables that Nix sets in its sandbox for every
/// build.
const NIX_ENVIRONMENT_VARS: [(&str, &str); 12] = [
("HOME", "/homeless-shelter"),
("NIX_BUILD_CORES", "0"), // TODO: make this configurable?
("NIX_BUILD_TOP", "/"),
("NIX_LOG_FD", "2"),
("NIX_STORE", "/nix/store"),
("PATH", "/path-not-set"),
("PWD", "/build"),
("TEMP", "/build"),
("TEMPDIR", "/build"),
("TERM", "xterm-256color"),
("TMP", "/build"),
("TMPDIR", "/build"),
];
/// Takes a [Derivation] and turns it into a [BuildRequest].
/// It assumes the Derivation has been validated.
/// It needs two lookup functions:
/// - one translating input sources to a castore node
/// (`fn_input_sources_to_node`)
/// - one translating input derivations and (a subset of their) output names to
/// a castore node (`fn_input_drvs_to_node`).
#[allow(dead_code)]
fn derivation_to_build_request<FIS, FID>(
derivation: &Derivation,
fn_input_sources_to_node: FIS,
fn_input_drvs_to_node: FID,
) -> BuildRequest
where
FIS: Fn(StorePathRef) -> Node,
FID: Fn(StorePathRef, &[&str]) -> Node,
{
debug_assert!(derivation.validate(true).is_ok(), "drv must validate");
// produce command_args, which is builder and arguments in a Vec.
let mut command_args: Vec<String> = Vec::with_capacity(derivation.arguments.len() + 1);
command_args.push(derivation.builder.clone());
command_args.extend_from_slice(&derivation.arguments);
// produce output_paths, which is the basename of each output (sorted)
// since Derivation is validated, we know output paths can be parsed.
// TODO: b/264 will remove the need to parse them here
let mut outputs: Vec<String> = derivation
.outputs
.values()
.map(|output| {
let output_storepath = StorePathRef::from_absolute_path(output.path.as_bytes())
.expect("invalid output storepath");
output_storepath.to_string()
})
.collect();
// Sort the outputs. We can use sort_unstable, as these are unique strings.
outputs.sort_unstable();
// Produce environment_vars. We use a BTreeMap while producing them, so the
// resulting Vec is sorted by key.
let mut environment_vars: BTreeMap<String, Bytes> = BTreeMap::new();
// Start with some the ones that nix magically sets:
environment_vars.extend(
NIX_ENVIRONMENT_VARS
.iter()
.map(|(k, v)| (k.to_string(), Bytes::from_static(v.as_bytes()))),
);
// extend / overwrite with the keys set in the derivation environment itself.
// TODO: check if this order is correct, and environment vars set in the
// *Derivation actually* have priority.
environment_vars.extend(
derivation
.environment
.iter()
.map(|(k, v)| (k.clone(), Bytes::from(v.to_vec()))),
);
// Turn this into a sorted-by-key Vec<EnvVar>.
let environment_vars = Vec::from_iter(
environment_vars
.into_iter()
.map(|(k, v)| EnvVar { key: k, value: v }),
);
// Produce inputs. As we refer to the contents here, not just plain store path strings,
// we need to perform lookups.
// FUTUREWORK: should we also model input_derivations and input_sources with StorePath?
let mut inputs: Vec<Node> = Vec::new();
// since Derivation is validated, we know input sources can be parsed.
for input_source in derivation.input_sources.iter() {
let sp = StorePathRef::from_absolute_path(input_source.as_bytes())
.expect("invalid input source path");
let node = fn_input_sources_to_node(sp);
inputs.push(node);
}
// since Derivation is validated, we know input derivations can be parsed.
for (input_derivation, output_names) in derivation.input_derivations.iter() {
let sp = StorePathRef::from_absolute_path(input_derivation.as_bytes())
.expect("invalid input derivation path");
let output_names: Vec<&str> = output_names.iter().map(|e| e.as_str()).collect();
let node = fn_input_drvs_to_node(sp, output_names.as_slice());
inputs.push(node);
}
// validate all nodes are actually Some.
debug_assert!(
inputs.iter().all(|input| input.node.is_some()),
"input nodes must be some"
);
// sort inputs by their name
inputs.sort_by(|a, b| {
a.node
.as_ref()
.unwrap()
.get_name()
.cmp(b.node.as_ref().unwrap().get_name())
});
// Produce constraints. We currently only put platform in here.
// Maybe more things need to be added here in the future.
let constraints = Some(BuildConstraints {
system: derivation.system.clone(),
min_memory: 0,
available_ro_paths: vec![],
});
BuildRequest {
command_args,
outputs,
environment_vars,
inputs,
constraints,
}
}
#[cfg(test)]
mod test {
use bytes::Bytes;
use nix_compat::derivation::Derivation;
use tvix_build::proto::{
build_request::{BuildConstraints, EnvVar},
BuildRequest,
};
use tvix_castore::{
fixtures::DUMMY_DIGEST,
proto::{DirectoryNode, Node},
};
use crate::tvix_build::NIX_ENVIRONMENT_VARS;
use super::derivation_to_build_request;
use lazy_static::lazy_static;
lazy_static! {
static ref INPUT_NODE_FOO: Node = Node {
node: Some(tvix_castore::proto::node::Node::Directory(DirectoryNode {
name: Bytes::from("mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar"),
digest: DUMMY_DIGEST.clone().into(),
size: 42,
})),
};
}
#[test]
fn test_derivation_to_build_request() {
let aterm_bytes = include_bytes!("tests/ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv");
let derivation = Derivation::from_aterm_bytes(aterm_bytes).expect("must parse");
let build_request = derivation_to_build_request(
&derivation,
|_| unreachable!(),
|input_drv, output_names| {
// expected to be called with ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv only
if input_drv.to_string() != "ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv" {
panic!("called with unexpected input_drv: {}", input_drv);
}
// expect to be called with ["out"]
if output_names != ["out"] {
panic!("called with unexpected output_names: {:?}", output_names);
}
// all good, reply with INPUT_NODE_FOO
INPUT_NODE_FOO.clone()
},
);
let mut expected_environment_vars = vec![
EnvVar {
key: "bar".to_string(),
value: Bytes::from("/nix/store/mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar"),
},
EnvVar {
key: "builder".to_string(),
value: Bytes::from(":"),
},
EnvVar {
key: "name".to_string(),
value: Bytes::from("foo"),
},
EnvVar {
key: "out".to_string(),
value: Bytes::from("/nix/store/fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo"),
},
EnvVar {
key: "system".to_string(),
value: Bytes::from(":"),
},
];
expected_environment_vars.extend(NIX_ENVIRONMENT_VARS.iter().map(|(k, v)| EnvVar {
key: k.to_string(),
value: Bytes::from_static(v.as_bytes()),
}));
expected_environment_vars.sort_unstable_by_key(|e| e.key.to_owned());
assert_eq!(
BuildRequest {
command_args: vec![":".to_string()],
outputs: vec!["fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo".to_string()],
environment_vars: expected_environment_vars,
inputs: vec![INPUT_NODE_FOO.clone()],
constraints: Some(BuildConstraints {
system: derivation.system.clone(),
min_memory: 0,
available_ro_paths: vec![],
}),
},
build_request
);
}
}