mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/glue): contextful derivation

Browse source 
We calculate the input context by performing the union of context
over all input of the derivation.

Then, we just pass the rest of it to the remaining machinery.

Finally, we re-emit an `outPath` and a `drvPath` containing the expected
contexts.

Change-Id: I74905fb258b5bee8b08d1208c9eb87f51b92a890
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10436
Autosubmit: raitobezarius <tvl@lahfa.xyz>
Reviewed-by: tazjin <tazjin@tvl.su>
Tested-by: BuildkiteCI
This commit is contained in:
Ryan Lahfa 2023-12-26 02:03:05 +01:00 committed by clbot
parent 15309c7deb
commit 099ca6b7c0
5 changed files with 86 additions and 223 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
tvix/cli/src/main.rs
View file

@ -91,15 +91,12 @@ fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> b
let known_paths: Rc<RefCell<KnownPaths>> = Default::default();
add_derivation_builtins(&mut eval, known_paths.clone());
configure_nix_path(&mut eval, &args.nix_search_path);
eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new(
known_paths,
TvixStoreIO::new(
blob_service,
directory_service,
path_info_service.into(), // we need an Arc<_> here.
tokio_runtime.handle().clone(),
),
));
eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new(TvixStoreIO::new(
blob_service,
directory_service,
path_info_service.into(), // we need an Arc<_> here.
tokio_runtime.handle().clone(),
)));
let source_map = eval.source_map();
let result = {

17
tvix/glue/benches/eval.rs
View file

@ -29,7 +29,7 @@ fn interpret(code: &str) {
let mut eval = tvix_eval::Evaluation::new_impure();
let known_paths: Rc<RefCell<KnownPaths>> = Default::default();
add_derivation_builtins(&mut eval, known_paths.clone());
add_derivation_builtins(&mut eval, known_paths);
configure_nix_path(
&mut eval,
// The benchmark requires TVIX_BENCH_NIX_PATH to be set, so barf out
@ -37,15 +37,12 @@ fn interpret(code: &str) {
&Some(env::var("TVIX_BENCH_NIX_PATH").expect("TVIX_BENCH_NIX_PATH must be set")),
);
eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new(
known_paths.clone(),
TvixStoreIO::new(
BLOB_SERVICE.clone(),
DIRECTORY_SERVICE.clone(),
PATH_INFO_SERVICE.clone(),
TOKIO_RUNTIME.handle().clone(),
),
));
eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new(TvixStoreIO::new(
BLOB_SERVICE.clone(),
DIRECTORY_SERVICE.clone(),
PATH_INFO_SERVICE.clone(),
TOKIO_RUNTIME.handle().clone(),
)));
let result = eval.evaluate(code, None);

127
tvix/glue/src/builtins/derivation.rs
View file

@ -1,6 +1,6 @@
//! Implements `builtins.derivation`, the core of what makes Nix build packages.
use crate::builtins::DerivationError;
use crate::known_paths::{KnownPaths, PathKind, PathName};
use crate::known_paths::KnownPaths;
use nix_compat::derivation::{Derivation, Output};
use nix_compat::nixhash;
use std::cell::RefCell;
@ -9,7 +9,8 @@ use std::rc::Rc;
use tvix_eval::builtin_macros::builtins;
use tvix_eval::generators::{self, emit_warning_kind, GenCo};
use tvix_eval::{
AddContext, CatchableErrorKind, CoercionKind, ErrorKind, NixAttrs, NixList, Value, WarningKind,
AddContext, CatchableErrorKind, CoercionKind, ErrorKind, NixAttrs, NixContext,
NixContextElement, NixList, Value, WarningKind,
};
// Constants used for strangely named fields in derivation inputs.
@ -46,20 +47,15 @@ async fn populate_outputs(
}
/// Populate the inputs of a derivation from the build references
/// found when scanning the derivation's parameters.
fn populate_inputs<I: IntoIterator<Item = PathName>>(
drv: &mut Derivation,
known_paths: &KnownPaths,
references: I,
) {
for reference in references.into_iter() {
let reference = &known_paths[&reference];
match &reference.kind {
PathKind::Plain => {
drv.input_sources.insert(reference.path.clone());
/// found when scanning the derivation's parameters and extracting their contexts.
fn populate_inputs(drv: &mut Derivation, full_context: NixContext) {
for element in full_context.iter() {
match element {
NixContextElement::Plain(source) => {
drv.input_sources.insert(source.clone());
}
PathKind::Output { name, derivation } => {
NixContextElement::Single { name, derivation } => {
match drv.input_derivations.entry(derivation.clone()) {
btree_map::Entry::Vacant(entry) => {
entry.insert(BTreeSet::from([name.clone()]));
@ -71,16 +67,12 @@ fn populate_inputs<I: IntoIterator<Item = PathName>>(
}
}
PathKind::Derivation { output_names } => {
match drv.input_derivations.entry(reference.path.clone()) {
btree_map::Entry::Vacant(entry) => {
entry.insert(output_names.clone());
}
btree_map::Entry::Occupied(mut entry) => {
entry.get_mut().extend(output_names.clone().into_iter());
}
}
NixContextElement::Derivation(_drv_path) => {
// This is a hard one, it means that
// we are depending on a drvPath of ourselves
// *or* another derivation's drvPath.
// What to do here?
panic!("please do not depend on drvPath, I have 2 hours of sleep in blood");
}
}
}
@ -232,6 +224,7 @@ pub(crate) mod derivation_builtins {
use super::*;
use nix_compat::store_path::hash_placeholder;
use tvix_eval::generators::Gen;
use tvix_eval::{NixContext, NixContextElement, NixString};
#[builtin("placeholder")]
async fn builtin_placeholder(co: GenCo, input: Value) -> Result<Value, ErrorKind> {
@ -299,15 +292,31 @@ pub(crate) mod derivation_builtins {
Ok(Ok(None))
}
let mut input_context = NixContext::new();
for (name, value) in input.clone().into_iter_sorted() {
let value = generators::request_force(&co, value).await;
if ignore_nulls && matches!(value, Value::Null) {
continue;
}
match strong_importing_coerce_to_string(&co, value.clone()).await? {
match generators::request_string_coerce(
&co,
value.clone(),
CoercionKind {
strong: true,
import_paths: true,
},
)
.await
{
Err(cek) => return Ok(Value::Catchable(cek)),
Ok(val_str) => {
// Learn about this derivation references
// by looking at its context.
input_context.mimic(&val_str);
let val_str = val_str.as_str().to_string();
// handle_derivation_parameters tells us whether the
// argument should be added to the environment; continue
// to the next one otherwise
@ -370,21 +379,6 @@ pub(crate) mod derivation_builtins {
}
}
// Scan references in relevant attributes to detect any build-references.
let references = {
let state = state.borrow();
if state.is_empty() {
// skip reference scanning, create an empty result
Default::default()
} else {
let mut refscan = state.reference_scanner();
drv.arguments.iter().for_each(|s| refscan.scan(s));
drv.environment.values().for_each(|s| refscan.scan(s));
refscan.scan(&drv.builder);
refscan.finalise()
}
};
// Each output name needs to exist in the environment, at this
// point initialised as an empty string because that is the
// way of Golang ;)
@ -398,8 +392,8 @@ pub(crate) mod derivation_builtins {
}
}
populate_inputs(&mut drv, input_context);
let mut known_paths = state.borrow_mut();
populate_inputs(&mut drv, &known_paths, references);
// At this point, derivation fields are fully populated from
// eval data structures.
@ -428,25 +422,35 @@ pub(crate) mod derivation_builtins {
&derivation_or_fod_hash_final,
);
// mark all the new paths as known
let output_names: Vec<String> = drv.outputs.keys().map(Clone::clone).collect();
known_paths.drv(derivation_path.to_absolute_path(), &output_names);
for (output_name, output) in &drv.outputs {
known_paths.output(
&output.path,
output_name,
derivation_path.to_absolute_path(),
);
}
let mut new_attrs: Vec<(String, String)> = drv
let mut new_attrs: Vec<(String, NixString)> = drv
.outputs
.into_iter()
.map(|(name, output)| (name, output.path))
.map(|(name, output)| {
(
name.clone(),
(
output.path,
Some(
NixContextElement::Single {
name,
derivation: derivation_path.to_absolute_path(),
}
.into(),
),
)
.into(),
)
})
.collect();
new_attrs.push(("drvPath".to_string(), derivation_path.to_absolute_path()));
new_attrs.push((
"drvPath".to_string(),
(
derivation_path.to_absolute_path(),
Some(NixContextElement::Derivation(derivation_path.to_absolute_path()).into()),
)
.into(),
));
Ok(Value::Attrs(Box::new(NixAttrs::from_iter(
new_attrs.into_iter(),
@ -454,12 +458,7 @@ pub(crate) mod derivation_builtins {
}
#[builtin("toFile")]
async fn builtin_to_file(
state: Rc<RefCell<KnownPaths>>,
co: GenCo,
name: Value,
content: Value,
) -> Result<Value, ErrorKind> {
async fn builtin_to_file(co: GenCo, name: Value, content: Value) -> Result<Value, ErrorKind> {
let name = name
.to_str()
.context("evaluating the `name` parameter of builtins.toFile")?;
@ -482,11 +481,11 @@ pub(crate) mod derivation_builtins {
.map_err(DerivationError::InvalidDerivation)?
.to_absolute_path();
state.borrow_mut().plain(&path);
let context: NixContext = NixContextElement::Plain(path.clone()).into();
// TODO: actually persist the file in the store at that path ...
Ok(Value::String(path.into()))
Ok(Value::String(NixString::new_context_from(context, &path)))
}
}

131
tvix/glue/src/known_paths.rs
View file

@ -11,12 +11,9 @@
//! Please see //tvix/eval/docs/build-references.md for more
//! information.
use crate::refscan::{ReferenceScanner, STORE_PATH_LEN};
use crate::refscan::STORE_PATH_LEN;
use nix_compat::nixhash::NixHash;
use std::{
collections::{hash_map, BTreeSet, HashMap},
ops::Index,
};
use std::collections::{BTreeSet, HashMap};
#[derive(Debug, PartialEq)]
pub enum PathKind {
@ -36,12 +33,6 @@ pub struct KnownPath {
pub kind: PathKind,
}
impl KnownPath {
fn new(path: String, kind: PathKind) -> Self {
KnownPath { path, kind }
}
}
/// Internal struct to prevent accidental leaks of the truncated path
/// names.
#[repr(transparent)]
@ -54,6 +45,12 @@ impl From<&str> for PathName {
}
}
impl From<String> for PathName {
fn from(s: String) -> Self {
s.as_str().into()
}
}
/// This instance is required to pass PathName instances as needles to
/// the reference scanner.
impl AsRef<[u8]> for PathName {
@ -64,125 +61,13 @@ impl AsRef<[u8]> for PathName {
#[derive(Debug, Default)]
pub struct KnownPaths {
/// All known paths, keyed by a truncated version of their store
/// path used for reference scanning.
paths: HashMap<PathName, KnownPath>,
/// All known derivation or FOD hashes.
///
/// Keys are derivation paths, values is the NixHash.
derivation_or_fod_hashes: HashMap<String, NixHash>,
}
impl Index<&PathName> for KnownPaths {
type Output = KnownPath;
fn index(&self, index: &PathName) -> &Self::Output {
&self.paths[index]
}
}
impl KnownPaths {
fn insert_path(&mut self, path: String, path_kind: PathKind) {
match self.paths.entry(path.as_str().into()) {
hash_map::Entry::Vacant(entry) => {
entry.insert(KnownPath::new(path, path_kind));
}
hash_map::Entry::Occupied(mut entry) => {
match (path_kind, &mut entry.get_mut().kind) {
// These variant combinations require no "merging action".
(PathKind::Plain, PathKind::Plain) => (),
#[allow(unused_variables)]
(
PathKind::Output {
name: name1,
derivation: drv1,
},
PathKind::Output {
name: ref name2,
derivation: ref drv2,
},
) => {
#[cfg(debug_assertions)]
{
if &name1 != name2 {
panic!(
"inserted path {} with two different names: {} and {}",
path, name1, name2
);
}
if &drv1 != drv2 {
panic!(
"inserted path {} with two different derivations: {} and {}",
path, drv1, drv2
);
}
}
}
(
PathKind::Derivation { output_names: new },
PathKind::Derivation {
output_names: ref mut old,
},
) => {
old.extend(new);
}
_ => panic!(
"path '{}' inserted twice with different types",
entry.key().0
),
};
}
};
}
/// Mark a plain path as known.
pub fn plain<S: ToString>(&mut self, path: S) {
self.insert_path(path.to_string(), PathKind::Plain);
}
/// Mark a derivation as known.
pub fn drv<P: ToString, O: ToString>(&mut self, path: P, outputs: &[O]) {
self.insert_path(
path.to_string(),
PathKind::Derivation {
output_names: outputs.iter().map(ToString::to_string).collect(),
},
);
}
/// Mark a derivation output path as known.
pub fn output<P: ToString, N: ToString, D: ToString>(
&mut self,
output_path: P,
name: N,
drv_path: D,
) {
self.insert_path(
output_path.to_string(),
PathKind::Output {
name: name.to_string(),
derivation: drv_path.to_string(),
},
);
}
/// Checks whether there are any known paths. If not, a reference
/// scanner can not be created.
pub fn is_empty(&self) -> bool {
self.paths.is_empty()
}
/// Create a reference scanner from the current set of known paths.
pub fn reference_scanner(&self) -> ReferenceScanner<PathName> {
let candidates = self.paths.keys().map(Clone::clone).collect();
ReferenceScanner::new(candidates)
}
/// Fetch the opaque "hash derivation modulo" for a given derivation path.
pub fn get_hash_derivation_modulo(&self, drv_path: &str) -> NixHash {
// TODO: we rely on an invariant that things *should* have

19
tvix/glue/src/tvix_io.rs
View file

@ -8,30 +8,19 @@
//! otherwise fundamental features like nixpkgs bootstrapping and hash
//! calculation will not work.
use std::cell::RefCell;
use std::io;
use std::path::{Path, PathBuf};
use std::rc::Rc;
use tvix_eval::{EvalIO, FileType};
use crate::known_paths::KnownPaths;
// TODO: Merge this together with TvixStoreIO?
pub struct TvixIO<T: EvalIO> {
/// Ingested paths must be reported to this known paths tracker
/// for accurate build reference scanning.
known_paths: Rc<RefCell<KnownPaths>>,
// Actual underlying [EvalIO] implementation.
actual: T,
}
impl<T: EvalIO> TvixIO<T> {
pub fn new(known_paths: Rc<RefCell<KnownPaths>>, actual: T) -> Self {
Self {
known_paths,
actual,
}
pub fn new(actual: T) -> Self {
Self { actual }
}
}
@ -42,10 +31,6 @@ impl<T: EvalIO> EvalIO for TvixIO<T> {
fn import_path(&self, path: &Path) -> io::Result<PathBuf> {
let imported_path = self.actual.import_path(path)?;
self.known_paths
.borrow_mut()
.plain(imported_path.to_string_lossy());
Ok(imported_path)
}