feat(tvix/cli): Add derivation file dumping functionality

Provides a derivation file dumping functionality for tvix-cli that can
be used when passing the --drv-dumpdir CLI arg to tvix-cli.

This will dump all the known derivation files into the specified
directory, making it easier to debug derivation divergences between Tvix
generated drvs and the drvs generated by Nix.

Supersedes: https://cl.tvl.fyi/c/depot/+/11265

Change-Id: I0e10b26eba22032b84ac543af0d4150ad87aed3e
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12192
Autosubmit: Ilan Joselevich <personal@ilanjoselevich.com>
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>
This commit is contained in:
Ilan Joselevich 2024-08-12 00:02:16 +03:00 committed by clbot
parent fd64df0511
commit d2a80dda88
4 changed files with 49 additions and 2 deletions

View file

@ -72,4 +72,15 @@ pub struct Args {
#[arg(long, env, default_value = "dummy://")] #[arg(long, env, default_value = "dummy://")]
pub build_service_addr: String, pub build_service_addr: String,
/// An optional path in which Derivations encountered during evaluation
/// are dumped into, after evaluation. If it doesn't exist, the directory is created.
///
/// Files dumped there are named like they would show up in `/nix/store`,
/// if produced by Nix. Existing files are not overwritten.
///
/// This is only for debugging and diffing purposes for post-eval inspection;
/// Tvix does not read from these.
#[clap(long)]
pub drv_dumpdir: Option<PathBuf>,
} }

View file

@ -112,7 +112,7 @@ pub fn evaluate(
eval_builder = eval_builder.add_builtins(impure_builtins()); eval_builder = eval_builder.add_builtins(impure_builtins());
eval_builder = add_derivation_builtins(eval_builder, Rc::clone(&tvix_store_io)); eval_builder = add_derivation_builtins(eval_builder, Rc::clone(&tvix_store_io));
eval_builder = add_fetcher_builtins(eval_builder, Rc::clone(&tvix_store_io)); eval_builder = add_fetcher_builtins(eval_builder, Rc::clone(&tvix_store_io));
eval_builder = add_import_builtins(eval_builder, tvix_store_io); eval_builder = add_import_builtins(eval_builder, Rc::clone(&tvix_store_io));
} }
}; };
eval_builder = configure_nix_path(eval_builder, &args.nix_search_path); eval_builder = configure_nix_path(eval_builder, &args.nix_search_path);
@ -173,6 +173,21 @@ pub fn evaluate(
} }
} }
if let Some(dumpdir) = &args.drv_dumpdir {
// Dump all known derivations files to `dumpdir`.
std::fs::create_dir_all(dumpdir).expect("failed to create drv dumpdir");
tvix_store_io
.known_paths
.borrow()
.get_derivations()
// Skip already dumped derivations.
.filter(|(drv_path, _)| !dumpdir.join(drv_path.to_string()).exists())
.for_each(|(drv_path, drv)| {
std::fs::write(dumpdir.join(drv_path.to_string()), drv.to_aterm_bytes())
.expect("failed to write drv to dumpdir");
})
}
Ok(EvalResult { Ok(EvalResult {
globals, globals,
value: result.value, value: result.value,

View file

@ -129,6 +129,11 @@ impl KnownPaths {
.get(output_path) .get(output_path)
.map(|(name, fetch)| (name.to_owned(), fetch.to_owned())) .map(|(name, fetch)| (name.to_owned(), fetch.to_owned()))
} }
/// Returns an iterator over all known derivations and their store path.
pub fn get_derivations(&self) -> impl Iterator<Item = (&StorePath, &Derivation)> {
self.derivations.iter().map(|(k, v)| (k, &v.1))
}
} }
#[cfg(test)] #[cfg(test)]
@ -269,5 +274,21 @@ mod tests {
); );
} }
#[test]
fn get_derivations_working() {
let mut known_paths = KnownPaths::default();
// Add BAR_DRV
known_paths.add_derivation(BAR_DRV_PATH.clone(), BAR_DRV.clone());
// We should be able to find BAR_DRV_PATH and BAR_DRV as a pair in get_derivations.
assert_eq!(
Some((&BAR_DRV_PATH.clone(), &BAR_DRV.clone())),
known_paths
.get_derivations()
.find(|(s, d)| (*s, *d) == (&BAR_DRV_PATH, &BAR_DRV))
);
}
// TODO: add test panicking about missing digest // TODO: add test panicking about missing digest
} }

View file

@ -67,7 +67,7 @@ pub struct TvixStoreIO {
>, >,
// Paths known how to produce, by building or fetching. // Paths known how to produce, by building or fetching.
pub(crate) known_paths: RefCell<KnownPaths>, pub known_paths: RefCell<KnownPaths>,
} }
impl TvixStoreIO { impl TvixStoreIO {