From 3196fe0143b6ff729c177fa5d17fa03c9e9627c9 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Fri, 3 Nov 2023 13:34:37 +0200 Subject: [PATCH] refactor(tvix): move tvix glue code into glue crate There's various bits and pieces in tvix-cli that use both the store and evaluator, as well as nix-compat. For example, builtins.derivation, as well as the reference scanning implementation. This "glue code" currently isn't accessible from anywhere else, but it'd be very useful if it were. Move it out into a `glue` crate, and make `tvix-cli` a consumer of it. All the KnownPaths setup and passing around, as well as NIX_PATH handling is also something that should probably be moved into the glue crate as well, but that's something left for a future CL. Change-Id: I080ed3d1825ab23790666486840f301f00856277 Reviewed-on: https://cl.tvl.fyi/c/depot/+/9908 Autosubmit: flokli Tested-by: BuildkiteCI Reviewed-by: raitobezarius --- tvix/Cargo.lock | 17 ++++++ tvix/Cargo.nix | 71 +++++++++++++++++++++++++ tvix/Cargo.toml | 1 + tvix/cli/Cargo.toml | 1 + tvix/cli/src/main.rs | 26 +++------ tvix/glue/Cargo.toml | 20 +++++++ tvix/glue/default.nix | 5 ++ tvix/{cli => glue}/src/.skip-subtree | 0 tvix/{cli => glue}/src/derivation.nix | 0 tvix/{cli => glue}/src/derivation.rs | 0 tvix/{cli => glue}/src/errors.rs | 0 tvix/{cli => glue}/src/fetchurl.nix | 0 tvix/{cli => glue}/src/known_paths.rs | 0 tvix/glue/src/lib.rs | 28 ++++++++++ tvix/{cli => glue}/src/refscan.rs | 0 tvix/{cli => glue}/src/tvix_io.rs | 7 +-- tvix/{cli => glue}/src/tvix_store_io.rs | 0 17 files changed, 154 insertions(+), 22 deletions(-) create mode 100644 tvix/glue/Cargo.toml create mode 100644 tvix/glue/default.nix rename tvix/{cli => glue}/src/.skip-subtree (100%) rename tvix/{cli => glue}/src/derivation.nix (100%) rename tvix/{cli => glue}/src/derivation.rs (100%) rename tvix/{cli => glue}/src/errors.rs (100%) rename tvix/{cli => glue}/src/fetchurl.nix (100%) rename tvix/{cli => glue}/src/known_paths.rs (100%) create mode 100644 tvix/glue/src/lib.rs rename tvix/{cli => glue}/src/refscan.rs (100%) rename tvix/{cli => glue}/src/tvix_io.rs (94%) rename tvix/{cli => glue}/src/tvix_store_io.rs (100%) diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index 4748c3abc..9dd66f83f 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -2899,6 +2899,7 @@ dependencies = [ "tracing", "tvix-castore", "tvix-eval", + "tvix-glue", "tvix-store", "wu-manber", ] @@ -2945,6 +2946,22 @@ dependencies = [ "tvix-eval", ] +[[package]] +name = "tvix-glue" +version = "0.1.0" +dependencies = [ + "bytes", + "nix-compat", + "test-case", + "thiserror", + "tokio", + "tracing", + "tvix-castore", + "tvix-eval", + "tvix-store", + "wu-manber", +] + [[package]] name = "tvix-serde" version = "0.1.0" diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index dab411ebb..ad3eac33b 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -93,6 +93,16 @@ rec { # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; }; + "tvix-glue" = rec { + packageId = "tvix-glue"; + build = internal.buildRustCrateWithFeatures { + packageId = "tvix-glue"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; "tvix-serde" = rec { packageId = "tvix-serde"; build = internal.buildRustCrateWithFeatures { @@ -8730,6 +8740,10 @@ rec { name = "tvix-eval"; packageId = "tvix-eval"; } + { + name = "tvix-glue"; + packageId = "tvix-glue"; + } { name = "tvix-store"; packageId = "tvix-store"; @@ -8920,6 +8934,63 @@ rec { } ]; + }; + "tvix-glue" = rec { + crateName = "tvix-glue"; + version = "0.1.0"; + edition = "2021"; + # We can't filter paths with references in Nix 2.4 + # See https://github.com/NixOS/nix/issues/5410 + src = + if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion)) + then lib.cleanSourceWith { filter = sourceFilter; src = ./glue; } + else ./glue; + dependencies = [ + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "nix-compat"; + packageId = "nix-compat"; + } + { + name = "thiserror"; + packageId = "thiserror"; + } + { + name = "tokio"; + packageId = "tokio"; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "tvix-castore"; + packageId = "tvix-castore"; + } + { + name = "tvix-eval"; + packageId = "tvix-eval"; + } + { + name = "tvix-store"; + packageId = "tvix-store"; + usesDefaultFeatures = false; + } + { + name = "wu-manber"; + packageId = "wu-manber"; + } + ]; + devDependencies = [ + { + name = "test-case"; + packageId = "test-case"; + } + ]; + }; "tvix-serde" = rec { crateName = "tvix-serde"; diff --git a/tvix/Cargo.toml b/tvix/Cargo.toml index 645aac362..23486a921 100644 --- a/tvix/Cargo.toml +++ b/tvix/Cargo.toml @@ -23,6 +23,7 @@ members = [ "cli", "eval", "eval/builtin-macros", + "glue", "nix_cli", "nix-compat", "serde", diff --git a/tvix/cli/Cargo.toml b/tvix/cli/Cargo.toml index 8bdd26ff4..7a27c148f 100644 --- a/tvix/cli/Cargo.toml +++ b/tvix/cli/Cargo.toml @@ -12,6 +12,7 @@ nix-compat = { path = "../nix-compat" } tvix-castore = { path = "../castore" } tvix-store = { path = "../store", default-features = false, features = []} tvix-eval = { path = "../eval" } +tvix-glue = { path = "../glue" } bytes = "1.4.0" clap = { version = "4.0", features = ["derive", "env"] } dirs = "4.0.0" diff --git a/tvix/cli/src/main.rs b/tvix/cli/src/main.rs index ebcfe4b80..1ffa2be25 100644 --- a/tvix/cli/src/main.rs +++ b/tvix/cli/src/main.rs @@ -1,24 +1,18 @@ -mod derivation; -mod errors; -mod known_paths; -mod refscan; -mod tvix_io; -mod tvix_store_io; - use std::cell::RefCell; use std::rc::Rc; use std::sync::Arc; use std::{fs, path::PathBuf}; +use tvix_glue::add_derivation_builtins; +use tvix_glue::known_paths::KnownPaths; use clap::Parser; -use known_paths::KnownPaths; use rustyline::{error::ReadlineError, Editor}; use tvix_castore::blobservice::MemoryBlobService; use tvix_castore::directoryservice::MemoryDirectoryService; use tvix_eval::observer::{DisassemblingObserver, TracingObserver}; use tvix_eval::Value; +use tvix_glue::tvix_store_io::TvixStoreIO; use tvix_store::pathinfoservice::MemoryPathInfoService; -use tvix_store_io::TvixStoreIO; #[derive(Parser)] struct Args { @@ -69,7 +63,6 @@ struct Args { /// evaluation succeeded. fn interpret(code: &str, path: Option, args: &Args, explain: bool) -> bool { let mut eval = tvix_eval::Evaluation::new_impure(code, path); - let known_paths: Rc> = Default::default(); eval.strict = args.strict; @@ -80,9 +73,11 @@ fn interpret(code: &str, path: Option, args: &Args, explain: bool) -> b directory_service.clone(), )); - let tokio_runtime = tokio::runtime::Runtime::new().unwrap(); + let known_paths: Rc> = Default::default(); + add_derivation_builtins(&mut eval, known_paths.clone()); - eval.io_handle = Box::new(tvix_io::TvixIO::new( + let tokio_runtime = tokio::runtime::Runtime::new().unwrap(); + eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new( known_paths.clone(), TvixStoreIO::new( blob_service, @@ -100,13 +95,6 @@ fn interpret(code: &str, path: Option, args: &Args, explain: bool) -> b .map(|p| format!("nix=/__corepkgs__:{}", p)) .or_else(|| Some("nix=/__corepkgs__".to_string())); - eval.builtins - .extend(derivation::derivation_builtins(known_paths)); - - // Add the actual `builtins.derivation` from compiled Nix code - eval.src_builtins - .push(("derivation", include_str!("derivation.nix"))); - let source_map = eval.source_map(); let result = { let mut compiler_observer = diff --git a/tvix/glue/Cargo.toml b/tvix/glue/Cargo.toml new file mode 100644 index 000000000..4ebfda870 --- /dev/null +++ b/tvix/glue/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "tvix-glue" +version = "0.1.0" +edition = "2021" + +[dependencies] +nix-compat = { path = "../nix-compat" } +tvix-eval = { path = "../eval" } +tvix-castore = { path = "../castore" } +tvix-store = { path = "../store", default-features = false, features = []} +bytes = "1.4.0" +tracing = "0.1.37" +tokio = "1.28.0" +thiserror = "1.0.38" + +[dependencies.wu-manber] +git = "https://github.com/tvlfyi/wu-manber.git" + +[dev-dependencies] +test-case = "2.2.2" diff --git a/tvix/glue/default.nix b/tvix/glue/default.nix new file mode 100644 index 000000000..975b0a5a7 --- /dev/null +++ b/tvix/glue/default.nix @@ -0,0 +1,5 @@ +{ depot, ... }: + +(depot.tvix.crates.workspaceMembers.tvix-glue.build.override { + runTests = true; +}) diff --git a/tvix/cli/src/.skip-subtree b/tvix/glue/src/.skip-subtree similarity index 100% rename from tvix/cli/src/.skip-subtree rename to tvix/glue/src/.skip-subtree diff --git a/tvix/cli/src/derivation.nix b/tvix/glue/src/derivation.nix similarity index 100% rename from tvix/cli/src/derivation.nix rename to tvix/glue/src/derivation.nix diff --git a/tvix/cli/src/derivation.rs b/tvix/glue/src/derivation.rs similarity index 100% rename from tvix/cli/src/derivation.rs rename to tvix/glue/src/derivation.rs diff --git a/tvix/cli/src/errors.rs b/tvix/glue/src/errors.rs similarity index 100% rename from tvix/cli/src/errors.rs rename to tvix/glue/src/errors.rs diff --git a/tvix/cli/src/fetchurl.nix b/tvix/glue/src/fetchurl.nix similarity index 100% rename from tvix/cli/src/fetchurl.nix rename to tvix/glue/src/fetchurl.nix diff --git a/tvix/cli/src/known_paths.rs b/tvix/glue/src/known_paths.rs similarity index 100% rename from tvix/cli/src/known_paths.rs rename to tvix/glue/src/known_paths.rs diff --git a/tvix/glue/src/lib.rs b/tvix/glue/src/lib.rs new file mode 100644 index 000000000..acb81d314 --- /dev/null +++ b/tvix/glue/src/lib.rs @@ -0,0 +1,28 @@ +use std::{cell::RefCell, rc::Rc}; + +use known_paths::KnownPaths; + +pub mod derivation; +pub mod errors; +pub mod known_paths; +pub mod refscan; +pub mod tvix_io; +pub mod tvix_store_io; + +/// Adds derivation-related builtins to the passed [tvix_eval::Evaluation]. +/// +/// These are `derivation` and `derivationStrict`. +/// +/// As they need to interact with `known_paths`, we also need to pass in +/// `known_paths`. +pub fn add_derivation_builtins( + eval: &mut tvix_eval::Evaluation, + known_paths: Rc>, +) { + eval.builtins + .extend(derivation::derivation_builtins(known_paths)); + + // Add the actual `builtins.derivation` from compiled Nix code + eval.src_builtins + .push(("derivation", include_str!("derivation.nix"))); +} diff --git a/tvix/cli/src/refscan.rs b/tvix/glue/src/refscan.rs similarity index 100% rename from tvix/cli/src/refscan.rs rename to tvix/glue/src/refscan.rs diff --git a/tvix/cli/src/tvix_io.rs b/tvix/glue/src/tvix_io.rs similarity index 94% rename from tvix/cli/src/tvix_io.rs rename to tvix/glue/src/tvix_io.rs index 74f91138b..caadbeb5e 100644 --- a/tvix/cli/src/tvix_io.rs +++ b/tvix/glue/src/tvix_io.rs @@ -8,15 +8,16 @@ //! otherwise fundamental features like nixpkgs bootstrapping and hash //! calculation will not work. -use crate::KnownPaths; use std::cell::RefCell; use std::io; use std::path::{Path, PathBuf}; use std::rc::Rc; use tvix_eval::{EvalIO, FileType}; +use crate::known_paths::KnownPaths; + // TODO: Merge this together with TvixStoreIO? -pub(crate) struct TvixIO { +pub struct TvixIO { /// Ingested paths must be reported to this known paths tracker /// for accurate build reference scanning. known_paths: Rc>, @@ -26,7 +27,7 @@ pub(crate) struct TvixIO { } impl TvixIO { - pub(crate) fn new(known_paths: Rc>, actual: T) -> Self { + pub fn new(known_paths: Rc>, actual: T) -> Self { Self { known_paths, actual, diff --git a/tvix/cli/src/tvix_store_io.rs b/tvix/glue/src/tvix_store_io.rs similarity index 100% rename from tvix/cli/src/tvix_store_io.rs rename to tvix/glue/src/tvix_store_io.rs