refactor(tvix): move tvix glue code into glue crate

There's various bits and pieces in tvix-cli that use both the store and
evaluator, as well as nix-compat. For example, builtins.derivation, as
well as the reference scanning implementation.

This "glue code" currently isn't accessible from anywhere else, but it'd
be very useful if it were.

Move it out into a `glue` crate, and make `tvix-cli` a consumer of it.

All the KnownPaths setup and passing around, as well as NIX_PATH
handling is also something that should probably be moved into the glue
crate as well, but that's something left for a future CL.

Change-Id: I080ed3d1825ab23790666486840f301f00856277
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9908
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: raitobezarius <tvl@lahfa.xyz>

tvix/Cargo.lock

@@ -2899,6 +2899,7 @@ dependencies = [
  "tracing",
  "tvix-castore",
  "tvix-eval",
+ "tvix-glue",
  "tvix-store",
  "wu-manber",
 ]
@@ -2945,6 +2946,22 @@ dependencies = [
  "tvix-eval",
 ]
 
+[[package]]
+name = "tvix-glue"
+version = "0.1.0"
+dependencies = [
+ "bytes",
+ "nix-compat",
+ "test-case",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "tvix-castore",
+ "tvix-eval",
+ "tvix-store",
+ "wu-manber",
+]
+
 [[package]]
 name = "tvix-serde"
 version = "0.1.0"

tvix/Cargo.nix

@@ -93,6 +93,16 @@ rec {
       # File a bug if you depend on any for non-debug work!
       debug = internal.debugCrate { inherit packageId; };
     };
+    "tvix-glue" = rec {
+      packageId = "tvix-glue";
+      build = internal.buildRustCrateWithFeatures {
+        packageId = "tvix-glue";
+      };
+
+      # Debug support which might change between releases.
+      # File a bug if you depend on any for non-debug work!
+      debug = internal.debugCrate { inherit packageId; };
+    };
     "tvix-serde" = rec {
       packageId = "tvix-serde";
       build = internal.buildRustCrateWithFeatures {
@@ -8730,6 +8740,10 @@ rec {
             name = "tvix-eval";
             packageId = "tvix-eval";
           }
+          {
+            name = "tvix-glue";
+            packageId = "tvix-glue";
+          }
           {
             name = "tvix-store";
             packageId = "tvix-store";
@@ -8920,6 +8934,63 @@ rec {
           }
         ];
 
+      };
+      "tvix-glue" = rec {
+        crateName = "tvix-glue";
+        version = "0.1.0";
+        edition = "2021";
+        # We can't filter paths with references in Nix 2.4
+        # See https://github.com/NixOS/nix/issues/5410
+        src =
+          if ((lib.versionOlder builtins.nixVersion "2.4pre20211007") || (lib.versionOlder "2.5" builtins.nixVersion))
+          then lib.cleanSourceWith { filter = sourceFilter; src = ./glue; }
+          else ./glue;
+        dependencies = [
+          {
+            name = "bytes";
+            packageId = "bytes";
+          }
+          {
+            name = "nix-compat";
+            packageId = "nix-compat";
+          }
+          {
+            name = "thiserror";
+            packageId = "thiserror";
+          }
+          {
+            name = "tokio";
+            packageId = "tokio";
+          }
+          {
+            name = "tracing";
+            packageId = "tracing";
+          }
+          {
+            name = "tvix-castore";
+            packageId = "tvix-castore";
+          }
+          {
+            name = "tvix-eval";
+            packageId = "tvix-eval";
+          }
+          {
+            name = "tvix-store";
+            packageId = "tvix-store";
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "wu-manber";
+            packageId = "wu-manber";
+          }
+        ];
+        devDependencies = [
+          {
+            name = "test-case";
+            packageId = "test-case";
+          }
+        ];
+
       };
       "tvix-serde" = rec {
         crateName = "tvix-serde";

tvix/Cargo.toml

@@ -23,6 +23,7 @@ members = [
   "cli",
   "eval",
   "eval/builtin-macros",
+  "glue",
   "nix_cli",
   "nix-compat",
   "serde",

tvix/cli/Cargo.toml

@@ -12,6 +12,7 @@ nix-compat = { path = "../nix-compat" }
 tvix-castore = { path = "../castore" }
 tvix-store = { path = "../store", default-features = false, features = []}
 tvix-eval = { path = "../eval" }
+tvix-glue = { path = "../glue" }
 bytes = "1.4.0"
 clap = { version = "4.0", features = ["derive", "env"] }
 dirs = "4.0.0"

tvix/cli/src/main.rs

@@ -1,24 +1,18 @@
-mod derivation;
-mod errors;
-mod known_paths;
-mod refscan;
-mod tvix_io;
-mod tvix_store_io;
-
 use std::cell::RefCell;
 use std::rc::Rc;
 use std::sync::Arc;
 use std::{fs, path::PathBuf};
+use tvix_glue::add_derivation_builtins;
+use tvix_glue::known_paths::KnownPaths;
 
 use clap::Parser;
-use known_paths::KnownPaths;
 use rustyline::{error::ReadlineError, Editor};
 use tvix_castore::blobservice::MemoryBlobService;
 use tvix_castore::directoryservice::MemoryDirectoryService;
 use tvix_eval::observer::{DisassemblingObserver, TracingObserver};
 use tvix_eval::Value;
+use tvix_glue::tvix_store_io::TvixStoreIO;
 use tvix_store::pathinfoservice::MemoryPathInfoService;
-use tvix_store_io::TvixStoreIO;
 
 #[derive(Parser)]
 struct Args {
@@ -69,7 +63,6 @@ struct Args {
 /// evaluation succeeded.
 fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> bool {
     let mut eval = tvix_eval::Evaluation::new_impure(code, path);
-    let known_paths: Rc<RefCell<KnownPaths>> = Default::default();
 
     eval.strict = args.strict;
 
@@ -80,9 +73,11 @@ fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> b
         directory_service.clone(),
     ));
 
-    let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
+    let known_paths: Rc<RefCell<KnownPaths>> = Default::default();
+    add_derivation_builtins(&mut eval, known_paths.clone());
 
-    eval.io_handle = Box::new(tvix_io::TvixIO::new(
+    let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
+    eval.io_handle = Box::new(tvix_glue::tvix_io::TvixIO::new(
         known_paths.clone(),
         TvixStoreIO::new(
             blob_service,
@@ -100,13 +95,6 @@ fn interpret(code: &str, path: Option<PathBuf>, args: &Args, explain: bool) -> b
         .map(|p| format!("nix=/__corepkgs__:{}", p))
         .or_else(|| Some("nix=/__corepkgs__".to_string()));
 
-    eval.builtins
-        .extend(derivation::derivation_builtins(known_paths));
-
-    // Add the actual `builtins.derivation` from compiled Nix code
-    eval.src_builtins
-        .push(("derivation", include_str!("derivation.nix")));
-
     let source_map = eval.source_map();
     let result = {
         let mut compiler_observer =

tvix/glue/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "tvix-glue"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+nix-compat = { path = "../nix-compat" }
+tvix-eval = { path = "../eval" }
+tvix-castore = { path = "../castore" }
+tvix-store = { path = "../store", default-features = false, features = []}
+bytes = "1.4.0"
+tracing = "0.1.37"
+tokio = "1.28.0"
+thiserror = "1.0.38"
+
+[dependencies.wu-manber]
+git = "https://github.com/tvlfyi/wu-manber.git"
+
+[dev-dependencies]
+test-case = "2.2.2"

tvix/glue/default.nix

@@ -0,0 +1,5 @@
+{ depot, ... }:
+
+(depot.tvix.crates.workspaceMembers.tvix-glue.build.override {
+  runTests = true;
+})

tvix/glue/src/lib.rs

@@ -0,0 +1,28 @@
+use std::{cell::RefCell, rc::Rc};
+
+use known_paths::KnownPaths;
+
+pub mod derivation;
+pub mod errors;
+pub mod known_paths;
+pub mod refscan;
+pub mod tvix_io;
+pub mod tvix_store_io;
+
+/// Adds derivation-related builtins to the passed [tvix_eval::Evaluation].
+///
+/// These are `derivation` and `derivationStrict`.
+///
+/// As they need to interact with `known_paths`, we also need to pass in
+/// `known_paths`.
+pub fn add_derivation_builtins(
+    eval: &mut tvix_eval::Evaluation,
+    known_paths: Rc<RefCell<KnownPaths>>,
+) {
+    eval.builtins
+        .extend(derivation::derivation_builtins(known_paths));
+
+    // Add the actual `builtins.derivation` from compiled Nix code
+    eval.src_builtins
+        .push(("derivation", include_str!("derivation.nix")));
+}

tvix/glue/src/tvix_io.rs

@@ -8,15 +8,16 @@
 //! otherwise fundamental features like nixpkgs bootstrapping and hash
 //! calculation will not work.
 
-use crate::KnownPaths;
 use std::cell::RefCell;
 use std::io;
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use tvix_eval::{EvalIO, FileType};
 
+use crate::known_paths::KnownPaths;
+
 // TODO: Merge this together with TvixStoreIO?
-pub(crate) struct TvixIO<T: EvalIO> {
+pub struct TvixIO<T: EvalIO> {
     /// Ingested paths must be reported to this known paths tracker
     /// for accurate build reference scanning.
     known_paths: Rc<RefCell<KnownPaths>>,
@@ -26,7 +27,7 @@ pub(crate) struct TvixIO<T: EvalIO> {
 }
 
 impl<T: EvalIO> TvixIO<T> {
-    pub(crate) fn new(known_paths: Rc<RefCell<KnownPaths>>, actual: T) -> Self {
+    pub fn new(known_paths: Rc<RefCell<KnownPaths>>, actual: T) -> Self {
         Self {
             known_paths,
             actual,