feat(tvix/eval): initial implementation of builtins.import

This adds an initial working version of builtins.import which
encapsulates the entire functionality of `import` within the builtin
itself, without requiring any changes in the compiler or VM.

The key insight that enables this is that we can simply return a Thunk
from `import` that is constructed from the output of running the
compiler and - ta-da! - no other component needs to know about it.

A couple of notes:

* builtins.import needs to capture variables like the SourceCode
  structure. This means it can not currently be constructed the same
  way as other builtins and has special handling, which leaks out to
  `eval.rs`. I have postponed dealing with that until we have this
  working a bit more.

* the `globals` are not yet passed through

* the error representation for the new variants is absolutely not done
  yet, we probably want to switch to something that supports
  cause-chaining now (like miette)

* there is no mechanism for emitting warnings at runtime; we need to
  add that

Change-Id: I3117a7ae3ff2432bf44f5ff05ad35f47faca31d5
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6857
Reviewed-by: sterni <sternenseemann@systemli.org>
Reviewed-by: wpcarro <wpcarro@gmail.com>
Tested-by: BuildkiteCI
Reviewed-by: grfn <grfn@gws.fyi>

tvix/eval/src/builtins/impure.rs

@@ -1,19 +1,23 @@
 use std::{
-    collections::BTreeMap,
+    collections::{BTreeMap, HashMap},
+    rc::Rc,
     time::{SystemTime, UNIX_EPOCH},
 };
 
 use crate::{
-    value::{Builtin, NixString},
-    Value,
+    errors::ErrorKind,
+    observer::NoOpObserver,
+    value::{Builtin, NixString, Thunk},
+    vm::VM,
+    SourceCode, Value,
 };
 
 fn impure_builtins() -> Vec<Builtin> {
     vec![]
 }
 
-/// Return all impure builtins, that is all builtins which may perform I/O outside of the VM and so
-/// cannot be used in all contexts (e.g. WASM).
+/// Return all impure builtins, that is all builtins which may perform I/O
+/// outside of the VM and so cannot be used in all contexts (e.g. WASM).
 pub(super) fn builtins() -> BTreeMap<NixString, Value> {
     let mut map: BTreeMap<NixString, Value> = impure_builtins()
         .into_iter()
@@ -34,3 +38,73 @@ pub(super) fn builtins() -> BTreeMap<NixString, Value> {
 
     map
 }
+
+/// Constructs and inserts the `import` builtin. This builtin is special in that
+/// it needs to capture the [crate::SourceCode] structure to correctly track
+/// source code locations while invoking a compiler.
+// TODO: need to be able to pass through a CompilationObserver, too.
+pub fn builtins_import(source: SourceCode) -> Builtin {
+    Builtin::new(
+        "import",
+        &[true],
+        move |mut args: Vec<Value>, _: &mut VM| {
+            let path = match args.pop().unwrap() {
+                Value::Path(path) => path,
+                Value::String(_) => {
+                    return Err(ErrorKind::NotImplemented("importing from string-paths"))
+                }
+                other => {
+                    return Err(ErrorKind::TypeError {
+                        expected: "path or string",
+                        actual: other.type_of(),
+                    })
+                }
+            };
+
+            let contents =
+                std::fs::read_to_string(&path).map_err(|err| ErrorKind::ReadFileError {
+                    path: path.clone(),
+                    error: Rc::new(err),
+                })?;
+
+            let parsed = rnix::ast::Root::parse(&contents);
+            let errors = parsed.errors();
+
+            if !errors.is_empty() {
+                return Err(ErrorKind::ImportParseError {
+                    path,
+                    errors: errors.to_vec(),
+                });
+            }
+
+            let file = source.add_file(path.to_string_lossy().to_string(), contents);
+
+            let result = crate::compile(
+                &parsed.tree().expr().unwrap(),
+                Some(path.clone()),
+                file,
+                HashMap::new(), // TODO: pass through globals
+                &mut NoOpObserver::default(),
+            )
+            .map_err(|err| ErrorKind::ImportCompilerError {
+                path: path.clone(),
+                errors: vec![err],
+            })?;
+
+            if !result.errors.is_empty() {
+                return Err(ErrorKind::ImportCompilerError {
+                    path,
+                    errors: result.errors,
+                });
+            }
+
+            // TODO: deal with runtime *warnings* (most likely through an
+            // emit_warning function on the VM that might return it together with
+            // the result)
+
+            // Compilation succeeded, we can construct a thunk from whatever it spat
+            // out and return that.
+            Ok(Value::Thunk(Thunk::new(result.lambda)))
+        },
+    )
+}

tvix/eval/src/builtins/mod.rs

@@ -19,7 +19,7 @@ use crate::{arithmetic_op, cmp_op};
 use self::versions::{VersionPart, VersionPartsIter};
 
 #[cfg(feature = "impure")]
-mod impure;
+pub mod impure;
 pub mod versions;
 
 /// Coerce a Nix Value to a plain path, e.g. in order to access the file it

tvix/eval/src/errors.rs

@@ -1,5 +1,6 @@
 use crate::value::CoercionKind;
 use std::path::PathBuf;
+use std::rc::Rc;
 use std::{fmt::Display, num::ParseIntError};
 
 use codemap::Span;
@@ -99,6 +100,24 @@ pub enum ErrorKind {
     /// literal attribute sets.
     UnmergeableValue,
 
+    /// Tvix failed to read a file from disk for some reason.
+    ReadFileError {
+        path: PathBuf,
+        error: Rc<std::io::Error>,
+    },
+
+    /// Parse errors occured while importing a file.
+    ImportParseError {
+        path: PathBuf,
+        errors: Vec<rnix::parser::ParseError>,
+    },
+
+    /// Compilation errors occured while importing a file.
+    ImportCompilerError {
+        path: PathBuf,
+        errors: Vec<Error>,
+    },
+
     /// Tvix internal warning for features triggered by users that are
     /// not actually implemented yet, and without which eval can not
     /// proceed.
@@ -271,6 +290,34 @@ to a missing value in the attribute set(s) included via `with`."#,
                     .into()
             }
 
+            ErrorKind::ReadFileError { path, error } => {
+                format!(
+                    "failed to read file '{}': {}",
+                    path.to_string_lossy(),
+                    error
+                )
+            }
+
+            ErrorKind::ImportParseError { errors, path } => {
+                format!(
+                    "{} parse errors occured while importing '{}'",
+                    errors.len(),
+                    path.to_string_lossy()
+                )
+            }
+
+            ErrorKind::ImportCompilerError { errors, path } => {
+                // TODO: chain display of these errors, though this is
+                // probably not the right place for that (should
+                // branch into a more elaborate diagnostic() call
+                // below).
+                format!(
+                    "{} errors occured while importing '{}'",
+                    errors.len(),
+                    path.to_string_lossy()
+                )
+            }
+
             ErrorKind::NotImplemented(feature) => {
                 format!("feature not yet implemented in Tvix: {}", feature)
             }
@@ -305,6 +352,11 @@ to a missing value in the attribute set(s) included via `with`."#,
             ErrorKind::TailEmptyList { .. } => "E023",
             ErrorKind::UnmergeableInherit { .. } => "E024",
             ErrorKind::UnmergeableValue => "E025",
+            ErrorKind::ReadFileError { .. } => "E026",
+            ErrorKind::ImportParseError { .. } => "E027",
+            ErrorKind::ImportCompilerError { .. } => "E028",
+
+            // Placeholder error while Tvix is under construction.
             ErrorKind::NotImplemented(_) => "E999",
 
             // TODO: thunk force errors should yield a chained

tvix/eval/src/eval.rs

@@ -58,12 +58,21 @@ pub fn interpret(code: &str, location: Option<PathBuf>, options: Options) -> Eva
         println!("{:?}", root_expr);
     }
 
+    // TODO: encapsulate this import weirdness in builtins
+    let mut builtins = global_builtins();
+
+    #[cfg(feature = "impure")]
+    builtins.insert(
+        "import",
+        Value::Builtin(crate::builtins::impure::builtins_import(source.clone())),
+    );
+
     let result = if options.dump_bytecode {
         crate::compiler::compile(
             &root_expr,
             location,
             file.clone(),
-            global_builtins(),
+            builtins,
             &mut DisassemblingObserver::new(source.clone(), std::io::stderr()),
         )
     } else {
@@ -71,7 +80,7 @@ pub fn interpret(code: &str, location: Option<PathBuf>, options: Options) -> Eva
             &root_expr,
             location,
             file.clone(),
-            global_builtins(),
+            builtins,
             &mut NoOpObserver::default(),
         )
     }?;