From 8821746d6c6c1b71774727fe1103425d903952bb Mon Sep 17 00:00:00 2001 From: Aspen Smith Date: Sun, 7 Jul 2024 09:21:52 -0400 Subject: [PATCH] fix(tvix/repl): Share globals and sourcemap across evaluations Now that we can bind (potentially lazy, potentially lambda-containing) values in the REPL and then reference them in subsequent evaluations, it's important that the values to which we construct shared references are shared across those subsequent evaluations - otherwise, we get panics due to unknown source map locations, or dropped weak references to globals. This change assigns both the globals and the source map as fields on the Repl after the first evaluation, and then passes those in (to the EvaluationBuilder) on subsequent evaluations. On the EvaluationBuilder side, there's some panicking introduced - this is intentional, as my intent is for the builder to be configured statically enough that panicking is the best way to report errors here (it's always a bug to misconfigure an Evaluation, and we'd never want to handle it dynamically). Change-Id: I37225697235c22b683ca48a17d30fa8fedd12d1b Reviewed-on: https://cl.tvl.fyi/c/depot/+/11960 Reviewed-by: flokli Autosubmit: aspen Tested-by: BuildkiteCI --- tvix/cli/src/lib.rs | 67 +++++++++++---- tvix/cli/src/main.rs | 4 + tvix/cli/src/repl.rs | 34 ++++++-- tvix/cli/tests/.skip-tree | 0 tvix/cli/tests/import.nix | 1 + tvix/cli/tests/repl.rs | 16 ++++ tvix/cli/tests/six.nix | 1 + tvix/eval/src/compiler/mod.rs | 2 +- tvix/eval/src/lib.rs | 151 +++++++++++++++++++++++++--------- tvix/utils.nix | 5 +- 10 files changed, 221 insertions(+), 60 deletions(-) create mode 100644 tvix/cli/tests/.skip-tree create mode 100644 tvix/cli/tests/import.nix create mode 100644 tvix/cli/tests/six.nix diff --git a/tvix/cli/src/lib.rs b/tvix/cli/src/lib.rs index 008593c5d..800ffb4e0 100644 --- a/tvix/cli/src/lib.rs +++ b/tvix/cli/src/lib.rs @@ -8,7 +8,7 @@ use tvix_build::buildservice; use tvix_eval::{ builtins::impure_builtins, observer::{DisassemblingObserver, TracingObserver}, - ErrorKind, EvalIO, Value, + ErrorKind, EvalIO, GlobalsMap, SourceCode, Value, }; use tvix_glue::{ builtins::{add_derivation_builtins, add_fetcher_builtins, add_import_builtins}, @@ -83,7 +83,13 @@ impl AllowIncomplete { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct IncompleteInput; +pub struct EvalResult { + value: Option, + globals: Rc, +} + /// Interprets the given code snippet, printing out warnings and errors and returning the result +#[allow(clippy::too_many_arguments)] pub fn evaluate( tvix_store_io: Rc, code: &str, @@ -91,7 +97,9 @@ pub fn evaluate( args: &Args, allow_incomplete: AllowIncomplete, env: Option<&HashMap>, -) -> Result, IncompleteInput> { + globals: Option>, + source_map: Option, +) -> Result { let span = Span::current(); span.pb_start(); span.pb_set_style(&tvix_tracing::PB_SPINNER_STYLE); @@ -102,16 +110,27 @@ pub fn evaluate( )) as Box) .enable_import() .with_strict(args.strict) - .add_builtins(impure_builtins()) .env(env); - eval_builder = add_derivation_builtins(eval_builder, Rc::clone(&tvix_store_io)); - eval_builder = add_fetcher_builtins(eval_builder, Rc::clone(&tvix_store_io)); - eval_builder = add_import_builtins(eval_builder, tvix_store_io); - eval_builder = configure_nix_path(eval_builder, &args.nix_search_path); + match globals { + Some(globals) => { + eval_builder = eval_builder.with_globals(globals); + } + None => { + eval_builder = eval_builder.add_builtins(impure_builtins()); + eval_builder = add_derivation_builtins(eval_builder, Rc::clone(&tvix_store_io)); + eval_builder = add_fetcher_builtins(eval_builder, Rc::clone(&tvix_store_io)); + eval_builder = add_import_builtins(eval_builder, tvix_store_io); + eval_builder = configure_nix_path(eval_builder, &args.nix_search_path); + } + }; + + if let Some(source_map) = source_map { + eval_builder = eval_builder.with_source_map(source_map); + } let source_map = eval_builder.source_map().clone(); - let result = { + let (result, globals) = { let mut compiler_observer = DisassemblingObserver::new(source_map.clone(), std::io::stderr()); if args.dump_bytecode { @@ -129,7 +148,9 @@ pub fn evaluate( span.pb_set_message("Evaluating…"); let eval = eval_builder.build(); - eval.evaluate(code, path) + let globals = eval.globals(); + let result = eval.evaluate(code, path); + (result, globals) }; if allow_incomplete.allow() @@ -160,19 +181,24 @@ pub fn evaluate( } } - Ok(result.value) + Ok(EvalResult { + globals, + value: result.value, + }) } pub struct InterpretResult { output: String, success: bool, + pub(crate) globals: Option>, } impl InterpretResult { - pub fn empty_success() -> Self { + pub fn empty_success(globals: Option>) -> Self { Self { output: String::new(), success: true, + globals, } } @@ -194,6 +220,7 @@ impl InterpretResult { /// and the result itself. The return value indicates whether /// evaluation succeeded. #[instrument(skip_all, fields(indicatif.pb_show=1))] +#[allow(clippy::too_many_arguments)] pub fn interpret( tvix_store_io: Rc, code: &str, @@ -202,11 +229,22 @@ pub fn interpret( explain: bool, allow_incomplete: AllowIncomplete, env: Option<&HashMap>, + globals: Option>, + source_map: Option, ) -> Result { let mut output = String::new(); - let result = evaluate(tvix_store_io, code, path, args, allow_incomplete, env)?; + let result = evaluate( + tvix_store_io, + code, + path, + args, + allow_incomplete, + env, + globals, + source_map, + )?; - if let Some(value) = result.as_ref() { + if let Some(value) = result.value.as_ref() { if explain { writeln!(&mut output, "=> {}", value.explain()).unwrap(); } else if args.raw { @@ -219,6 +257,7 @@ pub fn interpret( // inform the caller about any errors Ok(InterpretResult { output, - success: result.is_some(), + success: result.value.is_some(), + globals: Some(result.globals), }) } diff --git a/tvix/cli/src/main.rs b/tvix/cli/src/main.rs index 0bd3be37e..f927665ae 100644 --- a/tvix/cli/src/main.rs +++ b/tvix/cli/src/main.rs @@ -75,6 +75,8 @@ fn main() { false, AllowIncomplete::RequireComplete, None, // TODO(aspen): Pass in --arg/--argstr here + None, + None, ) .unwrap() .finalize() @@ -104,6 +106,8 @@ fn run_file(io_handle: Rc, mut path: PathBuf, args: &Args) { false, AllowIncomplete::RequireComplete, None, + None, + None, ) .unwrap() .finalize() diff --git a/tvix/cli/src/repl.rs b/tvix/cli/src/repl.rs index 5098fbaee..6b34b6552 100644 --- a/tvix/cli/src/repl.rs +++ b/tvix/cli/src/repl.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, path::PathBuf}; use rustyline::{error::ReadlineError, Editor}; use smol_str::SmolStr; -use tvix_eval::Value; +use tvix_eval::{GlobalsMap, SourceCode, Value}; use tvix_glue::tvix_store_io::TvixStoreIO; use crate::{ @@ -92,6 +92,8 @@ pub struct Repl<'a> { io_handle: Rc, args: &'a Args, + source_map: SourceCode, + globals: Option>, } impl<'a> Repl<'a> { @@ -103,6 +105,8 @@ impl<'a> Repl<'a> { env: HashMap::new(), io_handle, args, + source_map: Default::default(), + globals: None, } } @@ -179,7 +183,7 @@ impl<'a> Repl<'a> { } ReplCommand::Help => { println!("{}", ReplCommand::HELP); - Ok(InterpretResult::empty_success()) + Ok(InterpretResult::empty_success(None)) } ReplCommand::Expr(input) => interpret( Rc::clone(&self.io_handle), @@ -189,6 +193,8 @@ impl<'a> Repl<'a> { false, AllowIncomplete::Allow, Some(&self.env), + self.globals.clone(), + Some(self.source_map.clone()), ), ReplCommand::Assign(Assignment { ident, value }) => { match evaluate( @@ -198,12 +204,15 @@ impl<'a> Repl<'a> { self.args, AllowIncomplete::Allow, Some(&self.env), + self.globals.clone(), + Some(self.source_map.clone()), ) { - Ok(Some(value)) => { - self.env.insert(ident.into(), value); - Ok(InterpretResult::empty_success()) + Ok(result) => { + if let Some(value) = result.value { + self.env.insert(ident.into(), value); + } + Ok(InterpretResult::empty_success(Some(result.globals))) } - Ok(None) => Ok(InterpretResult::empty_success()), Err(incomplete) => Err(incomplete), } } @@ -215,6 +224,8 @@ impl<'a> Repl<'a> { true, AllowIncomplete::Allow, Some(&self.env), + self.globals.clone(), + Some(self.source_map.clone()), ), ReplCommand::Print(input) => interpret( Rc::clone(&self.io_handle), @@ -227,13 +238,22 @@ impl<'a> Repl<'a> { false, AllowIncomplete::Allow, Some(&self.env), + self.globals.clone(), + Some(self.source_map.clone()), ), }; match res { - Ok(InterpretResult { output, .. }) => { + Ok(InterpretResult { + output, + globals, + success: _, + }) => { self.rl.add_history_entry(input); self.multiline_input = None; + if globals.is_some() { + self.globals = globals; + } CommandResult { output, continue_: true, diff --git a/tvix/cli/tests/.skip-tree b/tvix/cli/tests/.skip-tree new file mode 100644 index 000000000..e69de29bb diff --git a/tvix/cli/tests/import.nix b/tvix/cli/tests/import.nix new file mode 100644 index 000000000..9ac2d0232 --- /dev/null +++ b/tvix/cli/tests/import.nix @@ -0,0 +1 @@ +{ }: import ./six.nix { } diff --git a/tvix/cli/tests/repl.rs b/tvix/cli/tests/repl.rs index a14f4bff7..c66443309 100644 --- a/tvix/cli/tests/repl.rs +++ b/tvix/cli/tests/repl.rs @@ -53,6 +53,22 @@ test_repl!(bind_lazy() { "#]]; }); +test_repl!(bind_lazy_errors() { + r#"x = (_: "x" + 1)"# => expect![[""]]; + "x null" => expect![[""]]; +}); + +test_repl!(bind_referencing_import() { + "six = import ./tests/six.nix {}" => expect![[""]]; + "six.six" => expect![[r#" + => 6 :: int + "#]]; + "imported = import ./tests/import.nix" => expect![[""]]; + "(imported {}).six" => expect![[r#" + => 6 :: int + "#]]; +}); + test_repl!(deep_print() { "builtins.map (x: x + 1) [ 1 2 3 ]" => expect![[r#" => [ ] :: list diff --git a/tvix/cli/tests/six.nix b/tvix/cli/tests/six.nix new file mode 100644 index 000000000..d466abe06 --- /dev/null +++ b/tvix/cli/tests/six.nix @@ -0,0 +1 @@ +{ }: { six = builtins.foldl' (x: y: x + y) 0 [ 1 2 3 ]; } diff --git a/tvix/eval/src/compiler/mod.rs b/tvix/eval/src/compiler/mod.rs index 1ec47599f..3a25052aa 100644 --- a/tvix/eval/src/compiler/mod.rs +++ b/tvix/eval/src/compiler/mod.rs @@ -117,7 +117,7 @@ impl TrackedFormal { /// The map of globally available functions and other values that /// should implicitly be resolvable in the global scope. -pub(crate) type GlobalsMap = HashMap<&'static str, Value>; +pub type GlobalsMap = HashMap<&'static str, Value>; /// Set of builtins that (if they exist) should be made available in /// the global scope, meaning that they can be accessed not just diff --git a/tvix/eval/src/lib.rs b/tvix/eval/src/lib.rs index 00dc7918d..a53a2a02d 100644 --- a/tvix/eval/src/lib.rs +++ b/tvix/eval/src/lib.rs @@ -42,13 +42,12 @@ use std::rc::Rc; use std::str::FromStr; use std::sync::Arc; -use crate::compiler::GlobalsMap; use crate::observer::{CompilerObserver, RuntimeObserver}; use crate::value::Lambda; use crate::vm::run_lambda; // Re-export the public interface used by other crates. -pub use crate::compiler::{compile, prepare_globals, CompilationOutput}; +pub use crate::compiler::{compile, prepare_globals, CompilationOutput, GlobalsMap}; pub use crate::errors::{AddContext, CatchableErrorKind, Error, ErrorKind, EvalResult}; pub use crate::io::{DummyIO, EvalIO, FileType}; pub use crate::pretty_ast::pretty_print_expr; @@ -64,6 +63,16 @@ pub use crate::value::{Builtin, CoercionKind, NixAttrs, NixList, NixString, Valu #[cfg(feature = "impure")] pub use crate::io::StdIO; +struct BuilderBuiltins { + builtins: Vec<(&'static str, Value)>, + src_builtins: Vec<(&'static str, &'static str)>, +} + +enum BuilderGlobals { + Builtins(BuilderBuiltins), + Globals(Rc), +} + /// Builder for building an [`Evaluation`]. /// /// Construct an [`EvaluationBuilder`] by calling one of: @@ -75,9 +84,8 @@ pub use crate::io::StdIO; /// Then configure the fields by calling the various methods on [`EvaluationBuilder`], and finally /// call [`build`](Self::build) to construct an [`Evaluation`] pub struct EvaluationBuilder<'co, 'ro, 'env, IO> { - source_map: SourceCode, - builtins: Vec<(&'static str, Value)>, - src_builtins: Vec<(&'static str, &'static str)>, + source_map: Option, + globals: BuilderGlobals, env: Option<&'env HashMap>, io_handle: IO, enable_import: bool, @@ -98,21 +106,31 @@ where /// - Adds a `"storeDir"` builtin containing the store directory of the configured IO handle /// - Sets up globals based on the configured builtins /// - Copies all other configured fields to the [`Evaluation`] - pub fn build(mut self) -> Evaluation<'co, 'ro, 'env, IO> { - // Insert a storeDir builtin *iff* a store directory is present. - if let Some(store_dir) = self.io_handle.as_ref().store_dir() { - self.builtins.push(("storeDir", store_dir.into())); - } + pub fn build(self) -> Evaluation<'co, 'ro, 'env, IO> { + let source_map = self.source_map.unwrap_or_default(); - let globals = crate::compiler::prepare_globals( - self.builtins, - self.src_builtins, - self.source_map.clone(), - self.enable_import, - ); + let globals = match self.globals { + BuilderGlobals::Globals(globals) => globals, + BuilderGlobals::Builtins(BuilderBuiltins { + mut builtins, + src_builtins, + }) => { + // Insert a storeDir builtin *iff* a store directory is present. + if let Some(store_dir) = self.io_handle.as_ref().store_dir() { + builtins.push(("storeDir", store_dir.into())); + } + + crate::compiler::prepare_globals( + builtins, + src_builtins, + source_map.clone(), + self.enable_import, + ) + } + }; Evaluation { - source_map: self.source_map, + source_map, globals, env: self.env, io_handle: self.io_handle, @@ -132,11 +150,13 @@ impl<'co, 'ro, 'env, IO> EvaluationBuilder<'co, 'ro, 'env, IO> { builtins.extend(builtins::placeholders()); // these are temporary Self { - source_map: SourceCode::default(), + source_map: None, enable_import: false, io_handle, - builtins, - src_builtins: vec![], + globals: BuilderGlobals::Builtins(BuilderBuiltins { + builtins, + src_builtins: vec![], + }), env: None, strict: false, nix_path: None, @@ -149,8 +169,7 @@ impl<'co, 'ro, 'env, IO> EvaluationBuilder<'co, 'ro, 'env, IO> { EvaluationBuilder { io_handle, source_map: self.source_map, - builtins: self.builtins, - src_builtins: self.src_builtins, + globals: self.globals, env: self.env, enable_import: self.enable_import, strict: self.strict, @@ -175,14 +194,64 @@ impl<'co, 'ro, 'env, IO> EvaluationBuilder<'co, 'ro, 'env, IO> { self.with_enable_import(true) } + fn builtins_mut(&mut self) -> &mut BuilderBuiltins { + match &mut self.globals { + BuilderGlobals::Builtins(builtins) => builtins, + BuilderGlobals::Globals(_) => { + panic!("Cannot modify builtins on an EvaluationBuilder with globals configured") + } + } + } + + /// Add additional builtins (represented as tuples of name and [`Value`]) to this evaluation + /// builder. + /// + /// # Panics + /// + /// Panics if this evaluation builder has had globals set via [`with_globals`] pub fn add_builtins(mut self, builtins: I) -> Self where I: IntoIterator, { - self.builtins.extend(builtins); + self.builtins_mut().builtins.extend(builtins); self } + /// Add additional builtins that are implemented in Nix source code (represented as tuples of + /// name and nix source) to this evaluation builder. + /// + /// # Panics + /// + /// Panics if this evaluation builder has had globals set via [`with_globals`] + pub fn add_src_builtin(mut self, name: &'static str, src: &'static str) -> Self { + self.builtins_mut().src_builtins.push((name, src)); + self + } + + /// Set the globals for this evaluation builder to a previously-constructed globals map. + /// Intended to allow sharing globals across multiple evaluations (eg for the REPL). + /// + /// Discards any builtins previously configured via [`add_builtins`] and [`add_src_builtins`]. + /// If either of those methods is called on the evaluation builder after this one, they will + /// panic. + pub fn with_globals(self, globals: Rc) -> Self { + Self { + globals: BuilderGlobals::Globals(globals), + ..self + } + } + + pub fn with_source_map(self, source_map: SourceCode) -> Self { + debug_assert!( + self.source_map.is_none(), + "Cannot set the source_map on an EvaluationBuilder twice" + ); + Self { + source_map: Some(source_map), + ..self + } + } + pub fn with_strict(self, strict: bool) -> Self { Self { strict, ..self } } @@ -191,11 +260,6 @@ impl<'co, 'ro, 'env, IO> EvaluationBuilder<'co, 'ro, 'env, IO> { self.with_strict(true) } - pub fn add_src_builtin(mut self, name: &'static str, src: &'static str) -> Self { - self.src_builtins.push((name, src)); - self - } - pub fn nix_path(self, nix_path: Option) -> Self { Self { nix_path, ..self } } @@ -234,8 +298,8 @@ impl<'co, 'ro, 'env, IO> EvaluationBuilder<'co, 'ro, 'env, IO> { } impl<'co, 'ro, 'env, IO> EvaluationBuilder<'co, 'ro, 'env, IO> { - pub fn source_map(&self) -> &SourceCode { - &self.source_map + pub fn source_map(&mut self) -> &SourceCode { + self.source_map.get_or_insert_with(SourceCode::default) } } @@ -255,7 +319,9 @@ impl<'co, 'ro, 'env> EvaluationBuilder<'co, 'ro, 'env, Box> { pub fn enable_impure(mut self, io: Option>) -> Self { self.io_handle = io.unwrap_or_else(|| Box::new(StdIO) as Box); self.enable_import = true; - self.builtins.extend(builtins::impure_builtins()); + self.builtins_mut() + .builtins + .extend(builtins::impure_builtins()); // Make `NIX_PATH` resolutions work by default, unless the // user already overrode this with something else. @@ -332,9 +398,26 @@ pub struct EvaluationResult { } impl<'co, 'ro, 'env, IO> Evaluation<'co, 'ro, 'env, IO> { + /// Make a new [builder][] for configuring an evaluation + /// + /// [builder]: EvaluationBuilder pub fn builder(io_handle: IO) -> EvaluationBuilder<'co, 'ro, 'env, IO> { EvaluationBuilder::new(io_handle) } + + /// Clone the reference to the map of Nix globals for this evaluation. If [`Value`]s are shared + /// across subsequent [`Evaluation`]s, it is important that those evaluations all have the same + /// underlying globals map. + pub fn globals(&self) -> Rc { + self.globals.clone() + } + + /// Clone the reference to the contained source code map. This is used after an evaluation for + /// pretty error printing. Also, if [`Value`]s are shared across subsequent [`Evaluation`]s, it + /// is important that those evaluations all have the same underlying source code map. + pub fn source_map(&self) -> SourceCode { + self.source_map.clone() + } } impl<'co, 'ro, 'env> Evaluation<'co, 'ro, 'env, Box> { @@ -352,12 +435,6 @@ impl<'co, 'ro, 'env, IO> Evaluation<'co, 'ro, 'env, IO> where IO: AsRef + 'static, { - /// Clone the reference to the contained source code map. This is used after - /// an evaluation for pretty error printing. - pub fn source_map(&self) -> SourceCode { - self.source_map.clone() - } - /// Only compile the provided source code, at an optional location of the /// source code (i.e. path to the file it was read from; used for error /// reporting, and for resolving relative paths in impure functions) diff --git a/tvix/utils.nix b/tvix/utils.nix index 7adf8fe99..1548d4de0 100644 --- a/tvix/utils.nix +++ b/tvix/utils.nix @@ -78,7 +78,10 @@ }; tvix-cli = prev: { - src = depot.tvix.utils.filterRustCrateSrc { root = prev.src.origSrc; }; + src = depot.tvix.utils.filterRustCrateSrc rec { + root = prev.src.origSrc; + extraFileset = root + "/tests"; + }; buildInputs = lib.optional pkgs.stdenv.isDarwin commonDarwinDeps; };