feat(tvix/eval): detect deferred upvalue capturing

Uses the threaded through slot offset to determine whether
initialisation of a captured local upvalue must be defered to a later
point where all values of a scope are available.

This adds a new data representation to the opcode for this situation,
but the equivalent runtime handling is not yet implemented. This is in
part because there is more compiler machinery needed to find the
resolution point.

Change-Id: Ifd0c393f76abfe6e2d91483faf0f58947ab1dedc
Reviewed-on: https://cl.tvl.fyi/c/depot/+/6329
Reviewed-by: sterni <sternenseemann@systemli.org>
Tested-by: BuildkiteCI

tvix/eval/src/compiler/mod.rs

@@ -122,7 +122,7 @@ impl Compiler {
             ast::Expr::LetIn(let_in) => self.compile_let_in(let_in),
             ast::Expr::Ident(ident) => self.compile_ident(ident),
             ast::Expr::With(with) => self.compile_with(with),
-            ast::Expr::Lambda(lambda) => self.compile_lambda(lambda),
+            ast::Expr::Lambda(lambda) => self.compile_lambda(slot, lambda),
             ast::Expr::Apply(apply) => self.compile_apply(apply),
 
             // Parenthesized expressions are simply unwrapped, leaving
@@ -778,7 +778,7 @@ impl Compiler {
         self.end_scope();
     }
 
-    fn compile_lambda(&mut self, node: ast::Lambda) {
+    fn compile_lambda(&mut self, slot: Option<usize>, node: ast::Lambda) {
         // Open new lambda context in compiler, which has its own
         // scope etc.
         self.contexts.push(LambdaCtx::new());
@@ -833,9 +833,22 @@ impl Compiler {
         self.chunk().push_op(OpCode::OpClosure(closure_idx));
         for upvalue in compiled.scope.upvalues {
             match upvalue {
-                Upvalue::Stack(idx) => {
+                Upvalue::Stack(idx) if slot.is_none() => {
                     self.chunk().push_op(OpCode::DataLocalIdx(idx));
                 }
+
+                Upvalue::Stack(idx) => {
+                    // If the upvalue slot is located *after* the
+                    // closure, the upvalue resolution must be
+                    // deferred until the scope is fully initialised
+                    // and can be finalised.
+                    if slot.unwrap() < idx.0 {
+                        self.chunk().push_op(OpCode::DataDeferredLocal(idx));
+                    } else {
+                        self.chunk().push_op(OpCode::DataLocalIdx(idx));
+                    }
+                }
+
                 Upvalue::Upvalue(idx) => {
                     self.chunk().push_op(OpCode::DataUpvalueIdx(idx));
                 }

tvix/eval/src/opcode.rs

@@ -115,6 +115,7 @@ pub enum OpCode {
     // The VM skips over these by advancing the instruction pointer
     // according to the count.
     DataLocalIdx(StackIdx),
+    DataDeferredLocal(StackIdx),
     DataUpvalueIdx(UpvalueIdx),
     DataDynamicIdx(ConstantIdx),
     DataDynamicAncestor(UpvalueIdx),

tvix/eval/src/vm.rs

@@ -426,6 +426,10 @@ impl VM {
                                 closure.push_upvalue(value);
                             }
 
+                            OpCode::DataDeferredLocal(_idx) => {
+                                todo!("deferred local initialisation")
+                            }
+
                             _ => panic!("compiler error: missing closure operand"),
                         }
                     }
@@ -434,6 +438,7 @@ impl VM {
                 // Data-carrying operands should never be executed,
                 // that is a critical error in the VM.
                 OpCode::DataLocalIdx(_)
+                | OpCode::DataDeferredLocal(_)
                 | OpCode::DataUpvalueIdx(_)
                 | OpCode::DataDynamicIdx(_)
                 | OpCode::DataDynamicAncestor(_) => {