mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/eval): contextful JSON operations

Browse source 
`toJSON` transform a Nix structure into a JSON string.

For each context in that Nix structure, the JSON string must possess it.

Thus, it is necessary to take the union of all contexts and attach it to
the final structure.

Unfortunately, the return type of `into_json` is a serde's JSON object,
not a string. Therefore, it is not possible to reuse `NixString`
machinery.

Context tests are reinforced as Nix does not test those behaviors.

Fixes b/393.

Change-Id: I5afdbc4e18dd70469192c1aa657d1049ba330149
Signed-off-by: Ryan Lahfa <tvl@lahfa.xyz>
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11266
Reviewed-by: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
This commit is contained in:
Ryan Lahfa 2024-03-25 03:36:32 +01:00 committed by raitobezarius
parent 45cf7ae657
commit 863c4207cc
11 changed files with 102 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
tvix/eval/src/builtins/mod.rs
View file

@ -453,11 +453,11 @@ mod pure_builtins {
#[builtin("toJSON")]
async fn builtin_to_json(co: GenCo, val: Value) -> Result<Value, ErrorKind> {
match val.into_json(&co).await? {
match val.into_contextful_json(&co).await? {
Err(cek) => Ok(Value::from(cek)),
Ok(json_value) => {
Ok((json_value, ctx)) => {
let json_str = serde_json::to_string(&json_value)?;
Ok(json_str.into())
Ok(NixString::new_context_from(ctx, json_str).into())
}
}
}

2
tvix/eval/src/builtins/to_xml.rs
View file

@ -137,7 +137,7 @@ fn value_variant_to_xml<W: Write>(w: &mut EventWriter<W>, value: &Value) -> Resu
| Value::Blueprint(_)
| Value::DeferredUpvalue(_)
| Value::UnresolvedPath(_)
| Value::Json(_)
| Value::Json(..)
| Value::FinaliseRequest(_) => {
return Err(ErrorKind::TvixBug {
msg: "internal value variant encountered in builtins.toXML",

63
tvix/eval/src/value/json.rs
View file

@ -6,6 +6,7 @@
use super::{CoercionKind, Value};
use crate::errors::{CatchableErrorKind, ErrorKind};
use crate::generators::{self, GenCo};
use crate::NixContext;
use bstr::ByteSlice;
use serde_json::value::to_value;
@ -13,22 +14,32 @@ use serde_json::Value as Json; // name clash with *our* `Value`
use serde_json::{Map, Number};
impl Value {
pub async fn into_json(
/// Transforms the structure into a JSON
/// and accumulate all encountered context in the second's element
/// of the return type.
pub async fn into_contextful_json(
self,
co: &GenCo,
) -> Result<Result<Json, CatchableErrorKind>, ErrorKind> {
) -> Result<Result<(Json, NixContext), CatchableErrorKind>, ErrorKind> {
let self_forced = generators::request_force(co, self).await;
let mut context = NixContext::new();
let value = match self_forced {
Value::Null => Json::Null,
Value::Bool(b) => Json::Bool(b),
Value::Integer(i) => Json::Number(Number::from(i)),
Value::Float(f) => to_value(f)?,
Value::String(s) => Json::String(s.to_str()?.to_owned()),
Value::String(s) => {
context.mimic(&s);
Json::String(s.to_str()?.to_owned())
}
Value::Path(p) => {
let imported = generators::request_path_import(co, *p).await;
Json::String(imported.to_string_lossy().to_string())
let path = imported.to_string_lossy().to_string();
context = context.append(crate::NixContextElement::Plain(path.clone()));
Json::String(path)
}
Value::List(l) => {
@ -36,7 +47,10 @@ impl Value {
for val in l.into_iter() {
match generators::request_to_json(co, val).await {
Ok(v) => out.push(v),
Ok((v, mut ctx)) => {
context = context.join(&mut ctx);
out.push(v)
}
Err(cek) => return Ok(Err(cek)),
}
}
@ -62,7 +76,14 @@ impl Value {
.await?
{
Value::Catchable(cek) => return Ok(Err(*cek)),
Value::String(s) => return Ok(Ok(Json::String(s.to_str()?.to_owned()))),
Value::String(s) => {
// We need a fresh context here because `__toString` will discard
// everything.
let mut fresh = NixContext::new();
fresh.mimic(&s);
return Ok(Ok((Json::String(s.to_str()?.to_owned()), fresh)));
}
_ => panic!("Value::coerce_to_string_() returned a non-string!"),
}
}
@ -79,7 +100,10 @@ impl Value {
out.insert(
name.to_str()?.to_owned(),
match generators::request_to_json(co, value).await {
Ok(v) => v,
Ok((v, mut ctx)) => {
context = context.join(&mut ctx);
v
}
Err(cek) => return Ok(Err(cek)),
},
);
@ -97,21 +121,34 @@ impl Value {
| val @ Value::Blueprint(_)
| val @ Value::DeferredUpvalue(_)
| val @ Value::UnresolvedPath(_)
| val @ Value::Json(_)
| val @ Value::Json(..)
| val @ Value::FinaliseRequest(_) => {
return Err(ErrorKind::NotSerialisableToJson(val.type_of()))
}
};
Ok(Ok(value))
Ok(Ok((value, context)))
}
/// Generator version of the above, which wraps responses in
/// Value::Json.
pub(crate) async fn into_json_generator(self, co: GenCo) -> Result<Value, ErrorKind> {
match self.into_json(&co).await? {
/// [`Value::Json`].
pub(crate) async fn into_contextful_json_generator(
self,
co: GenCo,
) -> Result<Value, ErrorKind> {
match self.into_contextful_json(&co).await? {
Err(cek) => Ok(Value::from(cek)),
Ok(json) => Ok(Value::Json(Box::new(json))),
Ok((json, ctx)) => Ok(Value::Json(Box::new((json, ctx)))),
}
}
/// Transforms the structure into a JSON
/// All the accumulated context is ignored, use [`into_contextful_json`]
/// to obtain the resulting context of the JSON object.
pub async fn into_json(
self,
co: &GenCo,
) -> Result<Result<Json, CatchableErrorKind>, ErrorKind> {
Ok(self.into_contextful_json(co).await?.map(|(json, _)| json))
}
}

12
tvix/eval/src/value/mod.rs
View file

@ -78,7 +78,7 @@ pub enum Value {
#[serde(skip)]
UnresolvedPath(Box<PathBuf>),
#[serde(skip)]
Json(Box<serde_json::Value>),
Json(Box<(serde_json::Value, NixContext)>),
#[serde(skip)]
FinaliseRequest(bool),
@ -294,7 +294,7 @@ impl Value {
| Value::Blueprint(_)
| Value::DeferredUpvalue(_)
| Value::UnresolvedPath(_)
| Value::Json(_)
| Value::Json(..)
| Value::FinaliseRequest(_) => panic!(
"Tvix bug: internal value left on stack: {}",
value.type_of()
@ -444,7 +444,7 @@ impl Value {
| (Value::Blueprint(_), _)
| (Value::DeferredUpvalue(_), _)
| (Value::UnresolvedPath(_), _)
| (Value::Json(_), _)
| (Value::Json(..), _)
| (Value::FinaliseRequest(_), _) => {
panic!("tvix bug: .coerce_to_string() called on internal value")
}
@ -681,7 +681,7 @@ impl Value {
Value::Blueprint(_) => "internal[blueprint]",
Value::DeferredUpvalue(_) => "internal[deferred_upvalue]",
Value::UnresolvedPath(_) => "internal[unresolved_path]",
Value::Json(_) => "internal[json]",
Value::Json(..) => "internal[json]",
Value::FinaliseRequest(_) => "internal[finaliser_sentinel]",
Value::Catchable(_) => "internal[catchable]",
}
@ -877,7 +877,7 @@ impl Value {
| Value::Blueprint(_)
| Value::DeferredUpvalue(_)
| Value::UnresolvedPath(_)
| Value::Json(_)
| Value::Json(..)
| Value::FinaliseRequest(_) => "an internal Tvix evaluator value".into(),
}
}
@ -991,7 +991,7 @@ impl TotalDisplay for Value {
Value::Blueprint(_) => f.write_str("internal[blueprint]"),
Value::DeferredUpvalue(_) => f.write_str("internal[deferred_upvalue]"),
Value::UnresolvedPath(_) => f.write_str("internal[unresolved_path]"),
Value::Json(_) => f.write_str("internal[json]"),
Value::Json(..) => f.write_str("internal[json]"),
Value::FinaliseRequest(_) => f.write_str("internal[finaliser_sentinel]"),
// Delegate thunk display to the type, as it must handle

6
tvix/eval/src/vm/generators.rs
View file

@ -493,7 +493,7 @@ where
VMRequest::ToJson(value) => {
self.reenqueue_generator(name, span.clone(), generator);
self.enqueue_generator("to_json", span, |co| {
value.into_json_generator(co)
value.into_contextful_json_generator(co)
});
return Ok(false);
}
@ -778,9 +778,9 @@ pub(crate) async fn request_span(co: &GenCo) -> LightSpan {
pub(crate) async fn request_to_json(
co: &GenCo,
value: Value,
) -> Result<serde_json::Value, CatchableErrorKind> {
) -> Result<(serde_json::Value, NixContext), CatchableErrorKind> {
match co.yield_(VMRequest::ToJson(value)).await {
VMResponse::Value(Value::Json(json)) => Ok(*json),
VMResponse::Value(Value::Json(json_with_ctx)) => Ok(*json_with_ctx),
VMResponse::Value(Value::Catchable(cek)) => Err(*cek),
msg => panic!(
"Tvix bug: VM responded with incorrect generator message: {}",

7
tvix/glue/src/builtins/derivation.rs
View file

@ -372,14 +372,13 @@ pub(crate) mod derivation_builtins {
return Ok(val);
}
// TODO(raitobezarius): context for json values?
// input_context.mimic(&val);
let val_json = match val.into_json(&co).await? {
let (val_json, mut context) = match val.into_contextful_json(&co).await? {
Ok(v) => v,
Err(cek) => return Ok(Value::from(cek)),
};
input_context = input_context.join(&mut context);
// No need to check for dups, we only iterate over every attribute name once
structured_attrs.insert(arg_name.to_owned(), val_json);
} else {

2
tvix/glue/src/tests/tvix_tests/eval-okay-context-introspection.exp
View file

@ -1 +1 @@
[ true true true true true true true true true true true ]
[ true true true true true true true true true true true true true ]

10
tvix/glue/src/tests/tvix_tests/eval-okay-context-introspection.nix
View file

@ -41,6 +41,13 @@ let
reconstructed-path = appendContextFrom combo-path
(builtins.unsafeDiscardStringContext combo-path);
an-str = {
a = "${drv}";
};
an-list = {
b = [ drv ];
};
# Eta rule for strings with context.
etaRule = str:
str == appendContextFrom
@ -70,4 +77,7 @@ in
(etaRule "foo")
(etaRule drv.drvPath)
(etaRule drv.foo.outPath)
# `toJSON` tests
(builtins.hasContext (builtins.toJSON an-str))
(builtins.hasContext (builtins.toJSON an-list))
]

2
tvix/glue/src/tests/tvix_tests/eval-okay-context-propagation.exp
View file

@ -1 +1 @@
[ true true true true true true true true true true true true true true true true true true true true true true true true true true true true ]
[ true true true true true true true true true true true true true true true true true true true true true true true true true true true true true true true true true true ]

23
tvix/glue/src/tests/tvix_tests/eval-okay-context-propagation.nix
View file

@ -93,4 +93,27 @@ in
(preserveContext other-drv (builtins.concatStringsSep "${other-drv}" [ "abc" "def" ]))
# `attrNames` will never ever produce context.
(preserveContext "abc" (toString (builtins.attrNames { a = { }; b = { }; c = { }; })))
# `toJSON` preserves context of its inputs.
(preserveContexts [ drv other-drv ] (builtins.toJSON {
a = [ drv ];
b = [ other-drv ];
}))
(preserveContexts [ drv other-drv ] (builtins.toJSON {
a.deep = [ drv ];
b = [ other-drv ];
}))
(preserveContexts [ drv other-drv ] (builtins.toJSON {
a = "${drv}";
b = [ other-drv ];
}))
(preserveContexts [ drv other-drv ] (builtins.toJSON {
a.deep = "${drv}";
b = [ other-drv ];
}))
(preserveContexts [ drv other-drv ] (builtins.toJSON {
a = "${drv} ${other-drv}";
}))
(preserveContexts [ drv other-drv ] (builtins.toJSON {
a.b.c.d.e.f = "${drv} ${other-drv}";
}))
]

2
tvix/serde/src/de.rs
View file

@ -107,7 +107,7 @@ impl<'de> de::Deserializer<'de> for NixDeserializer {
| Value::Blueprint(_)
| Value::DeferredUpvalue(_)
| Value::UnresolvedPath(_)
| Value::Json(_)
| Value::Json(..)
| Value::Catchable(_)
| Value::FinaliseRequest(_) => Err(Error::Unserializable {
value_type: self.value.type_of(),