feat(tvix/eval): Implement builtins.fromJSON
Using `serde_json` for parsing JSON here, plus an `impl FromJSON for Value`. The latter is primarily to stay "dependency light" for now - likely going with an actual serde `Deserialize` impl in the future is going to be way better as it allows saving significantly on intermediary allocations. Change-Id: I152a0448ff7c87cf7ebaac927c38912b99de1c18 Reviewed-on: https://cl.tvl.fyi/c/depot/+/6920 Tested-by: BuildkiteCI Reviewed-by: tazjin <tazjin@tvl.su>
This commit is contained in:
parent
277c69cbe5
commit
5eb89be682
12 changed files with 123 additions and 13 deletions
5
corp/tvixbolt/Cargo.lock
generated
5
corp/tvixbolt/Cargo.lock
generated
|
@ -478,9 +478,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde_json"
|
name = "serde_json"
|
||||||
version = "1.0.85"
|
version = "1.0.86"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
|
checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"itoa",
|
"itoa",
|
||||||
"ryu",
|
"ryu",
|
||||||
|
@ -582,6 +582,7 @@ dependencies = [
|
||||||
"path-clean",
|
"path-clean",
|
||||||
"rnix",
|
"rnix",
|
||||||
"rowan",
|
"rowan",
|
||||||
|
"serde_json",
|
||||||
"smol_str",
|
"smol_str",
|
||||||
"tabwriter",
|
"tabwriter",
|
||||||
]
|
]
|
||||||
|
|
5
tvix/eval/Cargo.lock
generated
5
tvix/eval/Cargo.lock
generated
|
@ -997,9 +997,9 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde_json"
|
name = "serde_json"
|
||||||
version = "1.0.85"
|
version = "1.0.86"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
|
checksum = "41feea4228a6f1cd09ec7a3593a682276702cd67b5273544757dae23c096f074"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"itoa 1.0.3",
|
"itoa 1.0.3",
|
||||||
"ryu",
|
"ryu",
|
||||||
|
@ -1211,6 +1211,7 @@ dependencies = [
|
||||||
"rnix",
|
"rnix",
|
||||||
"rowan",
|
"rowan",
|
||||||
"rustyline",
|
"rustyline",
|
||||||
|
"serde_json",
|
||||||
"smol_str",
|
"smol_str",
|
||||||
"tabwriter",
|
"tabwriter",
|
||||||
"tempdir",
|
"tempdir",
|
||||||
|
|
|
@ -24,6 +24,7 @@ codemap-diagnostic = "0.1.1"
|
||||||
proptest = { version = "1.0.0", default_features = false, features = ["std", "alloc", "break-dead-code", "tempfile"], optional = true }
|
proptest = { version = "1.0.0", default_features = false, features = ["std", "alloc", "break-dead-code", "tempfile"], optional = true }
|
||||||
test-strategy = { version = "0.2.1", optional = true }
|
test-strategy = { version = "0.2.1", optional = true }
|
||||||
clap = { version = "3.2.22", optional = true, features = ["derive", "env"] }
|
clap = { version = "3.2.22", optional = true, features = ["derive", "env"] }
|
||||||
|
serde_json = "1.0.86"
|
||||||
|
|
||||||
# rnix has not been released in a while (as of 2022-09-23), we will
|
# rnix has not been released in a while (as of 2022-09-23), we will
|
||||||
# use it from git.
|
# use it from git.
|
||||||
|
|
|
@ -106,11 +106,15 @@ lib.fix (self: depot.third_party.naersk.buildPackage (lib.fix (naerskArgs: {
|
||||||
|
|
||||||
base="$(dirname "$i")/$(basename "$i" ".nix")"
|
base="$(dirname "$i")/$(basename "$i" ".nix")"
|
||||||
|
|
||||||
if [[ "$(basename "$i")" == "eval-okay-search-path.nix" ]]; then
|
case "$(basename $i)" in
|
||||||
# TODO(sterni): fix this test
|
eval-okay-search-path.nix) ;&
|
||||||
echo "SKIPPED: $i"
|
eval-okay-fromjson.nix)
|
||||||
continue
|
# TODO(sterni,grfn): fix these tests
|
||||||
fi
|
echo "SKIPPED: $i"
|
||||||
|
continue
|
||||||
|
;;
|
||||||
|
*) ;;
|
||||||
|
esac
|
||||||
|
|
||||||
if test -e $base.exp; then
|
if test -e $base.exp; then
|
||||||
flags=
|
flags=
|
||||||
|
|
|
@ -271,6 +271,11 @@ fn pure_builtins() -> Vec<Builtin> {
|
||||||
Ok(res)
|
Ok(res)
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
Builtin::new("fromJSON", &[true], |args: Vec<Value>, _: &mut VM| {
|
||||||
|
let json_str = args[0].to_str()?;
|
||||||
|
let json: serde_json::Value = serde_json::from_str(&json_str)?;
|
||||||
|
json.try_into()
|
||||||
|
}),
|
||||||
Builtin::new("genList", &[true, true], |args: Vec<Value>, vm: &mut VM| {
|
Builtin::new("genList", &[true, true], |args: Vec<Value>, vm: &mut VM| {
|
||||||
let len = args[1].as_int()?;
|
let len = args[1].as_int()?;
|
||||||
(0..len)
|
(0..len)
|
||||||
|
|
|
@ -129,6 +129,9 @@ pub enum ErrorKind {
|
||||||
error: Rc<io::Error>,
|
error: Rc<io::Error>,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/// Errors converting JSON to a value
|
||||||
|
FromJsonError(String),
|
||||||
|
|
||||||
/// Tvix internal warning for features triggered by users that are
|
/// Tvix internal warning for features triggered by users that are
|
||||||
/// not actually implemented yet, and without which eval can not
|
/// not actually implemented yet, and without which eval can not
|
||||||
/// proceed.
|
/// proceed.
|
||||||
|
@ -176,6 +179,13 @@ impl ErrorKind {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl From<serde_json::Error> for ErrorKind {
|
||||||
|
fn from(err: serde_json::Error) -> Self {
|
||||||
|
// Can't just put the `serde_json::Error` in the ErrorKind since it doesn't impl `Clone`
|
||||||
|
Self::FromJsonError(format!("Error parsing JSON: {err}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct Error {
|
pub struct Error {
|
||||||
pub kind: ErrorKind,
|
pub kind: ErrorKind,
|
||||||
|
@ -343,6 +353,10 @@ to a missing value in the attribute set(s) included via `with`."#,
|
||||||
write!(f, "{error}")
|
write!(f, "{error}")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ErrorKind::FromJsonError(msg) => {
|
||||||
|
write!(f, "Error converting JSON to a Nix value: {msg}")
|
||||||
|
}
|
||||||
|
|
||||||
ErrorKind::NotImplemented(feature) => {
|
ErrorKind::NotImplemented(feature) => {
|
||||||
write!(f, "feature not yet implemented in Tvix: {}", feature)
|
write!(f, "feature not yet implemented in Tvix: {}", feature)
|
||||||
}
|
}
|
||||||
|
@ -621,6 +635,7 @@ impl Error {
|
||||||
| ErrorKind::ImportParseError { .. }
|
| ErrorKind::ImportParseError { .. }
|
||||||
| ErrorKind::ImportCompilerError { .. }
|
| ErrorKind::ImportCompilerError { .. }
|
||||||
| ErrorKind::IO { .. }
|
| ErrorKind::IO { .. }
|
||||||
|
| ErrorKind::FromJsonError(_)
|
||||||
| ErrorKind::NotImplemented(_) => return None,
|
| ErrorKind::NotImplemented(_) => return None,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -659,6 +674,7 @@ impl Error {
|
||||||
ErrorKind::ImportParseError { .. } => "E027",
|
ErrorKind::ImportParseError { .. } => "E027",
|
||||||
ErrorKind::ImportCompilerError { .. } => "E028",
|
ErrorKind::ImportCompilerError { .. } => "E028",
|
||||||
ErrorKind::IO { .. } => "E029",
|
ErrorKind::IO { .. } => "E029",
|
||||||
|
ErrorKind::FromJsonError { .. } => "E030",
|
||||||
|
|
||||||
// Placeholder error while Tvix is under construction.
|
// Placeholder error while Tvix is under construction.
|
||||||
ErrorKind::NotImplemented(_) => "E999",
|
ErrorKind::NotImplemented(_) => "E999",
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
"quote \" reverse solidus \\ solidus / backspace formfeed newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace 1 char unicode encoded e with accent é 2 char unicode encoded s with caron š 3 char unicode encoded rightwards arrow →"
|
|
@ -0,0 +1,3 @@
|
||||||
|
# This string contains all supported escapes in a JSON string, per json.org
|
||||||
|
# \b and \f are not supported by Nix
|
||||||
|
builtins.fromJSON ''"quote \" reverse solidus \\ solidus \/ backspace \b formfeed \f newline \n carriage return \r horizontal tab \t 1 char unicode encoded backspace \u0008 1 char unicode encoded e with accent \u00e9 2 char unicode encoded s with caron \u0161 3 char unicode encoded rightwards arrow \u2192"''
|
1
tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.exp
Normal file
1
tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.exp
Normal file
|
@ -0,0 +1 @@
|
||||||
|
[ { Image = { Animated = false; Height = 600; IDs = [ 116 943 234 38793 true false null -100 ]; Latitude = 37.7668; Longitude = -122.3959; Thumbnail = { Height = 125; Url = "http://www.example.com/image/481989943"; Width = 100; }; Title = "View from 15th Floor"; Width = 800; }; } { name = "a"; value = "b"; } ]
|
23
tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.nix
Normal file
23
tvix/eval/src/tests/tvix_tests/eval-okay-fromjson.nix
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
[
|
||||||
|
# RFC 7159, section 13.
|
||||||
|
(builtins.fromJSON
|
||||||
|
''
|
||||||
|
{
|
||||||
|
"Image": {
|
||||||
|
"Width": 800,
|
||||||
|
"Height": 600,
|
||||||
|
"Title": "View from 15th Floor",
|
||||||
|
"Thumbnail": {
|
||||||
|
"Url": "http://www.example.com/image/481989943",
|
||||||
|
"Height": 125,
|
||||||
|
"Width": 100
|
||||||
|
},
|
||||||
|
"Animated" : false,
|
||||||
|
"IDs": [116, 943, 234, 38793, true ,false,null, -100],
|
||||||
|
"Latitude": 37.7668,
|
||||||
|
"Longitude": -122.3959
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'')
|
||||||
|
(builtins.fromJSON ''{"name": "a", "value": "b"}'')
|
||||||
|
]
|
|
@ -274,6 +274,12 @@ impl NixAttrs {
|
||||||
NixAttrs(AttrsRep::Map(map))
|
NixAttrs(AttrsRep::Map(map))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Construct an optimized "KV"-style attribute set given the value for the
|
||||||
|
/// `"name"` key, and the value for the `"value"` key
|
||||||
|
pub(crate) fn from_kv(name: Value, value: Value) -> Self {
|
||||||
|
NixAttrs(AttrsRep::KV { name, value })
|
||||||
|
}
|
||||||
|
|
||||||
/// Compare `self` against `other` for equality using Nix equality semantics
|
/// Compare `self` against `other` for equality using Nix equality semantics
|
||||||
pub fn nix_eq(&self, other: &Self, vm: &mut VM) -> Result<bool, ErrorKind> {
|
pub fn nix_eq(&self, other: &Self, vm: &mut VM) -> Result<bool, ErrorKind> {
|
||||||
match (&self.0, &other.0) {
|
match (&self.0, &other.0) {
|
||||||
|
@ -376,10 +382,10 @@ fn attempt_optimise_kv(slice: &mut [Value]) -> Option<NixAttrs> {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
Some(NixAttrs(AttrsRep::KV {
|
Some(NixAttrs::from_kv(
|
||||||
name: slice[name_idx].clone(),
|
slice[name_idx].clone(),
|
||||||
value: slice[value_idx].clone(),
|
slice[value_idx].clone(),
|
||||||
}))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Set an attribute on an in-construction attribute set, while
|
/// Set an attribute on an in-construction attribute set, while
|
||||||
|
|
|
@ -390,6 +390,54 @@ impl From<PathBuf> for Value {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl From<Vec<Value>> for Value {
|
||||||
|
fn from(val: Vec<Value>) -> Self {
|
||||||
|
Self::List(NixList::from(val))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TryFrom<serde_json::Value> for Value {
|
||||||
|
type Error = ErrorKind;
|
||||||
|
|
||||||
|
fn try_from(value: serde_json::Value) -> Result<Self, Self::Error> {
|
||||||
|
// TODO(grfn): Replace with a real serde::Deserialize impl (for perf)
|
||||||
|
match value {
|
||||||
|
serde_json::Value::Null => Ok(Self::Null),
|
||||||
|
serde_json::Value::Bool(b) => Ok(Self::Bool(b)),
|
||||||
|
serde_json::Value::Number(n) => {
|
||||||
|
if let Some(i) = n.as_i64() {
|
||||||
|
Ok(Self::Integer(i))
|
||||||
|
} else if let Some(f) = n.as_f64() {
|
||||||
|
Ok(Self::Float(f))
|
||||||
|
} else {
|
||||||
|
Err(ErrorKind::FromJsonError(format!(
|
||||||
|
"JSON number not representable as Nix value: {n}"
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serde_json::Value::String(s) => Ok(s.into()),
|
||||||
|
serde_json::Value::Array(a) => Ok(a
|
||||||
|
.into_iter()
|
||||||
|
.map(Value::try_from)
|
||||||
|
.collect::<Result<Vec<_>, _>>()?
|
||||||
|
.into()),
|
||||||
|
serde_json::Value::Object(obj) => {
|
||||||
|
match (obj.len(), obj.get("name"), obj.get("value")) {
|
||||||
|
(2, Some(name), Some(value)) => Ok(Self::attrs(NixAttrs::from_kv(
|
||||||
|
name.clone().try_into()?,
|
||||||
|
value.clone().try_into()?,
|
||||||
|
))),
|
||||||
|
_ => Ok(Self::attrs(NixAttrs::from_map(
|
||||||
|
obj.into_iter()
|
||||||
|
.map(|(k, v)| Ok((k.into(), v.try_into()?)))
|
||||||
|
.collect::<Result<_, ErrorKind>>()?,
|
||||||
|
))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn type_error(expected: &'static str, actual: &Value) -> ErrorKind {
|
fn type_error(expected: &'static str, actual: &Value) -> ErrorKind {
|
||||||
ErrorKind::TypeError {
|
ErrorKind::TypeError {
|
||||||
expected,
|
expected,
|
||||||
|
|
Loading…
Reference in a new issue