2022-08-04 15:29:38 +02:00
|
|
|
[package]
|
|
|
|
|
name = "tvix-eval"
|
|
|
|
|
version = "0.1.0"
|
|
|
|
|
edition = "2021"
|
|
|
|
|
|
2022-09-03 18:40:01 +02:00
|
|
|
[lib]
|
|
|
|
|
name = "tvix_eval"
|
|
|
|
|
|
2022-08-04 15:29:38 +02:00
|
|
|
[dependencies]
|
2022-12-29 12:44:09 +01:00
|
|
|
builtin-macros = { path = "./builtin-macros", package = "tvix-eval-builtin-macros" }
|
2023-07-19 17:52:50 +02:00
|
|
|
bytes = "1.4.0"
|
fix(tvix): Represent strings as byte arrays
C++ nix uses C-style zero-terminated char pointers to represent strings
internally - however, up to this point, tvix has used Rust `String` and
`str` for string values. Since those are required to be valid utf-8, we
haven't been able to properly represent all the string values that Nix
supports.
To fix that, this change converts the internal representation of the
NixString struct from `Box<str>` to `BString`, from the `bstr` crate -
this is a wrapper around a `Vec<u8>` with extra functions for treating
that byte vector as a "morally string-like" value, which is basically
exactly what we need.
Since this changes a pretty fundamental assumption about a pretty core
type, there are a *lot* of changes in a lot of places to make this work,
but I've tried to keep the general philosophy and intent of most of the
code in most places intact. Most notably, there's nothing that's been
done to make the derivation stuff in //tvix/glue work with non-utf8
strings everywhere, instead opting to just convert to String/str when
passing things into that - there *might* be something to be done there,
but I don't know what the rules should be and I don't want to figure
them out in this change.
To deal with OS-native paths in a way that also works in WASM for
tvixbolt, this also adds a dependency on the "os_str_bytes" crate.
Fixes: b/189
Fixes: b/337
Change-Id: I5e6eb29c62f47dd91af954f5e12bfc3d186f5526
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10200
Reviewed-by: tazjin <tazjin@tvl.su>
Reviewed-by: flokli <flokli@flokli.de>
Reviewed-by: sterni <sternenseemann@systemli.org>
Autosubmit: aspen <root@gws.fyi>
Tested-by: BuildkiteCI
2023-12-05 23:25:52 +01:00
|
|
|
bstr = { version = "1.8.0", features = ["serde"] }
|
2022-09-01 14:52:48 +02:00
|
|
|
codemap = "0.1.3"
|
2022-09-11 23:34:25 +02:00
|
|
|
codemap-diagnostic = "0.1.1"
|
2022-12-29 12:44:09 +01:00
|
|
|
dirs = "4.0.0"
|
2024-08-07 13:09:37 +02:00
|
|
|
genawaiter = { version = "0.99.1", default-features = false }
|
2024-07-07 16:51:26 +02:00
|
|
|
imbl = { version = "3.0", features = [ "serde" ] }
|
fix(tvix/eval): `getContext` merges underlying values
Previously, we were assembling very naively an attribute set composed of context we saw.
But it was forgetting that `"${drv}${drv.drvPath}"` would contain 2 contexts with the same key, but
with different values, one with `outputs = [ "out" ];` and `allOutputs = true;`.
Following this reasoning and comparing with what Nix does, we ought to merge underlying values systematically.
Hence, I bring `itertools` to perform a group by on the key and merge everything on the fly, it's not
beautiful but it's the best I could find, notice that I don't use
`group_by` but I talk about group by, that is, because `group_by` is a
`group_by_consecutive`, see
https://github.com/rust-itertools/itertools/issues/374.
Initially, I tried to do it without a `into_grouping_map_by`, it was akin to assemble the final `NixAttrs` directly,
it was less readable and harder to pull out because we don't have a lot of in-place mutable functions on
our data structures.
Change-Id: I9933c9bd88ffe04de50dda14f21879b60d8b8cd4
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10620
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
2024-01-14 02:40:07 +01:00
|
|
|
itertools = "0.12.0"
|
feat(tvix/eval): use lexical-core to format float
Apparently our naive implementation of float formatting, which simply
used {:.5}, and trimmed trailing "0" strings not sufficient.
It wrongly trimmed numbers with zeroes but no decimal point, like
`10000` got trimmed to `1`.
Nix uses `std::to_string` on the double, which according to
https://en.cppreference.com/w/cpp/string/basic_string/to_string
is equivalent to `std::sprintf(buf, "%f", value)`.
https://en.cppreference.com/w/cpp/io/c/fprintf mentions this is treated
like this:
> Precision specifies the exact number of digits to appear after
> the decimal point character. The default precision is 6. In the
> alternative implementation decimal point character is written even if
> no digits follow it. For infinity and not-a-number conversion style
> see notes.
This doesn't seem to be the case though, and Nix uses scientific
notation in some cases.
There's a whole bunch of strategies to determine which is a more compact
notation, and which notation should be used for a given number.
https://github.com/rust-lang/rust/issues/24556 provides some pointers
into various rabbit holes for those interested.
This gist seems to be that currently a different formatting is not
exposed in rust directly, at least not for public consumption.
There is the
[lexical-core](https://github.com/Alexhuszagh/rust-lexical) crate
though, which provides a way to format floats with various strategies
and formats.
Change our implementation of `TotalDisplay` for the `Value::Float` case
to use that. We still need to do some post-processing, because Nix
always adds the sign in scientific notation (and there's no way to
configure lexical-core to do that), and lexical-core in some cases keeps
the trailing zeros.
Even with all that in place, there as a difference in `eval-okay-
fromjson.nix` (from tvix-tests), which I couldn't get to work. I updated
the fixture to a less problematic number.
With this, the testsuite passes again, and does for the upcoming CL
introducing builtins.fromTOML, and enabling the nix testsuite bits for
it, too.
Change-Id: Ie6fba5619e1d9fd7ce669a51594658b029057acc
Reviewed-on: https://cl.tvl.fyi/c/depot/+/7922
Tested-by: BuildkiteCI
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: tazjin <tazjin@tvl.su>
2023-01-24 19:27:20 +01:00
|
|
|
lazy_static = "1.4.0"
|
|
|
|
|
lexical-core = { version = "0.8.5", features = ["format", "parse-floats"] }
|
fix(tvix): Represent strings as byte arrays
C++ nix uses C-style zero-terminated char pointers to represent strings
internally - however, up to this point, tvix has used Rust `String` and
`str` for string values. Since those are required to be valid utf-8, we
haven't been able to properly represent all the string values that Nix
supports.
To fix that, this change converts the internal representation of the
NixString struct from `Box<str>` to `BString`, from the `bstr` crate -
this is a wrapper around a `Vec<u8>` with extra functions for treating
that byte vector as a "morally string-like" value, which is basically
exactly what we need.
Since this changes a pretty fundamental assumption about a pretty core
type, there are a *lot* of changes in a lot of places to make this work,
but I've tried to keep the general philosophy and intent of most of the
code in most places intact. Most notably, there's nothing that's been
done to make the derivation stuff in //tvix/glue work with non-utf8
strings everywhere, instead opting to just convert to String/str when
passing things into that - there *might* be something to be done there,
but I don't know what the rules should be and I don't want to figure
them out in this change.
To deal with OS-native paths in a way that also works in WASM for
tvixbolt, this also adds a dependency on the "os_str_bytes" crate.
Fixes: b/189
Fixes: b/337
Change-Id: I5e6eb29c62f47dd91af954f5e12bfc3d186f5526
Reviewed-on: https://cl.tvl.fyi/c/depot/+/10200
Reviewed-by: tazjin <tazjin@tvl.su>
Reviewed-by: flokli <flokli@flokli.de>
Reviewed-by: sterni <sternenseemann@systemli.org>
Autosubmit: aspen <root@gws.fyi>
Tested-by: BuildkiteCI
2023-12-05 23:25:52 +01:00
|
|
|
os_str_bytes = { version = "6.3", features = ["conversions"] }
|
2022-12-29 12:44:09 +01:00
|
|
|
path-clean = "0.1"
|
2024-08-07 13:09:37 +02:00
|
|
|
proptest = { version = "1.3.0", default-features = false, features = ["std", "alloc", "tempfile"], optional = true }
|
2022-10-13 00:45:52 +02:00
|
|
|
regex = "1.6.0"
|
2022-12-06 14:31:09 +01:00
|
|
|
rnix = "0.11.0"
|
2022-12-29 12:44:09 +01:00
|
|
|
rowan = "*" # pinned by rnix
|
2022-12-24 18:18:26 +01:00
|
|
|
serde = { version = "1.0", features = [ "rc", "derive" ] }
|
2022-12-29 12:44:09 +01:00
|
|
|
serde_json = "1.0"
|
2023-05-11 17:06:32 +02:00
|
|
|
smol_str = "0.2.0"
|
2022-12-29 12:44:09 +01:00
|
|
|
tabwriter = "1.2"
|
|
|
|
|
test-strategy = { version = "0.2.1", optional = true }
|
2023-01-24 15:54:29 +01:00
|
|
|
toml = "0.6.0"
|
2024-02-21 17:49:07 +01:00
|
|
|
sha2 = "0.10.8"
|
|
|
|
|
sha1 = "0.10.6"
|
|
|
|
|
md-5 = "0.10.6"
|
2024-06-30 18:54:27 +02:00
|
|
|
data-encoding = "2.6.0"
|
2024-07-28 18:11:41 +02:00
|
|
|
rustc-hash = "2.0.0"
|
2024-07-28 18:50:09 +02:00
|
|
|
nohash-hasher = "0.2.0"
|
2022-08-10 17:52:42 +02:00
|
|
|
|
|
|
|
|
[dev-dependencies]
|
2023-12-09 18:26:40 +01:00
|
|
|
criterion = "0.5"
|
2024-01-06 00:33:36 +01:00
|
|
|
itertools = "0.12.0"
|
2024-08-07 23:21:26 +02:00
|
|
|
mimalloc = "0.1.43"
|
2022-08-16 14:33:50 +02:00
|
|
|
pretty_assertions = "1.2.1"
|
2024-04-10 13:37:45 +02:00
|
|
|
rstest = "0.19.0"
|
2023-10-08 13:03:22 +02:00
|
|
|
tempfile = "3.3.0"
|
2022-08-10 18:18:01 +02:00
|
|
|
|
|
|
|
|
[features]
|
2023-03-11 23:39:31 +01:00
|
|
|
default = ["impure", "arbitrary", "nix_tests"]
|
2022-09-03 18:40:01 +02:00
|
|
|
|
2022-08-10 18:18:01 +02:00
|
|
|
# Enables running the Nix language test suite from the original C++
|
|
|
|
|
# Nix implementation (at version 2.3) against Tvix.
|
|
|
|
|
nix_tests = []
|
2022-08-25 17:04:07 +02:00
|
|
|
|
2022-09-18 22:34:41 +02:00
|
|
|
# Enables operations in the VM which depend on the ability to perform I/O
|
|
|
|
|
impure = []
|
|
|
|
|
|
2022-09-17 19:52:02 +02:00
|
|
|
# Enables Arbitrary impls for internal types (required to run tests)
|
2023-01-24 15:54:29 +01:00
|
|
|
arbitrary = ["proptest", "test-strategy", "imbl/proptest"]
|
2022-09-17 19:52:02 +02:00
|
|
|
|
2024-07-28 17:37:50 +02:00
|
|
|
# Don't leak strings (enable this if you care about peak memory usage of eval)
|
|
|
|
|
#
|
|
|
|
|
# This is intended as a stop-gap until we have a garbage collector
|
|
|
|
|
no_leak = []
|
|
|
|
|
|
2022-08-25 17:04:07 +02:00
|
|
|
[[bench]]
|
|
|
|
|
name = "eval"
|
|
|
|
|
harness = false
|
