mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/eval): implement builtins.hashString

Browse source 
Implements md5, sha1, sha256 and sha512 using the related crates from
the RustCrypto hashes project (https://github.com/RustCrypto/hashes)

Change-Id: I00730dea44ec9ef85309edc27addab0ae88814b8
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11005
Tested-by: BuildkiteCI
Reviewed-by: aspen <root@gws.fyi>
This commit is contained in:
Padraic-O-Mhuiris 2024-02-21 16:49:07 +00:00 committed by Pádraic Ó Mhuiris
parent ffb134398d
commit 5c3065b43a
14 changed files with 263 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
tvix/Cargo.lock generated
View file

@ -1416,6 +1416,16 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
"digest",
]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.7.1" version = "2.7.1"
@ -2543,6 +2553,17 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "sha1"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]] [[package]]
name = "sha2" name = "sha2"
version = "0.10.8" version = "0.10.8"
@ -3346,12 +3367,14 @@ dependencies = [
"codemap", "codemap",
"codemap-diagnostic", "codemap-diagnostic",
"criterion", "criterion",
"data-encoding",
"dirs", "dirs",
"genawaiter", "genawaiter",
"imbl", "imbl",
"itertools 0.12.0", "itertools 0.12.0",
"lazy_static", "lazy_static",
"lexical-core", "lexical-core",
"md-5",
"os_str_bytes", "os_str_bytes",
"path-clean", "path-clean",
"pretty_assertions", "pretty_assertions",
@ -3362,6 +3385,8 @@ dependencies = [
"rstest", "rstest",
"serde", "serde",
"serde_json", "serde_json",
"sha1",
"sha2",
"smol_str", "smol_str",
"tabwriter", "tabwriter",
"tempfile", "tempfile",

90
tvix/Cargo.nix
View file

@ -4191,6 +4191,41 @@ rec {
features = { }; features = { };
resolvedDefaultFeatures = [ "default" ]; resolvedDefaultFeatures = [ "default" ];
}; };
"md-5" = rec {
crateName = "md-5";
version = "0.10.6";
edition = "2018";
sha256 = "1kvq5rnpm4fzwmyv5nmnxygdhhb2369888a06gdc9pxyrzh7x7nq";
libName = "md5";
authors = [
"RustCrypto Developers"
];
dependencies = [
{
name = "cfg-if";
packageId = "cfg-if";
}
{
name = "digest";
packageId = "digest";
}
];
devDependencies = [
{
name = "digest";
packageId = "digest";
features = [ "dev" ];
}
];
features = {
"asm" = [ "md5-asm" ];
"default" = [ "std" ];
"md5-asm" = [ "dep:md5-asm" ];
"oid" = [ "digest/oid" ];
"std" = [ "digest/std" ];
};
resolvedDefaultFeatures = [ "default" "std" ];
};
"memchr" = rec { "memchr" = rec {
crateName = "memchr"; crateName = "memchr";
version = "2.7.1"; version = "2.7.1";
@ -7763,6 +7798,45 @@ rec {
]; ];
}; };
"sha1" = rec {
crateName = "sha1";
version = "0.10.6";
edition = "2018";
sha256 = "1fnnxlfg08xhkmwf2ahv634as30l1i3xhlhkvxflmasi5nd85gz3";
authors = [
"RustCrypto Developers"
];
dependencies = [
{
name = "cfg-if";
packageId = "cfg-if";
}
{
name = "cpufeatures";
packageId = "cpufeatures";
target = { target, features }: (("aarch64" == target."arch" or null) || ("x86" == target."arch" or null) || ("x86_64" == target."arch" or null));
}
{
name = "digest";
packageId = "digest";
}
];
devDependencies = [
{
name = "digest";
packageId = "digest";
features = [ "dev" ];
}
];
features = {
"asm" = [ "sha1-asm" ];
"default" = [ "std" ];
"oid" = [ "digest/oid" ];
"sha1-asm" = [ "dep:sha1-asm" ];
"std" = [ "digest/std" ];
};
resolvedDefaultFeatures = [ "default" "std" ];
};
"sha2" = rec { "sha2" = rec {
crateName = "sha2"; crateName = "sha2";
version = "0.10.8"; version = "0.10.8";
@ -10493,6 +10567,10 @@ rec {
name = "codemap-diagnostic"; name = "codemap-diagnostic";
packageId = "codemap-diagnostic"; packageId = "codemap-diagnostic";
} }
{
name = "data-encoding";
packageId = "data-encoding";
}
{ {
name = "dirs"; name = "dirs";
packageId = "dirs"; packageId = "dirs";
@ -10520,6 +10598,10 @@ rec {
packageId = "lexical-core"; packageId = "lexical-core";
features = [ "format" "parse-floats" ]; features = [ "format" "parse-floats" ];
} }
{
name = "md-5";
packageId = "md-5";
}
{ {
name = "os_str_bytes"; name = "os_str_bytes";
packageId = "os_str_bytes"; packageId = "os_str_bytes";
@ -10557,6 +10639,14 @@ rec {
name = "serde_json"; name = "serde_json";
packageId = "serde_json"; packageId = "serde_json";
} }
{
name = "sha1";
packageId = "sha1";
}
{
name = "sha2";
packageId = "sha2";
}
{ {
name = "smol_str"; name = "smol_str";
packageId = "smol_str"; packageId = "smol_str";

4
tvix/eval/Cargo.toml
View file

@ -33,6 +33,10 @@ tabwriter = "1.2"
test-strategy = { version = "0.2.1", optional = true } test-strategy = { version = "0.2.1", optional = true }
toml = "0.6.0" toml = "0.6.0"
xml-rs = "0.8.4" xml-rs = "0.8.4"
sha2 = "0.10.8"
sha1 = "0.10.6"
md-5 = "0.10.6"
data-encoding = "2.5.0"
[dev-dependencies] [dev-dependencies]
criterion = "0.5" criterion = "0.5"

2
tvix/eval/docs/builtins.md
View file

@ -66,7 +66,7 @@ The `impl` column indicates implementation status in tvix:
| hasAttr | false | | | | | hasAttr | false | | | |
| hasContext | false | | | | | hasContext | false | | | |
| hashFile | false | | false | todo | | hashFile | false | | false | todo |
| hashString | false | | | todo | | hashString | false | | | |
| head | false | | | | | head | false | | | |
| import | true | | | | | import | true | | | |
| intersectAttrs | false | | | | | intersectAttrs | false | | | |

32
tvix/eval/src/builtins/mod.rs
View file

@ -5,9 +5,14 @@
use bstr::{ByteSlice, ByteVec}; use bstr::{ByteSlice, ByteVec};
use builtin_macros::builtins; use builtin_macros::builtins;
use data_encoding::HEXLOWER;
use genawaiter::rc::Gen; use genawaiter::rc::Gen;
use imbl::OrdMap; use imbl::OrdMap;
use md5::Md5;
use regex::Regex; use regex::Regex;
use sha1::Sha1;
use sha2::digest::Output;
use sha2::{Digest, Sha256, Sha512};
use std::cmp::{self, Ordering}; use std::cmp::{self, Ordering};
use std::collections::VecDeque; use std::collections::VecDeque;
use std::collections::{BTreeMap, HashSet}; use std::collections::{BTreeMap, HashSet};
@ -686,15 +691,24 @@ mod pure_builtins {
#[builtin("hashString")] #[builtin("hashString")]
#[allow(non_snake_case)] #[allow(non_snake_case)]
async fn builtin_hashString( async fn builtin_hashString(co: GenCo, algo: Value, s: Value) -> Result<Value, ErrorKind> {
co: GenCo, fn hash<D: Digest>(b: &[u8]) -> Output<D> {
_algo: Value, let mut hasher = D::new();
_string: Value, hasher.update(b);
) -> Result<Value, ErrorKind> { hasher.finalize()
// FIXME: propagate contexts here. }
Ok(Value::from(CatchableErrorKind::UnimplementedFeature(
"hashString".into(), let s = s.to_str()?;
)))
let encoded_hash = match algo.to_str()?.as_bytes() {
b"md5" => HEXLOWER.encode(hash::<Md5>(&s).as_bstr()),
b"sha1" => HEXLOWER.encode(hash::<Sha1>(&s).as_bstr()),
b"sha256" => HEXLOWER.encode(hash::<Sha256>(&s).as_bstr()),
b"sha512" => HEXLOWER.encode(hash::<Sha512>(&s).as_bstr()),
_ => return Err(ErrorKind::UnknownHashType(s.into())),
};
Ok(Value::from(encoded_hash))
} }
#[builtin("head")] #[builtin("head")]

10
tvix/eval/src/errors.rs
View file

@ -229,6 +229,10 @@ pub enum ErrorKind {
/// tvix-eval when returning a result to the user, never inside of /// tvix-eval when returning a result to the user, never inside of
/// eval code. /// eval code.
CatchableError(CatchableErrorKind), CatchableError(CatchableErrorKind),
/// Invalid hash type specified, must be one of "md5", "sha1", "sha256"
/// or "sha512"
UnknownHashType(String),
} }
impl error::Error for Error { impl error::Error for Error {
@ -533,6 +537,10 @@ to a missing value in the attribute set(s) included via `with`."#,
ErrorKind::CatchableError(inner) => { ErrorKind::CatchableError(inner) => {
write!(f, "{}", inner) write!(f, "{}", inner)
} }
ErrorKind::UnknownHashType(hash_type) => {
write!(f, "unknown hash type '{}'", hash_type)
}
} }
} }
} }
@ -821,6 +829,7 @@ impl Error {
| ErrorKind::TvixBug { .. } | ErrorKind::TvixBug { .. }
| ErrorKind::NotImplemented(_) | ErrorKind::NotImplemented(_)
| ErrorKind::WithContext { .. } | ErrorKind::WithContext { .. }
| ErrorKind::UnknownHashType(_)
| ErrorKind::CatchableError(_) => return None, | ErrorKind::CatchableError(_) => return None,
}; };
@ -866,6 +875,7 @@ impl Error {
ErrorKind::NotSerialisableToJson(_) => "E036", ErrorKind::NotSerialisableToJson(_) => "E036",
ErrorKind::UnexpectedContext => "E037", ErrorKind::UnexpectedContext => "E037",
ErrorKind::Utf8 => "E038", ErrorKind::Utf8 => "E038",
ErrorKind::UnknownHashType(_) => "E039",
// Special error code for errors from other Tvix // Special error code for errors from other Tvix
// components. We may want to introduce a code namespacing // components. We may want to introduce a code namespacing

0
tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-groupBy.exp → tvix/eval/src/tests/nix_tests/eval-okay-groupBy.exp
View file

2
tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-groupBy.nix → tvix/eval/src/tests/nix_tests/eval-okay-groupBy.nix
View file

@ -1,4 +1,4 @@
with import ./../lib.nix; with import ./lib.nix;
builtins.groupBy (n: builtins.groupBy (n:
builtins.substring 0 1 (builtins.hashString "sha256" (toString n)) builtins.substring 0 1 (builtins.hashString "sha256" (toString n))

0
tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.exp → tvix/eval/src/tests/nix_tests/eval-okay-hashstring.exp
View file

0
tvix/eval/src/tests/nix_tests/notyetpassing/eval-okay-hashstring.nix → tvix/eval/src/tests/nix_tests/eval-okay-hashstring.nix
View file

1
tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hashString.exp Normal file
View file

@ -0,0 +1 @@
[ "8a0614b4eaa4cffb7515ec101847e198" "8bd218cf61321d8aa05b3602b99f90d2d8cef3d6" "80ac06d74cb6c5d14af718ce8c3c1255969a1a595b76a3cf92354a95331a879a" "0edac513b6b0454705b553deda4c9b055da0939d26d2f73548862817ebeac5378cf64ff7a752ce1a0590a736735d3bbd9e8a7f04d93617cdf514313f5ab5baa4" ]

6
tvix/eval/src/tests/tvix_tests/eval-okay-builtins-hashString.nix Normal file
View file

@ -0,0 +1,6 @@
[
(builtins.hashString "md5" "tvix")
(builtins.hashString "sha1" "tvix")
(builtins.hashString "sha256" "tvix")
(builtins.hashString "sha512" "tvix")
]

6
tvix/eval/src/value/string.rs
View file

@ -524,6 +524,12 @@ impl<'a> From<&'a NixString> for &'a BStr {
} }
} }
impl From<NixString> for String {
fn from(s: NixString) -> Self {
s.to_string()
}
}
impl From<NixString> for BString { impl From<NixString> for BString {
fn from(s: NixString) -> Self { fn from(s: NixString) -> Self {
s.as_bstr().to_owned() s.as_bstr().to_owned()

96
web/tvixbolt/Cargo.lock generated
View file

@ -35,6 +35,15 @@ version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "703642b98a00b3b90513279a8ede3fcfa479c126c5fb46e78f3051522f021403" checksum = "703642b98a00b3b90513279a8ede3fcfa479c126c5fb46e78f3051522f021403"
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]] [[package]]
name = "boolinator" name = "boolinator"
version = "2.4.0" version = "2.4.0"
@ -102,6 +111,41 @@ version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636"
[[package]]
name = "cpufeatures"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
dependencies = [
"libc",
]
[[package]]
name = "crypto-common"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "data-encoding"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5"
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
]
[[package]] [[package]]
name = "dirs" name = "dirs"
version = "4.0.0" version = "4.0.0"
@ -173,6 +217,16 @@ version = "0.99.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc"
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.2.12" version = "0.2.12"
@ -458,6 +512,16 @@ version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
"digest",
]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.7.1" version = "2.7.1"
@ -725,6 +789,28 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "sha1"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "sha2"
version = "0.10.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.9" version = "0.4.9"
@ -857,12 +943,14 @@ dependencies = [
"bytes", "bytes",
"codemap", "codemap",
"codemap-diagnostic", "codemap-diagnostic",
"data-encoding",
"dirs", "dirs",
"genawaiter", "genawaiter",
"imbl", "imbl",
"itertools", "itertools",
"lazy_static", "lazy_static",
"lexical-core", "lexical-core",
"md-5",
"os_str_bytes", "os_str_bytes",
"path-clean", "path-clean",
"regex", "regex",
@ -870,6 +958,8 @@ dependencies = [
"rowan", "rowan",
"serde", "serde",
"serde_json", "serde_json",
"sha1",
"sha2",
"smol_str", "smol_str",
"tabwriter", "tabwriter",
"toml", "toml",
@ -901,6 +991,12 @@ dependencies = [
"yew-router", "yew-router",
] ]
[[package]]
name = "typenum"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]] [[package]]
name = "unicode-ident" name = "unicode-ident"
version = "1.0.12" version = "1.0.12"