tvl-depot/users/edef/weave/Cargo.toml

[package]
name = "weave"
version = "0.1.0"
edition = "2021"

[workspace]
members = ["."]

# TODO(edef): cut down on required features, this is kind of a grab bag right now
[dependencies]
anyhow = { version = "1.0.79", features = ["backtrace"] }
hashbrown = "0.14.3"
nix-compat = { version = "0.1.0", path = "../../../tvix/nix-compat" }
safer_owning_ref = "0.5.0"
rayon = "1.8.1"
rustc-hash = "2.0.0"
tvix-tracing = { version = "0.1.0", path = "../../../tvix/tracing" }
tracing = "0.1.40"
tracing-indicatif = "0.3.6"

[dependencies.polars]
version = "0.36.2"
features = ["parquet", "lazy", "streaming"]
feat(tvix/tools/weave): init Scalable tracing GC for the cache.nixos.org dataset. Change-Id: I6c7852796f28e1a1c7607384ffb55f44407e1185 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10765 Tested-by: BuildkiteCI Reviewed-by: flokli <flokli@flokli.de> 2024-02-15 23:20:47 +01:00			`[package]`
			`name = "weave"`
			`version = "0.1.0"`
			`edition = "2021"`

			`[workspace]`
			`members = ["."]`

			`# TODO(edef): cut down on required features, this is kind of a grab bag right now`
			`[dependencies]`
			`anyhow = { version = "1.0.79", features = ["backtrace"] }`
			`hashbrown = "0.14.3"`
chore(tvix/tools): move weave to //users/edef This is not a core Tvix tool, it's a tool that uses a Tvix component. Change-Id: I705f2c4ab87f1512e005007c933e16b84ed4279f Reviewed-on: https://cl.tvl.fyi/c/depot/+/12605 Reviewed-by: edef <edef@edef.eu> Tested-by: BuildkiteCI Autosubmit: tazjin <tazjin@tvl.su> 2024-10-13 00:57:58 +02:00			`nix-compat = { version = "0.1.0", path = "../../../tvix/nix-compat" }`
fix(users/edef/weave): use safer_owning_ref owning_ref has serious unsoundness. Change-Id: Ie760697cd6399e6bc75f1ad17c9bb74adc077a35 Reviewed-on: https://cl.tvl.fyi/c/depot/+/12656 Reviewed-by: edef <edef@edef.eu> Tested-by: BuildkiteCI Autosubmit: edef <edef@edef.eu> Reviewed-by: flokli <flokli@flokli.de> 2024-10-18 00:37:05 +02:00			`safer_owning_ref = "0.5.0"`
feat(tvix/tools/weave): init Scalable tracing GC for the cache.nixos.org dataset. Change-Id: I6c7852796f28e1a1c7607384ffb55f44407e1185 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10765 Tested-by: BuildkiteCI Reviewed-by: flokli <flokli@flokli.de> 2024-02-15 23:20:47 +01:00			`rayon = "1.8.1"`
feat(users/edef/weave): use FxHashSet and dedupe early Deduping early saves a fair bit of memory, but the extra hashing is costly. We switch to FxHash, since we don't need a DoS-proof hash, but we do need it to be fast. Change-Id: Ic6b7010874c417862baa9b882593208c8dd1d5e6 Reviewed-on: https://cl.tvl.fyi/c/depot/+/12648 Reviewed-by: flokli <flokli@flokli.de> Tested-by: BuildkiteCI 2024-10-19 15:58:53 +02:00			`rustc-hash = "2.0.0"`
feat(users/edef/weave): use tracing_indicatif for progress Progress bars :3 Change-Id: I770d0f8381521b6efc8b38c0db4d59c771887fee Reviewed-on: https://cl.tvl.fyi/c/depot/+/12673 Tested-by: BuildkiteCI Reviewed-by: flokli <flokli@flokli.de> 2024-10-19 21:23:56 +02:00			`tvix-tracing = { version = "0.1.0", path = "../../../tvix/tracing" }`
			`tracing = "0.1.40"`
			`tracing-indicatif = "0.3.6"`
feat(tvix/tools/weave): init Scalable tracing GC for the cache.nixos.org dataset. Change-Id: I6c7852796f28e1a1c7607384ffb55f44407e1185 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10765 Tested-by: BuildkiteCI Reviewed-by: flokli <flokli@flokli.de> 2024-02-15 23:20:47 +01:00
			`[dependencies.polars]`
			`version = "0.36.2"`
refactor(users/edef/weave/swizzle): use polars streaming This vastly reduces the memory requirements, so we can run in ~40G RAM. Change-Id: I4952a780df294bd852a8b4682ba2fd59b9bae675 Reviewed-on: https://cl.tvl.fyi/c/depot/+/12667 Reviewed-by: flokli <flokli@flokli.de> Tested-by: BuildkiteCI 2024-10-17 15:26:01 +02:00			`features = ["parquet", "lazy", "streaming"]`