9498ac936e
Use a proper graph library to ensure all nodes are reachable from the root. We had a bit of that handrolled during add(), as well as later, which had an annoying bug: Redundant nodes were omitted during insert, but when returning the list during finalize, we did not properly account they need to be introduced before their parents are sent. We now simply populate a petgraph DiGraph during insert (skipping inserting nodes we already saw), and use petgraph's DfsPostOrder to traverse the graph during finalize. If the number of returned indices equals the total number of nodes in the graph, all nodes are reachable from the root, we can consume the graph and return the nodes as a vec, in the same order as the traversal (and insertion). Providing a regression test for the initial bug is challenging, as the current code uses a bunch of HashSets. I manually tested ingesting a full NixOS closure using this mechanism (via gRPC, which exposes this problem, as it validates twice), and it now works. Change-Id: Ic1d5e3e981f2993cc08c5c6b60ad895e578326dc Reviewed-on: https://cl.tvl.fyi/c/depot/+/11418 Autosubmit: flokli <flokli@flokli.de> Reviewed-by: Connor Brewster <cbrewster@hey.com> Tested-by: BuildkiteCI
113 lines
2.4 KiB
TOML
113 lines
2.4 KiB
TOML
[package]
|
|
name = "tvix-castore"
|
|
version = "0.1.0"
|
|
edition = "2021"
|
|
|
|
[dependencies]
|
|
async-stream = "0.3.5"
|
|
async-tempfile = "0.4.0"
|
|
blake3 = { version = "1.3.1", features = ["rayon", "std", "traits-preview"] }
|
|
bstr = "1.6.0"
|
|
bytes = "1.4.0"
|
|
data-encoding = "2.3.3"
|
|
digest = "0.10.7"
|
|
fastcdc = { version = "3.1.0", features = ["tokio"] }
|
|
futures = "0.3.30"
|
|
lazy_static = "1.4.0"
|
|
object_store = { version = "0.9.1", features = ["http"] }
|
|
parking_lot = "0.12.1"
|
|
pin-project-lite = "0.2.13"
|
|
prost = "0.12.1"
|
|
sled = { version = "0.34.7" }
|
|
thiserror = "1.0.38"
|
|
tokio-stream = { version = "0.1.14", features = ["fs", "net"] }
|
|
tokio-util = { version = "0.7.9", features = ["io", "io-util"] }
|
|
tokio = { version = "1.32.0", features = ["fs", "macros", "net", "rt", "rt-multi-thread", "signal"] }
|
|
tonic = "0.11.0"
|
|
tower = "0.4.13"
|
|
tracing = "0.1.37"
|
|
url = "2.4.0"
|
|
walkdir = "2.4.0"
|
|
zstd = "0.13.0"
|
|
serde = { version = "1.0.197", features = [ "derive" ] }
|
|
serde_with = "3.7.0"
|
|
serde_qs = "0.12.0"
|
|
petgraph = "0.6.4"
|
|
|
|
[dependencies.bigtable_rs]
|
|
optional = true
|
|
# https://github.com/liufuyang/bigtable_rs/pull/72
|
|
git = "https://github.com/flokli/bigtable_rs"
|
|
rev = "0af404741dfc40eb9fa99cf4d4140a09c5c20df7"
|
|
|
|
[dependencies.fuse-backend-rs]
|
|
optional = true
|
|
version = "0.11.0"
|
|
|
|
[dependencies.libc]
|
|
optional = true
|
|
version = "0.2.144"
|
|
|
|
[dependencies.tonic-reflection]
|
|
optional = true
|
|
version = "0.11.0"
|
|
|
|
[dependencies.vhost]
|
|
optional = true
|
|
version = "0.6"
|
|
|
|
[dependencies.vhost-user-backend]
|
|
optional = true
|
|
version = "0.8"
|
|
|
|
[dependencies.virtio-queue]
|
|
optional = true
|
|
version = "0.7"
|
|
|
|
[dependencies.vm-memory]
|
|
optional = true
|
|
version = "0.10"
|
|
|
|
[dependencies.vmm-sys-util]
|
|
optional = true
|
|
version = "0.11"
|
|
|
|
[dependencies.virtio-bindings]
|
|
optional = true
|
|
version = "0.2.1"
|
|
|
|
[build-dependencies]
|
|
prost-build = "0.12.1"
|
|
tonic-build = "0.11.0"
|
|
|
|
[dev-dependencies]
|
|
async-process = "2.1.0"
|
|
rstest = "0.19.0"
|
|
tempfile = "3.3.0"
|
|
tokio-retry = "0.3.0"
|
|
hex-literal = "0.4.1"
|
|
rstest_reuse = "0.6.0"
|
|
xattr = "1.3.1"
|
|
|
|
[features]
|
|
default = []
|
|
cloud = [
|
|
"dep:bigtable_rs",
|
|
"object_store/aws",
|
|
"object_store/azure",
|
|
"object_store/gcp",
|
|
]
|
|
fs = ["dep:libc", "dep:fuse-backend-rs"]
|
|
virtiofs = [
|
|
"fs",
|
|
"dep:vhost",
|
|
"dep:vhost-user-backend",
|
|
"dep:virtio-queue",
|
|
"dep:vm-memory",
|
|
"dep:vmm-sys-util",
|
|
"dep:virtio-bindings",
|
|
"fuse-backend-rs?/vhost-user-fs", # impl FsCacheReqHandler for SlaveFsCacheReq
|
|
"fuse-backend-rs?/virtiofs",
|
|
]
|
|
fuse = ["fs"]
|
|
tonic-reflection = ["dep:tonic-reflection"]
|