chore(tvix/[ca]store): Drop sled support completely in favor of redb
Over the past couple of months we've been using redb instead of sled as the default filesystem-based database in PS and DS. I am confident that we can get rid of sled completely now, and just keep redb. Change-Id: I11fa1e4453e280253855f8eade990b37eb6965ae Reviewed-on: https://cl.tvl.fyi/c/depot/+/12567 Reviewed-by: yuka <yuka@yuka.dev> Reviewed-by: flokli <flokli@flokli.de> Tested-by: BuildkiteCI Autosubmit: Ilan Joselevich <personal@ilanjoselevich.com>
This commit is contained in:
Ilan Joselevich
parent
56b306f884
commit
ab6e8d28aa
14 changed files with 18 additions and 812 deletions
87
tvix/Cargo.lock
generated
87
tvix/Cargo.lock
generated
|
|
@ -524,12 +524,6 @@ version = "3.14.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
|
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "byteorder"
|
|
||||||
version = "1.5.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bytes"
|
name = "bytes"
|
||||||
version = "1.7.1"
|
version = "1.7.1"
|
||||||
|
|
@ -1323,16 +1317,6 @@ dependencies = [
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fs2"
|
|
||||||
version = "0.4.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
|
||||||
dependencies = [
|
|
||||||
"libc",
|
|
||||||
"winapi",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fuse-backend-rs"
|
name = "fuse-backend-rs"
|
||||||
version = "0.12.0"
|
version = "0.12.0"
|
||||||
|
|
@ -1463,15 +1447,6 @@ dependencies = [
|
||||||
"slab",
|
"slab",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fxhash"
|
|
||||||
version = "0.2.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
|
||||||
dependencies = [
|
|
||||||
"byteorder",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "gcp_auth"
|
name = "gcp_auth"
|
||||||
version = "0.12.2"
|
version = "0.12.2"
|
||||||
|
|
@ -2284,7 +2259,7 @@ dependencies = [
|
||||||
"lru",
|
"lru",
|
||||||
"mimalloc",
|
"mimalloc",
|
||||||
"nix-compat",
|
"nix-compat",
|
||||||
"parking_lot 0.12.3",
|
"parking_lot",
|
||||||
"prost",
|
"prost",
|
||||||
"prost-build",
|
"prost-build",
|
||||||
"rstest",
|
"rstest",
|
||||||
|
|
@ -2514,7 +2489,7 @@ dependencies = [
|
||||||
"hyper",
|
"hyper",
|
||||||
"itertools 0.13.0",
|
"itertools 0.13.0",
|
||||||
"md-5",
|
"md-5",
|
||||||
"parking_lot 0.12.3",
|
"parking_lot",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"quick-xml",
|
"quick-xml",
|
||||||
"rand",
|
"rand",
|
||||||
|
|
@ -2703,17 +2678,6 @@ version = "2.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
|
checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "parking_lot"
|
|
||||||
version = "0.11.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
|
|
||||||
dependencies = [
|
|
||||||
"instant",
|
|
||||||
"lock_api",
|
|
||||||
"parking_lot_core 0.8.6",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parking_lot"
|
name = "parking_lot"
|
||||||
version = "0.12.3"
|
version = "0.12.3"
|
||||||
|
|
@ -2721,21 +2685,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
|
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"lock_api",
|
"lock_api",
|
||||||
"parking_lot_core 0.9.9",
|
"parking_lot_core",
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "parking_lot_core"
|
|
||||||
version = "0.8.6"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
|
|
||||||
dependencies = [
|
|
||||||
"cfg-if",
|
|
||||||
"instant",
|
|
||||||
"libc",
|
|
||||||
"redox_syscall 0.2.16",
|
|
||||||
"smallvec",
|
|
||||||
"winapi",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -3233,15 +3183,6 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "redox_syscall"
|
|
||||||
version = "0.2.16"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 1.3.2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "redox_syscall"
|
name = "redox_syscall"
|
||||||
version = "0.3.5"
|
version = "0.3.5"
|
||||||
|
|
@ -3842,22 +3783,6 @@ dependencies = [
|
||||||
"autocfg",
|
"autocfg",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "sled"
|
|
||||||
version = "0.34.7"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
|
|
||||||
dependencies = [
|
|
||||||
"crc32fast",
|
|
||||||
"crossbeam-epoch",
|
|
||||||
"crossbeam-utils",
|
|
||||||
"fs2",
|
|
||||||
"fxhash",
|
|
||||||
"libc",
|
|
||||||
"log",
|
|
||||||
"parking_lot 0.11.2",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "smallvec"
|
name = "smallvec"
|
||||||
version = "1.13.1"
|
version = "1.13.1"
|
||||||
|
|
@ -4698,7 +4623,7 @@ dependencies = [
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"libc",
|
"libc",
|
||||||
"object_store",
|
"object_store",
|
||||||
"parking_lot 0.12.3",
|
"parking_lot",
|
||||||
"petgraph",
|
"petgraph",
|
||||||
"pin-project",
|
"pin-project",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
|
|
@ -4712,7 +4637,6 @@ dependencies = [
|
||||||
"serde_qs",
|
"serde_qs",
|
||||||
"serde_tagged",
|
"serde_tagged",
|
||||||
"serde_with",
|
"serde_with",
|
||||||
"sled",
|
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"threadpool",
|
"threadpool",
|
||||||
|
|
@ -4895,7 +4819,7 @@ dependencies = [
|
||||||
"lru",
|
"lru",
|
||||||
"mimalloc",
|
"mimalloc",
|
||||||
"nix-compat",
|
"nix-compat",
|
||||||
"parking_lot 0.12.3",
|
"parking_lot",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"prost",
|
"prost",
|
||||||
"prost-build",
|
"prost-build",
|
||||||
|
|
@ -4909,7 +4833,6 @@ dependencies = [
|
||||||
"serde_qs",
|
"serde_qs",
|
||||||
"serde_with",
|
"serde_with",
|
||||||
"sha2",
|
"sha2",
|
||||||
"sled",
|
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|
|
||||||
237
tvix/Cargo.nix
237
tvix/Cargo.nix
|
|
@ -1911,19 +1911,6 @@ rec {
|
||||||
};
|
};
|
||||||
resolvedDefaultFeatures = [ "default" ];
|
resolvedDefaultFeatures = [ "default" ];
|
||||||
};
|
};
|
||||||
"byteorder" = rec {
|
|
||||||
crateName = "byteorder";
|
|
||||||
version = "1.5.0";
|
|
||||||
edition = "2021";
|
|
||||||
sha256 = "0jzncxyf404mwqdbspihyzpkndfgda450l0893pz5xj685cg5l0z";
|
|
||||||
authors = [
|
|
||||||
"Andrew Gallant <jamslam@gmail.com>"
|
|
||||||
];
|
|
||||||
features = {
|
|
||||||
"default" = [ "std" ];
|
|
||||||
};
|
|
||||||
resolvedDefaultFeatures = [ "default" "std" ];
|
|
||||||
};
|
|
||||||
"bytes" = rec {
|
"bytes" = rec {
|
||||||
crateName = "bytes";
|
crateName = "bytes";
|
||||||
version = "1.7.1";
|
version = "1.7.1";
|
||||||
|
|
@ -2820,7 +2807,7 @@ rec {
|
||||||
"nightly" = [ "crossbeam-utils/nightly" ];
|
"nightly" = [ "crossbeam-utils/nightly" ];
|
||||||
"std" = [ "alloc" "crossbeam-utils/std" ];
|
"std" = [ "alloc" "crossbeam-utils/std" ];
|
||||||
};
|
};
|
||||||
resolvedDefaultFeatures = [ "alloc" "default" "std" ];
|
resolvedDefaultFeatures = [ "alloc" "std" ];
|
||||||
};
|
};
|
||||||
"crossbeam-utils" = rec {
|
"crossbeam-utils" = rec {
|
||||||
crateName = "crossbeam-utils";
|
crateName = "crossbeam-utils";
|
||||||
|
|
@ -4149,29 +4136,6 @@ rec {
|
||||||
};
|
};
|
||||||
resolvedDefaultFeatures = [ "alloc" "default" "std" ];
|
resolvedDefaultFeatures = [ "alloc" "default" "std" ];
|
||||||
};
|
};
|
||||||
"fs2" = rec {
|
|
||||||
crateName = "fs2";
|
|
||||||
version = "0.4.3";
|
|
||||||
edition = "2015";
|
|
||||||
sha256 = "04v2hwk7035c088f19mfl5b1lz84gnvv2hv6m935n0hmirszqr4m";
|
|
||||||
authors = [
|
|
||||||
"Dan Burkert <dan@danburkert.com>"
|
|
||||||
];
|
|
||||||
dependencies = [
|
|
||||||
{
|
|
||||||
name = "libc";
|
|
||||||
packageId = "libc";
|
|
||||||
target = { target, features }: (target."unix" or false);
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "winapi";
|
|
||||||
packageId = "winapi";
|
|
||||||
target = { target, features }: (target."windows" or false);
|
|
||||||
features = [ "handleapi" "processthreadsapi" "winerror" "fileapi" "winbase" "std" ];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
};
|
|
||||||
"fuse-backend-rs" = rec {
|
"fuse-backend-rs" = rec {
|
||||||
crateName = "fuse-backend-rs";
|
crateName = "fuse-backend-rs";
|
||||||
version = "0.12.0";
|
version = "0.12.0";
|
||||||
|
|
@ -4625,23 +4589,6 @@ rec {
|
||||||
};
|
};
|
||||||
resolvedDefaultFeatures = [ "alloc" "async-await" "async-await-macro" "channel" "default" "futures-channel" "futures-io" "futures-macro" "futures-sink" "io" "memchr" "sink" "slab" "std" ];
|
resolvedDefaultFeatures = [ "alloc" "async-await" "async-await-macro" "channel" "default" "futures-channel" "futures-io" "futures-macro" "futures-sink" "io" "memchr" "sink" "slab" "std" ];
|
||||||
};
|
};
|
||||||
"fxhash" = rec {
|
|
||||||
crateName = "fxhash";
|
|
||||||
version = "0.2.1";
|
|
||||||
edition = "2015";
|
|
||||||
sha256 = "037mb9ichariqi45xm6mz0b11pa92gj38ba0409z3iz239sns6y3";
|
|
||||||
libPath = "lib.rs";
|
|
||||||
authors = [
|
|
||||||
"cbreeden <github@u.breeden.cc>"
|
|
||||||
];
|
|
||||||
dependencies = [
|
|
||||||
{
|
|
||||||
name = "byteorder";
|
|
||||||
packageId = "byteorder";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
};
|
|
||||||
"gcp_auth" = rec {
|
"gcp_auth" = rec {
|
||||||
crateName = "gcp_auth";
|
crateName = "gcp_auth";
|
||||||
version = "0.12.2";
|
version = "0.12.2";
|
||||||
|
|
@ -7087,7 +7034,7 @@ rec {
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "parking_lot";
|
name = "parking_lot";
|
||||||
packageId = "parking_lot 0.12.3";
|
packageId = "parking_lot";
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "prost";
|
name = "prost";
|
||||||
|
|
@ -7877,7 +7824,7 @@ rec {
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "parking_lot";
|
name = "parking_lot";
|
||||||
packageId = "parking_lot 0.12.3";
|
packageId = "parking_lot";
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "percent-encoding";
|
name = "percent-encoding";
|
||||||
|
|
@ -8625,40 +8572,7 @@ rec {
|
||||||
"loom" = [ "dep:loom" ];
|
"loom" = [ "dep:loom" ];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
"parking_lot 0.11.2" = rec {
|
"parking_lot" = rec {
|
||||||
crateName = "parking_lot";
|
|
||||||
version = "0.11.2";
|
|
||||||
edition = "2018";
|
|
||||||
sha256 = "16gzf41bxmm10x82bla8d6wfppy9ym3fxsmdjyvn61m66s0bf5vx";
|
|
||||||
authors = [
|
|
||||||
"Amanieu d'Antras <amanieu@gmail.com>"
|
|
||||||
];
|
|
||||||
dependencies = [
|
|
||||||
{
|
|
||||||
name = "instant";
|
|
||||||
packageId = "instant";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "lock_api";
|
|
||||||
packageId = "lock_api";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "parking_lot_core";
|
|
||||||
packageId = "parking_lot_core 0.8.6";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
features = {
|
|
||||||
"arc_lock" = [ "lock_api/arc_lock" ];
|
|
||||||
"deadlock_detection" = [ "parking_lot_core/deadlock_detection" ];
|
|
||||||
"nightly" = [ "parking_lot_core/nightly" "lock_api/nightly" ];
|
|
||||||
"owning_ref" = [ "lock_api/owning_ref" ];
|
|
||||||
"serde" = [ "lock_api/serde" ];
|
|
||||||
"stdweb" = [ "instant/stdweb" ];
|
|
||||||
"wasm-bindgen" = [ "instant/wasm-bindgen" ];
|
|
||||||
};
|
|
||||||
resolvedDefaultFeatures = [ "default" ];
|
|
||||||
};
|
|
||||||
"parking_lot 0.12.3" = rec {
|
|
||||||
crateName = "parking_lot";
|
crateName = "parking_lot";
|
||||||
version = "0.12.3";
|
version = "0.12.3";
|
||||||
edition = "2021";
|
edition = "2021";
|
||||||
|
|
@ -8673,7 +8587,7 @@ rec {
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "parking_lot_core";
|
name = "parking_lot_core";
|
||||||
packageId = "parking_lot_core 0.9.9";
|
packageId = "parking_lot_core";
|
||||||
}
|
}
|
||||||
];
|
];
|
||||||
features = {
|
features = {
|
||||||
|
|
@ -8685,52 +8599,7 @@ rec {
|
||||||
};
|
};
|
||||||
resolvedDefaultFeatures = [ "default" ];
|
resolvedDefaultFeatures = [ "default" ];
|
||||||
};
|
};
|
||||||
"parking_lot_core 0.8.6" = rec {
|
"parking_lot_core" = rec {
|
||||||
crateName = "parking_lot_core";
|
|
||||||
version = "0.8.6";
|
|
||||||
edition = "2018";
|
|
||||||
sha256 = "1p2nfcbr0b9lm9rglgm28k6mwyjwgm4knipsmqbgqaxdy3kcz8k0";
|
|
||||||
authors = [
|
|
||||||
"Amanieu d'Antras <amanieu@gmail.com>"
|
|
||||||
];
|
|
||||||
dependencies = [
|
|
||||||
{
|
|
||||||
name = "cfg-if";
|
|
||||||
packageId = "cfg-if";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "instant";
|
|
||||||
packageId = "instant";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "libc";
|
|
||||||
packageId = "libc";
|
|
||||||
target = { target, features }: (target."unix" or false);
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "redox_syscall";
|
|
||||||
packageId = "redox_syscall 0.2.16";
|
|
||||||
target = { target, features }: ("redox" == target."os" or null);
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "smallvec";
|
|
||||||
packageId = "smallvec";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "winapi";
|
|
||||||
packageId = "winapi";
|
|
||||||
target = { target, features }: (target."windows" or false);
|
|
||||||
features = [ "winnt" "ntstatus" "minwindef" "winerror" "winbase" "errhandlingapi" "handleapi" ];
|
|
||||||
}
|
|
||||||
];
|
|
||||||
features = {
|
|
||||||
"backtrace" = [ "dep:backtrace" ];
|
|
||||||
"deadlock_detection" = [ "petgraph" "thread-id" "backtrace" ];
|
|
||||||
"petgraph" = [ "dep:petgraph" ];
|
|
||||||
"thread-id" = [ "dep:thread-id" ];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
"parking_lot_core 0.9.9" = rec {
|
|
||||||
crateName = "parking_lot_core";
|
crateName = "parking_lot_core";
|
||||||
version = "0.9.9";
|
version = "0.9.9";
|
||||||
edition = "2018";
|
edition = "2018";
|
||||||
|
|
@ -10207,23 +10076,6 @@ rec {
|
||||||
"python" = [ "dep:pyo3" "dep:pyo3-build-config" ];
|
"python" = [ "dep:pyo3" "dep:pyo3-build-config" ];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
"redox_syscall 0.2.16" = rec {
|
|
||||||
crateName = "redox_syscall";
|
|
||||||
version = "0.2.16";
|
|
||||||
edition = "2018";
|
|
||||||
sha256 = "16jicm96kjyzm802cxdd1k9jmcph0db1a4lhslcnhjsvhp0mhnpv";
|
|
||||||
libName = "syscall";
|
|
||||||
authors = [
|
|
||||||
"Jeremy Soller <jackpot51@gmail.com>"
|
|
||||||
];
|
|
||||||
dependencies = [
|
|
||||||
{
|
|
||||||
name = "bitflags";
|
|
||||||
packageId = "bitflags 1.3.2";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
};
|
|
||||||
"redox_syscall 0.3.5" = rec {
|
"redox_syscall 0.3.5" = rec {
|
||||||
crateName = "redox_syscall";
|
crateName = "redox_syscall";
|
||||||
version = "0.3.5";
|
version = "0.3.5";
|
||||||
|
|
@ -12340,69 +12192,6 @@ rec {
|
||||||
};
|
};
|
||||||
resolvedDefaultFeatures = [ "default" "std" ];
|
resolvedDefaultFeatures = [ "default" "std" ];
|
||||||
};
|
};
|
||||||
"sled" = rec {
|
|
||||||
crateName = "sled";
|
|
||||||
version = "0.34.7";
|
|
||||||
edition = "2018";
|
|
||||||
sha256 = "0dcr2s7cylj5mb33ci3kpx7fz797jwvysnl5airrir9cgirv95kz";
|
|
||||||
authors = [
|
|
||||||
"Tyler Neely <t@jujit.su>"
|
|
||||||
];
|
|
||||||
dependencies = [
|
|
||||||
{
|
|
||||||
name = "crc32fast";
|
|
||||||
packageId = "crc32fast";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "crossbeam-epoch";
|
|
||||||
packageId = "crossbeam-epoch";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "crossbeam-utils";
|
|
||||||
packageId = "crossbeam-utils";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "fs2";
|
|
||||||
packageId = "fs2";
|
|
||||||
target = { target, features }: (("linux" == target."os" or null) || ("macos" == target."os" or null) || ("windows" == target."os" or null));
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "fxhash";
|
|
||||||
packageId = "fxhash";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "libc";
|
|
||||||
packageId = "libc";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "log";
|
|
||||||
packageId = "log";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
name = "parking_lot";
|
|
||||||
packageId = "parking_lot 0.11.2";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
devDependencies = [
|
|
||||||
{
|
|
||||||
name = "log";
|
|
||||||
packageId = "log";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
features = {
|
|
||||||
"backtrace" = [ "dep:backtrace" ];
|
|
||||||
"color-backtrace" = [ "dep:color-backtrace" ];
|
|
||||||
"compression" = [ "zstd" ];
|
|
||||||
"default" = [ "no_metrics" ];
|
|
||||||
"io_uring" = [ "rio" ];
|
|
||||||
"no_logs" = [ "log/max_level_off" ];
|
|
||||||
"pretty_backtrace" = [ "color-backtrace" ];
|
|
||||||
"rio" = [ "dep:rio" ];
|
|
||||||
"testing" = [ "event_log" "lock_free_delays" "compression" "failpoints" "backtrace" ];
|
|
||||||
"zstd" = [ "dep:zstd" ];
|
|
||||||
};
|
|
||||||
resolvedDefaultFeatures = [ "default" "no_metrics" ];
|
|
||||||
};
|
|
||||||
"smallvec" = rec {
|
"smallvec" = rec {
|
||||||
crateName = "smallvec";
|
crateName = "smallvec";
|
||||||
version = "1.13.1";
|
version = "1.13.1";
|
||||||
|
|
@ -15409,7 +15198,7 @@ rec {
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "parking_lot";
|
name = "parking_lot";
|
||||||
packageId = "parking_lot 0.12.3";
|
packageId = "parking_lot";
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "petgraph";
|
name = "petgraph";
|
||||||
|
|
@ -15448,10 +15237,6 @@ rec {
|
||||||
name = "serde_with";
|
name = "serde_with";
|
||||||
packageId = "serde_with";
|
packageId = "serde_with";
|
||||||
}
|
}
|
||||||
{
|
|
||||||
name = "sled";
|
|
||||||
packageId = "sled";
|
|
||||||
}
|
|
||||||
{
|
{
|
||||||
name = "thiserror";
|
name = "thiserror";
|
||||||
packageId = "thiserror";
|
packageId = "thiserror";
|
||||||
|
|
@ -16212,7 +15997,7 @@ rec {
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "parking_lot";
|
name = "parking_lot";
|
||||||
packageId = "parking_lot 0.12.3";
|
packageId = "parking_lot";
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
name = "pin-project-lite";
|
name = "pin-project-lite";
|
||||||
|
|
@ -16257,10 +16042,6 @@ rec {
|
||||||
name = "sha2";
|
name = "sha2";
|
||||||
packageId = "sha2";
|
packageId = "sha2";
|
||||||
}
|
}
|
||||||
{
|
|
||||||
name = "sled";
|
|
||||||
packageId = "sled";
|
|
||||||
}
|
|
||||||
{
|
{
|
||||||
name = "thiserror";
|
name = "thiserror";
|
||||||
packageId = "thiserror";
|
packageId = "thiserror";
|
||||||
|
|
@ -17930,7 +17711,7 @@ rec {
|
||||||
features = {
|
features = {
|
||||||
"debug" = [ "impl-debug" ];
|
"debug" = [ "impl-debug" ];
|
||||||
};
|
};
|
||||||
resolvedDefaultFeatures = [ "basetsd" "consoleapi" "errhandlingapi" "fileapi" "handleapi" "knownfolders" "minwindef" "ntstatus" "objbase" "processenv" "processthreadsapi" "shellapi" "shlobj" "std" "stringapiset" "synchapi" "sysinfoapi" "winbase" "wincon" "winerror" "winnt" "winuser" ];
|
resolvedDefaultFeatures = [ "basetsd" "consoleapi" "errhandlingapi" "fileapi" "handleapi" "knownfolders" "minwindef" "objbase" "processenv" "shellapi" "shlobj" "std" "stringapiset" "synchapi" "sysinfoapi" "winbase" "wincon" "winerror" "winnt" "winuser" ];
|
||||||
};
|
};
|
||||||
"winapi-i686-pc-windows-gnu" = rec {
|
"winapi-i686-pc-windows-gnu" = rec {
|
||||||
crateName = "winapi-i686-pc-windows-gnu";
|
crateName = "winapi-i686-pc-windows-gnu";
|
||||||
|
|
|
||||||
|
|
@ -123,7 +123,6 @@ serde_tagged = "0.3.0"
|
||||||
serde_with = "3.9.0"
|
serde_with = "3.9.0"
|
||||||
sha1 = "0.10.6"
|
sha1 = "0.10.6"
|
||||||
sha2 = "0.10.8"
|
sha2 = "0.10.8"
|
||||||
sled = "0.34.7"
|
|
||||||
smol_str = "0.2.2"
|
smol_str = "0.2.2"
|
||||||
tabwriter = "1.4"
|
tabwriter = "1.4"
|
||||||
tempfile = "3.12.0"
|
tempfile = "3.12.0"
|
||||||
|
|
|
||||||
|
|
@ -36,8 +36,8 @@ Now, spin up tvix-daemon, connecting to some (local) backends:
|
||||||
```
|
```
|
||||||
tvix-store --otlp=false daemon \
|
tvix-store --otlp=false daemon \
|
||||||
--blob-service-addr=objectstore+file://$PWD/blobs \
|
--blob-service-addr=objectstore+file://$PWD/blobs \
|
||||||
--directory-service-addr=sled://$PWD/directories.sled \
|
--directory-service-addr=redb://$PWD/directories.redb \
|
||||||
--path-info-service-addr=sled://$PWD/pathinfo.sled &
|
--path-info-service-addr=redb://$PWD/pathinfo.redb &
|
||||||
```
|
```
|
||||||
|
|
||||||
Copy some data into tvix-store (we use `nar-bridge` for this for now):
|
Copy some data into tvix-store (we use `nar-bridge` for this for now):
|
||||||
|
|
@ -58,8 +58,8 @@ In case you want to have `tvix-store virtiofs` open the stores directly, kill
|
||||||
```
|
```
|
||||||
pkill tvix-store
|
pkill tvix-store
|
||||||
export BLOB_SERVICE_ADDR=objectstore+file://$PWD/blobs
|
export BLOB_SERVICE_ADDR=objectstore+file://$PWD/blobs
|
||||||
export DIRECTORY_SERVICE_ADDR=sled://$PWD/directories.sled
|
export DIRECTORY_SERVICE_ADDR=redb://$PWD/directories.redb
|
||||||
export PATH_INFO_SERVICE_ADDR=sled://$PWD/pathinfo.sled
|
export PATH_INFO_SERVICE_ADDR=redb://$PWD/pathinfo.redb
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Interactive shell
|
#### Interactive shell
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,6 @@ object_store = { workspace = true, features = ["http"] }
|
||||||
parking_lot = { workspace = true }
|
parking_lot = { workspace = true }
|
||||||
pin-project-lite = { workspace = true }
|
pin-project-lite = { workspace = true }
|
||||||
prost = { workspace = true }
|
prost = { workspace = true }
|
||||||
sled = { workspace = true }
|
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
tokio-stream = { workspace = true, features = ["fs", "net"] }
|
tokio-stream = { workspace = true, features = ["fs", "net"] }
|
||||||
tokio-util = { workspace = true, features = ["io", "io-util", "codec"] }
|
tokio-util = { workspace = true, features = ["io", "io-util", "codec"] }
|
||||||
|
|
|
||||||
|
|
@ -13,10 +13,6 @@ use super::DirectoryService;
|
||||||
/// The following URIs are supported:
|
/// The following URIs are supported:
|
||||||
/// - `memory:`
|
/// - `memory:`
|
||||||
/// Uses a in-memory implementation.
|
/// Uses a in-memory implementation.
|
||||||
/// - `sled:`
|
|
||||||
/// Uses a in-memory sled implementation.
|
|
||||||
/// - `sled:///absolute/path/to/somewhere`
|
|
||||||
/// Uses sled, using a path on the disk for persistency. Can be only opened
|
|
||||||
/// from one process at the same time.
|
/// from one process at the same time.
|
||||||
/// - `redb:`
|
/// - `redb:`
|
||||||
/// Uses a in-memory redb implementation.
|
/// Uses a in-memory redb implementation.
|
||||||
|
|
@ -55,8 +51,6 @@ mod tests {
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
lazy_static! {
|
lazy_static! {
|
||||||
static ref TMPDIR_SLED_1: TempDir = TempDir::new().unwrap();
|
|
||||||
static ref TMPDIR_SLED_2: TempDir = TempDir::new().unwrap();
|
|
||||||
static ref TMPDIR_REDB_1: TempDir = TempDir::new().unwrap();
|
static ref TMPDIR_REDB_1: TempDir = TempDir::new().unwrap();
|
||||||
static ref TMPDIR_REDB_2: TempDir = TempDir::new().unwrap();
|
static ref TMPDIR_REDB_2: TempDir = TempDir::new().unwrap();
|
||||||
}
|
}
|
||||||
|
|
@ -64,16 +58,6 @@ mod tests {
|
||||||
#[rstest]
|
#[rstest]
|
||||||
/// This uses an unsupported scheme.
|
/// This uses an unsupported scheme.
|
||||||
#[case::unsupported_scheme("http://foo.example/test", false)]
|
#[case::unsupported_scheme("http://foo.example/test", false)]
|
||||||
/// This configures sled in temporary mode.
|
|
||||||
#[case::sled_valid_temporary("sled://", true)]
|
|
||||||
/// This configures sled with /, which should fail.
|
|
||||||
#[case::sled_invalid_root("sled:///", false)]
|
|
||||||
/// This configures sled with a host, not path, which should fail.
|
|
||||||
#[case::sled_invalid_host("sled://foo.example", false)]
|
|
||||||
/// This configures sled with a valid path path, which should succeed.
|
|
||||||
#[case::sled_valid_path(&format!("sled://{}", &TMPDIR_SLED_1.path().to_str().unwrap()), true)]
|
|
||||||
/// This configures sled with a host, and a valid path path, which should fail.
|
|
||||||
#[case::sled_invalid_host_with_valid_path(&format!("sled://foo.example{}", &TMPDIR_SLED_2.path().to_str().unwrap()), false)]
|
|
||||||
/// This correctly sets the scheme, and doesn't set a path.
|
/// This correctly sets the scheme, and doesn't set a path.
|
||||||
#[case::memory_valid("memory://", true)]
|
#[case::memory_valid("memory://", true)]
|
||||||
/// This sets a memory url host to `foo`
|
/// This sets a memory url host to `foo`
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,6 @@ mod object_store;
|
||||||
mod order_validator;
|
mod order_validator;
|
||||||
mod redb;
|
mod redb;
|
||||||
mod simple_putter;
|
mod simple_putter;
|
||||||
mod sled;
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub mod tests;
|
pub mod tests;
|
||||||
mod traverse;
|
mod traverse;
|
||||||
|
|
@ -27,7 +26,6 @@ pub use self::object_store::{ObjectStoreDirectoryService, ObjectStoreDirectorySe
|
||||||
pub use self::order_validator::{LeavesToRootValidator, OrderValidator, RootToLeavesValidator};
|
pub use self::order_validator::{LeavesToRootValidator, OrderValidator, RootToLeavesValidator};
|
||||||
pub use self::redb::{RedbDirectoryService, RedbDirectoryServiceConfig};
|
pub use self::redb::{RedbDirectoryService, RedbDirectoryServiceConfig};
|
||||||
pub use self::simple_putter::SimplePutter;
|
pub use self::simple_putter::SimplePutter;
|
||||||
pub use self::sled::{SledDirectoryService, SledDirectoryServiceConfig};
|
|
||||||
pub use self::traverse::descend_to;
|
pub use self::traverse::descend_to;
|
||||||
pub use self::utils::traverse_directory;
|
pub use self::utils::traverse_directory;
|
||||||
|
|
||||||
|
|
@ -138,7 +136,6 @@ pub(crate) fn register_directory_services(reg: &mut Registry) {
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>, super::directoryservice::MemoryDirectoryServiceConfig>("memory");
|
reg.register::<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>, super::directoryservice::MemoryDirectoryServiceConfig>("memory");
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>, super::directoryservice::CacheConfig>("cache");
|
reg.register::<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>, super::directoryservice::CacheConfig>("cache");
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>, super::directoryservice::GRPCDirectoryServiceConfig>("grpc");
|
reg.register::<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>, super::directoryservice::GRPCDirectoryServiceConfig>("grpc");
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>, super::directoryservice::SledDirectoryServiceConfig>("sled");
|
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>, super::directoryservice::RedbDirectoryServiceConfig>("redb");
|
reg.register::<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>, super::directoryservice::RedbDirectoryServiceConfig>("redb");
|
||||||
#[cfg(feature = "cloud")]
|
#[cfg(feature = "cloud")]
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -1,263 +0,0 @@
|
||||||
use futures::stream::BoxStream;
|
|
||||||
use prost::Message;
|
|
||||||
use std::ops::Deref;
|
|
||||||
use std::path::Path;
|
|
||||||
use std::sync::Arc;
|
|
||||||
use tonic::async_trait;
|
|
||||||
use tracing::{instrument, warn};
|
|
||||||
|
|
||||||
use super::utils::traverse_directory;
|
|
||||||
use super::{Directory, DirectoryGraph, DirectoryPutter, DirectoryService, LeavesToRootValidator};
|
|
||||||
use crate::composition::{CompositionContext, ServiceBuilder};
|
|
||||||
use crate::{proto, B3Digest, Error};
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
|
||||||
pub struct SledDirectoryService {
|
|
||||||
db: sled::Db,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SledDirectoryService {
|
|
||||||
pub fn new<P: AsRef<Path>>(p: P) -> Result<Self, sled::Error> {
|
|
||||||
if p.as_ref() == Path::new("/") {
|
|
||||||
return Err(sled::Error::Unsupported(
|
|
||||||
"cowardly refusing to open / with sled".to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
let config = sled::Config::default()
|
|
||||||
.use_compression(false) // is a required parameter
|
|
||||||
.path(p);
|
|
||||||
let db = config.open()?;
|
|
||||||
|
|
||||||
Ok(Self { db })
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new_temporary() -> Result<Self, sled::Error> {
|
|
||||||
let config = sled::Config::default().temporary(true);
|
|
||||||
let db = config.open()?;
|
|
||||||
|
|
||||||
Ok(Self { db })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl DirectoryService for SledDirectoryService {
|
|
||||||
#[instrument(skip(self, digest), fields(directory.digest = %digest))]
|
|
||||||
async fn get(&self, digest: &B3Digest) -> Result<Option<Directory>, Error> {
|
|
||||||
let resp = tokio::task::spawn_blocking({
|
|
||||||
let db = self.db.clone();
|
|
||||||
let digest = digest.clone();
|
|
||||||
move || db.get(digest.as_slice())
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
.map_err(|e| {
|
|
||||||
warn!("failed to retrieve directory: {}", e);
|
|
||||||
Error::StorageError(format!("failed to retrieve directory: {}", e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
match resp {
|
|
||||||
// The directory was not found, return
|
|
||||||
None => Ok(None),
|
|
||||||
|
|
||||||
// The directory was found, try to parse the data as Directory message
|
|
||||||
Some(data) => match proto::Directory::decode(&*data) {
|
|
||||||
Ok(directory) => {
|
|
||||||
// Validate the retrieved Directory indeed has the
|
|
||||||
// digest we expect it to have, to detect corruptions.
|
|
||||||
let actual_digest = directory.digest();
|
|
||||||
if actual_digest != *digest {
|
|
||||||
return Err(Error::StorageError(format!(
|
|
||||||
"requested directory with digest {}, but got {}",
|
|
||||||
digest, actual_digest
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
let directory = directory.try_into().map_err(|e| {
|
|
||||||
warn!("failed to retrieve directory: {}", e);
|
|
||||||
Error::StorageError(format!("failed to retrieve directory: {}", e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
Ok(Some(directory))
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
warn!("unable to parse directory {}: {}", digest, e);
|
|
||||||
Err(Error::StorageError(e.to_string()))
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[instrument(skip(self, directory), fields(directory.digest = %directory.digest()))]
|
|
||||||
async fn put(&self, directory: Directory) -> Result<B3Digest, Error> {
|
|
||||||
tokio::task::spawn_blocking({
|
|
||||||
let db = self.db.clone();
|
|
||||||
move || {
|
|
||||||
let digest = directory.digest();
|
|
||||||
|
|
||||||
// store it
|
|
||||||
db.insert(
|
|
||||||
digest.as_slice(),
|
|
||||||
proto::Directory::from(directory).encode_to_vec(),
|
|
||||||
)
|
|
||||||
.map_err(|e| Error::StorageError(e.to_string()))?;
|
|
||||||
|
|
||||||
Ok(digest)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
}
|
|
||||||
|
|
||||||
#[instrument(skip_all, fields(directory.digest = %root_directory_digest))]
|
|
||||||
fn get_recursive(
|
|
||||||
&self,
|
|
||||||
root_directory_digest: &B3Digest,
|
|
||||||
) -> BoxStream<'static, Result<Directory, Error>> {
|
|
||||||
traverse_directory(self.clone(), root_directory_digest)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[instrument(skip_all)]
|
|
||||||
fn put_multiple_start(&self) -> Box<(dyn DirectoryPutter + 'static)>
|
|
||||||
where
|
|
||||||
Self: Clone,
|
|
||||||
{
|
|
||||||
Box::new(SledDirectoryPutter {
|
|
||||||
tree: self.db.deref().clone(),
|
|
||||||
directory_validator: Some(Default::default()),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(serde::Deserialize)]
|
|
||||||
#[serde(deny_unknown_fields)]
|
|
||||||
pub struct SledDirectoryServiceConfig {
|
|
||||||
is_temporary: bool,
|
|
||||||
#[serde(default)]
|
|
||||||
/// required when is_temporary = false
|
|
||||||
path: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TryFrom<url::Url> for SledDirectoryServiceConfig {
|
|
||||||
type Error = Box<dyn std::error::Error + Send + Sync>;
|
|
||||||
fn try_from(url: url::Url) -> Result<Self, Self::Error> {
|
|
||||||
// sled doesn't support host, and a path can be provided (otherwise
|
|
||||||
// it'll live in memory only).
|
|
||||||
if url.has_host() {
|
|
||||||
return Err(Error::StorageError("no host allowed".to_string()).into());
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: expose compression and other parameters as URL parameters?
|
|
||||||
|
|
||||||
Ok(if url.path().is_empty() {
|
|
||||||
SledDirectoryServiceConfig {
|
|
||||||
is_temporary: true,
|
|
||||||
path: None,
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
SledDirectoryServiceConfig {
|
|
||||||
is_temporary: false,
|
|
||||||
path: Some(url.path().to_string()),
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl ServiceBuilder for SledDirectoryServiceConfig {
|
|
||||||
type Output = dyn DirectoryService;
|
|
||||||
async fn build<'a>(
|
|
||||||
&'a self,
|
|
||||||
_instance_name: &str,
|
|
||||||
_context: &CompositionContext,
|
|
||||||
) -> Result<Arc<dyn DirectoryService>, Box<dyn std::error::Error + Send + Sync + 'static>> {
|
|
||||||
match self {
|
|
||||||
SledDirectoryServiceConfig {
|
|
||||||
is_temporary: true,
|
|
||||||
path: None,
|
|
||||||
} => Ok(Arc::new(SledDirectoryService::new_temporary()?)),
|
|
||||||
SledDirectoryServiceConfig {
|
|
||||||
is_temporary: true,
|
|
||||||
path: Some(_),
|
|
||||||
} => Err(Error::StorageError(
|
|
||||||
"Temporary SledDirectoryService can not have path".into(),
|
|
||||||
)
|
|
||||||
.into()),
|
|
||||||
SledDirectoryServiceConfig {
|
|
||||||
is_temporary: false,
|
|
||||||
path: None,
|
|
||||||
} => Err(Error::StorageError("SledDirectoryService is missing path".into()).into()),
|
|
||||||
SledDirectoryServiceConfig {
|
|
||||||
is_temporary: false,
|
|
||||||
path: Some(path),
|
|
||||||
} => Ok(Arc::new(SledDirectoryService::new(path)?)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Buffers Directory messages to be uploaded and inserts them in a batch
|
|
||||||
/// transaction on close.
|
|
||||||
pub struct SledDirectoryPutter {
|
|
||||||
tree: sled::Tree,
|
|
||||||
|
|
||||||
/// The directories (inside the directory validator) that we insert later,
|
|
||||||
/// or None, if they were already inserted.
|
|
||||||
directory_validator: Option<DirectoryGraph<LeavesToRootValidator>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl DirectoryPutter for SledDirectoryPutter {
|
|
||||||
#[instrument(level = "trace", skip_all, fields(directory.digest=%directory.digest()), err)]
|
|
||||||
async fn put(&mut self, directory: Directory) -> Result<(), Error> {
|
|
||||||
match self.directory_validator {
|
|
||||||
None => return Err(Error::StorageError("already closed".to_string())),
|
|
||||||
Some(ref mut validator) => {
|
|
||||||
validator
|
|
||||||
.add(directory)
|
|
||||||
.map_err(|e| Error::StorageError(e.to_string()))?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[instrument(level = "trace", skip_all, ret, err)]
|
|
||||||
async fn close(&mut self) -> Result<B3Digest, Error> {
|
|
||||||
match self.directory_validator.take() {
|
|
||||||
None => Err(Error::InvalidRequest("already closed".to_string())),
|
|
||||||
Some(validator) => {
|
|
||||||
// Insert all directories as a batch.
|
|
||||||
tokio::task::spawn_blocking({
|
|
||||||
let tree = self.tree.clone();
|
|
||||||
move || {
|
|
||||||
// retrieve the validated directories.
|
|
||||||
let directories = validator
|
|
||||||
.validate()
|
|
||||||
.map_err(|e| Error::StorageError(e.to_string()))?
|
|
||||||
.drain_leaves_to_root()
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
|
|
||||||
// Get the root digest, which is at the end (cf. insertion order)
|
|
||||||
let root_digest = directories
|
|
||||||
.last()
|
|
||||||
.ok_or_else(|| Error::InvalidRequest("got no directories".to_string()))?
|
|
||||||
.digest();
|
|
||||||
|
|
||||||
let mut batch = sled::Batch::default();
|
|
||||||
for directory in directories {
|
|
||||||
batch.insert(
|
|
||||||
directory.digest().as_slice(),
|
|
||||||
proto::Directory::from(directory).encode_to_vec(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
tree.apply_batch(batch).map_err(|e| {
|
|
||||||
Error::StorageError(format!("unable to apply batch: {}", e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
Ok(root_digest)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -23,7 +23,6 @@ use self::utils::make_grpc_directory_service_client;
|
||||||
#[rstest]
|
#[rstest]
|
||||||
#[case::grpc(make_grpc_directory_service_client().await)]
|
#[case::grpc(make_grpc_directory_service_client().await)]
|
||||||
#[case::memory(directoryservice::from_addr("memory://").await.unwrap())]
|
#[case::memory(directoryservice::from_addr("memory://").await.unwrap())]
|
||||||
#[case::sled(directoryservice::from_addr("sled://").await.unwrap())]
|
|
||||||
#[case::redb(directoryservice::from_addr("redb://").await.unwrap())]
|
#[case::redb(directoryservice::from_addr("redb://").await.unwrap())]
|
||||||
#[case::objectstore(directoryservice::from_addr("objectstore+memory://").await.unwrap())]
|
#[case::objectstore(directoryservice::from_addr("objectstore+memory://").await.unwrap())]
|
||||||
#[cfg_attr(all(feature = "cloud", feature = "integration"), case::bigtable(directoryservice::from_addr("bigtable://instance-1?project_id=project-1&table_name=table-1&family_name=cf1").await.unwrap()))]
|
#[cfg_attr(all(feature = "cloud", feature = "integration"), case::bigtable(directoryservice::from_addr("bigtable://instance-1?project_id=project-1&table_name=table-1&family_name=cf1").await.unwrap()))]
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,6 @@ serde_json = { workspace = true }
|
||||||
serde_with = { workspace = true }
|
serde_with = { workspace = true }
|
||||||
serde_qs = { workspace = true }
|
serde_qs = { workspace = true }
|
||||||
sha2 = { workspace = true }
|
sha2 = { workspace = true }
|
||||||
sled = { workspace = true }
|
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
tokio = { workspace = true, features = ["fs", "macros", "net", "rt", "rt-multi-thread", "signal"] }
|
tokio = { workspace = true, features = ["fs", "macros", "net", "rt", "rt-multi-thread", "signal"] }
|
||||||
tokio-listener = { workspace = true, features = ["clap", "multi-listener", "sd_listen", "tonic012"] }
|
tokio-listener = { workspace = true, features = ["clap", "multi-listener", "sd_listen", "tonic012"] }
|
||||||
|
|
|
||||||
|
|
@ -12,11 +12,6 @@ use url::Url;
|
||||||
/// The following URIs are supported:
|
/// The following URIs are supported:
|
||||||
/// - `memory:`
|
/// - `memory:`
|
||||||
/// Uses a in-memory implementation.
|
/// Uses a in-memory implementation.
|
||||||
/// - `sled:`
|
|
||||||
/// Uses a in-memory sled implementation.
|
|
||||||
/// - `sled:///absolute/path/to/somewhere`
|
|
||||||
/// Uses sled, using a path on the disk for persistency. Can be only opened
|
|
||||||
/// from one process at the same time.
|
|
||||||
/// - `redb:`
|
/// - `redb:`
|
||||||
/// Uses a in-memory redb implementation.
|
/// Uses a in-memory redb implementation.
|
||||||
/// - `redb:///absolute/path/to/somewhere`
|
/// - `redb:///absolute/path/to/somewhere`
|
||||||
|
|
@ -66,8 +61,6 @@ mod tests {
|
||||||
use tvix_castore::directoryservice::{DirectoryService, MemoryDirectoryServiceConfig};
|
use tvix_castore::directoryservice::{DirectoryService, MemoryDirectoryServiceConfig};
|
||||||
|
|
||||||
lazy_static! {
|
lazy_static! {
|
||||||
static ref TMPDIR_SLED_1: TempDir = TempDir::new().unwrap();
|
|
||||||
static ref TMPDIR_SLED_2: TempDir = TempDir::new().unwrap();
|
|
||||||
static ref TMPDIR_REDB_1: TempDir = TempDir::new().unwrap();
|
static ref TMPDIR_REDB_1: TempDir = TempDir::new().unwrap();
|
||||||
static ref TMPDIR_REDB_2: TempDir = TempDir::new().unwrap();
|
static ref TMPDIR_REDB_2: TempDir = TempDir::new().unwrap();
|
||||||
}
|
}
|
||||||
|
|
@ -77,16 +70,6 @@ mod tests {
|
||||||
#[rstest]
|
#[rstest]
|
||||||
/// This uses a unsupported scheme.
|
/// This uses a unsupported scheme.
|
||||||
#[case::unsupported_scheme("http://foo.example/test", false)]
|
#[case::unsupported_scheme("http://foo.example/test", false)]
|
||||||
/// This configures sled in temporary mode.
|
|
||||||
#[case::sled_temporary("sled://", true)]
|
|
||||||
/// This configures sled with /, which should fail.
|
|
||||||
#[case::sled_invalid_root("sled:///", false)]
|
|
||||||
/// This configures sled with a host, not path, which should fail.
|
|
||||||
#[case::sled_invalid_host("sled://foo.example", false)]
|
|
||||||
/// This configures sled with a valid path path, which should succeed.
|
|
||||||
#[case::sled_valid_path(&format!("sled://{}", &TMPDIR_SLED_1.path().to_str().unwrap()), true)]
|
|
||||||
/// This configures sled with a host, and a valid path path, which should fail.
|
|
||||||
#[case::sled_invalid_host_with_valid_path(&format!("sled://foo.example{}", &TMPDIR_SLED_2.path().to_str().unwrap()), false)]
|
|
||||||
/// This correctly sets the scheme, and doesn't set a path.
|
/// This correctly sets the scheme, and doesn't set a path.
|
||||||
#[case::memory_valid("memory://", true)]
|
#[case::memory_valid("memory://", true)]
|
||||||
/// This sets a memory url host to `foo`
|
/// This sets a memory url host to `foo`
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,6 @@ mod memory;
|
||||||
mod nix_http;
|
mod nix_http;
|
||||||
mod redb;
|
mod redb;
|
||||||
mod signing_wrapper;
|
mod signing_wrapper;
|
||||||
mod sled;
|
|
||||||
|
|
||||||
#[cfg(any(feature = "fuse", feature = "virtiofs"))]
|
#[cfg(any(feature = "fuse", feature = "virtiofs"))]
|
||||||
mod fs;
|
mod fs;
|
||||||
|
|
@ -32,7 +31,6 @@ pub use self::memory::{MemoryPathInfoService, MemoryPathInfoServiceConfig};
|
||||||
pub use self::nix_http::{NixHTTPPathInfoService, NixHTTPPathInfoServiceConfig};
|
pub use self::nix_http::{NixHTTPPathInfoService, NixHTTPPathInfoServiceConfig};
|
||||||
pub use self::redb::{RedbPathInfoService, RedbPathInfoServiceConfig};
|
pub use self::redb::{RedbPathInfoService, RedbPathInfoServiceConfig};
|
||||||
pub use self::signing_wrapper::{KeyFileSigningPathInfoServiceConfig, SigningPathInfoService};
|
pub use self::signing_wrapper::{KeyFileSigningPathInfoServiceConfig, SigningPathInfoService};
|
||||||
pub use self::sled::{SledPathInfoService, SledPathInfoServiceConfig};
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use self::signing_wrapper::test_signing_service;
|
pub(crate) use self::signing_wrapper::test_signing_service;
|
||||||
|
|
@ -94,7 +92,6 @@ pub(crate) fn register_pathinfo_services(reg: &mut Registry) {
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, LruPathInfoServiceConfig>("lru");
|
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, LruPathInfoServiceConfig>("lru");
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, MemoryPathInfoServiceConfig>("memory");
|
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, MemoryPathInfoServiceConfig>("memory");
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, NixHTTPPathInfoServiceConfig>("nix");
|
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, NixHTTPPathInfoServiceConfig>("nix");
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, SledPathInfoServiceConfig>("sled");
|
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, RedbPathInfoServiceConfig>("redb");
|
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, RedbPathInfoServiceConfig>("redb");
|
||||||
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, KeyFileSigningPathInfoServiceConfig>("keyfile-signing");
|
reg.register::<Box<dyn ServiceBuilder<Output = dyn PathInfoService>>, KeyFileSigningPathInfoServiceConfig>("keyfile-signing");
|
||||||
#[cfg(feature = "cloud")]
|
#[cfg(feature = "cloud")]
|
||||||
|
|
|
||||||
|
|
@ -1,190 +0,0 @@
|
||||||
use super::PathInfoService;
|
|
||||||
use crate::proto::PathInfo;
|
|
||||||
use async_stream::try_stream;
|
|
||||||
use futures::stream::BoxStream;
|
|
||||||
use nix_compat::nixbase32;
|
|
||||||
use prost::Message;
|
|
||||||
use std::path::Path;
|
|
||||||
use std::sync::Arc;
|
|
||||||
use tonic::async_trait;
|
|
||||||
use tracing::{instrument, warn};
|
|
||||||
use tvix_castore::composition::{CompositionContext, ServiceBuilder};
|
|
||||||
use tvix_castore::Error;
|
|
||||||
|
|
||||||
/// SledPathInfoService stores PathInfo in a [sled](https://github.com/spacejam/sled).
|
|
||||||
///
|
|
||||||
/// The PathInfo messages are stored as encoded protos, and keyed by their output hash,
|
|
||||||
/// as that's currently the only request type available.
|
|
||||||
pub struct SledPathInfoService {
|
|
||||||
db: sled::Db,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl SledPathInfoService {
|
|
||||||
pub fn new<P: AsRef<Path>>(p: P) -> Result<Self, sled::Error> {
|
|
||||||
if p.as_ref() == Path::new("/") {
|
|
||||||
return Err(sled::Error::Unsupported(
|
|
||||||
"cowardly refusing to open / with sled".to_string(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
|
|
||||||
let config = sled::Config::default()
|
|
||||||
.use_compression(false) // is a required parameter
|
|
||||||
.path(p);
|
|
||||||
let db = config.open()?;
|
|
||||||
|
|
||||||
Ok(Self { db })
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn new_temporary() -> Result<Self, sled::Error> {
|
|
||||||
let config = sled::Config::default().temporary(true);
|
|
||||||
let db = config.open()?;
|
|
||||||
|
|
||||||
Ok(Self { db })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl PathInfoService for SledPathInfoService {
|
|
||||||
#[instrument(level = "trace", skip_all, fields(path_info.digest = nixbase32::encode(&digest)))]
|
|
||||||
async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> {
|
|
||||||
let resp = tokio::task::spawn_blocking({
|
|
||||||
let db = self.db.clone();
|
|
||||||
move || db.get(digest.as_slice())
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
.map_err(|e| {
|
|
||||||
warn!("failed to retrieve PathInfo: {}", e);
|
|
||||||
Error::StorageError(format!("failed to retrieve PathInfo: {}", e))
|
|
||||||
})?;
|
|
||||||
match resp {
|
|
||||||
None => Ok(None),
|
|
||||||
Some(data) => {
|
|
||||||
let path_info = PathInfo::decode(&*data).map_err(|e| {
|
|
||||||
warn!("failed to decode stored PathInfo: {}", e);
|
|
||||||
Error::StorageError(format!("failed to decode stored PathInfo: {}", e))
|
|
||||||
})?;
|
|
||||||
Ok(Some(path_info))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[instrument(level = "trace", skip_all, fields(path_info.root_node = ?path_info.node))]
|
|
||||||
async fn put(&self, path_info: PathInfo) -> Result<PathInfo, Error> {
|
|
||||||
// Call validate on the received PathInfo message.
|
|
||||||
let store_path = path_info
|
|
||||||
.validate()
|
|
||||||
.map_err(|e| Error::InvalidRequest(format!("failed to validate PathInfo: {}", e)))?;
|
|
||||||
|
|
||||||
// In case the PathInfo is valid, we were able to parse a StorePath.
|
|
||||||
// Store it in the database, keyed by its digest.
|
|
||||||
// This overwrites existing PathInfo objects.
|
|
||||||
tokio::task::spawn_blocking({
|
|
||||||
let db = self.db.clone();
|
|
||||||
let k = *store_path.digest();
|
|
||||||
let data = path_info.encode_to_vec();
|
|
||||||
move || db.insert(k, data)
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
.map_err(|e| {
|
|
||||||
warn!("failed to insert PathInfo: {}", e);
|
|
||||||
Error::StorageError(format! {
|
|
||||||
"failed to insert PathInfo: {}", e
|
|
||||||
})
|
|
||||||
})?;
|
|
||||||
|
|
||||||
Ok(path_info)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn list(&self) -> BoxStream<'static, Result<PathInfo, Error>> {
|
|
||||||
let db = self.db.clone();
|
|
||||||
let mut it = db.iter().values();
|
|
||||||
|
|
||||||
Box::pin(try_stream! {
|
|
||||||
// Don't block the executor while waiting for .next(), so wrap that
|
|
||||||
// in a spawn_blocking call.
|
|
||||||
// We need to pass around it to be able to reuse it.
|
|
||||||
while let (Some(elem), new_it) = tokio::task::spawn_blocking(move || {
|
|
||||||
(it.next(), it)
|
|
||||||
}).await? {
|
|
||||||
it = new_it;
|
|
||||||
let data = elem.map_err(|e| {
|
|
||||||
warn!("failed to retrieve PathInfo: {}", e);
|
|
||||||
Error::StorageError(format!("failed to retrieve PathInfo: {}", e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let path_info = PathInfo::decode(&*data).map_err(|e| {
|
|
||||||
warn!("failed to decode stored PathInfo: {}", e);
|
|
||||||
Error::StorageError(format!("failed to decode stored PathInfo: {}", e))
|
|
||||||
})?;
|
|
||||||
|
|
||||||
yield path_info
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(serde::Deserialize)]
|
|
||||||
#[serde(deny_unknown_fields)]
|
|
||||||
pub struct SledPathInfoServiceConfig {
|
|
||||||
is_temporary: bool,
|
|
||||||
#[serde(default)]
|
|
||||||
/// required when is_temporary = false
|
|
||||||
path: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TryFrom<url::Url> for SledPathInfoServiceConfig {
|
|
||||||
type Error = Box<dyn std::error::Error + Send + Sync>;
|
|
||||||
fn try_from(url: url::Url) -> Result<Self, Self::Error> {
|
|
||||||
// sled doesn't support host, and a path can be provided (otherwise
|
|
||||||
// it'll live in memory only).
|
|
||||||
if url.has_host() {
|
|
||||||
return Err(Error::StorageError("no host allowed".to_string()).into());
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: expose compression and other parameters as URL parameters?
|
|
||||||
|
|
||||||
Ok(if url.path().is_empty() {
|
|
||||||
SledPathInfoServiceConfig {
|
|
||||||
is_temporary: true,
|
|
||||||
path: None,
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
SledPathInfoServiceConfig {
|
|
||||||
is_temporary: false,
|
|
||||||
path: Some(url.path().to_string()),
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl ServiceBuilder for SledPathInfoServiceConfig {
|
|
||||||
type Output = dyn PathInfoService;
|
|
||||||
async fn build<'a>(
|
|
||||||
&'a self,
|
|
||||||
_instance_name: &str,
|
|
||||||
_context: &CompositionContext,
|
|
||||||
) -> Result<Arc<dyn PathInfoService>, Box<dyn std::error::Error + Send + Sync + 'static>> {
|
|
||||||
match self {
|
|
||||||
SledPathInfoServiceConfig {
|
|
||||||
is_temporary: true,
|
|
||||||
path: None,
|
|
||||||
} => Ok(Arc::new(SledPathInfoService::new_temporary()?)),
|
|
||||||
SledPathInfoServiceConfig {
|
|
||||||
is_temporary: true,
|
|
||||||
path: Some(_),
|
|
||||||
} => Err(
|
|
||||||
Error::StorageError("Temporary SledPathInfoService can not have path".into())
|
|
||||||
.into(),
|
|
||||||
),
|
|
||||||
SledPathInfoServiceConfig {
|
|
||||||
is_temporary: false,
|
|
||||||
path: None,
|
|
||||||
} => Err(Error::StorageError("SledPathInfoService is missing path".into()).into()),
|
|
||||||
SledPathInfoServiceConfig {
|
|
||||||
is_temporary: false,
|
|
||||||
path: Some(path),
|
|
||||||
} => Ok(Arc::new(SledPathInfoService::new(path)?)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -9,7 +9,6 @@ use rstest_reuse::{self, *};
|
||||||
use super::PathInfoService;
|
use super::PathInfoService;
|
||||||
use crate::pathinfoservice::redb::RedbPathInfoService;
|
use crate::pathinfoservice::redb::RedbPathInfoService;
|
||||||
use crate::pathinfoservice::MemoryPathInfoService;
|
use crate::pathinfoservice::MemoryPathInfoService;
|
||||||
use crate::pathinfoservice::SledPathInfoService;
|
|
||||||
use crate::proto::PathInfo;
|
use crate::proto::PathInfo;
|
||||||
use crate::tests::fixtures::DUMMY_PATH_DIGEST;
|
use crate::tests::fixtures::DUMMY_PATH_DIGEST;
|
||||||
use tvix_castore::proto as castorepb;
|
use tvix_castore::proto as castorepb;
|
||||||
|
|
@ -29,7 +28,6 @@ use self::utils::make_bigtable_path_info_service;
|
||||||
let (_, _, svc) = make_grpc_path_info_service_client().await;
|
let (_, _, svc) = make_grpc_path_info_service_client().await;
|
||||||
svc
|
svc
|
||||||
})]
|
})]
|
||||||
#[case::sled(SledPathInfoService::new_temporary().unwrap())]
|
|
||||||
#[case::redb(RedbPathInfoService::new_temporary().unwrap())]
|
#[case::redb(RedbPathInfoService::new_temporary().unwrap())]
|
||||||
#[case::signing(test_signing_service())]
|
#[case::signing(test_signing_service())]
|
||||||
#[cfg_attr(all(feature = "cloud",feature="integration"), case::bigtable(make_bigtable_path_info_service().await))]
|
#[cfg_attr(all(feature = "cloud",feature="integration"), case::bigtable(make_bigtable_path_info_service().await))]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue