From 9748543f1c3edd88e8b05f6a772d6e9be18a7be9 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 11 Dec 2023 21:25:34 +0200 Subject: [PATCH] chore(tvix): drop sled compression support It's been a while since the last sled release, and that one binds to a pretty old version of zstd, requiring workarounds like cl/10090. Upstream sled main branch currently has zstd halfway patched out (it's a no-op, but the feature flag and options are still there), and it's in that state for a year. Rather than maintaining our own fork of sled, let's just stop using the compression feature in sled, dropping the version pin to zstd that way, removing the need for cl/10090. This doesn't mean we won't reintroduce per-blob compression - but we probably just won't let sled take care of the compression, but do it ourselves - which is necessary for more chunked blob storage anyways. Even though we do drop the feature flag, we still need to explicitly use use_compression(false). Change-Id: I0e4892d29e41c76653272dc1a3625180da6fee12 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10257 Autosubmit: flokli Reviewed-by: raitobezarius Tested-by: BuildkiteCI --- tvix/Cargo.lock | 1 - tvix/Cargo.nix | 9 +-------- tvix/castore/Cargo.toml | 2 +- tvix/castore/src/blobservice/from_addr.rs | 2 +- tvix/castore/src/blobservice/sled.rs | 4 +++- tvix/castore/src/directoryservice/sled.rs | 4 +++- tvix/store/Cargo.toml | 2 +- tvix/store/src/pathinfoservice/from_addr.rs | 2 +- tvix/store/src/pathinfoservice/sled.rs | 4 +++- 9 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tvix/Cargo.lock b/tvix/Cargo.lock index 5d15a9436..ce6f10bb7 100644 --- a/tvix/Cargo.lock +++ b/tvix/Cargo.lock @@ -2448,7 +2448,6 @@ dependencies = [ "libc", "log", "parking_lot 0.11.2", - "zstd", ] [[package]] diff --git a/tvix/Cargo.nix b/tvix/Cargo.nix index 826ebf320..5e4b54d9d 100644 --- a/tvix/Cargo.nix +++ b/tvix/Cargo.nix @@ -7426,11 +7426,6 @@ rec { name = "parking_lot"; packageId = "parking_lot 0.11.2"; } - { - name = "zstd"; - packageId = "zstd"; - optional = true; - } ]; devDependencies = [ { @@ -7450,7 +7445,7 @@ rec { "testing" = [ "event_log" "lock_free_delays" "compression" "failpoints" "backtrace" ]; "zstd" = [ "dep:zstd" ]; }; - resolvedDefaultFeatures = [ "compression" "default" "no_metrics" "zstd" ]; + resolvedDefaultFeatures = [ "default" "no_metrics" ]; }; "smallvec" = rec { crateName = "smallvec"; @@ -9457,7 +9452,6 @@ rec { { name = "sled"; packageId = "sled"; - features = [ "compression" ]; } { name = "thiserror"; @@ -9980,7 +9974,6 @@ rec { { name = "sled"; packageId = "sled"; - features = [ "compression" ]; } { name = "thiserror"; diff --git a/tvix/castore/Cargo.toml b/tvix/castore/Cargo.toml index 573c7daa0..468ef8ec3 100644 --- a/tvix/castore/Cargo.toml +++ b/tvix/castore/Cargo.toml @@ -12,7 +12,7 @@ futures = "0.3.28" lazy_static = "1.4.0" pin-project-lite = "0.2.13" prost = "0.12.1" -sled = { version = "0.34.7", features = ["compression"] } +sled = { version = "0.34.7" } thiserror = "1.0.38" tokio-stream = { version = "0.1.14", features = ["fs", "net"] } tokio-util = { version = "0.7.9", features = ["io", "io-util"] } diff --git a/tvix/castore/src/blobservice/from_addr.rs b/tvix/castore/src/blobservice/from_addr.rs index 2834d25a1..106edce04 100644 --- a/tvix/castore/src/blobservice/from_addr.rs +++ b/tvix/castore/src/blobservice/from_addr.rs @@ -36,7 +36,7 @@ pub async fn from_addr(uri: &str) -> Result, crate::Error> )); } - // TODO: expose compression and other parameters as URL parameters? + // TODO: expose other parameters as URL parameters? if url.path().is_empty() { return Ok(Arc::new( diff --git a/tvix/castore/src/blobservice/sled.rs b/tvix/castore/src/blobservice/sled.rs index 12391ed72..a6fdbac49 100644 --- a/tvix/castore/src/blobservice/sled.rs +++ b/tvix/castore/src/blobservice/sled.rs @@ -15,7 +15,9 @@ pub struct SledBlobService { impl SledBlobService { pub fn new(p: PathBuf) -> Result { - let config = sled::Config::default().use_compression(true).path(p); + let config = sled::Config::default() + .use_compression(false) // is a required parameter + .path(p); let db = config.open()?; Ok(Self { db }) diff --git a/tvix/castore/src/directoryservice/sled.rs b/tvix/castore/src/directoryservice/sled.rs index 400d2fed5..50e58e7e7 100644 --- a/tvix/castore/src/directoryservice/sled.rs +++ b/tvix/castore/src/directoryservice/sled.rs @@ -18,7 +18,9 @@ pub struct SledDirectoryService { impl SledDirectoryService { pub fn new(p: PathBuf) -> Result { - let config = sled::Config::default().use_compression(true).path(p); + let config = sled::Config::default() + .use_compression(false) // is a required parameter + .path(p); let db = config.open()?; Ok(Self { db }) diff --git a/tvix/store/Cargo.toml b/tvix/store/Cargo.toml index d5ea10a5b..86d40d72b 100644 --- a/tvix/store/Cargo.toml +++ b/tvix/store/Cargo.toml @@ -18,7 +18,7 @@ parking_lot = "0.12.1" pin-project-lite = "0.2.13" prost = "0.12.1" sha2 = "0.10.6" -sled = { version = "0.34.7", features = ["compression"] } +sled = { version = "0.34.7" } thiserror = "1.0.38" tokio = { version = "1.32.0", features = ["fs", "macros", "net", "rt", "rt-multi-thread", "signal"] } tokio-listener = { version = "0.2.2", features = [ "tonic010" ] } diff --git a/tvix/store/src/pathinfoservice/from_addr.rs b/tvix/store/src/pathinfoservice/from_addr.rs index ee6a6925a..1505b3e42 100644 --- a/tvix/store/src/pathinfoservice/from_addr.rs +++ b/tvix/store/src/pathinfoservice/from_addr.rs @@ -54,7 +54,7 @@ pub async fn from_addr( )); } - // TODO: expose compression and other parameters as URL parameters? + // TODO: expose other parameters as URL parameters? if url.path().is_empty() { return Ok(Arc::new( diff --git a/tvix/store/src/pathinfoservice/sled.rs b/tvix/store/src/pathinfoservice/sled.rs index a12f918c8..1382c3c29 100644 --- a/tvix/store/src/pathinfoservice/sled.rs +++ b/tvix/store/src/pathinfoservice/sled.rs @@ -26,7 +26,9 @@ impl SledPathInfoService { blob_service: Arc, directory_service: Arc, ) -> Result { - let config = sled::Config::default().use_compression(true).path(p); + let config = sled::Config::default() + .use_compression(false) // is a required parameter + .path(p); let db = config.open()?; Ok(Self {