mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/castore): introduce "cloud" feature flag

Browse source 
This controls whether tvix-castore has support for various cloud
backends or not.

Use this to control the set of feature flags for the object_store
backend, and only enable the aws, azure and gcp ones if it's set.
In the future this can be used to enable/disable other cloud backends
too.

Without feature flags, `object_store` already supports the `InMemory`
and `LocalFilesystem` backends, and we also want to unconditionally
enable the `http` one. Make sure at least the construction of these
services is covered in the tests.

Similarly, the tvix-store crate, which provides the tvix-store CLI has a
`cloud` feature flag too (defaulting to enabled).

Change-Id: I9fb9c87b740e7dc83f8ff7a0862905d036d513f2
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11204
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2024-03-19 11:46:12 +02:00 • committed by clbot
parent 591edf0d5b
commit 2798803f76
4 changed files with 34 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

7
tvix/castore/Cargo.toml
View file

@ -14,7 +14,7 @@ digest = "0.10.7"
fastcdc = { version = "3.1.0", features = ["tokio"] }
futures = "0.3.30"
lazy_static = "1.4.0"
object_store = { version = "0.9.1", features = ["aws", "azure", "gcp", "http"] }
object_store = { version = "0.9.1", features = ["http"] }
parking_lot = "0.12.1"
pin-project-lite = "0.2.13"
prost = "0.12.1"
@ -78,6 +78,11 @@ hex-literal = "0.4.1"
[features]
default = []
cloud = [
"object_store/aws",
"object_store/azure",
"object_store/gcp",
]
fs = ["dep:libc", "dep:fuse-backend-rs"]
virtiofs = [
"fs",

28
tvix/castore/src/blobservice/from_addr.rs
View file

@ -131,15 +131,13 @@ mod tests {
/// Correct scheme to connect to localhost over http, without specifying a port.
#[test_case("grpc+https://localhost", true; "grpc valid https host without port")]
/// Correct scheme to connect to localhost over http, but with additional path, which is invalid.
#[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")]
/// An example for object store (Memory)
#[test_case("grpc+http://localhost/some-path", false; "grpc invalid has path")]
/// An example for object store (InMemory)
#[test_case("objectstore+memory:///", true; "objectstore valid memory url")]
/// An example for object store (File)
#[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")] // ??
/// An example for object store (S3)
#[test_case("objectstore+s3://bucket/path", true; "objectstore valid s3 url")]
/// An example for object store (GCS)
#[test_case("objectstore+gs://bucket/path", true; "objectstore valid gcs url")]
/// An example for object store (LocalFileSystem)
#[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")]
// An example for object store (HTTP / WebDAV)
#[test_case("objectstore+https://localhost:8080/some-path", true; "objectstore valid http url")]
#[tokio::test]
async fn test_from_addr_tokio(uri_str: &str, exp_succeed: bool) {
if exp_succeed {
@ -148,4 +146,18 @@ mod tests {
assert!(from_addr(uri_str).await.is_err(), "should fail");
}
}
#[cfg(feature = "cloud")]
/// An example for object store (S3)
#[test_case("objectstore+s3://bucket/path", true; "objectstore valid s3 url")]
/// An example for object store (GCS)
#[test_case("objectstore+gs://bucket/path", true; "objectstore valid gcs url")]
#[tokio::test]
async fn test_from_addr_tokio_cloud(uri_str: &str, exp_succeed: bool) {
if exp_succeed {
from_addr(uri_str).await.expect("should succeed");
} else {
assert!(from_addr(uri_str).await.is_err(), "should fail");
}
}
}