refactor(tvix/castore): introduce "cloud" feature flag

This controls whether tvix-castore has support for various cloud
backends or not.

Use this to control the set of feature flags for the object_store
backend, and only enable the aws, azure and gcp ones if it's set.
In the future this can be used to enable/disable other cloud backends
too.

Without feature flags, `object_store` already supports the `InMemory`
and `LocalFilesystem` backends, and we also want to unconditionally
enable the `http` one. Make sure at least the construction of these
services is covered in the tests.

Similarly, the tvix-store crate, which provides the tvix-store CLI has a
`cloud` feature flag too (defaulting to enabled).

Change-Id: I9fb9c87b740e7dc83f8ff7a0862905d036d513f2
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11204
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2024-03-19 11:46:12 +02:00 committed by clbot
parent 591edf0d5b
commit 2798803f76
4 changed files with 34 additions and 14 deletions

View file

@ -12117,7 +12117,7 @@ rec {
{
name = "object_store";
packageId = "object_store";
features = [ "aws" "azure" "gcp" "http" ];
features = [ "http" ];
}
{
name = "parking_lot";
@ -12243,12 +12243,13 @@ rec {
}
];
features = {
"cloud" = [ "object_store/aws" "object_store/azure" "object_store/gcp" ];
"fs" = [ "dep:libc" "dep:fuse-backend-rs" ];
"fuse" = [ "fs" ];
"tonic-reflection" = [ "dep:tonic-reflection" ];
"virtiofs" = [ "fs" "dep:vhost" "dep:vhost-user-backend" "dep:virtio-queue" "dep:vm-memory" "dep:vmm-sys-util" "dep:virtio-bindings" "fuse-backend-rs?/vhost-user-fs" "fuse-backend-rs?/virtiofs" ];
};
resolvedDefaultFeatures = [ "default" "fs" "fuse" "tonic-reflection" "virtiofs" ];
resolvedDefaultFeatures = [ "cloud" "default" "fs" "fuse" "tonic-reflection" "virtiofs" ];
};
"tvix-cli" = rec {
crateName = "tvix-cli";
@ -12927,13 +12928,14 @@ rec {
}
];
features = {
"default" = [ "fuse" "otlp" "tonic-reflection" ];
"cloud" = [ "tvix-castore/cloud" ];
"default" = [ "cloud" "fuse" "otlp" "tonic-reflection" ];
"fuse" = [ "tvix-castore/fuse" ];
"otlp" = [ "dep:opentelemetry" "dep:opentelemetry-otlp" "dep:opentelemetry_sdk" ];
"tonic-reflection" = [ "dep:tonic-reflection" "tvix-castore/tonic-reflection" ];
"virtiofs" = [ "tvix-castore/virtiofs" ];
};
resolvedDefaultFeatures = [ "default" "fuse" "otlp" "tonic-reflection" "virtiofs" ];
resolvedDefaultFeatures = [ "cloud" "default" "fuse" "otlp" "tonic-reflection" "virtiofs" ];
};
"typenum" = rec {
crateName = "typenum";

View file

@ -14,7 +14,7 @@ digest = "0.10.7"
fastcdc = { version = "3.1.0", features = ["tokio"] }
futures = "0.3.30"
lazy_static = "1.4.0"
object_store = { version = "0.9.1", features = ["aws", "azure", "gcp", "http"] }
object_store = { version = "0.9.1", features = ["http"] }
parking_lot = "0.12.1"
pin-project-lite = "0.2.13"
prost = "0.12.1"
@ -78,6 +78,11 @@ hex-literal = "0.4.1"
[features]
default = []
cloud = [
"object_store/aws",
"object_store/azure",
"object_store/gcp",
]
fs = ["dep:libc", "dep:fuse-backend-rs"]
virtiofs = [
"fs",

View file

@ -131,15 +131,13 @@ mod tests {
/// Correct scheme to connect to localhost over http, without specifying a port.
#[test_case("grpc+https://localhost", true; "grpc valid https host without port")]
/// Correct scheme to connect to localhost over http, but with additional path, which is invalid.
#[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")]
/// An example for object store (Memory)
#[test_case("grpc+http://localhost/some-path", false; "grpc invalid has path")]
/// An example for object store (InMemory)
#[test_case("objectstore+memory:///", true; "objectstore valid memory url")]
/// An example for object store (File)
#[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")] // ??
/// An example for object store (S3)
#[test_case("objectstore+s3://bucket/path", true; "objectstore valid s3 url")]
/// An example for object store (GCS)
#[test_case("objectstore+gs://bucket/path", true; "objectstore valid gcs url")]
/// An example for object store (LocalFileSystem)
#[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")]
// An example for object store (HTTP / WebDAV)
#[test_case("objectstore+https://localhost:8080/some-path", true; "objectstore valid http url")]
#[tokio::test]
async fn test_from_addr_tokio(uri_str: &str, exp_succeed: bool) {
if exp_succeed {
@ -148,4 +146,18 @@ mod tests {
assert!(from_addr(uri_str).await.is_err(), "should fail");
}
}
#[cfg(feature = "cloud")]
/// An example for object store (S3)
#[test_case("objectstore+s3://bucket/path", true; "objectstore valid s3 url")]
/// An example for object store (GCS)
#[test_case("objectstore+gs://bucket/path", true; "objectstore valid gcs url")]
#[tokio::test]
async fn test_from_addr_tokio_cloud(uri_str: &str, exp_succeed: bool) {
if exp_succeed {
from_addr(uri_str).await.expect("should succeed");
} else {
assert!(from_addr(uri_str).await.is_err(), "should fail");
}
}
}

View file

@ -53,7 +53,8 @@ tempfile = "3.3.0"
tokio-retry = "0.3.0"
[features]
default = ["fuse", "otlp", "tonic-reflection"]
default = ["cloud", "fuse", "otlp", "tonic-reflection"]
cloud = ["tvix-castore/cloud"]
fuse = ["tvix-castore/fuse"]
otlp = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk"]
tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"]