refactor(tvix/castore): introduce "cloud" feature flag
This controls whether tvix-castore has support for various cloud backends or not. Use this to control the set of feature flags for the object_store backend, and only enable the aws, azure and gcp ones if it's set. In the future this can be used to enable/disable other cloud backends too. Without feature flags, `object_store` already supports the `InMemory` and `LocalFilesystem` backends, and we also want to unconditionally enable the `http` one. Make sure at least the construction of these services is covered in the tests. Similarly, the tvix-store crate, which provides the tvix-store CLI has a `cloud` feature flag too (defaulting to enabled). Change-Id: I9fb9c87b740e7dc83f8ff7a0862905d036d513f2 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11204 Autosubmit: flokli <flokli@flokli.de> Reviewed-by: raitobezarius <tvl@lahfa.xyz> Tested-by: BuildkiteCI
This commit is contained in:
parent
591edf0d5b
commit
2798803f76
4 changed files with 34 additions and 14 deletions
|
@ -12117,7 +12117,7 @@ rec {
|
|||
{
|
||||
name = "object_store";
|
||||
packageId = "object_store";
|
||||
features = [ "aws" "azure" "gcp" "http" ];
|
||||
features = [ "http" ];
|
||||
}
|
||||
{
|
||||
name = "parking_lot";
|
||||
|
@ -12243,12 +12243,13 @@ rec {
|
|||
}
|
||||
];
|
||||
features = {
|
||||
"cloud" = [ "object_store/aws" "object_store/azure" "object_store/gcp" ];
|
||||
"fs" = [ "dep:libc" "dep:fuse-backend-rs" ];
|
||||
"fuse" = [ "fs" ];
|
||||
"tonic-reflection" = [ "dep:tonic-reflection" ];
|
||||
"virtiofs" = [ "fs" "dep:vhost" "dep:vhost-user-backend" "dep:virtio-queue" "dep:vm-memory" "dep:vmm-sys-util" "dep:virtio-bindings" "fuse-backend-rs?/vhost-user-fs" "fuse-backend-rs?/virtiofs" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "default" "fs" "fuse" "tonic-reflection" "virtiofs" ];
|
||||
resolvedDefaultFeatures = [ "cloud" "default" "fs" "fuse" "tonic-reflection" "virtiofs" ];
|
||||
};
|
||||
"tvix-cli" = rec {
|
||||
crateName = "tvix-cli";
|
||||
|
@ -12927,13 +12928,14 @@ rec {
|
|||
}
|
||||
];
|
||||
features = {
|
||||
"default" = [ "fuse" "otlp" "tonic-reflection" ];
|
||||
"cloud" = [ "tvix-castore/cloud" ];
|
||||
"default" = [ "cloud" "fuse" "otlp" "tonic-reflection" ];
|
||||
"fuse" = [ "tvix-castore/fuse" ];
|
||||
"otlp" = [ "dep:opentelemetry" "dep:opentelemetry-otlp" "dep:opentelemetry_sdk" ];
|
||||
"tonic-reflection" = [ "dep:tonic-reflection" "tvix-castore/tonic-reflection" ];
|
||||
"virtiofs" = [ "tvix-castore/virtiofs" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "default" "fuse" "otlp" "tonic-reflection" "virtiofs" ];
|
||||
resolvedDefaultFeatures = [ "cloud" "default" "fuse" "otlp" "tonic-reflection" "virtiofs" ];
|
||||
};
|
||||
"typenum" = rec {
|
||||
crateName = "typenum";
|
||||
|
|
|
@ -14,7 +14,7 @@ digest = "0.10.7"
|
|||
fastcdc = { version = "3.1.0", features = ["tokio"] }
|
||||
futures = "0.3.30"
|
||||
lazy_static = "1.4.0"
|
||||
object_store = { version = "0.9.1", features = ["aws", "azure", "gcp", "http"] }
|
||||
object_store = { version = "0.9.1", features = ["http"] }
|
||||
parking_lot = "0.12.1"
|
||||
pin-project-lite = "0.2.13"
|
||||
prost = "0.12.1"
|
||||
|
@ -78,6 +78,11 @@ hex-literal = "0.4.1"
|
|||
|
||||
[features]
|
||||
default = []
|
||||
cloud = [
|
||||
"object_store/aws",
|
||||
"object_store/azure",
|
||||
"object_store/gcp",
|
||||
]
|
||||
fs = ["dep:libc", "dep:fuse-backend-rs"]
|
||||
virtiofs = [
|
||||
"fs",
|
||||
|
|
|
@ -131,15 +131,13 @@ mod tests {
|
|||
/// Correct scheme to connect to localhost over http, without specifying a port.
|
||||
#[test_case("grpc+https://localhost", true; "grpc valid https host without port")]
|
||||
/// Correct scheme to connect to localhost over http, but with additional path, which is invalid.
|
||||
#[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")]
|
||||
/// An example for object store (Memory)
|
||||
#[test_case("grpc+http://localhost/some-path", false; "grpc invalid has path")]
|
||||
/// An example for object store (InMemory)
|
||||
#[test_case("objectstore+memory:///", true; "objectstore valid memory url")]
|
||||
/// An example for object store (File)
|
||||
#[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")] // ??
|
||||
/// An example for object store (S3)
|
||||
#[test_case("objectstore+s3://bucket/path", true; "objectstore valid s3 url")]
|
||||
/// An example for object store (GCS)
|
||||
#[test_case("objectstore+gs://bucket/path", true; "objectstore valid gcs url")]
|
||||
/// An example for object store (LocalFileSystem)
|
||||
#[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")]
|
||||
// An example for object store (HTTP / WebDAV)
|
||||
#[test_case("objectstore+https://localhost:8080/some-path", true; "objectstore valid http url")]
|
||||
#[tokio::test]
|
||||
async fn test_from_addr_tokio(uri_str: &str, exp_succeed: bool) {
|
||||
if exp_succeed {
|
||||
|
@ -148,4 +146,18 @@ mod tests {
|
|||
assert!(from_addr(uri_str).await.is_err(), "should fail");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "cloud")]
|
||||
/// An example for object store (S3)
|
||||
#[test_case("objectstore+s3://bucket/path", true; "objectstore valid s3 url")]
|
||||
/// An example for object store (GCS)
|
||||
#[test_case("objectstore+gs://bucket/path", true; "objectstore valid gcs url")]
|
||||
#[tokio::test]
|
||||
async fn test_from_addr_tokio_cloud(uri_str: &str, exp_succeed: bool) {
|
||||
if exp_succeed {
|
||||
from_addr(uri_str).await.expect("should succeed");
|
||||
} else {
|
||||
assert!(from_addr(uri_str).await.is_err(), "should fail");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -53,7 +53,8 @@ tempfile = "3.3.0"
|
|||
tokio-retry = "0.3.0"
|
||||
|
||||
[features]
|
||||
default = ["fuse", "otlp", "tonic-reflection"]
|
||||
default = ["cloud", "fuse", "otlp", "tonic-reflection"]
|
||||
cloud = ["tvix-castore/cloud"]
|
||||
fuse = ["tvix-castore/fuse"]
|
||||
otlp = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk"]
|
||||
tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"]
|
||||
|
|
Loading…
Reference in a new issue