mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/castore/blobsvc/from_addr): support object_store

Browse source 
The object_store crate supports a ton of different stores, with different schemes.

For now, use a objectstore+ scheme prefix to enable these.

Change-Id: I946f76e32a0fb0867ef59060217894cda5b959b9
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11080
Tested-by: BuildkiteCI
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink 2024-03-02 18:12:40 +02:00 committed by flokli
parent 1c2db676a0
commit f22d5b3d11
1 changed files with 21 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
tvix/castore/src/blobservice/from_addr.rs
View file

@ -3,7 +3,8 @@ use url::Url;
use crate::{proto::blob_service_client::BlobServiceClient, Error};
use super::{
BlobService, GRPCBlobService, MemoryBlobService, SimpleFilesystemBlobService, SledBlobService,
BlobService, GRPCBlobService, MemoryBlobService, ObjectStoreBlobService,
SimpleFilesystemBlobService, SledBlobService,
};
/// Constructs a new instance of a [BlobService] from an URI.
@ -62,6 +63,17 @@ pub async fn from_addr(uri: &str) -> Result<Box<dyn BlobService>, crate::Error>
}
Box::new(SimpleFilesystemBlobService::new(url.path().into()).await?)
} else if let Some(_trimmed_scheme) = url.scheme().strip_prefix("objectstore+") {
// We need to convert the URL to string, strip the prefix there, and then
// parse it back as url, as Url::set_scheme() rejects some of the transitions we want to do.
let trimmed_url = {
let s = url.to_string();
Url::parse(s.strip_prefix("objectstore+").unwrap()).unwrap()
};
return Ok(Box::new(
ObjectStoreBlobService::parse_url(&trimmed_url)
.map_err(|e| Error::StorageError(e.to_string()))?,
));
} else {
Err(crate::Error::StorageError(format!(
"unknown scheme: {}",
@ -114,6 +126,14 @@ mod tests {
#[test_case("grpc+https://localhost", true; "grpc valid https host without port")]
/// Correct scheme to connect to localhost over http, but with additional path, which is invalid.
#[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")]
/// An example for object store (Memory)
#[test_case("objectstore+memory:///", true; "objectstore valid memory url")]
/// An example for object store (File)
#[test_case("objectstore+file:///foo/bar", true; "objectstore valid file url")] // ??
/// An example for object store (S3)
#[test_case("objectstore+s3://bucket/path", true; "objectstore valid s3 url")]
/// An example for object store (GCS)
#[test_case("objectstore+gs://bucket/path", true; "objectstore valid gcs url")]
#[tokio::test]
async fn test_from_addr_tokio(uri_str: &str, exp_succeed: bool) {
if exp_succeed {