2022-11-11 23:48:24 +01:00
|
|
|
[package]
|
2023-03-14 22:36:10 +01:00
|
|
|
name = "tvix-store"
|
2022-11-11 23:48:24 +01:00
|
|
|
version = "0.1.0"
|
|
|
|
|
edition = "2021"
|
|
|
|
|
|
|
|
|
|
[dependencies]
|
2022-12-27 18:10:46 +01:00
|
|
|
anyhow = "1.0.68"
|
2022-11-12 00:40:09 +01:00
|
|
|
blake3 = { version = "1.3.1", features = ["rayon", "std"] }
|
2023-02-13 16:47:22 +01:00
|
|
|
clap = { version = "4.0", features = ["derive", "env"] }
|
|
|
|
|
count-write = "0.1.0"
|
2022-12-29 22:47:02 +01:00
|
|
|
data-encoding = "2.3.3"
|
|
|
|
|
lazy_static = "1.4.0"
|
2023-03-01 18:30:54 +01:00
|
|
|
nix-compat = { path = "../nix-compat" }
|
2022-11-12 00:07:48 +01:00
|
|
|
prost = "0.11.2"
|
2023-03-01 18:30:54 +01:00
|
|
|
rayon = "1.6.1"
|
2023-02-13 16:47:22 +01:00
|
|
|
sha2 = "0.10.6"
|
2022-12-29 21:39:28 +01:00
|
|
|
sled = { version = "0.34.7", features = ["compression"] }
|
2022-12-27 18:10:46 +01:00
|
|
|
thiserror = "1.0.38"
|
2023-05-01 16:29:20 +02:00
|
|
|
tokio-stream = "0.1.14"
|
refactor(tvix/store/blobsvc): make BlobStore async
We previously kept the trait of a BlobService sync.
This however had some annoying consequences:
- It became more and more complicated to track when we're in a context
with an async runtime in the context or not, producing bugs like
https://b.tvl.fyi/issues/304
- The sync trait shielded away async clients from async worloads,
requiring manual block_on code inside the gRPC client code, and
spawn_blocking calls in consumers of the trait, even if they were
async (like the gRPC server)
- We had to write our own custom glue code (SyncReadIntoAsyncRead)
to convert a sync io::Read into a tokio::io::AsyncRead, which already
existed in tokio internally, but upstream ia hesitant to expose.
This now makes the BlobService trait async (via the async_trait macro,
like we already do in various gRPC parts), and replaces the sync readers
and writers with their async counterparts.
Tests interacting with a BlobService now need to have an async runtime
available, the easiest way for this is to mark the test functions
with the tokio::test macro, allowing us to directly .await in the test
function.
In places where we don't have an async runtime available from context
(like tvix-cli), we can pass one down explicitly.
Now that we don't provide a sync interface anymore, the (sync) FUSE
library now holds a pointer to a tokio runtime handle, and needs to at
least have 2 threads available when talking to a blob service (which is
why some of the tests now use the multi_thread flavor).
The FUSE tests got a bit more verbose, as we couldn't use the
setup_and_mount function accepting a callback anymore. We can hopefully
move some of the test fixture setup to rstest in the future to make this
less repetitive.
Co-Authored-By: Connor Brewster <cbrewster@hey.com>
Change-Id: Ia0501b606e32c852d0108de9c9016b21c94a3c05
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9329
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Tested-by: BuildkiteCI
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
2023-09-13 14:20:21 +02:00
|
|
|
tokio = { version = "1.28.0", features = ["fs", "net", "rt-multi-thread", "signal"] }
|
2022-11-13 00:23:14 +01:00
|
|
|
tonic = "0.8.2"
|
2022-12-28 17:17:53 +01:00
|
|
|
tracing = "0.1.37"
|
2023-02-16 16:49:22 +01:00
|
|
|
tracing-subscriber = { version = "0.3.16", features = ["json"] }
|
2023-09-05 16:08:50 +02:00
|
|
|
walkdir = "2.4.0"
|
2023-05-01 16:29:20 +02:00
|
|
|
tokio-util = { version = "0.7.8", features = ["io", "io-util"] }
|
feat(tvix/store/directorysvc): add gRPC client
This provides a GRPCDirectoryService struct implementing
DirectoryService, allowing a client to Directory objects from a (remote)
tvix-store.
Remote in this case is anything outside the current process, be it
another process, or an endpoint on the network.
To keep the sync interface in the `DirectoryService` trait, a handle to
some tokio runtime needs to be passed into the constructor, and the two
methods use `self.tokio_handle.spawn` to start an async function, and
`self.tokio_handle.block_on` to wait for its completion.
The client handle, called `grpc_client` itself is easy to clone, and
treats concurrent requests internally. This means, even though we keep
the `DirectoryService` trait sync, there's nothing preventing it from
being used concurrently, let's say from multiple threads.
There's still two limitations for now:
1) The trait doesn't make use of the `recursive` request, which
currently leads to a N+1 query problem. This can be fixed
by `GRPCDirectoryService` having a reference to another
`DirectoryService` acting as the local side.
I want to wait for general store composition code to pop up before
manually coding this here.
2) It's currently only possible to put() leaf directory nodes, as the
request normally requires uploading a whole closure. We might want
to add another batch function to upload a whole closure, and/or do
this batching in certain cases. This still needs some more thinking.
Change-Id: I7ffec791610b72c0960cf5307cefbb12ec946dc9
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8336
Tested-by: BuildkiteCI
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: tazjin <tazjin@tvl.su>
2023-03-23 13:49:57 +01:00
|
|
|
tower = "0.4.13"
|
refactor(tvix/store): remove ChunkService
Whether chunking is involved or not, is an implementation detail of each
Blobstore. Consumers of a whole blob shouldn't need to worry about that.
It currently is not visible in the gRPC interface either. It
shouldn't bleed into everything.
Let the BlobService trait provide `open_read` and `open_write` methods,
which return handles providing io::Read or io::Write, and leave the
details up to the implementation.
This means, our custom BlobReader module can go away, and all the
chunking bits in there, too.
In the future, we might still want to add more chunking-aware syncing,
but as a syncing strategy some stores can expose, not as a fundamental
protocol component.
This currently needs "SyncReadIntoAsyncRead", taken and vendored in from
https://github.com/tokio-rs/tokio/pull/5669.
It provides a AsyncRead for a sync Read, which is necessary to connect
our (sync) BlobReader interface to a GRPC server implementation.
As an alternative, we could also make the BlobReader itself async, and
let consumers of the trait (EvalIO) deal with the async-ness, but this
is less of a change for now.
In terms of vendoring, I initially tried to move our tokio crate to
these commits, but ended up in version incompatibilities, so let's
vendor it in for now.
Change-Id: I5969ebbc4c0e1ceece47981be3b9e7cfb3f59ad0
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8551
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
2023-05-11 14:49:01 +02:00
|
|
|
futures = "0.3.28"
|
|
|
|
|
bytes = "1.4.0"
|
2023-05-11 17:01:17 +02:00
|
|
|
smol_str = "0.2.0"
|
|
|
|
|
serde_json = "1.0"
|
2023-06-09 18:07:00 +02:00
|
|
|
url = "2.4.0"
|
refactor(tvix/store/blobsvc): make BlobStore async
We previously kept the trait of a BlobService sync.
This however had some annoying consequences:
- It became more and more complicated to track when we're in a context
with an async runtime in the context or not, producing bugs like
https://b.tvl.fyi/issues/304
- The sync trait shielded away async clients from async worloads,
requiring manual block_on code inside the gRPC client code, and
spawn_blocking calls in consumers of the trait, even if they were
async (like the gRPC server)
- We had to write our own custom glue code (SyncReadIntoAsyncRead)
to convert a sync io::Read into a tokio::io::AsyncRead, which already
existed in tokio internally, but upstream ia hesitant to expose.
This now makes the BlobService trait async (via the async_trait macro,
like we already do in various gRPC parts), and replaces the sync readers
and writers with their async counterparts.
Tests interacting with a BlobService now need to have an async runtime
available, the easiest way for this is to mark the test functions
with the tokio::test macro, allowing us to directly .await in the test
function.
In places where we don't have an async runtime available from context
(like tvix-cli), we can pass one down explicitly.
Now that we don't provide a sync interface anymore, the (sync) FUSE
library now holds a pointer to a tokio runtime handle, and needs to at
least have 2 threads available when talking to a blob service (which is
why some of the tests now use the multi_thread flavor).
The FUSE tests got a bit more verbose, as we couldn't use the
setup_and_mount function accepting a callback anymore. We can hopefully
move some of the test fixture setup to rstest in the future to make this
less repetitive.
Co-Authored-By: Connor Brewster <cbrewster@hey.com>
Change-Id: Ia0501b606e32c852d0108de9c9016b21c94a3c05
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9329
Reviewed-by: Connor Brewster <cbrewster@hey.com>
Tested-by: BuildkiteCI
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
2023-09-13 14:20:21 +02:00
|
|
|
pin-project-lite = "0.2.13"
|
2022-11-12 00:07:48 +01:00
|
|
|
|
2023-05-28 09:22:08 +02:00
|
|
|
[dependencies.fuser]
|
|
|
|
|
optional = true
|
|
|
|
|
version = "0.12.0"
|
|
|
|
|
|
2022-11-26 02:14:02 +01:00
|
|
|
[dependencies.tonic-reflection]
|
|
|
|
|
optional = true
|
|
|
|
|
version = "0.5.0"
|
|
|
|
|
|
2023-05-28 09:22:08 +02:00
|
|
|
[dependencies.libc]
|
|
|
|
|
optional = true
|
|
|
|
|
version = "0.2.144"
|
|
|
|
|
|
2022-11-12 00:07:48 +01:00
|
|
|
[build-dependencies]
|
|
|
|
|
prost-build = "0.11.2"
|
2022-11-13 00:23:14 +01:00
|
|
|
tonic-build = "0.8.2"
|
2022-12-29 22:47:02 +01:00
|
|
|
|
|
|
|
|
[dev-dependencies]
|
|
|
|
|
test-case = "2.2.2"
|
2022-12-29 21:39:28 +01:00
|
|
|
tempfile = "3.3.0"
|
2023-01-18 20:46:33 +01:00
|
|
|
tonic-mock = { git = "https://github.com/brainrake/tonic-mock", branch = "bump-dependencies" }
|
2022-11-26 02:14:02 +01:00
|
|
|
|
|
|
|
|
[features]
|
2023-05-28 09:22:08 +02:00
|
|
|
default = ["fuse", "reflection"]
|
|
|
|
|
fuse = ["dep:fuser", "dep:libc"]
|
2023-01-21 14:18:51 +01:00
|
|
|
reflection = ["tonic-reflection"]
|