2022-11-11 23:48:24 +01:00
|
|
|
[package]
|
2023-03-14 22:36:10 +01:00
|
|
|
name = "tvix-store"
|
2022-11-11 23:48:24 +01:00
|
|
|
version = "0.1.0"
|
|
|
|
|
edition = "2021"
|
|
|
|
|
|
|
|
|
|
[dependencies]
|
2022-12-27 18:10:46 +01:00
|
|
|
anyhow = "1.0.68"
|
2022-11-12 00:40:09 +01:00
|
|
|
blake3 = { version = "1.3.1", features = ["rayon", "std"] }
|
2023-02-13 16:47:22 +01:00
|
|
|
clap = { version = "4.0", features = ["derive", "env"] }
|
|
|
|
|
count-write = "0.1.0"
|
2022-12-29 22:47:02 +01:00
|
|
|
data-encoding = "2.3.3"
|
|
|
|
|
lazy_static = "1.4.0"
|
2023-03-01 18:30:54 +01:00
|
|
|
nix-compat = { path = "../nix-compat" }
|
2022-11-12 00:07:48 +01:00
|
|
|
prost = "0.11.2"
|
2023-03-01 18:30:54 +01:00
|
|
|
rayon = "1.6.1"
|
2023-02-13 16:47:22 +01:00
|
|
|
sha2 = "0.10.6"
|
2022-12-29 21:39:28 +01:00
|
|
|
sled = { version = "0.34.7", features = ["compression"] }
|
2022-12-27 18:10:46 +01:00
|
|
|
thiserror = "1.0.38"
|
2023-05-01 16:29:20 +02:00
|
|
|
tokio-stream = "0.1.14"
|
2023-09-03 16:39:44 +02:00
|
|
|
tokio = { version = "1.28.0", features = ["net", "rt-multi-thread", "signal"] }
|
2022-11-13 00:23:14 +01:00
|
|
|
tonic = "0.8.2"
|
2022-12-28 17:17:53 +01:00
|
|
|
tracing = "0.1.37"
|
2023-02-16 16:49:22 +01:00
|
|
|
tracing-subscriber = { version = "0.3.16", features = ["json"] }
|
2023-09-05 16:08:50 +02:00
|
|
|
walkdir = "2.4.0"
|
2023-05-01 16:29:20 +02:00
|
|
|
tokio-util = { version = "0.7.8", features = ["io", "io-util"] }
|
feat(tvix/store/directorysvc): add gRPC client
This provides a GRPCDirectoryService struct implementing
DirectoryService, allowing a client to Directory objects from a (remote)
tvix-store.
Remote in this case is anything outside the current process, be it
another process, or an endpoint on the network.
To keep the sync interface in the `DirectoryService` trait, a handle to
some tokio runtime needs to be passed into the constructor, and the two
methods use `self.tokio_handle.spawn` to start an async function, and
`self.tokio_handle.block_on` to wait for its completion.
The client handle, called `grpc_client` itself is easy to clone, and
treats concurrent requests internally. This means, even though we keep
the `DirectoryService` trait sync, there's nothing preventing it from
being used concurrently, let's say from multiple threads.
There's still two limitations for now:
1) The trait doesn't make use of the `recursive` request, which
currently leads to a N+1 query problem. This can be fixed
by `GRPCDirectoryService` having a reference to another
`DirectoryService` acting as the local side.
I want to wait for general store composition code to pop up before
manually coding this here.
2) It's currently only possible to put() leaf directory nodes, as the
request normally requires uploading a whole closure. We might want
to add another batch function to upload a whole closure, and/or do
this batching in certain cases. This still needs some more thinking.
Change-Id: I7ffec791610b72c0960cf5307cefbb12ec946dc9
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8336
Tested-by: BuildkiteCI
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: tazjin <tazjin@tvl.su>
2023-03-23 13:49:57 +01:00
|
|
|
tower = "0.4.13"
|
refactor(tvix/store): remove ChunkService
Whether chunking is involved or not, is an implementation detail of each
Blobstore. Consumers of a whole blob shouldn't need to worry about that.
It currently is not visible in the gRPC interface either. It
shouldn't bleed into everything.
Let the BlobService trait provide `open_read` and `open_write` methods,
which return handles providing io::Read or io::Write, and leave the
details up to the implementation.
This means, our custom BlobReader module can go away, and all the
chunking bits in there, too.
In the future, we might still want to add more chunking-aware syncing,
but as a syncing strategy some stores can expose, not as a fundamental
protocol component.
This currently needs "SyncReadIntoAsyncRead", taken and vendored in from
https://github.com/tokio-rs/tokio/pull/5669.
It provides a AsyncRead for a sync Read, which is necessary to connect
our (sync) BlobReader interface to a GRPC server implementation.
As an alternative, we could also make the BlobReader itself async, and
let consumers of the trait (EvalIO) deal with the async-ness, but this
is less of a change for now.
In terms of vendoring, I initially tried to move our tokio crate to
these commits, but ended up in version incompatibilities, so let's
vendor it in for now.
Change-Id: I5969ebbc4c0e1ceece47981be3b9e7cfb3f59ad0
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8551
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
2023-05-11 14:49:01 +02:00
|
|
|
futures = "0.3.28"
|
|
|
|
|
bytes = "1.4.0"
|
2023-05-11 17:01:17 +02:00
|
|
|
smol_str = "0.2.0"
|
|
|
|
|
serde_json = "1.0"
|
2023-06-09 18:07:00 +02:00
|
|
|
url = "2.4.0"
|
2022-11-12 00:07:48 +01:00
|
|
|
|
2023-05-28 09:22:08 +02:00
|
|
|
[dependencies.fuser]
|
|
|
|
|
optional = true
|
|
|
|
|
version = "0.12.0"
|
|
|
|
|
|
2022-11-26 02:14:02 +01:00
|
|
|
[dependencies.tonic-reflection]
|
|
|
|
|
optional = true
|
|
|
|
|
version = "0.5.0"
|
|
|
|
|
|
2023-05-28 09:22:08 +02:00
|
|
|
[dependencies.libc]
|
|
|
|
|
optional = true
|
|
|
|
|
version = "0.2.144"
|
|
|
|
|
|
2022-11-12 00:07:48 +01:00
|
|
|
[build-dependencies]
|
|
|
|
|
prost-build = "0.11.2"
|
2022-11-13 00:23:14 +01:00
|
|
|
tonic-build = "0.8.2"
|
2022-12-29 22:47:02 +01:00
|
|
|
|
|
|
|
|
[dev-dependencies]
|
|
|
|
|
test-case = "2.2.2"
|
2022-12-29 21:39:28 +01:00
|
|
|
tempfile = "3.3.0"
|
2023-01-18 20:46:33 +01:00
|
|
|
tonic-mock = { git = "https://github.com/brainrake/tonic-mock", branch = "bump-dependencies" }
|
2022-11-26 02:14:02 +01:00
|
|
|
|
|
|
|
|
[features]
|
2023-05-28 09:22:08 +02:00
|
|
|
default = ["fuse", "reflection"]
|
|
|
|
|
fuse = ["dep:fuser", "dep:libc"]
|
2023-01-21 14:18:51 +01:00
|
|
|
reflection = ["tonic-reflection"]
|