feat(tvix/castore/directory): add bigtable backend
This adds a Directory service using https://cloud.google.com/bigtable/docs/ as a K/V store. Directory (closures) are put in individual keys. We don't do any bucketed upload of directory closures (yet), as castore/ fs does query individually, does not request recursively (and buffers). This will be addressed by store composition at some point. Change-Id: I7fada45bf386a78b7ec93be38c5f03879a2a6e22 Reviewed-on: https://cl.tvl.fyi/c/depot/+/11212 Tested-by: BuildkiteCI Reviewed-by: Connor Brewster <cbrewster@hey.com> Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink
clbot
parent
84ad8a0bbd
commit
17849c5c00
12 changed files with 2767 additions and 22 deletions
565
tvix/Cargo.lock
generated
565
tvix/Cargo.lock
generated
|
|
@ -119,6 +119,19 @@ version = "0.7.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
|
||||
|
||||
[[package]]
|
||||
name = "async-channel"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3"
|
||||
dependencies = [
|
||||
"concurrent-queue",
|
||||
"event-listener 5.2.0",
|
||||
"event-listener-strategy 0.5.0",
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.4.6"
|
||||
|
|
@ -134,6 +147,63 @@ dependencies = [
|
|||
"xz2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-io"
|
||||
version = "2.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884"
|
||||
dependencies = [
|
||||
"async-lock 3.3.0",
|
||||
"cfg-if",
|
||||
"concurrent-queue",
|
||||
"futures-io",
|
||||
"futures-lite",
|
||||
"parking",
|
||||
"polling",
|
||||
"rustix",
|
||||
"slab",
|
||||
"tracing",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-lock"
|
||||
version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b"
|
||||
dependencies = [
|
||||
"event-listener 2.5.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-lock"
|
||||
version = "3.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b"
|
||||
dependencies = [
|
||||
"event-listener 4.0.3",
|
||||
"event-listener-strategy 0.4.0",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-process"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "451e3cf68011bd56771c79db04a9e333095ab6349f7e47592b788e9b98720cc8"
|
||||
dependencies = [
|
||||
"async-channel",
|
||||
"async-io",
|
||||
"async-lock 3.3.0",
|
||||
"async-signal",
|
||||
"blocking",
|
||||
"cfg-if",
|
||||
"event-listener 5.2.0",
|
||||
"futures-lite",
|
||||
"rustix",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-recursion"
|
||||
version = "1.0.5"
|
||||
|
|
@ -145,6 +215,24 @@ dependencies = [
|
|||
"syn 2.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-signal"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5"
|
||||
dependencies = [
|
||||
"async-io",
|
||||
"async-lock 2.8.0",
|
||||
"atomic-waker",
|
||||
"cfg-if",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"rustix",
|
||||
"signal-hook-registry",
|
||||
"slab",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-stream"
|
||||
version = "0.3.5"
|
||||
|
|
@ -167,6 +255,12 @@ dependencies = [
|
|||
"syn 2.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-task"
|
||||
version = "4.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799"
|
||||
|
||||
[[package]]
|
||||
name = "async-tempfile"
|
||||
version = "0.4.0"
|
||||
|
|
@ -188,6 +282,12 @@ dependencies = [
|
|||
"syn 2.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atomic-waker"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.1.0"
|
||||
|
|
@ -321,6 +421,29 @@ version = "1.6.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
|
||||
|
||||
[[package]]
|
||||
name = "bigtable_rs"
|
||||
version = "0.2.9"
|
||||
source = "git+https://github.com/flokli/bigtable_rs?rev=0af404741dfc40eb9fa99cf4d4140a09c5c20df7#0af404741dfc40eb9fa99cf4d4140a09c5c20df7"
|
||||
dependencies = [
|
||||
"gcp_auth",
|
||||
"http 0.2.11",
|
||||
"log",
|
||||
"prost 0.12.3",
|
||||
"prost-build",
|
||||
"prost-types",
|
||||
"prost-wkt",
|
||||
"prost-wkt-build",
|
||||
"prost-wkt-types",
|
||||
"serde",
|
||||
"serde_with",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tonic 0.11.0",
|
||||
"tonic-build",
|
||||
"tower",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-set"
|
||||
version = "0.5.3"
|
||||
|
|
@ -378,6 +501,22 @@ dependencies = [
|
|||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blocking"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118"
|
||||
dependencies = [
|
||||
"async-channel",
|
||||
"async-lock 3.3.0",
|
||||
"async-task",
|
||||
"fastrand",
|
||||
"futures-io",
|
||||
"futures-lite",
|
||||
"piper",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bstr"
|
||||
version = "1.9.0"
|
||||
|
|
@ -468,8 +607,10 @@ checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b"
|
|||
dependencies = [
|
||||
"android-tzdata",
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"serde",
|
||||
"wasm-bindgen",
|
||||
"windows-targets 0.52.0",
|
||||
]
|
||||
|
||||
|
|
@ -573,6 +714,15 @@ version = "1.0.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
|
||||
|
||||
[[package]]
|
||||
name = "concurrent-queue"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.9.6"
|
||||
|
|
@ -739,6 +889,41 @@ dependencies = [
|
|||
"syn 2.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.20.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.20.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"ident_case",
|
||||
"proc-macro2 1.0.76",
|
||||
"quote 1.0.35",
|
||||
"strsim",
|
||||
"syn 2.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.20.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"quote 1.0.35",
|
||||
"syn 2.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "data-encoding"
|
||||
version = "2.5.0"
|
||||
|
|
@ -755,6 +940,16 @@ dependencies = [
|
|||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
|
||||
dependencies = [
|
||||
"powerfmt",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "diff"
|
||||
version = "0.1.13"
|
||||
|
|
@ -890,6 +1085,15 @@ version = "1.0.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
|
||||
|
||||
[[package]]
|
||||
name = "erased-serde"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b73807008a3c7f171cc40312f37d95ef0396e048b5848d775f54b1a4dd4a0d3"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.3.8"
|
||||
|
|
@ -910,6 +1114,54 @@ dependencies = [
|
|||
"str-buf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "event-listener"
|
||||
version = "2.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
|
||||
|
||||
[[package]]
|
||||
name = "event-listener"
|
||||
version = "4.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e"
|
||||
dependencies = [
|
||||
"concurrent-queue",
|
||||
"parking",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "event-listener"
|
||||
version = "5.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b5fb89194fa3cad959b833185b3063ba881dbfc7030680b314250779fb4cc91"
|
||||
dependencies = [
|
||||
"concurrent-queue",
|
||||
"parking",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "event-listener-strategy"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3"
|
||||
dependencies = [
|
||||
"event-listener 4.0.3",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "event-listener-strategy"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "feedafcaa9b749175d5ac357452a9d41ea2911da598fde46ce1fe02c37751291"
|
||||
dependencies = [
|
||||
"event-listener 5.2.0",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fastcdc"
|
||||
version = "3.1.0"
|
||||
|
|
@ -1066,6 +1318,19 @@ version = "0.3.30"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
|
||||
|
||||
[[package]]
|
||||
name = "futures-lite"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"parking",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.30"
|
||||
|
|
@ -1122,6 +1387,31 @@ dependencies = [
|
|||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gcp_auth"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "de2c71ea685b88a1aa50e9fb66fe0e1cb29d755f58cca41fb8c91ef604d4f4d4"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
"chrono",
|
||||
"home",
|
||||
"hyper 0.14.28",
|
||||
"hyper-rustls",
|
||||
"ring",
|
||||
"rustls 0.21.10",
|
||||
"rustls-pemfile 1.0.4",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-futures",
|
||||
"url",
|
||||
"which 5.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "genawaiter"
|
||||
version = "0.99.1"
|
||||
|
|
@ -1238,6 +1528,12 @@ version = "0.3.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f"
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "hex-literal"
|
||||
version = "0.4.1"
|
||||
|
|
@ -1381,6 +1677,7 @@ dependencies = [
|
|||
"http 0.2.11",
|
||||
"hyper 0.14.28",
|
||||
"rustls 0.21.10",
|
||||
"rustls-native-certs 0.6.3",
|
||||
"tokio",
|
||||
"tokio-rustls 0.24.1",
|
||||
]
|
||||
|
|
@ -1436,6 +1733,12 @@ dependencies = [
|
|||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ident_case"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.5.0"
|
||||
|
|
@ -1478,6 +1781,7 @@ checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
|
|||
dependencies = [
|
||||
"autocfg",
|
||||
"hashbrown 0.12.3",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1488,6 +1792,7 @@ checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
|
|||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.14.3",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1499,6 +1804,12 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inventory"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767"
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.9.0"
|
||||
|
|
@ -1924,6 +2235,12 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.18"
|
||||
|
|
@ -2104,6 +2421,12 @@ version = "0.1.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
|
||||
|
||||
[[package]]
|
||||
name = "parking"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.11.2"
|
||||
|
|
@ -2206,6 +2529,17 @@ version = "0.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "piper"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"fastrand",
|
||||
"futures-io",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkcs8"
|
||||
version = "0.10.2"
|
||||
|
|
@ -2256,6 +2590,26 @@ dependencies = [
|
|||
"plotters-backend",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polling"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30054e72317ab98eddd8561db0f6524df3367636884b7b21b703e4b280a84a14"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"concurrent-queue",
|
||||
"pin-project-lite",
|
||||
"rustix",
|
||||
"tracing",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
|
|
@ -2356,10 +2710,12 @@ dependencies = [
|
|||
"prettyplease",
|
||||
"prost 0.12.3",
|
||||
"prost-types",
|
||||
"pulldown-cmark",
|
||||
"pulldown-cmark-to-cmark",
|
||||
"regex",
|
||||
"syn 2.0.48",
|
||||
"tempfile",
|
||||
"which",
|
||||
"which 4.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2397,6 +2753,72 @@ dependencies = [
|
|||
"prost 0.12.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-wkt"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d8ef9c3f0f1dab910d2b7e2c24a8e4322e122eba6d7a1921eeebcebbc046c40"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"inventory",
|
||||
"prost 0.12.3",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"typetag",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-wkt-build"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b31cae9a54ca84fee1504740a82eebf2479532905e106f63ca0c3bc8d780321"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"prost 0.12.3",
|
||||
"prost-build",
|
||||
"prost-types",
|
||||
"quote 1.0.35",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-wkt-types"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "435be4a8704091b4c5fb1d79799de7f2dbff53af05edf29385237f8cf7ab37ee"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"prost 0.12.3",
|
||||
"prost-build",
|
||||
"prost-types",
|
||||
"prost-wkt",
|
||||
"prost-wkt-build",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pulldown-cmark"
|
||||
version = "0.9.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b"
|
||||
dependencies = [
|
||||
"bitflags 2.4.2",
|
||||
"memchr",
|
||||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pulldown-cmark-to-cmark"
|
||||
version = "10.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0194e6e1966c23cc5fd988714f85b18d548d773e81965413555d96569931833d"
|
||||
dependencies = [
|
||||
"pulldown-cmark",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "1.2.3"
|
||||
|
|
@ -2959,18 +3381,18 @@ checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0"
|
|||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.195"
|
||||
version = "1.0.197"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02"
|
||||
checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.195"
|
||||
version = "1.0.197"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c"
|
||||
checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.76",
|
||||
"quote 1.0.35",
|
||||
|
|
@ -2998,6 +3420,17 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_qs"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0431a35568651e363364210c91983c1da5eb29404d9f0928b67d4ebcfa7d330c"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
"serde",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "0.6.5"
|
||||
|
|
@ -3019,6 +3452,36 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_with"
|
||||
version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"chrono",
|
||||
"hex",
|
||||
"indexmap 1.9.3",
|
||||
"indexmap 2.1.0",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"serde_with_macros",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_with_macros"
|
||||
version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"proc-macro2 1.0.76",
|
||||
"quote 1.0.35",
|
||||
"syn 2.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.6"
|
||||
|
|
@ -3386,6 +3849,37 @@ dependencies = [
|
|||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"itoa",
|
||||
"num-conv",
|
||||
"powerfmt",
|
||||
"serde",
|
||||
"time-core",
|
||||
"time-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time-core"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774"
|
||||
dependencies = [
|
||||
"num-conv",
|
||||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinytemplate"
|
||||
version = "1.2.1"
|
||||
|
|
@ -3739,6 +4233,16 @@ dependencies = [
|
|||
"valuable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-futures"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
|
||||
dependencies = [
|
||||
"pin-project",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-log"
|
||||
version = "0.2.0"
|
||||
|
|
@ -3831,8 +4335,10 @@ dependencies = [
|
|||
name = "tvix-castore"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-process",
|
||||
"async-stream",
|
||||
"async-tempfile",
|
||||
"bigtable_rs",
|
||||
"blake3",
|
||||
"bstr",
|
||||
"bytes",
|
||||
|
|
@ -3851,6 +4357,9 @@ dependencies = [
|
|||
"prost-build",
|
||||
"rstest",
|
||||
"rstest_reuse",
|
||||
"serde",
|
||||
"serde_qs",
|
||||
"serde_with",
|
||||
"sled",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
|
|
@ -4045,12 +4554,45 @@ version = "1.17.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
|
||||
|
||||
[[package]]
|
||||
name = "typetag"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "661d18414ec032a49ece2d56eee03636e43c4e8d577047ab334c0ba892e29aaf"
|
||||
dependencies = [
|
||||
"erased-serde",
|
||||
"inventory",
|
||||
"once_cell",
|
||||
"serde",
|
||||
"typetag-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typetag-impl"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac73887f47b9312552aa90ef477927ff014d63d1920ca8037c6c1951eab64bb1"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.76",
|
||||
"quote 1.0.35",
|
||||
"syn 2.0.48",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unarray"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
|
||||
dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.15"
|
||||
|
|
@ -4363,6 +4905,19 @@ dependencies = [
|
|||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bf3ea8596f3a0dd5980b46430f2058dfe2c36a27ccfbb1845d6fbfcd9ba6e14"
|
||||
dependencies = [
|
||||
"either",
|
||||
"home",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
|
|
|
|||
1788
tvix/Cargo.nix
1788
tvix/Cargo.nix
File diff suppressed because it is too large
Load diff
|
|
@ -29,6 +29,15 @@ tracing = "0.1.37"
|
|||
url = "2.4.0"
|
||||
walkdir = "2.4.0"
|
||||
zstd = "0.13.0"
|
||||
serde = { version = "1.0.197", features = [ "derive" ] }
|
||||
serde_with = "3.7.0"
|
||||
serde_qs = "0.12.0"
|
||||
|
||||
[dependencies.bigtable_rs]
|
||||
optional = true
|
||||
# https://github.com/liufuyang/bigtable_rs/pull/72
|
||||
git = "https://github.com/flokli/bigtable_rs"
|
||||
rev = "0af404741dfc40eb9fa99cf4d4140a09c5c20df7"
|
||||
|
||||
[dependencies.fuse-backend-rs]
|
||||
optional = true
|
||||
|
|
@ -71,6 +80,7 @@ prost-build = "0.12.1"
|
|||
tonic-build = "0.11.0"
|
||||
|
||||
[dev-dependencies]
|
||||
async-process = "2.1.0"
|
||||
rstest = "0.18.2"
|
||||
tempfile = "3.3.0"
|
||||
tokio-retry = "0.3.0"
|
||||
|
|
@ -80,6 +90,7 @@ rstest_reuse = "0.6.0"
|
|||
[features]
|
||||
default = []
|
||||
cloud = [
|
||||
"dep:bigtable_rs",
|
||||
"object_store/aws",
|
||||
"object_store/azure",
|
||||
"object_store/gcp",
|
||||
|
|
|
|||
|
|
@ -4,5 +4,9 @@ depot.tvix.crates.workspaceMembers.tvix-castore.build.override {
|
|||
runTests = true;
|
||||
testPreRun = ''
|
||||
export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt;
|
||||
export PATH="$PATH:${pkgs.lib.makeBinPath [pkgs.cbtemulator pkgs.google-cloud-bigtable-tool]}"
|
||||
'';
|
||||
|
||||
# enable some optional features.
|
||||
features = [ "default" "cloud" ];
|
||||
}
|
||||
|
|
|
|||
355
tvix/castore/src/directoryservice/bigtable.rs
Normal file
355
tvix/castore/src/directoryservice/bigtable.rs
Normal file
|
|
@ -0,0 +1,355 @@
|
|||
use bigtable_rs::{bigtable, google::bigtable::v2 as bigtable_v2};
|
||||
use bytes::Bytes;
|
||||
use data_encoding::HEXLOWER;
|
||||
use futures::stream::BoxStream;
|
||||
use prost::Message;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_with::{serde_as, DurationSeconds};
|
||||
use tonic::async_trait;
|
||||
use tracing::{instrument, trace, warn};
|
||||
|
||||
use super::{utils::traverse_directory, DirectoryPutter, DirectoryService, SimplePutter};
|
||||
use crate::{proto, B3Digest, Error};
|
||||
|
||||
/// There should not be more than 10 MiB in a single cell.
|
||||
/// https://cloud.google.com/bigtable/docs/schema-design#cells
|
||||
const CELL_SIZE_LIMIT: u64 = 10 * 1024 * 1024;
|
||||
|
||||
/// Provides a [DirectoryService] implementation using
|
||||
/// [Bigtable](https://cloud.google.com/bigtable/docs/)
|
||||
/// as an underlying K/V store.
|
||||
///
|
||||
/// # Data format
|
||||
/// We use Bigtable as a plain K/V store.
|
||||
/// The row key is the digest of the directory, in hexlower.
|
||||
/// Inside the row, we currently have a single column/cell, again using the
|
||||
/// hexlower directory digest.
|
||||
/// Its value is the Directory message, serialized in canonical protobuf.
|
||||
/// We currently only populate this column.
|
||||
///
|
||||
/// In the future, we might want to introduce "bucketing", essentially storing
|
||||
/// all directories inserted via `put_multiple_start` in a batched form.
|
||||
/// This will prevent looking up intermediate Directories, which are not
|
||||
/// directly at the root, so rely on store composition.
|
||||
#[derive(Clone)]
|
||||
pub struct BigtableDirectoryService {
|
||||
client: bigtable::BigTable,
|
||||
params: BigtableParameters,
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(dead_code)]
|
||||
/// Holds the temporary directory containing the unix socket, and the
|
||||
/// spawned emulator process.
|
||||
emulator: std::sync::Arc<(tempfile::TempDir, async_process::Child)>,
|
||||
}
|
||||
|
||||
/// Represents configuration of [BigtableDirectoryService].
|
||||
/// This currently conflates both connect parameters and data model/client
|
||||
/// behaviour parameters.
|
||||
#[serde_as]
|
||||
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
|
||||
pub struct BigtableParameters {
|
||||
project_id: String,
|
||||
instance_name: String,
|
||||
#[serde(default)]
|
||||
is_read_only: bool,
|
||||
#[serde(default = "default_channel_size")]
|
||||
channel_size: usize,
|
||||
|
||||
#[serde_as(as = "Option<DurationSeconds<String>>")]
|
||||
#[serde(default = "default_timeout")]
|
||||
timeout: Option<std::time::Duration>,
|
||||
table_name: String,
|
||||
family_name: String,
|
||||
|
||||
#[serde(default = "default_app_profile_id")]
|
||||
app_profile_id: String,
|
||||
}
|
||||
|
||||
fn default_app_profile_id() -> String {
|
||||
"default".to_owned()
|
||||
}
|
||||
|
||||
fn default_channel_size() -> usize {
|
||||
4
|
||||
}
|
||||
|
||||
fn default_timeout() -> Option<std::time::Duration> {
|
||||
Some(std::time::Duration::from_secs(4))
|
||||
}
|
||||
|
||||
impl BigtableDirectoryService {
|
||||
#[cfg(not(test))]
|
||||
pub async fn connect(params: BigtableParameters) -> Result<Self, bigtable::Error> {
|
||||
let connection = bigtable::BigTableConnection::new(
|
||||
¶ms.project_id,
|
||||
¶ms.instance_name,
|
||||
params.is_read_only,
|
||||
params.channel_size,
|
||||
params.timeout,
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(Self {
|
||||
client: connection.client(),
|
||||
params,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub async fn connect(params: BigtableParameters) -> Result<Self, bigtable::Error> {
|
||||
use std::time::Duration;
|
||||
|
||||
use async_process::{Command, Stdio};
|
||||
use tempfile::TempDir;
|
||||
use tokio_retry::{strategy::ExponentialBackoff, Retry};
|
||||
|
||||
let tmpdir = TempDir::new().unwrap();
|
||||
|
||||
let socket_path = tmpdir.path().join("cbtemulator.sock");
|
||||
|
||||
let emulator_process = Command::new("cbtemulator")
|
||||
.arg("-address")
|
||||
.arg(socket_path.clone())
|
||||
.stderr(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.kill_on_drop(true)
|
||||
.spawn()
|
||||
.expect("failed to spwan emulator");
|
||||
|
||||
Retry::spawn(
|
||||
ExponentialBackoff::from_millis(20).max_delay(Duration::from_secs(1)),
|
||||
|| async {
|
||||
if socket_path.exists() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(())
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("failed to wait for socket");
|
||||
|
||||
// populate the emulator
|
||||
for cmd in &[
|
||||
vec!["createtable", ¶ms.table_name],
|
||||
vec!["createfamily", ¶ms.table_name, ¶ms.family_name],
|
||||
] {
|
||||
Command::new("cbt")
|
||||
.args({
|
||||
let mut args = vec![
|
||||
"-instance",
|
||||
¶ms.instance_name,
|
||||
"-project",
|
||||
¶ms.project_id,
|
||||
];
|
||||
args.extend_from_slice(cmd);
|
||||
args
|
||||
})
|
||||
.env(
|
||||
"BIGTABLE_EMULATOR_HOST",
|
||||
format!("unix://{}", socket_path.to_string_lossy()),
|
||||
)
|
||||
.output()
|
||||
.await
|
||||
.expect("failed to run cbt setup command");
|
||||
}
|
||||
|
||||
let connection = bigtable_rs::bigtable::BigTableConnection::new_with_emulator(
|
||||
&format!("unix://{}", socket_path.to_string_lossy()),
|
||||
¶ms.project_id,
|
||||
¶ms.instance_name,
|
||||
params.is_read_only,
|
||||
params.timeout,
|
||||
)?;
|
||||
|
||||
Ok(Self {
|
||||
client: connection.client(),
|
||||
params,
|
||||
emulator: (tmpdir, emulator_process).into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Derives the row/column key for a given blake3 digest.
|
||||
/// We use hexlower encoding, also because it can't be misinterpreted as RE2.
|
||||
fn derive_directory_key(digest: &B3Digest) -> String {
|
||||
HEXLOWER.encode(digest.as_slice())
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl DirectoryService for BigtableDirectoryService {
|
||||
#[instrument(skip(self, digest), err, fields(directory.digest = %digest))]
|
||||
async fn get(&self, digest: &B3Digest) -> Result<Option<proto::Directory>, Error> {
|
||||
let mut client = self.client.clone();
|
||||
let directory_key = derive_directory_key(digest);
|
||||
|
||||
let request = bigtable_v2::ReadRowsRequest {
|
||||
app_profile_id: self.params.app_profile_id.to_string(),
|
||||
table_name: client.get_full_table_name(&self.params.table_name),
|
||||
rows_limit: 1,
|
||||
rows: Some(bigtable_v2::RowSet {
|
||||
row_keys: vec![directory_key.clone().into()],
|
||||
row_ranges: vec![],
|
||||
}),
|
||||
// Filter selected family name, and column qualifier matching our digest.
|
||||
// This is to ensure we don't fail once we start bucketing.
|
||||
filter: Some(bigtable_v2::RowFilter {
|
||||
filter: Some(bigtable_v2::row_filter::Filter::Chain(
|
||||
bigtable_v2::row_filter::Chain {
|
||||
filters: vec![
|
||||
bigtable_v2::RowFilter {
|
||||
filter: Some(
|
||||
bigtable_v2::row_filter::Filter::FamilyNameRegexFilter(
|
||||
self.params.family_name.to_string(),
|
||||
),
|
||||
),
|
||||
},
|
||||
bigtable_v2::RowFilter {
|
||||
filter: Some(
|
||||
bigtable_v2::row_filter::Filter::ColumnQualifierRegexFilter(
|
||||
directory_key.clone().into(),
|
||||
),
|
||||
),
|
||||
},
|
||||
],
|
||||
},
|
||||
)),
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut response = client
|
||||
.read_rows(request)
|
||||
.await
|
||||
.map_err(|e| Error::StorageError(format!("unable to read rows: {}", e)))?;
|
||||
|
||||
if response.len() != 1 {
|
||||
if response.len() > 1 {
|
||||
// This shouldn't happen, we limit number of rows to 1
|
||||
return Err(Error::StorageError(
|
||||
"got more than one row from bigtable".into(),
|
||||
));
|
||||
}
|
||||
// else, this is simply a "not found".
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let (row_key, mut row_cells) = response.pop().unwrap();
|
||||
if row_key != directory_key.as_bytes() {
|
||||
// This shouldn't happen, we requested this row key.
|
||||
return Err(Error::StorageError(
|
||||
"got wrong row key from bigtable".into(),
|
||||
));
|
||||
}
|
||||
|
||||
let row_cell = row_cells
|
||||
.pop()
|
||||
.ok_or_else(|| Error::StorageError("found no cells".into()))?;
|
||||
|
||||
// Ensure there's only one cell (so no more left after the pop())
|
||||
// This shouldn't happen, We filter out other cells in our query.
|
||||
if !row_cells.is_empty() {
|
||||
return Err(Error::StorageError(
|
||||
"more than one cell returned from bigtable".into(),
|
||||
));
|
||||
}
|
||||
|
||||
// We also require the qualifier to be correct in the filter above,
|
||||
// so this shouldn't happen.
|
||||
if directory_key.as_bytes() != row_cell.qualifier {
|
||||
return Err(Error::StorageError("unexpected cell qualifier".into()));
|
||||
}
|
||||
|
||||
// For the data in that cell, ensure the digest matches what's requested, before parsing.
|
||||
let got_digest = B3Digest::from(blake3::hash(&row_cell.value).as_bytes());
|
||||
if got_digest != *digest {
|
||||
return Err(Error::StorageError(format!(
|
||||
"invalid digest: {}",
|
||||
got_digest
|
||||
)));
|
||||
}
|
||||
|
||||
// Try to parse the value into a Directory message.
|
||||
let directory = proto::Directory::decode(Bytes::from(row_cell.value))
|
||||
.map_err(|e| Error::StorageError(format!("unable to decode directory proto: {}", e)))?;
|
||||
|
||||
// validate the Directory.
|
||||
directory
|
||||
.validate()
|
||||
.map_err(|e| Error::StorageError(format!("invalid Directory message: {}", e)))?;
|
||||
|
||||
Ok(Some(directory))
|
||||
}
|
||||
|
||||
#[instrument(skip(self, directory), err, fields(directory.digest = %directory.digest()))]
|
||||
async fn put(&self, directory: proto::Directory) -> Result<B3Digest, Error> {
|
||||
let directory_digest = directory.digest();
|
||||
let mut client = self.client.clone();
|
||||
let directory_key = derive_directory_key(&directory_digest);
|
||||
|
||||
// Ensure the directory we're trying to upload passes validation
|
||||
directory
|
||||
.validate()
|
||||
.map_err(|e| Error::InvalidRequest(format!("directory is invalid: {}", e)))?;
|
||||
|
||||
let data = directory.encode_to_vec();
|
||||
if data.len() as u64 > CELL_SIZE_LIMIT {
|
||||
return Err(Error::StorageError(
|
||||
"Directory exceeds cell limit on Bigtable".into(),
|
||||
));
|
||||
}
|
||||
|
||||
let resp = client
|
||||
.check_and_mutate_row(bigtable_v2::CheckAndMutateRowRequest {
|
||||
table_name: client.get_full_table_name(&self.params.table_name),
|
||||
app_profile_id: self.params.app_profile_id.to_string(),
|
||||
row_key: directory_key.clone().into(),
|
||||
predicate_filter: Some(bigtable_v2::RowFilter {
|
||||
filter: Some(bigtable_v2::row_filter::Filter::ColumnQualifierRegexFilter(
|
||||
directory_key.clone().into(),
|
||||
)),
|
||||
}),
|
||||
// If the column was already found, do nothing.
|
||||
true_mutations: vec![],
|
||||
// Else, do the insert.
|
||||
false_mutations: vec![
|
||||
// https://cloud.google.com/bigtable/docs/writes
|
||||
bigtable_v2::Mutation {
|
||||
mutation: Some(bigtable_v2::mutation::Mutation::SetCell(
|
||||
bigtable_v2::mutation::SetCell {
|
||||
family_name: self.params.family_name.to_string(),
|
||||
column_qualifier: directory_key.clone().into(),
|
||||
timestamp_micros: -1, // use server time to fill timestamp
|
||||
value: data,
|
||||
},
|
||||
)),
|
||||
},
|
||||
],
|
||||
})
|
||||
.await
|
||||
.map_err(|e| Error::StorageError(format!("unable to mutate rows: {}", e)))?;
|
||||
|
||||
if resp.predicate_matched {
|
||||
trace!("already existed")
|
||||
}
|
||||
|
||||
Ok(directory_digest)
|
||||
}
|
||||
|
||||
#[instrument(skip_all, fields(directory.digest = %root_directory_digest))]
|
||||
fn get_recursive(
|
||||
&self,
|
||||
root_directory_digest: &B3Digest,
|
||||
) -> BoxStream<Result<proto::Directory, Error>> {
|
||||
traverse_directory(self.clone(), root_directory_digest)
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
fn put_multiple_start(&self) -> Box<(dyn DirectoryPutter + 'static)>
|
||||
where
|
||||
Self: Clone,
|
||||
{
|
||||
Box::new(SimplePutter::new(self.clone()))
|
||||
}
|
||||
}
|
||||
|
|
@ -19,7 +19,8 @@ use super::{DirectoryService, GRPCDirectoryService, MemoryDirectoryService, Sled
|
|||
/// - `grpc+http://host:port`, `grpc+https://host:port`
|
||||
/// Connects to a (remote) tvix-store gRPC service.
|
||||
pub async fn from_addr(uri: &str) -> Result<Box<dyn DirectoryService>, crate::Error> {
|
||||
let url = Url::parse(uri)
|
||||
#[allow(unused_mut)]
|
||||
let mut url = Url::parse(uri)
|
||||
.map_err(|e| crate::Error::StorageError(format!("unable to parse url: {}", e)))?;
|
||||
|
||||
let directory_service: Box<dyn DirectoryService> = match url.scheme() {
|
||||
|
|
@ -62,6 +63,30 @@ pub async fn from_addr(uri: &str) -> Result<Box<dyn DirectoryService>, crate::Er
|
|||
let client = DirectoryServiceClient::new(crate::tonic::channel_from_url(&url).await?);
|
||||
Box::new(GRPCDirectoryService::from_client(client))
|
||||
}
|
||||
#[cfg(feature = "cloud")]
|
||||
"bigtable" => {
|
||||
use super::bigtable::BigtableParameters;
|
||||
use super::BigtableDirectoryService;
|
||||
|
||||
// parse the instance name from the hostname.
|
||||
let instance_name = url
|
||||
.host_str()
|
||||
.ok_or_else(|| Error::StorageError("instance name missing".into()))?
|
||||
.to_string();
|
||||
|
||||
// … but add it to the query string now, so we just need to parse that.
|
||||
url.query_pairs_mut()
|
||||
.append_pair("instance_name", &instance_name);
|
||||
|
||||
let params: BigtableParameters = serde_qs::from_str(url.query().unwrap_or_default())
|
||||
.map_err(|e| Error::InvalidRequest(format!("failed to parse parameters: {}", e)))?;
|
||||
|
||||
Box::new(
|
||||
BigtableDirectoryService::connect(params)
|
||||
.await
|
||||
.map_err(|e| Error::StorageError(e.to_string()))?,
|
||||
)
|
||||
}
|
||||
_ => {
|
||||
return Err(crate::Error::StorageError(format!(
|
||||
"unknown scheme: {}",
|
||||
|
|
@ -117,6 +142,27 @@ mod tests {
|
|||
#[case::grpc_valid_https_host_without_port("grpc+https://localhost", true)]
|
||||
/// Correct scheme to connect to localhost over http, but with additional path, which is invalid.
|
||||
#[case::grpc_invalid_host_and_path("grpc+http://localhost/some-path", false)]
|
||||
/// A valid example for Bigtable
|
||||
#[cfg_attr(
|
||||
feature = "cloud",
|
||||
case::bigtable_valid_url(
|
||||
"bigtable://instance-1?project_id=project-1&table_name=table-1&family_name=cf1",
|
||||
true
|
||||
)
|
||||
)]
|
||||
/// A valid example for Bigtable, specifying a custom channel size and timeout
|
||||
#[cfg_attr(
|
||||
feature = "cloud",
|
||||
case::bigtable_valid_url(
|
||||
"bigtable://instance-1?project_id=project-1&table_name=table-1&family_name=cf1&channel_size=10&timeout=10",
|
||||
true
|
||||
)
|
||||
)]
|
||||
/// A invalid Bigtable example (missing fields)
|
||||
#[cfg_attr(
|
||||
feature = "cloud",
|
||||
case::bigtable_invalid_url("bigtable://instance-1", false)
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn test_from_addr_tokio(#[case] uri_str: &str, #[case] exp_succeed: bool) {
|
||||
if exp_succeed {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,12 @@ pub use self::sled::SledDirectoryService;
|
|||
pub use self::traverse::descend_to;
|
||||
pub use self::utils::traverse_directory;
|
||||
|
||||
#[cfg(feature = "cloud")]
|
||||
mod bigtable;
|
||||
|
||||
#[cfg(feature = "cloud")]
|
||||
pub use self::bigtable::BigtableDirectoryService;
|
||||
|
||||
/// The base trait all Directory services need to implement.
|
||||
/// This is a simple get and put of [crate::proto::Directory], returning their
|
||||
/// digest.
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ use self::utils::make_grpc_directory_service_client;
|
|||
#[case::grpc(make_grpc_directory_service_client().await)]
|
||||
#[case::memory(directoryservice::from_addr("memory://").await.unwrap())]
|
||||
#[case::sled(directoryservice::from_addr("sled://").await.unwrap())]
|
||||
#[cfg_attr(feature = "cloud", case::bigtable(directoryservice::from_addr("bigtable://instance-1?project_id=project-1&table_name=table-1&family_name=cf1").await.unwrap()))]
|
||||
pub fn directory_services(#[case] directory_service: impl DirectoryService) {}
|
||||
|
||||
/// Ensures asking for a directory that doesn't exist returns a Ok(None).
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ use crate::{
|
|||
directoryservice::MemoryDirectoryService,
|
||||
proto::directory_service_server::DirectoryServiceServer,
|
||||
};
|
||||
|
||||
use tonic::transport::{Endpoint, Server, Uri};
|
||||
|
||||
/// Constructs and returns a gRPC DirectoryService.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
{
|
||||
"bigtable_rs 0.2.9 (git+https://github.com/flokli/bigtable_rs?rev=0af404741dfc40eb9fa99cf4d4140a09c5c20df7#0af404741dfc40eb9fa99cf4d4140a09c5c20df7)": "1njjam1lx2xlnm7a41lga8601vmjgqz0fvc77x24gd04pc7avxll",
|
||||
"test-generator 0.3.0 (git+https://github.com/JamesGuthrie/test-generator.git?rev=82e799979980962aec1aa324ec6e0e4cad781f41#82e799979980962aec1aa324ec6e0e4cad781f41)": "08brp3qqa55hijc7xby3lam2cc84hvx1zzfqv6lj7smlczh8k32y",
|
||||
"wu-manber 0.1.0 (git+https://github.com/tvlfyi/wu-manber.git#0d5b22bea136659f7de60b102a7030e0daaa503d)": "1zhk83lbq99xzyjwphv2qrb8f8qgfqwa5bbbvyzm0z0bljsjv0pd"
|
||||
}
|
||||
}
|
||||
|
|
@ -43,6 +43,10 @@ let
|
|||
nativeBuildInputs = protobufDep prev;
|
||||
};
|
||||
|
||||
prost-wkt-types = prev: {
|
||||
nativeBuildInputs = protobufDep prev;
|
||||
};
|
||||
|
||||
tonic-reflection = prev: {
|
||||
nativeBuildInputs = protobufDep prev;
|
||||
};
|
||||
|
|
@ -84,6 +88,7 @@ let
|
|||
(crateName:
|
||||
(lib.nameValuePair "${crateName}-${crates.internal.crates.${crateName}.version}" crates.internal.crates.${crateName}.src.outputHash)
|
||||
) [
|
||||
"bigtable_rs"
|
||||
"test-generator"
|
||||
"wu-manber"
|
||||
]);
|
||||
|
|
|
|||
|
|
@ -19,10 +19,12 @@ pkgs.mkShell {
|
|||
pkgs.cargo
|
||||
pkgs.cargo-machete
|
||||
pkgs.cargo-expand
|
||||
pkgs.cbtemulator
|
||||
pkgs.clippy
|
||||
pkgs.evans
|
||||
pkgs.fuse
|
||||
pkgs.go
|
||||
pkgs.google-cloud-bigtable-tool
|
||||
pkgs.grpcurl
|
||||
pkgs.hyperfine
|
||||
pkgs.mdbook
|
||||
|
|
|
|||
Loading…
Reference in a new issue