feat(tvix/nar-bridge): init
This adds an implementation of nar-bridge in Rust. Currently, only the GET parts are implemented. Contrary to the Go variant, this doesn't try to keep a mapping from nar hashes to root node in memory, it simply encodes the root node itself (stripped by its basename) into the URL. This pulls in a more recent version of axum than what we use in tonic, causing two versions of http and hyper, however dealing with `Body::from_stream` in axum 0.6 is much more annoying, and https://github.com/hyperium/tonic/pull/1740 suggests this will be fixed soon. Change-Id: Ia4c2dbda7cd3fdbe47a75f3e33544d19eac6e44e Reviewed-on: https://cl.tvl.fyi/c/depot/+/11898 Autosubmit: flokli <flokli@flokli.de> Reviewed-by: Brian Olsen <me@griff.name> Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink
clbot
parent
0244ae6eaf
commit
861cc1f341
9 changed files with 1460 additions and 87 deletions
268
tvix/Cargo.lock
generated
268
tvix/Cargo.lock
generated
|
|
@ -310,13 +310,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core",
|
||||
"axum-core 0.3.4",
|
||||
"bitflags 1.3.2",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"http 0.2.11",
|
||||
"http-body 0.4.6",
|
||||
"hyper 0.14.28",
|
||||
"itoa",
|
||||
"matchit",
|
||||
"memchr",
|
||||
|
|
@ -325,12 +325,46 @@ dependencies = [
|
|||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"serde",
|
||||
"sync_wrapper",
|
||||
"sync_wrapper 0.1.2",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core 0.4.3",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.1.0",
|
||||
"http-body 1.0.0",
|
||||
"http-body-util",
|
||||
"hyper 1.3.1",
|
||||
"hyper-util",
|
||||
"itoa",
|
||||
"matchit",
|
||||
"memchr",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_path_to_error",
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper 1.0.1",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.3.4"
|
||||
|
|
@ -340,14 +374,35 @@ dependencies = [
|
|||
"async-trait",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http 0.2.11",
|
||||
"http-body 0.4.6",
|
||||
"mime",
|
||||
"rustversion",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.1.0",
|
||||
"http-body 1.0.0",
|
||||
"http-body-util",
|
||||
"mime",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"sync_wrapper 0.1.2",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.69"
|
||||
|
|
@ -382,7 +437,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "12ccba0acd0a82afb2b1fe89181b0776ff18aef5e355030631fe534c8ae8ec73"
|
||||
dependencies = [
|
||||
"gcp_auth",
|
||||
"http",
|
||||
"http 0.2.11",
|
||||
"log",
|
||||
"prost",
|
||||
"prost-build",
|
||||
|
|
@ -1388,7 +1443,7 @@ dependencies = [
|
|||
"base64",
|
||||
"chrono",
|
||||
"home",
|
||||
"hyper",
|
||||
"hyper 0.14.28",
|
||||
"hyper-rustls",
|
||||
"ring",
|
||||
"rustls 0.21.12",
|
||||
|
|
@ -1478,7 +1533,26 @@ dependencies = [
|
|||
"futures-core",
|
||||
"futures-sink",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http 0.2.11",
|
||||
"indexmap 2.1.0",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"futures-util",
|
||||
"http 1.1.0",
|
||||
"indexmap 2.1.0",
|
||||
"slab",
|
||||
"tokio",
|
||||
|
|
@ -1552,6 +1626,17 @@ dependencies = [
|
|||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "0.4.6"
|
||||
|
|
@ -1559,7 +1644,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http",
|
||||
"http 0.2.11",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http 1.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body-util"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"http 1.1.0",
|
||||
"http-body 1.0.0",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
|
|
@ -1597,9 +1705,9 @@ dependencies = [
|
|||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"h2 0.3.26",
|
||||
"http 0.2.11",
|
||||
"http-body 0.4.6",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa",
|
||||
|
|
@ -1611,6 +1719,26 @@ dependencies = [
|
|||
"want",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "1.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"h2 0.4.4",
|
||||
"http 1.1.0",
|
||||
"http-body 1.0.0",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa",
|
||||
"pin-project-lite",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-rustls"
|
||||
version = "0.24.2"
|
||||
|
|
@ -1618,8 +1746,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"http",
|
||||
"hyper",
|
||||
"http 0.2.11",
|
||||
"hyper 0.14.28",
|
||||
"rustls 0.21.12",
|
||||
"rustls-native-certs 0.6.3",
|
||||
"tokio",
|
||||
|
|
@ -1632,12 +1760,28 @@ version = "0.4.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
|
||||
dependencies = [
|
||||
"hyper",
|
||||
"hyper 0.14.28",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tokio-io-timeout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-util"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.1.0",
|
||||
"http-body 1.0.0",
|
||||
"hyper 1.3.1",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.60"
|
||||
|
|
@ -2094,6 +2238,35 @@ version = "0.8.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
|
||||
|
||||
[[package]]
|
||||
name = "nar-bridge"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"axum 0.7.5",
|
||||
"bytes",
|
||||
"clap",
|
||||
"data-encoding",
|
||||
"hex-literal",
|
||||
"itertools 0.12.0",
|
||||
"nix-compat",
|
||||
"prost",
|
||||
"prost-build",
|
||||
"rstest",
|
||||
"serde",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-listener",
|
||||
"tokio-util",
|
||||
"tonic",
|
||||
"tonic-build",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"tvix-castore",
|
||||
"tvix-store",
|
||||
"tvix-tracing",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nibble_vec"
|
||||
version = "0.1.0"
|
||||
|
|
@ -2260,7 +2433,7 @@ dependencies = [
|
|||
"chrono",
|
||||
"futures",
|
||||
"humantime",
|
||||
"hyper",
|
||||
"hyper 0.14.28",
|
||||
"itertools 0.12.0",
|
||||
"md-5",
|
||||
"parking_lot 0.12.2",
|
||||
|
|
@ -2320,7 +2493,7 @@ checksum = "7690dc77bf776713848c4faa6501157469017eaf332baccd4eb1cea928743d94"
|
|||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"http",
|
||||
"http 0.2.11",
|
||||
"opentelemetry",
|
||||
]
|
||||
|
||||
|
|
@ -2332,7 +2505,7 @@ checksum = "1a016b8d9495c639af2145ac22387dcb88e44118e45320d9238fbf4e7889abcb"
|
|||
dependencies = [
|
||||
"async-trait",
|
||||
"futures-core",
|
||||
"http",
|
||||
"http 0.2.11",
|
||||
"opentelemetry",
|
||||
"opentelemetry-proto",
|
||||
"opentelemetry-semantic-conventions",
|
||||
|
|
@ -2981,10 +3154,10 @@ dependencies = [
|
|||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"h2 0.3.26",
|
||||
"http 0.2.11",
|
||||
"http-body 0.4.6",
|
||||
"hyper 0.14.28",
|
||||
"hyper-rustls",
|
||||
"ipnet",
|
||||
"js-sys",
|
||||
|
|
@ -3021,7 +3194,7 @@ checksum = "5a735987236a8e238bf0296c7e351b999c188ccc11477f311b82b55c93984216"
|
|||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"http",
|
||||
"http 0.2.11",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"task-local-extensions",
|
||||
|
|
@ -3372,18 +3545,18 @@ checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0"
|
|||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.197"
|
||||
version = "1.0.204"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
|
||||
checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.197"
|
||||
version = "1.0.204"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
|
||||
checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -3401,6 +3574,16 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_path_to_error"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_qs"
|
||||
version = "0.12.0"
|
||||
|
|
@ -3685,6 +3868,12 @@ version = "0.1.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
|
||||
|
||||
[[package]]
|
||||
name = "sync_wrapper"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
|
||||
|
||||
[[package]]
|
||||
name = "system-configuration"
|
||||
version = "0.5.1"
|
||||
|
|
@ -3913,16 +4102,21 @@ version = "0.4.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4134661e12ec11c6276be73544a43144a357b08dfab5c41fd226e15b5bc9a6b2"
|
||||
dependencies = [
|
||||
"axum 0.7.5",
|
||||
"clap",
|
||||
"document-features",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"hyper 1.3.1",
|
||||
"hyper-util",
|
||||
"nix 0.26.4",
|
||||
"pin-project",
|
||||
"socket2",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tonic",
|
||||
"tower",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
|
|
@ -4065,13 +4259,13 @@ checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13"
|
|||
dependencies = [
|
||||
"async-stream",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"axum 0.6.20",
|
||||
"base64",
|
||||
"bytes",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"h2 0.3.26",
|
||||
"http 0.2.11",
|
||||
"http-body 0.4.6",
|
||||
"hyper 0.14.28",
|
||||
"hyper-timeout",
|
||||
"percent-encoding",
|
||||
"pin-project",
|
||||
|
|
@ -4144,8 +4338,8 @@ dependencies = [
|
|||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http 0.2.11",
|
||||
"http-body 0.4.6",
|
||||
"http-range-header",
|
||||
"pin-project-lite",
|
||||
"tower-layer",
|
||||
|
|
@ -4571,7 +4765,7 @@ dependencies = [
|
|||
name = "tvix-tracing"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"http",
|
||||
"http 0.2.11",
|
||||
"indicatif",
|
||||
"lazy_static",
|
||||
"opentelemetry",
|
||||
|
|
|
|||
885
tvix/Cargo.nix
885
tvix/Cargo.nix
File diff suppressed because it is too large
Load diff
|
|
@ -25,6 +25,7 @@ members = [
|
|||
"eval",
|
||||
"eval/builtin-macros",
|
||||
"glue",
|
||||
"nar-bridge",
|
||||
"nix-compat",
|
||||
"serde",
|
||||
"store",
|
||||
|
|
|
|||
40
tvix/nar-bridge/Cargo.toml
Normal file
40
tvix/nar-bridge/Cargo.toml
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
[package]
|
||||
name = "nar-bridge"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
axum = { version = "0.7.5", features = ["http2"] }
|
||||
bytes = "1.4.0"
|
||||
clap = { version = "4.0", features = ["derive", "env"] }
|
||||
data-encoding = "2.3.3"
|
||||
itertools = "0.12.0"
|
||||
prost = "0.12.1"
|
||||
nix-compat = { path = "../nix-compat", features = ["async"] }
|
||||
thiserror = "1.0.56"
|
||||
tokio = { version = "1.32.0" }
|
||||
tokio-listener = { version = "0.4.2", features = [ "axum07", "clap", "multi-listener", "sd_listen" ] }
|
||||
tokio-util = { version = "0.7.9", features = ["io", "io-util", "compat"] }
|
||||
tonic = { version = "0.11.0", features = ["tls", "tls-roots"] }
|
||||
tvix-castore = { path = "../castore" }
|
||||
tvix-store = { path = "../store" }
|
||||
tvix-tracing = { path = "../tracing", features = ["tonic"] }
|
||||
tracing = "0.1.37"
|
||||
tracing-subscriber = "0.3.16"
|
||||
url = "2.4.0"
|
||||
serde = { version = "1.0.204", features = ["derive"] }
|
||||
|
||||
[build-dependencies]
|
||||
prost-build = "0.12.1"
|
||||
tonic-build = "0.11.0"
|
||||
|
||||
[features]
|
||||
default = ["otlp"]
|
||||
otlp = ["tvix-tracing/otlp"]
|
||||
|
||||
[dev-dependencies]
|
||||
hex-literal = "0.4.1"
|
||||
rstest = "0.19.0"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
11
tvix/nar-bridge/default.nix
Normal file
11
tvix/nar-bridge/default.nix
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
{ depot, lib, ... }:
|
||||
|
||||
(depot.tvix.crates.workspaceMembers.nar-bridge.build.override {
|
||||
runTests = true;
|
||||
}).overrideAttrs (old: rec {
|
||||
meta.ci.targets = lib.filter (x: lib.hasPrefix "with-features" x || x == "no-features") (lib.attrNames passthru);
|
||||
passthru = (depot.tvix.utils.mkFeaturePowerset {
|
||||
inherit (old) crateName;
|
||||
features = [ "otlp" ];
|
||||
});
|
||||
})
|
||||
84
tvix/nar-bridge/src/bin/nar-bridge.rs
Normal file
84
tvix/nar-bridge/src/bin/nar-bridge.rs
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
use clap::Parser;
|
||||
use nar_bridge::AppState;
|
||||
use tracing::info;
|
||||
|
||||
/// Expose the Nix HTTP Binary Cache protocol for a tvix-store.
|
||||
#[derive(Parser)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
struct Cli {
|
||||
#[arg(long, env, default_value = "grpc+http://[::1]:8000")]
|
||||
blob_service_addr: String,
|
||||
|
||||
#[arg(long, env, default_value = "grpc+http://[::1]:8000")]
|
||||
directory_service_addr: String,
|
||||
|
||||
#[arg(long, env, default_value = "grpc+http://[::1]:8000")]
|
||||
path_info_service_addr: String,
|
||||
|
||||
/// The priority to announce at the `nix-cache-info` endpoint.
|
||||
/// A lower number means it's *more preferred.
|
||||
#[arg(long, env, default_value_t = 39)]
|
||||
priority: u64,
|
||||
|
||||
/// The address to listen on.
|
||||
#[clap(flatten)]
|
||||
listen_args: tokio_listener::ListenerAddressLFlag,
|
||||
|
||||
#[cfg(feature = "otlp")]
|
||||
/// Whether to configure OTLP. Set --otlp=false to disable.
|
||||
#[arg(long, default_missing_value = "true", default_value = "true", num_args(0..=1), require_equals(true), action(clap::ArgAction::Set))]
|
||||
otlp: bool,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
let _tracing_handle = {
|
||||
#[allow(unused_mut)]
|
||||
let mut builder = tvix_tracing::TracingBuilder::default();
|
||||
#[cfg(feature = "otlp")]
|
||||
{
|
||||
if cli.otlp {
|
||||
builder = builder.enable_otlp("tvix.store");
|
||||
}
|
||||
}
|
||||
builder.build()?
|
||||
};
|
||||
|
||||
// initialize stores
|
||||
let (blob_service, directory_service, path_info_service, _nar_calculation_service) =
|
||||
tvix_store::utils::construct_services(
|
||||
cli.blob_service_addr,
|
||||
cli.directory_service_addr,
|
||||
cli.path_info_service_addr,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let state = AppState::new(blob_service, directory_service, path_info_service.into());
|
||||
|
||||
let app = nar_bridge::gen_router(cli.priority).with_state(state);
|
||||
|
||||
let listen_address = &cli.listen_args.listen_address.unwrap_or_else(|| {
|
||||
"[::]:8000"
|
||||
.parse()
|
||||
.expect("invalid fallback listen address")
|
||||
});
|
||||
|
||||
let listener = tokio_listener::Listener::bind(
|
||||
listen_address,
|
||||
&Default::default(),
|
||||
&cli.listen_args.listener_options,
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!(listen_address=%listen_address, "starting daemon");
|
||||
|
||||
tokio_listener::axum07::serve(
|
||||
listener,
|
||||
app.into_make_service_with_connect_info::<tokio_listener::SomeSocketAddrClonable>(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
50
tvix/nar-bridge/src/lib.rs
Normal file
50
tvix/nar-bridge/src/lib.rs
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
use axum::routing::head;
|
||||
use axum::{routing::get, Router};
|
||||
use std::sync::Arc;
|
||||
use tvix_castore::blobservice::BlobService;
|
||||
use tvix_castore::directoryservice::DirectoryService;
|
||||
use tvix_store::pathinfoservice::PathInfoService;
|
||||
|
||||
mod nar;
|
||||
mod narinfo;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
blob_service: Arc<dyn BlobService>,
|
||||
directory_service: Arc<dyn DirectoryService>,
|
||||
path_info_service: Arc<dyn PathInfoService>,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub fn new(
|
||||
blob_service: Arc<dyn BlobService>,
|
||||
directory_service: Arc<dyn DirectoryService>,
|
||||
path_info_service: Arc<dyn PathInfoService>,
|
||||
) -> Self {
|
||||
Self {
|
||||
blob_service,
|
||||
directory_service,
|
||||
path_info_service,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn gen_router(priority: u64) -> Router<AppState> {
|
||||
Router::new()
|
||||
.route("/", get(root))
|
||||
.route("/nar/tvix-castore/:root_node_enc", get(nar::get))
|
||||
.route("/:narinfo_str", get(narinfo::get))
|
||||
.route("/:narinfo_str", head(narinfo::head))
|
||||
.route("/nix-cache-info", get(move || nix_cache_info(priority)))
|
||||
}
|
||||
|
||||
async fn root() -> &'static str {
|
||||
"Hello from nar-bridge"
|
||||
}
|
||||
|
||||
async fn nix_cache_info(priority: u64) -> String {
|
||||
format!(
|
||||
"StoreDir: /nix/store\nWantMassQuery: 1\nPriority: {}\n",
|
||||
priority
|
||||
)
|
||||
}
|
||||
77
tvix/nar-bridge/src/nar.rs
Normal file
77
tvix/nar-bridge/src/nar.rs
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
use axum::body::Body;
|
||||
use axum::extract::Query;
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::Response;
|
||||
use bytes::Bytes;
|
||||
use data_encoding::BASE64URL_NOPAD;
|
||||
use serde::Deserialize;
|
||||
use tokio_util::io::ReaderStream;
|
||||
use tracing::{instrument, warn};
|
||||
|
||||
use crate::AppState;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct GetNARParams {
|
||||
#[serde(rename = "narsize")]
|
||||
nar_size: u64,
|
||||
}
|
||||
|
||||
#[instrument(skip(blob_service, directory_service))]
|
||||
pub async fn get(
|
||||
axum::extract::Path(root_node_enc): axum::extract::Path<String>,
|
||||
axum::extract::Query(GetNARParams { nar_size }): Query<GetNARParams>,
|
||||
axum::extract::State(AppState {
|
||||
blob_service,
|
||||
directory_service,
|
||||
..
|
||||
}): axum::extract::State<AppState>,
|
||||
) -> Result<Response, StatusCode> {
|
||||
use prost::Message;
|
||||
// b64decode the root node passed *by the user*
|
||||
let root_node_proto = BASE64URL_NOPAD
|
||||
.decode(root_node_enc.as_bytes())
|
||||
.map_err(|e| {
|
||||
warn!(err=%e, "unable to decode root node b64");
|
||||
StatusCode::NOT_FOUND
|
||||
})?;
|
||||
|
||||
// check the proto size to be somewhat reasonable before parsing it.
|
||||
if root_node_proto.len() > 4096 {
|
||||
warn!("rejected too large root node");
|
||||
return Err(StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
// parse the proto
|
||||
let root_node: tvix_castore::proto::Node = Message::decode(Bytes::from(root_node_enc))
|
||||
.map_err(|e| {
|
||||
warn!(err=%e, "unable to decode root node proto");
|
||||
StatusCode::NOT_FOUND
|
||||
})?;
|
||||
|
||||
// validate it.
|
||||
let root_node = root_node
|
||||
.validate()
|
||||
.map_err(|e| {
|
||||
warn!(err=%e, "root node validation failed");
|
||||
StatusCode::BAD_REQUEST
|
||||
})?
|
||||
.to_owned();
|
||||
|
||||
let (w, r) = tokio::io::duplex(1024 * 8);
|
||||
|
||||
// spawn a task rendering the NAR to the client
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) =
|
||||
tvix_store::nar::write_nar(w, &root_node, blob_service, directory_service).await
|
||||
{
|
||||
warn!(err=%e, "failed to write out NAR");
|
||||
}
|
||||
});
|
||||
|
||||
Ok(Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("cache-control", "max-age=31536000, immutable")
|
||||
.header("content-length", nar_size)
|
||||
.body(Body::from_stream(ReaderStream::new(r)))
|
||||
.unwrap())
|
||||
}
|
||||
131
tvix/nar-bridge/src/narinfo.rs
Normal file
131
tvix/nar-bridge/src/narinfo.rs
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
use axum::http::StatusCode;
|
||||
use nix_compat::nixbase32;
|
||||
use tracing::{instrument, warn, Span};
|
||||
use tvix_castore::proto::node::Node;
|
||||
|
||||
use crate::AppState;
|
||||
|
||||
#[instrument(skip(path_info_service))]
|
||||
pub async fn head(
|
||||
axum::extract::Path(narinfo_str): axum::extract::Path<String>,
|
||||
axum::extract::State(AppState {
|
||||
path_info_service, ..
|
||||
}): axum::extract::State<AppState>,
|
||||
) -> Result<&'static str, StatusCode> {
|
||||
let digest = parse_narinfo_str(&narinfo_str)?;
|
||||
Span::current().record("path_info.digest", &narinfo_str[0..32]);
|
||||
|
||||
if path_info_service
|
||||
.get(digest)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!(err=%e, "failed to get PathInfo");
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?
|
||||
.is_some()
|
||||
{
|
||||
Ok("")
|
||||
} else {
|
||||
warn!("PathInfo not found");
|
||||
Err(StatusCode::NOT_FOUND)
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip(path_info_service))]
|
||||
pub async fn get(
|
||||
axum::extract::Path(narinfo_str): axum::extract::Path<String>,
|
||||
axum::extract::State(AppState {
|
||||
path_info_service, ..
|
||||
}): axum::extract::State<AppState>,
|
||||
) -> Result<String, StatusCode> {
|
||||
let digest = parse_narinfo_str(&narinfo_str)?;
|
||||
Span::current().record("path_info.digest", &narinfo_str[0..32]);
|
||||
|
||||
// fetch the PathInfo
|
||||
let path_info = path_info_service
|
||||
.get(digest)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!(err=%e, "failed to get PathInfo");
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?
|
||||
.ok_or(StatusCode::NOT_FOUND)?;
|
||||
|
||||
let store_path = path_info.validate().map_err(|e| {
|
||||
warn!(err=%e, "invalid PathInfo");
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
|
||||
let mut narinfo = path_info.to_narinfo(store_path).ok_or_else(|| {
|
||||
warn!(path_info=?path_info, "PathInfo contained no NAR data");
|
||||
StatusCode::INTERNAL_SERVER_ERROR
|
||||
})?;
|
||||
|
||||
// encode the (unnamed) root node in the NAR url itself.
|
||||
let root_node = path_info
|
||||
.node
|
||||
.as_ref()
|
||||
.and_then(|n| n.node.as_ref())
|
||||
.expect("root node must not be none")
|
||||
.clone()
|
||||
.rename("".into());
|
||||
|
||||
let mut buf = Vec::new();
|
||||
Node::encode(&root_node, &mut buf);
|
||||
|
||||
let url = format!(
|
||||
"nar/tvix-castore/{}?narsize={}",
|
||||
data_encoding::BASE64URL_NOPAD.encode(&buf),
|
||||
narinfo.nar_size,
|
||||
);
|
||||
|
||||
narinfo.url = &url;
|
||||
|
||||
Ok(narinfo.to_string())
|
||||
}
|
||||
|
||||
/// Parses a `3mzh8lvgbynm9daj7c82k2sfsfhrsfsy.narinfo` string and returns the
|
||||
/// nixbase32-decoded digest.
|
||||
fn parse_narinfo_str(s: &str) -> Result<[u8; 20], StatusCode> {
|
||||
if !s.is_char_boundary(32) {
|
||||
warn!("invalid string, no char boundary at 32");
|
||||
return Err(StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
Ok(match s.split_at(32) {
|
||||
(hash_str, ".narinfo") => {
|
||||
// we know this is 32 bytes
|
||||
let hash_str_fixed: [u8; 32] = hash_str.as_bytes().try_into().unwrap();
|
||||
nixbase32::decode_fixed(hash_str_fixed).map_err(|e| {
|
||||
warn!(err=%e, "invalid digest");
|
||||
StatusCode::NOT_FOUND
|
||||
})?
|
||||
}
|
||||
_ => {
|
||||
warn!("invalid string");
|
||||
return Err(StatusCode::NOT_FOUND);
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::parse_narinfo_str;
|
||||
use hex_literal::hex;
|
||||
|
||||
#[test]
|
||||
fn success() {
|
||||
assert_eq!(
|
||||
hex!("8a12321522fd91efbd60ebb2481af88580f61600"),
|
||||
parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44la.narinfo").unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure() {
|
||||
assert!(parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44la").is_err());
|
||||
assert!(parse_narinfo_str("/00bgd045z0d4icpbc2yyz4gx48ak44la").is_err());
|
||||
assert!(parse_narinfo_str("000000").is_err());
|
||||
assert!(parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44l🦊.narinfo").is_err());
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue