feat(tvix/store/pathinfoservice): implement NixHTTPPathInfoService
NixHTTPPathInfoService acts as a bridge in between the Nix HTTP Binary cache protocol provided by Nix binary caches such as cache.nixos.org, and the Tvix Store Model. It implements the [PathInfoService] trait in an interesting way: Every [PathInfoService::get] fetches the .narinfo and referred NAR file, inserting components into a [BlobService] and [DirectoryService], then returning a [PathInfo] struct with the root. Due to this being quite a costly operation, clients are expected to layer this service with store composition, so they're only ingested once. The client is expected to be (indirectly) using the same [BlobService] and [DirectoryService], so able to fetch referred Directories and Blobs. [PathInfoService::put] and [PathInfoService::nar] are not implemented and return an error if called. This behaves very similar to the nar-bridge-pathinfo code in nar-bridge, except it's now in Rust. Change-Id: Ia03d4fed9d0657965d100299af97cd917a03f2f0 Reviewed-on: https://cl.tvl.fyi/c/depot/+/10069 Tested-by: BuildkiteCI Autosubmit: flokli <flokli@flokli.de> Reviewed-by: raitobezarius <tvl@lahfa.xyz>
This commit is contained in:
Florian Klink
flokli
parent
4e9e4b19ef
commit
be48ba75ab
6 changed files with 1043 additions and 10 deletions
173
tvix/Cargo.lock
generated
173
tvix/Cargo.lock
generated
|
|
@ -693,6 +693,15 @@ version = "1.8.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "endian-type"
|
||||
version = "0.1.2"
|
||||
|
|
@ -1064,6 +1073,20 @@ dependencies = [
|
|||
"want",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-rustls"
|
||||
version = "0.24.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"http",
|
||||
"hyper",
|
||||
"rustls",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-timeout"
|
||||
version = "0.4.1"
|
||||
|
|
@ -1140,6 +1163,12 @@ dependencies = [
|
|||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
|
||||
|
||||
[[package]]
|
||||
name = "is-terminal"
|
||||
version = "0.4.7"
|
||||
|
|
@ -1298,6 +1327,17 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lzma-sys"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.7.0"
|
||||
|
|
@ -1633,6 +1673,12 @@ version = "0.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
|
||||
|
||||
[[package]]
|
||||
name = "plotters"
|
||||
version = "0.3.4"
|
||||
|
|
@ -1959,6 +2005,48 @@ version = "0.7.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.11.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"ipnet",
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustls",
|
||||
"rustls-pemfile",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"system-configuration",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"wasm-streams",
|
||||
"web-sys",
|
||||
"webpki-roots",
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.16.20"
|
||||
|
|
@ -2209,6 +2297,18 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_urlencoded"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.6"
|
||||
|
|
@ -2391,6 +2491,27 @@ version = "0.1.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
|
||||
|
||||
[[package]]
|
||||
name = "system-configuration"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"core-foundation",
|
||||
"system-configuration-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system-configuration-sys"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tabwriter"
|
||||
version = "1.2.1"
|
||||
|
|
@ -2977,6 +3098,7 @@ dependencies = [
|
|||
"pin-project-lite",
|
||||
"prost",
|
||||
"prost-build",
|
||||
"reqwest",
|
||||
"sha2",
|
||||
"sled",
|
||||
"tempfile",
|
||||
|
|
@ -3002,6 +3124,7 @@ dependencies = [
|
|||
"vm-memory",
|
||||
"vmm-sys-util",
|
||||
"walkdir",
|
||||
"xz2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3222,6 +3345,18 @@ dependencies = [
|
|||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.84"
|
||||
|
|
@ -3251,6 +3386,19 @@ version = "0.2.84"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-streams"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.61"
|
||||
|
|
@ -3261,6 +3409,12 @@ dependencies = [
|
|||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.25.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc"
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "4.4.0"
|
||||
|
|
@ -3435,6 +3589,16 @@ version = "0.48.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
version = "0.50.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wu-manber"
|
||||
version = "0.1.0"
|
||||
|
|
@ -3446,6 +3610,15 @@ version = "0.8.18"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bab77e97b50aee93da431f2cee7cd0f43b4d1da3c408042f2d7d164187774f0a"
|
||||
|
||||
[[package]]
|
||||
name = "xz2"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
|
||||
dependencies = [
|
||||
"lzma-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yansi"
|
||||
version = "0.5.1"
|
||||
|
|
|
|||
631
tvix/Cargo.nix
631
tvix/Cargo.nix
|
|
@ -2015,6 +2015,29 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "default" "use_std" ];
|
||||
};
|
||||
"encoding_rs" = rec {
|
||||
crateName = "encoding_rs";
|
||||
version = "0.8.33";
|
||||
edition = "2018";
|
||||
sha256 = "1qa5k4a0ipdrxq4xg9amms9r9pnnfn7nfh2i9m3mw0ka563b6s3j";
|
||||
authors = [
|
||||
"Henri Sivonen <hsivonen@hsivonen.fi>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "cfg-if";
|
||||
packageId = "cfg-if";
|
||||
}
|
||||
];
|
||||
features = {
|
||||
"default" = [ "alloc" ];
|
||||
"fast-legacy-encode" = [ "fast-hangul-encode" "fast-hanja-encode" "fast-kanji-encode" "fast-gb-hanzi-encode" "fast-big5-hanzi-encode" ];
|
||||
"packed_simd" = [ "dep:packed_simd" ];
|
||||
"serde" = [ "dep:serde" ];
|
||||
"simd-accel" = [ "packed_simd" "packed_simd/into_bits" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "alloc" "default" ];
|
||||
};
|
||||
"endian-type" = rec {
|
||||
crateName = "endian-type";
|
||||
version = "0.1.2";
|
||||
|
|
@ -3171,6 +3194,75 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "client" "default" "full" "h2" "http1" "http2" "runtime" "server" "socket2" "stream" "tcp" ];
|
||||
};
|
||||
"hyper-rustls" = rec {
|
||||
crateName = "hyper-rustls";
|
||||
version = "0.24.2";
|
||||
edition = "2021";
|
||||
sha256 = "1475j4a2nczz4aajzzsq3hpwg1zacmzbqg393a14j80ff8izsgpc";
|
||||
dependencies = [
|
||||
{
|
||||
name = "futures-util";
|
||||
packageId = "futures-util";
|
||||
usesDefaultFeatures = false;
|
||||
}
|
||||
{
|
||||
name = "http";
|
||||
packageId = "http";
|
||||
}
|
||||
{
|
||||
name = "hyper";
|
||||
packageId = "hyper";
|
||||
usesDefaultFeatures = false;
|
||||
features = [ "client" ];
|
||||
}
|
||||
{
|
||||
name = "rustls";
|
||||
packageId = "rustls";
|
||||
usesDefaultFeatures = false;
|
||||
}
|
||||
{
|
||||
name = "tokio";
|
||||
packageId = "tokio";
|
||||
}
|
||||
{
|
||||
name = "tokio-rustls";
|
||||
packageId = "tokio-rustls";
|
||||
usesDefaultFeatures = false;
|
||||
}
|
||||
];
|
||||
devDependencies = [
|
||||
{
|
||||
name = "hyper";
|
||||
packageId = "hyper";
|
||||
features = [ "full" ];
|
||||
}
|
||||
{
|
||||
name = "rustls";
|
||||
packageId = "rustls";
|
||||
usesDefaultFeatures = false;
|
||||
features = [ "tls12" ];
|
||||
}
|
||||
{
|
||||
name = "tokio";
|
||||
packageId = "tokio";
|
||||
features = [ "io-std" "macros" "net" "rt-multi-thread" ];
|
||||
}
|
||||
];
|
||||
features = {
|
||||
"acceptor" = [ "hyper/server" "tokio-runtime" ];
|
||||
"default" = [ "native-tokio" "http1" "tls12" "logging" "acceptor" ];
|
||||
"http1" = [ "hyper/http1" ];
|
||||
"http2" = [ "hyper/http2" ];
|
||||
"log" = [ "dep:log" ];
|
||||
"logging" = [ "log" "tokio-rustls/logging" "rustls/logging" ];
|
||||
"native-tokio" = [ "tokio-runtime" "rustls-native-certs" ];
|
||||
"rustls-native-certs" = [ "dep:rustls-native-certs" ];
|
||||
"tls12" = [ "tokio-rustls/tls12" "rustls/tls12" ];
|
||||
"tokio-runtime" = [ "hyper/runtime" ];
|
||||
"webpki-roots" = [ "dep:webpki-roots" ];
|
||||
"webpki-tokio" = [ "tokio-runtime" "webpki-roots" ];
|
||||
};
|
||||
};
|
||||
"hyper-timeout" = rec {
|
||||
crateName = "hyper-timeout";
|
||||
version = "0.4.1";
|
||||
|
|
@ -3422,6 +3514,24 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "close" "default" "hermit-abi" "libc" "windows-sys" ];
|
||||
};
|
||||
"ipnet" = rec {
|
||||
crateName = "ipnet";
|
||||
version = "2.9.0";
|
||||
edition = "2018";
|
||||
sha256 = "1hzrcysgwf0knf83ahb3535hrkw63mil88iqc6kjaryfblrqylcg";
|
||||
authors = [
|
||||
"Kris Price <kris@krisprice.nz>"
|
||||
];
|
||||
features = {
|
||||
"default" = [ "std" ];
|
||||
"heapless" = [ "dep:heapless" ];
|
||||
"json" = [ "serde" "schemars" ];
|
||||
"schemars" = [ "dep:schemars" ];
|
||||
"ser_as_str" = [ "heapless" ];
|
||||
"serde" = [ "dep:serde" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "default" "std" ];
|
||||
};
|
||||
"is-terminal" = rec {
|
||||
crateName = "is-terminal";
|
||||
version = "0.4.7";
|
||||
|
|
@ -3875,6 +3985,32 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "std" ];
|
||||
};
|
||||
"lzma-sys" = rec {
|
||||
crateName = "lzma-sys";
|
||||
version = "0.1.20";
|
||||
edition = "2018";
|
||||
sha256 = "09sxp20waxyglgn3cjz8qjkspb3ryz2fwx4rigkwvrk46ymh9njz";
|
||||
authors = [
|
||||
"Alex Crichton <alex@alexcrichton.com>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "libc";
|
||||
packageId = "libc";
|
||||
}
|
||||
];
|
||||
buildDependencies = [
|
||||
{
|
||||
name = "cc";
|
||||
packageId = "cc";
|
||||
}
|
||||
{
|
||||
name = "pkg-config";
|
||||
packageId = "pkg-config";
|
||||
}
|
||||
];
|
||||
features = { };
|
||||
};
|
||||
"matchit" = rec {
|
||||
crateName = "matchit";
|
||||
version = "0.7.0";
|
||||
|
|
@ -4814,6 +4950,16 @@ rec {
|
|||
"Josef Brandl <mail@josefbrandl.de>"
|
||||
];
|
||||
|
||||
};
|
||||
"pkg-config" = rec {
|
||||
crateName = "pkg-config";
|
||||
version = "0.3.27";
|
||||
edition = "2015";
|
||||
sha256 = "0r39ryh1magcq4cz5g9x88jllsnxnhcqr753islvyk4jp9h2h1r6";
|
||||
authors = [
|
||||
"Alex Crichton <alex@alexcrichton.com>"
|
||||
];
|
||||
|
||||
};
|
||||
"plotters" = rec {
|
||||
crateName = "plotters";
|
||||
|
|
@ -5776,6 +5922,275 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "default" "std" "unicode" "unicode-age" "unicode-bool" "unicode-case" "unicode-gencat" "unicode-perl" "unicode-script" "unicode-segment" ];
|
||||
};
|
||||
"reqwest" = rec {
|
||||
crateName = "reqwest";
|
||||
version = "0.11.22";
|
||||
edition = "2018";
|
||||
sha256 = "0nx9mczsf11pcjicfpwad0l8drf2nn72dbpcvp42lv644s4djv04";
|
||||
authors = [
|
||||
"Sean McArthur <sean@seanmonstar.com>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "base64";
|
||||
packageId = "base64";
|
||||
}
|
||||
{
|
||||
name = "bytes";
|
||||
packageId = "bytes";
|
||||
}
|
||||
{
|
||||
name = "encoding_rs";
|
||||
packageId = "encoding_rs";
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "futures-core";
|
||||
packageId = "futures-core";
|
||||
usesDefaultFeatures = false;
|
||||
}
|
||||
{
|
||||
name = "futures-util";
|
||||
packageId = "futures-util";
|
||||
usesDefaultFeatures = false;
|
||||
}
|
||||
{
|
||||
name = "h2";
|
||||
packageId = "h2";
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "http";
|
||||
packageId = "http";
|
||||
}
|
||||
{
|
||||
name = "http-body";
|
||||
packageId = "http-body";
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "hyper";
|
||||
packageId = "hyper";
|
||||
usesDefaultFeatures = false;
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
features = [ "tcp" "http1" "http2" "client" "runtime" ];
|
||||
}
|
||||
{
|
||||
name = "hyper-rustls";
|
||||
packageId = "hyper-rustls";
|
||||
optional = true;
|
||||
usesDefaultFeatures = false;
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "ipnet";
|
||||
packageId = "ipnet";
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "js-sys";
|
||||
packageId = "js-sys";
|
||||
target = { target, features }: ("wasm32" == target."arch");
|
||||
}
|
||||
{
|
||||
name = "log";
|
||||
packageId = "log";
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "mime";
|
||||
packageId = "mime";
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "once_cell";
|
||||
packageId = "once_cell";
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "percent-encoding";
|
||||
packageId = "percent-encoding";
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "pin-project-lite";
|
||||
packageId = "pin-project-lite";
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "rustls";
|
||||
packageId = "rustls";
|
||||
optional = true;
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
features = [ "dangerous_configuration" ];
|
||||
}
|
||||
{
|
||||
name = "rustls-pemfile";
|
||||
packageId = "rustls-pemfile";
|
||||
optional = true;
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "serde";
|
||||
packageId = "serde";
|
||||
}
|
||||
{
|
||||
name = "serde_json";
|
||||
packageId = "serde_json";
|
||||
optional = true;
|
||||
}
|
||||
{
|
||||
name = "serde_json";
|
||||
packageId = "serde_json";
|
||||
target = { target, features }: ("wasm32" == target."arch");
|
||||
}
|
||||
{
|
||||
name = "serde_urlencoded";
|
||||
packageId = "serde_urlencoded";
|
||||
}
|
||||
{
|
||||
name = "system-configuration";
|
||||
packageId = "system-configuration";
|
||||
target = { target, features }: ("macos" == target."os");
|
||||
}
|
||||
{
|
||||
name = "tokio";
|
||||
packageId = "tokio";
|
||||
usesDefaultFeatures = false;
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
features = [ "net" "time" ];
|
||||
}
|
||||
{
|
||||
name = "tokio-rustls";
|
||||
packageId = "tokio-rustls";
|
||||
optional = true;
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "tokio-util";
|
||||
packageId = "tokio-util";
|
||||
optional = true;
|
||||
usesDefaultFeatures = false;
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
features = [ "codec" "io" ];
|
||||
}
|
||||
{
|
||||
name = "tower-service";
|
||||
packageId = "tower-service";
|
||||
}
|
||||
{
|
||||
name = "url";
|
||||
packageId = "url";
|
||||
}
|
||||
{
|
||||
name = "wasm-bindgen";
|
||||
packageId = "wasm-bindgen";
|
||||
target = { target, features }: ("wasm32" == target."arch");
|
||||
}
|
||||
{
|
||||
name = "wasm-bindgen-futures";
|
||||
packageId = "wasm-bindgen-futures";
|
||||
target = { target, features }: ("wasm32" == target."arch");
|
||||
}
|
||||
{
|
||||
name = "wasm-streams";
|
||||
packageId = "wasm-streams";
|
||||
optional = true;
|
||||
target = { target, features }: ("wasm32" == target."arch");
|
||||
}
|
||||
{
|
||||
name = "web-sys";
|
||||
packageId = "web-sys";
|
||||
target = { target, features }: ("wasm32" == target."arch");
|
||||
features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" ];
|
||||
}
|
||||
{
|
||||
name = "webpki-roots";
|
||||
packageId = "webpki-roots";
|
||||
optional = true;
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
}
|
||||
{
|
||||
name = "winreg";
|
||||
packageId = "winreg";
|
||||
target = { target, features }: (target."windows" or false);
|
||||
}
|
||||
];
|
||||
devDependencies = [
|
||||
{
|
||||
name = "hyper";
|
||||
packageId = "hyper";
|
||||
usesDefaultFeatures = false;
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
features = [ "tcp" "stream" "http1" "http2" "client" "server" "runtime" ];
|
||||
}
|
||||
{
|
||||
name = "serde";
|
||||
packageId = "serde";
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
features = [ "derive" ];
|
||||
}
|
||||
{
|
||||
name = "tokio";
|
||||
packageId = "tokio";
|
||||
usesDefaultFeatures = false;
|
||||
target = { target, features }: (!("wasm32" == target."arch"));
|
||||
features = [ "macros" "rt-multi-thread" ];
|
||||
}
|
||||
{
|
||||
name = "wasm-bindgen";
|
||||
packageId = "wasm-bindgen";
|
||||
target = { target, features }: ("wasm32" == target."arch");
|
||||
features = [ "serde-serialize" ];
|
||||
}
|
||||
];
|
||||
features = {
|
||||
"__rustls" = [ "hyper-rustls" "tokio-rustls" "rustls" "__tls" "rustls-pemfile" ];
|
||||
"async-compression" = [ "dep:async-compression" ];
|
||||
"blocking" = [ "futures-util/io" "tokio/rt-multi-thread" "tokio/sync" ];
|
||||
"brotli" = [ "async-compression" "async-compression/brotli" "tokio-util" ];
|
||||
"cookie_crate" = [ "dep:cookie_crate" ];
|
||||
"cookie_store" = [ "dep:cookie_store" ];
|
||||
"cookies" = [ "cookie_crate" "cookie_store" ];
|
||||
"default" = [ "default-tls" ];
|
||||
"default-tls" = [ "hyper-tls" "native-tls-crate" "__tls" "tokio-native-tls" ];
|
||||
"deflate" = [ "async-compression" "async-compression/zlib" "tokio-util" ];
|
||||
"futures-channel" = [ "dep:futures-channel" ];
|
||||
"gzip" = [ "async-compression" "async-compression/gzip" "tokio-util" ];
|
||||
"h3" = [ "dep:h3" ];
|
||||
"h3-quinn" = [ "dep:h3-quinn" ];
|
||||
"http3" = [ "rustls-tls-manual-roots" "h3" "h3-quinn" "quinn" "futures-channel" ];
|
||||
"hyper-rustls" = [ "dep:hyper-rustls" ];
|
||||
"hyper-tls" = [ "dep:hyper-tls" ];
|
||||
"json" = [ "serde_json" ];
|
||||
"mime_guess" = [ "dep:mime_guess" ];
|
||||
"multipart" = [ "mime_guess" ];
|
||||
"native-tls" = [ "default-tls" ];
|
||||
"native-tls-alpn" = [ "native-tls" "native-tls-crate/alpn" ];
|
||||
"native-tls-crate" = [ "dep:native-tls-crate" ];
|
||||
"native-tls-vendored" = [ "native-tls" "native-tls-crate/vendored" ];
|
||||
"quinn" = [ "dep:quinn" ];
|
||||
"rustls" = [ "dep:rustls" ];
|
||||
"rustls-native-certs" = [ "dep:rustls-native-certs" ];
|
||||
"rustls-pemfile" = [ "dep:rustls-pemfile" ];
|
||||
"rustls-tls" = [ "rustls-tls-webpki-roots" ];
|
||||
"rustls-tls-manual-roots" = [ "__rustls" ];
|
||||
"rustls-tls-native-roots" = [ "rustls-native-certs" "__rustls" ];
|
||||
"rustls-tls-webpki-roots" = [ "webpki-roots" "__rustls" ];
|
||||
"serde_json" = [ "dep:serde_json" ];
|
||||
"socks" = [ "tokio-socks" ];
|
||||
"stream" = [ "tokio/fs" "tokio-util" "wasm-streams" ];
|
||||
"tokio-native-tls" = [ "dep:tokio-native-tls" ];
|
||||
"tokio-rustls" = [ "dep:tokio-rustls" ];
|
||||
"tokio-socks" = [ "dep:tokio-socks" ];
|
||||
"tokio-util" = [ "dep:tokio-util" ];
|
||||
"trust-dns" = [ "trust-dns-resolver" ];
|
||||
"trust-dns-resolver" = [ "dep:trust-dns-resolver" ];
|
||||
"wasm-streams" = [ "dep:wasm-streams" ];
|
||||
"webpki-roots" = [ "dep:webpki-roots" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "__rustls" "__tls" "hyper-rustls" "rustls" "rustls-pemfile" "rustls-tls" "rustls-tls-webpki-roots" "stream" "tokio-rustls" "tokio-util" "wasm-streams" "webpki-roots" ];
|
||||
};
|
||||
"ring" = rec {
|
||||
crateName = "ring";
|
||||
version = "0.16.20";
|
||||
|
|
@ -6094,7 +6509,7 @@ rec {
|
|||
"read_buf" = [ "rustversion" ];
|
||||
"rustversion" = [ "dep:rustversion" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "default" "log" "logging" "tls12" ];
|
||||
resolvedDefaultFeatures = [ "dangerous_configuration" "default" "log" "logging" "tls12" ];
|
||||
};
|
||||
"rustls-native-certs" = rec {
|
||||
crateName = "rustls-native-certs";
|
||||
|
|
@ -6577,6 +6992,34 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "serde" ];
|
||||
};
|
||||
"serde_urlencoded" = rec {
|
||||
crateName = "serde_urlencoded";
|
||||
version = "0.7.1";
|
||||
edition = "2018";
|
||||
sha256 = "1zgklbdaysj3230xivihs30qi5vkhigg323a9m62k8jwf4a1qjfk";
|
||||
authors = [
|
||||
"Anthony Ramine <n.oxyde@gmail.com>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "form_urlencoded";
|
||||
packageId = "form_urlencoded";
|
||||
}
|
||||
{
|
||||
name = "itoa";
|
||||
packageId = "itoa";
|
||||
}
|
||||
{
|
||||
name = "ryu";
|
||||
packageId = "ryu";
|
||||
}
|
||||
{
|
||||
name = "serde";
|
||||
packageId = "serde";
|
||||
}
|
||||
];
|
||||
|
||||
};
|
||||
"sha2" = rec {
|
||||
crateName = "sha2";
|
||||
version = "0.10.6";
|
||||
|
|
@ -7072,6 +7515,50 @@ rec {
|
|||
"futures-core" = [ "dep:futures-core" ];
|
||||
};
|
||||
};
|
||||
"system-configuration" = rec {
|
||||
crateName = "system-configuration";
|
||||
version = "0.5.1";
|
||||
edition = "2021";
|
||||
sha256 = "1rz0r30xn7fiyqay2dvzfy56cvaa3km74hnbz2d72p97bkf3lfms";
|
||||
authors = [
|
||||
"Mullvad VPN"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "bitflags";
|
||||
packageId = "bitflags 1.3.2";
|
||||
}
|
||||
{
|
||||
name = "core-foundation";
|
||||
packageId = "core-foundation";
|
||||
}
|
||||
{
|
||||
name = "system-configuration-sys";
|
||||
packageId = "system-configuration-sys";
|
||||
}
|
||||
];
|
||||
|
||||
};
|
||||
"system-configuration-sys" = rec {
|
||||
crateName = "system-configuration-sys";
|
||||
version = "0.5.0";
|
||||
edition = "2021";
|
||||
sha256 = "1jckxvdr37bay3i9v52izgy52dg690x5xfg3hd394sv2xf4b2px7";
|
||||
authors = [
|
||||
"Mullvad VPN"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "core-foundation-sys";
|
||||
packageId = "core-foundation-sys";
|
||||
}
|
||||
{
|
||||
name = "libc";
|
||||
packageId = "libc";
|
||||
}
|
||||
];
|
||||
|
||||
};
|
||||
"tabwriter" = rec {
|
||||
crateName = "tabwriter";
|
||||
version = "1.2.1";
|
||||
|
|
@ -9139,6 +9626,12 @@ rec {
|
|||
name = "prost";
|
||||
packageId = "prost";
|
||||
}
|
||||
{
|
||||
name = "reqwest";
|
||||
packageId = "reqwest";
|
||||
usesDefaultFeatures = false;
|
||||
features = [ "rustls-tls" "stream" ];
|
||||
}
|
||||
{
|
||||
name = "sha2";
|
||||
packageId = "sha2";
|
||||
|
|
@ -9237,6 +9730,10 @@ rec {
|
|||
name = "walkdir";
|
||||
packageId = "walkdir";
|
||||
}
|
||||
{
|
||||
name = "xz2";
|
||||
packageId = "xz2";
|
||||
}
|
||||
];
|
||||
buildDependencies = [
|
||||
{
|
||||
|
|
@ -9812,6 +10309,39 @@ rec {
|
|||
};
|
||||
resolvedDefaultFeatures = [ "spans" ];
|
||||
};
|
||||
"wasm-bindgen-futures" = rec {
|
||||
crateName = "wasm-bindgen-futures";
|
||||
version = "0.4.34";
|
||||
edition = "2018";
|
||||
sha256 = "0m0lnnnhs9ni4dn9vz74prsjz8bdcf8dvnznd5ljch5s279f06gj";
|
||||
authors = [
|
||||
"The wasm-bindgen Developers"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "cfg-if";
|
||||
packageId = "cfg-if";
|
||||
}
|
||||
{
|
||||
name = "js-sys";
|
||||
packageId = "js-sys";
|
||||
}
|
||||
{
|
||||
name = "wasm-bindgen";
|
||||
packageId = "wasm-bindgen";
|
||||
}
|
||||
{
|
||||
name = "web-sys";
|
||||
packageId = "web-sys";
|
||||
target = { target, features }: (builtins.elem "atomics" targetFeatures);
|
||||
features = [ "MessageEvent" "Worker" ];
|
||||
}
|
||||
];
|
||||
features = {
|
||||
"futures-core" = [ "dep:futures-core" ];
|
||||
"futures-core-03-stream" = [ "futures-core" ];
|
||||
};
|
||||
};
|
||||
"wasm-bindgen-macro" = rec {
|
||||
crateName = "wasm-bindgen-macro";
|
||||
version = "0.2.84";
|
||||
|
|
@ -9883,6 +10413,48 @@ rec {
|
|||
"The wasm-bindgen Developers"
|
||||
];
|
||||
|
||||
};
|
||||
"wasm-streams" = rec {
|
||||
crateName = "wasm-streams";
|
||||
version = "0.3.0";
|
||||
edition = "2021";
|
||||
sha256 = "1iqa4kmhbsjj8k4q15i1x0x4p3xda0dhbg7zw51mydr4g129sq5l";
|
||||
type = [ "cdylib" "rlib" ];
|
||||
authors = [
|
||||
"Mattias Buelens <mattias@buelens.com>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "futures-util";
|
||||
packageId = "futures-util";
|
||||
features = [ "io" "sink" ];
|
||||
}
|
||||
{
|
||||
name = "js-sys";
|
||||
packageId = "js-sys";
|
||||
}
|
||||
{
|
||||
name = "wasm-bindgen";
|
||||
packageId = "wasm-bindgen";
|
||||
}
|
||||
{
|
||||
name = "wasm-bindgen-futures";
|
||||
packageId = "wasm-bindgen-futures";
|
||||
}
|
||||
{
|
||||
name = "web-sys";
|
||||
packageId = "web-sys";
|
||||
features = [ "AbortSignal" ];
|
||||
}
|
||||
];
|
||||
devDependencies = [
|
||||
{
|
||||
name = "web-sys";
|
||||
packageId = "web-sys";
|
||||
features = [ "console" "AbortSignal" "Response" "ReadableStream" "Window" ];
|
||||
}
|
||||
];
|
||||
|
||||
};
|
||||
"web-sys" = rec {
|
||||
crateName = "web-sys";
|
||||
|
|
@ -10339,7 +10911,14 @@ rec {
|
|||
"XrViewerPose" = [ "XrPose" ];
|
||||
"XrWebGlLayer" = [ "EventTarget" "XrLayer" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "CanvasRenderingContext2d" "Crypto" "Document" "DomRect" "DomRectReadOnly" "Element" "EventTarget" "HtmlCanvasElement" "HtmlElement" "Node" "Window" ];
|
||||
resolvedDefaultFeatures = [ "AbortController" "AbortSignal" "Blob" "BlobPropertyBag" "CanvasRenderingContext2d" "Crypto" "Document" "DomRect" "DomRectReadOnly" "Element" "Event" "EventTarget" "File" "FormData" "Headers" "HtmlCanvasElement" "HtmlElement" "MessageEvent" "Node" "ReadableStream" "Request" "RequestCredentials" "RequestInit" "RequestMode" "Response" "ServiceWorkerGlobalScope" "Window" "Worker" "WorkerGlobalScope" ];
|
||||
};
|
||||
"webpki-roots" = rec {
|
||||
crateName = "webpki-roots";
|
||||
version = "0.25.2";
|
||||
edition = "2018";
|
||||
sha256 = "1z13850xvsijjxxvzx1wq3m6pz78ih5q6wjcp7gpgwz4gfspn90l";
|
||||
|
||||
};
|
||||
"which" = rec {
|
||||
crateName = "which";
|
||||
|
|
@ -11008,7 +11587,7 @@ rec {
|
|||
"Win32_Web" = [ "Win32" ];
|
||||
"Win32_Web_InternetExplorer" = [ "Win32_Web" ];
|
||||
};
|
||||
resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ];
|
||||
resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_NetworkManagement" "Win32_NetworkManagement_IpHelper" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_Diagnostics" "Win32_System_Diagnostics_Debug" "Win32_System_IO" "Win32_System_Memory" "Win32_System_Pipes" "Win32_System_Registry" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_Time" "Win32_System_WindowsProgramming" "default" ];
|
||||
};
|
||||
"windows-targets 0.42.2" = rec {
|
||||
crateName = "windows-targets";
|
||||
|
|
@ -11269,6 +11848,31 @@ rec {
|
|||
];
|
||||
|
||||
};
|
||||
"winreg" = rec {
|
||||
crateName = "winreg";
|
||||
version = "0.50.0";
|
||||
edition = "2018";
|
||||
sha256 = "1cddmp929k882mdh6i9f2as848f13qqna6czwsqzkh1pqnr5fkjj";
|
||||
authors = [
|
||||
"Igor Shaula <gentoo90@gmail.com>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "cfg-if";
|
||||
packageId = "cfg-if";
|
||||
}
|
||||
{
|
||||
name = "windows-sys";
|
||||
packageId = "windows-sys 0.48.0";
|
||||
features = [ "Win32_Foundation" "Win32_System_Time" "Win32_System_Registry" "Win32_Security" "Win32_Storage_FileSystem" "Win32_System_Diagnostics_Debug" ];
|
||||
}
|
||||
];
|
||||
features = {
|
||||
"chrono" = [ "dep:chrono" ];
|
||||
"serde" = [ "dep:serde" ];
|
||||
"serialization-serde" = [ "transactions" "serde" ];
|
||||
};
|
||||
};
|
||||
"wu-manber" = rec {
|
||||
crateName = "wu-manber";
|
||||
version = "0.1.0";
|
||||
|
|
@ -11296,6 +11900,27 @@ rec {
|
|||
];
|
||||
|
||||
};
|
||||
"xz2" = rec {
|
||||
crateName = "xz2";
|
||||
version = "0.1.7";
|
||||
edition = "2018";
|
||||
sha256 = "1qk7nzpblizvayyq4xzi4b0zacmmbqr6vb9fc0v1avyp17f4931q";
|
||||
authors = [
|
||||
"Alex Crichton <alex@alexcrichton.com>"
|
||||
];
|
||||
dependencies = [
|
||||
{
|
||||
name = "lzma-sys";
|
||||
packageId = "lzma-sys";
|
||||
}
|
||||
];
|
||||
features = {
|
||||
"futures" = [ "dep:futures" ];
|
||||
"static" = [ "lzma-sys/static" ];
|
||||
"tokio" = [ "tokio-io" "futures" ];
|
||||
"tokio-io" = [ "dep:tokio-io" ];
|
||||
};
|
||||
};
|
||||
"yansi" = rec {
|
||||
crateName = "yansi";
|
||||
version = "0.5.1";
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ tvix-castore = { path = "../castore" }
|
|||
url = "2.4.0"
|
||||
walkdir = "2.4.0"
|
||||
async-recursion = "1.0.5"
|
||||
reqwest = { version = "0.11.22", features = ["rustls-tls", "stream"], default-features = false }
|
||||
xz2 = "0.1.7"
|
||||
|
||||
[dependencies.fuse-backend-rs]
|
||||
optional = true
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
use crate::proto::path_info_service_client::PathInfoServiceClient;
|
||||
|
||||
use super::{GRPCPathInfoService, MemoryPathInfoService, PathInfoService, SledPathInfoService};
|
||||
use super::{
|
||||
GRPCPathInfoService, MemoryPathInfoService, NixHTTPPathInfoService, PathInfoService,
|
||||
SledPathInfoService,
|
||||
};
|
||||
|
||||
use std::sync::Arc;
|
||||
use tvix_castore::{blobservice::BlobService, directoryservice::DirectoryService, Error};
|
||||
|
|
@ -62,6 +65,16 @@ pub async fn from_addr(
|
|||
SledPathInfoService::new(url.path().into(), blob_service, directory_service)
|
||||
.map_err(|e| Error::StorageError(e.to_string()))?,
|
||||
));
|
||||
} else if url.scheme() == "nix+http" || url.scheme() == "nix+https" {
|
||||
// Stringify the URL and remove the nix+ prefix.
|
||||
// We can't use `url.set_scheme(rest)`, as it disallows
|
||||
// setting something http(s) that previously wasn't.
|
||||
let url = Url::parse(url.to_string().strip_prefix("nix+").unwrap()).unwrap();
|
||||
Arc::new(NixHTTPPathInfoService::new(
|
||||
url,
|
||||
blob_service,
|
||||
directory_service,
|
||||
))
|
||||
} else if url.scheme().starts_with("grpc+") {
|
||||
// schemes starting with grpc+ go to the GRPCPathInfoService.
|
||||
// That's normally grpc+unix for unix sockets, and grpc+http(s) for the HTTP counterparts.
|
||||
|
|
@ -113,6 +126,14 @@ mod tests {
|
|||
#[test_case("memory:///", false; "memory invalid root path")]
|
||||
/// This sets a memory url path to "/foo", which is invalid.
|
||||
#[test_case("memory:///foo", false; "memory invalid root path foo")]
|
||||
/// Correct Scheme for the cache.nixos.org binary cache.
|
||||
#[test_case("nix+https://cache.nixos.org", true; "correct nix+https")]
|
||||
/// Correct Scheme for the cache.nixos.org binary cache (HTTP URL).
|
||||
#[test_case("nix+http://cache.nixos.org", true; "correct nix+http")]
|
||||
/// Correct Scheme for Nix HTTP Binary cache, with a subpath.
|
||||
#[test_case("nix+http://192.0.2.1/foo", true; "correct nix http with subpath")]
|
||||
/// Correct Scheme for Nix HTTP Binary cache, with a subpath and port.
|
||||
#[test_case("nix+http://[::1]:8080/foo", true; "correct nix http with subpath and port")]
|
||||
/// Correct scheme to connect to a unix socket.
|
||||
#[test_case("grpc+unix:///path/to/somewhere", true; "grpc valid unix socket")]
|
||||
/// Correct scheme for unix socket, but setting a host too, which is invalid.
|
||||
|
|
@ -127,11 +148,8 @@ mod tests {
|
|||
#[test_case("grpc+http://localhost/some-path", false; "grpc valid invalid host and path")]
|
||||
#[tokio::test]
|
||||
async fn test_from_addr_tokio(uri_str: &str, is_ok: bool) {
|
||||
assert_eq!(
|
||||
from_addr(uri_str, gen_blob_service(), gen_directory_service())
|
||||
.await
|
||||
.is_ok(),
|
||||
is_ok
|
||||
)
|
||||
let resp = from_addr(uri_str, gen_blob_service(), gen_directory_service()).await;
|
||||
|
||||
assert_eq!(resp.is_ok(), is_ok);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
mod from_addr;
|
||||
mod grpc;
|
||||
mod memory;
|
||||
mod nix_http;
|
||||
mod sled;
|
||||
|
||||
use futures::Stream;
|
||||
|
|
@ -14,6 +15,7 @@ use crate::proto::PathInfo;
|
|||
pub use self::from_addr::from_addr;
|
||||
pub use self::grpc::GRPCPathInfoService;
|
||||
pub use self::memory::MemoryPathInfoService;
|
||||
pub use self::nix_http::NixHTTPPathInfoService;
|
||||
pub use self::sled::SledPathInfoService;
|
||||
|
||||
/// The base trait all PathInfo services need to implement.
|
||||
|
|
|
|||
213
tvix/store/src/pathinfoservice/nix_http.rs
Normal file
213
tvix/store/src/pathinfoservice/nix_http.rs
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
use std::{
|
||||
io::{self, BufRead},
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use data_encoding::BASE64;
|
||||
use futures::{Stream, TryStreamExt};
|
||||
use nix_compat::{narinfo::NarInfo, nixbase32};
|
||||
use reqwest::StatusCode;
|
||||
use tonic::async_trait;
|
||||
use tracing::{debug, instrument, warn};
|
||||
use tvix_castore::{
|
||||
blobservice::BlobService, directoryservice::DirectoryService, proto as castorepb, Error,
|
||||
};
|
||||
|
||||
use crate::proto::PathInfo;
|
||||
|
||||
use super::PathInfoService;
|
||||
|
||||
/// NixHTTPPathInfoService acts as a bridge in between the Nix HTTP Binary cache
|
||||
/// protocol provided by Nix binary caches such as cache.nixos.org, and the Tvix
|
||||
/// Store Model.
|
||||
/// It implements the [PathInfoService] trait in an interesting way:
|
||||
/// Every [PathInfoService::get] fetches the .narinfo and referred NAR file,
|
||||
/// inserting components into a [BlobService] and [DirectoryService], then
|
||||
/// returning a [PathInfo] struct with the root.
|
||||
///
|
||||
/// Due to this being quite a costly operation, clients are expected to layer
|
||||
/// this service with store composition, so they're only ingested once.
|
||||
///
|
||||
/// The client is expected to be (indirectly) using the same [BlobService] and
|
||||
/// [DirectoryService], so able to fetch referred Directories and Blobs.
|
||||
/// [PathInfoService::put] and [PathInfoService::nar] are not implemented and
|
||||
/// return an error if called.
|
||||
/// TODO: what about reading from nix-cache-info?
|
||||
pub struct NixHTTPPathInfoService {
|
||||
base_url: url::Url,
|
||||
http_client: reqwest::Client,
|
||||
|
||||
blob_service: Arc<dyn BlobService>,
|
||||
directory_service: Arc<dyn DirectoryService>,
|
||||
}
|
||||
|
||||
impl NixHTTPPathInfoService {
|
||||
pub fn new(
|
||||
base_url: url::Url,
|
||||
blob_service: Arc<dyn BlobService>,
|
||||
directory_service: Arc<dyn DirectoryService>,
|
||||
) -> Self {
|
||||
Self {
|
||||
base_url,
|
||||
http_client: reqwest::Client::new(),
|
||||
blob_service,
|
||||
directory_service,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl PathInfoService for NixHTTPPathInfoService {
|
||||
#[instrument(skip_all, err, fields(path.digest=BASE64.encode(&digest)))]
|
||||
async fn get(&self, digest: [u8; 20]) -> Result<Option<PathInfo>, Error> {
|
||||
let narinfo_url = self
|
||||
.base_url
|
||||
.join(&format!("{}.narinfo", nixbase32::encode(&digest)))
|
||||
.map_err(|e| {
|
||||
warn!(e = %e, "unable to join URL");
|
||||
io::Error::new(io::ErrorKind::InvalidInput, "unable to join url")
|
||||
})?;
|
||||
|
||||
debug!(narinfo_url= %narinfo_url, "constructed NARInfo url");
|
||||
|
||||
let resp = self
|
||||
.http_client
|
||||
.get(narinfo_url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!(e=%e,"unable to send NARInfo request");
|
||||
io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"unable to send NARInfo request",
|
||||
)
|
||||
})?;
|
||||
|
||||
// In the case of a 404, return a NotFound.
|
||||
// We also return a NotFound in case of a 403 - this is to match the behaviour as Nix,
|
||||
// when querying nix-cache.s3.amazonaws.com directly, rather than cache.nixos.org.
|
||||
if resp.status() == StatusCode::NOT_FOUND || resp.status() == StatusCode::FORBIDDEN {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let narinfo_str = resp.text().await.map_err(|e| {
|
||||
warn!(e=%e,"unable to decode response as string");
|
||||
io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"unable to decode response as string",
|
||||
)
|
||||
})?;
|
||||
|
||||
// parse the received narinfo
|
||||
let narinfo = NarInfo::parse(&narinfo_str).map_err(|e| {
|
||||
warn!(e=%e,"unable to parse response as NarInfo");
|
||||
io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"unable to parse response as NarInfo",
|
||||
)
|
||||
})?;
|
||||
|
||||
// Convert to a (sparse) PathInfo. We still need to populate the node field,
|
||||
// and for this we need to download the NAR file.
|
||||
// FUTUREWORK: Keep some database around mapping from narsha256 to
|
||||
// (unnamed) rootnode, so we can use that (and the name from the
|
||||
// StorePath) and avoid downloading the same NAR a second time.
|
||||
let pathinfo: PathInfo = (&narinfo).into();
|
||||
|
||||
// create a request for the NAR file itself.
|
||||
let nar_url = self.base_url.join(narinfo.url).map_err(|e| {
|
||||
warn!(e = %e, "unable to join URL");
|
||||
io::Error::new(io::ErrorKind::InvalidInput, "unable to join url")
|
||||
})?;
|
||||
debug!(nar_url= %nar_url, "constructed NAR url");
|
||||
|
||||
let resp = self
|
||||
.http_client
|
||||
.get(nar_url.clone())
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!(e=%e,"unable to send NAR request");
|
||||
io::Error::new(io::ErrorKind::InvalidInput, "unable to send NAR request")
|
||||
})?;
|
||||
|
||||
// if the request is not successful, return an error.
|
||||
if !resp.status().is_success() {
|
||||
return Err(Error::StorageError(format!(
|
||||
"unable to retrieve NAR at {}, status {}",
|
||||
nar_url,
|
||||
resp.status()
|
||||
)));
|
||||
}
|
||||
|
||||
// get an AsyncRead of the response body.
|
||||
let async_r = tokio_util::io::StreamReader::new(resp.bytes_stream().map_err(|e| {
|
||||
let e = e.without_url();
|
||||
warn!(e=%e, "failed to get response body");
|
||||
io::Error::new(io::ErrorKind::BrokenPipe, e.to_string())
|
||||
}));
|
||||
let sync_r = std::io::BufReader::new(tokio_util::io::SyncIoBridge::new(async_r));
|
||||
|
||||
// handle decompression, by wrapping the reader.
|
||||
let mut sync_r: Box<dyn BufRead + Send> = match narinfo.compression {
|
||||
Some("none") => Box::new(sync_r),
|
||||
Some("xz") => Box::new(std::io::BufReader::new(xz2::read::XzDecoder::new(sync_r))),
|
||||
Some(comp) => {
|
||||
return Err(Error::InvalidRequest(
|
||||
format!("unsupported compression: {}", comp).to_string(),
|
||||
))
|
||||
}
|
||||
None => {
|
||||
return Err(Error::InvalidRequest(
|
||||
"unsupported compression: bzip2".to_string(),
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
let res = tokio::task::spawn_blocking({
|
||||
let blob_service = self.blob_service.clone();
|
||||
let directory_service = self.directory_service.clone();
|
||||
move || crate::nar::read_nar(&mut sync_r, blob_service, directory_service)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
match res {
|
||||
Ok(root_node) => Ok(Some(PathInfo {
|
||||
node: Some(castorepb::Node {
|
||||
// set the name of the root node to the digest-name of the store path.
|
||||
node: Some(root_node.rename(narinfo.store_path.to_string().to_owned().into())),
|
||||
}),
|
||||
references: pathinfo.references,
|
||||
narinfo: pathinfo.narinfo,
|
||||
})),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(skip_all, fields(path_info=?_path_info))]
|
||||
async fn put(&self, _path_info: PathInfo) -> Result<PathInfo, Error> {
|
||||
Err(Error::InvalidRequest(
|
||||
"put not supported for this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
#[instrument(skip_all, fields(root_node=?root_node))]
|
||||
async fn calculate_nar(
|
||||
&self,
|
||||
root_node: &castorepb::node::Node,
|
||||
) -> Result<(u64, [u8; 32]), Error> {
|
||||
Err(Error::InvalidRequest(
|
||||
"calculate_nar not supported for this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
fn list(&self) -> Pin<Box<dyn Stream<Item = Result<PathInfo, Error>> + Send>> {
|
||||
Box::pin(futures::stream::once(async {
|
||||
Err(Error::InvalidRequest(
|
||||
"list not supported for this backend".to_string(),
|
||||
))
|
||||
}))
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue