feat(tvix/store): add opentelemetry

This is behind the otlp feature flag (currently enabled by default).

By default, this will try to push traces to a OTLP collector running at
https://localhost:4317.

You can make one available by running:

```
docker run -d --name jaeger \
  -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
  -e COLLECTOR_OTLP_ENABLED=true \
  -p 6831:6831/udp \
  -p 6832:6832/udp \
  -p 5778:5778 \
  -p 16686:16686 \
  -p 4317:4317 \
  -p 4318:4318 \
  -p 14250:14250 \
  -p 14268:14268 \
  -p 14269:14269 \
  -p 9411:9411 --rm \
  jaegertracing/all-in-one:1.42
```

Started like that, jaeger brings a web interface at
http://localhost:16686/search

As documented in
https://docs.rs/opentelemetry-otlp/latest/opentelemetry_otlp/, you can
point this to another location by setting `OTEL_EXPORTER_OTLP_ENDPOINT`.

Change-Id: Id1dca367d70027b2ea98bb70bcf99a68363ec2be
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8194
Tested-by: BuildkiteCI
Reviewed-by: tazjin <tazjin@tvl.su>
Autosubmit: flokli <flokli@flokli.de>
Reviewed-by: aaqaishtyaq <aaqaishtyaq@gmail.com>
This commit is contained in:
Florian Klink 2023-03-01 00:37:08 +01:00 committed by clbot
parent 597a6b6205
commit 3307791855
5 changed files with 1135 additions and 94 deletions

277
tvix/Cargo.lock generated
View file

@ -771,6 +771,12 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
[[package]]
name = "equivalent"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
version = "0.3.1"
@ -1035,7 +1041,7 @@ dependencies = [
"futures-sink",
"futures-util",
"http",
"indexmap",
"indexmap 1.9.3",
"slab",
"tokio",
"tokio-util",
@ -1054,6 +1060,12 @@ version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
[[package]]
name = "heck"
version = "0.4.1"
@ -1215,7 +1227,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown",
"hashbrown 0.12.3",
]
[[package]]
name = "indexmap"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
dependencies = [
"equivalent",
"hashbrown 0.14.3",
]
[[package]]
@ -1282,9 +1304,9 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.61"
version = "0.3.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"
checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca"
dependencies = [
"wasm-bindgen",
]
@ -1634,6 +1656,93 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "opentelemetry"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
dependencies = [
"futures-core",
"futures-sink",
"indexmap 2.1.0",
"js-sys",
"once_cell",
"pin-project-lite",
"thiserror",
"urlencoding",
]
[[package]]
name = "opentelemetry-otlp"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930"
dependencies = [
"async-trait",
"futures-core",
"http",
"opentelemetry",
"opentelemetry-proto",
"opentelemetry-semantic-conventions",
"opentelemetry_sdk",
"prost 0.11.9",
"thiserror",
"tokio",
"tonic 0.9.2",
]
[[package]]
name = "opentelemetry-proto"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1"
dependencies = [
"opentelemetry",
"opentelemetry_sdk",
"prost 0.11.9",
"tonic 0.9.2",
]
[[package]]
name = "opentelemetry-semantic-conventions"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84"
dependencies = [
"opentelemetry",
]
[[package]]
name = "opentelemetry_sdk"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "968ba3f2ca03e90e5187f5e4f46c791ef7f2c163ae87789c8ce5f5ca3b7b7de5"
dependencies = [
"async-trait",
"crossbeam-channel",
"futures-channel",
"futures-executor",
"futures-util",
"glob",
"once_cell",
"opentelemetry",
"ordered-float",
"percent-encoding",
"rand",
"thiserror",
"tokio",
"tokio-stream",
]
[[package]]
name = "ordered-float"
version = "4.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e"
dependencies = [
"num-traits",
]
[[package]]
name = "overload"
version = "0.1.1"
@ -1707,7 +1816,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"
dependencies = [
"fixedbitset",
"indexmap",
"indexmap 1.9.3",
]
[[package]]
@ -1880,6 +1989,16 @@ dependencies = [
"unarray",
]
[[package]]
name = "prost"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
dependencies = [
"bytes",
"prost-derive 0.11.9",
]
[[package]]
name = "prost"
version = "0.12.1"
@ -1887,7 +2006,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d"
dependencies = [
"bytes",
"prost-derive",
"prost-derive 0.12.1",
]
[[package]]
@ -1904,7 +2023,7 @@ dependencies = [
"once_cell",
"petgraph",
"prettyplease",
"prost",
"prost 0.12.1",
"prost-types",
"regex",
"syn 2.0.39",
@ -1912,6 +2031,19 @@ dependencies = [
"which",
]
[[package]]
name = "prost-derive"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4"
dependencies = [
"anyhow",
"itertools",
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 1.0.109",
]
[[package]]
name = "prost-derive"
version = "0.12.1"
@ -1931,7 +2063,7 @@ version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf"
dependencies = [
"prost",
"prost 0.12.1",
]
[[package]]
@ -2172,7 +2304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64449cfef9483a475ed56ae30e2da5ee96448789fb2aa240a04beb6a055078bf"
dependencies = [
"countme",
"hashbrown",
"hashbrown 0.12.3",
"memoffset 0.8.0",
"rustc-hash",
"text-size",
@ -2830,7 +2962,7 @@ dependencies = [
"pin-project",
"socket2 0.5.4",
"tokio",
"tonic",
"tonic 0.10.2",
"tracing",
]
@ -2919,13 +3051,41 @@ version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b"
dependencies = [
"indexmap",
"indexmap 1.9.3",
"nom8",
"serde",
"serde_spanned",
"toml_datetime",
]
[[package]]
name = "tonic"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
dependencies = [
"async-trait",
"axum",
"base64",
"bytes",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"hyper",
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost 0.11.9",
"tokio",
"tokio-stream",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tonic"
version = "0.10.2"
@ -2944,7 +3104,7 @@ dependencies = [
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost",
"prost 0.12.1",
"rustls",
"rustls-native-certs",
"rustls-pemfile",
@ -2976,11 +3136,11 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fa37c513df1339d197f4ba21d28c918b9ef1ac1768265f11ecb6b7f1cba1b76"
dependencies = [
"prost",
"prost 0.12.1",
"prost-types",
"tokio",
"tokio-stream",
"tonic",
"tonic 0.10.2",
]
[[package]]
@ -2991,7 +3151,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
dependencies = [
"futures-core",
"futures-util",
"indexmap",
"indexmap 1.9.3",
"pin-project",
"pin-project-lite",
"rand",
@ -3060,6 +3220,35 @@ dependencies = [
"tracing-core",
]
[[package]]
name = "tracing-log"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"once_cell",
"tracing-core",
]
[[package]]
name = "tracing-opentelemetry"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596"
dependencies = [
"js-sys",
"once_cell",
"opentelemetry",
"opentelemetry_sdk",
"smallvec",
"tracing",
"tracing-core",
"tracing-log 0.2.0",
"tracing-subscriber",
"web-time",
]
[[package]]
name = "tracing-serde"
version = "0.1.3"
@ -3083,7 +3272,7 @@ dependencies = [
"smallvec",
"thread_local",
"tracing-core",
"tracing-log",
"tracing-log 0.1.3",
"tracing-serde",
]
@ -3098,9 +3287,9 @@ name = "tvix-build"
version = "0.1.0"
dependencies = [
"bytes",
"prost",
"prost 0.12.1",
"prost-build",
"tonic",
"tonic 0.10.2",
"tonic-build",
"tonic-reflection",
"tvix-castore",
@ -3123,7 +3312,7 @@ dependencies = [
"libc",
"parking_lot 0.12.1",
"pin-project-lite",
"prost",
"prost 0.12.1",
"prost-build",
"sled",
"tempfile",
@ -3133,7 +3322,7 @@ dependencies = [
"tokio-retry",
"tokio-stream",
"tokio-util",
"tonic",
"tonic 0.10.2",
"tonic-build",
"tonic-reflection",
"tower",
@ -3253,8 +3442,11 @@ dependencies = [
"futures",
"lazy_static",
"nix-compat",
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry_sdk",
"pin-project-lite",
"prost",
"prost 0.12.1",
"prost-build",
"reqwest",
"sha2",
@ -3267,11 +3459,12 @@ dependencies = [
"tokio-retry",
"tokio-stream",
"tokio-util",
"tonic",
"tonic 0.10.2",
"tonic-build",
"tonic-reflection",
"tower",
"tracing",
"tracing-opentelemetry",
"tracing-subscriber",
"tvix-castore",
"url",
@ -3347,6 +3540,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "urlencoding"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "utf8parse"
version = "0.2.1"
@ -3483,9 +3682,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.84"
version = "0.2.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@ -3493,16 +3692,16 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.84"
version = "0.2.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"
checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826"
dependencies = [
"bumpalo",
"log",
"once_cell",
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 1.0.109",
"syn 2.0.39",
"wasm-bindgen-shared",
]
@ -3520,9 +3719,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.84"
version = "0.2.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"
checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2"
dependencies = [
"quote 1.0.33",
"wasm-bindgen-macro-support",
@ -3530,22 +3729,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.84"
version = "0.2.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283"
dependencies = [
"proc-macro2 1.0.67",
"quote 1.0.33",
"syn 1.0.109",
"syn 2.0.39",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.84"
version = "0.2.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f"
[[package]]
name = "wasm-streams"
@ -3570,6 +3769,16 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.25.2"

File diff suppressed because it is too large Load diff

View file

@ -32,6 +32,10 @@ let
buildInputs = prev.buildInputs or [ ] ++ iconvDarwinDep;
};
opentelemetry-proto = prev: {
nativeBuildInputs = protobufDep prev;
};
prost-build = prev: {
nativeBuildInputs = protobufDep prev;
};

View file

@ -16,6 +16,9 @@ lazy_static = "1.4.0"
nix-compat = { path = "../nix-compat", features = ["async"] }
pin-project-lite = "0.2.13"
prost = "0.12.1"
opentelemetry = { version = "0.21.0", optional = true}
opentelemetry-otlp = { version = "0.14.0", optional = true }
opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio"], optional = true}
sha2 = "0.10.6"
sled = { version = "0.34.7" }
thiserror = "1.0.38"
@ -26,6 +29,7 @@ tokio-util = { version = "0.7.9", features = ["io", "io-util", "compat"] }
tonic = { version = "0.10.2", features = ["tls", "tls-roots"] }
tower = "0.4.13"
tracing = "0.1.37"
tracing-opentelemetry = "0.22.0"
tracing-subscriber = { version = "0.3.16", features = ["json"] }
tvix-castore = { path = "../castore" }
url = "2.4.0"
@ -48,7 +52,8 @@ tempfile = "3.3.0"
tokio-retry = "0.3.0"
[features]
default = ["fuse", "tonic-reflection"]
default = ["fuse", "otlp", "tonic-reflection"]
fuse = ["tvix-castore/fuse"]
virtiofs = ["tvix-castore/virtiofs"]
otlp = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk"]
tonic-reflection = ["dep:tonic-reflection", "tvix-castore/tonic-reflection"]
virtiofs = ["tvix-castore/virtiofs"]

View file

@ -1,6 +1,10 @@
use clap::Parser;
use clap::Subcommand;
use futures::future::try_join_all;
use tonic::transport::Server;
use tracing::info;
use tracing::Level;
use std::path::PathBuf;
use std::sync::Arc;
@ -9,6 +13,7 @@ use tokio_listener::SystemOptions;
use tokio_listener::UserOptions;
use tracing_subscriber::prelude::*;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
use tvix_castore::proto::blob_service_server::BlobServiceServer;
use tvix_castore::proto::directory_service_server::DirectoryServiceServer;
@ -32,10 +37,6 @@ use tvix_castore::proto::FILE_DESCRIPTOR_SET as CASTORE_FILE_DESCRIPTOR_SET;
#[cfg(feature = "tonic-reflection")]
use tvix_store::proto::FILE_DESCRIPTOR_SET;
use clap::Parser;
use tonic::{transport::Server, Result};
use tracing::{info, Level};
#[derive(Parser)]
#[command(author, version, about, long_about = None)]
struct Cli {
@ -170,7 +171,37 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
),
);
tracing::subscriber::set_global_default(subscriber).expect("Unable to set global subscriber");
// Add the otlp layer (when otlp is enabled), then init the registry.
// Or if the feature is disabled, just init without adding the layer.
// It's necessary to do this separately, as every with() call chains the
// layer into the type of the registry.
#[cfg(feature = "otlp")]
{
let opentelemetry_layer = {
let otlp_exporter = opentelemetry_otlp::new_exporter().tonic();
// TODO: re-add once https://github.com/open-telemetry/opentelemetry-rust/pull/1252 is solved.
// let mut metadata = tonic::metadata::MetadataMap::new();
// metadata.insert("service.name", "tvix.store".parse()?);
// otlp_exporter.with_metadata(metadata),
let tracer = opentelemetry_otlp::new_pipeline()
.tracing()
.with_exporter(otlp_exporter)
.install_batch(opentelemetry_sdk::runtime::Tokio)?;
// Create a tracing layer with the configured tracer
tracing_opentelemetry::layer().with_tracer(tracer)
};
let subscriber = subscriber.with(opentelemetry_layer);
subscriber.try_init()?;
}
// Init the registry (when otlp is not enabled)
#[cfg(not(feature = "otlp"))]
{
subscriber.try_init()?;
}
match cli.command {
Commands::Daemon {