feat(tvix/nar-bridge): wire up metrics layer

This provides some global HTTP statistics.

Change-Id: I8bd3e034123154a49d94720b0c8d0c3babde5ae3
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12557
Reviewed-by: Jonas Chevalier <zimbatm@zimbatm.com>
Tested-by: BuildkiteCI
Reviewed-by: flokli <flokli@flokli.de>

tvix/Cargo.lock

@@ -2196,6 +2196,7 @@ dependencies = [
  "lru",
  "mimalloc",
  "nix-compat",
+ "opentelemetry",
  "parking_lot",
  "prost",
  "prost-build",
@@ -2209,6 +2210,7 @@ dependencies = [
  "tonic-build",
  "tower 0.4.13",
  "tower-http",
+ "tower-otel-http-metrics",
  "tracing",
  "tracing-subscriber",
  "tvix-castore",
@@ -4369,6 +4371,21 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
 
+[[package]]
+name = "tower-otel-http-metrics"
+version = "0.8.0"
+source = "git+https://github.com/francoposa/tower-otel-http-metrics?rev=2023a58e7287a691872f8e75f433179d29d1b439#2023a58e7287a691872f8e75f433179d29d1b439"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "http",
+ "opentelemetry",
+ "pin-project-lite",
+ "tower 0.5.1",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "tower-service"
 version = "0.3.3"

tvix/Cargo.nix

@@ -6869,6 +6869,10 @@ rec {
             packageId = "nix-compat";
             features = [ "async" ];
           }
+          {
+            name = "opentelemetry";
+            packageId = "opentelemetry";
+          }
           {
             name = "parking_lot";
             packageId = "parking_lot";
@@ -6914,6 +6918,11 @@ rec {
             packageId = "tower-http";
             features = [ "trace" ];
           }
+          {
+            name = "tower-otel-http-metrics";
+            packageId = "tower-otel-http-metrics";
+            optional = true;
+          }
           {
             name = "tracing";
             packageId = "tracing";
@@ -6962,10 +6971,11 @@ rec {
         ];
         features = {
           "default" = [ "otlp" ];
-          "otlp" = [ "tvix-tracing/otlp" ];
+          "otlp" = [ "tvix-tracing/otlp" "tower-otel-http-metrics" ];
+          "tower-otel-http-metrics" = [ "dep:tower-otel-http-metrics" ];
           "xp-store-composition-cli" = [ "tvix-store/xp-composition-cli" ];
         };
-        resolvedDefaultFeatures = [ "default" "otlp" "xp-store-composition-cli" ];
+        resolvedDefaultFeatures = [ "default" "otlp" "tower-otel-http-metrics" "xp-store-composition-cli" ];
       };
       "nibble_vec" = rec {
         crateName = "nibble_vec";
@@ -14337,6 +14347,66 @@ rec {
         ];
 
       };
+      "tower-otel-http-metrics" = rec {
+        crateName = "tower-otel-http-metrics";
+        version = "0.8.0";
+        edition = "2021";
+        workspace_member = null;
+        src = pkgs.fetchgit {
+          url = "https://github.com/francoposa/tower-otel-http-metrics";
+          rev = "2023a58e7287a691872f8e75f433179d29d1b439";
+          sha256 = "1jiclkybx0fbgd6x2xfhzbq1xm3wba3vbixshqjy765c86jjffcg";
+        };
+        libName = "tower_otel_http_metrics";
+        dependencies = [
+          {
+            name = "futures-core";
+            packageId = "futures-core";
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "futures-util";
+            packageId = "futures-util";
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "http";
+            packageId = "http";
+            usesDefaultFeatures = false;
+            features = [ "std" ];
+          }
+          {
+            name = "opentelemetry";
+            packageId = "opentelemetry";
+            usesDefaultFeatures = false;
+            features = [ "metrics" ];
+          }
+          {
+            name = "pin-project-lite";
+            packageId = "pin-project-lite";
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "tower";
+            packageId = "tower 0.5.1";
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "tower-layer";
+            packageId = "tower-layer";
+            usesDefaultFeatures = false;
+          }
+          {
+            name = "tower-service";
+            packageId = "tower-service";
+            usesDefaultFeatures = false;
+          }
+        ];
+        features = {
+          "axum" = [ "dep:axum" ];
+        };
+        resolvedDefaultFeatures = [ "default" ];
+      };
       "tower-service" = rec {
         crateName = "tower-service";
         version = "0.3.3";

tvix/Cargo.toml

@@ -144,6 +144,8 @@ tonic-health = { version = "0.12.2", default-features = false }
 tonic-reflection = "0.12.2"
 tower = "0.4.13"
 tower-http = "0.5.2"
+# https://github.com/francoposa/tower-otel-http-metrics/pull/13
+tower-otel-http-metrics = { git = "https://github.com/francoposa/tower-otel-http-metrics", rev = "2023a58e7287a691872f8e75f433179d29d1b439"}
 tracing = "0.1.40"
 tracing-indicatif = "0.3.6"
 tracing-opentelemetry = "0.28.0"

tvix/crate-hashes.json

@@ -1,6 +1,7 @@
 {
   "git+https://github.com/TrueLayer/reqwest-middleware?rev=8a494c165734e24c62823714843e1c9347027e8a#0.4.0": "10gaxsvqld8dfwnm0nk93sqbbd2mjl7zfdf6vglp6waka7h87mba",
   "git+https://github.com/TrueLayer/reqwest-middleware?rev=8a494c165734e24c62823714843e1c9347027e8a#reqwest-tracing@0.5.4": "10gaxsvqld8dfwnm0nk93sqbbd2mjl7zfdf6vglp6waka7h87mba",
+  "git+https://github.com/francoposa/tower-otel-http-metrics?rev=2023a58e7287a691872f8e75f433179d29d1b439#0.8.0": "1jiclkybx0fbgd6x2xfhzbq1xm3wba3vbixshqjy765c86jjffcg",
   "git+https://github.com/liufuyang/bigtable_rs?rev=1818355a5373a5bc2c84287e3a4e3807154ac8ef#0.2.10": "0mn6iw1z7gdxbarsqiwscbdr25nplwlvzs0rs51vgnnjfsnbgl6q",
   "git+https://github.com/tvlfyi/wu-manber.git#wu-manber@0.1.0": "1zhk83lbq99xzyjwphv2qrb8f8qgfqwa5bbbvyzm0z0bljsjv0pd"
 }

tvix/default.nix

@@ -21,6 +21,7 @@ let
         "bigtable_rs"
         "reqwest-middleware"
         "reqwest-tracing"
+        "tower-otel-http-metrics"
         "wu-manber"
       ]);
   };

tvix/nar-bridge/Cargo.toml

@@ -16,11 +16,13 @@ futures = { workspace = true }
 itertools = { workspace = true }
 prost = { workspace = true }
 nix-compat = { path = "../nix-compat", features = ["async"] }
+opentelemetry = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
 tokio-listener = { workspace = true, features = ["axum07", "clap", "multi-listener", "sd_listen"] }
 tokio-util = { workspace = true, features = ["io", "io-util", "compat"] }
 tonic = { workspace = true, features = ["tls", "tls-roots"] }
+tower-otel-http-metrics = { workspace = true, optional = true }
 tvix-castore = { path = "../castore" }
 tvix-store = { path = "../store" }
 tvix-tracing = { path = "../tracing", features = ["tonic", "axum"] }
@@ -38,7 +40,7 @@ tonic-build = { workspace = true }
 
 [features]
 default = ["otlp"]
-otlp = ["tvix-tracing/otlp"]
+otlp = ["tvix-tracing/otlp", "tower-otel-http-metrics"]
 xp-store-composition-cli = ["tvix-store/xp-composition-cli"]
 
 [dev-dependencies]

tvix/nar-bridge/src/lib.rs

@@ -43,7 +43,16 @@ impl AppState {
 }
 
 pub fn gen_router(priority: u64) -> Router<AppState> {
-    Router::new()
+    #[cfg(feature = "otlp")]
+    let metrics_meter = opentelemetry::global::meter("nar-bridge");
+
+    #[cfg(feature = "otlp")]
+    let metrics_layer = tower_otel_http_metrics::HTTPMetricsLayerBuilder::new()
+        .with_meter(metrics_meter)
+        .build()
+        .unwrap();
+
+    let router = Router::new()
         .route("/", get(root))
         .route("/nar/:nar_str", get(four_o_four))
         .route("/nar/:nar_str", head(nar::head_root_nodes))
@@ -53,7 +62,12 @@ pub fn gen_router(priority: u64) -> Router<AppState> {
         .route("/:narinfo_str", get(narinfo::get))
         .route("/:narinfo_str", head(narinfo::head))
         .route("/:narinfo_str", put(narinfo::put))
-        .route("/nix-cache-info", get(move || nix_cache_info(priority)))
+        .route("/nix-cache-info", get(move || nix_cache_info(priority)));
+
+    #[cfg(feature = "otlp")]
+    return router.layer(metrics_layer);
+    #[cfg(not(feature = "otlp"))]
+    return router;
 }
 
 async fn root() -> &'static str {