mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(users/flokli/nixos/nixos-tvix-cache): collect metrics

Browse source 
This enables routing of metrics to an instance of VictoriaMetrics, and
configures opentelemetry-collector to route metrics there.

Change-Id: If765191a4cc70ddcaad821d45132b96a10a12148
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12812
Reviewed-by: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
Reviewed-by: Jonas Chevalier <zimbatm@zimbatm.com>
This commit is contained in:
Florian Klink 2024-11-19 17:17:32 +02:00 committed by flokli
parent 52a8e47ac1
commit 6f1d059c7d
1 changed files with 56 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
users/flokli/nixos/nixos-tvix-cache/monitoring.nix
View file

@ -1,4 +1,4 @@
{ config, ... }: { config, pkgs, ... }:
let let
domain = config.machine.domain; domain = config.machine.domain;
in in
@ -18,12 +18,13 @@ in
grpc_server_max_send_msg_size = 67108864; grpc_server_max_send_msg_size = 67108864;
log_level = "warn"; log_level = "warn";
}; };
distributor.receivers = {
otlp.protocols = { # move the otlp listener to another port than 4317, and disable the 4318 one.
grpc = { }; # *:4317 # opentelemetry-connector binds on both 4317 and 4318.
http = { }; # *:4318 distributor.receivers.otlp.protocols = {
}; grpc.endpoint = "127.0.0.1:4319";
}; };
storage.trace = { storage.trace = {
backend = "local"; backend = "local";
wal.path = "/var/lib/tempo/wal"; wal.path = "/var/lib/tempo/wal";
@ -33,55 +34,51 @@ in
}; };
}; };
# No need, tempo collects the traces directly. services.opentelemetry-collector = {
# enable = true;
# services.opentelemetry-collector = { settings = {
# enable = true; receivers = {
otlp.protocols.grpc.endpoint = "127.0.0.1:4317";
otlp.protocols.http.endpoint = "127.0.0.1:4318";
};
# settings = { processors = {
# receivers = { batch = { };
# otlp.protocols.grpc.endpoint = "127.0.0.1:4317"; };
# otlp.protocols.http.endpoint = "127.0.0.1:4318";
# };
# processors = { exporters = {
# batch = { }; otlp = {
# }; endpoint = "127.0.0.1:4319"; # Tempo otlp-grpc
tls.insecure = true;
};
"otlphttp/metrics" = {
compression = "gzip";
encoding = "proto";
endpoint = "http://localhost:8428/opentelemetry";
tls.insecure = true;
# exporters = { };
# otlp = { };
# endpoint = "127.0.0.1:9080"; # Tempo
# };
# };
# extensions = { service = {
# zpages = { }; pipelines = {
# }; traces = {
receivers = [ "otlp" ];
processors = [ "batch" ];
exporters = [ "otlp" ];
};
metrics = {
receivers = [ "otlp" ];
processors = [ "batch" ];
exporters = [ "otlphttp/metrics" ];
};
};
};
};
};
services.victoriametrics.enable = true;
# service = {
# extensions = [
# "zpages"
# ];
# pipelines = {
# traces = {
# receivers = [ "otlp" ];
# processors = [ "batch" ];
# exporters = [ "otlp" ];
# };
# metrics = {
# receivers = [ "otlp" ];
# processors = [ "batch" ];
# exporters = [ "otlp" ];
# };
# logs = {
# receivers = [ "otlp" ];
# processors = [ "batch" ];
# exporters = [ "otlp" ];
# };
# };
# };
# };
# };
services.grafana = { services.grafana = {
enable = true; enable = true;
@ -122,19 +119,25 @@ in
name = "Tempo"; name = "Tempo";
type = "tempo"; type = "tempo";
uid = "traces"; uid = "traces";
url = "http://127.0.0.1:3200"; url = "http://127.0.0.1:9080";
access = "proxy"; access = "proxy";
timeout = "300"; timeout = "300";
jsonData = { jsonData = {
nodeGraph.enabled = true; nodeGraph.enabled = true;
# tracesToLogs.datasourceUid = "logs"; # tracesToLogs.datasourceUid = "logs";
# tracesToMetrics.datasourceUid = "metrics"; tracesToMetrics.datasourceUid = "metrics";
# serviceMap.datasourceUid = "metrics"; # serviceMap.datasourceUid = "metrics";
# nodeGraph.enabled = true; # nodeGraph.enabled = true;
# lokiSearch.datasourceUid = "logs"; # lokiSearch.datasourceUid = "logs";
}; };
} }
{
name = "prometheus";
type = "prometheus";
uid = "metrics";
url = "http://localhost:8428/";
}
]; ];
}; };
}; };
@ -143,5 +146,6 @@ in
services.nginx.virtualHosts."${domain}".locations."/grafana" = { services.nginx.virtualHosts."${domain}".locations."/grafana" = {
proxyPass = "http://localhost:3000"; proxyPass = "http://localhost:3000";
proxyWebsockets = true;
}; };
} }