cst1/infrastructure
1
0
Fork 0
forked from DGNum/infrastructure
Code Pull requests Activity

feat(monitoring): drop prometheus in favor of victorialogs

Browse source
This commit is contained in:
sinavir 2025-03-27 12:29:49 +01:00 • committed by thubrecht
parent e2b916e649
commit aed96b16e1
14 changed files with 141 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

1
machines/nixos/rescue01/secrets/secrets.nix
View file

@ -8,4 +8,5 @@
# List of secrets for rescue01
"netbird-relay_environment_file"
"stateless-uptime-kuma-password"
"vmagent-uptime_kuma_apikey"
]

28
machines/nixos/rescue01/secrets/vmagent-uptime_kuma_apikey Normal file
View file

@ -0,0 +1,28 @@
age-encryption.org/v1
-> ssh-ed25519 jIXfPA kqyRXA2pHUxClljkjBwyvVVlfCvw1I+t/KHeulAZ9VA
YNQSmZA3sdAQUk49H0Irud0tsA6zbhrSkBRGDe2hlo8
-> ssh-ed25519 QlRB9Q 5z+dG8KPAuRWYOWrHdSM8shnM9KR3Kz6plfNN0KCBEc
69u6n8zgnQgS/fLQYJKC7TDPytnjvDT90zpS16T0oOI
-> ssh-ed25519 r+nK/Q FsjzUCGKyJb4/SQRkB8dEicKfXEjHC04t0o2PmHAKjY
pBHTvHER0lFNv2cCFW5uZ6Nc7cMacMyLjdS8c1ojIm0
-> ssh-rsa krWCLQ
iTCYGfPEkLvgJ+uW92N7LFPkSI4mC293xns2TV83rtKr7zszYJQa7Xx+bu6KDjB0
/+3E1pH0o+iMF0PPJNcQ1lBxefnPtsQfgW6RAqgu+DwGW/J13uoKBezliFicTh/u
uaT5z7At+T6wq/vEdd5KGP9RdSDdfoKbED1tIxLzKdUqWDurayhuRPMr1JLm4Tar
77oONkDuPFYI8FsyaibFJdqr+outomiAUEu8i0v62Y1WRMwxHmT4sqJaTIiYr5vD
8dheZ5804mrekXrlis3VDrTBTValrrkq9qs78Ly9QR9z6BNREEje+s6TV3KqaYev
AomZKYyh+gtdac2caK1WxA
-> ssh-ed25519 /vwQcQ iFmz9CqY4+SuyspXG4+z0kuO75FKZ0wyBqTmar90T1U
yAuwWlEozFtSNAlGdXRFwj/uekIuhMkoWVUtBfWfcCs
-> ssh-ed25519 0R97PA /mXGDXr1Bgxg5RNR0L2vZdo/aXVjl3R89kwLY/8k1Gc
yCOvDa+Fel7Uv3RURubLTx3yxJ1L7c12yuSmD2HIMus
-> ssh-ed25519 JGx7Ng 9q8YiaK/VKpY8f8OUISi6GU4h1j178T/inCZAemchgk
J2ODND+4nw6r5sXWMndb/CT3GHIaBTvKYolHd29Cfko
-> ssh-ed25519 bUjjig PNhrLR1pDruxTL/i/tNT2zPSNK3GfjiuaFC2YY4qelg
ISupvlviQKPbgrD3+sr+i9yWmpsDJzpAc/9XJqD9ulY
-> ssh-ed25519 +MNHsw GrdwnnaZWy8s5nsy8EC52F87JmuLWDjAKjLCJBKyqw0
BmYMybzWs4a3TvgDs61RzNgjhDLyioVnfFDrxcUhx2U
-> ~8@'-grease 8p8,zY xRg`tl F+$W!$n {n/
CpBIH2q+yIcEz8MR
--- 2d05zWr4b0J/wzQE+Xo9mOvwLtkZPqfotYCrWRPzMnM
ÜÇ,Õi7"ë<>Ž†òSÀknp¡ª¼‘Òî÷âÚÙ¸`K}<7D>ìðž4ÍÄ ˜6uàá²d0_Ï,[¥3ùkÐ1Ÿ<31>„s<k¨õ«?c

22
machines/nixos/rescue01/uptime-kuma.nix
View file

@ -47,11 +47,6 @@ let
extraProbes = {
monitors = {
"prometheus.dgnum.eu" = {
type = mkForce "http";
accepted_statuscodes = [ "401" ];
};
"ollama01.beta.dgnum.eu" = {
type = mkForce "http";
accepted_statuscodes = [ "401" ];
@ -159,4 +154,21 @@ in
passwordFile = config.age.secrets."stateless-uptime-kuma-password".path;
enableService = true;
};
services.vmagent.prometheusConfig = {
scrape_configs = [
{
job_name = "uptime_kuma";
scheme = "http";
static_configs = [ { targets = [ "localhost:${builtins.toString port}" ]; } ];
basic_auth = {
username = "prometheus";
password_file = "%{CREDENTIALS_DIRECTORY}/uptime_kuma_apikey";
};
}
];
};
systemd.services.vmagent.serviceConfig.LoadCredential = [
"uptime_kuma_apikey:${config.age.secrets."vmagent-uptime_kuma_apikey".path}"
];
}

1
machines/nixos/storage01/_configuration.nix
View file

@ -22,7 +22,6 @@ lib.extra.mkConfig {
"netbird"
"openbao"
"peertube"
"prometheus"
"redirections"
"victorialogs"
"victoriametrics"

94
machines/nixos/storage01/garage.nix
View file

@ -3,6 +3,7 @@
# SPDX-FileContributor: Ryan Lahfa <ryan.lahfa@dgnum.eu>
#
# SPDX-License-Identifier: EUPL-1.2
{ config, ... }:
let
host = "s3.dgnum.eu";
@ -51,50 +52,65 @@ in
metadata_dir = "/data/fast/garage/meta";
};
services.garage.settings = {
s3_api.root_domain = ".${host}";
s3_web.root_domain = ".${webHost}";
};
services.nginx.virtualHosts = {
"s3-admin.dgnum.eu" = {
enableACME = true;
forceSSL = true;
locations."/".extraConfig = ''
proxy_pass http://127.0.0.1:${builtins.toString ports.admin_api};
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
'';
services = {
garage.settings = {
s3_api.root_domain = ".${host}";
s3_web.root_domain = ".${webHost}";
};
${host} = {
enableACME = true;
forceSSL = true;
nginx.virtualHosts = {
"s3-admin.dgnum.eu" = {
enableACME = true;
forceSSL = true;
serverAliases = mkHosted host buckets;
locations."/".extraConfig = ''
proxy_pass http://127.0.0.1:${builtins.toString ports.admin_api};
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
'';
};
${host} = {
enableACME = true;
forceSSL = true;
serverAliases = mkHosted host buckets;
locations."/".extraConfig = ''
proxy_pass http://127.0.0.1:${builtins.toString ports.s3_api};
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
# Disable buffering to a temporary file.
proxy_max_temp_file_size 0;
client_max_body_size 5G;
'';
};
${webHost} = {
enableACME = true;
forceSSL = true;
serverAliases = domains ++ (mkHosted webHost buckets);
locations."/".extraConfig = ''
proxy_pass http://127.0.0.1:${builtins.toString ports.s3_web};
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
'';
};
locations."/".extraConfig = ''
proxy_pass http://127.0.0.1:${builtins.toString ports.s3_api};
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
# Disable buffering to a temporary file.
proxy_max_temp_file_size 0;
client_max_body_size 5G;
'';
};
${webHost} = {
enableACME = true;
forceSSL = true;
serverAliases = domains ++ (mkHosted webHost buckets);
locations."/".extraConfig = ''
proxy_pass http://127.0.0.1:${builtins.toString ports.s3_web};
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
'';
vmagent.prometheusConfig = {
scrape_configs = [
{
job_name = "garage";
static_configs = [ { targets = [ "localhost:3903" ]; } ];
bearer_token_file = "%{CREDENTIALS_DIRECTORY}/garage_api}";
}
];
};
};
systemd.services.vmagent.serviceConfig.LoadCredential = [
"garage_api:${config.age.secrets."vmagent-garage_api".path}"
];
}

92
machines/nixos/storage01/prometheus.nix
View file

@ -1,92 +0,0 @@
# SPDX-FileCopyrightText: 2024 Maurice Debray <maurice.debray@dgnum.eu>
#
# SPDX-License-Identifier: EUPL-1.2
{
config,
serverNodes,
lib,
...
}:
let
host = "prometheus.dgnum.eu";
port = 9091;
nodeExporterConfigs = lib.flatten (
lib.mapAttrsToList (
node:
{ config, ... }:
lib.optional config.dgn-monitoring.exporters.enable {
targets = map (p: "${node}.dgnum:${builtins.toString p}") (
builtins.attrValues config.dgn-monitoring.exporters.ports
);
labels = {
host = node;
};
}
) serverNodes
);
in
{
services.prometheus = {
enable = true;
inherit port;
checkConfig = "syntax-only";
enableReload = true;
listenAddress = "127.0.0.1";
webConfigFile = config.age.secrets."prometheus-web_config_file".path;
webExternalUrl = "https://${host}";
retentionTime = "1y";
extraFlags = [ "--storage.tsdb.retention.size=20GB" ];
globalConfig = {
scrape_interval = "15s"; # if you change this settings, please do it in grafana also
};
scrapeConfigs = [
{
job_name = "prometheus";
static_configs = [ { targets = [ "localhost:9090" ]; } ];
}
{
job_name = "node_exporter";
static_configs = nodeExporterConfigs;
}
{
job_name = "uptime_kuma";
scheme = "https";
static_configs = [ { targets = [ "status.dgnum.eu" ]; } ];
basic_auth = {
username = "prometheus";
password_file = config.age.secrets."prometheus-uptime-kuma-apikey".path;
};
}
{
job_name = "hyp01_ups";
metrics_path = "/ups_metrics";
static_configs = [ { targets = [ "100.80.255.180:9199" ]; } ];
}
{
job_name = "garage";
static_configs = [ { targets = [ "localhost:3903" ]; } ];
bearer_token_file = config.age.secrets."prometheus-garage_api".path;
}
];
};
dgn-web.simpleProxies.prometheus = {
inherit host port;
proxyWebsockets = true;
};
age-secrets.autoMatch = [ "prometheus" ];
}

BIN
machines/nixos/storage01/secrets/prometheus-garage_api
View file

Binary file not shown.

BIN
machines/nixos/storage01/secrets/prometheus-uptime-kuma-apikey
View file

Binary file not shown.

24
machines/nixos/storage01/secrets/prometheus-web_config_file
View file

@ -1,24 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 rHotTw RBYiRMlYSSJqICOKMUSCKb5r5o05rS9WCS4iiC7c1k0
dWd+dS0UqpFmlSPAsoUPcslPbMhTulikbpzDD9LRCVQ
-> ssh-ed25519 jIXfPA clKVYR1MMXYIjDcmZQuCP69U04wdgyJ0JVWVtT/pkjI
JygjZZkaJwJ4HIYJL31PEL1+5DPmhh+z94NlO1zrG0E
-> ssh-ed25519 QlRB9Q SYHBZy0l+fTuJdzroI6mQPvupbiYp8RuuX/7cg38ZhE
x7aqS5IYPC1bpGYIl7yy/KPBF0Z6xgV1PWzbVvxzXII
-> ssh-ed25519 r+nK/Q xTImKbcn53ib31IDa/Q8LA1E7doiY+I2b6RkH7At/kI
iNYdewo4J2o0QrnViG5E4gsXEkKPeWFoT/6WiU11njE
-> ssh-rsa krWCLQ
T/J5iYdgBoWsf8jgWForcwBq0MJhGZgMNdATzsAoa77eV12FV9nDgfLiJzU4WTRA
FLruL0h/5uqT2wzZri8AC3acHsR9iAScz+GK20EX2iokNciwYZl6vRlcDIfaQkyz
94xJrC0Rpo6K1zdICi3A9wtn/CYxRD5bo7MfJskbPmF3WwieiR2Fg1Oam7AZzwRm
eOCj9yNgNjI1vTIB5ioaPpB+nUSRSri6ovTpTX2ycZE/QRPjZiK0G6aPwDNF3GxQ
Bjs4l8ko0YdaPGvst5HGSSuycYDbJPi6+qEZKKAQtSHqoboiTpFd/8nhrOGBsf+x
eFPoTDPRY2WV3rVr4wZBYg
-> ssh-ed25519 /vwQcQ V/OLrAVr+KFWZ76alax3vFvdcb4A9oVbglgz9qqUozQ
jPnZPOGnMWV8njpLD1F6i+8Ls/HTv+EkGYV2JQVE8Ik
-> ssh-ed25519 0R97PA rbjVky5a9248+2JkFtyFrD0++B+GWAJHdZSDVmgl3hg
5pyYsn9W3oFZwbT7oBHgN36KF+4rLT5F7aSqRI0Q7ko
-> ssh-ed25519 JGx7Ng irOl4ZTWNXSM/WF202o+K8tg/hoirSxnfhhZnA4o+0U
jxIwKHqklEIdp+rSglvnY425LH/UDcIWcl3BobbOJFY
--- Sr6TXmrdeYpLMuEe3xDH2RYittqf2lqHzwakvlOohY4
yš¶%$†'[‡>ÿ´F3T"#þN}2Of3]e¨>*¢{Ð5tÖQ0M»<NÜZØöw8w:<3A>û‚ƾê8ø^‰ý»üÍÄ2¥OÔb<C394>k‚w雟aZÇQÖèà/8^¢sôjfÔíp/ù3–Y&Pˆª¾XÊ$

4
machines/nixos/storage01/secrets/secrets.nix
View file

@ -21,8 +21,6 @@
"peertube-secrets_file"
"peertube-service_environment_file"
"peertube-smtp_password_file"
"prometheus-garage_api"
"prometheus-uptime-kuma-apikey"
"prometheus-web_config_file"
"vmagent-garage_api"
"tvix-store-infra-signing-key"
]

BIN
machines/nixos/storage01/secrets/vmagent-garage_api Normal file
View file

Binary file not shown.

4
machines/nixos/vault01/ups.nix
View file

@ -6,7 +6,6 @@
config,
lib,
pkgs,
meta,
name,
...
}:
@ -94,9 +93,10 @@
services.prometheus.exporters.nut = {
enable = true;
listenAddress = meta.network.${name}.netbirdIp;
port = 9199;
};
dgn-monitoring.scrapeConfigs.nut.metrics_path = "/ups_metrics";
networking.firewall.interfaces.wt0.allowedTCPPorts = [ 9199 ];
}

1
meta/dns.nix
View file

@ -106,7 +106,6 @@ let
"git" # Forgejo
"influx" # InfluxDB
"netbird" # Netbird
"prometheus" # Prometheus
"victoria-metrics" # Victoria Metrics
"videos" # Peertube
"pub"

60
modules/nixos/dgn-monitoring/default.nix
View file

@ -5,6 +5,7 @@
{
config,
lib,
pkgs,
meta,
name,
nodeMeta,
@ -14,12 +15,17 @@
let
inherit (lib)
filterAttrs
mapAttrs
mapAttrsToList
mkDefault
mkEnableOption
mkForce
mkIf
mkOption
;
inherit (lib.types) attrsOf;
cfg = config.dgn-monitoring;
in
@ -30,9 +36,38 @@ in
enable = mkEnableOption "the DGNum monitoring system" // {
default = true;
};
scrapeConfigs = mkOption {
type = attrsOf (pkgs.formats.yaml { }).type;
description = ''
Specifications of `scrape_config` sections.
'';
};
};
config = mkIf cfg.enable {
dgn-monitoring.scrapeConfigs =
mapAttrs
(_: cfg: {
static_configs = mkDefault [ { targets = [ "127.0.0.1:${builtins.toString cfg.port}" ]; } ];
})
(
filterAttrs (
name: cfg:
!(builtins.elem name [
"assertions"
"warnings"
"blackbox"
"unifi-poller"
"domain"
"minio"
"idrac"
"pve"
"tor"
])
&& cfg.enable
) config.services.prometheus.exporters
);
services.vmagent = {
enable = true;
@ -42,29 +77,8 @@ in
};
prometheusConfig = {
scrape_configs =
mapAttrsToList
(job_name: cfg: {
inherit job_name;
static_configs = [ { targets = [ "127.0.0.1:${builtins.toString cfg.port}" ]; } ];
})
(
filterAttrs (
name: cfg:
!(builtins.elem name [
"assertions"
"warnings"
"blackbox"
"unifi-poller"
"domain"
"minio"
"idrac"
"pve"
"tor"
])
&& cfg.enable
) config.services.prometheus.exporters
);
scrape_configs = mapAttrsToList (job_name: value: value // { inherit job_name; }) cfg.scrapeConfigs;
global = {
scrape_interval = "15s";
external_labels.hostname = "${name}.${nodeMeta.site}.infra.dgnum.eu";