feat(monitoring): drop prometheus in favor of victorialogs

machines/nixos/rescue01/secrets/secrets.nix

@@ -8,4 +8,5 @@
     # List of secrets for rescue01
     "netbird-relay_environment_file"
     "stateless-uptime-kuma-password"
+    "vmagent-uptime_kuma_apikey"
   ]

machines/nixos/rescue01/secrets/vmagent-uptime_kuma_apikey

@@ -0,0 +1,28 @@
+age-encryption.org/v1
+-> ssh-ed25519 jIXfPA kqyRXA2pHUxClljkjBwyvVVlfCvw1I+t/KHeulAZ9VA
+YNQSmZA3sdAQUk49H0Irud0tsA6zbhrSkBRGDe2hlo8
+-> ssh-ed25519 QlRB9Q 5z+dG8KPAuRWYOWrHdSM8shnM9KR3Kz6plfNN0KCBEc
+69u6n8zgnQgS/fLQYJKC7TDPytnjvDT90zpS16T0oOI
+-> ssh-ed25519 r+nK/Q FsjzUCGKyJb4/SQRkB8dEicKfXEjHC04t0o2PmHAKjY
+pBHTvHER0lFNv2cCFW5uZ6Nc7cMacMyLjdS8c1ojIm0
+-> ssh-rsa krWCLQ
+iTCYGfPEkLvgJ+uW92N7LFPkSI4mC293xns2TV83rtKr7zszYJQa7Xx+bu6KDjB0
+/+3E1pH0o+iMF0PPJNcQ1lBxefnPtsQfgW6RAqgu+DwGW/J13uoKBezliFicTh/u
+uaT5z7At+T6wq/vEdd5KGP9RdSDdfoKbED1tIxLzKdUqWDurayhuRPMr1JLm4Tar
+77oONkDuPFYI8FsyaibFJdqr+outomiAUEu8i0v62Y1WRMwxHmT4sqJaTIiYr5vD
+8dheZ5804mrekXrlis3VDrTBTValrrkq9qs78Ly9QR9z6BNREEje+s6TV3KqaYev
+AomZKYyh+gtdac2caK1WxA
+-> ssh-ed25519 /vwQcQ iFmz9CqY4+SuyspXG4+z0kuO75FKZ0wyBqTmar90T1U
+yAuwWlEozFtSNAlGdXRFwj/uekIuhMkoWVUtBfWfcCs
+-> ssh-ed25519 0R97PA /mXGDXr1Bgxg5RNR0L2vZdo/aXVjl3R89kwLY/8k1Gc
+yCOvDa+Fel7Uv3RURubLTx3yxJ1L7c12yuSmD2HIMus
+-> ssh-ed25519 JGx7Ng 9q8YiaK/VKpY8f8OUISi6GU4h1j178T/inCZAemchgk
+J2ODND+4nw6r5sXWMndb/CT3GHIaBTvKYolHd29Cfko
+-> ssh-ed25519 bUjjig PNhrLR1pDruxTL/i/tNT2zPSNK3GfjiuaFC2YY4qelg
+ISupvlviQKPbgrD3+sr+i9yWmpsDJzpAc/9XJqD9ulY
+-> ssh-ed25519 +MNHsw GrdwnnaZWy8s5nsy8EC52F87JmuLWDjAKjLCJBKyqw0
+BmYMybzWs4a3TvgDs61RzNgjhDLyioVnfFDrxcUhx2U
+-> ~8@'-grease 8p8,zY xRg`tl F+$W!$n {n/
+CpBIH2q+yIcEz8MR
+--- 2d05zWr4b0J/wzQE+Xo9mOvwLtkZPqfotYCrWRPzMnM
+ÜÇ,Õi7"ë<>Ž†òSÀknp¡ª¼‘Òî÷âÚÙ¸`K}<7D>ìðž4ÍÄ	˜6uàá²d0_Ï,[¥3ùkÐ1Ÿ<31>„s<k¨õ«?c

machines/nixos/rescue01/uptime-kuma.nix

@@ -47,11 +47,6 @@ let
 
   extraProbes = {
     monitors = {
-      "prometheus.dgnum.eu" = {
-        type = mkForce "http";
-        accepted_statuscodes = [ "401" ];
-      };
-
       "ollama01.beta.dgnum.eu" = {
         type = mkForce "http";
         accepted_statuscodes = [ "401" ];
@@ -159,4 +154,21 @@ in
     passwordFile = config.age.secrets."stateless-uptime-kuma-password".path;
     enableService = true;
   };
+
+  services.vmagent.prometheusConfig = {
+    scrape_configs = [
+      {
+        job_name = "uptime_kuma";
+        scheme = "http";
+        static_configs = [ { targets = [ "localhost:${builtins.toString port}" ]; } ];
+        basic_auth = {
+          username = "prometheus";
+          password_file = "%{CREDENTIALS_DIRECTORY}/uptime_kuma_apikey";
+        };
+      }
+    ];
+  };
+  systemd.services.vmagent.serviceConfig.LoadCredential = [
+    "uptime_kuma_apikey:${config.age.secrets."vmagent-uptime_kuma_apikey".path}"
+  ];
 }

machines/nixos/storage01/_configuration.nix

@@ -22,7 +22,6 @@ lib.extra.mkConfig {
     "netbird"
     "openbao"
     "peertube"
-    "prometheus"
     "redirections"
     "victorialogs"
     "victoriametrics"

machines/nixos/storage01/garage.nix

@@ -3,6 +3,7 @@
 # SPDX-FileContributor: Ryan Lahfa <ryan.lahfa@dgnum.eu>
 #
 # SPDX-License-Identifier: EUPL-1.2
+{ config, ... }:
 
 let
   host = "s3.dgnum.eu";
@@ -51,50 +52,65 @@ in
     metadata_dir = "/data/fast/garage/meta";
   };
 
-  services.garage.settings = {
-    s3_api.root_domain = ".${host}";
-    s3_web.root_domain = ".${webHost}";
-  };
-
-  services.nginx.virtualHosts = {
-    "s3-admin.dgnum.eu" = {
-      enableACME = true;
-      forceSSL = true;
-
-      locations."/".extraConfig = ''
-        proxy_pass http://127.0.0.1:${builtins.toString ports.admin_api};
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $host;
-      '';
+  services = {
+    garage.settings = {
+      s3_api.root_domain = ".${host}";
+      s3_web.root_domain = ".${webHost}";
     };
 
-    ${host} = {
-      enableACME = true;
-      forceSSL = true;
+    nginx.virtualHosts = {
+      "s3-admin.dgnum.eu" = {
+        enableACME = true;
+        forceSSL = true;
 
-      serverAliases = mkHosted host buckets;
+        locations."/".extraConfig = ''
+          proxy_pass http://127.0.0.1:${builtins.toString ports.admin_api};
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header Host $host;
+        '';
+      };
+
+      ${host} = {
+        enableACME = true;
+        forceSSL = true;
+
+        serverAliases = mkHosted host buckets;
+
+        locations."/".extraConfig = ''
+          proxy_pass http://127.0.0.1:${builtins.toString ports.s3_api};
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header Host $host;
+          # Disable buffering to a temporary file.
+          proxy_max_temp_file_size 0;
+          client_max_body_size 5G;
+        '';
+      };
+
+      ${webHost} = {
+        enableACME = true;
+        forceSSL = true;
+
+        serverAliases = domains ++ (mkHosted webHost buckets);
+
+        locations."/".extraConfig = ''
+          proxy_pass http://127.0.0.1:${builtins.toString ports.s3_web};
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header Host $host;
+        '';
+      };
 
-      locations."/".extraConfig = ''
-        proxy_pass http://127.0.0.1:${builtins.toString ports.s3_api};
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $host;
-        # Disable buffering to a temporary file.
-        proxy_max_temp_file_size 0;
-        client_max_body_size 5G;
-      '';
     };
-
-    ${webHost} = {
-      enableACME = true;
-      forceSSL = true;
-
-      serverAliases = domains ++ (mkHosted webHost buckets);
-
-      locations."/".extraConfig = ''
-        proxy_pass http://127.0.0.1:${builtins.toString ports.s3_web};
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $host;
-      '';
+    vmagent.prometheusConfig = {
+      scrape_configs = [
+        {
+          job_name = "garage";
+          static_configs = [ { targets = [ "localhost:3903" ]; } ];
+          bearer_token_file = "%{CREDENTIALS_DIRECTORY}/garage_api}";
+        }
+      ];
     };
   };
+  systemd.services.vmagent.serviceConfig.LoadCredential = [
+    "garage_api:${config.age.secrets."vmagent-garage_api".path}"
+  ];
 }

machines/nixos/storage01/prometheus.nix

@@ -1,92 +0,0 @@
-# SPDX-FileCopyrightText: 2024 Maurice Debray <maurice.debray@dgnum.eu>
-#
-# SPDX-License-Identifier: EUPL-1.2
-
-{
-  config,
-  serverNodes,
-  lib,
-  ...
-}:
-
-let
-  host = "prometheus.dgnum.eu";
-  port = 9091;
-
-  nodeExporterConfigs = lib.flatten (
-    lib.mapAttrsToList (
-      node:
-      { config, ... }:
-      lib.optional config.dgn-monitoring.exporters.enable {
-        targets = map (p: "${node}.dgnum:${builtins.toString p}") (
-          builtins.attrValues config.dgn-monitoring.exporters.ports
-        );
-        labels = {
-          host = node;
-        };
-      }
-    ) serverNodes
-  );
-in
-
-{
-  services.prometheus = {
-    enable = true;
-
-    inherit port;
-
-    checkConfig = "syntax-only";
-    enableReload = true;
-
-    listenAddress = "127.0.0.1";
-
-    webConfigFile = config.age.secrets."prometheus-web_config_file".path;
-
-    webExternalUrl = "https://${host}";
-
-    retentionTime = "1y";
-
-    extraFlags = [ "--storage.tsdb.retention.size=20GB" ];
-
-    globalConfig = {
-      scrape_interval = "15s"; # if you change this settings, please do it in grafana also
-    };
-
-    scrapeConfigs = [
-      {
-        job_name = "prometheus";
-        static_configs = [ { targets = [ "localhost:9090" ]; } ];
-      }
-      {
-        job_name = "node_exporter";
-        static_configs = nodeExporterConfigs;
-      }
-      {
-        job_name = "uptime_kuma";
-        scheme = "https";
-        static_configs = [ { targets = [ "status.dgnum.eu" ]; } ];
-        basic_auth = {
-          username = "prometheus";
-          password_file = config.age.secrets."prometheus-uptime-kuma-apikey".path;
-        };
-      }
-      {
-        job_name = "hyp01_ups";
-        metrics_path = "/ups_metrics";
-        static_configs = [ { targets = [ "100.80.255.180:9199" ]; } ];
-      }
-      {
-        job_name = "garage";
-        static_configs = [ { targets = [ "localhost:3903" ]; } ];
-        bearer_token_file = config.age.secrets."prometheus-garage_api".path;
-      }
-    ];
-  };
-
-  dgn-web.simpleProxies.prometheus = {
-    inherit host port;
-    proxyWebsockets = true;
-  };
-
-  age-secrets.autoMatch = [ "prometheus" ];
-}

machines/nixos/storage01/secrets/prometheus-web_config_file

@@ -1,24 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 rHotTw RBYiRMlYSSJqICOKMUSCKb5r5o05rS9WCS4iiC7c1k0
-dWd+dS0UqpFmlSPAsoUPcslPbMhTulikbpzDD9LRCVQ
--> ssh-ed25519 jIXfPA clKVYR1MMXYIjDcmZQuCP69U04wdgyJ0JVWVtT/pkjI
-JygjZZkaJwJ4HIYJL31PEL1+5DPmhh+z94NlO1zrG0E
--> ssh-ed25519 QlRB9Q SYHBZy0l+fTuJdzroI6mQPvupbiYp8RuuX/7cg38ZhE
-x7aqS5IYPC1bpGYIl7yy/KPBF0Z6xgV1PWzbVvxzXII
--> ssh-ed25519 r+nK/Q xTImKbcn53ib31IDa/Q8LA1E7doiY+I2b6RkH7At/kI
-iNYdewo4J2o0QrnViG5E4gsXEkKPeWFoT/6WiU11njE
--> ssh-rsa krWCLQ
-T/J5iYdgBoWsf8jgWForcwBq0MJhGZgMNdATzsAoa77eV12FV9nDgfLiJzU4WTRA
-FLruL0h/5uqT2wzZri8AC3acHsR9iAScz+GK20EX2iokNciwYZl6vRlcDIfaQkyz
-94xJrC0Rpo6K1zdICi3A9wtn/CYxRD5bo7MfJskbPmF3WwieiR2Fg1Oam7AZzwRm
-eOCj9yNgNjI1vTIB5ioaPpB+nUSRSri6ovTpTX2ycZE/QRPjZiK0G6aPwDNF3GxQ
-Bjs4l8ko0YdaPGvst5HGSSuycYDbJPi6+qEZKKAQtSHqoboiTpFd/8nhrOGBsf+x
-eFPoTDPRY2WV3rVr4wZBYg
--> ssh-ed25519 /vwQcQ V/OLrAVr+KFWZ76alax3vFvdcb4A9oVbglgz9qqUozQ
-jPnZPOGnMWV8njpLD1F6i+8Ls/HTv+EkGYV2JQVE8Ik
--> ssh-ed25519 0R97PA rbjVky5a9248+2JkFtyFrD0++B+GWAJHdZSDVmgl3hg
-5pyYsn9W3oFZwbT7oBHgN36KF+4rLT5F7aSqRI0Q7ko
--> ssh-ed25519 JGx7Ng irOl4ZTWNXSM/WF202o+K8tg/hoirSxnfhhZnA4o+0U
-jxIwKHqklEIdp+rSglvnY425LH/UDcIWcl3BobbOJFY
---- Sr6TXmrdeYpLMuEe3xDH2RYittqf2lqHzwakvlOohY4
-yš¶%$†'[‡>ÿ´F3T"#þN}2Of3]e¨>*¢{Ð5tÖQ0M»<NÜZØöw8w:<3A>û‚ƾê8ø^‰ý»üÍÄ2¥OÔb<C394>k‚w雟aZÇQÖèà/8^¢sôjfÔíp/ù3–Y&Pˆª¾XÊ$

machines/nixos/storage01/secrets/secrets.nix

@@ -21,8 +21,6 @@
     "peertube-secrets_file"
     "peertube-service_environment_file"
     "peertube-smtp_password_file"
-    "prometheus-garage_api"
-    "prometheus-uptime-kuma-apikey"
-    "prometheus-web_config_file"
+    "vmagent-garage_api"
     "tvix-store-infra-signing-key"
   ]

machines/nixos/vault01/ups.nix

@@ -6,7 +6,6 @@
   config,
   lib,
   pkgs,
-  meta,
   name,
   ...
 }:
@@ -94,9 +93,10 @@
 
   services.prometheus.exporters.nut = {
     enable = true;
-    listenAddress = meta.network.${name}.netbirdIp;
     port = 9199;
   };
 
+  dgn-monitoring.scrapeConfigs.nut.metrics_path = "/ups_metrics";
+
   networking.firewall.interfaces.wt0.allowedTCPPorts = [ 9199 ];
 }