feat(monitoring): Add uptime-kuma to prometheus targets

machines/storage01/prometheus.nix

@@ -1,8 +1,26 @@
-{ config, ... }:
+{
+  config,
+  nodes,
+  lib,
+  ...
+}:
 
 let
   host = "prometheus.dgnum.eu";
   port = 9091;
+
+  nodeExporterConfigs = lib.flatten (
+    lib.mapAttrsToList (
+      node:
+      { config, ... }:
+      lib.optional config.dgn-node-monitoring.enable {
+        targets = [ "${node}.dgnum:${builtins.toString config.dgn-node-monitoring.port}" ];
+        labels = {
+          host = node;
+        };
+      }
+    ) nodes
+  );
 in
 
 {
@@ -20,11 +38,32 @@ in
 
     webExternalUrl = "https://${host}";
 
+    retentionTime = "6m";
+
+    extraFlags = [ "--storage.tsdb.retention.size=20GB" ];
+
+    globalConfig = {
+      scrape_interval = "15s"; # if you change this settings, please do it in grafana also
+    };
+
     scrapeConfigs = [
       {
         job_name = "prometheus";
         static_configs = [ { targets = [ "localhost:9090" ]; } ];
       }
+      {
+        job_name = "node_exporter";
+        static_configs = nodeExporterConfigs;
+      }
+      {
+        job_name = "uptime_kuma";
+        scheme = "https";
+        static_configs = [ { targets = [ "status.dgnum.eu" ]; } ];
+        basic_auth = {
+          username = "prometheus";
+          password_file = config.age.secrets."prometheus-uptime-kuma-apikey".path;
+        };
+      }
     ];
   };
 

machines/storage01/secrets/secrets.nix

@@ -17,4 +17,5 @@ lib.setDefault { inherit publicKeys; } [
   "peertube-service_environment_file"
   "peertube-smtp_password_file"
   "prometheus-web_config_file"
+  "prometheus-uptime-kuma-apikey"
 ]

machines/web02/_configuration.nix

@@ -8,6 +8,7 @@ lib.extra.mkConfig {
 
   enabledServices = [
     # List of services to enable
+    "monitoring"
   ];
 
   extraConfig = {

machines/web02/monitoring.nix

@@ -0,0 +1 @@
+{ dgn-node-monitoring.enable = false; }

modules/default.nix

@@ -43,6 +43,7 @@
       "dgn-console"
       "dgn-fail2ban"
       "dgn-hardware"
+      "dgn-node-monitoring"
       "dgn-notify"
       "dgn-netbox-agent"
       "dgn-network"

modules/dgn-node-monitoring.nix

@@ -0,0 +1,43 @@
+{ config, lib, ... }:
+let
+  inherit (lib)
+    mkEnableOption
+    mkIf
+    mkOption
+
+    types
+    ;
+  cfg = config.dgn-node-monitoring;
+in
+
+{
+  options.dgn-node-monitoring = {
+    enable = mkEnableOption "DGNum nodes monitoring (needs a valid netbird tunnel)" // {
+      default = true;
+    };
+    port = mkOption {
+      type = types.port;
+      default = 9002;
+      description = lib.mdDoc ''
+        Port to listen on.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.prometheus = {
+      exporters = {
+        node = {
+          enable = true;
+          enabledCollectors = [
+            "processes"
+            "systemd"
+          ];
+          inherit (cfg) port;
+          listenAddress = "0.0.0.0";
+        };
+      };
+    };
+    networking.firewall.interfaces.wt0.allowedTCPPorts = [ cfg.port ];
+  };
+}