infrastructure/machines/storage01/prometheus.nix

{
  config,
  nodes,
  lib,
  ...
}:

let
  host = "prometheus.dgnum.eu";
  port = 9091;

  nodeExporterConfigs = lib.flatten (
    lib.mapAttrsToList (
      node:
      { config, ... }:
      lib.optional config.dgn-node-monitoring.enable {
        targets = [ "${node}.dgnum:${builtins.toString config.dgn-node-monitoring.port}" ];
        labels = {
          host = node;
        };
      }
    ) nodes
  );
in

{
  services.prometheus = {
    enable = true;

    inherit port;

    checkConfig = "syntax-only";
    enableReload = true;

    listenAddress = "127.0.0.1";

    webConfigFile = config.age.secrets."prometheus-web_config_file".path;

    webExternalUrl = "https://${host}";

    retentionTime = "1y";

    extraFlags = [ "--storage.tsdb.retention.size=20GB" ];

    globalConfig = {
      scrape_interval = "15s"; # if you change this settings, please do it in grafana also
    };

    scrapeConfigs = [
      {
        job_name = "prometheus";
        static_configs = [ { targets = [ "localhost:9090" ]; } ];
      }
      {
        job_name = "node_exporter";
        static_configs = nodeExporterConfigs;
      }
      {
        job_name = "uptime_kuma";
        scheme = "https";
        static_configs = [ { targets = [ "status.dgnum.eu" ]; } ];
        basic_auth = {
          username = "prometheus";
          password_file = config.age.secrets."prometheus-uptime-kuma-apikey".path;
        };
      }
      {
        job_name = "hyp01_ups";
        metrics_path = "/ups_metrics";
        static_configs = [ { targets = [ "100.80.255.180:9199" ]; } ];
      }
      {
        job_name = "garage";
        static_configs = [ { targets = [ "localhost:3903" ]; } ];
        bearer_token_file = config.age.secrets."prometheus-garage_api".path;
      }
    ];
  };

  services.nginx.virtualHosts.${host} = {
    enableACME = true;
    forceSSL = true;

    locations."/" = {
      proxyPass = "http://127.0.0.1:${builtins.toString port}";
      proxyWebsockets = true;
      recommendedProxySettings = true;
    };
  };

  age-secrets.autoMatch = [ "prometheus" ];
}