infrastructure/modules/nixos/dgn-monitoring/default.nix

# SPDX-FileCopyrightText: 2024 Tom Hubrecht <tom.hubrecht@dgnum.eu>
#
# SPDX-License-Identifier: EUPL-1.2

{
  config,
  lib,
  pkgs,
  meta,
  name,
  nodeMeta,
  ...
}:

let
  inherit (lib)
    filterAttrs
    mapAttrs
    mapAttrsToList
    mkDefault
    mkEnableOption
    mkForce
    mkIf
    mkOption
    ;

  inherit (lib.types) attrsOf;

  cfg = config.dgn-monitoring;
in

{
  imports = [ ./exporters.nix ];

  options.dgn-monitoring = {
    enable = mkEnableOption "the DGNum monitoring system" // {
      default = true;
    };
    scrapeConfigs = mkOption {
      type = attrsOf (pkgs.formats.yaml { }).type;
      description = ''
        Specifications of `scrape_config` sections.
      '';
    };
  };

  config = mkIf cfg.enable {
    dgn-monitoring.scrapeConfigs =
      mapAttrs
        (_: cfg: {
          static_configs = mkDefault [ { targets = [ "127.0.0.1:${builtins.toString cfg.port}" ]; } ];
        })
        (
          filterAttrs (
            name: cfg:
            !(builtins.elem name [
              "assertions"
              "warnings"
              "blackbox"
              "unifi-poller"
              "domain"
              "minio"
              "idrac"
              "pve"
              "tor"
            ])
            && cfg.enable
          ) config.services.prometheus.exporters
        );

    services.vmagent = {
      enable = true;

      flags = {
        "remoteWrite.url" = "http://${meta.network.storage01.netbirdIp}:8428/api/v1/write";
        "remoteWrite.label" = "node=${name}";
      };

      prometheusConfig = {
        scrape_configs = mapAttrsToList (job_name: value: value // { inherit job_name; }) cfg.scrapeConfigs;

        global = {
          scrape_interval = "15s";
          external_labels.hostname = "${name}.${nodeMeta.site}.infra.dgnum.eu";
        };
      };
    };

    services.journald.upload = {
      enable = true;
      settings = {
        Upload.URL = "http://${meta.network.storage01.netbirdIp}:9428/insert/journald";
      };
    };

    # Don't restart too often to reduce e-mail notifications when the network or the database is down
    systemd.services.systemd-journal-upload.serviceConfig.RestartSec = mkForce 60;
  };
}