infrastructure/machines/core-services-01/monitoring.nix

{ config, pkgs, lib, ... }:
let
  my = config.my;
  realm = "ClubReseau";
  mkChildNode = { uuid, allowFrom }: { ... }@options: ''
    [${uuid}]
      enabled = yes
      default history = 10000
      default memory mode = dbengine
      health enabled by default = auto
      allow from = ${allowFrom}
  '';
  testClusterHypervisors = lib.attrValues {
    pve01 = {
      uuid = "ff9a34ec-2bf4-4389-a01a-6e242424e675";
      allowFrom = "*";
      # allowFrom = "fd85:27e8:0fc9::2";
    };
    pve02 = {
      uuid = "ed393d76-e325-48c4-be90-3d7a1d3066ee";
      allowFrom = "*";
      # allowFrom = "fd85:27e8:0fc9::3";
    };
    pve03 = {
      uuid = "abeeab1f-d4f4-4ca7-aabb-54ff28031f82";
      allowFrom = "*";
      # allowFrom = "fd85:27e8:0fc9::4";
    };
    pve04 = {
      uuid = "ee0f7cec-86f8-4fa2-8258-f7bf4172eb4b";
      allowFrom = "*";
      # allowFrom = "fd85:27e8:0fc9::5";
    };
  };
in
{
  services.netdata = {
    enable = true;
    package = pkgs.netdata.overrideAttrs (old: {
      version = "1.35.0-nightly";
      src = pkgs.fetchFromGitHub {
        owner = "netdata";
        repo = "netdata";
        rev = "955d1a7c61f4eab802648d563f84d978ebeb8fc4";
        sha256 = "sha256-gN44Df+k384NummsHls2JY1CinBaFK3cpdmmdZMbA44=";
        fetchSubmodules = true;
      };
    });
  };

  # Allow WireGuard VPN
  networking.firewall.allowedUDPPorts = [ 51820 ];
  # Allow access to the raw netdata
  networking.firewall.interfaces.wgmon.allowedUDPPorts = [ 19999 ];
  networking.firewall.interfaces.wgmon.allowedTCPPorts = [ 19999 ];
  networking.wireguard.interfaces.wgmon = {
    ips = [ "fd85:27e8:0fc9::1/48" ];

    listenPort = 51820;

    privateKeyFile = "/etc/secrets/wgmon";
    generatePrivateKeyFile = true;

    peers = [
      { publicKey = "6IHA4e+UcCSx9+e5BZwLvzeZv5RWwqO1CCLJedN2nU4="; allowedIPs = [ "fd85:27e8:fc9::2/128" ]; }
      { publicKey = "xRdfylDpi8c+BRwDCxenRs6i4XWesdd75keWfKItZFo="; allowedIPs = [ "fd85:27e8:fc9::3/128" ]; }
      { publicKey = "rjodopHTEyD+DyDsNp8xyNC0KeZGH462Ls495NXT1VI="; allowedIPs = [ "fd85:27e8:fc9::4/128" ];}
      { publicKey = "IJRsrhzCRAHpaEHLZRNdPuDp25FXzuAm+CGmZDsRThk="; allowedIPs = [ "fd85:27e8:fc9::5/128" ]; }
      { publicKey = "oYsN1Qy+a7dwVOKapN5s5KJOmhSflLHZqh+GLMeNpHw="; allowedIPs = [ "fd85:27e8:fc9::6/128" ]; }
      # { publicKey = ""; allowedIPs = [ "fd85:27e8:fc9::7/128" ]; }
    ];
  };

  systemd.services.netdata.restartTriggers = map (v: config.environment.etc."netdata/${v}.conf".source) [
    "netdata"
    "stream"
    "health_alarm_notify"
  ];

  environment.etc."netdata/netdata.conf" = lib.mkForce {
    user = "netdata";
    group = "netdata";
    mode = "0600";
    text = ''
      [db]
      mode = dbengine
      storage tiers = 3
      update every = 1
      dbengine multihost disk space MB = 23000
      dbengine page cache size MB = 384
      dbengine tier 1 update every iterations = 60
      dbengine tier 1 multihost disk space MB = 10000
      dbengine tier 1 page cache size MB = 384
      dbengine tier 2 update every iterations = 3600
      dbengine tier 2 multihost disk space MB = 5000
      dbengine tier 2 page cache size MB = 384
    '';
  };

  environment.etc."netdata/stream.conf" = {
    user = "netdata";
    group = "netdata";
    mode = "0600";
    text = (lib.concatMapStringsSep "\n" (cfg: mkChildNode cfg {})
      ([
        # PVE01 hypervisor
        {
          uuid = "e245097d-bf52-4f66-9c10-984e8d5ee178";
          allowFrom = "10.1.1.10";
        }
        # Public COF server
        {
          uuid = "c48e6ef1-5cdf-408d-ae2f-86aadb14e3fe";
          allowFrom = "10.1.1.21";
        }
      ] ++ testClusterHypervisors));
  };

  environment.etc."netdata/health_alarm_notify.conf" = {
    user = "netdata";
    group = "netdata";
    mode = "0600";
    text = ''
      # External tools
      nc="${pkgs.netcat}/bin/nc"

      # IRC configuration
      SEND_IRC="YES"
      DEFAULT_RECIPIENT_IRC="#réseau"
      IRC_NETWORK="ens.wtf"
      IRC_NICKNAME="core-services-01"
      IRC_REALNAME="KlubRZ Core Services 01"
    '';
  };

  services.oauth2_proxy = {
    enable = true;
    keyFile = config.age.secrets.oauth2ProxyKeyFile.path;
    provider = "keycloak-oidc";
    email.domains = [ "*" ];
    cookie = {
      name = "_oauth2_proxy_ensrz";
      domain = ".rz.ens.wtf";
    };
    setXauthrequest = true;
    scope = "openid";
    redirectURL = "https://monitoring.${my.subZone}/oauth2/callback";
    reverseProxy = true;
    passHostHeader = true;

    extraConfig = {
      whitelist-domain = [ ".rz.ens.wtf" ];
      oidc-issuer-url = "https://auth.${my.subZone}/auth/realms/${realm}";
      # insecure-oidc-allow-unverified-email = true;
      show-debug-on-error = true;
    };

    nginx = {
      virtualHosts = [ "monitoring.${my.subZone}" ];
    };
  };

  services.nginx = {
    enable = true;

    virtualHosts."monitoring.${my.subZone}" = {
      enableACME = true;
      forceSSL = true;

      locations."/" = {
        proxyPass = "http://localhost:19999";
        extraConfig = ''
          # For large authentication-authorization headers
          proxy_buffer_size          128k;
          proxy_buffers              4 256k;
          proxy_busy_buffers_size    256k;
        '';
      };
    };
  };

  # services.smartd = {
  #   enable = true;
  #   extraOptions = [ "-A /var/log/smartd/" ]; # For netdata
  # };
}