tvl-depot/users/sterni/machines/ingeborg/monitoring.nix

{ pkgs, lib, config, ... }:

let
  ircChannel = "#sterni.lv";
  irccatPort =
    builtins.replaceStrings [ ":" ] [ "" ]
      config.services.depot.irccat.config.tcp.listen;

  mkIrcMessager =
    { name
    , msgExpr
    }:
    pkgs.writeShellScript name ''
      set -euo pipefail
      printf '%s %s\n' ${lib.escapeShellArg ircChannel} ${msgExpr} | \
        ${lib.getBin pkgs.netcat-openbsd}/bin/nc -N localhost ${irccatPort}
    '';

  netdataPort = 19999;
in

{
  imports = [
    ./irccat.nix
  ];

  config = {
    services.depot.irccat.config.irc.channels = [
      ircChannel
    ];

    # Since we have irccat we can wire up mdadm --monitor
    boot.swraid.mdadmConf = ''
      PROGRAM ${
        mkIrcMessager {
          name = "mdmonitor-to-irc";
          # prog EVENT MD_DEVICE COMPONENT_DEVICE
          msgExpr = ''"mdmonitor: $1($2''${3:+, $3})"'';
        }
      }
    '';

    # TODO(sterni): irc notifications (?)
    services = {
      smartd = {
        enable = true;
        autodetect = true;
        # Short self test every day 03:00
        # Long self test every tuesday 05:00
        defaults.autodetected = "-a -o on -s (S/../.././03|L/../../2/05)";
        extraOptions = [
          "-A"
          "/var/log/smartd/"
        ];
      };

      netdata = {
        enable = true;
        config = {
          logs = {
            access = "syslog";
            error = "syslog";
            debug = "syslog";
            health = "syslog";
            collector = "syslog";
          };
          web = {
            "default port" = toString netdataPort;
            "bind to" = "localhost:${toString netdataPort}";
          };
          health = {
            "script to execute on alarm" = pkgs.writeShellScript "simple-alarm-notify" ''
              set -euo pipefail

              # This humongous list is copied over from netdata's alarm-notify.sh
              roles="''${1}"               # the roles that should be notified for this event
              args_host="''${2}"           # the host generated this event
              unique_id="''${3}"           # the unique id of this event
              alarm_id="''${4}"            # the unique id of the alarm that generated this event
              event_id="''${5}"            # the incremental id of the event, for this alarm id
              when="''${6}"                # the timestamp this event occurred
              name="''${7}"                # the name of the alarm, as given in netdata health.d entries
              chart="''${8}"               # the name of the chart (type.id)
              status="''${9}"              # the current status : REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
              old_status="''${10}"         # the previous status: REMOVED, UNINITIALIZED, UNDEFINED, CLEAR, WARNING, CRITICAL
              value="''${11}"              # the current value of the alarm
              old_value="''${12}"          # the previous value of the alarm
              src="''${13}"                # the line number and file the alarm has been configured
              duration="''${14}"           # the duration in seconds of the previous alarm state
              non_clear_duration="''${15}" # the total duration in seconds this is/was non-clear
              units="''${16}"              # the units of the value
              info="''${17}"               # a short description of the alarm
              value_string="''${18}"       # friendly value (with units)
              # shellcheck disable=SC2034
              # variable is unused, but https://github.com/netdata/netdata/pull/5164#discussion_r255572947
              old_value_string="''${19}"   # friendly old value (with units), previously named "old_value_string"
              calc_expression="''${20}"    # contains the expression that was evaluated to trigger the alarm
              calc_param_values="''${21}"  # the values of the parameters in the expression, at the time of the evaluation
              total_warnings="''${22}"     # Total number of alarms in WARNING state
              total_critical="''${23}"     # Total number of alarms in CRITICAL state
              total_warn_alarms="''${24}"  # List of alarms in warning state
              total_crit_alarms="''${25}"  # List of alarms in critical state
              classification="''${26}"     # The class field from .conf files
              edit_command_line="''${27}"  # The command to edit the alarm, with the line number
              child_machine_guid="''${28}" # the machine_guid of the child
              transition_id="''${29}"      # the transition_id of the alert
              summary="''${30}"            # the summary text field of the alert

              # Verify that they haven't extended the arg list
              ARG_COUNT_EXPECTED=30

              if [[ "$#" != "$ARG_COUNT_EXPECTED" ]]; then
                echo "$0: WARNING: unexpected number of arguments: $#. Did netdata add more?" >&2
              fi

              MSG="netdata: $status ''${name//_/ } ($chart): ''${summary//_/ } = $value_string"

              # Filter rules by chart name. This is necessary, since the "enabled alarms"
              # filter only allows for filtering alarm types, not specific alarms
              # belonging to that alarm.
              case "$chart" in
                # netdata prefers the automatically assigned names (dm-<n>, md<n>,
                # sd<c>) over ids for alerts, so this configuration assumes that
                # we have two physical disks which we kind of assert using the
                # grub configuration (it is more difficult with the soft raid
                # config).
                # ${assert builtins.length config.boot.loader.grub.devices == 2; ""}
                disk_util.sda | disk_util.sdb | disk_backlog.sda | disk_backlog.sdb)

                  ;;
                disk_util.* | disk_backlog.*)
                  echo "$0: INFO: DISCARDING message: $MSG" >&2
                  exit 0
                  ;;
                *)
                  ;;
              esac

              echo "$0: INFO: sending message: $MSG" >&2
              ${
                mkIrcMessager {
                  name = "trivial-send-to-irc";
                  msgExpr = "\"$1\"";
                }
              } "$MSG"
            '';
          };
        };
      };
    };
  };
}