feat(monitoring): Add simple systemd monitoring

modules/default.nix

@@ -43,6 +43,7 @@
       "dgn-console"
       "dgn-fail2ban"
       "dgn-hardware"
+      "dgn-notify"
       "dgn-netbox-agent"
       "dgn-network"
       "dgn-records"
@@ -60,5 +61,6 @@
       "services/crabfit"
       "services/forgejo-nix-runners"
       "services/bupstash"
+      "services/systemd-notify"
     ]);
 }

modules/dgn-notify/default.nix

@@ -0,0 +1,46 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+{
+
+  options.dgn-notify = {
+    enable = lib.mkEnableOption "DGNum email notification cli" // {
+      default = true;
+    };
+  };
+  config = lib.mkIf config.dgn-notify.enable {
+    programs.msmtp = {
+      enable = true;
+      setSendmail = false;
+      accounts.default = {
+        tls = true;
+        tls_starttls = false;
+        port = 465;
+        auth = true;
+        host = "kurisu.lahfa.xyz";
+        from = "noreply@infra.dgnum.eu";
+        user = "web-services@infra.dgnum.eu";
+        passwordeval = "cat ${config.age.secrets.mail.path}";
+      };
+    };
+    services.systemd-notify = {
+      enable = true;
+      command = builtins.toString (
+        pkgs.writeShellScript "sendmail" ''
+          ${pkgs.msmtp}/bin/sendmail -i -t <<ERRMAIL
+          To: admins+monitoring@dgnum.eu
+          Subject: [$HOSTNAME] Systemd failure: $1
+          Content-Transfer-Encoding: 8bit
+          Content-Type: text/plain; charset=UTF-8
+
+          $(systemctl status --full "$1")
+          ERRMAIL
+        ''
+      );
+    };
+    age-secrets.sources = [ ./. ];
+  };
+}

modules/dgn-notify/mail

@@ -0,0 +1,42 @@
+age-encryption.org/v1
+-> ssh-ed25519 jIXfPA OQPfJlhjA2mE6CtPownzhUh4yUYb+he03z2/K+PcniA
+vti9qw1oGB7EDqo2SYPwvPAnhQKzW1NVYMQaYSYaN/Q
+-> ssh-ed25519 QlRB9Q PUeKoFYp0epk8zczDrSCGTIfYZaX7kHA2PIeY8YcvR0
+vyRB+qsAZ3kFNzPECH5FkAgeMk1o6IXRiiaat/UJuL8
+-> ssh-ed25519 r+nK/Q 4Sa0MIETabNeIIGyZ6WEZ154bi6gj+BwlneBsHvLmVo
+L01kFjZu50Npncn/QEfMQLNv1ljIYnBzI+9XtjFM2yM
+-> ssh-rsa krWCLQ
+o62CV+7mRx20Y3Q7lqeorquJ29cNSdwgQzuEysNeFMm03xANdSZeZ9TRvWUMx5y5
+SPhv5EMorNf+A5LbfElI3j6wKGgE2g0iIdDNT8xS2ySfmXeVVrT/hAY9cTYOx3GB
+4XwopgSqhcyywMaLll7KBivj/mr9UYdX1qyi7SfNsWocKSpkNF8Pe1YRCTU47sYS
+uxIEGbBcPTDfIGPveLHT9bBZYpbttGsnjnIe6I079Om88xjfpo8qZwAGlzpF9hXQ
+vMChPrRwQvMwhUrvNAFiwY4e3X1oqIcv/KSqV0gslF2VhvzwDcbbEiCOkM6TQHj/
+RpiDdNkiY9lBmOZg3wNjTw
+-> ssh-ed25519 /vwQcQ 40PBWkw+DULVemPULITTSKk7zGOD7rN97A/crAostgk
+loDeyiqPoeUSFNtxmkWTgy5cyqVtGngr3OoSlHlRiTc
+-> ssh-ed25519 0R97PA xN+zMW9dexNxODhcbs2FrgxBDtWwunt9pUtPi4pJbwM
+5Pzw5LS4Bb1ksCQhPv3qvXe/TvKvrf7XBdKdFbldaow
+-> ssh-ed25519 JGx7Ng 7eYBQBzo9sm/5W/lO1gVamQBDf/WsguFZGFuvXeYiSM
+dFPjeIi1YyfHgssP7CPhlAHXLrQ7iBEafBVxAfMIqtU
+-> ssh-ed25519 5SY7Kg mC8h039gQX/u0EwcH/DJBxpOxiMZsX/wX8ZAYLOzCUE
+PrYbbegyx7WRPRQaK9nKkHtNfSH/zlgjeGMlNoDbagQ
+-> ssh-ed25519 p/Mg4Q 6bLGoqivdYz4gCbaU4cwmQbI+KVdTDGYYjQ/SCLLRU0
+O1R7w4ZTVlbw0p7IFQRuvu13eNxzJr59xKRKPJbJbsU
+-> ssh-ed25519 tDqJRg 6rbXSrt5nDpP55Y0MwQFuXbdPkgC7JP8VdycsTLf03k
+cIV19Jdp1rNZoo0z+yFwtXStsyOiXHCq5ztlrejf8Qk
+-> ssh-ed25519 9pVK7Q gzkXr47zOzLsGnK2gESaitvWpLVViZ8wJPNWDe36ah0
+jJdXBcFPjz9jPsSqJAxB/Iya/l/O3w96MbJK7f1aHfE
+-> ssh-ed25519 /BRpBQ 9Swk1Oio1fLYNCX5I1XvdDHMkFHfmgfMpGF3FSsawU4
+7Ab6e40dnUcMdC7fAjFUY3KNDxw8GmtkO1gmni1G0aI
+-> ssh-ed25519 +MNHsw P3hMfxkpPDJgEbTHRc2C//irQVhL1vTHa+VXCiqY/xI
+VYFCdH0aSVLoWi9GXyo92sUvn4hzF9ROkYhE3BRmGDg
+-> ssh-ed25519 rHotTw Z9R0jIPKxEYIKIYU1sfxvYSQKSnBZJWt2cBvQDn+lWA
+tJVyOEmpVx3EYLMznQjlFzyJ9JqJzLmSXdogFjcLTaM
+-> ssh-ed25519 +mFdtQ oU3LPyIpaRkFKV4cOfoLvvELpzFrHr7ap3lc1SnKP2s
+p001TBOwpMUhtRUoMQHLQ0FC3VWG3JIssJokJsVMwr8
+-> ssh-ed25519 0IVRbA e93ze1tPZJxX70VFUHjPFzLcaSpMrcg/z/hWOaO2cU4
+9LYn8LHBVvpeuAFavlE9WKS2+yYdmlSUVZfPpHCqExY
+-> ssh-ed25519 8V9fnQ YFio60+aQsv99gPWKurFdQ5GU2tnTU+k0sUcZLoIo0c
+FJrDp468+s+DSbn7QhXPa0qQJFwEi8MgCc+IPc8gbsU
+--- P4Ebv9iP0ZaqpuQg0N0jE4H3ebL4zYkoFIIJ8xtjTpA
+S*‡eñ—Â3Qé!
T'X¼Ú·´¾<C2B4>w1vž»ËnÆvws¾úäLó9-»k*€Öı:耤/»o@îöù—J

modules/dgn-notify/secrets.nix

@@ -0,0 +1 @@
+{ mail.publicKeys = (import ../../lib { }).machineKeys; }