org: deploy monitoring of clock

machines/hackens-org/_configuration.nix

@@ -20,6 +20,8 @@
       ./static-sites.nix
       ./legacy-redir.nix
       ./webpass.nix
+      ./prometheus.nix
+      ./grafana.nix
     ];
 
   time.timeZone = "Europe/Paris";

machines/hackens-org/grafana.nix

@@ -0,0 +1,59 @@
+{ config, ... }:
+
+let
+  host = "grafana.hackens.org";
+  port = 3033;
+
+in
+
+{
+  services = {
+    grafana = {
+      enable = true;
+
+      settings = {
+        database = {
+          type = "postgres";
+          user = "grafana";
+          host = "/run/postgresql";
+        };
+
+        server = {
+          domain = host;
+          enable_gzip = true;
+          enforce_domain = true;
+          http_port = port;
+          root_url = "https://${host}";
+          router_logging = true;
+        };
+
+        users = {
+          default_theme = "system";
+          default_language = "en-GB";
+        };
+      };
+    };
+
+    postgresql = {
+      enable = true;
+      ensureDatabases = [ "grafana" ];
+      ensureUsers = [
+        {
+          name = "grafana";
+          ensureDBOwnership = true;
+        }
+      ];
+    };
+
+    nginx.virtualHosts.${host} = {
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${builtins.toString port}";
+        proxyWebsockets = true;
+        recommendedProxySettings = true;
+      };
+    };
+  };
+}

machines/hackens-org/prometheus.nix

@@ -0,0 +1,58 @@
+{ config, ... }:
+
+let
+  host = "prometheus.hackens.org";
+  port = 9091;
+in
+
+{
+  services.prometheus = {
+    enable = true;
+
+    inherit port;
+
+    checkConfig = "syntax-only";
+    enableReload = true;
+
+    listenAddress = "127.0.0.1";
+
+    webConfigFile = config.age.secrets."prometheus-webconf".path;
+
+    webExternalUrl = "https://${host}";
+
+    rules = [ ''
+      groups:
+        - name: Chrony
+          rules:
+            - record: instance:chrony_clock_error_seconds:abs
+              expr: >
+                abs(chrony_tracking_last_offset_seconds)
+                +
+                chrony_tracking_root_dispersion_seconds
+                +
+                (0.5 * chrony_tracking_root_delay_seconds)
+    ''];
+
+    scrapeConfigs = [
+      {
+        job_name = "prometheus";
+        static_configs = [ { targets = [ "localhost:9090" ]; } ];
+      }
+      {
+        job_name = "chrony";
+        static_configs = [ { targets = [ "10.10.10.3:9123" ]; } ];
+      }
+    ];
+  };
+
+  services.nginx.virtualHosts.${host} = {
+    enableACME = true;
+    forceSSL = true;
+
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:${builtins.toString port}";
+      proxyWebsockets = true;
+      recommendedProxySettings = true;
+    };
+  };
+}

machines/hackens-org/secrets/default.nix

@@ -11,4 +11,8 @@
     file = ./wg-key.age;
     owner = "systemd-network";
   };
+  age.secrets."prometheus-webconf" = {
+    file = ./prometheus-webconf;
+    owner = "prometheus";
+  };
 }

machines/hackens-org/secrets/prometheus-webconf

@@ -0,0 +1,28 @@
+age-encryption.org/v1
+-> ssh-ed25519 JGx7Ng 3DWZHBY1KZEvOumwmuIX5xcKhB2xpFJvg+uVmXoGfFo
+Hrn6QdBr9FSgd4+Z+rxnGThb0uSHFjRwmyjqyy1hwBs
+-> ssh-ed25519 kXobKQ xnMZxvtqDeHnp2UZ2FIj04ph9BrR3kqM9Fm8caK6sEw
+Jlm74DMV2YWV8LlHsLyM2yeLr6fcJ3T6T4iOf6iG7RY
+-> ssh-ed25519 7hZk0g a1lS6NN3Iwu4wV+BE6wmDuoG0rImD7LEY497/wl0QnM
+ophBz/eeIfQCxhrRgjCdjVGku24ZPKR6S1vTllBEZOc
+-> ssh-rsa krWCLQ
+kC94KuzwbLmSynjU6ZtOV5ETjpsHUPQvfxTOWuo0Z1ngdri0CBSlu/D0eaD+JYTo
+4wtJPb35ehcypzp065tpbOePRsHNv2R5bV18GF5ohcvLafqn4soXc73kvsGzsFyT
+hKK1mD5LHn7aP2KEDkusinFLWE/FPbRB2MKwZwBPITGtE4T217T0bcn9MdnnV1YW
+7YH2MYKbYT+FfDg/t+l8omafk9pRaDRkHsFZNPa8j1z1i6jHWhOJ8KYQnxRgoVYM
+ofRK749B5K6dsbRlU4J1sIOlrEPfoNLTvDwkyrCAdF9ZMGH4TLK76om9u14fG+jB
+2Ln/7md8jj4XRGOUyfep5Q
+-> ssh-ed25519 /vwQcQ GXa7l/Y8yBUXiv08TQOUrhoFFrxQHF3ZewPPe1vWLyU
+7wTZTr0iGbfvgxAEQtTq4BPQtAbdZ+Hej8QIBCtw/JE
+-> ssh-ed25519 0R97PA bjG6ig3F8snfLM2Azjz1WUEaafbeq4Hv0mFzIrC3Plo
+nKnnKJQ+FcTQfQV7nZPu0n+F8VmcgQn7C8IRl9wMINc
+-> ssh-ed25519 cvTB5g CznWsKDtd1s7ccFl0eJXXVkUz81CeJ6I72IfpG5ikw0
+zeTvRKwLjHWRzeeVb6NUMuwkZeZ3WQSD37uoHV5sedY
+-> ssh-ed25519 Wu8JLQ S44TDpf416SC8zGXQH3gN9ixLAY6j/bTMksyItbX+Sg
+JV/RaJieIrr7nfj8IPAQitBBjq4M6tmflEx1eqbsQmI
+-> ssh-ed25519 EIt1vA 5iBYXBsV5FGSrHt+cDc4PKZu/nE6mIPYWzdazFT4oHM
+1PWYe9H3ZcRl8QwjRPbU2COpyV4JkGd694B/dT+6obY
+-> ssh-ed25519 X51wxg lAgdRTLkS9+y9JcPYz0R8IJrSnKsD0xMgBRle2ivlHM
+lZQCJdyq4uRCtKAOrW3CT5fKbMQ+BOVRaJExnWAAsMo
+--- ddO9KM2/rYdcUbhYcAXTwriNVsZMJ+x3C9gTM0J5JkY
+k｡釀崋@微	s/r$ﾕｫ･ｨつｽﾟ､5ｾ Оﾓ｢))<29>O遏?<3F>:･M<>ｿ張>ﾅ>磁葯ｩiW]ｅ.T3暈ﾓ朗ｫｶ&ﾍOh0Eﾜrﾓ4ﾛq
<10><>=居.ﾄ!pｼ選v7]+ﾝLy<4C>#[快3B

machines/hackens-org/secrets/secrets.nix

@@ -14,4 +14,7 @@ in
   "wg-key.age".publicKeys = (readpubkeys "sinavir")
     ++ (readpubkeys "hackens-host") ++ (readpubkeys "raito")
     ++ (readpubkeys "gdd") ++ (readpubkeys "backslash");
+  "prometheus-webconf".publicKeys = (readpubkeys "sinavir")
+    ++ (readpubkeys "hackens-host") ++ (readpubkeys "raito")
+    ++ (readpubkeys "gdd") ++ (readpubkeys "backslash");
 }

machines/hackens-org/wireguard.nix

@@ -35,6 +35,14 @@
         };
 
         wireguardPeers = [
+          { #hackens-desktop
+            wireguardPeerConfig = {
+              AllowedIPs = [
+                "10.10.10.3/32"
+              ];
+              PublicKey = "h4Nf+e4JIjqOMuM5JtLN298BF/fym9fWKGtRZmS5MVA=";
+            };
+          }
           { #bakham (AGB)
             wireguardPeerConfig = {
               AllowedIPs = [

meta.nix

@@ -22,7 +22,7 @@ let
       };
       hackens-org = {
         deployment = {
-          targetHost = "server1.hackens.org"; # todo make something with ens firewall
+          targetHost = "10.10.10.1"; # todo make something with ens firewall
           tags = [ "server" ];
           targetPort = 2222;
         };