feat: improve cli and scrapping (breaking change in json format)

lib/default.nix

@@ -0,0 +1,69 @@
+{ lib }:
+rec {
+  mkFqdn = _: cfg: cfg.networking.fqdn;
+
+  fromHive =
+    {
+      builder,
+      nodes,
+      excludes ? [ ],
+    }:
+    lib.mkMerge (
+      builtins.map (node: builder node nodes.${node}) (
+        lib.subtractLists excludes (builtins.attrNames nodes)
+      )
+    );
+
+  pingProbesFromHive =
+    {
+      mkHost,
+      nodes,
+      prefix ? "Ping ",
+      excludes ? [ ],
+      tags ? [ ],
+    }:
+    fromHive {
+      builder = (
+        node: module: {
+          monitors = {
+            ${prefix + node} = {
+              type = "ping";
+              inherit tags;
+              hostname = mkHost node module.config;
+            };
+          };
+        }
+      );
+      inherit nodes excludes;
+    };
+
+  httpProbesFromConfig =
+    {
+      config,
+      excludes ? [ ],
+      prefix ? "",
+      type ? "keyword",
+      tags ? [ ],
+    }:
+    let
+      filter = k: v: !builtins.elem k excludes && v.globalRedirect == null;
+    in
+    {
+      monitors = lib.mapAttrs' (
+        vhostName: vhost:
+        let
+          hasSSL = vhost.onlySSL || vhost.enableSSL || vhost.addSSL || vhost.forceSSL;
+          serverName = if vhost.serverName != null then vhost.serverName else vhostName;
+        in
+        {
+          name = prefix + serverName;
+          value = {
+            inherit type;
+            inherit tags;
+            url = "http${lib.optionalString hasSSL "s"}://${serverName}";
+            method = "get";
+          };
+        }
+      ) (lib.filterAttrs filter config.services.nginx.virtualHosts);
+    };
+}

nixos/module.nix

@@ -22,29 +22,31 @@ in
         Extra arguments to use for executing `stateless-uptime-kuma`.
       '';
     };
+    lib = lib.mkOption { type = lib.types.raw; };
     probesConfig = {
       monitors = lib.mkOption {
-        inherit (probesFormat) type;
-        default = [ ];
+        type = with lib.types; attrsOf probesFormat.type;
+        default = { };
       };
       tags = lib.mkOption {
-        inherit (probesFormat) type;
-        default = [ ];
+        type = with lib.types; attrsOf probesFormat.type;
+        default = { };
       };
       notifications = lib.mkOption {
-        inherit (probesFormat) type;
-        default = [ ];
+        type = with lib.types; attrsOf probesFormat.type;
+        default = { };
       };
     };
   };
   config.statelessUptimeKuma = {
+    lib = import ../lib { inherit lib; };
     build = {
       json = probesFormat.generate "probes.json" cfg.probesConfig;
       script = pkgs.writeShellApplication {
         name = "deploy-uptime-kuma-probes";
         runtimeInputs = [ pkgs.statelessUptimeKuma ];
         text = ''
-          stateless-uptime-kuma apply-json -f ${cfg.build.json} ${cfg.extraFlags}
+          stateless-uptime-kuma apply-json -f ${cfg.build.json} ${builtins.concatStringsSep " " cfg.extraFlags}
         '';
       };
     };

stateless_uptime_kuma/cli.py

@@ -35,19 +35,34 @@ def cli():
     help="Scrape keywords for http probe",
     default=False,
 )
-def apply_json(file, scrape_http_keywords):
+@click.option(
+    "--keywords-fallback/--no-keywords-fallback",
+    help="When scrapping keywords, fallback to http if no keywords found",
+    default=True,
+)
+@click.option(
+    "--no-autocreate-tags",
+    "-t",
+    is_flag=True,
+    help="Don't automatically create tags if not in tags section of input",
+    default=False,
+)
+def apply_json(file, scrape_http_keywords, no_autocreate_tags, keywords_fallback):
     """
     Apply json probes
     """
+    logger.debug(
+        f"Flags value:\n - scrape_http_keywords: {scrape_http_keywords}\n - no_autocreate_tags: {no_autocreate_tags}\n - keywords_fallback: {keywords_fallback}"
+    )
     with UptimeKumaApi("http://localhost:3001") as api:
-        api.login("admin", "123456789a")
         logging.debug("Reading json")
         data = json.load(file)
         logging.debug("Parsing json")
-        tree = from_dict(api, data)
+        tree = from_dict(api, data, not no_autocreate_tags)
         if scrape_http_keywords:
             hydrate_http_probes(tree)
         logging.debug("Sync probes")
+        api.login("admin", "123456789a")
         Manager(api, tree).process()
 
 

stateless_uptime_kuma/hydratation.py

@@ -8,10 +8,10 @@ from uptime_kuma_api import MonitorType
 logger = logging.getLogger(__name__)
 
 
-def hydrate_http_probes(tree, excludes=[]):
+def hydrate_http_probes(tree, excludes=[], fallback_to_http = True):
     for probe in tree.get("monitors", []):
         if "type" not in probe.kwargs:
-            logger.error("Fatal: probes must have a 'type' parameter")
+            logger.error(f"Fatal: probe {probe.name}  must have a 'type' parameter")
             sys.exit(1)
         if (
             probe.kwargs["type"] == MonitorType.KEYWORD
@@ -28,9 +28,20 @@ def hydrate_http_probes(tree, excludes=[]):
             method = probe.kwargs["method"]
             headers = probe.kwargs.get("headers")
             body = probe.kwargs.get("body")
-            content = requests.request(method, url, headers=headers, data=body).text
-            m = re.search("<title>(.*?)</title>", content)
+            req = requests.request(
+                method, url, headers=headers, data=body, allow_redirects=True
+            )
+            req.encoding = req.apparent_encoding
+            if req.status_code not in probe.get_status_codes():
+                logger.error(f"{probe.name} is not returning the right status code")
+            m = re.search("(?s)<title[^>]*>(.*?)</title>", req.text)
             if m is None:
-                logger.info(f"Didn't find keywords for probe {probe.name}, skipping")
-            probe.kwargs["keyword"] = m.group(1)
-            logger.debug(f"New keyword: {m.group(1)}")
+                logger.warn(
+                    f"Didn't find keywords for probe {probe.name}, falling back on classic HTTP probe"
+                )
+                logging.debug(req.text)
+                if fallback_to_http:
+                    probe.kwargs["type"] = "http"
+            else:
+                probe.kwargs["keyword"] = m.group(1).strip()
+                logger.debug(f"New keyword: {m.group(1).strip()}")

stateless_uptime_kuma/tree_gen.py

@@ -17,31 +17,33 @@ def die_tag_format_error():
     sys.exit(1)
 
 
-def from_dict(api, tree):
+def from_dict(api, tree, autocreate_tags=True):
     notif = tree.get("notifications", [])
-    indexed_notifications = {n["name"]: Notification(api, **n) for n in notif}
+    indexed_notifications = {
+        name: Notification(api, name, **kwargs) for name, kwargs in notif.items()
+    }
     tags = tree.get("tags", [])
-    indexed_tags = {t["name"]: Tag(api, **t) for t in tags}
+    indexed_tags = {name: Tag(api, name, **kwargs) for name, kwargs in tags.items()}
     monitors = tree.get("monitors", [])
     indexed_monitors = {}
-    for m in monitors:
+    for monitor_name, monitor_kwargs in monitors.items():
         associated_tags = []
-        for tag in m.get("tags", []):
+        for tag in monitor_kwargs.get("tags", []):
             if not isinstance(tag, dict) or "name" not in tag:
                 die_tag_format_error()
             try:
+                if autocreate_tags and tag["name"] not in indexed_tags:
+                    indexed_tags[tag["name"]] = Tag(api, name=tag["name"])
                 associated_tags.append((indexed_tags[tag["name"]], tag.get("value")))
             except IndexError:
                 die_tag_format_error()
-        m["tags"] = associated_tags
+        monitor_kwargs["tags"] = associated_tags
         associated_notifications = [
-            indexed_notifications[notif] for notif in m.get("notifications", [])
+            indexed_notifications[notif]
+            for notif in monitor_kwargs.get("notifications", [])
         ]
-        m["notifications"] = associated_notifications
-        if "name" not in m:
-            logger.error("Fatal: All monitors must have a name")
-            sys.exit(1)
-        indexed_monitors[m["name"]] = Monitor(api, **m)
+        monitor_kwargs["notifications"] = associated_notifications
+        indexed_monitors[monitor_name] = Monitor(api, monitor_name, **monitor_kwargs)
     return {
         "monitors": indexed_monitors.values(),
         "tags": indexed_tags.values(),

stateless_uptime_kuma/uptime_kuma.py

@@ -3,6 +3,7 @@ Classes to make the needed operations to reach the specified state.
 """
 
 import logging
+import sys
 
 logger = logging.getLogger(__name__)
 
@@ -85,6 +86,24 @@ class Monitor(Item):
         self.notifications = notifications
         self.saved = False
 
+    def get_status_codes(self):
+        if "accepted_statuscodes" not in self.kwargs:
+            return set(range(200, 300))
+        accepted_statuscodes = set()
+        for codes in self.kwargs["accepted_statuscodes"]:
+            if "-" in codes:
+                c = codes.split("-")
+                if len(c) != 2:
+                    logger.error(
+                        f"Fatal: status codes for {self} must be in the format 'number' or 'number-number'"
+                    )
+                    sys.exit(1)
+                a, b = int(c[0]), int(c[1])
+                accepted_statuscodes.update(range(a, b))
+            else:
+                accepted_statuscodes.add(int(codes))
+        return accepted_statuscodes
+
     def save(self):
         if self.saved:
             return