feat: improve cli and scrapping (breaking change in json format)

nixos/module.nix

@@ -25,16 +25,16 @@ in
     lib = lib.mkOption { type = lib.types.raw; };
     probesConfig = {
       monitors = lib.mkOption {
-        inherit (probesFormat) type;
+        type = with lib.types; attrsOf probesFormat.type;
-        default = [ ];
+        default = { };
       };
       tags = lib.mkOption {
-        inherit (probesFormat) type;
+        type = with lib.types; attrsOf probesFormat.type;
-        default = [ ];
+        default = { };
       };
       notifications = lib.mkOption {
-        inherit (probesFormat) type;
+        type = with lib.types; attrsOf probesFormat.type;
-        default = [ ];
+        default = { };
       };
     };
   };
@@ -46,7 +46,7 @@ in
         name = "deploy-uptime-kuma-probes";
         runtimeInputs = [ pkgs.statelessUptimeKuma ];
         text = ''
-          stateless-uptime-kuma apply-json -f ${cfg.build.json} ${cfg.extraFlags}
+          stateless-uptime-kuma apply-json -f ${cfg.build.json} ${builtins.concatStringsSep " " cfg.extraFlags}
         '';
       };
     };

stateless_uptime_kuma/cli.py

@@ -35,19 +35,34 @@ def cli():
     help="Scrape keywords for http probe",
     default=False,
 )
-def apply_json(file, scrape_http_keywords):
+@click.option(
+    "--keywords-fallback/--no-keywords-fallback",
+    help="When scrapping keywords, fallback to http if no keywords found",
+    default=True,
+)
+@click.option(
+    "--no-autocreate-tags",
+    "-t",
+    is_flag=True,
+    help="Don't automatically create tags if not in tags section of input",
+    default=False,
+)
+def apply_json(file, scrape_http_keywords, no_autocreate_tags, keywords_fallback):
     """
     Apply json probes
     """
+    logger.debug(
+        f"Flags value:\n - scrape_http_keywords: {scrape_http_keywords}\n - no_autocreate_tags: {no_autocreate_tags}\n - keywords_fallback: {keywords_fallback}"
+    )
     with UptimeKumaApi("http://localhost:3001") as api:
-        api.login("admin", "123456789a")
         logging.debug("Reading json")
         data = json.load(file)
         logging.debug("Parsing json")
-        tree = from_dict(api, data)
+        tree = from_dict(api, data, not no_autocreate_tags)
         if scrape_http_keywords:
             hydrate_http_probes(tree)
         logging.debug("Sync probes")
+        api.login("admin", "123456789a")
         Manager(api, tree).process()
 
 

stateless_uptime_kuma/hydratation.py

@@ -8,10 +8,10 @@ from uptime_kuma_api import MonitorType
 logger = logging.getLogger(__name__)
 
 
-def hydrate_http_probes(tree, excludes=[]):
+def hydrate_http_probes(tree, excludes=[], fallback_to_http = True):
     for probe in tree.get("monitors", []):
         if "type" not in probe.kwargs:
-            logger.error("Fatal: probes must have a 'type' parameter")
+            logger.error(f"Fatal: probe {probe.name}  must have a 'type' parameter")
             sys.exit(1)
         if (
             probe.kwargs["type"] == MonitorType.KEYWORD
@@ -28,9 +28,20 @@ def hydrate_http_probes(tree, excludes=[]):
             method = probe.kwargs["method"]
             headers = probe.kwargs.get("headers")
             body = probe.kwargs.get("body")
-            content = requests.request(method, url, headers=headers, data=body).text
+            req = requests.request(
-            m = re.search("<title>(.*?)</title>", content)
+                method, url, headers=headers, data=body, allow_redirects=True
+            )
+            req.encoding = req.apparent_encoding
+            if req.status_code not in probe.get_status_codes():
+                logger.error(f"{probe.name} is not returning the right status code")
+            m = re.search("(?s)<title[^>]*>(.*?)</title>", req.text)
             if m is None:
-                logger.info(f"Didn't find keywords for probe {probe.name}, skipping")
+                logger.warn(
-            probe.kwargs["keyword"] = m.group(1)
+                    f"Didn't find keywords for probe {probe.name}, falling back on classic HTTP probe"
-            logger.debug(f"New keyword: {m.group(1)}")
+                )
+                logging.debug(req.text)
+                if fallback_to_http:
+                    probe.kwargs["type"] = "http"
+            else:
+                probe.kwargs["keyword"] = m.group(1).strip()
+                logger.debug(f"New keyword: {m.group(1).strip()}")

stateless_uptime_kuma/tree_gen.py

@@ -17,31 +17,33 @@ def die_tag_format_error():
     sys.exit(1)
 
 
-def from_dict(api, tree):
+def from_dict(api, tree, autocreate_tags=True):
     notif = tree.get("notifications", [])
-    indexed_notifications = {n["name"]: Notification(api, **n) for n in notif}
+    indexed_notifications = {
+        name: Notification(api, name, **kwargs) for name, kwargs in notif.items()
+    }
     tags = tree.get("tags", [])
-    indexed_tags = {t["name"]: Tag(api, **t) for t in tags}
+    indexed_tags = {name: Tag(api, name, **kwargs) for name, kwargs in tags.items()}
     monitors = tree.get("monitors", [])
     indexed_monitors = {}
-    for m in monitors:
+    for monitor_name, monitor_kwargs in monitors.items():
         associated_tags = []
-        for tag in m.get("tags", []):
+        for tag in monitor_kwargs.get("tags", []):
             if not isinstance(tag, dict) or "name" not in tag:
                 die_tag_format_error()
             try:
+                if autocreate_tags and tag["name"] not in indexed_tags:
+                    indexed_tags[tag["name"]] = Tag(api, name=tag["name"])
                 associated_tags.append((indexed_tags[tag["name"]], tag.get("value")))
             except IndexError:
                 die_tag_format_error()
-        m["tags"] = associated_tags
+        monitor_kwargs["tags"] = associated_tags
         associated_notifications = [
-            indexed_notifications[notif] for notif in m.get("notifications", [])
+            indexed_notifications[notif]
+            for notif in monitor_kwargs.get("notifications", [])
         ]
-        m["notifications"] = associated_notifications
+        monitor_kwargs["notifications"] = associated_notifications
-        if "name" not in m:
+        indexed_monitors[monitor_name] = Monitor(api, monitor_name, **monitor_kwargs)
-            logger.error("Fatal: All monitors must have a name")
-            sys.exit(1)
-        indexed_monitors[m["name"]] = Monitor(api, **m)
     return {
         "monitors": indexed_monitors.values(),
         "tags": indexed_tags.values(),

stateless_uptime_kuma/uptime_kuma.py

@@ -3,6 +3,7 @@ Classes to make the needed operations to reach the specified state.
 """
 
 import logging
+import sys
 
 logger = logging.getLogger(__name__)
 
@@ -85,6 +86,24 @@ class Monitor(Item):
         self.notifications = notifications
         self.saved = False
 
+    def get_status_codes(self):
+        if "accepted_statuscodes" not in self.kwargs:
+            return set(range(200, 300))
+        accepted_statuscodes = set()
+        for codes in self.kwargs["accepted_statuscodes"]:
+            if "-" in codes:
+                c = codes.split("-")
+                if len(c) != 2:
+                    logger.error(
+                        f"Fatal: status codes for {self} must be in the format 'number' or 'number-number'"
+                    )
+                    sys.exit(1)
+                a, b = int(c[0]), int(c[1])
+                accepted_statuscodes.update(range(a, b))
+            else:
+                accepted_statuscodes.add(int(codes))
+        return accepted_statuscodes
+
     def save(self):
         if self.saved:
             return