From c1863033bccc50f1b0d7209fc26eb89672894163 Mon Sep 17 00:00:00 2001 From: sinavir Date: Sat, 20 Apr 2024 13:37:45 +0200 Subject: [PATCH] feat: improve cli and scrapping (breaking change in json format) --- nixos/module.nix | 14 +++++++------- stateless_uptime_kuma/cli.py | 21 ++++++++++++++++++--- stateless_uptime_kuma/hydratation.py | 25 ++++++++++++++++++------- stateless_uptime_kuma/tree_gen.py | 26 ++++++++++++++------------ stateless_uptime_kuma/uptime_kuma.py | 19 +++++++++++++++++++ 5 files changed, 76 insertions(+), 29 deletions(-) diff --git a/nixos/module.nix b/nixos/module.nix index eef273c..dbee0ad 100644 --- a/nixos/module.nix +++ b/nixos/module.nix @@ -25,16 +25,16 @@ in lib = lib.mkOption { type = lib.types.raw; }; probesConfig = { monitors = lib.mkOption { - inherit (probesFormat) type; - default = [ ]; + type = with lib.types; attrsOf probesFormat.type; + default = { }; }; tags = lib.mkOption { - inherit (probesFormat) type; - default = [ ]; + type = with lib.types; attrsOf probesFormat.type; + default = { }; }; notifications = lib.mkOption { - inherit (probesFormat) type; - default = [ ]; + type = with lib.types; attrsOf probesFormat.type; + default = { }; }; }; }; @@ -46,7 +46,7 @@ in name = "deploy-uptime-kuma-probes"; runtimeInputs = [ pkgs.statelessUptimeKuma ]; text = '' - stateless-uptime-kuma apply-json -f ${cfg.build.json} ${cfg.extraFlags} + stateless-uptime-kuma apply-json -f ${cfg.build.json} ${builtins.concatStringsSep " " cfg.extraFlags} ''; }; }; diff --git a/stateless_uptime_kuma/cli.py b/stateless_uptime_kuma/cli.py index 541b254..d003740 100644 --- a/stateless_uptime_kuma/cli.py +++ b/stateless_uptime_kuma/cli.py @@ -35,19 +35,34 @@ def cli(): help="Scrape keywords for http probe", default=False, ) -def apply_json(file, scrape_http_keywords): +@click.option( + "--keywords-fallback/--no-keywords-fallback", + help="When scrapping keywords, fallback to http if no keywords found", + default=True, +) +@click.option( + "--no-autocreate-tags", + "-t", + is_flag=True, + help="Don't automatically create tags if not in tags section of input", + default=False, +) +def apply_json(file, scrape_http_keywords, no_autocreate_tags, keywords_fallback): """ Apply json probes """ + logger.debug( + f"Flags value:\n - scrape_http_keywords: {scrape_http_keywords}\n - no_autocreate_tags: {no_autocreate_tags}\n - keywords_fallback: {keywords_fallback}" + ) with UptimeKumaApi("http://localhost:3001") as api: - api.login("admin", "123456789a") logging.debug("Reading json") data = json.load(file) logging.debug("Parsing json") - tree = from_dict(api, data) + tree = from_dict(api, data, not no_autocreate_tags) if scrape_http_keywords: hydrate_http_probes(tree) logging.debug("Sync probes") + api.login("admin", "123456789a") Manager(api, tree).process() diff --git a/stateless_uptime_kuma/hydratation.py b/stateless_uptime_kuma/hydratation.py index ab79c8d..b77d53f 100644 --- a/stateless_uptime_kuma/hydratation.py +++ b/stateless_uptime_kuma/hydratation.py @@ -8,10 +8,10 @@ from uptime_kuma_api import MonitorType logger = logging.getLogger(__name__) -def hydrate_http_probes(tree, excludes=[]): +def hydrate_http_probes(tree, excludes=[], fallback_to_http = True): for probe in tree.get("monitors", []): if "type" not in probe.kwargs: - logger.error("Fatal: probes must have a 'type' parameter") + logger.error(f"Fatal: probe {probe.name} must have a 'type' parameter") sys.exit(1) if ( probe.kwargs["type"] == MonitorType.KEYWORD @@ -28,9 +28,20 @@ def hydrate_http_probes(tree, excludes=[]): method = probe.kwargs["method"] headers = probe.kwargs.get("headers") body = probe.kwargs.get("body") - content = requests.request(method, url, headers=headers, data=body).text - m = re.search("(.*?)", content) + req = requests.request( + method, url, headers=headers, data=body, allow_redirects=True + ) + req.encoding = req.apparent_encoding + if req.status_code not in probe.get_status_codes(): + logger.error(f"{probe.name} is not returning the right status code") + m = re.search("(?s)]*>(.*?)", req.text) if m is None: - logger.info(f"Didn't find keywords for probe {probe.name}, skipping") - probe.kwargs["keyword"] = m.group(1) - logger.debug(f"New keyword: {m.group(1)}") + logger.warn( + f"Didn't find keywords for probe {probe.name}, falling back on classic HTTP probe" + ) + logging.debug(req.text) + if fallback_to_http: + probe.kwargs["type"] = "http" + else: + probe.kwargs["keyword"] = m.group(1).strip() + logger.debug(f"New keyword: {m.group(1).strip()}") diff --git a/stateless_uptime_kuma/tree_gen.py b/stateless_uptime_kuma/tree_gen.py index 4897167..97fc2fb 100644 --- a/stateless_uptime_kuma/tree_gen.py +++ b/stateless_uptime_kuma/tree_gen.py @@ -17,31 +17,33 @@ def die_tag_format_error(): sys.exit(1) -def from_dict(api, tree): +def from_dict(api, tree, autocreate_tags=True): notif = tree.get("notifications", []) - indexed_notifications = {n["name"]: Notification(api, **n) for n in notif} + indexed_notifications = { + name: Notification(api, name, **kwargs) for name, kwargs in notif.items() + } tags = tree.get("tags", []) - indexed_tags = {t["name"]: Tag(api, **t) for t in tags} + indexed_tags = {name: Tag(api, name, **kwargs) for name, kwargs in tags.items()} monitors = tree.get("monitors", []) indexed_monitors = {} - for m in monitors: + for monitor_name, monitor_kwargs in monitors.items(): associated_tags = [] - for tag in m.get("tags", []): + for tag in monitor_kwargs.get("tags", []): if not isinstance(tag, dict) or "name" not in tag: die_tag_format_error() try: + if autocreate_tags and tag["name"] not in indexed_tags: + indexed_tags[tag["name"]] = Tag(api, name=tag["name"]) associated_tags.append((indexed_tags[tag["name"]], tag.get("value"))) except IndexError: die_tag_format_error() - m["tags"] = associated_tags + monitor_kwargs["tags"] = associated_tags associated_notifications = [ - indexed_notifications[notif] for notif in m.get("notifications", []) + indexed_notifications[notif] + for notif in monitor_kwargs.get("notifications", []) ] - m["notifications"] = associated_notifications - if "name" not in m: - logger.error("Fatal: All monitors must have a name") - sys.exit(1) - indexed_monitors[m["name"]] = Monitor(api, **m) + monitor_kwargs["notifications"] = associated_notifications + indexed_monitors[monitor_name] = Monitor(api, monitor_name, **monitor_kwargs) return { "monitors": indexed_monitors.values(), "tags": indexed_tags.values(), diff --git a/stateless_uptime_kuma/uptime_kuma.py b/stateless_uptime_kuma/uptime_kuma.py index bd3f181..c9f03b0 100644 --- a/stateless_uptime_kuma/uptime_kuma.py +++ b/stateless_uptime_kuma/uptime_kuma.py @@ -3,6 +3,7 @@ Classes to make the needed operations to reach the specified state. """ import logging +import sys logger = logging.getLogger(__name__) @@ -85,6 +86,24 @@ class Monitor(Item): self.notifications = notifications self.saved = False + def get_status_codes(self): + if "accepted_statuscodes" not in self.kwargs: + return set(range(200, 300)) + accepted_statuscodes = set() + for codes in self.kwargs["accepted_statuscodes"]: + if "-" in codes: + c = codes.split("-") + if len(c) != 2: + logger.error( + f"Fatal: status codes for {self} must be in the format 'number' or 'number-number'" + ) + sys.exit(1) + a, b = int(c[0]), int(c[1]) + accepted_statuscodes.update(range(a, b)) + else: + accepted_statuscodes.add(int(codes)) + return accepted_statuscodes + def save(self): if self.saved: return