From e0cfd1ceb0bb4c33c158c120be1c3129eb8f3102 Mon Sep 17 00:00:00 2001
From: Tom Hubrecht <tom.hubrecht@dgnum.eu>
Date: Mon, 6 Jan 2025 00:11:56 +0100
Subject: [PATCH] feat(firewall): Ban AI crawlers

---
 modules/nixos/dgn-firewall/streams.nix | 42 +++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/modules/nixos/dgn-firewall/streams.nix b/modules/nixos/dgn-firewall/streams.nix
index 34ca50c..654582c 100644
--- a/modules/nixos/dgn-firewall/streams.nix
+++ b/modules/nixos/dgn-firewall/streams.nix
@@ -25,6 +25,43 @@ let
   };
 
   available = {
+    ai-crawlers = {
+      cmd = [
+        "tail"
+        "-n0"
+        "-f"
+        "/var/log/nginx/access.log"
+      ];
+
+      filters.bots = {
+        regex = builtins.map (name: ''^<ip>.*"[^"]*${name}[^"]*"$'') [
+          "AI2Bot"
+          "Amazonbot"
+          "Applebot"
+          "Applebot-Extended"
+          "Bytespider"
+          "CCBot"
+          "ChatGPT-User"
+          "ClaudeBot"
+          "Diffbot"
+          "DuckAssistBot"
+          "FacebookBot"
+          "GPTBot"
+          "Google-Extended"
+          "Kangaroo Bot"
+          "Meta-ExternalAgent"
+          "Meta-ExternalFetcher"
+          "OAI-SearchBot"
+          "PerplexityBot"
+          "Timpibot"
+          "Webzio-Extended"
+          "YouBot"
+          "omgili"
+        ];
+        actions = ban "720h";
+      };
+    };
+
     ssh = {
       cmd = journalctl "sshd";
 
@@ -46,5 +83,8 @@ let
 in
 
 builtins.mapAttrs (_: builtins.foldl' (a: s: a // { ${s} = available.${s}; }) { }) {
-  default = [ "ssh" ];
+  default = [
+    "ai-crawlers"
+    "ssh"
+  ];
 }