feat: Add options for multiple decoupled caches with same storage backend

modules/default.nix

@@ -6,36 +6,7 @@
 }:
 let
   cfg = config.services.tvix-binary-cache;
-in
+  systemdHardening = {
-{
-  options = {
-    services.tvix-binary-cache = {
-      enable = lib.mkEnableOption "BinaryCache using tvix ca-store";
-      port = lib.mkOption {
-        type = lib.types.port;
-        default = 9000;
-      };
-    };
-  };
-
-  config = {
-
-    systemd.services =
-      let
-        stateDir = "tvix-binary-cache";
-      in
-      lib.mkIf cfg.enable {
-        nar-bridge = {
-          wants = [ "tvix-store.service" ];
-          wantedBy = [ "multi-user.target" ];
-          after = [ "tvix-store.service" ];
-          serviceConfig = rec {
-            ExecStart = "${lib.getExe pkgs.nar-bridge-go} --otlp=false --listen-addr=\"[::1]:${builtins.toString cfg.port}\" --store-addr=\"unix://%t/${stateDir}/socket\"";
-
-            DynamicUser = true;
-            User = "tvix-binary-cache";
-            Group = "nginx";
-
     PrivateDevices = true;
     PrivateTmp = true;
     ProtectControlGroups = true;
@@ -51,11 +22,95 @@ in
     RuntimeDirectoryMode = "0750";
     StateDirectoryMode = "0750";
   };
+in
+{
+  options = {
+    services.tvix-binary-cache = {
+      enable = lib.mkEnableOption "BinaryCache using tvix ca-store";
+      blob-service-addr = lib.mkOption {
+        type = lib.types.str;
+        default = "objectstore+file://%S/tvix-castore/blobs.object-store";
+        description = ''
+          `blob-service-addr` option for the castore (please read tvix source/docs to learn more).
+        '';
       };
-        tvix-store = {
+      directory-service-addr = lib.mkOption {
+        type = lib.types.str;
+        default = "sled://%S/tvix-castore/directories.sled";
+        description = ''
+          `directory-service-addr` option for the castore (please read tvix source/docs to learn more).
+        '';
+      };
+      caches = lib.mkOption {
+        type = lib.types.attrsOf (
+          lib.types.submodule (
+            { name, ... }:
+            {
+              options = {
+                port = lib.mkOption {
+                  type = lib.types.port;
+                  default = 9000;
+                };
+                name = lib.mkOption {
+                  type = lib.types.str;
+                  description = "Name of the cache";
+                  default = name;
+                  defaultText = lib.literalMD "Defaults to attribute name in services.tvix-binary-cache.caches";
+                };
+              };
+            }
+          )
+        );
+
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    systemd.services = lib.mkMerge (
+      (lib.singleton {
+        tvix-castore = {
+          wants = [ "tvix-castore.service" ];
+          after = [ "tvix-castore.service" ];
           environment = {
-            BLOB_SERVICE_ADDR = "objectstore+file://%S/${stateDir}/blobs.object_store";
+            BLOB_SERVICE_ADDR = cfg.blob-service-addr;
-            DIRECTORY_SERVICE_ADDR = "sled://%S/${stateDir}/directories.sled";
+            DIRECTORY_SERVICE_ADDR = cfg.directory-service-addr;
+            PATH_INFO_SERVICE_ADDR = "sled://%S/tvix-castore/pathinfo.sled"; # Unused but probably needed
+          };
+          serviceConfig = {
+            ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/tvix-castore/socket\"";
+            DynamicUser = true;
+            User = "tvix-binary-cache";
+            StateDirectory = "tvix-castore";
+            RuntimeDirectory = "tvix-castore";
+          } // systemdHardening;
+
+        };
+      })
+      ++ (lib.mapAttrsToList (
+        name: cfg:
+        let
+          stateDir = "tvix-binary-cache-${cfg.name}";
+        in
+        {
+          "nar-bridge-${cfg.name}" = {
+            wants = [ "tvix-store-${cfg.name}.service" ];
+            wantedBy = [ "multi-user.target" ];
+            after = [ "tvix-store-${cfg.name}.service" ];
+            serviceConfig = rec {
+              ExecStart = "${lib.getExe pkgs.nar-bridge-go} --otlp=false --listen-addr=\"[::1]:${builtins.toString cfg.port}\" --store-addr=\"unix://%t/${stateDir}/socket\"";
+
+              DynamicUser = true;
+              User = "tvix-binary-cache";
+            } // systemdHardening;
+          };
+          "tvix-store-${cfg.name}" = {
+            wants = [ "tvix-castore.service" ];
+            after = [ "tvix-castore.service" ];
+            environment = {
+              BLOB_SERVICE_ADDR = "grpc+unix://%t/tvix-castore/socket";
+              DIRECTORY_SERVICE_ADDR = "grpc+unix://%t/tvix-castore/socket";
               PATH_INFO_SERVICE_ADDR = "sled://%S/${stateDir}/pathinfo.sled";
             };
             serviceConfig = {
@@ -63,28 +118,14 @@ in
 
               DynamicUser = true;
               User = "tvix-binary-cache";
-            Group = "nginx";
               StateDirectory = stateDir;
               RuntimeDirectory = stateDir;
-
+            } // systemdHardening;
-            PrivateDevices = true;
-            PrivateTmp = true;
-            ProtectControlGroups = true;
-            ProtectKernelTunables = true;
-            RestrictSUIDSGID = true;
-
-            ProtectSystem = "strict";
-            ProtectKernelLogs = true;
-            ProtectProc = "invisible";
-            PrivateUsers = true;
-            ProtectHome = true;
-            UMask = "0077";
-            RuntimeDirectoryMode = "0750";
-            StateDirectoryMode = "0750";
-          };
           };
 
-      };
+        }
+      ) cfg.caches)
+    );
 
   };
 }

test.nix

@@ -1,53 +0,0 @@
-let
-  sources = import ./npins;
-  inherit (sources) nixpkgs;
-  pkgs = import nixpkgs { overlays = [ (import ./pkgs/overlay.nix) ]; };
-  inherit (pkgs) hello;
-in
-pkgs.testers.runNixOSTest (_: {
-  name = "cache smoke test";
-  nodes.machine =
-    { config, ... }:
-    {
-      imports = [ ./modules ];
-
-      system.extraDependencies = [ hello ];
-
-      services.tvix-binary-cache.enable = true;
-
-      services.nginx = {
-        enable = true;
-        recommendedProxySettings = true;
-        virtualHosts.cachix = {
-          default = true;
-          locations."/" = {
-            proxyPass = "http://localhost:${toString config.services.tvix-binary-cache.port}";
-            extraConfig = ''
-              client_max_body_size 10G;
-            '';
-          };
-
-        };
-      };
-    };
-  testScript = ''
-    import sys
-    import time
-    start_all()
-    machine.wait_for_unit("tvix-store.service")
-    machine.wait_for_unit("nginx.service")
-    machine.wait_for_unit("nar-bridge.service")
-    time.sleep(1)
-    with subtest("Nar bridge home"):
-        out = machine.succeed("curl http://127.0.0.1/")
-        print(repr(out))
-        if out != "nar-bridge":
-            sys.exit(1)
-    with subtest("Nar upload"):
-        machine.succeed("nix copy --extra-experimental-features nix-command --to 'http://127.0.0.1/?compression=none' ${hello}")
-    with subtest("narinfo retrieve"):
-        narHash = "${hello}"[11:11+32]
-        machine.succeed("curl 'http://127.0.0.1/{narHash}.narinfo'")
-
-  '';
-})

tests/basic.nix

@@ -0,0 +1,62 @@
+{ pkgs }:
+let
+  inherit (pkgs) hello;
+in
+pkgs.testers.runNixOSTest (_: {
+  name = "cache smoke test";
+  nodes.machine =
+    { config, ... }:
+    {
+      imports = [ ../modules ];
+
+      system.extraDependencies = [ hello ];
+
+      services.tvix-binary-cache = {
+        enable = true;
+        caches = {
+          one.port = 8000;
+          two.port = 8001;
+        };
+      };
+
+      services.nginx = {
+        enable = true;
+        recommendedProxySettings = true;
+        virtualHosts.cachix = {
+          default = true;
+          locations = {
+            "/one".return = "302 /one/";
+            "/one/" = {
+              proxyPass = "http://localhost:${toString config.services.tvix-binary-cache.caches.one.port}/";
+            };
+            "/two".return = "302 /two/";
+            "/two/" = {
+              proxyPass = "http://localhost:${toString config.services.tvix-binary-cache.caches.two.port}/";
+            };
+          };
+          extraConfig = "client_max_body_size 1G;";
+
+        };
+      };
+    };
+  testScript = ''
+    import sys
+    import time
+    start_all()
+    machine.wait_for_unit("nginx.service")
+    machine.wait_for_unit("nar-bridge-one.service")
+    machine.wait_for_unit("nar-bridge-two.service")
+    time.sleep(1)
+    with subtest("Nar bridge home"):
+        out = machine.succeed("curl -L http://127.0.0.1/one")
+        if out != "nar-bridge":
+            sys.exit(1)
+    with subtest("Nar upload"):
+        machine.succeed("nix copy --extra-experimental-features nix-command --to 'http://127.0.0.1/one/?compression=none' ${hello}")
+    with subtest("narinfo retrieve"):
+        narHash = "${hello}"[11:11+32]
+        machine.succeed(f"curl -f 'http://127.0.0.1/one/{narHash}.narinfo'")
+        machine.fail(f"curl -f 'http://127.0.0.1/two/{narHash}.narinfo'")
+
+  '';
+})

tests/default.nix

@@ -0,0 +1,8 @@
+let
+  sources = import ../npins;
+  inherit (sources) nixpkgs;
+  pkgs = import nixpkgs { overlays = [ (import ../pkgs/overlay.nix) ]; };
+in
+{
+  basic = pkgs.callPackage ./basic.nix { };
+}