feat: Add options for multiple decoupled caches with same storage backend
This commit is contained in:
sinavir
parent
e7f078780a
commit
279a3d67e5
4 changed files with 172 additions and 114 deletions
|
|
@ -6,85 +6,126 @@
|
|||
}:
|
||||
let
|
||||
cfg = config.services.tvix-binary-cache;
|
||||
systemdHardening = {
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectKernelTunables = true;
|
||||
RestrictSUIDSGID = true;
|
||||
|
||||
ProtectSystem = "strict";
|
||||
ProtectKernelLogs = true;
|
||||
ProtectProc = "invisible";
|
||||
PrivateUsers = true;
|
||||
ProtectHome = true;
|
||||
UMask = "0077";
|
||||
RuntimeDirectoryMode = "0750";
|
||||
StateDirectoryMode = "0750";
|
||||
};
|
||||
in
|
||||
{
|
||||
options = {
|
||||
services.tvix-binary-cache = {
|
||||
enable = lib.mkEnableOption "BinaryCache using tvix ca-store";
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 9000;
|
||||
blob-service-addr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "objectstore+file://%S/tvix-castore/blobs.object-store";
|
||||
description = ''
|
||||
`blob-service-addr` option for the castore (please read tvix source/docs to learn more).
|
||||
'';
|
||||
};
|
||||
directory-service-addr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "sled://%S/tvix-castore/directories.sled";
|
||||
description = ''
|
||||
`directory-service-addr` option for the castore (please read tvix source/docs to learn more).
|
||||
'';
|
||||
};
|
||||
caches = lib.mkOption {
|
||||
type = lib.types.attrsOf (
|
||||
lib.types.submodule (
|
||||
{ name, ... }:
|
||||
{
|
||||
options = {
|
||||
port = lib.mkOption {
|
||||
type = lib.types.port;
|
||||
default = 9000;
|
||||
};
|
||||
name = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "Name of the cache";
|
||||
default = name;
|
||||
defaultText = lib.literalMD "Defaults to attribute name in services.tvix-binary-cache.caches";
|
||||
};
|
||||
};
|
||||
}
|
||||
)
|
||||
);
|
||||
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
config = lib.mkIf cfg.enable {
|
||||
|
||||
systemd.services =
|
||||
let
|
||||
stateDir = "tvix-binary-cache";
|
||||
in
|
||||
lib.mkIf cfg.enable {
|
||||
nar-bridge = {
|
||||
wants = [ "tvix-store.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "tvix-store.service" ];
|
||||
serviceConfig = rec {
|
||||
ExecStart = "${lib.getExe pkgs.nar-bridge-go} --otlp=false --listen-addr=\"[::1]:${builtins.toString cfg.port}\" --store-addr=\"unix://%t/${stateDir}/socket\"";
|
||||
|
||||
DynamicUser = true;
|
||||
User = "tvix-binary-cache";
|
||||
Group = "nginx";
|
||||
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectKernelTunables = true;
|
||||
RestrictSUIDSGID = true;
|
||||
|
||||
ProtectSystem = "strict";
|
||||
ProtectKernelLogs = true;
|
||||
ProtectProc = "invisible";
|
||||
PrivateUsers = true;
|
||||
ProtectHome = true;
|
||||
UMask = "0077";
|
||||
RuntimeDirectoryMode = "0750";
|
||||
StateDirectoryMode = "0750";
|
||||
};
|
||||
};
|
||||
tvix-store = {
|
||||
systemd.services = lib.mkMerge (
|
||||
(lib.singleton {
|
||||
tvix-castore = {
|
||||
wants = [ "tvix-castore.service" ];
|
||||
after = [ "tvix-castore.service" ];
|
||||
environment = {
|
||||
BLOB_SERVICE_ADDR = "objectstore+file://%S/${stateDir}/blobs.object_store";
|
||||
DIRECTORY_SERVICE_ADDR = "sled://%S/${stateDir}/directories.sled";
|
||||
PATH_INFO_SERVICE_ADDR = "sled://%S/${stateDir}/pathinfo.sled";
|
||||
BLOB_SERVICE_ADDR = cfg.blob-service-addr;
|
||||
DIRECTORY_SERVICE_ADDR = cfg.directory-service-addr;
|
||||
PATH_INFO_SERVICE_ADDR = "sled://%S/tvix-castore/pathinfo.sled"; # Unused but probably needed
|
||||
};
|
||||
serviceConfig = {
|
||||
ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/${stateDir}/socket\"";
|
||||
|
||||
ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/tvix-castore/socket\"";
|
||||
DynamicUser = true;
|
||||
User = "tvix-binary-cache";
|
||||
Group = "nginx";
|
||||
StateDirectory = stateDir;
|
||||
RuntimeDirectory = stateDir;
|
||||
StateDirectory = "tvix-castore";
|
||||
RuntimeDirectory = "tvix-castore";
|
||||
} // systemdHardening;
|
||||
|
||||
PrivateDevices = true;
|
||||
PrivateTmp = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectKernelTunables = true;
|
||||
RestrictSUIDSGID = true;
|
||||
|
||||
ProtectSystem = "strict";
|
||||
ProtectKernelLogs = true;
|
||||
ProtectProc = "invisible";
|
||||
PrivateUsers = true;
|
||||
ProtectHome = true;
|
||||
UMask = "0077";
|
||||
RuntimeDirectoryMode = "0750";
|
||||
StateDirectoryMode = "0750";
|
||||
};
|
||||
};
|
||||
})
|
||||
++ (lib.mapAttrsToList (
|
||||
name: cfg:
|
||||
let
|
||||
stateDir = "tvix-binary-cache-${cfg.name}";
|
||||
in
|
||||
{
|
||||
"nar-bridge-${cfg.name}" = {
|
||||
wants = [ "tvix-store-${cfg.name}.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "tvix-store-${cfg.name}.service" ];
|
||||
serviceConfig = rec {
|
||||
ExecStart = "${lib.getExe pkgs.nar-bridge-go} --otlp=false --listen-addr=\"[::1]:${builtins.toString cfg.port}\" --store-addr=\"unix://%t/${stateDir}/socket\"";
|
||||
|
||||
};
|
||||
DynamicUser = true;
|
||||
User = "tvix-binary-cache";
|
||||
} // systemdHardening;
|
||||
};
|
||||
"tvix-store-${cfg.name}" = {
|
||||
wants = [ "tvix-castore.service" ];
|
||||
after = [ "tvix-castore.service" ];
|
||||
environment = {
|
||||
BLOB_SERVICE_ADDR = "grpc+unix://%t/tvix-castore/socket";
|
||||
DIRECTORY_SERVICE_ADDR = "grpc+unix://%t/tvix-castore/socket";
|
||||
PATH_INFO_SERVICE_ADDR = "sled://%S/${stateDir}/pathinfo.sled";
|
||||
};
|
||||
serviceConfig = {
|
||||
ExecStart = "${pkgs.tvix-store}/bin/tvix-store --otlp=false daemon --listen-address=\"%t/${stateDir}/socket\"";
|
||||
|
||||
DynamicUser = true;
|
||||
User = "tvix-binary-cache";
|
||||
StateDirectory = stateDir;
|
||||
RuntimeDirectory = stateDir;
|
||||
} // systemdHardening;
|
||||
};
|
||||
|
||||
}
|
||||
) cfg.caches)
|
||||
);
|
||||
|
||||
};
|
||||
}
|
||||
|
|
|
|||
53
test.nix
53
test.nix
|
|
@ -1,53 +0,0 @@
|
|||
let
|
||||
sources = import ./npins;
|
||||
inherit (sources) nixpkgs;
|
||||
pkgs = import nixpkgs { overlays = [ (import ./pkgs/overlay.nix) ]; };
|
||||
inherit (pkgs) hello;
|
||||
in
|
||||
pkgs.testers.runNixOSTest (_: {
|
||||
name = "cache smoke test";
|
||||
nodes.machine =
|
||||
{ config, ... }:
|
||||
{
|
||||
imports = [ ./modules ];
|
||||
|
||||
system.extraDependencies = [ hello ];
|
||||
|
||||
services.tvix-binary-cache.enable = true;
|
||||
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
recommendedProxySettings = true;
|
||||
virtualHosts.cachix = {
|
||||
default = true;
|
||||
locations."/" = {
|
||||
proxyPass = "http://localhost:${toString config.services.tvix-binary-cache.port}";
|
||||
extraConfig = ''
|
||||
client_max_body_size 10G;
|
||||
'';
|
||||
};
|
||||
|
||||
};
|
||||
};
|
||||
};
|
||||
testScript = ''
|
||||
import sys
|
||||
import time
|
||||
start_all()
|
||||
machine.wait_for_unit("tvix-store.service")
|
||||
machine.wait_for_unit("nginx.service")
|
||||
machine.wait_for_unit("nar-bridge.service")
|
||||
time.sleep(1)
|
||||
with subtest("Nar bridge home"):
|
||||
out = machine.succeed("curl http://127.0.0.1/")
|
||||
print(repr(out))
|
||||
if out != "nar-bridge":
|
||||
sys.exit(1)
|
||||
with subtest("Nar upload"):
|
||||
machine.succeed("nix copy --extra-experimental-features nix-command --to 'http://127.0.0.1/?compression=none' ${hello}")
|
||||
with subtest("narinfo retrieve"):
|
||||
narHash = "${hello}"[11:11+32]
|
||||
machine.succeed("curl 'http://127.0.0.1/{narHash}.narinfo'")
|
||||
|
||||
'';
|
||||
})
|
||||
62
tests/basic.nix
Normal file
62
tests/basic.nix
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
{ pkgs }:
|
||||
let
|
||||
inherit (pkgs) hello;
|
||||
in
|
||||
pkgs.testers.runNixOSTest (_: {
|
||||
name = "cache smoke test";
|
||||
nodes.machine =
|
||||
{ config, ... }:
|
||||
{
|
||||
imports = [ ../modules ];
|
||||
|
||||
system.extraDependencies = [ hello ];
|
||||
|
||||
services.tvix-binary-cache = {
|
||||
enable = true;
|
||||
caches = {
|
||||
one.port = 8000;
|
||||
two.port = 8001;
|
||||
};
|
||||
};
|
||||
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
recommendedProxySettings = true;
|
||||
virtualHosts.cachix = {
|
||||
default = true;
|
||||
locations = {
|
||||
"/one".return = "302 /one/";
|
||||
"/one/" = {
|
||||
proxyPass = "http://localhost:${toString config.services.tvix-binary-cache.caches.one.port}/";
|
||||
};
|
||||
"/two".return = "302 /two/";
|
||||
"/two/" = {
|
||||
proxyPass = "http://localhost:${toString config.services.tvix-binary-cache.caches.two.port}/";
|
||||
};
|
||||
};
|
||||
extraConfig = "client_max_body_size 1G;";
|
||||
|
||||
};
|
||||
};
|
||||
};
|
||||
testScript = ''
|
||||
import sys
|
||||
import time
|
||||
start_all()
|
||||
machine.wait_for_unit("nginx.service")
|
||||
machine.wait_for_unit("nar-bridge-one.service")
|
||||
machine.wait_for_unit("nar-bridge-two.service")
|
||||
time.sleep(1)
|
||||
with subtest("Nar bridge home"):
|
||||
out = machine.succeed("curl -L http://127.0.0.1/one")
|
||||
if out != "nar-bridge":
|
||||
sys.exit(1)
|
||||
with subtest("Nar upload"):
|
||||
machine.succeed("nix copy --extra-experimental-features nix-command --to 'http://127.0.0.1/one/?compression=none' ${hello}")
|
||||
with subtest("narinfo retrieve"):
|
||||
narHash = "${hello}"[11:11+32]
|
||||
machine.succeed(f"curl -f 'http://127.0.0.1/one/{narHash}.narinfo'")
|
||||
machine.fail(f"curl -f 'http://127.0.0.1/two/{narHash}.narinfo'")
|
||||
|
||||
'';
|
||||
})
|
||||
8
tests/default.nix
Normal file
8
tests/default.nix
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
let
|
||||
sources = import ../npins;
|
||||
inherit (sources) nixpkgs;
|
||||
pkgs = import nixpkgs { overlays = [ (import ../pkgs/overlay.nix) ]; };
|
||||
in
|
||||
{
|
||||
basic = pkgs.callPackage ./basic.nix { };
|
||||
}
|
||||
Loading…
Reference in a new issue