feat(whitby): switch from nix-serve to harmonia for the cache

Harmonia is, ostensibly, faster and better and, most importantly, not a giant
pile of wonky Perl.

I've tested locally that Harmonia works with Nix 2.3 (on both ends), so I think
we should be good to go here.

We have a vendored copy of the upstream module for now. We need to fix Nix 2.3
compatibility in upstream for the module, but the service itself works fine.

Change-Id: I3897bb02b83bd466b6fe7077c05728ac49ea4406
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12517
Tested-by: BuildkiteCI
Autosubmit: tazjin <tazjin@tvl.su>
Reviewed-by: sterni <sternenseemann@systemli.org>
This commit is contained in:
Vincent Ambo 2024-09-26 20:04:53 +03:00 committed by tazjin
parent 2c2a6c9060
commit 143f35e003
4 changed files with 125 additions and 15 deletions

View file

@ -14,6 +14,7 @@ in
(mod "cheddar.nix")
(mod "clbot.nix")
(mod "gerrit-autosubmit.nix")
(mod "harmonia.nix")
(mod "irccat.nix")
(mod "josh.nix")
(mod "journaldriver.nix")
@ -229,12 +230,17 @@ in
grafana.file = secretFile "grafana";
irccat.file = secretFile "irccat";
keycloak-db.file = secretFile "keycloak-db";
nix-cache-priv.file = secretFile "nix-cache-priv";
owothia.file = secretFile "owothia";
panettone.file = secretFile "panettone";
smtprelay.file = secretFile "smtprelay";
teleirc.file = secretFile "teleirc";
nix-cache-priv = {
file = secretFile "nix-cache-priv";
mode = "0440";
group = "harmonia";
};
buildkite-agent-token = {
file = secretFile "buildkite-agent-token";
mode = "0440";
@ -477,11 +483,14 @@ in
];
};
services.nix-serve = {
# Run a Harmonia binary cache.
#
# TODO(tazjin): switch to upstream module after fix for Nix 2.3
services.depot.harmonia = {
enable = true;
port = 6443;
secretKeyFile = config.age.secretsDir + "/nix-cache-priv";
bindAddress = "localhost";
signKeyPaths = [ (config.age.secretsDir + "/nix-cache-priv") ];
settings.bind = "127.0.0.1:6443";
settings.priority = 50;
};
services.fail2ban.enable = true;

110
ops/modules/harmonia.nix Normal file
View file

@ -0,0 +1,110 @@
# This is a fork of the nixpkgs module for Harmonia, which adds compatibility
# with Nix 2.3.
#
# We will upstream this eventually.
{ config, pkgs, lib, ... }:
let
cfg = config.services.depot.harmonia;
format = pkgs.formats.toml { };
credentials = lib.imap0
(i: signKeyPath: {
id = "sign-key-${builtins.toString i}";
path = signKeyPath;
})
cfg.signKeyPaths;
in
{
options = {
services.depot.harmonia = {
enable = lib.mkEnableOption "Harmonia: Nix binary cache written in Rust";
signKeyPaths = lib.mkOption {
type = lib.types.listOf lib.types.path;
default = [ ];
description = "Paths to the signing keys to use for signing the cache";
};
package = lib.mkPackageOption pkgs "harmonia" { };
settings = lib.mkOption {
inherit (format) type;
default = { };
description = ''
Settings to merge with the default configuration.
For the list of the default configuration, see <https://github.com/nix-community/harmonia/tree/master#configuration>.
'';
};
};
};
config = lib.mkIf cfg.enable {
users.users.harmonia = {
isSystemUser = true;
group = "harmonia";
};
users.groups.harmonia = { };
systemd.services.harmonia = {
description = "harmonia binary cache service";
requires = [ "nix-daemon.socket" ];
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
environment = {
CONFIG_FILE = format.generate "harmonia.toml" cfg.settings;
SIGN_KEY_PATHS = lib.strings.concatMapStringsSep " "
(
credential: "%d/${credential.id}"
)
credentials;
# Note: it's important to set this for nix-store, because it wants to use
# $HOME in order to use a temporary cache dir. bizarre failures will occur
# otherwise
HOME = "/run/harmonia";
};
serviceConfig = {
ExecStart = lib.getExe cfg.package;
User = "harmonia";
Group = "harmonia";
Restart = "on-failure";
PrivateUsers = true;
DeviceAllow = [ "" ];
UMask = "0066";
RuntimeDirectory = "harmonia";
LoadCredential = builtins.map (credential: "${credential.id}:${credential.path}") credentials;
SystemCallFilter = [
"@system-service"
"~@privileged"
"~@resources"
];
CapabilityBoundingSet = "";
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectControlGroups = true;
ProtectKernelLogs = true;
ProtectHostname = true;
ProtectClock = true;
RestrictRealtime = true;
MemoryDenyWriteExecute = true;
ProcSubset = "pid";
ProtectProc = "invisible";
RestrictNamespaces = true;
SystemCallArchitectures = "native";
PrivateNetwork = false;
PrivateTmp = true;
PrivateDevices = true;
PrivateMounts = true;
NoNewPrivileges = true;
ProtectSystem = "strict";
ProtectHome = true;
LockPersonality = true;
RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6";
LimitNOFILE = 65536;
};
};
};
}

View file

@ -17,13 +17,8 @@
alias /run/agenix/nix-cache-pub;
}
location = /nix-cache-info {
add_header Content-Type text/plain;
return 200 "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 50\n";
}
location / {
proxy_pass http://localhost:${toString config.services.nix-serve.port};
proxy_pass http://${config.services.depot.harmonia.settings.bind};
}
'';
};

View file

@ -79,10 +79,6 @@ depot.nix.readTree.drvTargets {
};
});
# nix-serve does not work with nix 2.4
# https://github.com/edolstra/nix-serve/issues/28
nix-serve = super.nix-serve.override { nix = self.nix_2_3; };
# Avoid builds of mkShell derivations in CI.
mkShell = super.lib.makeOverridable (args: (super.mkShell args).overrideAttrs (_: {
passthru = {