feat(ops/modules): Add module for automatically collecting garbage

Adds a module that automatically collects garbage based on disk space
thresholds, and configures it to run hourly on whitby.

This is implemented as an alternative to cl/2937, which I've been told
uses a Nix feature that doesn't actually work.

Under-the-hood this is simply a systemd timer running a shell script
which checks available disk space and runs GC when necessary.

Change-Id: I3c6b5de85b74ea52e7e16c53f2f900e0911c9805
Reviewed-on: https://cl.tvl.fyi/c/depot/+/3014
Tested-by: BuildkiteCI
Reviewed-by: lukegb <lukegb@tvl.fyi>
This commit is contained in:
Vincent Ambo 2021-04-14 18:16:42 +02:00 committed by tazjin
parent 605302091d
commit cc903bc3b0
2 changed files with 100 additions and 0 deletions

View file

@ -6,6 +6,7 @@ let
inherit (lib) range;
in {
imports = [
"${depot.path}/ops/modules/automatic-gc.nix"
"${depot.path}/ops/modules/clbot.nix"
"${depot.path}/ops/modules/irccat.nix"
"${depot.path}/ops/modules/monorepo-gerrit.nix"
@ -187,6 +188,15 @@ in {
challengeResponseAuthentication = false;
};
# Automatically collect garbage from the Nix store.
services.depot.automatic-gc = {
enable = true;
interval = "1 hour";
diskThreshold = 200; # GiB
maxFreed = 420; # GiB
preserveGenerations = "90d";
};
# Run a handful of Buildkite agents to support parallel builds.
services.depot.buildkite = {
enable = true;

View file

@ -0,0 +1,90 @@
# Defines a service for automatically collecting Nix garbage
# periodically, without relying on the (ostensibly broken) Nix options
# for min/max space available.
{ config, lib, pkgs, ... }:
let
cfg = config.services.depot.automatic-gc;
description = "Automatically collect Nix garbage";
GiBtoKiB = n: n * 1024 * 1024;
GiBtoBytes = n: n * 1024 * 1024 * 1024;
gcScript = pkgs.writeShellScript "automatic-nix-gc" ''
set -ueo pipefail
readonly MIN_THRESHOLD_KIB="${toString (GiBtoKiB cfg.diskThreshold)}"
readonly MAX_FREED_BYTES="${toString (GiBtoBytes cfg.maxFreed)}"
readonly GEN_THRESHOLD="${cfg.preserveGenerations}"
readonly AVAILABLE_KIB=$(df --sync /nix --output=avail | tail -n1)
if [ "''${AVAILABLE_KIB}" -lt "''${MIN_THRESHOLD_KIB}" ]; then
echo "Have ''${AVAILABLE_KIB} KiB, but want ''${MIN_THRESHOLD_KIB} KiB."
echo "Triggering Nix garbage collection up to ''${MAX_FREED_BYTES} bytes."
set -x
nix-collect-garbage \
--delete-older-than "''${GEN_THRESHOLD}" \
--max-freed "''${MAX_FREED_BYTES}"
else
echo "Skipping GC, enough space available"
fi
'';
in {
options.services.depot.automatic-gc = {
enable = lib.mkEnableOption description;
interval = lib.mkOption {
type = lib.types.str;
example = "1h";
description = ''
Interval between garbage collection runs, specified in
systemd.time(7) format.
'';
};
diskThreshold = lib.mkOption {
type = lib.types.int;
example = "100";
description = ''
Minimum amount of space that needs to be available (in GiB) on
the partition holding /nix. Garbage collection is triggered if
it falls below this.
'';
};
maxFreed = lib.mkOption {
type = lib.types.int;
example = "420";
description = ''
Maximum amount of space to free in a single GC run, in GiB.
'';
};
preserveGenerations = lib.mkOption {
type = lib.types.str;
default = "90d";
description = ''
Preserve NixOS generations younger than the specified value,
in the format expected by nix-collect-garbage(1).
'';
};
};
config = lib.mkIf cfg.enable {
systemd.services.automatic-gc = {
inherit description;
script = "${gcScript}";
serviceConfig.Type = "oneshot";
};
systemd.timers.automatic-gc = {
inherit description;
wantedBy = [ "multi-user.target" ];
timerConfig = {
OnActiveSec = "1";
OnUnitActiveSec = cfg.interval;
};
};
};
}