feat(ops/modules): Add module for automatically collecting garbage
Adds a module that automatically collects garbage based on disk space thresholds, and configures it to run hourly on whitby. This is implemented as an alternative to cl/2937, which I've been told uses a Nix feature that doesn't actually work. Under-the-hood this is simply a systemd timer running a shell script which checks available disk space and runs GC when necessary. Change-Id: I3c6b5de85b74ea52e7e16c53f2f900e0911c9805 Reviewed-on: https://cl.tvl.fyi/c/depot/+/3014 Tested-by: BuildkiteCI Reviewed-by: lukegb <lukegb@tvl.fyi>
This commit is contained in:
parent
605302091d
commit
cc903bc3b0
2 changed files with 100 additions and 0 deletions
|
@ -6,6 +6,7 @@ let
|
|||
inherit (lib) range;
|
||||
in {
|
||||
imports = [
|
||||
"${depot.path}/ops/modules/automatic-gc.nix"
|
||||
"${depot.path}/ops/modules/clbot.nix"
|
||||
"${depot.path}/ops/modules/irccat.nix"
|
||||
"${depot.path}/ops/modules/monorepo-gerrit.nix"
|
||||
|
@ -187,6 +188,15 @@ in {
|
|||
challengeResponseAuthentication = false;
|
||||
};
|
||||
|
||||
# Automatically collect garbage from the Nix store.
|
||||
services.depot.automatic-gc = {
|
||||
enable = true;
|
||||
interval = "1 hour";
|
||||
diskThreshold = 200; # GiB
|
||||
maxFreed = 420; # GiB
|
||||
preserveGenerations = "90d";
|
||||
};
|
||||
|
||||
# Run a handful of Buildkite agents to support parallel builds.
|
||||
services.depot.buildkite = {
|
||||
enable = true;
|
||||
|
|
90
ops/modules/automatic-gc.nix
Normal file
90
ops/modules/automatic-gc.nix
Normal file
|
@ -0,0 +1,90 @@
|
|||
# Defines a service for automatically collecting Nix garbage
|
||||
# periodically, without relying on the (ostensibly broken) Nix options
|
||||
# for min/max space available.
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.services.depot.automatic-gc;
|
||||
description = "Automatically collect Nix garbage";
|
||||
|
||||
GiBtoKiB = n: n * 1024 * 1024;
|
||||
GiBtoBytes = n: n * 1024 * 1024 * 1024;
|
||||
|
||||
gcScript = pkgs.writeShellScript "automatic-nix-gc" ''
|
||||
set -ueo pipefail
|
||||
|
||||
readonly MIN_THRESHOLD_KIB="${toString (GiBtoKiB cfg.diskThreshold)}"
|
||||
readonly MAX_FREED_BYTES="${toString (GiBtoBytes cfg.maxFreed)}"
|
||||
readonly GEN_THRESHOLD="${cfg.preserveGenerations}"
|
||||
readonly AVAILABLE_KIB=$(df --sync /nix --output=avail | tail -n1)
|
||||
|
||||
if [ "''${AVAILABLE_KIB}" -lt "''${MIN_THRESHOLD_KIB}" ]; then
|
||||
echo "Have ''${AVAILABLE_KIB} KiB, but want ''${MIN_THRESHOLD_KIB} KiB."
|
||||
echo "Triggering Nix garbage collection up to ''${MAX_FREED_BYTES} bytes."
|
||||
set -x
|
||||
nix-collect-garbage \
|
||||
--delete-older-than "''${GEN_THRESHOLD}" \
|
||||
--max-freed "''${MAX_FREED_BYTES}"
|
||||
else
|
||||
echo "Skipping GC, enough space available"
|
||||
fi
|
||||
'';
|
||||
in {
|
||||
options.services.depot.automatic-gc = {
|
||||
enable = lib.mkEnableOption description;
|
||||
|
||||
interval = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "1h";
|
||||
description = ''
|
||||
Interval between garbage collection runs, specified in
|
||||
systemd.time(7) format.
|
||||
'';
|
||||
};
|
||||
|
||||
diskThreshold = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
example = "100";
|
||||
description = ''
|
||||
Minimum amount of space that needs to be available (in GiB) on
|
||||
the partition holding /nix. Garbage collection is triggered if
|
||||
it falls below this.
|
||||
'';
|
||||
};
|
||||
|
||||
maxFreed = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
example = "420";
|
||||
description = ''
|
||||
Maximum amount of space to free in a single GC run, in GiB.
|
||||
'';
|
||||
};
|
||||
|
||||
preserveGenerations = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "90d";
|
||||
description = ''
|
||||
Preserve NixOS generations younger than the specified value,
|
||||
in the format expected by nix-collect-garbage(1).
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
systemd.services.automatic-gc = {
|
||||
inherit description;
|
||||
script = "${gcScript}";
|
||||
serviceConfig.Type = "oneshot";
|
||||
};
|
||||
|
||||
systemd.timers.automatic-gc = {
|
||||
inherit description;
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
||||
timerConfig = {
|
||||
OnActiveSec = "1";
|
||||
OnUnitActiveSec = cfg.interval;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
Loading…
Reference in a new issue