feat(ops/modules): Add module for automatically collecting garbage
Adds a module that automatically collects garbage based on disk space thresholds, and configures it to run hourly on whitby. This is implemented as an alternative to cl/2937, which I've been told uses a Nix feature that doesn't actually work. Under-the-hood this is simply a systemd timer running a shell script which checks available disk space and runs GC when necessary. Change-Id: I3c6b5de85b74ea52e7e16c53f2f900e0911c9805 Reviewed-on: https://cl.tvl.fyi/c/depot/+/3014 Tested-by: BuildkiteCI Reviewed-by: lukegb <lukegb@tvl.fyi>
This commit is contained in:
parent
605302091d
commit
cc903bc3b0
2 changed files with 100 additions and 0 deletions
|
@ -6,6 +6,7 @@ let
|
||||||
inherit (lib) range;
|
inherit (lib) range;
|
||||||
in {
|
in {
|
||||||
imports = [
|
imports = [
|
||||||
|
"${depot.path}/ops/modules/automatic-gc.nix"
|
||||||
"${depot.path}/ops/modules/clbot.nix"
|
"${depot.path}/ops/modules/clbot.nix"
|
||||||
"${depot.path}/ops/modules/irccat.nix"
|
"${depot.path}/ops/modules/irccat.nix"
|
||||||
"${depot.path}/ops/modules/monorepo-gerrit.nix"
|
"${depot.path}/ops/modules/monorepo-gerrit.nix"
|
||||||
|
@ -187,6 +188,15 @@ in {
|
||||||
challengeResponseAuthentication = false;
|
challengeResponseAuthentication = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
# Automatically collect garbage from the Nix store.
|
||||||
|
services.depot.automatic-gc = {
|
||||||
|
enable = true;
|
||||||
|
interval = "1 hour";
|
||||||
|
diskThreshold = 200; # GiB
|
||||||
|
maxFreed = 420; # GiB
|
||||||
|
preserveGenerations = "90d";
|
||||||
|
};
|
||||||
|
|
||||||
# Run a handful of Buildkite agents to support parallel builds.
|
# Run a handful of Buildkite agents to support parallel builds.
|
||||||
services.depot.buildkite = {
|
services.depot.buildkite = {
|
||||||
enable = true;
|
enable = true;
|
||||||
|
|
90
ops/modules/automatic-gc.nix
Normal file
90
ops/modules/automatic-gc.nix
Normal file
|
@ -0,0 +1,90 @@
|
||||||
|
# Defines a service for automatically collecting Nix garbage
|
||||||
|
# periodically, without relying on the (ostensibly broken) Nix options
|
||||||
|
# for min/max space available.
|
||||||
|
{ config, lib, pkgs, ... }:
|
||||||
|
|
||||||
|
let
|
||||||
|
cfg = config.services.depot.automatic-gc;
|
||||||
|
description = "Automatically collect Nix garbage";
|
||||||
|
|
||||||
|
GiBtoKiB = n: n * 1024 * 1024;
|
||||||
|
GiBtoBytes = n: n * 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
gcScript = pkgs.writeShellScript "automatic-nix-gc" ''
|
||||||
|
set -ueo pipefail
|
||||||
|
|
||||||
|
readonly MIN_THRESHOLD_KIB="${toString (GiBtoKiB cfg.diskThreshold)}"
|
||||||
|
readonly MAX_FREED_BYTES="${toString (GiBtoBytes cfg.maxFreed)}"
|
||||||
|
readonly GEN_THRESHOLD="${cfg.preserveGenerations}"
|
||||||
|
readonly AVAILABLE_KIB=$(df --sync /nix --output=avail | tail -n1)
|
||||||
|
|
||||||
|
if [ "''${AVAILABLE_KIB}" -lt "''${MIN_THRESHOLD_KIB}" ]; then
|
||||||
|
echo "Have ''${AVAILABLE_KIB} KiB, but want ''${MIN_THRESHOLD_KIB} KiB."
|
||||||
|
echo "Triggering Nix garbage collection up to ''${MAX_FREED_BYTES} bytes."
|
||||||
|
set -x
|
||||||
|
nix-collect-garbage \
|
||||||
|
--delete-older-than "''${GEN_THRESHOLD}" \
|
||||||
|
--max-freed "''${MAX_FREED_BYTES}"
|
||||||
|
else
|
||||||
|
echo "Skipping GC, enough space available"
|
||||||
|
fi
|
||||||
|
'';
|
||||||
|
in {
|
||||||
|
options.services.depot.automatic-gc = {
|
||||||
|
enable = lib.mkEnableOption description;
|
||||||
|
|
||||||
|
interval = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
example = "1h";
|
||||||
|
description = ''
|
||||||
|
Interval between garbage collection runs, specified in
|
||||||
|
systemd.time(7) format.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
diskThreshold = lib.mkOption {
|
||||||
|
type = lib.types.int;
|
||||||
|
example = "100";
|
||||||
|
description = ''
|
||||||
|
Minimum amount of space that needs to be available (in GiB) on
|
||||||
|
the partition holding /nix. Garbage collection is triggered if
|
||||||
|
it falls below this.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
maxFreed = lib.mkOption {
|
||||||
|
type = lib.types.int;
|
||||||
|
example = "420";
|
||||||
|
description = ''
|
||||||
|
Maximum amount of space to free in a single GC run, in GiB.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
preserveGenerations = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = "90d";
|
||||||
|
description = ''
|
||||||
|
Preserve NixOS generations younger than the specified value,
|
||||||
|
in the format expected by nix-collect-garbage(1).
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
config = lib.mkIf cfg.enable {
|
||||||
|
systemd.services.automatic-gc = {
|
||||||
|
inherit description;
|
||||||
|
script = "${gcScript}";
|
||||||
|
serviceConfig.Type = "oneshot";
|
||||||
|
};
|
||||||
|
|
||||||
|
systemd.timers.automatic-gc = {
|
||||||
|
inherit description;
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
|
||||||
|
timerConfig = {
|
||||||
|
OnActiveSec = "1";
|
||||||
|
OnUnitActiveSec = cfg.interval;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
Loading…
Add table
Reference in a new issue