From 0b606aad46e1d96da36d4831df63ad90f11d21c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Sep 2017 20:43:42 +0200 Subject: [PATCH] Add automatic garbage collection Nix can now automatically run the garbage collector during builds or while adding paths to the store. The option "min-free = " specifies that Nix should run the garbage collector whenever free space in the Nix store drops below . It will then delete garbage until "max-free" bytes are available. Garbage collection during builds is asynchronous; running builds are not paused and new builds are not blocked. However, there also is a synchronous GC run prior to the first build/substitution. Currently, no old GC roots are deleted (as in "nix-collect-garbage -d"). --- doc/manual/release-notes/rl-1.12.xml | 4 ++ src/libstore/build.cc | 5 ++ src/libstore/gc.cc | 70 ++++++++++++++++++++++++++++ src/libstore/globals.hh | 10 +++- src/libstore/local-store.cc | 18 +++++++ src/libstore/local-store.hh | 21 +++++++++ 6 files changed, 127 insertions(+), 1 deletion(-) diff --git a/doc/manual/release-notes/rl-1.12.xml b/doc/manual/release-notes/rl-1.12.xml index d9bdd9edd..033c9b971 100644 --- a/doc/manual/release-notes/rl-1.12.xml +++ b/doc/manual/release-notes/rl-1.12.xml @@ -393,6 +393,10 @@ configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev" package repository. + + Automatic garbage collection. + + This release has contributions from TBD. diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ce41752e6..ddf4bf00d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3957,6 +3957,8 @@ void Worker::run(const Goals & _topGoals) checkInterrupt(); + store.autoGC(false); + /* Call every wake goal (in the ordering established by CompareGoalPtrs). */ while (!awake.empty() && !topGoals.empty()) { @@ -4014,6 +4016,9 @@ void Worker::waitForInput() is a build timeout, then wait for input until the first deadline for any child. */ auto nearest = steady_time_point::max(); // nearest deadline + if (settings.minFree.get() != 0) + // Periodicallty wake up to see if we need to run the garbage collector. + nearest = before + std::chrono::seconds(10); for (auto & i : children) { if (!i.respectTimeouts) continue; if (0 != settings.maxSilentTime) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 534db8c6e..cf95f7f45 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,6 +1,7 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" +#include "finally.hh" #include #include @@ -9,6 +10,7 @@ #include #include +#include #include #include #include @@ -845,4 +847,72 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } +void LocalStore::autoGC(bool sync) +{ + auto getAvail = [this]() { + struct statvfs st; + if (statvfs(realStoreDir.c_str(), &st)) + throw SysError("getting filesystem info about '%s'", realStoreDir); + + return (uint64_t) st.f_bavail * st.f_bsize; + }; + + std::shared_future future; + + { + auto state(_state.lock()); + + if (state->gcRunning) { + future = state->gcFuture; + debug("waiting for auto-GC to finish"); + goto sync; + } + + auto now = std::chrono::steady_clock::now(); + + if (now < state->lastGCCheck + std::chrono::seconds(5)) return; + + auto avail = getAvail(); + + state->lastGCCheck = now; + + if (avail >= settings.minFree || avail >= settings.maxFree) return; + + if (avail > state->availAfterGC * 0.97) return; + + state->gcRunning = true; + + std::promise promise; + future = state->gcFuture = promise.get_future().share(); + + std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable { + + /* Wake up any threads waiting for the auto-GC to finish. */ + Finally wakeup([&]() { + auto state(_state.lock()); + state->gcRunning = false; + state->lastGCCheck = std::chrono::steady_clock::now(); + promise.set_value(); + }); + + printInfo("running auto-GC to free %d bytes", settings.maxFree - avail); + + GCOptions options; + options.maxFreed = settings.maxFree - avail; + + GCResults results; + + collectGarbage(options, results); + + _state.lock()->availAfterGC = getAvail(); + + }).detach(); + } + + sync: + // Wait for the future outside of the state lock. + if (sync) future.get(); +} + + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index c20d147f5..41d332311 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -4,8 +4,9 @@ #include "config.hh" #include -#include +#include +#include namespace nix { @@ -342,6 +343,13 @@ public: Setting hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", "A list of servers used by builtins.fetchurl to fetch files by hash."}; + + Setting minFree{this, 0, "min-free", + "Automatically run the garbage collector when free disk space drops below the specified amount."}; + + Setting maxFree{this, std::numeric_limits::max(), "max-free", + "Stop deleting garbage when free disk space is above the specified amount."}; + }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5ca776099..7afecc1cf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -244,6 +244,18 @@ LocalStore::LocalStore(const Params & params) LocalStore::~LocalStore() { + std::shared_future future; + + { + auto state(_state.lock()); + if (state->gcRunning) + future = state->gcFuture; + } + + if (future.valid()) { + printError("waiting for auto-GC to finish on exit..."); + future.get(); + } try { auto state(_state.lock()); @@ -991,6 +1003,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref & StringSource source(*nar); restorePath(realPath, source); + autoGC(); + canonicalisePathMetaData(realPath, -1); optimisePath(realPath); // FIXME: combine with hashPath() @@ -1025,6 +1039,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, deletePath(realPath); + autoGC(); + if (recursive) { StringSource source(dump); restorePath(realPath, source); @@ -1097,6 +1113,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s, deletePath(realPath); + autoGC(); + writeFile(realPath, s); canonicalisePathMetaData(realPath, -1); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 04519bfca..4973bd9a9 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -7,6 +7,8 @@ #include "sync.hh" #include "util.hh" +#include +#include #include #include @@ -60,6 +62,21 @@ private: /* The file to which we write our temporary roots. */ AutoCloseFD fdTempRoots; + + /* The last time we checked whether to do an auto-GC, or an + auto-GC finished. */ + std::chrono::time_point lastGCCheck; + + /* Whether auto-GC is running. If so, get gcFuture to wait for + the GC to finish. */ + bool gcRunning = false; + std::shared_future gcFuture; + + /* How much disk space was available after the previous + auto-GC. If the current available disk space is below + minFree but not much below availAfterGC, then there is no + point in starting a new GC. */ + uint64_t availAfterGC = std::numeric_limits::max(); }; Sync _state; @@ -196,6 +213,10 @@ public: void addSignatures(const Path & storePath, const StringSet & sigs) override; + /* If free disk space in /nix/store if below minFree, delete + garbage until it exceeds maxFree. */ + void autoGC(bool sync = true); + private: int getSchema();