refactor(3p/nix): Only initialise garbage-collector where needed
Only libexpr depends on the garbage collector, specifically only instantiations of EvalState actually require the GC to be initialised. Rather than always starting it for the whole program, even if it is not needed, this change moves the GC initialisation into libexpr, guarded by absl::call_once. This should make it possible to run the nix daemon without the garbage collector interfering, granted that things are correcty separated and the daemon does not actually invoke the evaluator. Based on my investigation so far, the daemon logic itself does not require libexpr to be present at all - so I think it is safe - but the current monobinary might have some tricks up its sleeve that will cause problems for us. We can deal with those if they arise. Relates to https://b.tvl.fyi/issues/30 Change-Id: I61c745f96420c02e089bd3c362ac3ccb117d3073 Reviewed-on: https://cl.tvl.fyi/c/depot/+/1584 Tested-by: BuildkiteCI Reviewed-by: edef <edef@edef.eu> Reviewed-by: kanepyork <rikingcoding@gmail.com>
This commit is contained in:
parent
6e04b23506
commit
46c78aa0f9
6 changed files with 72 additions and 70 deletions
125
third_party/nix/src/libexpr/eval.cc
vendored
125
third_party/nix/src/libexpr/eval.cc
vendored
|
@ -12,6 +12,7 @@
|
||||||
|
|
||||||
#define GC_INCLUDE_NEW
|
#define GC_INCLUDE_NEW
|
||||||
|
|
||||||
|
#include <absl/base/call_once.h>
|
||||||
#include <absl/container/flat_hash_set.h>
|
#include <absl/container/flat_hash_set.h>
|
||||||
#include <absl/strings/match.h>
|
#include <absl/strings/match.h>
|
||||||
#include <gc/gc.h>
|
#include <gc/gc.h>
|
||||||
|
@ -34,6 +35,65 @@
|
||||||
#include "libutil/visitor.hh"
|
#include "libutil/visitor.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
// Called when the Boehm GC runs out of memory.
|
||||||
|
static void* BoehmOomHandler(size_t requested) {
|
||||||
|
/* Convert this to a proper C++ exception. */
|
||||||
|
LOG(FATAL) << "Garbage collector ran out of memory; requested " << requested
|
||||||
|
<< " bytes";
|
||||||
|
throw std::bad_alloc();
|
||||||
|
}
|
||||||
|
|
||||||
|
void ConfigureBoehmGc() {
|
||||||
|
/* Don't look for interior pointers. This reduces the odds of
|
||||||
|
misdetection a bit. */
|
||||||
|
GC_set_all_interior_pointers(0);
|
||||||
|
|
||||||
|
/* We don't have any roots in data segments, so don't scan from
|
||||||
|
there. */
|
||||||
|
GC_set_no_dls(1);
|
||||||
|
|
||||||
|
GC_INIT();
|
||||||
|
|
||||||
|
GC_set_oom_fn(BoehmOomHandler);
|
||||||
|
|
||||||
|
/* Set the initial heap size to something fairly big (25% of
|
||||||
|
physical RAM, up to a maximum of 384 MiB) so that in most cases
|
||||||
|
we don't need to garbage collect at all. (Collection has a
|
||||||
|
fairly significant overhead.) The heap size can be overridden
|
||||||
|
through libgc's GC_INITIAL_HEAP_SIZE environment variable. We
|
||||||
|
should probably also provide a nix.conf setting for this. Note
|
||||||
|
that GC_expand_hp() causes a lot of virtual, but not physical
|
||||||
|
(resident) memory to be allocated. This might be a problem on
|
||||||
|
systems that don't overcommit. */
|
||||||
|
if (getenv("GC_INITIAL_HEAP_SIZE") == nullptr) {
|
||||||
|
size_t size = 32 * 1024 * 1024;
|
||||||
|
#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
|
||||||
|
size_t maxSize = 384 * 1024 * 1024;
|
||||||
|
long pageSize = sysconf(_SC_PAGESIZE);
|
||||||
|
long pages = sysconf(_SC_PHYS_PAGES);
|
||||||
|
if (pageSize != -1) {
|
||||||
|
size = (pageSize * pages) / 4;
|
||||||
|
} // 25% of RAM
|
||||||
|
if (size > maxSize) {
|
||||||
|
size = maxSize;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
DLOG(INFO) << "setting initial heap size to " << size << " bytes";
|
||||||
|
GC_expand_hp(size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
namespace expr {
|
||||||
|
|
||||||
|
absl::once_flag gc_flag;
|
||||||
|
|
||||||
|
void InitGC() { absl::call_once(gc_flag, &ConfigureBoehmGc); }
|
||||||
|
|
||||||
|
} // namespace expr
|
||||||
|
|
||||||
static char* dupString(const char* s) {
|
static char* dupString(const char* s) {
|
||||||
char* t;
|
char* t;
|
||||||
|
@ -187,14 +247,6 @@ std::string showType(const Value& v) {
|
||||||
abort();
|
abort();
|
||||||
}
|
}
|
||||||
|
|
||||||
#if HAVE_BOEHMGC
|
|
||||||
/* Called when the Boehm GC runs out of memory. */
|
|
||||||
static void* oomHandler(size_t requested) {
|
|
||||||
/* Convert this to a proper C++ exception. */
|
|
||||||
throw std::bad_alloc();
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
static Symbol getName(const AttrName& name, EvalState& state, Env& env) {
|
static Symbol getName(const AttrName& name, EvalState& state, Env& env) {
|
||||||
return std::visit(
|
return std::visit(
|
||||||
util::overloaded{[&](const Symbol& name) -> Symbol { return name; },
|
util::overloaded{[&](const Symbol& name) -> Symbol { return name; },
|
||||||
|
@ -207,59 +259,6 @@ static Symbol getName(const AttrName& name, EvalState& state, Env& env) {
|
||||||
name);
|
name);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool gcInitialised = false;
|
|
||||||
|
|
||||||
void initGC() {
|
|
||||||
if (gcInitialised) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
#if HAVE_BOEHMGC
|
|
||||||
/* Initialise the Boehm garbage collector. */
|
|
||||||
|
|
||||||
/* Don't look for interior pointers. This reduces the odds of
|
|
||||||
misdetection a bit. */
|
|
||||||
GC_set_all_interior_pointers(0);
|
|
||||||
|
|
||||||
/* We don't have any roots in data segments, so don't scan from
|
|
||||||
there. */
|
|
||||||
GC_set_no_dls(1);
|
|
||||||
|
|
||||||
GC_INIT();
|
|
||||||
|
|
||||||
GC_set_oom_fn(oomHandler);
|
|
||||||
|
|
||||||
/* Set the initial heap size to something fairly big (25% of
|
|
||||||
physical RAM, up to a maximum of 384 MiB) so that in most cases
|
|
||||||
we don't need to garbage collect at all. (Collection has a
|
|
||||||
fairly significant overhead.) The heap size can be overridden
|
|
||||||
through libgc's GC_INITIAL_HEAP_SIZE environment variable. We
|
|
||||||
should probably also provide a nix.conf setting for this. Note
|
|
||||||
that GC_expand_hp() causes a lot of virtual, but not physical
|
|
||||||
(resident) memory to be allocated. This might be a problem on
|
|
||||||
systems that don't overcommit. */
|
|
||||||
if (getenv("GC_INITIAL_HEAP_SIZE") == nullptr) {
|
|
||||||
size_t size = 32 * 1024 * 1024;
|
|
||||||
#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES)
|
|
||||||
size_t maxSize = 384 * 1024 * 1024;
|
|
||||||
long pageSize = sysconf(_SC_PAGESIZE);
|
|
||||||
long pages = sysconf(_SC_PHYS_PAGES);
|
|
||||||
if (pageSize != -1) {
|
|
||||||
size = (pageSize * pages) / 4;
|
|
||||||
} // 25% of RAM
|
|
||||||
if (size > maxSize) {
|
|
||||||
size = maxSize;
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
DLOG(INFO) << "setting initial heap size to " << size << " bytes";
|
|
||||||
GC_expand_hp(size);
|
|
||||||
}
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
||||||
gcInitialised = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Very hacky way to parse $NIX_PATH, which is colon-separated, but
|
/* Very hacky way to parse $NIX_PATH, which is colon-separated, but
|
||||||
can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
|
can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
|
||||||
static Strings parseNixPath(const std::string& s) {
|
static Strings parseNixPath(const std::string& s) {
|
||||||
|
@ -334,9 +333,9 @@ EvalState::EvalState(const Strings& _searchPath, const ref<Store>& store)
|
||||||
store(store),
|
store(store),
|
||||||
baseEnv(allocEnv(128)),
|
baseEnv(allocEnv(128)),
|
||||||
staticBaseEnv(false, nullptr) {
|
staticBaseEnv(false, nullptr) {
|
||||||
countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0";
|
expr::InitGC();
|
||||||
|
|
||||||
assert(gcInitialised);
|
countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0";
|
||||||
|
|
||||||
/* Initialise the Nix expression search path. */
|
/* Initialise the Nix expression search path. */
|
||||||
if (!evalSettings.pureEval) {
|
if (!evalSettings.pureEval) {
|
||||||
|
|
10
third_party/nix/src/libexpr/eval.hh
vendored
10
third_party/nix/src/libexpr/eval.hh
vendored
|
@ -16,6 +16,13 @@
|
||||||
#include "libutil/hash.hh"
|
#include "libutil/hash.hh"
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
namespace expr {
|
||||||
|
|
||||||
|
// Initialise the Boehm GC once per program instance. This should be
|
||||||
|
// called in places that require the garbage collector.
|
||||||
|
void InitGC();
|
||||||
|
|
||||||
|
} // namespace expr
|
||||||
|
|
||||||
class Store;
|
class Store;
|
||||||
class EvalState;
|
class EvalState;
|
||||||
|
@ -59,9 +66,6 @@ std::ostream& operator<<(std::ostream& str, const Value& v);
|
||||||
typedef std::pair<std::string, std::string> SearchPathElem;
|
typedef std::pair<std::string, std::string> SearchPathElem;
|
||||||
typedef std::list<SearchPathElem> SearchPath;
|
typedef std::list<SearchPathElem> SearchPath;
|
||||||
|
|
||||||
/* Initialise the Boehm GC, if applicable. */
|
|
||||||
void initGC();
|
|
||||||
|
|
||||||
typedef std::map<Path, Expr*, std::less<Path>,
|
typedef std::map<Path, Expr*, std::less<Path>,
|
||||||
traceable_allocator<std::pair<const Path, Expr*>>>
|
traceable_allocator<std::pair<const Path, Expr*>>>
|
||||||
FileParseCache;
|
FileParseCache;
|
||||||
|
|
1
third_party/nix/src/nix/main.cc
vendored
1
third_party/nix/src/nix/main.cc
vendored
|
@ -126,7 +126,6 @@ void mainWrapped(int argc, char** argv) {
|
||||||
}
|
}
|
||||||
|
|
||||||
initNix();
|
initNix();
|
||||||
initGC();
|
|
||||||
|
|
||||||
programPath = argv[0];
|
programPath = argv[0];
|
||||||
std::string programName = baseNameOf(programPath);
|
std::string programName = baseNameOf(programPath);
|
||||||
|
|
2
third_party/nix/src/tests/attr-set.cc
vendored
2
third_party/nix/src/tests/attr-set.cc
vendored
|
@ -115,7 +115,7 @@ class AttrSetTest : public ::testing::Test {
|
||||||
protected:
|
protected:
|
||||||
EvalState* eval_state_;
|
EvalState* eval_state_;
|
||||||
void SetUp() override {
|
void SetUp() override {
|
||||||
nix::initGC();
|
nix::expr::InitGC();
|
||||||
auto store = std::make_shared<DummyStore>();
|
auto store = std::make_shared<DummyStore>();
|
||||||
eval_state_ = new EvalState({"."}, ref<Store>(store));
|
eval_state_ = new EvalState({"."}, ref<Store>(store));
|
||||||
symbol_table = &eval_state_->symbols;
|
symbol_table = &eval_state_->symbols;
|
||||||
|
|
2
third_party/nix/src/tests/language-tests.cc
vendored
2
third_party/nix/src/tests/language-tests.cc
vendored
|
@ -111,7 +111,7 @@ class NixEnvironment : public testing::Environment {
|
||||||
public:
|
public:
|
||||||
void SetUp() override {
|
void SetUp() override {
|
||||||
google::InitGoogleLogging("--logtostderr=false");
|
google::InitGoogleLogging("--logtostderr=false");
|
||||||
nix::initGC();
|
nix::expr::InitGC();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
2
third_party/nix/src/tests/value-to-json.cc
vendored
2
third_party/nix/src/tests/value-to-json.cc
vendored
|
@ -12,7 +12,7 @@
|
||||||
|
|
||||||
class ValueTest : public ::testing::Test {
|
class ValueTest : public ::testing::Test {
|
||||||
protected:
|
protected:
|
||||||
static void SetUpTestCase() { nix::initGC(); }
|
static void SetUpTestCase() { nix::expr::InitGC(); }
|
||||||
|
|
||||||
static void TearDownTestCase() {}
|
static void TearDownTestCase() {}
|
||||||
};
|
};
|
||||||
|
|
Loading…
Reference in a new issue