mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(3p/nix/libstore): Replace logging.h with glog

Browse source
This commit is contained in:
Vincent Ambo 2020-05-19 01:02:44 +01:00
parent ce99ba42df
commit 505b6b044b
25 changed files with 514 additions and 464 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
third_party/nix/src/libstore/binary-cache-store.cc vendored
View file

@ -6,6 +6,7 @@
#include "derivations.hh" #include "derivations.hh"
#include "fs-accessor.hh" #include "fs-accessor.hh"
#include "globals.hh" #include "globals.hh"
#include "glog/logging.h"
#include "json.hh" #include "json.hh"
#include "nar-accessor.hh" #include "nar-accessor.hh"
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
@ -176,12 +177,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo& info,
auto duration = auto duration =
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count(); .count();
printMsg(lvlTalkative, DLOG(INFO) << "copying path '" << narInfo->path << "' (" << narInfo->narSize
format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% " << " bytes, compressed "
"ms) to binary cache") % << ((1.0 - (double)narCompressed->size() / nar->size()) * 100.0)
narInfo->path % narInfo->narSize % << "% in " << duration << "ms) to binary cache";
((1.0 - (double)narCompressed->size() / nar->size()) * 100.0) %
duration);
/* Atomically write the NAR file. */ /* Atomically write the NAR file. */
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" +
@ -243,35 +242,28 @@ void BinaryCacheStore::queryPathInfoUncached(
const Path& storePath, const Path& storePath,
Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {
auto uri = getUri(); auto uri = getUri();
auto act = std::make_shared<Activity>( LOG(INFO) << "querying info about '" << storePath << "' on '" << uri << "'";
*logger, lvlTalkative, actQueryPathInfo,
fmt("querying info about '%s' on '%s'", storePath, uri),
Logger::Fields{storePath, uri});
PushActivity pact(act->id);
auto narInfoFile = narInfoFileFor(storePath); auto narInfoFile = narInfoFileFor(storePath);
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
getFile( getFile(narInfoFile, {[=](std::future<std::shared_ptr<std::string>> fut) {
narInfoFile, {[=](std::future<std::shared_ptr<std::string>> fut) { try {
try { auto data = fut.get();
auto data = fut.get();
if (!data) return (*callbackPtr)(nullptr); if (!data) return (*callbackPtr)(nullptr);
stats.narInfoRead++; stats.narInfoRead++;
(*callbackPtr)( (*callbackPtr)(
(std::shared_ptr<ValidPathInfo>)std::make_shared<NarInfo>( (std::shared_ptr<ValidPathInfo>)std::make_shared<NarInfo>(
*this, *data, narInfoFile)); *this, *data, narInfoFile));
(void) } catch (...) {
act; // force Activity into this lambda to ensure it stays alive callbackPtr->rethrow();
} catch (...) { }
callbackPtr->rethrow(); }});
}
}});
} }
Path BinaryCacheStore::addToStore(const string& name, const Path& srcPath, Path BinaryCacheStore::addToStore(const string& name, const Path& srcPath,
@ -356,7 +348,8 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path& path) {
auto logPath = "log/" + baseNameOf(drvPath); auto logPath = "log/" + baseNameOf(drvPath);
debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); DLOG(INFO) << "fetching build log from binary cache '" << getUri() << "/"
<< logPath << "'";
return getFile(logPath); return getFile(logPath);
} }

296
third_party/nix/src/libstore/build.cc vendored
View file

@ -31,6 +31,7 @@
#include "download.hh" #include "download.hh"
#include "finally.hh" #include "finally.hh"
#include "globals.hh" #include "globals.hh"
#include "glog/logging.h"
#include "json.hh" #include "json.hh"
#include "local-store.hh" #include "local-store.hh"
#include "machines.hh" #include "machines.hh"
@ -226,10 +227,6 @@ class Worker {
std::map<Path, bool> pathContentsGoodCache; std::map<Path, bool> pathContentsGoodCache;
public: public:
const Activity act;
const Activity actDerivations;
const Activity actSubstitutions;
/* Set if at least one derivation had a BuildError (i.e. permanent /* Set if at least one derivation had a BuildError (i.e. permanent
failure). */ failure). */
bool permanentFailure; bool permanentFailure;
@ -327,16 +324,6 @@ class Worker {
bool pathContentsGood(const Path& path); bool pathContentsGood(const Path& path);
void markContentsGood(const Path& path); void markContentsGood(const Path& path);
void updateProgress() {
actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds,
runningBuilds, failedBuilds);
actSubstitutions.progress(doneSubstitutions,
expectedSubstitutions + doneSubstitutions,
runningSubstitutions, failedSubstitutions);
act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize);
act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
}
}; };
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
@ -397,7 +384,9 @@ void Goal::amDone(ExitCode result) {
worker.removeGoal(shared_from_this()); worker.removeGoal(shared_from_this());
} }
void Goal::trace(const FormatOrString& fs) { debug("%1%: %2%", name, fs.s); } void Goal::trace(const FormatOrString& fs) {
DLOG(INFO) << name << ": " << fs.s;
}
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
@ -445,9 +434,11 @@ void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath,
fmt("diff-hook program '%1%' %2%", diffHook, fmt("diff-hook program '%1%' %2%", diffHook,
statusToString(diffRes.first))); statusToString(diffRes.first)));
if (diffRes.second != "") printError(chomp(diffRes.second)); if (diffRes.second != "") {
LOG(ERROR) << chomp(diffRes.second);
}
} catch (Error& error) { } catch (Error& error) {
printError("diff hook execution failed: %s", error.what()); LOG(ERROR) << "diff hook execution failed: " << error.what();
} }
} }
} }
@ -507,7 +498,7 @@ UserLock::UserLock() {
/* Copy the result of getgrnam. */ /* Copy the result of getgrnam. */
Strings users; Strings users;
for (char** p = gr->gr_mem; *p; ++p) { for (char** p = gr->gr_mem; *p; ++p) {
debug(format("found build user '%1%'") % *p); DLOG(INFO) << "found build user " << *p;
users.push_back(*p); users.push_back(*p);
} }
@ -518,7 +509,7 @@ UserLock::UserLock() {
/* Find a user account that isn't currently in use for another /* Find a user account that isn't currently in use for another
build. */ build. */
for (auto& i : users) { for (auto& i : users) {
debug(format("trying user '%1%'") % i); DLOG(INFO) << "trying user " << i;
struct passwd* pw = getpwnam(i.c_str()); struct passwd* pw = getpwnam(i.c_str());
if (!pw) if (!pw)
@ -607,15 +598,13 @@ struct HookInstance {
FdSink sink; FdSink sink;
std::map<ActivityId, Activity> activities;
HookInstance(); HookInstance();
~HookInstance(); ~HookInstance();
}; };
HookInstance::HookInstance() { HookInstance::HookInstance() {
debug("starting build hook '%s'", settings.buildHook); DLOG(INFO) << "starting build hook " << settings.buildHook;
/* Create a pipe to get the output of the child. */ /* Create a pipe to get the output of the child. */
fromHook.create(); fromHook.create();
@ -630,24 +619,29 @@ HookInstance::HookInstance() {
pid = startProcess([&]() { pid = startProcess([&]() {
commonChildInit(fromHook); commonChildInit(fromHook);
if (chdir("/") == -1) throw SysError("changing into /"); if (chdir("/") == -1) {
throw SysError("changing into /");
}
/* Dup the communication pipes. */ /* Dup the communication pipes. */
if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) {
throw SysError("dupping to-hook read side"); throw SysError("dupping to-hook read side");
}
/* Use fd 4 for the builder's stdout/stderr. */ /* Use fd 4 for the builder's stdout/stderr. */
if (dup2(builderOut.writeSide.get(), 4) == -1) if (dup2(builderOut.writeSide.get(), 4) == -1) {
throw SysError("dupping builder's stdout/stderr"); throw SysError("dupping builder's stdout/stderr");
}
/* Hack: pass the read side of that fd to allow build-remote /* Hack: pass the read side of that fd to allow build-remote
to read SSH error messages. */ to read SSH error messages. */
if (dup2(builderOut.readSide.get(), 5) == -1) if (dup2(builderOut.readSide.get(), 5) == -1) {
throw SysError("dupping builder's stdout/stderr"); throw SysError("dupping builder's stdout/stderr");
}
Strings args = { Strings args = {
baseNameOf(settings.buildHook), baseNameOf(settings.buildHook),
std::to_string(verbosity), // std::to_string(verbosity), // TODO(tazjin): what?
}; };
execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data());
@ -842,10 +836,6 @@ class DerivationGoal : public Goal {
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
std::unique_ptr<Activity> act;
std::map<ActivityId, Activity> builderActivities;
/* The remote machine on which we're building. */ /* The remote machine on which we're building. */
std::string machineName; std::string machineName;
@ -972,7 +962,6 @@ DerivationGoal::DerivationGoal(const Path& drvPath,
mcExpectedBuilds = mcExpectedBuilds =
std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
worker.updateProgress();
} }
DerivationGoal::DerivationGoal(const Path& drvPath, const BasicDerivation& drv, DerivationGoal::DerivationGoal(const Path& drvPath, const BasicDerivation& drv,
@ -988,7 +977,6 @@ DerivationGoal::DerivationGoal(const Path& drvPath, const BasicDerivation& drv,
mcExpectedBuilds = mcExpectedBuilds =
std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
worker.updateProgress();
/* Prevent the .chroot directory from being /* Prevent the .chroot directory from being
garbage-collected. (See isActiveTempFile() in gc.cc.) */ garbage-collected. (See isActiveTempFile() in gc.cc.) */
@ -1089,7 +1077,7 @@ void DerivationGoal::loadDerivation() {
trace("loading derivation"); trace("loading derivation");
if (nrFailed != 0) { if (nrFailed != 0) {
printError(format("cannot build missing derivation '%1%'") % drvPath); LOG(ERROR) << "cannot build missing derivation '" << drvPath << "'";
done(BuildResult::MiscFailure); done(BuildResult::MiscFailure);
return; return;
} }
@ -1241,9 +1229,8 @@ void DerivationGoal::repairClosure() {
PathSet broken; PathSet broken;
for (auto& i : outputClosure) { for (auto& i : outputClosure) {
if (worker.pathContentsGood(i)) continue; if (worker.pathContentsGood(i)) continue;
printError(format("found corrupted or missing path '%1%' in the output " LOG(ERROR) << "found corrupted or missing path '" << i
"closure of '%2%'") % << "' in the output closure of '" << drvPath << "'";
i % drvPath);
Path drvPath2 = outputsToDrv[i]; Path drvPath2 = outputsToDrv[i];
if (drvPath2 == "") if (drvPath2 == "")
addWaitee(worker.makeSubstitutionGoal(i, Repair)); addWaitee(worker.makeSubstitutionGoal(i, Repair));
@ -1274,9 +1261,8 @@ void DerivationGoal::inputsRealised() {
if (nrFailed != 0) { if (nrFailed != 0) {
if (!useDerivation) if (!useDerivation)
throw Error(format("some dependencies of '%1%' are missing") % drvPath); throw Error(format("some dependencies of '%1%' are missing") % drvPath);
printError(format("cannot build derivation '%1%': %2% dependencies " LOG(ERROR) << "cannot build derivation '" << drvPath << "': " << nrFailed
"couldn't be built") % << " dependencies couldn't be built";
drvPath % nrFailed);
done(BuildResult::DependencyFailed); done(BuildResult::DependencyFailed);
return; return;
} }
@ -1291,7 +1277,7 @@ void DerivationGoal::inputsRealised() {
/* The outputs are referenceable paths. */ /* The outputs are referenceable paths. */
for (auto& i : drv->outputs) { for (auto& i : drv->outputs) {
debug(format("building path '%1%'") % i.second.path); DLOG(INFO) << "building path " << i.second.path;
allPaths.insert(i.second.path); allPaths.insert(i.second.path);
} }
@ -1317,7 +1303,7 @@ void DerivationGoal::inputsRealised() {
/* Second, the input sources. */ /* Second, the input sources. */
worker.store.computeFSClosure(drv->inputSrcs, inputPaths); worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
debug(format("added input paths %1%") % showPaths(inputPaths)); DLOG(INFO) << "added input paths " << showPaths(inputPaths);
allPaths.insert(inputPaths.begin(), inputPaths.end()); allPaths.insert(inputPaths.begin(), inputPaths.end());
@ -1363,8 +1349,8 @@ void DerivationGoal::tryToBuild() {
build this derivation, so no further checks are necessary. */ build this derivation, so no further checks are necessary. */
validPaths = checkPathValidity(true, buildMode == bmRepair); validPaths = checkPathValidity(true, buildMode == bmRepair);
if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) {
debug(format("skipping build of derivation '%1%', someone beat us to it") % DLOG(INFO) << "skipping build of derivation '" << drvPath
drvPath); << "', someone beat us to it";
outputLocks.setDeletion(true); outputLocks.setDeletion(true);
done(BuildResult::AlreadyValid); done(BuildResult::AlreadyValid);
return; return;
@ -1379,7 +1365,7 @@ void DerivationGoal::tryToBuild() {
for (auto& i : drv->outputs) { for (auto& i : drv->outputs) {
Path path = i.second.path; Path path = i.second.path;
if (worker.store.isValidPath(path)) continue; if (worker.store.isValidPath(path)) continue;
debug(format("removing invalid path '%1%'") % path); DLOG(INFO) << "removing invalid path " << path;
deletePath(worker.store.toRealPath(path)); deletePath(worker.store.toRealPath(path));
} }
@ -1396,14 +1382,13 @@ void DerivationGoal::tryToBuild() {
: nrRounds > 1 ? "building '%s' (round %d/%d)" : nrRounds > 1 ? "building '%s' (round %d/%d)"
: "building '%s'", : "building '%s'",
drvPath, curRound, nrRounds); drvPath, curRound, nrRounds);
fmt("building '%s'", drvPath);
if (hook) msg += fmt(" on '%s'", machineName); if (hook) {
act = std::make_unique<Activity>( msg += fmt(" on '%s'", machineName);
*logger, lvlInfo, actBuild, msg, }
Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds}); LOG(INFO) << msg << "[" << drvPath << "]";
mcRunningBuilds = mcRunningBuilds =
std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
worker.updateProgress();
}; };
/* Is the build hook willing to accept this job? */ /* Is the build hook willing to accept this job? */
@ -1443,7 +1428,7 @@ void DerivationGoal::tryToBuild() {
startBuilder(); startBuilder();
} catch (BuildError& e) { } catch (BuildError& e) {
printError(e.msg()); LOG(ERROR) << e.msg();
outputLocks.unlock(); outputLocks.unlock();
buildUser.reset(); buildUser.reset();
worker.permanentFailure = true; worker.permanentFailure = true;
@ -1487,7 +1472,7 @@ MakeError(NotDeterministic, BuildError)
kill it. */ kill it. */
int status = hook ? hook->pid.kill() : pid.kill(); int status = hook ? hook->pid.kill() : pid.kill();
debug(format("builder process for '%1%' finished") % drvPath); DLOG(INFO) << "builder process for '" << drvPath << "' finished";
result.timesBuilt++; result.timesBuilt++;
result.stopTime = time(0); result.stopTime = time(0);
@ -1552,10 +1537,11 @@ MakeError(NotDeterministic, BuildError)
for (auto& line : logTail) msg += "\n " + line; for (auto& line : logTail) msg += "\n " + line;
} }
if (diskFull) if (diskFull) {
msg += msg +=
"\nnote: build failure may have been caused by lack of free disk " "\nnote: build failure may have been caused by lack of free disk "
"space"; "space";
}
throw BuildError(msg); throw BuildError(msg);
} }
@ -1565,10 +1551,8 @@ MakeError(NotDeterministic, BuildError)
registerOutputs(); registerOutputs();
if (settings.postBuildHook != "") { if (settings.postBuildHook != "") {
Activity act(*logger, lvlInfo, actPostBuildHook, LOG(INFO) << "running post-build-hook '" << settings.postBuildHook
fmt("running post-build-hook '%s'", settings.postBuildHook), << "' [" << drvPath << "]";
Logger::Fields{drvPath});
PushActivity pact(act.id);
auto outputPaths = drv->outputPaths(); auto outputPaths = drv->outputPaths();
std::map<std::string, std::string> hookEnvironment = getEnv(); std::map<std::string, std::string> hookEnvironment = getEnv();
@ -1580,11 +1564,8 @@ MakeError(NotDeterministic, BuildError)
opts.environment = hookEnvironment; opts.environment = hookEnvironment;
struct LogSink : Sink { struct LogSink : Sink {
Activity& act;
std::string currentLine; std::string currentLine;
LogSink(Activity& act) : act(act) {}
void operator()(const unsigned char* data, size_t len) override { void operator()(const unsigned char* data, size_t len) override {
for (size_t i = 0; i < len; i++) { for (size_t i = 0; i < len; i++) {
auto c = data[i]; auto c = data[i];
@ -1599,9 +1580,7 @@ MakeError(NotDeterministic, BuildError)
void flushLine() { void flushLine() {
if (settings.verboseBuild) { if (settings.verboseBuild) {
printError("post-build-hook: " + currentLine); LOG(ERROR) << "post-build-hook: " << currentLine;
} else {
act.result(resPostBuildLogLine, currentLine);
} }
currentLine.clear(); currentLine.clear();
} }
@ -1613,7 +1592,7 @@ MakeError(NotDeterministic, BuildError)
} }
} }
}; };
LogSink sink(act); LogSink sink;
opts.standardOut = &sink; opts.standardOut = &sink;
opts.mergeStderrToStdout = true; opts.mergeStderrToStdout = true;
@ -1649,7 +1628,7 @@ MakeError(NotDeterministic, BuildError)
outputLocks.unlock(); outputLocks.unlock();
} catch (BuildError& e) { } catch (BuildError& e) {
printError(e.msg()); LOG(ERROR) << e.msg();
outputLocks.unlock(); outputLocks.unlock();
@ -1696,38 +1675,37 @@ HookReply DerivationGoal::tryBuildHook() {
string reply; string reply;
while (true) { while (true) {
string s = readLine(worker.hook->fromHook.readSide.get()); string s = readLine(worker.hook->fromHook.readSide.get());
if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) if (string(s, 0, 2) == "# ") {
;
else if (string(s, 0, 2) == "# ") {
reply = string(s, 2); reply = string(s, 2);
break; break;
} else { } else {
s += "\n"; s += "\n";
writeToStderr(s); std::cerr << s;
} }
} }
debug(format("hook reply is '%1%'") % reply); DLOG(INFO) << "hook reply is " << reply;
if (reply == "decline") if (reply == "decline") {
return rpDecline; return rpDecline;
else if (reply == "decline-permanently") { } else if (reply == "decline-permanently") {
worker.tryBuildHook = false; worker.tryBuildHook = false;
worker.hook = 0; worker.hook = 0;
return rpDecline; return rpDecline;
} else if (reply == "postpone") } else if (reply == "postpone") {
return rpPostpone; return rpPostpone;
else if (reply != "accept") } else if (reply != "accept") {
throw Error(format("bad hook reply '%1%'") % reply); throw Error(format("bad hook reply '%1%'") % reply);
}
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo == EPIPE) { if (e.errNo == EPIPE) {
printError("build hook died unexpectedly: %s", LOG(ERROR) << "build hook died unexpectedly: "
chomp(drainFD(worker.hook->fromHook.readSide.get()))); << chomp(drainFD(worker.hook->fromHook.readSide.get()));
worker.hook = 0; worker.hook = 0;
return rpDecline; return rpDecline;
} else } else {
throw; throw;
}
} }
hook = std::move(worker.hook); hook = std::move(worker.hook);
@ -2007,8 +1985,7 @@ void DerivationGoal::startBuilder() {
/* Clean up the chroot directory automatically. */ /* Clean up the chroot directory automatically. */
autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir); autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
printMsg(lvlChatty, DLOG(INFO) << "setting up chroot environment in '" << chrootRootDir << "'";
format("setting up chroot environment in '%1%'") % chrootRootDir);
if (mkdir(chrootRootDir.c_str(), 0750) == -1) if (mkdir(chrootRootDir.c_str(), 0750) == -1)
throw SysError(format("cannot create '%1%'") % chrootRootDir); throw SysError(format("cannot create '%1%'") % chrootRootDir);
@ -2073,7 +2050,7 @@ void DerivationGoal::startBuilder() {
dirsInChroot[i] = r; dirsInChroot[i] = r;
else { else {
Path p = chrootRootDir + i; Path p = chrootRootDir + i;
debug("linking '%1%' to '%2%'", p, r); DLOG(INFO) << "linking '" << p << "' to '" << r << "'";
if (link(r.c_str(), p.c_str()) == -1) { if (link(r.c_str(), p.c_str()) == -1) {
/* Hard-linking fails if we exceed the maximum /* Hard-linking fails if we exceed the maximum
link count on a file (e.g. 32000 of ext3), link count on a file (e.g. 32000 of ext3),
@ -2132,8 +2109,7 @@ void DerivationGoal::startBuilder() {
if (useChroot && settings.preBuildHook != "" && if (useChroot && settings.preBuildHook != "" &&
dynamic_cast<Derivation*>(drv.get())) { dynamic_cast<Derivation*>(drv.get())) {
printMsg(lvlChatty, DLOG(INFO) << "executing pre-build hook '" << settings.preBuildHook << "'";
format("executing pre-build hook '%1%'") % settings.preBuildHook);
auto args = auto args =
useChroot ? Strings({drvPath, chrootRootDir}) : Strings({drvPath}); useChroot ? Strings({drvPath, chrootRootDir}) : Strings({drvPath});
enum BuildHookState { stBegin, stExtraChrootDirs }; enum BuildHookState { stBegin, stExtraChrootDirs };
@ -2165,7 +2141,7 @@ void DerivationGoal::startBuilder() {
} }
/* Run the builder. */ /* Run the builder. */
printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); DLOG(INFO) << "executing builder '" << drv->builder << "'";
/* Create the log file. */ /* Create the log file. */
Path logFile = openLogFile(); Path logFile = openLogFile();
@ -2364,7 +2340,7 @@ void DerivationGoal::startBuilder() {
if (msg.size() == 1) break; if (msg.size() == 1) break;
throw Error(string(msg, 1)); throw Error(string(msg, 1));
} }
debug(msg); DLOG(INFO) << msg;
} }
} }
@ -2604,10 +2580,10 @@ void setupSeccomp() {
throw SysError("unable to add X32 seccomp architecture"); throw SysError("unable to add X32 seccomp architecture");
if (nativeSystem == "aarch64-linux" && if (nativeSystem == "aarch64-linux" &&
seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) {
printError( LOG(ERROR) << "unable to add ARM seccomp architecture; this may result in "
"unable to add ARM seccomp architecture; this may result in spurious " << "spurious build failures if running 32-bit ARM processes";
"build failures if running 32-bit ARM processes"); }
/* Prevent builders from creating setuid/setgid binaries. */ /* Prevent builders from creating setuid/setgid binaries. */
for (int perm : {S_ISUID, S_ISGID}) { for (int perm : {S_ISUID, S_ISGID}) {
@ -2772,7 +2748,7 @@ void DerivationGoal::runChild() {
environment. */ environment. */
auto doBind = [&](const Path& source, const Path& target, auto doBind = [&](const Path& source, const Path& target,
bool optional = false) { bool optional = false) {
debug(format("bind mounting '%1%' to '%2%'") % source % target); DLOG(INFO) << "bind mounting '" << source << "' to '" << target << "'";
struct stat st; struct stat st;
if (stat(source.c_str(), &st) == -1) { if (stat(source.c_str(), &st) == -1) {
if (optional && errno == ENOENT) if (optional && errno == ENOENT)
@ -3082,8 +3058,6 @@ void DerivationGoal::runChild() {
/* Execute the program. This should not return. */ /* Execute the program. This should not return. */
if (drv->isBuiltin()) { if (drv->isBuiltin()) {
try { try {
logger = makeJSONLogger(*logger);
BasicDerivation drv2(*drv); BasicDerivation drv2(*drv);
for (auto& e : drv2.env) for (auto& e : drv2.env)
e.second = rewriteStrings(e.second, inputRewrites); e.second = rewriteStrings(e.second, inputRewrites);
@ -3215,8 +3189,7 @@ void DerivationGoal::registerOutputs() {
/* Apply hash rewriting if necessary. */ /* Apply hash rewriting if necessary. */
bool rewritten = false; bool rewritten = false;
if (!outputRewrites.empty()) { if (!outputRewrites.empty()) {
printError(format("warning: rewriting hashes in '%1%'; cross fingers") % LOG(WARNING) << "rewriting hashes in '" << path << "'; cross fingers";
path);
/* Canonicalise first. This ensures that the path we're /* Canonicalise first. This ensures that the path we're
rewriting doesn't contain a hard link to /etc/shadow or rewriting doesn't contain a hard link to /etc/shadow or
@ -3300,7 +3273,7 @@ void DerivationGoal::registerOutputs() {
contained in it. Compute the SHA-256 NAR hash at the same contained in it. Compute the SHA-256 NAR hash at the same
time. The hash is stored in the database so that we can time. The hash is stored in the database so that we can
verify later on whether nobody has messed with the store. */ verify later on whether nobody has messed with the store. */
debug("scanning for references inside '%1%'", path); DLOG(INFO) << "scanning for references inside '" << path << "'";
HashResult hash; HashResult hash;
PathSet references = scanForReferences(actualPath, allPaths, hash); PathSet references = scanForReferences(actualPath, allPaths, hash);
@ -3346,9 +3319,9 @@ void DerivationGoal::registerOutputs() {
for (auto& i : inputPaths) { for (auto& i : inputPaths) {
PathSet::iterator j = references.find(i); PathSet::iterator j = references.find(i);
if (j == references.end()) if (j == references.end())
debug(format("unreferenced input: '%1%'") % i); DLOG(INFO) << "unreferenced input: '" << i << "'";
else else
debug(format("referenced input: '%1%'") % i); DLOG(INFO) << "referenced input: '" << i << "'";
} }
if (curRound == nrRounds) { if (curRound == nrRounds) {
@ -3397,9 +3370,11 @@ void DerivationGoal::registerOutputs() {
buildUser ? buildUser->getGID() : getgid(), prev, buildUser ? buildUser->getGID() : getgid(), prev,
i->second.path, drvPath, tmpDir); i->second.path, drvPath, tmpDir);
if (settings.enforceDeterminism) throw NotDeterministic(msg); if (settings.enforceDeterminism) {
throw NotDeterministic(msg);
}
printError(msg); LOG(ERROR) << msg;
curRound = nrRounds; // we know enough, bail out early curRound = nrRounds; // we know enough, bail out early
} }
} }
@ -3644,8 +3619,7 @@ void DerivationGoal::deleteTmpDir(bool force) {
/* Don't keep temporary directories for builtins because they /* Don't keep temporary directories for builtins because they
might have privileged stuff (like a copy of netrc). */ might have privileged stuff (like a copy of netrc). */
if (settings.keepFailed && !force && !drv->isBuiltin()) { if (settings.keepFailed && !force && !drv->isBuiltin()) {
printError(format("note: keeping build directory '%2%'") % drvPath % LOG(INFO) << "note: keeping build directory '" << tmpDir << "'";
tmpDir);
chmod(tmpDir.c_str(), 0755); chmod(tmpDir.c_str(), 0755);
} else } else
deletePath(tmpDir); deletePath(tmpDir);
@ -3658,20 +3632,20 @@ void DerivationGoal::handleChildOutput(int fd, const string& data) {
(!hook && fd == builderOut.readSide.get())) { (!hook && fd == builderOut.readSide.get())) {
logSize += data.size(); logSize += data.size();
if (settings.maxLogSize && logSize > settings.maxLogSize) { if (settings.maxLogSize && logSize > settings.maxLogSize) {
printError( LOG(ERROR) << getName()
format("%1% killed after writing more than %2% bytes of log output") % << " killed after writing more than %2% bytes of log output"
getName() % settings.maxLogSize); << settings.maxLogSize;
killChild(); killChild();
done(BuildResult::LogLimitExceeded); done(BuildResult::LogLimitExceeded);
return; return;
} }
for (auto c : data) for (auto c : data)
if (c == '\r') if (c == '\r') {
currentLogLinePos = 0; currentLogLinePos = 0;
else if (c == '\n') } else if (c == '\n') {
flushLine(); flushLine();
else { } else {
if (currentLogLinePos >= currentLogLine.size()) if (currentLogLinePos >= currentLogLine.size())
currentLogLine.resize(currentLogLinePos + 1); currentLogLine.resize(currentLogLinePos + 1);
currentLogLine[currentLogLinePos++] = c; currentLogLine[currentLogLinePos++] = c;
@ -3683,11 +3657,10 @@ void DerivationGoal::handleChildOutput(int fd, const string& data) {
if (hook && fd == hook->fromHook.readSide.get()) { if (hook && fd == hook->fromHook.readSide.get()) {
for (auto c : data) for (auto c : data)
if (c == '\n') { if (c == '\n') {
handleJSONLogMessage(currentHookLine, worker.act, hook->activities,
true);
currentHookLine.clear(); currentHookLine.clear();
} else } else {
currentHookLine += c; currentHookLine += c;
}
} }
} }
@ -3697,19 +3670,11 @@ void DerivationGoal::handleEOF(int fd) {
} }
void DerivationGoal::flushLine() { void DerivationGoal::flushLine() {
if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) if (settings.verboseBuild && (settings.printRepeatedBuilds || curRound == 1))
; LOG(INFO) << currentLogLine;
else { else {
if (settings.verboseBuild && logTail.push_back(currentLogLine);
(settings.printRepeatedBuilds || curRound == 1)) if (logTail.size() > settings.logLines) logTail.pop_front();
printError(currentLogLine);
else {
logTail.push_back(currentLogLine);
if (logTail.size() > settings.logLines) logTail.pop_front();
}
act->result(resBuildLogLine, currentLogLine);
} }
currentLogLine = ""; currentLogLine = "";
@ -3719,10 +3684,14 @@ void DerivationGoal::flushLine() {
PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) {
PathSet result; PathSet result;
for (auto& i : drv->outputs) { for (auto& i : drv->outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue; if (!wantOutput(i.first, wantedOutputs)) {
continue;
}
bool good = worker.store.isValidPath(i.second.path) && bool good = worker.store.isValidPath(i.second.path) &&
(!checkHash || worker.pathContentsGood(i.second.path)); (!checkHash || worker.pathContentsGood(i.second.path));
if (good == returnValid) result.insert(i.second.path); if (good == returnValid) {
result.insert(i.second.path);
}
} }
return result; return result;
} }
@ -3758,8 +3727,6 @@ void DerivationGoal::done(BuildResult::Status status, const string& msg) {
} else { } else {
if (status != BuildResult::DependencyFailed) worker.failedBuilds++; if (status != BuildResult::DependencyFailed) worker.failedBuilds++;
} }
worker.updateProgress();
} }
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
@ -3891,9 +3858,9 @@ void SubstitutionGoal::tryNext() {
if (subs.size() == 0) { if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal /* None left. Terminate this goal and let someone else deal
with it. */ with it. */
debug(format("path '%1%' is required, but there is no substituter that can " DLOG(WARNING)
"build it") % << "path '" << storePath
storePath); << "' is required, but there is no substituter that can build it";
/* Hack: don't indicate failure if there were no substituters. /* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a In that case the calling derivation should just do a
@ -3902,7 +3869,6 @@ void SubstitutionGoal::tryNext() {
if (substituterFailed) { if (substituterFailed) {
worker.failedSubstitutions++; worker.failedSubstitutions++;
worker.updateProgress();
} }
return; return;
@ -3930,7 +3896,7 @@ void SubstitutionGoal::tryNext() {
throw; throw;
} catch (Error& e) { } catch (Error& e) {
if (settings.tryFallback) { if (settings.tryFallback) {
printError(e.what()); LOG(ERROR) << e.what();
tryNext(); tryNext();
return; return;
} }
@ -3949,17 +3915,14 @@ void SubstitutionGoal::tryNext() {
worker.expectedDownloadSize, narInfo->fileSize) worker.expectedDownloadSize, narInfo->fileSize)
: nullptr; : nullptr;
worker.updateProgress();
/* Bail out early if this substituter lacks a valid /* Bail out early if this substituter lacks a valid
signature. LocalStore::addToStore() also checks for this, but signature. LocalStore::addToStore() also checks for this, but
only after we've downloaded the path. */ only after we've downloaded the path. */
if (worker.store.requireSigs && !sub->isTrusted && if (worker.store.requireSigs && !sub->isTrusted &&
!info->checkSignatures(worker.store, worker.store.getPublicKeys())) { !info->checkSignatures(worker.store, worker.store.getPublicKeys())) {
printError( LOG(WARNING) << "substituter '" << sub->getUri()
"warning: substituter '%s' does not have a valid signature for path " << "' does not have a valid signature for path '" << storePath
"'%s'", << "'";
sub->getUri(), storePath);
tryNext(); tryNext();
return; return;
} }
@ -3980,8 +3943,8 @@ void SubstitutionGoal::referencesValid() {
trace("all references realised"); trace("all references realised");
if (nrFailed > 0) { if (nrFailed > 0) {
debug(format("some references of path '%1%' could not be realised") % DLOG(WARNING) << "some references of path '" << storePath
storePath); << "' could not be realised";
amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure
: ecFailed); : ecFailed);
return; return;
@ -4010,7 +3973,6 @@ void SubstitutionGoal::tryToRun() {
maintainRunningSubstitutions = maintainRunningSubstitutions =
std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
worker.updateProgress();
outPipe.create(); outPipe.create();
@ -4021,10 +3983,6 @@ void SubstitutionGoal::tryToRun() {
/* Wake up the worker loop when we're done. */ /* Wake up the worker loop when we're done. */
Finally updateStats([this]() { outPipe.writeSide = -1; }); Finally updateStats([this]() { outPipe.writeSide = -1; });
Activity act(*logger, actSubstitute,
Logger::Fields{storePath, sub->getUri()});
PushActivity pact(act.id);
copyStorePath(ref<Store>(sub), copyStorePath(ref<Store>(sub),
ref<Store>(worker.store.shared_from_this()), storePath, ref<Store>(worker.store.shared_from_this()), storePath,
repair, sub->isTrusted ? NoCheckSigs : CheckSigs); repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
@ -4050,7 +4008,7 @@ void SubstitutionGoal::finished() {
try { try {
promise.get_future().get(); promise.get_future().get();
} catch (std::exception& e) { } catch (std::exception& e) {
printError(e.what()); LOG(ERROR) << e.what();
/* Cause the parent build to fail unless --fallback is given, /* Cause the parent build to fail unless --fallback is given,
or the substitute has disappeared. The latter case behaves or the substitute has disappeared. The latter case behaves
@ -4071,8 +4029,7 @@ void SubstitutionGoal::finished() {
worker.markContentsGood(storePath); worker.markContentsGood(storePath);
printMsg(lvlChatty, DLOG(INFO) << "substitution of path '" << storePath << "' succeeded";
format("substitution of path '%1%' succeeded") % storePath);
maintainRunningSubstitutions.reset(); maintainRunningSubstitutions.reset();
@ -4088,8 +4045,6 @@ void SubstitutionGoal::finished() {
worker.doneNarSize += maintainExpectedNar->delta; worker.doneNarSize += maintainExpectedNar->delta;
maintainExpectedNar.reset(); maintainExpectedNar.reset();
worker.updateProgress();
amDone(ecSuccess); amDone(ecSuccess);
} }
@ -4103,11 +4058,7 @@ void SubstitutionGoal::handleEOF(int fd) {
static bool working = false; static bool working = false;
Worker::Worker(LocalStore& store) Worker::Worker(LocalStore& store) : store(store) {
: act(*logger, actRealise),
actDerivations(*logger, actBuilds),
actSubstitutions(*logger, actCopyPaths),
store(store) {
/* Debugging: prevent recursive workers. */ /* Debugging: prevent recursive workers. */
if (working) abort(); if (working) abort();
working = true; working = true;
@ -4242,7 +4193,7 @@ void Worker::childTerminated(Goal* goal, bool wakeSleepers) {
} }
void Worker::waitForBuildSlot(GoalPtr goal) { void Worker::waitForBuildSlot(GoalPtr goal) {
debug("wait for build slot"); DLOG(INFO) << "wait for build slot";
if (getNrLocalBuilds() < settings.maxBuildJobs) if (getNrLocalBuilds() < settings.maxBuildJobs)
wakeUp(goal); /* we can do it right away */ wakeUp(goal); /* we can do it right away */
else else
@ -4250,19 +4201,19 @@ void Worker::waitForBuildSlot(GoalPtr goal) {
} }
void Worker::waitForAnyGoal(GoalPtr goal) { void Worker::waitForAnyGoal(GoalPtr goal) {
debug("wait for any goal"); DLOG(INFO) << "wait for any goal";
addToWeakGoals(waitingForAnyGoal, goal); addToWeakGoals(waitingForAnyGoal, goal);
} }
void Worker::waitForAWhile(GoalPtr goal) { void Worker::waitForAWhile(GoalPtr goal) {
debug("wait for a while"); DLOG(INFO) << "wait for a while";
addToWeakGoals(waitingForAWhile, goal); addToWeakGoals(waitingForAWhile, goal);
} }
void Worker::run(const Goals& _topGoals) { void Worker::run(const Goals& _topGoals) {
for (auto& i : _topGoals) topGoals.insert(i); for (auto& i : _topGoals) topGoals.insert(i);
debug("entered goal loop"); DLOG(INFO) << "entered goal loop";
while (1) { while (1) {
checkInterrupt(); checkInterrupt();
@ -4308,7 +4259,7 @@ void Worker::run(const Goals& _topGoals) {
} }
void Worker::waitForInput() { void Worker::waitForInput() {
printMsg(lvlVomit, "waiting for children"); DLOG(INFO) << "waiting for children";
/* Process output from the file descriptors attached to the /* Process output from the file descriptors attached to the
children, namely log output and output path creation commands. children, namely log output and output path creation commands.
@ -4350,7 +4301,7 @@ void Worker::waitForInput() {
if (!waitingForAWhile.empty()) { if (!waitingForAWhile.empty()) {
useTimeout = true; useTimeout = true;
if (lastWokenUp == steady_time_point::min()) if (lastWokenUp == steady_time_point::min())
printError("waiting for locks or build slots..."); DLOG(WARNING) << "waiting for locks or build slots...";
if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) if (lastWokenUp == steady_time_point::min() || lastWokenUp > before)
lastWokenUp = before; lastWokenUp = before;
timeout.tv_sec = std::max( timeout.tv_sec = std::max(
@ -4361,7 +4312,9 @@ void Worker::waitForInput() {
} else } else
lastWokenUp = steady_time_point::min(); lastWokenUp = steady_time_point::min();
if (useTimeout) vomit("sleeping %d seconds", timeout.tv_sec); if (useTimeout) {
DLOG(INFO) << "sleeping " << timeout.tv_sec << " seconds";
}
/* Use select() to wait for the input side of any logger pipe to /* Use select() to wait for the input side of any logger pipe to
become `available'. Note that `available' (i.e., non-blocking) become `available'. Note that `available' (i.e., non-blocking)
@ -4403,15 +4356,14 @@ void Worker::waitForInput() {
// FIXME: is there a cleaner way to handle pt close // FIXME: is there a cleaner way to handle pt close
// than EIO? Is this even standard? // than EIO? Is this even standard?
if (rd == 0 || (rd == -1 && errno == EIO)) { if (rd == 0 || (rd == -1 && errno == EIO)) {
debug(format("%1%: got EOF") % goal->getName()); DLOG(WARNING) << goal->getName() << ": got EOF";
goal->handleEOF(k); goal->handleEOF(k);
j->fds.erase(k); j->fds.erase(k);
} else if (rd == -1) { } else if (rd == -1) {
if (errno != EINTR) if (errno != EINTR)
throw SysError("%s: read failed", goal->getName()); throw SysError("%s: read failed", goal->getName());
} else { } else {
printMsg(lvlVomit, DLOG(INFO) << goal->getName() << ": read " << rd << " bytes";
format("%1%: read %2% bytes") % goal->getName() % rd);
string data((char*)buffer.data(), rd); string data((char*)buffer.data(), rd);
j->lastOutput = after; j->lastOutput = after;
goal->handleChildOutput(k, data); goal->handleChildOutput(k, data);
@ -4422,8 +4374,8 @@ void Worker::waitForInput() {
if (goal->getExitCode() == Goal::ecBusy && 0 != settings.maxSilentTime && if (goal->getExitCode() == Goal::ecBusy && 0 != settings.maxSilentTime &&
j->respectTimeouts && j->respectTimeouts &&
after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) {
printError(format("%1% timed out after %2% seconds of silence") % LOG(ERROR) << goal->getName() << " timed out after "
goal->getName() % settings.maxSilentTime); << settings.maxSilentTime << " seconds of silence";
goal->timedOut(); goal->timedOut();
} }
@ -4431,8 +4383,8 @@ void Worker::waitForInput() {
0 != settings.buildTimeout && j->respectTimeouts && 0 != settings.buildTimeout && j->respectTimeouts &&
after - j->timeStarted >= after - j->timeStarted >=
std::chrono::seconds(settings.buildTimeout)) { std::chrono::seconds(settings.buildTimeout)) {
printError(format("%1% timed out after %2% seconds") % goal->getName() % LOG(ERROR) << goal->getName() << " timed out after "
settings.buildTimeout); << settings.buildTimeout << " seconds";
goal->timedOut(); goal->timedOut();
} }
} }
@ -4473,7 +4425,7 @@ unsigned int Worker::exitStatus() {
bool Worker::pathContentsGood(const Path& path) { bool Worker::pathContentsGood(const Path& path) {
std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
if (i != pathContentsGoodCache.end()) return i->second; if (i != pathContentsGoodCache.end()) return i->second;
printInfo(format("checking path '%1%'...") % path); LOG(INFO) << "checking path '" << path << "'...";
auto info = store.queryPathInfo(path); auto info = store.queryPathInfo(path);
bool res; bool res;
if (!pathExists(path)) if (!pathExists(path))
@ -4484,7 +4436,9 @@ bool Worker::pathContentsGood(const Path& path) {
res = info->narHash == nullHash || info->narHash == current.first; res = info->narHash == nullHash || info->narHash == current.first;
} }
pathContentsGoodCache[path] = res; pathContentsGoodCache[path] = res;
if (!res) printError(format("path '%1%' is corrupted or missing!") % path); if (!res) {
LOG(ERROR) << "path '" << path << "' is corrupted or missing!";
}
return res; return res;
} }

11
third_party/nix/src/libstore/builtins/buildenv.cc vendored
View file

@ -1,4 +1,5 @@
#include <fcntl.h> #include <fcntl.h>
#include <glog/logging.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
#include <algorithm> #include <algorithm>
@ -22,10 +23,8 @@ static void createLinks(const Path& srcDir, const Path& dstDir, int priority) {
srcFiles = readDirectory(srcDir); srcFiles = readDirectory(srcDir);
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo == ENOTDIR) { if (e.errNo == ENOTDIR) {
printError( LOG(ERROR) << "warning: not including '" << srcDir
"warning: not including '%s' in the user environment because it's " << "' in the user environment because it's not a directory";
"not a directory",
srcDir);
return; return;
} }
throw; throw;
@ -43,7 +42,7 @@ static void createLinks(const Path& srcDir, const Path& dstDir, int priority) {
throw SysError("getting status of '%1%'", srcFile); throw SysError("getting status of '%1%'", srcFile);
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo == ENOENT || e.errNo == ENOTDIR) { if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
printError("warning: skipping dangling symlink '%s'", dstFile); LOG(ERROR) << "warning: skipping dangling symlink '" << dstFile << "'";
continue; continue;
} }
throw; throw;
@ -200,7 +199,7 @@ void builtinBuildenv(const BasicDerivation& drv) {
for (const auto& pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++); for (const auto& pkgDir : pkgDirs) addPkg(pkgDir, priorityCounter++);
} }
printError("created %d symlinks in user environment", symlinks); LOG(INFO) << "created " << symlinks << " symlinks in user environment";
createSymlink(getAttr("manifest"), out + "/manifest.nix"); createSymlink(getAttr("manifest"), out + "/manifest.nix");
} }

3
third_party/nix/src/libstore/builtins/fetchurl.cc vendored
View file

@ -1,3 +1,4 @@
#include <glog/logging.h>
#include "archive.hh" #include "archive.hh"
#include "builtins.hh" #include "builtins.hh"
#include "compression.hh" #include "compression.hh"
@ -66,7 +67,7 @@ void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData) {
h.to_string(Base16, false)); h.to_string(Base16, false));
return; return;
} catch (Error& e) { } catch (Error& e) {
debug(e.what()); LOG(ERROR) << e.what();
} }
/* Otherwise try the specified URL. */ /* Otherwise try the specified URL. */

71
third_party/nix/src/libstore/download.cc vendored
View file

@ -15,6 +15,7 @@
#include <curl/curl.h> #include <curl/curl.h>
#include <fcntl.h> #include <fcntl.h>
#include <glog/logging.h>
#include <unistd.h> #include <unistd.h>
#include <algorithm> #include <algorithm>
#include <cmath> #include <cmath>
@ -50,7 +51,6 @@ struct CurlDownloader : public Downloader {
CurlDownloader& downloader; CurlDownloader& downloader;
DownloadRequest request; DownloadRequest request;
DownloadResult result; DownloadResult result;
Activity act;
bool done = false; // whether either the success or failure function has bool done = false; // whether either the success or failure function has
// been called // been called
Callback<DownloadResult> callback; Callback<DownloadResult> callback;
@ -77,10 +77,6 @@ struct CurlDownloader : public Downloader {
Callback<DownloadResult>&& callback) Callback<DownloadResult>&& callback)
: downloader(downloader), : downloader(downloader),
request(request), request(request),
act(*logger, lvlTalkative, actDownload,
fmt(request.data ? "uploading '%s'" : "downloading '%s'",
request.uri),
{request.uri}, request.parentAct),
callback(std::move(callback)), callback(std::move(callback)),
finalSink([this](const unsigned char* data, size_t len) { finalSink([this](const unsigned char* data, size_t len) {
if (this->request.dataCallback) { if (this->request.dataCallback) {
@ -97,6 +93,9 @@ struct CurlDownloader : public Downloader {
} else } else
this->result.data->append((char*)data, len); this->result.data->append((char*)data, len);
}) { }) {
LOG(INFO) << (request.data ? "uploading '" : "downloading '")
<< request.uri << "'";
if (!request.expectedETag.empty()) if (!request.expectedETag.empty())
requestHeaders = curl_slist_append( requestHeaders = curl_slist_append(
requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); requestHeaders, ("If-None-Match: " + request.expectedETag).c_str());
@ -162,8 +161,7 @@ struct CurlDownloader : public Downloader {
size_t headerCallback(void* contents, size_t size, size_t nmemb) { size_t headerCallback(void* contents, size_t size, size_t nmemb) {
size_t realSize = size * nmemb; size_t realSize = size * nmemb;
std::string line((char*)contents, realSize); std::string line((char*)contents, realSize);
printMsg(lvlVomit, DLOG(INFO) << "got header for '" << request.uri << "': " << trim(line);
format("got header for '%s': %s") % request.uri % trim(line));
if (line.compare(0, 5, "HTTP/") == 0) { // new response starts if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
result.etag = ""; result.etag = "";
auto ss = tokenizeString<vector<string>>(line, " "); auto ss = tokenizeString<vector<string>>(line, " ");
@ -184,8 +182,8 @@ struct CurlDownloader : public Downloader {
down the connection because we already have the down the connection because we already have the
data. */ data. */
if (result.etag == request.expectedETag && status == "200") { if (result.etag == request.expectedETag && status == "200") {
debug(format( DLOG(INFO)
"shutting down on 200 HTTP response with expected ETag")); << "shutting down on 200 HTTP response with expected ETag";
return 0; return 0;
} }
} else if (name == "content-encoding") } else if (name == "content-encoding")
@ -205,7 +203,7 @@ struct CurlDownloader : public Downloader {
int progressCallback(double dltotal, double dlnow) { int progressCallback(double dltotal, double dlnow) {
try { try {
act.progress(dlnow, dltotal); // TODO(tazjin): this had activity nonsense, clean it up
} catch (nix::Interrupted&) { } catch (nix::Interrupted&) {
assert(_isInterrupted); assert(_isInterrupted);
} }
@ -220,8 +218,9 @@ struct CurlDownloader : public Downloader {
static int debugCallback(CURL* handle, curl_infotype type, char* data, static int debugCallback(CURL* handle, curl_infotype type, char* data,
size_t size, void* userptr) { size_t size, void* userptr) {
if (type == CURLINFO_TEXT) if (type == CURLINFO_TEXT) {
vomit("curl: %s", chomp(std::string(data, size))); DLOG(INFO) << "curl: " << chomp(std::string(data, size));
}
return 0; return 0;
} }
@ -245,11 +244,12 @@ struct CurlDownloader : public Downloader {
curl_easy_reset(req); curl_easy_reset(req);
if (verbosity >= lvlVomit) { // TODO(tazjin): Add an Abseil flag for this
curl_easy_setopt(req, CURLOPT_VERBOSE, 1); // if (verbosity >= lvlVomit) {
curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, // curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
DownloadItem::debugCallback); // curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION,
} // DownloadItem::debugCallback);
// }
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L);
@ -329,10 +329,10 @@ struct CurlDownloader : public Downloader {
curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
if (effectiveUriCStr) result.effectiveUri = effectiveUriCStr; if (effectiveUriCStr) result.effectiveUri = effectiveUriCStr;
debug( DLOG(INFO) << "finished " << request.verb() << " of " << request.uri
"finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d " << "; curl status = " << code
"bytes", << ", HTTP status = " << httpStatus
request.verb(), request.uri, code, httpStatus, result.bodySize); << ", body = " << result.bodySize << " bytes";
if (decompressionSink) { if (decompressionSink) {
try { try {
@ -356,7 +356,6 @@ struct CurlDownloader : public Downloader {
httpStatus == 226 /* FTP */ || httpStatus == 226 /* FTP */ ||
httpStatus == 0 /* other protocol */)) { httpStatus == 0 /* other protocol */)) {
result.cached = httpStatus == 304; result.cached = httpStatus == 304;
act.progress(result.bodySize, result.bodySize);
done = true; done = true;
callback(std::move(result)); callback(std::move(result));
} }
@ -441,11 +440,12 @@ struct CurlDownloader : public Downloader {
std::pow(2.0f, attempt - 1 + std::pow(2.0f, attempt - 1 +
std::uniform_real_distribution<>( std::uniform_real_distribution<>(
0.0, 0.5)(downloader.mt19937)); 0.0, 0.5)(downloader.mt19937));
if (writtenToSink) if (writtenToSink) {
warn("%s; retrying from offset %d in %d ms", exc.what(), LOG(WARNING) << exc.what() << "; retrying from offset "
writtenToSink, ms); << writtenToSink << " in " << ms << "ms";
else } else {
warn("%s; retrying in %d ms", exc.what(), ms); LOG(WARNING) << exc.what() << "; retrying in " << ms << "ms";
}
embargo = embargo =
std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
downloader.enqueueItem(shared_from_this()); downloader.enqueueItem(shared_from_this());
@ -565,7 +565,7 @@ struct CurlDownloader : public Downloader {
nextWakeup - std::chrono::steady_clock::now()) nextWakeup - std::chrono::steady_clock::now())
.count()) .count())
: maxSleepTimeMs; : maxSleepTimeMs;
vomit("download thread waiting for %d ms", sleepTimeMs); DLOG(INFO) << "download thread waiting for " << sleepTimeMs << " ms";
mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds);
if (mc != CURLM_OK) if (mc != CURLM_OK)
throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % throw nix::Error(format("unexpected error from curl_multi_wait(): %s") %
@ -604,7 +604,8 @@ struct CurlDownloader : public Downloader {
} }
for (auto& item : incoming) { for (auto& item : incoming) {
debug("starting %s of %s", item->request.verb(), item->request.uri); DLOG(INFO) << "starting " << item->request.verb() << " of "
<< item->request.uri;
item->init(); item->init();
curl_multi_add_handle(curlm, item->req); curl_multi_add_handle(curlm, item->req);
item->active = true; item->active = true;
@ -612,7 +613,7 @@ struct CurlDownloader : public Downloader {
} }
} }
debug("download thread shutting down"); DLOG(INFO) << "download thread shutting down";
} }
void workerThreadEntry() { void workerThreadEntry() {
@ -620,7 +621,7 @@ struct CurlDownloader : public Downloader {
workerThreadMain(); workerThreadMain();
} catch (nix::Interrupted& e) { } catch (nix::Interrupted& e) {
} catch (std::exception& e) { } catch (std::exception& e) {
printError("unexpected error in download thread: %s", e.what()); LOG(ERROR) << "unexpected error in download thread: " << e.what();
} }
{ {
@ -762,7 +763,7 @@ void Downloader::download(DownloadRequest&& request, Sink& sink) {
download thread. (Hopefully sleeping will throttle the download thread. (Hopefully sleeping will throttle the
sender.) */ sender.) */
if (state->data.size() > 1024 * 1024) { if (state->data.size() > 1024 * 1024) {
debug("download buffer is full; going to sleep"); DLOG(INFO) << "download buffer is full; going to sleep";
state.wait_for(state->request, std::chrono::seconds(10)); state.wait_for(state->request, std::chrono::seconds(10));
} }
@ -870,7 +871,7 @@ CachedDownloadResult Downloader::downloadCached(
result.effectiveUri = request.uri; result.effectiveUri = request.uri;
result.etag = ss[1]; result.etag = ss[1];
} else if (!ss[1].empty()) { } else if (!ss[1].empty()) {
debug(format("verifying previous ETag '%1%'") % ss[1]); DLOG(INFO) << "verifying previous ETag: " << ss[1];
expectedETag = ss[1]; expectedETag = ss[1];
} }
} }
@ -908,7 +909,7 @@ CachedDownloadResult Downloader::downloadCached(
url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
} catch (DownloadError& e) { } catch (DownloadError& e) {
if (storePath.empty()) throw; if (storePath.empty()) throw;
warn("warning: %s; using cached result", e.msg()); LOG(WARNING) << e.msg() << "; using cached result";
result.etag = expectedETag; result.etag = expectedETag;
} }
} }
@ -924,7 +925,7 @@ CachedDownloadResult Downloader::downloadCached(
if (!store->isValidPath(unpackedStorePath)) unpackedStorePath = ""; if (!store->isValidPath(unpackedStorePath)) unpackedStorePath = "";
} }
if (unpackedStorePath.empty()) { if (unpackedStorePath.empty()) {
printInfo(format("unpacking '%1%'...") % url); LOG(INFO) << "unpacking '" << url << "' ...";
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
AutoDelete autoDelete(tmpDir, true); AutoDelete autoDelete(tmpDir, true);
// FIXME: this requires GNU tar for decompression. // FIXME: this requires GNU tar for decompression.

4
third_party/nix/src/libstore/download.hh vendored
View file

@ -46,14 +46,12 @@ struct DownloadRequest {
bool head = false; bool head = false;
size_t tries = downloadSettings.tries; size_t tries = downloadSettings.tries;
unsigned int baseRetryTimeMs = 250; unsigned int baseRetryTimeMs = 250;
ActivityId parentAct;
bool decompress = true; bool decompress = true;
std::shared_ptr<std::string> data; std::shared_ptr<std::string> data;
std::string mimeType; std::string mimeType;
std::function<void(char*, size_t)> dataCallback; std::function<void(char*, size_t)> dataCallback;
DownloadRequest(const std::string& uri) DownloadRequest(const std::string& uri) : uri(uri) {}
: uri(uri), parentAct(getCurActivity()) {}
std::string verb() { return data ? "upload" : "download"; } std::string verb() { return data ? "upload" : "download"; }
}; };

129
third_party/nix/src/libstore/gc.cc vendored
View file

@ -13,6 +13,7 @@
#include "derivations.hh" #include "derivations.hh"
#include "finally.hh" #include "finally.hh"
#include "globals.hh" #include "globals.hh"
#include "glog/logging.h"
#include "local-store.hh" #include "local-store.hh"
namespace nix { namespace nix {
@ -28,15 +29,16 @@ static string gcRootsDir = "gcroots";
AutoCloseFD LocalStore::openGCLock(LockType lockType) { AutoCloseFD LocalStore::openGCLock(LockType lockType) {
Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str(); Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str();
debug(format("acquiring global GC lock '%1%'") % fnGCLock); DLOG(INFO) << "acquiring global GC lock " << fnGCLock;
AutoCloseFD fdGCLock = AutoCloseFD fdGCLock =
open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
if (!fdGCLock) if (!fdGCLock) {
throw SysError(format("opening global GC lock '%1%'") % fnGCLock); throw SysError(format("opening global GC lock '%1%'") % fnGCLock);
}
if (!lockFile(fdGCLock.get(), lockType, false)) { if (!lockFile(fdGCLock.get(), lockType, false)) {
printError(format("waiting for the big garbage collector lock...")); LOG(ERROR) << "waiting for the big garbage collector lock...";
lockFile(fdGCLock.get(), lockType, true); lockFile(fdGCLock.get(), lockType, true);
} }
@ -116,12 +118,12 @@ Path LocalFSStore::addPermRoot(const Path& _storePath, const Path& _gcRoot,
gcroots directory. */ gcroots directory. */
if (settings.checkRootReachability) { if (settings.checkRootReachability) {
Roots roots = findRoots(false); Roots roots = findRoots(false);
if (roots[storePath].count(gcRoot) == 0) if (roots[storePath].count(gcRoot) == 0) {
printError( LOG(ERROR) << "warning: '" << gcRoot
format("warning: '%1%' is not in a directory where the garbage " << "' is not in a directory where the garbage "
"collector looks for roots; " << "collector looks for roots; therefore, '" << storePath
"therefore, '%2%' might be removed by the garbage collector") % << "' might be removed by the garbage collector";
gcRoot % storePath); }
} }
/* Grab the global GC root, causing us to block while a GC is in /* Grab the global GC root, causing us to block while a GC is in
@ -149,7 +151,7 @@ void LocalStore::addTempRoot(const Path& path) {
fdGCLock = -1; fdGCLock = -1;
debug(format("acquiring read lock on '%1%'") % fnTempRoots); DLOG(INFO) << "acquiring read lock on " << fnTempRoots;
lockFile(state->fdTempRoots.get(), ltRead, true); lockFile(state->fdTempRoots.get(), ltRead, true);
/* Check whether the garbage collector didn't get in our /* Check whether the garbage collector didn't get in our
@ -167,14 +169,14 @@ void LocalStore::addTempRoot(const Path& path) {
/* Upgrade the lock to a write lock. This will cause us to block /* Upgrade the lock to a write lock. This will cause us to block
if the garbage collector is holding our lock. */ if the garbage collector is holding our lock. */
debug(format("acquiring write lock on '%1%'") % fnTempRoots); DLOG(INFO) << "acquiring write lock on " << fnTempRoots;
lockFile(state->fdTempRoots.get(), ltWrite, true); lockFile(state->fdTempRoots.get(), ltWrite, true);
string s = path + '\0'; string s = path + '\0';
writeFull(state->fdTempRoots.get(), s); writeFull(state->fdTempRoots.get(), s);
/* Downgrade to a read lock. */ /* Downgrade to a read lock. */
debug(format("downgrading to read lock on '%1%'") % fnTempRoots); DLOG(INFO) << "downgrading to read lock on " << fnTempRoots;
lockFile(state->fdTempRoots.get(), ltRead, true); lockFile(state->fdTempRoots.get(), ltRead, true);
} }
@ -188,7 +190,7 @@ void LocalStore::findTempRoots(FDs& fds, Roots& tempRoots, bool censor) {
pid_t pid = std::stoi(i.name); pid_t pid = std::stoi(i.name);
debug(format("reading temporary root file '%1%'") % path); DLOG(INFO) << "reading temporary root file " << path;
FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666))); FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)));
if (!*fd) { if (!*fd) {
/* It's okay if the file has disappeared. */ /* It's okay if the file has disappeared. */
@ -204,7 +206,7 @@ void LocalStore::findTempRoots(FDs& fds, Roots& tempRoots, bool censor) {
only succeed if the owning process has died. In that case only succeed if the owning process has died. In that case
we don't care about its temporary roots. */ we don't care about its temporary roots. */
if (lockFile(fd->get(), ltWrite, false)) { if (lockFile(fd->get(), ltWrite, false)) {
printError(format("removing stale temporary roots file '%1%'") % path); LOG(ERROR) << "removing stale temporary roots file " << path;
unlink(path.c_str()); unlink(path.c_str());
writeFull(fd->get(), "d"); writeFull(fd->get(), "d");
continue; continue;
@ -213,7 +215,7 @@ void LocalStore::findTempRoots(FDs& fds, Roots& tempRoots, bool censor) {
/* Acquire a read lock. This will prevent the owning process /* Acquire a read lock. This will prevent the owning process
from upgrading to a write lock, therefore it will block in from upgrading to a write lock, therefore it will block in
addTempRoot(). */ addTempRoot(). */
debug(format("waiting for read lock on '%1%'") % path); DLOG(INFO) << "waiting for read lock on " << path;
lockFile(fd->get(), ltRead, true); lockFile(fd->get(), ltRead, true);
/* Read the entire file. */ /* Read the entire file. */
@ -224,7 +226,7 @@ void LocalStore::findTempRoots(FDs& fds, Roots& tempRoots, bool censor) {
while ((end = contents.find((char)0, pos)) != string::npos) { while ((end = contents.find((char)0, pos)) != string::npos) {
Path root(contents, pos, end - pos); Path root(contents, pos, end - pos);
debug("got temporary root '%s'", root); DLOG(INFO) << "got temporary root " << root;
assertStorePath(root); assertStorePath(root);
tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid));
pos = end + 1; pos = end + 1;
@ -240,8 +242,8 @@ void LocalStore::findRoots(const Path& path, unsigned char type, Roots& roots) {
if (isStorePath(storePath) && isValidPath(storePath)) if (isStorePath(storePath) && isValidPath(storePath))
roots[storePath].emplace(path); roots[storePath].emplace(path);
else else
printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % LOG(INFO) << "skipping invalid root from '" << path << "' to '"
storePath); << storePath << "'";
}; };
try { try {
@ -261,8 +263,8 @@ void LocalStore::findRoots(const Path& path, unsigned char type, Roots& roots) {
target = absPath(target, dirOf(path)); target = absPath(target, dirOf(path));
if (!pathExists(target)) { if (!pathExists(target)) {
if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) {
printInfo(format("removing stale link from '%1%' to '%2%'") % path % LOG(INFO) << "removing stale link from '" << path << "' to '"
target); << target << "'";
unlink(path.c_str()); unlink(path.c_str());
} }
} else { } else {
@ -284,10 +286,11 @@ void LocalStore::findRoots(const Path& path, unsigned char type, Roots& roots) {
catch (SysError& e) { catch (SysError& e) {
/* We only ignore permanent failures. */ /* We only ignore permanent failures. */
if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) {
printInfo(format("cannot read potential root '%1%'") % path); LOG(INFO) << "cannot read potential root '" << path << "'";
else } else {
throw; throw;
}
} }
} }
@ -319,16 +322,21 @@ try_again:
char buf[bufsiz]; char buf[bufsiz];
auto res = readlink(file.c_str(), buf, bufsiz); auto res = readlink(file.c_str(), buf, bufsiz);
if (res == -1) { if (res == -1) {
if (errno == ENOENT || errno == EACCES || errno == ESRCH) return; if (errno == ENOENT || errno == EACCES || errno == ESRCH) {
return;
}
throw SysError("reading symlink"); throw SysError("reading symlink");
} }
if (res == bufsiz) { if (res == bufsiz) {
if (SSIZE_MAX / 2 < bufsiz) throw Error("stupidly long symlink"); if (SSIZE_MAX / 2 < bufsiz) {
throw Error("stupidly long symlink");
}
bufsiz *= 2; bufsiz *= 2;
goto try_again; goto try_again;
} }
if (res > 0 && buf[0] == '/') if (res > 0 && buf[0] == '/') {
roots[std::string(static_cast<char*>(buf), res)].emplace(file); roots[std::string(static_cast<char*>(buf), res)].emplace(file);
}
} }
static string quoteRegexChars(const string& raw) { static string quoteRegexChars(const string& raw) {
@ -340,7 +348,9 @@ static void readFileRoots(const char* path, Roots& roots) {
try { try {
roots[readFile(path)].emplace(path); roots[readFile(path)].emplace(path);
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo != ENOENT && e.errNo != EACCES) throw; if (e.errNo != ENOENT && e.errNo != EACCES) {
throw;
}
} }
} }
@ -434,7 +444,7 @@ void LocalStore::findRuntimeRoots(Roots& roots, bool censor) {
if (isInStore(target)) { if (isInStore(target)) {
Path path = toStorePath(target); Path path = toStorePath(target);
if (isStorePath(path) && isValidPath(path)) { if (isStorePath(path) && isValidPath(path)) {
debug(format("got additional root '%1%'") % path); DLOG(INFO) << "got additional root " << path;
if (censor) if (censor)
roots[path].insert(censored); roots[path].insert(censored);
else else
@ -496,7 +506,7 @@ void LocalStore::deletePathRecursive(GCState& state, const Path& path) {
throw SysError(format("getting status of %1%") % realPath); throw SysError(format("getting status of %1%") % realPath);
} }
printInfo(format("deleting '%1%'") % path); LOG(INFO) << "deleting '" << path << "'";
state.results.paths.insert(path); state.results.paths.insert(path);
@ -519,8 +529,8 @@ void LocalStore::deletePathRecursive(GCState& state, const Path& path) {
state.bytesInvalidated += size; state.bytesInvalidated += size;
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo == ENOSPC) { if (e.errNo == ENOSPC) {
printInfo(format("note: can't create move '%1%': %2%") % realPath % LOG(INFO) << "note: can't create move '" << realPath
e.msg()); << "': " << e.msg();
deleteGarbage(state, realPath); deleteGarbage(state, realPath);
} }
} }
@ -529,8 +539,8 @@ void LocalStore::deletePathRecursive(GCState& state, const Path& path) {
if (state.results.bytesFreed + state.bytesInvalidated > if (state.results.bytesFreed + state.bytesInvalidated >
state.options.maxFreed) { state.options.maxFreed) {
printInfo(format("deleted or invalidated more than %1% bytes; stopping") % LOG(INFO) << "deleted or invalidated more than " << state.options.maxFreed
state.options.maxFreed); << " bytes; stopping";
throw GCLimitReached(); throw GCLimitReached();
} }
} }
@ -544,7 +554,7 @@ bool LocalStore::canReachRoot(GCState& state, PathSet& visited,
if (state.dead.count(path)) return false; if (state.dead.count(path)) return false;
if (state.roots.count(path)) { if (state.roots.count(path)) {
debug(format("cannot delete '%1%' because it's a root") % path); DLOG(INFO) << "cannot delete '" << path << "' because it's a root";
state.alive.insert(path); state.alive.insert(path);
return true; return true;
} }
@ -611,7 +621,7 @@ void LocalStore::tryToDelete(GCState& state, const Path& path) {
PathSet visited; PathSet visited;
if (canReachRoot(state, visited, path)) { if (canReachRoot(state, visited, path)) {
debug(format("cannot delete '%1%' because it's still reachable") % path); DLOG(INFO) << "cannot delete '" << path << "' because it's still reachable";
} else { } else {
/* No path we visited was a root, so everything is garbage. /* No path we visited was a root, so everything is garbage.
But we only delete ‘path’ and its referrers here so that But we only delete ‘path’ and its referrers here so that
@ -629,7 +639,9 @@ void LocalStore::tryToDelete(GCState& state, const Path& path) {
the link count. */ the link count. */
void LocalStore::removeUnusedLinks(const GCState& state) { void LocalStore::removeUnusedLinks(const GCState& state) {
AutoCloseDir dir(opendir(linksDir.c_str())); AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); if (!dir) {
throw SysError(format("opening directory '%1%'") % linksDir);
}
long long actualSize = 0, unsharedSize = 0; long long actualSize = 0, unsharedSize = 0;
@ -650,21 +662,26 @@ void LocalStore::removeUnusedLinks(const GCState& state) {
continue; continue;
} }
printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); LOG(INFO) << "deleting unused link " << path;
if (unlink(path.c_str()) == -1) if (unlink(path.c_str()) == -1) {
throw SysError(format("deleting '%1%'") % path); throw SysError(format("deleting '%1%'") % path);
}
state.results.bytesFreed += st.st_size; state.results.bytesFreed += st.st_size;
} }
struct stat st; struct stat st;
if (stat(linksDir.c_str(), &st) == -1) if (stat(linksDir.c_str(), &st) == -1) {
throw SysError(format("statting '%1%'") % linksDir); throw SysError(format("statting '%1%'") % linksDir);
}
long long overhead = st.st_blocks * 512ULL; long long overhead = st.st_blocks * 512ULL;
printInfo(format("note: currently hard linking saves %.2f MiB") % // TODO(tazjin): absl::StrFormat %.2f
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); LOG(INFO) << "note: currently hard linking saves "
<< ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))
<< " MiB";
} }
void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) { void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) {
@ -694,11 +711,15 @@ void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) {
/* Find the roots. Since we've grabbed the GC lock, the set of /* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */ permanent roots cannot increase now. */
printError(format("finding garbage collector roots...")); LOG(INFO) << "finding garbage collector roots...";
Roots rootMap; Roots rootMap;
if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true); if (!options.ignoreLiveness) {
findRootsNoTemp(rootMap, true);
}
for (auto& i : rootMap) state.roots.insert(i.first); for (auto& i : rootMap) {
state.roots.insert(i.first);
}
/* Read the temporary roots. This acquires read locks on all /* Read the temporary roots. This acquires read locks on all
per-process temporary root files. So after this point no paths per-process temporary root files. So after this point no paths
@ -719,7 +740,7 @@ void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) {
createDirs(trashDir); createDirs(trashDir);
} catch (SysError& e) { } catch (SysError& e) {
if (e.errNo == ENOSPC) { if (e.errNo == ENOSPC) {
printInfo(format("note: can't create trash directory: %1%") % e.msg()); LOG(INFO) << "note: can't create trash directory: " << e.msg();
state.moveToTrash = false; state.moveToTrash = false;
} }
} }
@ -739,14 +760,15 @@ void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) {
} else if (options.maxFreed > 0) { } else if (options.maxFreed > 0) {
if (state.shouldDelete) if (state.shouldDelete)
printError(format("deleting garbage...")); LOG(INFO) << "deleting garbage...";
else else
printError(format("determining live/dead paths...")); LOG(ERROR) << "determining live/dead paths...";
try { try {
AutoCloseDir dir(opendir(realStoreDir.c_str())); AutoCloseDir dir(opendir(realStoreDir.c_str()));
if (!dir) if (!dir) {
throw SysError(format("opening directory '%1%'") % realStoreDir); throw SysError(format("opening directory '%1%'") % realStoreDir);
}
/* Read the store and immediately delete all paths that /* Read the store and immediately delete all paths that
aren't valid. When using --max-freed etc., deleting aren't valid. When using --max-freed etc., deleting
@ -799,13 +821,13 @@ void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) {
fds.clear(); fds.clear();
/* Delete the trash directory. */ /* Delete the trash directory. */
printInfo(format("deleting '%1%'") % trashDir); LOG(INFO) << "deleting " << trashDir;
deleteGarbage(state, trashDir); deleteGarbage(state, trashDir);
/* Clean up the links directory. */ /* Clean up the links directory. */
if (options.action == GCOptions::gcDeleteDead || if (options.action == GCOptions::gcDeleteDead ||
options.action == GCOptions::gcDeleteSpecific) { options.action == GCOptions::gcDeleteSpecific) {
printError(format("deleting unused links...")); LOG(INFO) << "deleting unused links...";
removeUnusedLinks(state); removeUnusedLinks(state);
} }
@ -821,8 +843,9 @@ void LocalStore::autoGC(bool sync) {
return std::stoll(readFile(fakeFreeSpaceFile)); return std::stoll(readFile(fakeFreeSpaceFile));
struct statvfs st; struct statvfs st;
if (statvfs(realStoreDir.c_str(), &st)) if (statvfs(realStoreDir.c_str(), &st)) {
throw SysError("getting filesystem info about '%s'", realStoreDir); throw SysError("getting filesystem info about '%s'", realStoreDir);
}
return (uint64_t)st.f_bavail * st.f_bsize; return (uint64_t)st.f_bavail * st.f_bsize;
}; };
@ -834,7 +857,7 @@ void LocalStore::autoGC(bool sync) {
if (state->gcRunning) { if (state->gcRunning) {
future = state->gcFuture; future = state->gcFuture;
debug("waiting for auto-GC to finish"); DLOG(INFO) << "waiting for auto-GC to finish";
goto sync; goto sync;
} }
@ -870,7 +893,7 @@ void LocalStore::autoGC(bool sync) {
GCOptions options; GCOptions options;
options.maxFreed = settings.maxFree - avail; options.maxFreed = settings.maxFree - avail;
printInfo("running auto-GC to free %d bytes", options.maxFreed); LOG(INFO) << "running auto-GC to free " << options.maxFreed << " bytes";
GCResults results; GCResults results;

6
third_party/nix/src/libstore/http-binary-cache-store.cc vendored
View file

@ -1,3 +1,4 @@
#include <glog/logging.h>
#include "binary-cache-store.hh" #include "binary-cache-store.hh"
#include "download.hh" #include "download.hh"
#include "globals.hh" #include "globals.hh"
@ -45,7 +46,8 @@ class HttpBinaryCacheStore : public BinaryCacheStore {
auto state(_state.lock()); auto state(_state.lock());
if (state->enabled && settings.tryFallback) { if (state->enabled && settings.tryFallback) {
int t = 60; int t = 60;
printError("disabling binary cache '%s' for %s seconds", getUri(), t); LOG(WARNING) << "disabling binary cache '" << getUri() << "' for " << t
<< " seconds";
state->enabled = false; state->enabled = false;
state->disabledUntil = state->disabledUntil =
std::chrono::steady_clock::now() + std::chrono::seconds(t); std::chrono::steady_clock::now() + std::chrono::seconds(t);
@ -57,7 +59,7 @@ class HttpBinaryCacheStore : public BinaryCacheStore {
if (state->enabled) return; if (state->enabled) return;
if (std::chrono::steady_clock::now() > state->disabledUntil) { if (std::chrono::steady_clock::now() > state->disabledUntil) {
state->enabled = true; state->enabled = true;
debug("re-enabling binary cache '%s'", getUri()); DLOG(INFO) << "re-enabling binary cache '" << getUri() << "'";
return; return;
} }
throw SubstituterDisabled("substituter '%s' is disabled", getUri()); throw SubstituterDisabled("substituter '%s' is disabled", getUri());

7
third_party/nix/src/libstore/legacy-ssh-store.cc vendored
View file

@ -1,5 +1,6 @@
#include "archive.hh" #include "archive.hh"
#include "derivations.hh" #include "derivations.hh"
#include "glog/logging.h"
#include "pool.hh" #include "pool.hh"
#include "remote-store.hh" #include "remote-store.hh"
#include "serve-protocol.hh" #include "serve-protocol.hh"
@ -90,7 +91,8 @@ struct LegacySSHStore : public Store {
try { try {
auto conn(connections->get()); auto conn(connections->get());
debug("querying remote host '%s' for info on '%s'", host, path); DLOG(INFO) << "querying remote host '" << host << "' for info on '"
<< path << "'";
conn->to << cmdQueryPathInfos << PathSet{path}; conn->to << cmdQueryPathInfos << PathSet{path};
conn->to.flush(); conn->to.flush();
@ -125,7 +127,8 @@ struct LegacySSHStore : public Store {
void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair, void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair,
CheckSigsFlag checkSigs, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) override { std::shared_ptr<FSAccessor> accessor) override {
debug("adding path '%s' to remote host '%s'", info.path, host); DLOG(INFO) << "adding path '" << info.path << "' to remote host '" << host
<< "'";
auto conn(connections->get()); auto conn(connections->get());

87
third_party/nix/src/libstore/local-store.cc vendored
View file

@ -1,6 +1,7 @@
#include "local-store.hh" #include "local-store.hh"
#include <errno.h> #include <errno.h>
#include <fcntl.h> #include <fcntl.h>
#include <glog/logging.h>
#include <grp.h> #include <grp.h>
#include <stdio.h> #include <stdio.h>
#include <sys/select.h> #include <sys/select.h>
@ -81,11 +82,10 @@ LocalStore::LocalStore(const Params& params)
mode_t perm = 01775; mode_t perm = 01775;
struct group* gr = getgrnam(settings.buildUsersGroup.get().c_str()); struct group* gr = getgrnam(settings.buildUsersGroup.get().c_str());
if (!gr) if (!gr) {
printError(format("warning: the group '%1%' specified in " LOG(ERROR) << "warning: the group '" << settings.buildUsersGroup
"'build-users-group' does not exist") % << "' specified in 'build-users-group' does not exist";
settings.buildUsersGroup); } else {
else {
struct stat st; struct stat st;
if (stat(realStoreDir.c_str(), &st)) if (stat(realStoreDir.c_str(), &st))
throw SysError(format("getting attributes of path '%1%'") % throw SysError(format("getting attributes of path '%1%'") %
@ -147,7 +147,7 @@ LocalStore::LocalStore(const Params& params)
globalLock = openLockFile(globalLockPath.c_str(), true); globalLock = openLockFile(globalLockPath.c_str(), true);
if (!lockFile(globalLock.get(), ltRead, false)) { if (!lockFile(globalLock.get(), ltRead, false)) {
printError("waiting for the big Nix store lock..."); LOG(INFO) << "waiting for the big Nix store lock...";
lockFile(globalLock.get(), ltRead, true); lockFile(globalLock.get(), ltRead, true);
} }
@ -180,7 +180,7 @@ LocalStore::LocalStore(const Params& params)
"please upgrade Nix to version 1.11 first."); "please upgrade Nix to version 1.11 first.");
if (!lockFile(globalLock.get(), ltWrite, false)) { if (!lockFile(globalLock.get(), ltWrite, false)) {
printError("waiting for exclusive access to the Nix store..."); LOG(INFO) << "waiting for exclusive access to the Nix store...";
lockFile(globalLock.get(), ltWrite, true); lockFile(globalLock.get(), ltWrite, true);
} }
@ -272,7 +272,7 @@ LocalStore::~LocalStore() {
} }
if (future.valid()) { if (future.valid()) {
printError("waiting for auto-GC to finish on exit..."); LOG(INFO) << "waiting for auto-GC to finish on exit...";
future.get(); future.get();
} }
@ -850,8 +850,8 @@ void LocalStore::querySubstitutablePathInfos(const PathSet& paths,
if (sub->storeDir != storeDir) continue; if (sub->storeDir != storeDir) continue;
for (auto& path : paths) { for (auto& path : paths) {
if (infos.count(path)) continue; if (infos.count(path)) continue;
debug(format("checking substituter '%s' for path '%s'") % sub->getUri() % DLOG(INFO) << "checking substituter '" << sub->getUri() << "' for path '"
path); << path << "'";
try { try {
auto info = sub->queryPathInfo(path); auto info = sub->queryPathInfo(path);
auto narInfo = std::dynamic_pointer_cast<const NarInfo>( auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
@ -862,10 +862,11 @@ void LocalStore::querySubstitutablePathInfos(const PathSet& paths,
} catch (InvalidPath&) { } catch (InvalidPath&) {
} catch (SubstituterDisabled&) { } catch (SubstituterDisabled&) {
} catch (Error& e) { } catch (Error& e) {
if (settings.tryFallback) if (settings.tryFallback) {
printError(e.what()); LOG(ERROR) << e.what();
else } else {
throw; throw;
}
} }
} }
} }
@ -931,7 +932,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos& infos) {
/* Invalidate a path. The caller is responsible for checking that /* Invalidate a path. The caller is responsible for checking that
there are no referrers. */ there are no referrers. */
void LocalStore::invalidatePath(State& state, const Path& path) { void LocalStore::invalidatePath(State& state, const Path& path) {
debug(format("invalidating path '%1%'") % path); LOG(INFO) << "invalidating path '" << path << "'";
state.stmtInvalidatePath.use()(path).exec(); state.stmtInvalidatePath.use()(path).exec();
@ -954,12 +955,15 @@ const PublicKeys& LocalStore::getPublicKeys() {
void LocalStore::addToStore(const ValidPathInfo& info, Source& source, void LocalStore::addToStore(const ValidPathInfo& info, Source& source,
RepairFlag repair, CheckSigsFlag checkSigs, RepairFlag repair, CheckSigsFlag checkSigs,
std::shared_ptr<FSAccessor> accessor) { std::shared_ptr<FSAccessor> accessor) {
if (!info.narHash) if (!info.narHash) {
throw Error("cannot add path '%s' because it lacks a hash", info.path); throw Error("cannot add path '%s' because it lacks a hash", info.path);
}
if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) if (requireSigs && checkSigs &&
!info.checkSignatures(*this, getPublicKeys())) {
throw Error("cannot add path '%s' because it lacks a valid signature", throw Error("cannot add path '%s' because it lacks a valid signature",
info.path); info.path);
}
addTempRoot(info.path); addTempRoot(info.path);
@ -1168,7 +1172,7 @@ void LocalStore::invalidatePathChecked(const Path& path) {
} }
bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) {
printError(format("reading the Nix store...")); LOG(INFO) << "reading the Nix store...";
bool errors = false; bool errors = false;
@ -1180,7 +1184,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) {
for (auto& i : readDirectory(realStoreDir)) store.insert(i.name); for (auto& i : readDirectory(realStoreDir)) store.insert(i.name);
/* Check whether all valid paths actually exist. */ /* Check whether all valid paths actually exist. */
printInfo("checking path existence..."); LOG(INFO) << "checking path existence...";
PathSet validPaths2 = queryAllValidPaths(), validPaths, done; PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
@ -1191,7 +1195,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) {
/* Optionally, check the content hashes (slow). */ /* Optionally, check the content hashes (slow). */
if (checkContents) { if (checkContents) {
printInfo("checking hashes..."); LOG(INFO) << "checking hashes...";
Hash nullHash(htSHA256); Hash nullHash(htSHA256);
@ -1201,31 +1205,32 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) {
std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); std::shared_ptr<const ValidPathInfo>(queryPathInfo(i)));
/* Check the content hash (optionally - slow). */ /* Check the content hash (optionally - slow). */
printMsg(lvlTalkative, format("checking contents of '%1%'") % i); DLOG(INFO) << "checking contents of '" << i << "'";
HashResult current = hashPath(info->narHash.type, toRealPath(i)); HashResult current = hashPath(info->narHash.type, toRealPath(i));
if (info->narHash != nullHash && info->narHash != current.first) { if (info->narHash != nullHash && info->narHash != current.first) {
printError(format("path '%1%' was modified! " LOG(ERROR) << "path '" << i << "' was modified! expected hash '"
"expected hash '%2%', got '%3%'") % << info->narHash.to_string() << "', got '"
i % info->narHash.to_string() % current.first.to_string()); << current.first.to_string() << "'";
if (repair) if (repair) {
repairPath(i); repairPath(i);
else } else {
errors = true; errors = true;
}
} else { } else {
bool update = false; bool update = false;
/* Fill in missing hashes. */ /* Fill in missing hashes. */
if (info->narHash == nullHash) { if (info->narHash == nullHash) {
printError(format("fixing missing hash on '%1%'") % i); LOG(WARNING) << "fixing missing hash on '" << i << "'";
info->narHash = current.first; info->narHash = current.first;
update = true; update = true;
} }
/* Fill in missing narSize fields (from old stores). */ /* Fill in missing narSize fields (from old stores). */
if (info->narSize == 0) { if (info->narSize == 0) {
printError(format("updating size field on '%1%' to %2%") % i % LOG(ERROR) << "updating size field on '" << i << "' to "
current.second); << current.second;
info->narSize = current.second; info->narSize = current.second;
update = true; update = true;
} }
@ -1239,10 +1244,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) {
} catch (Error& e) { } catch (Error& e) {
/* It's possible that the path got GC'ed, so ignore /* It's possible that the path got GC'ed, so ignore
errors on invalid paths. */ errors on invalid paths. */
if (isValidPath(i)) if (isValidPath(i)) {
printError(format("error: %1%") % e.msg()); LOG(ERROR) << e.msg();
else } else {
printError(format("warning: %1%") % e.msg()); LOG(WARNING) << e.msg();
}
errors = true; errors = true;
} }
} }
@ -1256,11 +1262,13 @@ void LocalStore::verifyPath(const Path& path, const PathSet& store,
RepairFlag repair, bool& errors) { RepairFlag repair, bool& errors) {
checkInterrupt(); checkInterrupt();
if (done.find(path) != done.end()) return; if (done.find(path) != done.end()) {
return;
}
done.insert(path); done.insert(path);
if (!isStorePath(path)) { if (!isStorePath(path)) {
printError(format("path '%1%' is not in the Nix store") % path); LOG(ERROR) << "path '" << path << "' is not in the Nix store";
auto state(_state.lock()); auto state(_state.lock());
invalidatePath(*state, path); invalidatePath(*state, path);
return; return;
@ -1279,18 +1287,17 @@ void LocalStore::verifyPath(const Path& path, const PathSet& store,
} }
if (canInvalidate) { if (canInvalidate) {
printError(format("path '%1%' disappeared, removing from database...") % LOG(WARNING) << "path '" << path
path); << "' disappeared, removing from database...";
auto state(_state.lock()); auto state(_state.lock());
invalidatePath(*state, path); invalidatePath(*state, path);
} else { } else {
printError( LOG(ERROR) << "path '" << path
format("path '%1%' disappeared, but it still has valid referrers!") % << "' disappeared, but it still has valid referrers!";
path);
if (repair) try { if (repair) try {
repairPath(path); repairPath(path);
} catch (Error& e) { } catch (Error& e) {
printError(format("warning: %1%") % e.msg()); LOG(WARNING) << e.msg();
errors = true; errors = true;
} }
else else

2
third_party/nix/src/libstore/local-store.hh vendored
View file

@ -268,7 +268,7 @@ class LocalStore : public LocalFSStore {
InodeHash loadInodeHash(); InodeHash loadInodeHash();
Strings readDirectoryIgnoringInodes(const Path& path, Strings readDirectoryIgnoringInodes(const Path& path,
const InodeHash& inodeHash); const InodeHash& inodeHash);
void optimisePath_(Activity* act, OptimiseStats& stats, const Path& path, void optimisePath_(OptimiseStats& stats, const Path& path,
InodeHash& inodeHash); InodeHash& inodeHash);
// Internal versions that are not wrapped in retry_sqlite. // Internal versions that are not wrapped in retry_sqlite.

11
third_party/nix/src/libstore/machines.cc vendored
View file

@ -1,4 +1,5 @@
#include "machines.hh" #include "machines.hh"
#include <glog/logging.h>
#include <algorithm> #include <algorithm>
#include "globals.hh" #include "globals.hh"
#include "util.hh" #include "util.hh"
@ -53,15 +54,19 @@ void parseMachines(const std::string& s, Machines& machines) {
try { try {
parseMachines(readFile(file), machines); parseMachines(readFile(file), machines);
} catch (const SysError& e) { } catch (const SysError& e) {
if (e.errNo != ENOENT) throw; if (e.errNo != ENOENT) {
debug("cannot find machines file '%s'", file); throw;
}
DLOG(INFO) << "cannot find machines file: " << file;
} }
continue; continue;
} }
auto tokens = tokenizeString<std::vector<string>>(line); auto tokens = tokenizeString<std::vector<string>>(line);
auto sz = tokens.size(); auto sz = tokens.size();
if (sz < 1) throw FormatError("bad machine specification '%s'", line); if (sz < 1) {
throw FormatError("bad machine specification '%s'", line);
}
auto isSet = [&](size_t n) { auto isSet = [&](size_t n) {
return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; return tokens.size() > n && tokens[n] != "" && tokens[n] != "-";

1
third_party/nix/src/libstore/meson.build vendored
View file

@ -79,6 +79,7 @@ libstore_data = files(
#============================================================================ #============================================================================
libstore_dep_list = [ libstore_dep_list = [
glog_dep,
libbz2_dep, libbz2_dep,
libcurl_dep, libcurl_dep,
libdl_dep, libdl_dep,

4
third_party/nix/src/libstore/misc.cc vendored
View file

@ -1,3 +1,4 @@
#include <glog/logging.h>
#include "derivations.hh" #include "derivations.hh"
#include "globals.hh" #include "globals.hh"
#include "local-store.hh" #include "local-store.hh"
@ -99,8 +100,7 @@ void Store::queryMissing(const PathSet& targets, PathSet& willBuild_,
PathSet& willSubstitute_, PathSet& unknown_, PathSet& willSubstitute_, PathSet& unknown_,
unsigned long long& downloadSize_, unsigned long long& downloadSize_,
unsigned long long& narSize_) { unsigned long long& narSize_) {
Activity act(*logger, lvlDebug, actUnknown, LOG(INFO) << "querying info about missing paths";
"querying info about missing paths");
downloadSize_ = narSize_ = 0; downloadSize_ = narSize_ = 0;

5
third_party/nix/src/libstore/nar-info-disk-cache.cc vendored
View file

@ -1,4 +1,5 @@
#include "nar-info-disk-cache.hh" #include "nar-info-disk-cache.hh"
#include <glog/logging.h>
#include <sqlite3.h> #include <sqlite3.h>
#include "globals.hh" #include "globals.hh"
#include "sqlite.hh" #include "sqlite.hh"
@ -126,8 +127,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache {
now - settings.ttlPositiveNarInfoCache) now - settings.ttlPositiveNarInfoCache)
.exec(); .exec();
debug("deleted %d entries from the NAR info disk cache", DLOG(INFO) << "deleted " << sqlite3_changes(state->db)
sqlite3_changes(state->db)); << " entries from the NAR info disk cache";
SQLiteStmt( SQLiteStmt(
state->db, state->db,

82
third_party/nix/src/libstore/optimise-store.cc vendored
View file

@ -7,6 +7,7 @@
#include <cstring> #include <cstring>
#include <regex> #include <regex>
#include "globals.hh" #include "globals.hh"
#include "glog/logging.h"
#include "local-store.hh" #include "local-store.hh"
#include "util.hh" #include "util.hh"
@ -34,11 +35,13 @@ struct MakeReadOnly {
}; };
LocalStore::InodeHash LocalStore::loadInodeHash() { LocalStore::InodeHash LocalStore::loadInodeHash() {
debug("loading hash inodes in memory"); DLOG(INFO) << "loading hash inodes in memory";
InodeHash inodeHash; InodeHash inodeHash;
AutoCloseDir dir(opendir(linksDir.c_str())); AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); if (!dir) {
throw SysError(format("opening directory '%1%'") % linksDir);
}
struct dirent* dirent; struct dirent* dirent;
while (errno = 0, dirent = readdir(dir.get())) { /* sic */ while (errno = 0, dirent = readdir(dir.get())) { /* sic */
@ -46,9 +49,11 @@ LocalStore::InodeHash LocalStore::loadInodeHash() {
// We don't care if we hit non-hash files, anything goes // We don't care if we hit non-hash files, anything goes
inodeHash.insert(dirent->d_ino); inodeHash.insert(dirent->d_ino);
} }
if (errno) throw SysError(format("reading directory '%1%'") % linksDir); if (errno) {
throw SysError(format("reading directory '%1%'") % linksDir);
}
printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); DLOG(INFO) << "loaded " << inodeHash.size() << " hash inodes";
return inodeHash; return inodeHash;
} }
@ -58,14 +63,16 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path& path,
Strings names; Strings names;
AutoCloseDir dir(opendir(path.c_str())); AutoCloseDir dir(opendir(path.c_str()));
if (!dir) throw SysError(format("opening directory '%1%'") % path); if (!dir) {
throw SysError(format("opening directory '%1%'") % path);
}
struct dirent* dirent; struct dirent* dirent;
while (errno = 0, dirent = readdir(dir.get())) { /* sic */ while (errno = 0, dirent = readdir(dir.get())) { /* sic */
checkInterrupt(); checkInterrupt();
if (inodeHash.count(dirent->d_ino)) { if (inodeHash.count(dirent->d_ino)) {
debug(format("'%1%' is already linked") % dirent->d_name); DLOG(WARNING) << dirent->d_name << " is already linked";
continue; continue;
} }
@ -73,13 +80,15 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path& path,
if (name == "." || name == "..") continue; if (name == "." || name == "..") continue;
names.push_back(name); names.push_back(name);
} }
if (errno) throw SysError(format("reading directory '%1%'") % path); if (errno) {
throw SysError(format("reading directory '%1%'") % path);
}
return names; return names;
} }
void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats, void LocalStore::optimisePath_(OptimiseStats& stats, const Path& path,
const Path& path, InodeHash& inodeHash) { InodeHash& inodeHash) {
checkInterrupt(); checkInterrupt();
struct stat st; struct stat st;
@ -100,7 +109,7 @@ void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats,
if (S_ISDIR(st.st_mode)) { if (S_ISDIR(st.st_mode)) {
Strings names = readDirectoryIgnoringInodes(path, inodeHash); Strings names = readDirectoryIgnoringInodes(path, inodeHash);
for (auto& i : names) optimisePath_(act, stats, path + "/" + i, inodeHash); for (auto& i : names) optimisePath_(stats, path + "/" + i, inodeHash);
return; return;
} }
@ -117,14 +126,14 @@ void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats,
NixOS (example: $fontconfig/var/cache being modified). Skip NixOS (example: $fontconfig/var/cache being modified). Skip
those files. FIXME: check the modification time. */ those files. FIXME: check the modification time. */
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
printError(format("skipping suspicious writable file '%1%'") % path); LOG(WARNING) << "skipping suspicious writable file '" << path << "'";
return; return;
} }
/* This can still happen on top-level files. */ /* This can still happen on top-level files. */
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
debug(format("'%1%' is already linked, with %2% other file(s)") % path % DLOG(INFO) << path << " is already linked, with " << (st.st_nlink - 2)
(st.st_nlink - 2)); << " other file(s)";
return; return;
} }
@ -138,7 +147,7 @@ void LocalStore::optimisePath_(Activity* act, OptimiseStats& stats,
contents of the symlink (i.e. the result of readlink()), not contents of the symlink (i.e. the result of readlink()), not
the contents of the target (which may not even exist). */ the contents of the target (which may not even exist). */
Hash hash = hashPath(htSHA256, path).first; Hash hash = hashPath(htSHA256, path).first;
debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); LOG(INFO) << path << " has hash " << hash.to_string();
/* Check if this is a known hash. */ /* Check if this is a known hash. */
Path linkPath = linksDir + "/" + hash.to_string(Base32, false); Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
@ -162,8 +171,9 @@ retry:
full. When that happens, it's fine to ignore it: we full. When that happens, it's fine to ignore it: we
just effectively disable deduplication of this just effectively disable deduplication of this
file. */ file. */
printInfo("cannot link '%s' to '%s': %s", linkPath, path, LOG(WARNING) << "cannot link '" << linkPath << " to " << path << ": "
strerror(errno)); << strerror(errno);
return; return;
default: default:
@ -178,17 +188,17 @@ retry:
throw SysError(format("getting attributes of path '%1%'") % linkPath); throw SysError(format("getting attributes of path '%1%'") % linkPath);
if (st.st_ino == stLink.st_ino) { if (st.st_ino == stLink.st_ino) {
debug(format("'%1%' is already linked to '%2%'") % path % linkPath); DLOG(INFO) << path << " is already linked to " << linkPath;
return; return;
} }
if (st.st_size != stLink.st_size) { if (st.st_size != stLink.st_size) {
printError(format("removing corrupted link '%1%'") % linkPath); LOG(WARNING) << "removing corrupted link '" << linkPath << "'";
unlink(linkPath.c_str()); unlink(linkPath.c_str());
goto retry; goto retry;
} }
printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); DLOG(INFO) << "linking '" << path << "' to '" << linkPath << "'";
/* Make the containing directory writable, but only if it's not /* Make the containing directory writable, but only if it's not
the store itself (we don't want or need to mess with its the store itself (we don't want or need to mess with its
@ -209,8 +219,9 @@ retry:
/* Too many links to the same file (>= 32000 on most file /* Too many links to the same file (>= 32000 on most file
systems). This is likely to happen with empty files. systems). This is likely to happen with empty files.
Just shrug and ignore. */ Just shrug and ignore. */
if (st.st_size) if (st.st_size) {
printInfo(format("'%1%' has maximum number of links") % linkPath); LOG(WARNING) << linkPath << " has maximum number of links";
}
return; return;
} }
throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath);
@ -218,14 +229,16 @@ retry:
/* Atomically replace the old file with the new hard link. */ /* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) { if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (unlink(tempLink.c_str()) == -1) if (unlink(tempLink.c_str()) == -1) {
printError(format("unable to unlink '%1%'") % tempLink); LOG(ERROR) << "unable to unlink '" << tempLink << "'";
}
if (errno == EMLINK) { if (errno == EMLINK) {
/* Some filesystems generate too many links on the rename, /* Some filesystems generate too many links on the rename,
rather than on the original link. (Probably it rather than on the original link. (Probably it
temporarily increases the st_nlink field before temporarily increases the st_nlink field before
decreasing it again.) */ decreasing it again.) */
debug("'%s' has reached maximum number of links", linkPath); DLOG(WARNING) << "'" << linkPath
<< "' has reached maximum number of links";
return; return;
} }
throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path);
@ -234,30 +247,22 @@ retry:
stats.filesLinked++; stats.filesLinked++;
stats.bytesFreed += st.st_size; stats.bytesFreed += st.st_size;
stats.blocksFreed += st.st_blocks; stats.blocksFreed += st.st_blocks;
if (act) act->result(resFileLinked, st.st_size, st.st_blocks);
} }
void LocalStore::optimiseStore(OptimiseStats& stats) { void LocalStore::optimiseStore(OptimiseStats& stats) {
Activity act(*logger, actOptimiseStore);
PathSet paths = queryAllValidPaths(); PathSet paths = queryAllValidPaths();
InodeHash inodeHash = loadInodeHash(); InodeHash inodeHash = loadInodeHash();
act.progress(0, paths.size());
uint64_t done = 0; uint64_t done = 0;
for (auto& i : paths) { for (auto& i : paths) {
addTempRoot(i); addTempRoot(i);
if (!isValidPath(i)) continue; /* path was GC'ed, probably */ if (!isValidPath(i)) continue; /* path was GC'ed, probably */
{ {
Activity act(*logger, lvlTalkative, actUnknown, LOG(INFO) << "optimising path '" << i << "'";
fmt("optimising path '%s'", i)); optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
} }
done++; done++;
act.progress(done, paths.size());
} }
} }
@ -270,16 +275,17 @@ void LocalStore::optimiseStore() {
optimiseStore(stats); optimiseStore(stats);
printInfo(format("%1% freed by hard-linking %2% files") % LOG(INFO) << showBytes(stats.bytesFreed) << " freed by hard-linking "
showBytes(stats.bytesFreed) % stats.filesLinked); << stats.filesLinked << " files";
} }
void LocalStore::optimisePath(const Path& path) { void LocalStore::optimisePath(const Path& path) {
OptimiseStats stats; OptimiseStats stats;
InodeHash inodeHash; InodeHash inodeHash;
if (settings.autoOptimiseStore) if (settings.autoOptimiseStore) {
optimisePath_(nullptr, stats, path, inodeHash); optimisePath_(stats, path, inodeHash);
}
} }
} // namespace nix } // namespace nix

19
third_party/nix/src/libstore/pathlocks.cc vendored
View file

@ -1,5 +1,6 @@
#include "pathlocks.hh" #include "pathlocks.hh"
#include <fcntl.h> #include <fcntl.h>
#include <glog/logging.h>
#include <sys/file.h> #include <sys/file.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
@ -82,7 +83,7 @@ bool PathLocks::lockPaths(const PathSet& paths, const string& waitMsg,
checkInterrupt(); checkInterrupt();
Path lockPath = path + ".lock"; Path lockPath = path + ".lock";
debug(format("locking path '%1%'") % path); DLOG(INFO) << "locking path '" << path << "'";
AutoCloseFD fd; AutoCloseFD fd;
@ -93,7 +94,9 @@ bool PathLocks::lockPaths(const PathSet& paths, const string& waitMsg,
/* Acquire an exclusive lock. */ /* Acquire an exclusive lock. */
if (!lockFile(fd.get(), ltWrite, false)) { if (!lockFile(fd.get(), ltWrite, false)) {
if (wait) { if (wait) {
if (waitMsg != "") printError(waitMsg); if (waitMsg != "") {
LOG(WARNING) << waitMsg;
}
lockFile(fd.get(), ltWrite, true); lockFile(fd.get(), ltWrite, true);
} else { } else {
/* Failed to lock this path; release all other /* Failed to lock this path; release all other
@ -103,7 +106,7 @@ bool PathLocks::lockPaths(const PathSet& paths, const string& waitMsg,
} }
} }
debug(format("lock acquired on '%1%'") % lockPath); DLOG(INFO) << "lock acquired on '" << lockPath << "'";
/* Check that the lock file hasn't become stale (i.e., /* Check that the lock file hasn't become stale (i.e.,
hasn't been unlinked). */ hasn't been unlinked). */
@ -115,7 +118,7 @@ bool PathLocks::lockPaths(const PathSet& paths, const string& waitMsg,
a lock on a deleted file. This means that other a lock on a deleted file. This means that other
processes may create and acquire a lock on processes may create and acquire a lock on
`lockPath', and proceed. So we must retry. */ `lockPath', and proceed. So we must retry. */
debug(format("open lock file '%1%' has become stale") % lockPath); DLOG(INFO) << "open lock file '" << lockPath << "' has become stale";
else else
break; break;
} }
@ -139,11 +142,11 @@ void PathLocks::unlock() {
for (auto& i : fds) { for (auto& i : fds) {
if (deletePaths) deleteLockFile(i.second, i.first); if (deletePaths) deleteLockFile(i.second, i.first);
if (close(i.first) == -1) if (close(i.first) == -1) {
printError(format("error (ignored): cannot close lock file on '%1%'") % LOG(WARNING) << "cannot close lock file on '" << i.second << "'";
i.second); }
debug(format("lock released on '%1%'") % i.second); DLOG(INFO) << "lock released on '" << i.second << "'";
} }
fds.clear(); fds.clear();

9
third_party/nix/src/libstore/profiles.cc vendored
View file

@ -1,5 +1,6 @@
#include "profiles.hh" #include "profiles.hh"
#include <errno.h> #include <errno.h>
#include <glog/logging.h>
#include <stdio.h> #include <stdio.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
@ -110,10 +111,10 @@ void deleteGeneration(const Path& profile, unsigned int gen) {
static void deleteGeneration2(const Path& profile, unsigned int gen, static void deleteGeneration2(const Path& profile, unsigned int gen,
bool dryRun) { bool dryRun) {
if (dryRun) if (dryRun) {
printInfo(format("would remove generation %1%") % gen); LOG(INFO) << "would remove generation " << gen;
else { } else {
printInfo(format("removing generation %1%") % gen); LOG(INFO) << "removing generation " << gen;
deleteGeneration(profile, gen); deleteGeneration(profile, gen);
} }
} }

3
third_party/nix/src/libstore/references.cc vendored
View file

@ -1,4 +1,5 @@
#include "references.hh" #include "references.hh"
#include <glog/logging.h>
#include <cstdlib> #include <cstdlib>
#include <map> #include <map>
#include "archive.hh" #include "archive.hh"
@ -32,7 +33,7 @@ static void search(const unsigned char* s, size_t len, StringSet& hashes,
if (!match) continue; if (!match) continue;
string ref((const char*)s + i, refLength); string ref((const char*)s + i, refLength);
if (hashes.find(ref) != hashes.end()) { if (hashes.find(ref) != hashes.end()) {
debug(format("found reference to '%1%' at offset '%2%'") % ref % i); DLOG(INFO) << "found reference to '" << ref << "' at offset " << i;
seen.insert(ref); seen.insert(ref);
hashes.erase(ref); hashes.erase(ref);
} }

97
third_party/nix/src/libstore/remote-store.cc vendored
View file

@ -1,6 +1,7 @@
#include "remote-store.hh" #include "remote-store.hh"
#include <errno.h> #include <errno.h>
#include <fcntl.h> #include <fcntl.h>
#include <glog/logging.h>
#include <sys/socket.h> #include <sys/socket.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/types.h> #include <sys/types.h>
@ -13,6 +14,7 @@
#include "finally.hh" #include "finally.hh"
#include "globals.hh" #include "globals.hh"
#include "pool.hh" #include "pool.hh"
#include "prefork-compat.hh"
#include "serialise.hh" #include "serialise.hh"
#include "util.hh" #include "util.hh"
#include "worker-protocol.hh" #include "worker-protocol.hh"
@ -150,10 +152,14 @@ void RemoteStore::initConnection(Connection& conn) {
} }
void RemoteStore::setOptions(Connection& conn) { void RemoteStore::setOptions(Connection& conn) {
conn.to << wopSetOptions << settings.keepFailed << settings.keepGoing conn.to << wopSetOptions << settings.keepFailed
<< settings.tryFallback << verbosity << settings.maxBuildJobs << settings.keepGoing
<< settings.maxSilentTime << true // TODO(tazjin): Remove the verbosity stuff here.
<< (settings.verboseBuild ? lvlError : lvlVomit) << settings.tryFallback << compat::kInfo << settings.maxBuildJobs
<< settings.maxSilentTime
<< true
// TODO(tazjin): what behaviour does this toggle remotely?
<< (settings.verboseBuild ? compat::kError : compat::kVomit)
<< 0 // obsolete log type << 0 // obsolete log type
<< 0 /* obsolete print build trace */ << 0 /* obsolete print build trace */
<< settings.buildCores << settings.useSubstitutes; << settings.buildCores << settings.useSubstitutes;
@ -194,7 +200,8 @@ struct ConnectionHandle {
~ConnectionHandle() { ~ConnectionHandle() {
if (!daemonException && std::uncaught_exceptions()) { if (!daemonException && std::uncaught_exceptions()) {
handle.markBad(); handle.markBad();
debug("closing daemon connection because of an exception"); // TODO(tazjin): are these types of things supposed to be DEBUG?
DLOG(INFO) << "closing daemon connection because of an exception";
} }
} }
@ -625,19 +632,29 @@ RemoteStore::Connection::~Connection() {
} }
} }
static Logger::Fields readFields(Source& from) { // TODO(tazjin): these logger fields used to be passed to the JSON
Logger::Fields fields; // logger but I don't care about them, whatever sends them should
// also be fixed.
static void ignoreFields(Source& from) {
size_t size = readInt(from); size_t size = readInt(from);
// This ignores the fields simply by reading the data into nowhere.
for (size_t n = 0; n < size; n++) { for (size_t n = 0; n < size; n++) {
auto type = (decltype(Logger::Field::type))readInt(from); auto type_tag = readInt(from);
if (type == Logger::Field::tInt)
fields.push_back(readNum<uint64_t>(from)); switch (type_tag) {
else if (type == Logger::Field::tString) case 0: // previously: 0 ~ Logger::Field::tInt
fields.push_back(readString(from)); readNum<uint64_t>(from);
else break;
throw Error("got unsupported field type %x from Nix daemon", (int)type);
case 1: // previously: 1 ~ Logger::Field::tString
readString(from);
break;
default:
throw Error("got unsupported field type %x from Nix daemon", type_tag);
}
} }
return fields;
} }
std::exception_ptr RemoteStore::Connection::processStderr(Sink* sink, std::exception_ptr RemoteStore::Connection::processStderr(Sink* sink,
@ -667,36 +684,52 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink* sink,
return std::make_exception_ptr(Error(status, error)); return std::make_exception_ptr(Error(status, error));
} }
else if (msg == STDERR_NEXT) else if (msg == STDERR_NEXT) {
printError(chomp(readString(from))); LOG(ERROR) << chomp(readString(from));
}
else if (msg == STDERR_START_ACTIVITY) { else if (msg == STDERR_START_ACTIVITY) {
auto act = readNum<ActivityId>(from); // Various fields need to be ignored in this case, as the
auto lvl = (Verbosity)readInt(from); // activity stuff is being removed.
auto type = (ActivityType)readInt(from); readNum<uint64_t>(from); // used to be ActivityId
auto s = readString(from); const auto verbosity = static_cast<compat::Verbosity>(readInt(from));
auto fields = readFields(from); readInt(from); // activity type
auto parent = readNum<ActivityId>(from); const auto msg = readString(from);
logger->startActivity(act, lvl, type, s, fields, parent); ignoreFields(from);
readNum<uint64_t>(from); // ActivityId of "parent"
switch (verbosity) {
case compat::kError:
LOG(ERROR) << msg;
break;
case compat::kWarn:
LOG(WARNING) << msg;
break;
case compat::kInfo:
LOG(INFO) << msg;
break;
default:
DLOG(INFO) << msg;
}
} }
else if (msg == STDERR_STOP_ACTIVITY) { else if (msg == STDERR_STOP_ACTIVITY) {
auto act = readNum<ActivityId>(from); readNum<uint64_t>(from); // used to be ActivityId
logger->stopActivity(act);
} }
else if (msg == STDERR_RESULT) { else if (msg == STDERR_RESULT) {
auto act = readNum<ActivityId>(from); readNum<uint64_t>(from); // ActivityId
auto type = (ResultType)readInt(from); readInt(from); // ResultType
auto fields = readFields(from); ignoreFields(from);
logger->result(act, type, fields);
} }
else if (msg == STDERR_LAST) else if (msg == STDERR_LAST) {
break; break;
}
else else {
throw Error("got unknown message type %x from Nix daemon", msg); throw Error("got unknown message type %x from Nix daemon", msg);
}
} }
return nullptr; return nullptr;

3
third_party/nix/src/libstore/sqlite.cc vendored
View file

@ -1,4 +1,5 @@
#include "sqlite.hh" #include "sqlite.hh"
#include <glog/logging.h>
#include <sqlite3.h> #include <sqlite3.h>
#include <atomic> #include <atomic>
#include "util.hh" #include "util.hh"
@ -157,7 +158,7 @@ void handleSQLiteBusy(const SQLiteBusy& e) {
if (now > lastWarned + 10) { if (now > lastWarned + 10) {
lastWarned = now; lastWarned = now;
printError("warning: %s", e.what()); LOG(ERROR) << e.what();
} }
/* Sleep for a while since retrying the transaction right away /* Sleep for a while since retrying the transaction right away

11
third_party/nix/src/libstore/ssh.cc vendored
View file

@ -54,8 +54,13 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(
} else { } else {
args = {"ssh", host.c_str(), "-x", "-a"}; args = {"ssh", host.c_str(), "-x", "-a"};
addCommonSSHOpts(args); addCommonSSHOpts(args);
if (socketPath != "") args.insert(args.end(), {"-S", socketPath}); if (socketPath != "") {
if (verbosity >= lvlChatty) args.push_back("-v"); args.insert(args.end(), {"-S", socketPath});
}
// TODO(tazjin): Abseil verbosity flag
/*if (verbosity >= lvlChatty) {
args.push_back("-v");
}*/
} }
args.push_back(command); args.push_back(command);
@ -107,7 +112,7 @@ Path SSHMaster::startMaster() {
"-S", state->socketPath, "-S", state->socketPath,
"-o", "LocalCommand=echo started", "-o", "LocalCommand=echo started",
"-o", "PermitLocalCommand=yes"}; "-o", "PermitLocalCommand=yes"};
if (verbosity >= lvlChatty) args.push_back("-v"); // if (verbosity >= lvlChatty) args.push_back("-v");
addCommonSSHOpts(args); addCommonSSHOpts(args);
execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); execvp(args.begin()->c_str(), stringsToCharPtrs(args).data());

43
third_party/nix/src/libstore/store-api.cc vendored
View file

@ -1,4 +1,5 @@
#include "store-api.hh" #include "store-api.hh"
#include <glog/logging.h>
#include <future> #include <future>
#include "crypto.hh" #include "crypto.hh"
#include "derivations.hh" #include "derivations.hh"
@ -526,15 +527,16 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
auto srcUri = srcStore->getUri(); auto srcUri = srcStore->getUri();
auto dstUri = dstStore->getUri(); auto dstUri = dstStore->getUri();
Activity act(*logger, lvlInfo, actCopyPath, if (srcUri == "local" || srcUri == "daemon") {
srcUri == "local" || srcUri == "daemon" LOG(INFO) << "copying path '" << storePath << "' to '" << dstUri << "'";
? fmt("copying path '%s' to '%s'", storePath, dstUri) } else {
: dstUri == "local" || dstUri == "daemon" if (dstUri == "local" || dstUri == "daemon") {
? fmt("copying path '%s' from '%s'", storePath, srcUri) LOG(INFO) << "copying path '" << storePath << "' from '" << srcUri << "'";
: fmt("copying path '%s' from '%s' to '%s'", storePath, } else {
srcUri, dstUri), LOG(INFO) << "copying path '" << storePath << "' from '" << srcUri
{storePath, srcUri, dstUri}); << "' to '" << dstUri << "'";
PushActivity pact(act.id); }
}
auto info = srcStore->queryPathInfo(storePath); auto info = srcStore->queryPathInfo(storePath);
@ -565,7 +567,6 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
LambdaSink wrapperSink([&](const unsigned char* data, size_t len) { LambdaSink wrapperSink([&](const unsigned char* data, size_t len) {
sink(data, len); sink(data, len);
total += len; total += len;
act.progress(total, info->narSize);
}); });
srcStore->narFromPath({storePath}, wrapperSink); srcStore->narFromPath({storePath}, wrapperSink);
}, },
@ -588,18 +589,13 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore,
if (missing.empty()) return; if (missing.empty()) return;
Activity act(*logger, lvlInfo, actCopyPaths, LOG(INFO) << "copying " << missing.size() << " paths";
fmt("copying %d paths", missing.size()));
std::atomic<size_t> nrDone{0}; std::atomic<size_t> nrDone{0};
std::atomic<size_t> nrFailed{0}; std::atomic<size_t> nrFailed{0};
std::atomic<uint64_t> bytesExpected{0}; std::atomic<uint64_t> bytesExpected{0};
std::atomic<uint64_t> nrRunning{0}; std::atomic<uint64_t> nrRunning{0};
auto showProgress = [&]() {
act.progress(nrDone, missing.size(), nrRunning, nrFailed);
};
ThreadPool pool; ThreadPool pool;
processGraph<Path>( processGraph<Path>(
@ -608,14 +604,12 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore,
[&](const Path& storePath) { [&](const Path& storePath) {
if (dstStore->isValidPath(storePath)) { if (dstStore->isValidPath(storePath)) {
nrDone++; nrDone++;
showProgress();
return PathSet(); return PathSet();
} }
auto info = srcStore->queryPathInfo(storePath); auto info = srcStore->queryPathInfo(storePath);
bytesExpected += info->narSize; bytesExpected += info->narSize;
act.setExpected(actCopyPath, bytesExpected);
return info->references; return info->references;
}, },
@ -625,21 +619,17 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore,
if (!dstStore->isValidPath(storePath)) { if (!dstStore->isValidPath(storePath)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning); MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress();
try { try {
copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
} catch (Error& e) { } catch (Error& e) {
nrFailed++; nrFailed++;
if (!settings.keepGoing) throw e; if (!settings.keepGoing) throw e;
logger->log(lvlError, LOG(ERROR) << "could not copy " << storePath << ": " << e.what();
format("could not copy %s: %s") % storePath % e.what());
showProgress();
return; return;
} }
} }
nrDone++; nrDone++;
showProgress();
}); });
} }
@ -702,9 +692,8 @@ void ValidPathInfo::sign(const SecretKey& secretKey) {
bool ValidPathInfo::isContentAddressed(const Store& store) const { bool ValidPathInfo::isContentAddressed(const Store& store) const {
auto warn = [&]() { auto warn = [&]() {
printError( LOG(ERROR) << "warning: path '" << path
format("warning: path '%s' claims to be content-addressed but isn't") % << "' claims to be content-addressed but isn't";
path);
}; };
if (hasPrefix(ca, "text:")) { if (hasPrefix(ca, "text:")) {
@ -873,7 +862,7 @@ std::list<ref<Store>> getDefaultSubstituters() {
try { try {
stores.push_back(openStore(uri)); stores.push_back(openStore(uri));
} catch (Error& e) { } catch (Error& e) {
printError("warning: %s", e.what()); LOG(WARNING) << e.what();
} }
}; };

6
third_party/nix/src/libutil/meson.build vendored
View file

@ -11,7 +11,8 @@ libutil_src = files(
join_paths(meson.source_root(), 'src/libutil/serialise.cc'), join_paths(meson.source_root(), 'src/libutil/serialise.cc'),
join_paths(meson.source_root(), 'src/libutil/thread-pool.cc'), join_paths(meson.source_root(), 'src/libutil/thread-pool.cc'),
join_paths(meson.source_root(), 'src/libutil/util.cc'), join_paths(meson.source_root(), 'src/libutil/util.cc'),
join_paths(meson.source_root(), 'src/libutil/xml-writer.cc')) join_paths(meson.source_root(), 'src/libutil/xml-writer.cc'),
)
libutil_headers = files( libutil_headers = files(
join_paths(meson.source_root(), 'src/libutil/affinity.hh'), join_paths(meson.source_root(), 'src/libutil/affinity.hh'),
@ -27,13 +28,14 @@ libutil_headers = files(
join_paths(meson.source_root(), 'src/libutil/lru-cache.hh'), join_paths(meson.source_root(), 'src/libutil/lru-cache.hh'),
join_paths(meson.source_root(), 'src/libutil/monitor-fd.hh'), join_paths(meson.source_root(), 'src/libutil/monitor-fd.hh'),
join_paths(meson.source_root(), 'src/libutil/pool.hh'), join_paths(meson.source_root(), 'src/libutil/pool.hh'),
join_paths(meson.source_root(), 'src/libutil/prefork-compat.hh'),
join_paths(meson.source_root(), 'src/libutil/ref.hh'), join_paths(meson.source_root(), 'src/libutil/ref.hh'),
join_paths(meson.source_root(), 'src/libutil/serialise.hh'), join_paths(meson.source_root(), 'src/libutil/serialise.hh'),
join_paths(meson.source_root(), 'src/libutil/sync.hh'), join_paths(meson.source_root(), 'src/libutil/sync.hh'),
join_paths(meson.source_root(), 'src/libutil/thread-pool.hh'), join_paths(meson.source_root(), 'src/libutil/thread-pool.hh'),
join_paths(meson.source_root(), 'src/libutil/types.hh'), join_paths(meson.source_root(), 'src/libutil/types.hh'),
join_paths(meson.source_root(), 'src/libutil/util.hh'), join_paths(meson.source_root(), 'src/libutil/util.hh'),
join_paths(meson.source_root(), 'src/libutil/xml-writer.hh') join_paths(meson.source_root(), 'src/libutil/xml-writer.hh'),
) )
libutil_dep_list = [ libutil_dep_list = [

21
third_party/nix/src/libutil/prefork-compat.hh vendored Normal file
View file

@ -0,0 +1,21 @@
// This file exists to preserve compatibility with the pre-fork
// version of Nix (2.3.4).
//
// During the refactoring, various structures are getting ripped out
// and replaced with the dummies below while code is being cleaned up.
#ifndef NIX_SRC_LIBUTIL_PREFORK_COMPAT_H_
#define NIX_SRC_LIBUTIL_PREFORK_COMPAT_H_
namespace nix::compat {
// This is used in remote-store.cc for various things that expect the
// old logging protocol when talking over the wire. It will be removed
// hen the worker protocol is redone.
enum [[deprecated("old logging compat only")]] Verbosity{
kError = 0, kWarn, kInfo, kTalkative, kChatty, kDebug, kVomit,
};
} // namespace nix::compat
#endif // NIX_SRC_LIBUTIL_PREFORK_COMPAT_H_