* Added a switch `--fallback'. From the manual:

Whenever Nix attempts to realise a derivation for which a closure is
  already known, but this closure cannot be realised, fall back on
  normalising the derivation.

  The most common scenario in which this is useful is when we have
  registered substitutes in order to perform binary distribution from,
  say, a network repository.  If the repository is down, the
  realisation of the derivation will fail.  When this option is
  specified, Nix will build the derivation instead.  Thus, binary
  installation falls back on a source installation.  This option is
  not the default since it is generally not desirable for a transient
  failure in obtaining the substitutes to lead to a full build from
  source (with the related consumption of resources).
This commit is contained in:
Eelco Dolstra 2004-06-28 10:42:57 +00:00
parent b113edeab7
commit 91dc023665
17 changed files with 185 additions and 58 deletions

View file

@ -15,3 +15,4 @@
<arg><option>-k</option></arg> <arg><option>-k</option></arg>
<arg><option>--keep-failed</option></arg> <arg><option>--keep-failed</option></arg>
<arg><option>-K</option></arg> <arg><option>-K</option></arg>
<arg><option>--fallback</option></arg>

View file

@ -147,3 +147,28 @@
</listitem> </listitem>
</varlistentry> </varlistentry>
<varlistentry>
<term><option>--fallback</option></term>
<listitem>
<para>
Whenever Nix attempts to realise a derivation for which a
closure is already known, but this closure cannot be realised,
fall back on normalising the derivation.
</para>
<para>
The most common scenario in which this is useful is when we have
registered substitutes in order to perform binary distribution
from, say, a network repository. If the repository is down, the
realisation of the derivation will fail. When this option is
specified, Nix will build the derivation instead. Thus,
binary installation falls back on a source installation. This
option is not the default since it is generally not desirable
for a transient failure in obtaining the substitutes to lead to
a full build from source (with the related consumption of
resources).
</para>
</listitem>
</varlistentry>

View file

@ -139,6 +139,8 @@ static void initAndRun(int argc, char * * argv)
keepFailed = true; keepFailed = true;
else if (arg == "--keep-going" || arg == "-k") else if (arg == "--keep-going" || arg == "-k")
keepGoing = true; keepGoing = true;
else if (arg == "--fallback")
tryFallback = true;
else if (arg == "--max-jobs" || arg == "-j") { else if (arg == "--max-jobs" || arg == "-j") {
++i; ++i;
if (i == args.end()) throw UsageError("`--max-jobs' requires an argument"); if (i == args.end()) throw UsageError("`--max-jobs' requires an argument");

View file

@ -366,8 +366,11 @@ void Database::setString(const Transaction & txn, TableId table,
void Database::setStrings(const Transaction & txn, TableId table, void Database::setStrings(const Transaction & txn, TableId table,
const string & key, const Strings & data) const string & key, const Strings & data, bool deleteEmpty)
{ {
if (deleteEmpty && data.size() == 0)
delPair(txn, table, key);
else
setString(txn, table, key, packStrings(data)); setString(txn, table, key, packStrings(data));
} }

View file

@ -76,7 +76,8 @@ public:
const string & key, const string & data); const string & key, const string & data);
void setStrings(const Transaction & txn, TableId table, void setStrings(const Transaction & txn, TableId table,
const string & key, const Strings & data); const string & key, const Strings & data,
bool deleteEmpty = true);
void delPair(const Transaction & txn, TableId table, void delPair(const Transaction & txn, TableId table,
const string & key); const string & key);

View file

@ -10,6 +10,8 @@ bool keepFailed = false;
bool keepGoing = false; bool keepGoing = false;
bool tryFallback = false;
Verbosity buildVerbosity = lvlDebug; Verbosity buildVerbosity = lvlDebug;
unsigned int maxBuildJobs = 1; unsigned int maxBuildJobs = 1;

View file

@ -33,6 +33,10 @@ extern bool keepFailed;
of the same goal) fails. */ of the same goal) fails. */
extern bool keepGoing; extern bool keepGoing;
/* Whether, if we cannot realise the known closure corresponding to a
derivation, we should try to normalise the derivation instead. */
extern bool tryFallback;
/* Verbosity level for build output. */ /* Verbosity level for build output. */
extern Verbosity buildVerbosity; extern Verbosity buildVerbosity;

View file

@ -201,11 +201,28 @@ void Goal::waiteeDone(GoalPtr waitee, bool success)
{ {
assert(waitees.find(waitee) != waitees.end()); assert(waitees.find(waitee) != waitees.end());
waitees.erase(waitee); waitees.erase(waitee);
assert(nrWaitees > 0);
if (!success) ++nrFailed; if (!success) ++nrFailed;
if (!--nrWaitees || (!success && !keepGoing))
assert(nrWaitees > 0);
if (!--nrWaitees || (!success && !keepGoing)) {
/* If we failed and keepGoing is not set, we remove all
remaining waitees. */
for (Goals::iterator i = waitees.begin(); i != waitees.end(); ++i) {
GoalPtr goal = *i;
WeakGoals waiters2;
for (WeakGoals::iterator j = goal->waiters.begin();
j != goal->waiters.end(); ++j)
if (j->lock() != shared_from_this())
waiters2.insert(*j);
goal->waiters = waiters2;
}
waitees.clear();
worker.wakeUp(shared_from_this()); worker.wakeUp(shared_from_this());
} }
}
void Goal::amDone(bool success) void Goal::amDone(bool success)
@ -271,6 +288,17 @@ const char * * strings2CharPtrs(const Strings & ss)
} }
/* Should only be called after an expression has been normalised. */
Path queryNormalForm(const Path & nePath)
{
StoreExpr ne = storeExprFromPath(nePath);
if (ne.type == StoreExpr::neClosure) return nePath;
Path nfPath;
if (!querySuccessor(nePath, nfPath)) abort();
return nfPath;
}
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
@ -472,13 +500,7 @@ void NormalisationGoal::inputNormalised()
/* Inputs must also be realised before we can build this goal. */ /* Inputs must also be realised before we can build this goal. */
for (PathSet::iterator i = expr.derivation.inputs.begin(); for (PathSet::iterator i = expr.derivation.inputs.begin();
i != expr.derivation.inputs.end(); ++i) i != expr.derivation.inputs.end(); ++i)
{ addWaitee(worker.makeRealisationGoal(queryNormalForm(*i)));
Path neInput = *i, nfInput;
if (querySuccessor(neInput, nfInput))
neInput = nfInput;
/* Otherwise the input must be a closure. */
addWaitee(worker.makeRealisationGoal(neInput));
}
resetWaitees(expr.derivation.inputs.size()); resetWaitees(expr.derivation.inputs.size());
@ -829,8 +851,8 @@ bool NormalisationGoal::prepareBuild()
i != expr.derivation.inputs.end(); ++i) i != expr.derivation.inputs.end(); ++i)
{ {
checkInterrupt(); checkInterrupt();
Path nePath = *i, nfPath; Path nePath = *i;
if (!querySuccessor(nePath, nfPath)) nfPath = nePath; Path nfPath = queryNormalForm(nePath);
inputNFs.insert(nfPath); inputNFs.insert(nfPath);
if (nfPath != nePath) inputSucs[nePath] = nfPath; if (nfPath != nePath) inputSucs[nePath] = nfPath;
/* !!! nfPath should be a root of the garbage collector while /* !!! nfPath should be a root of the garbage collector while
@ -1174,9 +1196,15 @@ string NormalisationGoal::name()
class RealisationGoal : public Goal class RealisationGoal : public Goal
{ {
private: private:
/* The path of the closure store expression. */ /* The path of the store expression. */
Path nePath; Path nePath;
/* The normal form. */
Path nfPath;
/* Whether we should try to delete a broken successor mapping. */
bool tryFallback;
/* The store expression stored at nePath. */ /* The store expression stored at nePath. */
StoreExpr expr; StoreExpr expr;
@ -1191,9 +1219,12 @@ public:
/* The states. */ /* The states. */
void init(); void init();
void isNormalised();
void haveStoreExpr(); void haveStoreExpr();
void elemFinished(); void elemFinished();
void fallBack(const format & error);
string name(); string name();
}; };
@ -1202,6 +1233,7 @@ RealisationGoal::RealisationGoal(const Path & _nePath, Worker & _worker)
: Goal(_worker) : Goal(_worker)
{ {
nePath = _nePath; nePath = _nePath;
tryFallback = ::tryFallback;
state = &RealisationGoal::init; state = &RealisationGoal::init;
} }
@ -1221,11 +1253,36 @@ void RealisationGoal::init()
{ {
trace("init"); trace("init");
/* The first thing to do is to make sure that the store expression if (querySuccessor(nePath, nfPath)) {
exists. If it doesn't, it may be created through a isNormalised();
substitute. */ return;
}
/* First normalise the expression (which is a no-op if the
expression is already a closure). */
resetWaitees(1); resetWaitees(1);
addWaitee(worker.makeSubstitutionGoal(nePath)); addWaitee(worker.makeNormalisationGoal(nePath));
/* Since there is no successor right now, the normalisation goal
will perform an actual build. So there is no sense in trying a
fallback if the realisation of the closure fails (it can't
really fail). */
tryFallback = false;
state = &RealisationGoal::isNormalised;
}
void RealisationGoal::isNormalised()
{
trace("has been normalised");
nfPath = queryNormalForm(nePath);
/* Now make sure that the store expression exists. If it doesn't,
it may be created through a substitute. */
resetWaitees(1);
addWaitee(worker.makeSubstitutionGoal(nfPath));
state = &RealisationGoal::haveStoreExpr; state = &RealisationGoal::haveStoreExpr;
} }
@ -1236,21 +1293,18 @@ void RealisationGoal::haveStoreExpr()
trace("loading store expression"); trace("loading store expression");
if (nrFailed != 0) { if (nrFailed != 0) {
printMsg(lvlError, fallBack(format("cannot realise closure `%1%' since that file is missing") % nfPath);
format("cannot realise missing store expression `%1%'")
% nePath);
amDone(false);
return; return;
} }
assert(isValidPath(nePath)); assert(isValidPath(nfPath));
/* Get the store expression. */ /* Get the store expression. */
expr = storeExprFromPath(nePath); expr = storeExprFromPath(nfPath);
/* If this is a normal form (i.e., a closure) we are also done. */ /* If this is a normal form (i.e., a closure) we are also done. */
if (expr.type != StoreExpr::neClosure) if (expr.type != StoreExpr::neClosure)
throw Error(format("expected closure in `%1%'") % nePath); throw Error(format("expected closure in `%1%'") % nfPath);
/* Each path in the closure should exist, or should be creatable /* Each path in the closure should exist, or should be creatable
through a substitute. */ through a substitute. */
@ -1269,12 +1323,11 @@ void RealisationGoal::elemFinished()
trace("all closure elements present"); trace("all closure elements present");
if (nrFailed != 0) { if (nrFailed != 0) {
printMsg(lvlError, fallBack(
format("cannot realise closure `%1%': " format("cannot realise closure `%1%': "
"%2% closure element(s) are not present " "%2% closure element(s) are not present "
"and could not be substituted") "and could not be substituted")
% nePath % nrFailed); % nfPath % nrFailed);
amDone(false);
return; return;
} }
@ -1282,6 +1335,21 @@ void RealisationGoal::elemFinished()
} }
void RealisationGoal::fallBack(const format & error)
{
if (tryFallback && nePath != nfPath) {
printMsg(lvlError, format("%1%; trying to normalise derivation instead")
% error);
tryFallback = false;
unregisterSuccessor(nePath);
init();
} else {
printMsg(lvlError, format("%1%; maybe `--fallback' will help") % error);
amDone(false);
}
}
string RealisationGoal::name() string RealisationGoal::name()
{ {
return (format("realisation of `%1%'") % nePath).str(); return (format("realisation of `%1%'") % nePath).str();
@ -1409,8 +1477,7 @@ void SubstitutionGoal::exprNormalised()
} }
/* Realise the substitute store expression. */ /* Realise the substitute store expression. */
if (!querySuccessor(sub.storeExpr, nfSub)) nfSub = queryNormalForm(sub.storeExpr);
nfSub = sub.storeExpr;
addWaitee(worker.makeRealisationGoal(nfSub)); addWaitee(worker.makeRealisationGoal(nfSub));
resetWaitees(1); resetWaitees(1);
@ -1869,19 +1936,19 @@ Path normaliseStoreExpr(const Path & nePath)
if (!worker.run(worker.makeNormalisationGoal(nePath))) if (!worker.run(worker.makeNormalisationGoal(nePath)))
throw Error(format("normalisation of store expression `%1%' failed") % nePath); throw Error(format("normalisation of store expression `%1%' failed") % nePath);
Path nfPath; return queryNormalForm(nePath);
if (!querySuccessor(nePath, nfPath)) abort();
return nfPath;
} }
void realiseClosure(const Path & nePath) Path realiseStoreExpr(const Path & nePath)
{ {
startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath); startNest(nest, lvlDebug, format("realising `%1%'") % nePath);
Worker worker; Worker worker;
if (!worker.run(worker.makeRealisationGoal(nePath))) if (!worker.run(worker.makeRealisationGoal(nePath)))
throw Error(format("realisation of closure `%1%' failed") % nePath); throw Error(format("realisation of store expressions `%1%' failed") % nePath);
return queryNormalForm(nePath);
} }

View file

@ -10,13 +10,13 @@
successor is known. */ successor is known. */
Path normaliseStoreExpr(const Path & nePath); Path normaliseStoreExpr(const Path & nePath);
/* Realise a closure store expression in the file system. /* Realise a store expression. If the expression is a derivation, it
is first normalised into a closure. The closure is then realised
The pending paths are those that are already being realised. This in the file system (i.e., it is ensured that each path in the
prevents infinite recursion for paths realised through a substitute closure exists in the file system, if necessary by using the
(since when we build the substitute, we would first try to realise substitute mechanism). Returns the normal form of the expression
its output paths through substitutes... kaboom!). */ (i.e., its closure expression). */
void realiseClosure(const Path & nePath); Path realiseStoreExpr(const Path & nePath);
/* Ensure that a path exists, possibly by instantiating it by /* Ensure that a path exists, possibly by instantiating it by
realising a substitute. */ realising a substitute. */

View file

@ -237,6 +237,30 @@ void registerSuccessor(const Transaction & txn,
} }
void unregisterSuccessor(const Path & srcPath)
{
assertStorePath(srcPath);
Transaction txn(nixDB);
Path sucPath;
if (!nixDB.queryString(txn, dbSuccessors, srcPath, sucPath)) {
txn.abort();
return;
}
nixDB.delPair(txn, dbSuccessors, srcPath);
Paths revs;
nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs);
Paths::iterator i = find(revs.begin(), revs.end(), srcPath);
assert(i != revs.end());
revs.erase(i);
nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs);
txn.commit();
}
bool querySuccessor(const Path & srcPath, Path & sucPath) bool querySuccessor(const Path & srcPath, Path & sucPath)
{ {
return nixDB.queryString(noTxn, dbSuccessors, srcPath, sucPath); return nixDB.queryString(noTxn, dbSuccessors, srcPath, sucPath);
@ -294,9 +318,6 @@ static void writeSubstitutes(const Transaction & txn,
ss.push_back(packStrings(ss2)); ss.push_back(packStrings(ss2));
} }
if (ss.size() == 0)
nixDB.delPair(txn, dbSubstitutes, srcPath);
else
nixDB.setStrings(txn, dbSubstitutes, srcPath, ss); nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
} }

View file

@ -54,6 +54,9 @@ void copyPath(const Path & src, const Path & dst);
void registerSuccessor(const Transaction & txn, void registerSuccessor(const Transaction & txn,
const Path & srcPath, const Path & sucPath); const Path & srcPath, const Path & sucPath);
/* Remove a successor mapping. */
void unregisterSuccessor(const Path & srcPath);
/* Return the predecessors of the Nix expression stored at the given /* Return the predecessors of the Nix expression stored at the given
path. */ path. */
bool querySuccessor(const Path & srcPath, Path & sucPath); bool querySuccessor(const Path & srcPath, Path & sucPath);

View file

@ -213,8 +213,7 @@ void createUserEnv(EvalState & state, const DrvInfos & drvs,
/* Realise the resulting store expression. */ /* Realise the resulting store expression. */
debug(format("realising user environment")); debug(format("realising user environment"));
Path nfPath = normaliseStoreExpr(topLevelDrv.drvPath); Path nfPath = realiseStoreExpr(topLevelDrv.drvPath);
realiseClosure(nfPath);
/* Switch the current user environment to the output path. */ /* Switch the current user environment to the output path. */
debug(format("switching to new user environment")); debug(format("switching to new user environment"));

View file

@ -26,8 +26,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
for (Strings::iterator i = opArgs.begin(); for (Strings::iterator i = opArgs.begin();
i != opArgs.end(); i++) i != opArgs.end(); i++)
{ {
Path nfPath = normaliseStoreExpr(*i); Path nfPath = realiseStoreExpr(*i);
realiseClosure(nfPath);
cout << format("%1%\n") % (string) nfPath; cout << format("%1%\n") % (string) nfPath;
} }
} }
@ -58,8 +57,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
Path maybeNormalise(const Path & ne, bool normalise, bool realise) Path maybeNormalise(const Path & ne, bool normalise, bool realise)
{ {
if (realise) { if (realise) {
Path ne2 = normaliseStoreExpr(ne); Path ne2 = realiseStoreExpr(ne);
realiseClosure(ne2);
return normalise ? ne2 : ne; return normalise ? ne2 : ne;
} else } else
return normalise ? normaliseStoreExpr(ne) : ne; return normalise ? normaliseStoreExpr(ne) : ne;

View file

@ -19,10 +19,10 @@ parallel.sh: parallel.nix
build-hook.sh: build-hook.nix build-hook.sh: build-hook.nix
substitutes.sh: substitutes.nix substituter.nix substitutes.sh: substitutes.nix substituter.nix
substitutes2.sh: substitutes2.nix substituter.nix substituter2.nix substitutes2.sh: substitutes2.nix substituter.nix substituter2.nix
fall-back.sh: fall-back.nix fallback.sh: fallback.nix
TESTS = init.sh simple.sh dependencies.sh locking.sh parallel.sh \ TESTS = init.sh simple.sh dependencies.sh locking.sh parallel.sh \
build-hook.sh substitutes.sh substitutes2.sh build-hook.sh substitutes.sh substitutes2.sh fallback.sh verify.sh
XFAIL_TESTS = XFAIL_TESTS =
@ -36,4 +36,4 @@ EXTRA_DIST = $(TESTS) \
build-hook.nix.in build-hook.hook.sh \ build-hook.nix.in build-hook.hook.sh \
substitutes.nix.in substituter.nix.in substituter.builder.sh \ substitutes.nix.in substituter.nix.in substituter.builder.sh \
substitutes2.nix.in substituter2.nix.in substituter2.builder.sh \ substitutes2.nix.in substituter2.nix.in substituter2.builder.sh \
fall-back.nix.in fallback.nix.in

View file

@ -7,7 +7,7 @@ suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store
(echo $suc && echo $NIX_STORE_DIR/ffffffffffffffffffffffffffffffff.store && echo "/bla" && echo 0) | $TOP/src/nix-store/nix-store --substitute (echo $suc && echo $NIX_STORE_DIR/ffffffffffffffffffffffffffffffff.store && echo "/bla" && echo 0) | $TOP/src/nix-store/nix-store --substitute
$TOP/src/nix-store/nix-store --successor $storeExpr $suc $TOP/src/nix-store/nix-store --successor $storeExpr $suc
outPath=$($TOP/src/nix-store/nix-store -qnf "$storeExpr") outPath=$($TOP/src/nix-store/nix-store -qnf --fallback "$storeExpr")
echo "output path is $outPath" echo "output path is $outPath"

1
tests/verify.sh Normal file
View file

@ -0,0 +1 @@
$TOP/src/nix-store/nix-store --verify