From 5c9e9f732df6d95d712f25de9880b7461c53d6ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludovic=20Court=C3=A8s?= Date: Thu, 30 Jun 2011 15:19:13 +0000 Subject: [PATCH] Add support for the `build-timeout' and `--timeout' options. --- doc/manual/conf-file.xml | 17 +++++++++++++++++ doc/manual/opt-common-syn.xml | 4 ++++ doc/manual/opt-common.xml | 10 ++++++++++ doc/manual/release-notes.xml | 10 ++++++++++ src/libmain/shared.cc | 3 +++ src/libstore/build.cc | 34 ++++++++++++++++++++++++++++++---- src/libstore/globals.cc | 1 + src/libstore/globals.hh | 4 ++++ tests/Makefile.am | 3 ++- tests/timeout.builder.sh | 2 ++ tests/timeout.nix | 8 ++++++++ tests/timeout.sh | 24 ++++++++++++++++++++++++ 12 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 tests/timeout.builder.sh create mode 100644 tests/timeout.nix create mode 100644 tests/timeout.sh diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index cb47b9941..44a7ac388 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -134,6 +134,23 @@ env-keep-derivations = false + build-timeout + + + + This option defines the maximum number of seconds that a + builder can run. This is useful (for instance in a automated + build system) to catch builds that are stuck in an infinite loop + but keep writing to their standard output or standard error. It + can be overriden using the command line + switch. + + The value 0 means that there is no + timeout. This is also the default. + + + diff --git a/doc/manual/opt-common-syn.xml b/doc/manual/opt-common-syn.xml index da60288b2..23bd36dd6 100644 --- a/doc/manual/opt-common-syn.xml +++ b/doc/manual/opt-common-syn.xml @@ -21,6 +21,10 @@ number + + + number + diff --git a/doc/manual/opt-common.xml b/doc/manual/opt-common.xml index 2d67162d7..e86e4b412 100644 --- a/doc/manual/opt-common.xml +++ b/doc/manual/opt-common.xml @@ -132,6 +132,16 @@ + + + Sets the maximum number of seconds that a builder + can run. The default is specified by the build-timeout + configuration setting. 0 means no + timeout. + + + diff --git a/doc/manual/release-notes.xml b/doc/manual/release-notes.xml index 1e579a37b..0c29cae90 100644 --- a/doc/manual/release-notes.xml +++ b/doc/manual/release-notes.xml @@ -22,6 +22,16 @@ option. + + The option (corresponding to the + configuration setting build-timeout) allows you + to set an absolute timeout on builds — if a build runs for more than + the given number of seconds, it is terminated. This is useful for + recovering automatically from builds that are stuck in an infinite + loop but keep producing output, and for which + --max-silent-time is ineffective. + + diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 3110c9452..43ec4bcdd 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -142,6 +142,7 @@ static void initAndRun(int argc, char * * argv) maxBuildJobs = queryIntSetting("build-max-jobs", 1); buildCores = queryIntSetting("build-cores", 1); maxSilentTime = queryIntSetting("build-max-silent-time", 0); + buildTimeout = queryIntSetting("build-timeout", 0); /* Catch SIGINT. */ struct sigaction act; @@ -237,6 +238,8 @@ static void initAndRun(int argc, char * * argv) readOnlyMode = true; else if (arg == "--max-silent-time") maxSilentTime = getIntArg(arg, i, args.end()); + else if (arg == "--timeout") + buildTimeout = getIntArg(arg, i, args.end()); else if (arg == "--no-build-hook") useBuildHook = false; else if (arg == "--show-trace") diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 83bd6754a..4df62acea 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -209,7 +209,10 @@ private: /* Last time the goals in `waitingForAWhile' where woken up. */ time_t lastWokenUp; - + + /* Last time `waitForInput' was last called. */ + time_t lastWait; + public: bool cacheFailure; @@ -681,7 +684,8 @@ HookInstance::HookInstance() builderOut.readSide.close(); if (dup2(builderOut.writeSide, 4) == -1) throw SysError("dupping builder's stdout/stderr"); - + + /* XXX: Pass `buildTimeout' to the hook? */ execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(), (format("%1%") % maxSilentTime).str().c_str(), (format("%1%") % printBuildTrace).str().c_str(), @@ -2666,7 +2670,14 @@ void Worker::waitForInput() struct timeval timeout; timeout.tv_usec = 0; time_t before = time(0); - + + /* If a global timeout has been set, sleep until it's done. */ + if (buildTimeout != 0) { + useTimeout = true; + if (lastWait == 0 || lastWait > before) lastWait = before; + timeout.tv_sec = std::max((time_t) 0, lastWait + buildTimeout - before); + } + /* If we're monitoring for silence on stdout/stderr, sleep until the first deadline for any child. */ if (maxSilentTime != 0) { @@ -2678,8 +2689,11 @@ void Worker::waitForInput() } } if (oldest) { + time_t silenceTimeout = std::max((time_t) 0, oldest + maxSilentTime - before); + timeout.tv_sec = useTimeout + ? std::min(silenceTimeout, timeout.tv_sec) + : silenceTimeout; useTimeout = true; - timeout.tv_sec = std::max((time_t) 0, oldest + maxSilentTime - before); printMsg(lvlVomit, format("sleeping %1% seconds") % timeout.tv_sec); } } @@ -2717,6 +2731,9 @@ void Worker::waitForInput() time_t after = time(0); + /* Keep track of when we were last called. */ + lastWait = after; + /* Process all available file descriptors. */ /* Since goals may be canceled from inside the loop below (causing @@ -2765,6 +2782,15 @@ void Worker::waitForInput() % goal->getName() % maxSilentTime); goal->cancel(); } + + if (buildTimeout != 0 && + after - before >= (time_t) buildTimeout) + { + printMsg(lvlError, + format("%1% timed out after %2% seconds of activity") + % goal->getName() % buildTimeout); + goal->cancel(); + } } if (!waitingForAWhile.empty() && lastWokenUp + wakeUpInterval <= after) { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7069d104a..2e9dc8823 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -26,6 +26,7 @@ unsigned int buildCores = 1; bool readOnlyMode = false; string thisSystem = "unset"; time_t maxSilentTime = 0; +time_t buildTimeout = 0; Paths substituters; bool useBuildHook = true; bool printBuildTrace = false; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a74a741d6..231c1f850 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -72,6 +72,10 @@ extern string thisSystem; infinity. */ extern time_t maxSilentTime; +/* The maximum duration in seconds that a builder can run. 0 means + infinity. */ +extern time_t buildTimeout; + /* The substituters. There are programs that can somehow realise a store path without building, e.g., by downloading it or copying it from a CD. */ diff --git a/tests/Makefile.am b/tests/Makefile.am index fb4a2285c..d383bce24 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -8,7 +8,7 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh negative-caching.sh \ - binary-patching.sh + binary-patching.sh timeout.sh XFAIL_TESTS = @@ -33,5 +33,6 @@ EXTRA_DIST = $(TESTS) \ export-graph.nix \ negative-caching.nix \ binary-patching.nix \ + timeout.nix timeout.builder.sh \ $(wildcard lang/*.nix) $(wildcard lang/*.exp) $(wildcard lang/*.exp.xml) $(wildcard lang/*.flags) \ common.sh.in diff --git a/tests/timeout.builder.sh b/tests/timeout.builder.sh new file mode 100644 index 000000000..3e6e1fc04 --- /dev/null +++ b/tests/timeout.builder.sh @@ -0,0 +1,2 @@ +echo "\`timeout' builder entering an infinite loop" +while true ; do : ; done diff --git a/tests/timeout.nix b/tests/timeout.nix new file mode 100644 index 000000000..dec258801 --- /dev/null +++ b/tests/timeout.nix @@ -0,0 +1,8 @@ +with import ./config.nix; + +mkDerivation { + name = "timeout"; + builder = ./timeout.builder.sh; + PATH = ""; + goodPath = path; +} diff --git a/tests/timeout.sh b/tests/timeout.sh new file mode 100644 index 000000000..f27739fb2 --- /dev/null +++ b/tests/timeout.sh @@ -0,0 +1,24 @@ +# Test the `--timeout' option. + +source common.sh + +drvPath=$($nixinstantiate timeout.nix) + +test "$($nixstore -q --binding system "$drvPath")" = "$system" + +echo "derivation is $drvPath" + +failed=0 +messages="`$nixstore -r --timeout 2 $drvPath 2>&1 || failed=1`" +if test $failed -ne 0; then + echo "error: \`nix-store' succeeded; should have timed out" >&2 + exit 1 +fi + +if ! echo "$messages" | grep "timed out"; then + echo "error: \`nix-store' may have failed for reasons other than timeout" >&2 + echo >&2 + echo "output of \`nix-store' follows:" >&2 + echo "$messages" >&2 + exit 1 +fi