nix-store -l: Fetch build logs from the Internet

If a build log is not available locally, then ‘nix-store -l’ will now
try to download it from the servers listed in the ‘log-servers’ option
in nix.conf. For instance, if you have:

  log-servers = http://hydra.nixos.org/log

then it will try to get logs from http://hydra.nixos.org/log/<base
name of the store path>. So you can do things like:

  $ nix-store -l $(which xterm)

and get a log even if xterm wasn't built locally.

Makefile.config.in

@@ -10,6 +10,7 @@ PACKAGE_VERSION = @PACKAGE_VERSION@
 bash = @bash@
 bindir = @bindir@
 bsddiff_compat_include = @bsddiff_compat_include@
+curl = @curl@
 datadir = @datadir@
 datarootdir = @datarootdir@
 dblatex = @dblatex@

doc/manual/conf-file.xml

@@ -465,6 +465,20 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
   </varlistentry>
 
 
+  <varlistentry xml:id="conf-log-servers"><term><literal>log-servers</literal></term>
+
+    <listitem>
+
+      <para>A list of URL prefixes (such as
+      <literal>http://hydra.nixos.org/log</literal>) from which
+      <command>nix-store -l</command> will try to fetch build logs if
+      they’re not available locally.</para>
+
+    </listitem>
+
+  </varlistentry>
+
+
 </variablelist>
 
 </para>

doc/manual/nix-store.xml

@@ -1147,9 +1147,14 @@ the store path is used.</para>
 
 <para>Build logs are kept in
 <filename>/nix/var/log/nix/drvs</filename>.  However, there is no
-guarantee that a build log is available for any particular store
-path.  For instance, if the path was downloaded as a pre-built binary
-through a substitute, then the log is unavailable.</para>
+guarantee that a build log is available for any particular store path.
+For instance, if the path was downloaded as a pre-built binary through
+a substitute, then the log is unavailable. If the log is not available
+locally, then <command>nix-store</command> will try to download the
+log from the servers specified in the Nix option
+<option>log-servers</option>. For example, if it’s set to
+<literal>http://hydra.nixos.org/log</literal>, then Nix will check
+<literal>http://hydra.nixos.org/log/<replaceable>base-name</replaceable></literal>.</para>
 
 </refsection>
 

src/libstore/globals.cc

@@ -147,6 +147,7 @@ void Settings::update()
     get(envKeepDerivations, "env-keep-derivations");
     get(sshSubstituterHosts, "ssh-substituter-hosts");
     get(useSshSubstituter, "use-ssh-substituter");
+    get(logServers, "log-servers");
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
     if (subs == "default") {

src/libstore/globals.hh

@@ -197,6 +197,9 @@ struct Settings {
     /* Whether to show a stack trace if Nix evaluation fails. */
     bool showTrace;
 
+    /* A list of URL prefixes that can return Nix build logs. */
+    Strings logServers;
+
 private:
     SettingsMap settings, overrides;
 

src/libutil/util.cc

@@ -901,7 +901,7 @@ string runProgram(Path program, bool searchPath, const Strings & args)
     /* Wait for the child to finish. */
     int status = pid.wait(true);
     if (!statusOk(status))
-        throw Error(format("program `%1%' %2%")
+        throw ExecError(format("program `%1%' %2%")
             % program % statusToString(status));
 
     return result;

src/libutil/util.hh

@@ -257,6 +257,8 @@ void killUser(uid_t uid);
 string runProgram(Path program, bool searchPath = false,
     const Strings & args = Strings());
 
+MakeError(ExecError, Error)
+
 /* Close all file descriptors except stdin, stdout, stderr, and those
    listed in the given set.  Good practice in child processes. */
 void closeMostFDs(const set<int> & exceptions);

src/nix-store/local.mk

@@ -7,3 +7,5 @@ nix-store_SOURCES := $(wildcard $(d)/*.cc)
 nix-store_LIBS = libmain libstore libutil libformat
 
 nix-store_LDFLAGS = -lbz2
+
+nix-store_CXXFLAGS = -DCURL=\"$(curl)\"

src/nix-store/nix-store.cc

@@ -467,10 +467,11 @@ static void opReadLog(Strings opFlags, Strings opArgs)
     foreach (Strings::iterator, i, opArgs) {
         Path path = useDeriver(followLinksToStorePath(*i));
 
-        for (int j = 0; j <= 2; j++) {
-            if (j == 2) throw Error(format("build log of derivation `%1%' is not available") % path);
-
         string baseName = baseNameOf(path);
+        bool found = false;
+
+        for (int j = 0; j < 2; j++) {
+
             Path logPath =
                 j == 0
                 ? (format("%1%/%2%/%3%/%4%") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str()
@@ -481,6 +482,7 @@ static void opReadLog(Strings opFlags, Strings opArgs)
                 /* !!! Make this run in O(1) memory. */
                 string log = readFile(logPath);
                 writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size());
+                found = true;
                 break;
             }
 
@@ -500,9 +502,30 @@ static void opReadLog(Strings opFlags, Strings opArgs)
                     writeFull(STDOUT_FILENO, buf, n);
                 } while (err != BZ_STREAM_END);
                 BZ2_bzReadClose(&err, bz);
+                found = true;
                 break;
             }
         }
+
+        if (!found) {
+            for (auto & i : settings.logServers) {
+                string prefix = i;
+                if (!prefix.empty() && prefix.back() != '/') prefix += '/';
+                string url = prefix + baseName;
+                try {
+                    string log = runProgram(CURL, true, {"--fail", "--location", "--silent", "--", url});
+                    std::cout << "(using build log from " << url << ")" << std::endl;
+                    std::cout << log;
+                    found = true;
+                    break;
+                } catch (ExecError & e) {
+                    /* Ignore errors from curl. FIXME: actually, might be
+                       nice to print a warning on HTTP status != 404. */
+                }
+            }
+        }
+
+        if (!found) throw Error(format("build log of derivation `%1%' is not available") % path);
     }
 }