Merge branch 'master' of https://github.com/NixOS/nix into parallel-xz

doc/manual/expressions/builtins.xml

@@ -308,7 +308,8 @@ stdenv.mkDerivation { … }
   </varlistentry>
 
 
-  <varlistentry><term><function>builtins.filterSource</function>
+  <varlistentry xml:id='builtin-filterSource'>
+    <term><function>builtins.filterSource</function>
     <replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
 
     <listitem>
@@ -768,6 +769,75 @@ Evaluates to <literal>[ "foo" ]</literal>.
 
   </varlistentry>
 
+  <varlistentry>
+    <term>
+      <function>builtins.path</function>
+      <replaceable>args</replaceable>
+    </term>
+
+    <listitem>
+      <para>
+        An enrichment of the built-in path type, based on the attributes
+        present in <replaceable>args</replaceable>. All are optional
+        except <varname>path</varname>:
+      </para>
+
+      <variablelist>
+        <varlistentry>
+          <term>path</term>
+          <listitem>
+            <para>The underlying path.</para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>name</term>
+          <listitem>
+            <para>
+              The name of the path when added to the store. This can
+              used to reference paths that have nix-illegal characters
+              in their names, like <literal>@</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>filter</term>
+          <listitem>
+            <para>
+              A function of the type expected by
+              <link linkend="builtin-filterSource">builtins.filterSource</link>,
+              with the same semantics.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>recursive</term>
+          <listitem>
+            <para>
+              When <literal>false</literal>, when
+              <varname>path</varname> is added to the store it is with a
+              flat hash, rather than a hash of the NAR serialization of
+              the file. Thus, <varname>path</varname> must refer to a
+              regular file, not a directory. This allows similar
+              behavior to <literal>fetchurl</literal>. Defaults to
+              <literal>true</literal>.
+            </para>
+          </listitem>
+        </varlistentry>
+        <varlistentry>
+          <term>sha256</term>
+          <listitem>
+            <para>
+              When provided, this is the expected hash of the file at
+              the path. Evaluation will fail if the hash is incorrect,
+              and providing a hash allows
+              <literal>builtins.path</literal> to be used even when the
+              <literal>pure-eval</literal> nix config option is on.
+            </para>
+          </listitem>
+        </varlistentry>
+      </variablelist>
+    </listitem>
+  </varlistentry>
 
   <varlistentry><term><function>builtins.pathExists</function>
   <replaceable>path</replaceable></term>

release.nix

@@ -224,7 +224,9 @@ let
       nix = build.x86_64-linux; system = "x86_64-linux";
     });
 
-    tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system:
+    tests.setuid = pkgs.lib.genAttrs
+      ["i686-linux" "x86_64-linux"]
+      (system:
         import ./tests/setuid.nix rec {
           inherit nixpkgs;
           nix = build.${system}; inherit system;

src/libexpr/eval.cc

@@ -1578,7 +1578,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
         dstPath = srcToStore[path];
     else {
         dstPath = settings.readOnlyMode
-            ? store->computeStorePathForPath(checkSourcePath(path)).first
+            ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first
             : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
         srcToStore[path] = dstPath;
         printMsg(lvlChatty, format("copied source '%1%' -> '%2%'")

src/libexpr/primops.cc

@@ -1023,20 +1023,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
 }
 
 
-static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
+static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
+    Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
 {
-    PathSet context;
+    const auto path = settings.pureEval && expectedHash ?
-    Path path = state.coerceToPath(pos, *args[1], context);
+        path_ :
-    if (!context.empty())
+        state.checkSourcePath(path_);
-        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
+    PathFilter filter = filterFun ? ([&](const Path & path) {
-
-    state.forceValue(*args[0]);
-    if (args[0]->type != tLambda)
-        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
-
-    path = state.checkSourcePath(path);
-
-    PathFilter filter = [&](const Path & path) {
         auto st = lstat(path);
 
         /* Call the filter function.  The first argument is the path,
@@ -1045,7 +1038,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
         mkString(arg1, path);
 
         Value fun2;
-        state.callFunction(*args[0], arg1, fun2, noPos);
+        state.callFunction(*filterFun, arg1, fun2, noPos);
 
         Value arg2;
         mkString(arg2,
@@ -1058,16 +1051,79 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
         state.callFunction(fun2, arg2, res, noPos);
 
         return state.forceBool(res, pos);
-    };
+    }) : defaultPathFilter;
 
-    Path dstPath = settings.readOnlyMode
+    Path expectedStorePath;
-        ? state.store->computeStorePathForPath(path, true, htSHA256, filter).first
+    if (expectedHash) {
-        : state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair);
+        expectedStorePath =
+            state.store->makeFixedOutputPath(recursive, expectedHash, name);
+    }
+    Path dstPath;
+    if (!expectedHash || !state.store->isValidPath(expectedStorePath)) {
+        dstPath = settings.readOnlyMode
+            ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
+            : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair);
+        if (expectedHash && expectedStorePath != dstPath) {
+            throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path);
+        }
+    } else
+        dstPath = expectedStorePath;
 
     mkString(v, dstPath, {dstPath});
 }
 
 
+static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    PathSet context;
+    Path path = state.coerceToPath(pos, *args[1], context);
+    if (!context.empty())
+        throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
+
+    state.forceValue(*args[0]);
+    if (args[0]->type != tLambda)
+        throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
+
+    addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v);
+}
+
+static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+    state.forceAttrs(*args[0], pos);
+    Path path;
+    string name;
+    Value * filterFun = nullptr;
+    auto recursive = true;
+    Hash expectedHash;
+
+    for (auto & attr : *args[0]->attrs) {
+        const string & n(attr.name);
+        if (n == "path") {
+            PathSet context;
+            path = state.coerceToPath(*attr.pos, *attr.value, context);
+            if (!context.empty())
+                throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos);
+        } else if (attr.name == state.sName)
+            name = state.forceStringNoCtx(*attr.value, *attr.pos);
+        else if (n == "filter") {
+            state.forceValue(*attr.value);
+            filterFun = attr.value;
+        } else if (n == "recursive")
+            recursive = state.forceBool(*attr.value, *attr.pos);
+        else if (n == "sha256")
+            expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
+        else
+            throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos);
+    }
+    if (path.empty())
+        throw EvalError(format("'path' required, at %1%") % pos);
+    if (name.empty())
+        name = baseNameOf(path);
+
+    addPath(state, pos, name, path, filterFun, recursive, expectedHash, v);
+}
+
+
 /*************************************************************
  * Sets
  *************************************************************/
@@ -2085,6 +2141,7 @@ void EvalState::createBaseEnv()
     addPrimOp("__fromJSON", 1, prim_fromJSON);
     addPrimOp("__toFile", 2, prim_toFile);
     addPrimOp("__filterSource", 2, prim_filterSource);
+    addPrimOp("__path", 1, prim_path);
 
     // Sets
     addPrimOp("__attrNames", 1, prim_attrNames);

src/libstore/build.cc

@@ -3428,7 +3428,7 @@ void DerivationGoal::flushLine()
     else {
         if (settings.verboseBuild &&
             (settings.printRepeatedBuilds || curRound == 1))
-            printError(filterANSIEscapes(currentLogLine, true));
+            printError(currentLogLine);
         else {
             logTail.push_back(currentLogLine);
             if (logTail.size() > settings.logLines) logTail.pop_front();

src/libstore/store-api.cc

@@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
 }
 
 
-std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath,
+std::pair<Path, Hash> Store::computeStorePathForPath(const string & name,
-    bool recursive, HashType hashAlgo, PathFilter & filter) const
+    const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
 {
     Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
-    string name = baseNameOf(srcPath);
     Path dstPath = makeFixedOutputPath(recursive, h, name);
     return std::pair<Path, Hash>(dstPath, h);
 }

src/libstore/store-api.hh

@@ -307,9 +307,9 @@ public:
     /* This is the preparatory part of addToStore(); it computes the
        store path to which srcPath is to be copied.  Returns the store
        path and the cryptographic hash of the contents of srcPath. */
-    std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
+    std::pair<Path, Hash> computeStorePathForPath(const string & name,
-        bool recursive = true, HashType hashAlgo = htSHA256,
+        const Path & srcPath, bool recursive = true,
-        PathFilter & filter = defaultPathFilter) const;
+        HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
 
     /* Preparatory part of addTextToStore().
 

src/libutil/logging.cc

@@ -44,7 +44,7 @@ public:
             prefix = std::string("<") + c + ">";
         }
 
-        writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n");
+        writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n");
     }
 
     void startActivity(ActivityId act, Verbosity lvl, ActivityType type,

src/libutil/util.cc

@@ -1178,36 +1178,51 @@ void ignoreException()
 }
 
 
-string filterANSIEscapes(const string & s, bool nixOnly)
+std::string filterANSIEscapes(const std::string & s, unsigned int width)
 {
-    string t, r;
+    std::string t, e;
-    enum { stTop, stEscape, stCSI } state = stTop;
+    size_t w = 0;
-    for (auto c : s) {
+    auto i = s.begin();
-        if (state == stTop) {
+
-            if (c == '\e') {
+    while (w < (size_t) width && i != s.end()) {
-                state = stEscape;
+
-                r = c;
+        if (*i == '\e') {
-            } else
+            std::string e;
-                t += c;
+            e += *i++;
-        } else if (state == stEscape) {
+            char last = 0;
-            r += c;
+
-            if (c == '[')
+            if (i != s.end() && *i == '[') {
-                state = stCSI;
+                e += *i++;
-            else {
+                // eat parameter bytes
-                t += r;
+                while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++;
-                state = stTop;
+                // eat intermediate bytes
-            }
+                while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++;
+                // eat final byte
+                if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++;
             } else {
-            r += c;
+                if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
-            if (c >= 0x40 && c <= 0x7e) {
+            }
-                if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b'))
+
-                    t += r;
+            if (last == 'm')
-                state = stTop;
+                t += e;
-                r.clear();
+        }
+
+        else if (*i == '\t') {
+            i++; t += ' '; w++;
+            while (w < (size_t) width && w % 8) {
+                t += ' '; w++;
             }
         }
+
+        else if (*i == '\r')
+            // do nothing for now
+            ;
+
+        else {
+            t += *i++; w++;
         }
-    t += r;
+    }
+
     return t;
 }
 

src/libutil/util.hh

@@ -388,10 +388,12 @@ void ignoreException();
 #define ANSI_BLUE "\e[34;1m"
 
 
-/* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is
+/* Truncate a string to 'width' printable characters. Certain ANSI
-   set, only filter escape codes generated by Nixpkgs' stdenv (used to
+   escape sequences (such as colour setting) are copied but not
-   denote nesting etc.). */
+   included in the character count. Other ANSI escape sequences are
-string filterANSIEscapes(const string & s, bool nixOnly = false);
+   filtered. Also, tabs are expanded to spaces. */
+std::string filterANSIEscapes(const std::string & s,
+    unsigned int width = std::numeric_limits<unsigned int>::max());
 
 
 /* Base64 encoding/decoding. */

src/nix/progress-bar.cc

@@ -23,44 +23,6 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n)
     return fields[n].i;
 }
 
-/* Truncate a string to 'width' printable characters. ANSI escape
-   sequences are copied but not included in the character count. Also,
-   tabs are expanded to spaces. */
-static std::string ansiTruncate(const std::string & s, int width)
-{
-    if (width <= 0) return s;
-
-    std::string t;
-    size_t w = 0;
-    auto i = s.begin();
-
-    while (w < (size_t) width && i != s.end()) {
-        if (*i == '\e') {
-            t += *i++;
-            if (i != s.end() && *i == '[') {
-                t += *i++;
-                while (i != s.end() && (*i < 0x40 || *i > 0x7e)) {
-                    t += *i++;
-                }
-                if (i != s.end()) t += *i++;
-            }
-        }
-
-        else if (*i == '\t') {
-            t += ' '; w++;
-            while (w < (size_t) width && w & 8) {
-                t += ' '; w++;
-            }
-        }
-
-        else {
-            t += *i++; w++;
-        }
-    }
-
-    return t;
-}
-
 class ProgressBar : public Logger
 {
 private:
@@ -343,7 +305,10 @@ public:
             }
         }
 
-        writeToStderr("\r" + ansiTruncate(line, getWindowSize().second) + "\e[K");
+        auto width = getWindowSize().second;
+        if (width <= 0) std::numeric_limits<decltype(width)>::max();
+
+        writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K");
     }
 
     std::string getStatus(State & state)

tests/lang/data

@@ -0,0 +1 @@
+foo

tests/lang/eval-okay-path.nix

@@ -0,0 +1,7 @@
+builtins.path
+  { path = ./.;
+    filter = path: _: baseNameOf path == "data";
+    recursive = true;
+    sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw";
+    name = "output";
+  }

tests/misc.sh

@@ -16,4 +16,4 @@ nix-env --foo 2>&1 | grep "no operation"
 nix-env -q --foo 2>&1 | grep "unknown flag"
 
 # Eval Errors.
-nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at (string):1:15$"
+nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$"