mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of https://github.com/NixOS/nix into parallel-xz

Browse source
This commit is contained in:
AmineChikhaoui 2018-02-07 17:53:50 +01:00
commit 163e39547a
No known key found for this signature in database
GPG key ID: C0C8C84C15BCCD1C
14 changed files with 221 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
doc/manual/expressions/builtins.xml
View file

@ -308,8 +308,9 @@ stdenv.mkDerivation { … }
</varlistentry>
<varlistentry><term><function>builtins.filterSource</function>
<replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
<varlistentry xml:id='builtin-filterSource'>
<term><function>builtins.filterSource</function>
<replaceable>e1</replaceable> <replaceable>e2</replaceable></term>
<listitem>
@ -768,6 +769,75 @@ Evaluates to <literal>[ "foo" ]</literal>.
</varlistentry>
<varlistentry>
<term>
<function>builtins.path</function>
<replaceable>args</replaceable>
</term>
<listitem>
<para>
An enrichment of the built-in path type, based on the attributes
present in <replaceable>args</replaceable>. All are optional
except <varname>path</varname>:
</para>
<variablelist>
<varlistentry>
<term>path</term>
<listitem>
<para>The underlying path.</para>
</listitem>
</varlistentry>
<varlistentry>
<term>name</term>
<listitem>
<para>
The name of the path when added to the store. This can
used to reference paths that have nix-illegal characters
in their names, like <literal>@</literal>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>filter</term>
<listitem>
<para>
A function of the type expected by
<link linkend="builtin-filterSource">builtins.filterSource</link>,
with the same semantics.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>recursive</term>
<listitem>
<para>
When <literal>false</literal>, when
<varname>path</varname> is added to the store it is with a
flat hash, rather than a hash of the NAR serialization of
the file. Thus, <varname>path</varname> must refer to a
regular file, not a directory. This allows similar
behavior to <literal>fetchurl</literal>. Defaults to
<literal>true</literal>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>sha256</term>
<listitem>
<para>
When provided, this is the expected hash of the file at
the path. Evaluation will fail if the hash is incorrect,
and providing a hash allows
<literal>builtins.path</literal> to be used even when the
<literal>pure-eval</literal> nix config option is on.
</para>
</listitem>
</varlistentry>
</variablelist>
</listitem>
</varlistentry>
<varlistentry><term><function>builtins.pathExists</function>
<replaceable>path</replaceable></term>

12
release.nix
View file

@ -224,11 +224,13 @@ let
nix = build.x86_64-linux; system = "x86_64-linux";
});
tests.setuid = pkgs.lib.genAttrs (pkgs.lib.filter (pkgs.lib.hasSuffix "-linux") systems) (system:
import ./tests/setuid.nix rec {
inherit nixpkgs;
nix = build.${system}; inherit system;
});
tests.setuid = pkgs.lib.genAttrs
["i686-linux" "x86_64-linux"]
(system:
import ./tests/setuid.nix rec {
inherit nixpkgs;
nix = build.${system}; inherit system;
});
tests.binaryTarball =
with import nixpkgs { system = "x86_64-linux"; };

2
src/libexpr/eval.cc
View file

@ -1578,7 +1578,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
dstPath = srcToStore[path];
else {
dstPath = settings.readOnlyMode
? store->computeStorePathForPath(checkSourcePath(path)).first
? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first
: store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair);
srcToStore[path] = dstPath;
printMsg(lvlChatty, format("copied source '%1%' -> '%2%'")

93
src/libexpr/primops.cc
View file

@ -1023,20 +1023,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
}
static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
Value * filterFun, bool recursive, const Hash & expectedHash, Value & v)
{
PathSet context;
Path path = state.coerceToPath(pos, *args[1], context);
if (!context.empty())
throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
state.forceValue(*args[0]);
if (args[0]->type != tLambda)
throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
path = state.checkSourcePath(path);
PathFilter filter = [&](const Path & path) {
const auto path = settings.pureEval && expectedHash ?
path_ :
state.checkSourcePath(path_);
PathFilter filter = filterFun ? ([&](const Path & path) {
auto st = lstat(path);
/* Call the filter function. The first argument is the path,
@ -1045,7 +1038,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
mkString(arg1, path);
Value fun2;
state.callFunction(*args[0], arg1, fun2, noPos);
state.callFunction(*filterFun, arg1, fun2, noPos);
Value arg2;
mkString(arg2,
@ -1058,16 +1051,79 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
state.callFunction(fun2, arg2, res, noPos);
return state.forceBool(res, pos);
};
}) : defaultPathFilter;
Path dstPath = settings.readOnlyMode
? state.store->computeStorePathForPath(path, true, htSHA256, filter).first
: state.store->addToStore(baseNameOf(path), path, true, htSHA256, filter, state.repair);
Path expectedStorePath;
if (expectedHash) {
expectedStorePath =
state.store->makeFixedOutputPath(recursive, expectedHash, name);
}
Path dstPath;
if (!expectedHash || !state.store->isValidPath(expectedStorePath)) {
dstPath = settings.readOnlyMode
? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first
: state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair);
if (expectedHash && expectedStorePath != dstPath) {
throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path);
}
} else
dstPath = expectedStorePath;
mkString(v, dstPath, {dstPath});
}
static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
PathSet context;
Path path = state.coerceToPath(pos, *args[1], context);
if (!context.empty())
throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos);
state.forceValue(*args[0]);
if (args[0]->type != tLambda)
throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos);
addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v);
}
static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceAttrs(*args[0], pos);
Path path;
string name;
Value * filterFun = nullptr;
auto recursive = true;
Hash expectedHash;
for (auto & attr : *args[0]->attrs) {
const string & n(attr.name);
if (n == "path") {
PathSet context;
path = state.coerceToPath(*attr.pos, *attr.value, context);
if (!context.empty())
throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos);
} else if (attr.name == state.sName)
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "filter") {
state.forceValue(*attr.value);
filterFun = attr.value;
} else if (n == "recursive")
recursive = state.forceBool(*attr.value, *attr.pos);
else if (n == "sha256")
expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
else
throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos);
}
if (path.empty())
throw EvalError(format("'path' required, at %1%") % pos);
if (name.empty())
name = baseNameOf(path);
addPath(state, pos, name, path, filterFun, recursive, expectedHash, v);
}
/*************************************************************
* Sets
*************************************************************/
@ -2085,6 +2141,7 @@ void EvalState::createBaseEnv()
addPrimOp("__fromJSON", 1, prim_fromJSON);
addPrimOp("__toFile", 2, prim_toFile);
addPrimOp("__filterSource", 2, prim_filterSource);
addPrimOp("__path", 1, prim_path);
// Sets
addPrimOp("__attrNames", 1, prim_attrNames);

2
src/libstore/build.cc
View file

@ -3428,7 +3428,7 @@ void DerivationGoal::flushLine()
else {
if (settings.verboseBuild &&
(settings.printRepeatedBuilds || curRound == 1))
printError(filterANSIEscapes(currentLogLine, true));
printError(currentLogLine);
else {
logTail.push_back(currentLogLine);
if (logTail.size() > settings.logLines) logTail.pop_front();

5
src/libstore/store-api.cc
View file

@ -222,11 +222,10 @@ Path Store::makeTextPath(const string & name, const Hash & hash,
}
std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter) const
std::pair<Path, Hash> Store::computeStorePathForPath(const string & name,
const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const
{
Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath);
string name = baseNameOf(srcPath);
Path dstPath = makeFixedOutputPath(recursive, h, name);
return std::pair<Path, Hash>(dstPath, h);
}

6
src/libstore/store-api.hh
View file

@ -307,9 +307,9 @@ public:
/* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store
path and the cryptographic hash of the contents of srcPath. */
std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter) const;
std::pair<Path, Hash> computeStorePathForPath(const string & name,
const Path & srcPath, bool recursive = true,
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
/* Preparatory part of addTextToStore().

2
src/libutil/logging.cc
View file

@ -44,7 +44,7 @@ public:
prefix = std::string("<") + c + ">";
}
writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n");
writeToStderr(prefix + filterANSIEscapes(fs.s) + "\n");
}
void startActivity(ActivityId act, Verbosity lvl, ActivityType type,

65
src/libutil/util.cc
View file

@ -1178,36 +1178,51 @@ void ignoreException()
}
string filterANSIEscapes(const string & s, bool nixOnly)
std::string filterANSIEscapes(const std::string & s, unsigned int width)
{
string t, r;
enum { stTop, stEscape, stCSI } state = stTop;
for (auto c : s) {
if (state == stTop) {
if (c == '\e') {
state = stEscape;
r = c;
} else
t += c;
} else if (state == stEscape) {
r += c;
if (c == '[')
state = stCSI;
else {
t += r;
state = stTop;
std::string t, e;
size_t w = 0;
auto i = s.begin();
while (w < (size_t) width && i != s.end()) {
if (*i == '\e') {
std::string e;
e += *i++;
char last = 0;
if (i != s.end() && *i == '[') {
e += *i++;
// eat parameter bytes
while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++;
// eat intermediate bytes
while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++;
// eat final byte
if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++;
} else {
if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++;
}
} else {
r += c;
if (c >= 0x40 && c <= 0x7e) {
if (nixOnly && (c != 'p' && c != 'q' && c != 's' && c != 'a' && c != 'b'))
t += r;
state = stTop;
r.clear();
if (last == 'm')
t += e;
}
else if (*i == '\t') {
i++; t += ' '; w++;
while (w < (size_t) width && w % 8) {
t += ' '; w++;
}
}
else if (*i == '\r')
// do nothing for now
;
else {
t += *i++; w++;
}
}
t += r;
return t;
}

10
src/libutil/util.hh
View file

@ -388,10 +388,12 @@ void ignoreException();
#define ANSI_BLUE "\e[34;1m"
/* Filter out ANSI escape codes from the given string. If ‘nixOnly’ is
set, only filter escape codes generated by Nixpkgs' stdenv (used to
denote nesting etc.). */
string filterANSIEscapes(const string & s, bool nixOnly = false);
/* Truncate a string to 'width' printable characters. Certain ANSI
escape sequences (such as colour setting) are copied but not
included in the character count. Other ANSI escape sequences are
filtered. Also, tabs are expanded to spaces. */
std::string filterANSIEscapes(const std::string & s,
unsigned int width = std::numeric_limits<unsigned int>::max());
/* Base64 encoding/decoding. */

43
src/nix/progress-bar.cc
View file

@ -23,44 +23,6 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n)
return fields[n].i;
}
/* Truncate a string to 'width' printable characters. ANSI escape
sequences are copied but not included in the character count. Also,
tabs are expanded to spaces. */
static std::string ansiTruncate(const std::string & s, int width)
{
if (width <= 0) return s;
std::string t;
size_t w = 0;
auto i = s.begin();
while (w < (size_t) width && i != s.end()) {
if (*i == '\e') {
t += *i++;
if (i != s.end() && *i == '[') {
t += *i++;
while (i != s.end() && (*i < 0x40 || *i > 0x7e)) {
t += *i++;
}
if (i != s.end()) t += *i++;
}
}
else if (*i == '\t') {
t += ' '; w++;
while (w < (size_t) width && w & 8) {
t += ' '; w++;
}
}
else {
t += *i++; w++;
}
}
return t;
}
class ProgressBar : public Logger
{
private:
@ -343,7 +305,10 @@ public:
}
}
writeToStderr("\r" + ansiTruncate(line, getWindowSize().second) + "\e[K");
auto width = getWindowSize().second;
if (width <= 0) std::numeric_limits<decltype(width)>::max();
writeToStderr("\r" + filterANSIEscapes(line, width) + "\e[K");
}
std::string getStatus(State & state)

1
tests/lang/data Normal file
View file

@ -0,0 +1 @@
foo

7
tests/lang/eval-okay-path.nix Normal file
View file

@ -0,0 +1,7 @@
builtins.path
{ path = ./.;
filter = path: _: baseNameOf path == "data";
recursive = true;
sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw";
name = "output";
}

2
tests/misc.sh
View file

@ -16,4 +16,4 @@ nix-env --foo 2>&1 | grep "no operation"
nix-env -q --foo 2>&1 | grep "unknown flag"
# Eval Errors.
nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at (string):1:15$"
nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$"