* Realisation of File(...) expressions.

This commit is contained in:
Eelco Dolstra 2003-06-27 13:55:12 +00:00
parent bb03c45ca0
commit 3da9687854
11 changed files with 137 additions and 368 deletions

View file

@ -30,6 +30,7 @@ bool pathExists(string path)
} }
#if 0
/* Compute a derived value by running a program. */ /* Compute a derived value by running a program. */
static Hash computeDerived(Hash sourceHash, string targetName, static Hash computeDerived(Hash sourceHash, string targetName,
string platform, Hash prog, Environment env) string platform, Hash prog, Environment env)
@ -175,6 +176,7 @@ static Hash computeDerived(Hash sourceHash, string targetName,
return targetHash; return targetHash;
} }
#endif
/* Throw an exception if the given platform string is not supported by /* Throw an exception if the given platform string is not supported by
@ -182,54 +184,32 @@ static Hash computeDerived(Hash sourceHash, string targetName,
static void checkPlatform(string platform) static void checkPlatform(string platform)
{ {
if (platform != thisSystem) if (platform != thisSystem)
throw Error("a `" + platform + throw Error(format("a `%1%' is required, but I am a `%2%'")
"' is required, but I am a `" + thisSystem + "'"); % platform % thisSystem);
} }
string printExpr(Expr e) string printTerm(ATerm t)
{ {
char * s = ATwriteToString(e); char * s = ATwriteToString(t);
return s; return s;
} }
/* Throw an exception with an error message containing the given /* Throw an exception with an error message containing the given
aterm. */ aterm. */
static Error badTerm(const string & msg, Expr e) static Error badTerm(const format & f, ATerm t)
{ {
return Error(msg + ", in `" + printExpr(e) + "'"); return Error(format("%1%, in `%2%'") % f.str() % printTerm(t));
} }
Hash hashExpr(Expr e) Hash hashTerm(ATerm t)
{ {
return hashString(printExpr(e)); return hashString(printTerm(t));
} }
/* Evaluate an expression; the result must be a string. */
static string evalString(Expr e)
{
e = whNormalise(e);
char * s;
if (ATmatch(e, "Str(<str>)", &s)) return s;
else throw badTerm("string value expected", e);
}
#if 0
/* Evaluate an expression; the result must be a value reference. */
static Hash evalHash(Expr e)
{
e = evalValue(e);
char * s;
if (ATmatch(e, "Hash(<str>)", &s)) return parseHash(s);
else throw badTerm("value reference expected", e);
}
#endif
#if 0 #if 0
/* Evaluate a list of arguments into normal form. */ /* Evaluate a list of arguments into normal form. */
void evalArgs(ATermList args, ATermList & argsNF, Environment & env) void evalArgs(ATermList args, ATermList & argsNF, Environment & env)
@ -262,225 +242,63 @@ void evalArgs(ATermList args, ATermList & argsNF, Environment & env)
#endif #endif
Expr substExpr(string x, Expr rep, Expr e)
{
char * s;
Expr e2;
if (ATmatch(e, "Var(<str>)", &s))
if (x == s)
return rep;
else
return e;
if (ATmatch(e, "Lam(<str>, <term>)", &s, &e2))
if (x == s)
return e;
/* !!! unfair substitutions */
/* Generically substitute in subterms. */
if (ATgetType(e) == AT_APPL) {
AFun fun = ATgetAFun(e);
int arity = ATgetArity(fun);
ATermList args = ATempty;
for (int i = arity - 1; i >= 0; i--)
args = ATinsert(args, substExpr(x, rep, ATgetArgument(e, i)));
return (ATerm) ATmakeApplList(fun, args);
}
if (ATgetType(e) == AT_LIST) {
ATermList in = (ATermList) e;
ATermList out = ATempty;
while (!ATisEmpty(in)) {
out = ATinsert(out, substExpr(x, rep, ATgetFirst(in)));
in = ATgetNext(in);
}
return (ATerm) ATreverse(out);
}
throw badTerm("do not know how to substitute", e);
}
#if 0
Expr evalValue(Expr e)
{
char * s;
Expr eBuildPlatform, eProg, e2, e3, e4;
ATermList args;
/* Value references. */
if (ATmatch(e, "Hash(<str>)", &s)) {
parseHash(s); /* i.e., throw exception if not valid */
return e;
}
/* External expression. */
if (ATmatch(e, "Deref(<term>)", &e2)) {
string fn = queryValuePath(evalHash(e2));
ATerm e3 = ATreadFromNamedFile(fn.c_str());
if (!e3) throw Error("reading aterm from " + fn);
return evalValue(e3);
}
/* Execution primitive. */
if (ATmatch(e, "Exec(<term>, <term>, [<list>])",
&eBuildPlatform, &eProg, &args))
{
string buildPlatform = evalString(eBuildPlatform);
checkPlatform(buildPlatform);
Hash prog = evalHash(eProg);
Environment env;
ATermList argsNF;
evalArgs(args, argsNF, env);
Hash sourceHash = hashExpr(
ATmake("Exec(Str(<str>), Hash(<str>), <term>)",
buildPlatform.c_str(), ((string) prog).c_str(), argsNF));
/* Do we know a normal form for sourceHash? */
Hash targetHash;
string targetHashS;
if (queryDB(nixDB, dbNFs, sourceHash, targetHashS)) {
/* Yes. */
targetHash = parseHash(targetHashS);
debug("already built: " + (string) sourceHash
+ " -> " + (string) targetHash);
} else {
/* No, so we compute one. */
targetHash = computeDerived(sourceHash,
(string) sourceHash + "-nf", buildPlatform, prog, env);
}
return ATmake("Hash(<str>)", ((string) targetHash).c_str());
}
/* Barf. */
throw badTerm("invalid expression", e);
}
#endif
Expr whNormalise(Expr e)
{
char * s;
Expr e2, e3, e4, e5;
/* Normal forms. */
if (ATmatch(e, "Str(<str>)", &s) ||
ATmatch(e, "Bool(True)") ||
ATmatch(e, "Bool(False)") ||
ATmatch(e, "Lam(<str>, <term>)", &s, &e2) ||
ATmatch(e, "File(<str>, <term>, <term>)", &s, &e2, &e3) ||
ATmatch(e, "Derive(<term>, <term>, <term>, <term>)", &e2, &e3, &e4, &e5))
return e;
/* Application. */
if (ATmatch(e, "App(<term>, <term>)", &e2, &e3)) {
e2 = whNormalise(e2);
if (!ATmatch(e2, "Lam(<str>, <term>)", &s, &e4))
throw badTerm("expecting lambda", e2);
return whNormalise(substExpr(s, e3, e4));
}
throw badTerm("invalid expression", e);
}
Expr dNormalise(Expr e)
{
e = whNormalise(e);
/* !!! todo */
return e;
}
Expr fNormalise(Expr e)
{
e = dNormalise(e);
char * s;
Expr e2, e3;
if (ATmatch(e, "File(<str>, <term>, [<list>])", &s, &e2, &e3)) {
ATermList refs = (ATermList) e3, refs2 = ATempty;
while (!ATisEmpty(refs)) {
ATerm ref = ATgetFirst(refs);
refs2 = ATinsert(refs2, fNormalise(ref));
refs = ATgetNext(refs);
}
refs2 = ATreverse(refs2);
return ATmake("File(<str>, <term>, <term>)", s, e2, refs2);
}
else return e;
}
void writeContent(string path, Content content)
{
char * s;
if (ATmatch(content, "Regular(<str>)", &s)) {
int fd; /* !!! close on exception */
fd = open(path.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666);
if (fd == -1)
throw SysError("creating file " + path);
int len = strlen(s);
if (write(fd, s, len) != len)
throw SysError("writing file " + path);
close(fd);
}
else throw badTerm("ill-formed content", content);
}
struct RStatus struct RStatus
{ {
/* !!! the comparator of this hash should match the semantics of /* !!! the comparator of this hash should match the semantics of
the file system */ the file system */
map<string, Hash> paths; // map<string, Hash> paths;
}; };
static void realise2(RStatus & status, Expr e) static void realise(RStatus & status, FState fs)
{ {
char * s; char * s;
Content content; Content content;
ATermList refs; ATermList refs;
if (!ATmatch(e, "File(<str>, <term>, [<list>])", &s, &content, &refs)) if (ATmatch(fs, "File(<str>, <term>, [<list>])", &s, &content, &refs)) {
throw badTerm("not f-normalised", e); string path(s);
string path(s); if (path[0] != '/') throw Error("absolute path expected: " + path);
while (!ATisEmpty(refs)) { /* Realise referenced paths. */
realise2(status, ATgetFirst(refs)); while (!ATisEmpty(refs)) {
refs = ATgetNext(refs); realise(status, ATgetFirst(refs));
refs = ATgetNext(refs);
}
if (!ATmatch(content, "Hash(<str>)", &s))
throw badTerm("hash expected", content);
Hash hash = parseHash(s);
/* Perhaps the path already exists and has the right hash? */
if (pathExists(path)) {
if (hash == hashPath(path)) {
debug(format("path %1% already has hash %2%")
% path % (string) hash);
return;
}
throw Error(format("path %1% exists, but does not have hash %2%")
% path % (string) hash);
}
/* Do we know a path with that hash? If so, copy it. */
string path2 = queryFromStore(hash);
copyFile(path2, path);
} }
writeContent(path, content); else if (ATmatch(fs, "Derive()")) {
}
else throw badTerm("bad file system state expression", fs);
} }
void realise(Expr e) void realiseFState(FState fs)
{ {
RStatus status; RStatus status;
realise2(status, e); realise(status, fs);
} }

View file

@ -10,14 +10,12 @@ extern "C" {
using namespace std; using namespace std;
/* \section{Abstract syntax of Nix expressions} /* \section{Abstract syntax of Nix file system state expressions}
An expression describes a (partial) state of the file system in a A Nix file system state expression, or FState, describes a
referentially transparent way. The operational effect of (partial) state of the file system.
evaluating an expression is that the state described by the
expression is realised.
File : Path * Content * [Expr] -> Expr File : Path * Content * [FState] -> FState
File(path, content, refs) specifies a file object (its full path File(path, content, refs) specifies a file object (its full path
and contents), along with all file objects referenced by it (that and contents), along with all file objects referenced by it (that
@ -25,7 +23,7 @@ using namespace std;
self-referential. This prevents us from having to deal with self-referential. This prevents us from having to deal with
cycles. cycles.
Derive : String * Path * [Expr] * [Expr] * [Expr] -> Expr Derive : String * Path * [FState] * [Path] * [(String, String)] -> [FState]
Derive(platform, builder, ins, outs, env) specifies the creation of Derive(platform, builder, ins, outs, env) specifies the creation of
new file objects (in paths declared by `outs') by the execution of new file objects (in paths declared by `outs') by the execution of
@ -33,14 +31,6 @@ using namespace std;
place in a file system state given by `ins'. `env' specifies a place in a file system state given by `ins'. `env' specifies a
mapping of strings to strings. mapping of strings to strings.
Str : String -> Expr
A string constant.
Tup : Expr * Expr -> Expr
Tuples of expressions.
[ !!! NOT IMPLEMENTED [ !!! NOT IMPLEMENTED
Regular : String -> Content Regular : String -> Content
Directory : [(String, Content)] -> Content Directory : [(String, Content)] -> Content
@ -49,7 +39,11 @@ using namespace std;
CHash : Hash -> Content CHash : Hash -> Content
File content, given either in situ, or through an external reference File content, given either in situ, or through an external reference
to the file system or url-space decorated with a hash to preserve purity. to the file system or url-space decorated with a hash to preserve
purity.
A FState expression is in {\em $f$-normal form} if all Derive nodes
have been reduced to File nodes.
DISCUSSION: the idea is that a Regular/Directory is interchangeable DISCUSSION: the idea is that a Regular/Directory is interchangeable
with its CHash. This would appear to break referential with its CHash. This would appear to break referential
@ -60,63 +54,20 @@ using namespace std;
CHash, we should also export the file object referenced by that CHash, we should also export the file object referenced by that
CHash. CHash.
\section{Reduction rules}
...
\section{Normals forms}
An expression is in {\em weak head normal form} if it is a lambda,
a string or boolean value, or a File or Derive value.
An expression is in {\em $d$-normal form} if it matches the
signature FExpr:
File : String * Content * [DExpr] -> DExpr
Derive : String * Path * [Tup] * [Tup2] -> DExpr
Tup : Str * DExpr -> Tup
Tup : Str * Str -> Tup
Tup : Str * Str -> Tup2
Str : String -> Str
These are Nix expressions in which the file system result of Derive
expressions has not yet been computed. This is useful for, e.g.,
querying dependencies.
An expression is in {\em $f$-normal form} if it matches the
signature FExpr:
File : String * Content * [FExpr] -> FExpr
These are completely evaluated Nix expressions.
*/ */
typedef ATerm Expr; typedef ATerm FState;
typedef ATerm Content; typedef ATerm Content;
/* Expression normalisation. */
Expr whNormalise(Expr e);
Expr dNormalise(Expr e);
Expr fNormalise(Expr e);
/* Realise a $f$-normalised expression in the file system. */ /* Realise a $f$-normalised expression in the file system. */
void realise(Expr e); void realiseFState(FState fs);
/* Return a canonical textual representation of an expression. */ /* Return a canonical textual representation of an expression. */
string printExpr(Expr e); string printTerm(ATerm t);
/* Perform variable substitution. */ /* Hash an aterm. */
Expr substExpr(string x, Expr rep, Expr e); Hash hashTerm(ATerm t);
/* Hash an expression. */
Hash hashExpr(Expr e);
#endif /* !__EVAL_H */ #endif /* !__EVAL_H */

View file

@ -6,7 +6,7 @@ string dbRefs = "refs";
string dbNFs = "nfs"; string dbNFs = "nfs";
string dbNetSources = "netsources"; string dbNetSources = "netsources";
string nixValues = "/UNINIT"; string nixStore = "/UNINIT";
string nixLogDir = "/UNINIT"; string nixLogDir = "/UNINIT";
string nixDB = "/UNINIT"; string nixDB = "/UNINIT";

View file

@ -8,15 +8,10 @@ using namespace std;
/* Database names. */ /* Database names. */
/* dbRefs :: Hash -> FileName /* dbRefs :: Hash -> Path
Maintains a mapping from hashes to filenames within the NixValues Maintains a mapping from hashes to paths. This is what we use to
directory. This mapping is for performance only; it can be resolve CHash(hash) content descriptors. */
reconstructed unambiguously. The reason is that names in this
directory are not printed hashes but also might carry some
descriptive element (e.g., "aterm-2.0-ae749a..."). Without this
mapping, looking up a value would take O(n) time because we would
need to read the entire directory. */
extern string dbRefs; extern string dbRefs;
/* dbNFs :: Hash -> Hash /* dbNFs :: Hash -> Hash
@ -45,11 +40,11 @@ extern string dbNetSources;
/* Path names. */ /* Path names. */
/* nixValues is the directory where all Nix values (both files and /* nixStore is the directory where we generally store atomic and
directories, and both normal and non-normal forms) live. */ derived files. */
extern string nixValues; extern string nixStore;
/* nixLogDir is the directory where we log evaluations. */ /* nixLogDir is the directory where we log various operations. */
extern string nixLogDir; extern string nixLogDir;
/* nixDB is the file name of the Berkeley DB database where we /* nixDB is the file name of the Berkeley DB database where we

View file

@ -14,7 +14,7 @@ Hash::Hash()
} }
bool Hash::operator == (Hash & h2) bool Hash::operator == (Hash h2)
{ {
for (unsigned int i = 0; i < hashSize; i++) for (unsigned int i = 0; i < hashSize; i++)
if (hash[i] != h2.hash[i]) return false; if (hash[i] != h2.hash[i]) return false;
@ -22,7 +22,7 @@ bool Hash::operator == (Hash & h2)
} }
bool Hash::operator != (Hash & h2) bool Hash::operator != (Hash h2)
{ {
return !(*this == h2); return !(*this == h2);
} }

View file

@ -17,10 +17,10 @@ struct Hash
Hash(); Hash();
/* Check whether two hash are equal. */ /* Check whether two hash are equal. */
bool operator == (Hash & h2); bool operator == (Hash h2);
/* Check whether two hash are not equal. */ /* Check whether two hash are not equal. */
bool operator != (Hash & h2); bool operator != (Hash h2);
/* Convert a hash code into a hexadecimal representation. */ /* Convert a hash code into a hexadecimal representation. */
operator string() const; operator string() const;

View file

@ -11,20 +11,16 @@
#include "globals.hh" #include "globals.hh"
typedef Expr (* Normaliser) (Expr); void realise(FState fs)
void eval(Normaliser n, Expr e)
{ {
e = n(e); realiseFState(fs);
cout << (string) hashExpr(e) << ": " << printExpr(e) << endl;
} }
void evalFail(Normaliser n, Expr e) void realiseFail(FState fs)
{ {
try { try {
e = n(e); realiseFState(fs);
abort(); abort();
} catch (Error e) { } catch (Error e) {
cout << "error (expected): " << e.what() << endl; cout << "error (expected): " << e.what() << endl;
@ -96,7 +92,7 @@ void runTests()
string testDir = absPath("scratch"); string testDir = absPath("scratch");
cout << testDir << endl; cout << testDir << endl;
nixValues = testDir; nixStore = testDir;
nixLogDir = testDir; nixLogDir = testDir;
nixDB = testDir + "/db"; nixDB = testDir + "/db";
@ -104,6 +100,7 @@ void runTests()
/* Expression evaluation. */ /* Expression evaluation. */
#if 0
eval(whNormalise, eval(whNormalise,
ATmake("Str(\"Hello World\")")); ATmake("Str(\"Hello World\")"));
eval(whNormalise, eval(whNormalise,
@ -138,10 +135,27 @@ void runTests()
eval(fNormalise, e2); eval(fNormalise, e2);
realise(e2); realise(e2);
#endif
Hash builder1h;
string builder1fn;
addToStore("./test-builder-1.sh", builder1fn, builder1h);
FState fs1 = ATmake(
"File(<str>, Hash(<str>), [])",
builder1fn.c_str(),
((string) builder1h).c_str());
realiseFState(fs1);
realiseFState(fs1);
FState fs2 = ATmake(
"File(<str>, Hash(<str>), [])",
(builder1fn + "_bla").c_str(),
((string) builder1h).c_str());
realiseFState(fs2);
realiseFState(fs2);
#if 0 #if 0
Hash builder1 = addValue("./test-builder-1.sh");
Expr e1 = ATmake("Exec(Str(<str>), Hash(<str>), [])", Expr e1 = ATmake("Exec(Str(<str>), Hash(<str>), [])",
thisSystem.c_str(), ((string) builder1).c_str()); thisSystem.c_str(), ((string) builder1).c_str());

View file

@ -78,7 +78,7 @@ void deletePath(string path)
} }
void debug(string s) void debug(const format & f)
{ {
cerr << "debug: " << s << endl; cerr << format("debug: %1%\n") % f.str();
} }

View file

@ -7,7 +7,10 @@
#include <unistd.h> #include <unistd.h>
#include <boost/format.hpp>
using namespace std; using namespace std;
using namespace boost;
class Error : public exception class Error : public exception
@ -16,7 +19,7 @@ protected:
string err; string err;
public: public:
Error() { } Error() { }
Error(string _err) { err = _err; } Error(format f) { err = f.str(); }
~Error() throw () { } ~Error() throw () { }
const char * what() const throw () { return err.c_str(); } const char * what() const throw () { return err.c_str(); }
}; };
@ -59,7 +62,7 @@ string baseNameOf(string path);
void deletePath(string path); void deletePath(string path);
void debug(string s); void debug(const format & f);
#endif /* !__UTIL_H */ #endif /* !__UTIL_H */

View file

@ -34,7 +34,7 @@ struct CopySource : RestoreSource
}; };
static void copyFile(string src, string dst) void copyFile(string src, string dst)
{ {
/* Unfortunately C++ doesn't support coprocedures, so we have no /* Unfortunately C++ doesn't support coprocedures, so we have no
nice way to chain CopySink and CopySource together. Instead we nice way to chain CopySink and CopySource together. Instead we
@ -83,33 +83,25 @@ static void copyFile(string src, string dst)
} }
static string absValuePath(string s) void addToStore(string srcPath, string & dstPath, Hash & hash)
{ {
return nixValues + "/" + s; srcPath = absPath(srcPath);
}
hash = hashPath(srcPath);
Hash addValue(string path) string path;
{ if (queryDB(nixDB, dbRefs, hash, path)) {
path = absPath(path);
Hash hash = hashPath(path);
string name;
if (queryDB(nixDB, dbRefs, hash, name)) {
debug((string) hash + " already known"); debug((string) hash + " already known");
return hash; dstPath = path;
return;
} }
string baseName = baseNameOf(path); string baseName = baseNameOf(srcPath);
dstPath = nixStore + "/" + (string) hash + "-" + baseName;
string targetName = (string) hash + "-" + baseName; copyFile(srcPath, dstPath);
copyFile(path, absValuePath(targetName)); setDB(nixDB, dbRefs, hash, dstPath);
setDB(nixDB, dbRefs, hash, targetName);
return hash;
} }
@ -135,28 +127,28 @@ string fetchURL(string url)
#endif #endif
void deleteValue(Hash hash) void deleteFromStore(Hash hash)
{ {
string name; string fn;
if (queryDB(nixDB, dbRefs, hash, name)) { if (queryDB(nixDB, dbRefs, hash, fn)) {
string fn = absValuePath(name); string prefix = nixStore + "/";
if (string(fn, prefix.size()) != prefix)
throw Error("path " + fn + " is not in the store");
deletePath(fn); deletePath(fn);
delDB(nixDB, dbRefs, hash); delDB(nixDB, dbRefs, hash);
} }
} }
/* !!! bad name, "query" implies no side effect => getValuePath() */ string queryFromStore(Hash hash)
string queryValuePath(Hash hash)
{ {
bool checkedNet = false; bool checkedNet = false;
while (1) { while (1) {
string name, url; string fn, url;
if (queryDB(nixDB, dbRefs, hash, name)) { if (queryDB(nixDB, dbRefs, hash, fn)) {
string fn = absValuePath(name);
/* Verify that the file hasn't changed. !!! race !!! slow */ /* Verify that the file hasn't changed. !!! race !!! slow */
if (hashPath(fn) != hash) if (hashPath(fn) != hash)

View file

@ -8,21 +8,17 @@
using namespace std; using namespace std;
/* Copy a value to the nixValues directory and register it in dbRefs. void copyFile(string src, string dst);
/* Copy a file to the nixStore directory and register it in dbRefs.
Return the hash code of the value. */ Return the hash code of the value. */
Hash addValue(string pathName); void addToStore(string srcPath, string & dstPath, Hash & hash);
/* Delete a value from the nixStore directory. */
void deleteFromStore(Hash hash);
/* Delete a value from the nixValues directory. */ /* !!! */
void deleteValue(Hash hash); string queryFromStore(Hash hash);
/* Obtain the path of a value with the given hash. If a file with
that hash is known to exist in the local file system (as indicated
by the dbRefs database), we use that. Otherwise, we attempt to
fetch it from the network (using dbNetSources). We verify that the
file has the right hash. */
string queryValuePath(Hash hash);
#endif /* !__VALUES_H */ #endif /* !__VALUES_H */