tvl-depot/src/libstore/local-store.cc

1153 lines
34 KiB
C++
Raw Normal View History

#include "config.h"
#include "local-store.hh"
#include "util.hh"
#include "globals.hh"
#include "db.hh"
#include "archive.hh"
#include "pathlocks.hh"
#include "aterm.hh"
#include "derivations-ast.hh"
#include "worker-protocol.hh"
#include <iostream>
2003-12-22 17:40:46 +01:00
#include <algorithm>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <utime.h>
namespace nix {
/* Nix database. */
static Database nixDB;
/* Database tables. */
/* dbValidPaths :: Path -> ()
The existence of a key $p$ indicates that path $p$ is valid (that
is, produced by a succesful build). */
static TableId dbValidPaths = 0;
/* dbReferences :: Path -> [Path]
This table lists the outgoing file system references for each
output path that has been built by a Nix derivation. These are
found by scanning the path for the hash components of input
paths. */
static TableId dbReferences = 0;
/* dbReferrers :: Path -> Path
This table is just the reverse mapping of dbReferences. This table
can have duplicate keys, each corresponding value denoting a single
referrer. */
static TableId dbReferrers = 0;
/* dbDerivers :: Path -> [Path]
This table lists the derivation used to build a path. There can
only be multiple such paths for fixed-output derivations (i.e.,
derivations specifying an expected hash). */
static TableId dbDerivers = 0;
static void upgradeStore09();
static void upgradeStore11();
void checkStoreNotSymlink()
{
if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
Path path = nixStore;
struct stat st;
while (path != "/") {
if (lstat(path.c_str(), &st))
throw SysError(format("getting status of `%1%'") % path);
if (S_ISLNK(st.st_mode))
throw Error(format(
"the path `%1%' is a symlink; "
"this is not allowed for the Nix store and its parent directories")
% path);
path = dirOf(path);
}
}
LocalStore::LocalStore(bool reserveSpace)
{
substitutablePathsLoaded = false;
if (readOnlyMode) return;
checkStoreNotSymlink();
try {
Path reservedPath = nixDBPath + "/reserved";
string s = querySetting("gc-reserved-space", "");
int reservedSize;
if (!string2Int(s, reservedSize)) reservedSize = 1024 * 1024;
if (reserveSpace) {
struct stat st;
if (stat(reservedPath.c_str(), &st) == -1 ||
st.st_size != reservedSize)
writeFile(reservedPath, string(reservedSize, 'X'));
}
else
deletePath(reservedPath);
} catch (SysError & e) { /* don't care about errors */
}
try {
nixDB.open(nixDBPath);
} catch (DbNoPermission & e) {
printMsg(lvlTalkative, "cannot access Nix database; continuing anyway");
readOnlyMode = true;
return;
}
dbValidPaths = nixDB.openTable("validpaths");
dbReferences = nixDB.openTable("references");
dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */
dbDerivers = nixDB.openTable("derivers");
int curSchema = 0;
Path schemaFN = nixDBPath + "/schema";
if (pathExists(schemaFN)) {
string s = readFile(schemaFN);
if (!string2Int(s, curSchema))
throw Error(format("`%1%' is corrupt") % schemaFN);
}
if (curSchema > nixSchemaVersion)
throw Error(format("current Nix store schema is version %1%, but I only support %2%")
% curSchema % nixSchemaVersion);
if (curSchema < nixSchemaVersion) {
if (curSchema == 0) /* new store */
curSchema = nixSchemaVersion;
if (curSchema <= 1)
throw Error("your Nix store is no longer supported");
if (curSchema <= 2) upgradeStore09();
if (curSchema <= 3) upgradeStore11();
writeFile(schemaFN, (format("%1%") % nixSchemaVersion).str());
}
}
LocalStore::~LocalStore()
{
/* If the database isn't open, this is a NOP. */
try {
nixDB.close();
} catch (...) {
ignoreException();
}
}
void createStoreTransaction(Transaction & txn)
{
Transaction txn2(nixDB);
txn2.moveTo(txn);
}
void copyPath(const Path & src, const Path & dst, PathFilter & filter)
{
debug(format("copying `%1%' to `%2%'") % src % dst);
/* Dump an archive of the path `src' into a string buffer, then
restore the archive to `dst'. This is not a very good method
for very large paths, but `copyPath' is mainly used for small
files. */
StringSink sink;
dumpPath(src, sink, filter);
StringSource source(sink.s);
restorePath(dst, source);
}
static void _canonicalisePathMetaData(const Path & path, bool recurse)
{
checkInterrupt();
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
/* Change ownership to the current uid. If its a symlink, use
lchown if available, otherwise don't bother. Wrong ownership
of a symlink doesn't matter, since the owning user can't change
the symlink and can't delete it because the directory is not
writable. The only exception is top-level paths in the Nix
store (since that directory is group-writable for the Nix build
users group); we check for this case below. */
if (st.st_uid != geteuid()) {
#if HAVE_LCHOWN
2007-02-06 21:03:53 +01:00
if (lchown(path.c_str(), geteuid(), (gid_t) -1) == -1)
#else
if (!S_ISLNK(st.st_mode) &&
2007-02-06 21:03:53 +01:00
chown(path.c_str(), geteuid(), (gid_t) -1) == -1)
#endif
throw SysError(format("changing owner of `%1%' to %2%")
% path % geteuid());
}
if (!S_ISLNK(st.st_mode)) {
/* Mask out all type related bits. */
mode_t mode = st.st_mode & ~S_IFMT;
if (mode != 0444 && mode != 0555) {
mode = (st.st_mode & S_IFMT)
| 0444
| (st.st_mode & S_IXUSR ? 0111 : 0);
if (chmod(path.c_str(), mode) == -1)
throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
}
if (st.st_mtime != 0) {
struct utimbuf utimbuf;
utimbuf.actime = st.st_atime;
utimbuf.modtime = 0;
if (utime(path.c_str(), &utimbuf) == -1)
throw SysError(format("changing modification time of `%1%'") % path);
}
}
if (recurse && S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path);
for (Strings::iterator i = names.begin(); i != names.end(); ++i)
_canonicalisePathMetaData(path + "/" + *i, true);
}
}
void canonicalisePathMetaData(const Path & path)
{
_canonicalisePathMetaData(path, true);
/* On platforms that don't have lchown(), the top-level path can't
be a symlink, since we can't change its ownership. */
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
if (st.st_uid != geteuid()) {
assert(S_ISLNK(st.st_mode));
throw Error(format("wrong ownership of top-level store path `%1%'") % path);
}
}
bool isValidPathTxn(const Transaction & txn, const Path & path)
{
string s;
return nixDB.queryString(txn, dbValidPaths, path, s);
}
bool LocalStore::isValidPath(const Path & path)
{
return isValidPathTxn(noTxn, path);
}
PathSet LocalStore::queryValidPaths()
{
Paths paths;
nixDB.enumTable(noTxn, dbValidPaths, paths);
return PathSet(paths.begin(), paths.end());
}
static string addPrefix(const string & prefix, const string & s)
{
2005-12-25 03:02:29 +01:00
return prefix + string(1, (char) 0) + s;
}
static string stripPrefix(const string & prefix, const string & s)
{
if (s.size() <= prefix.size() ||
2005-12-25 00:32:59 +01:00
string(s, 0, prefix.size()) != prefix ||
s[prefix.size()] != 0)
throw Error(format("string `%1%' is missing prefix `%2%'")
% s % prefix);
return string(s, prefix.size() + 1);
}
static PathSet getReferrers(const Transaction & txn, const Path & storePath)
{
PathSet referrers;
Strings keys;
2005-12-25 03:02:29 +01:00
nixDB.enumTable(txn, dbReferrers, keys, storePath + string(1, (char) 0));
for (Strings::iterator i = keys.begin(); i != keys.end(); ++i)
referrers.insert(stripPrefix(storePath, *i));
return referrers;
}
void setReferences(const Transaction & txn, const Path & storePath,
const PathSet & references)
{
/* For invalid paths, we can only clear the references. */
if (references.size() > 0 && !isValidPathTxn(txn, storePath))
throw Error(
format("cannot set references for invalid path `%1%'") % storePath);
Paths oldReferences;
2005-01-27 18:48:14 +01:00
nixDB.queryStrings(txn, dbReferences, storePath, oldReferences);
PathSet oldReferences2(oldReferences.begin(), oldReferences.end());
if (oldReferences2 == references) return;
nixDB.setStrings(txn, dbReferences, storePath,
Paths(references.begin(), references.end()));
/* Update the referrers mappings of all new referenced paths. */
for (PathSet::const_iterator i = references.begin();
i != references.end(); ++i)
if (oldReferences2.find(*i) == oldReferences2.end())
nixDB.setString(txn, dbReferrers, addPrefix(*i, storePath), "");
/* Remove referrer mappings from paths that are no longer
references. */
for (Paths::iterator i = oldReferences.begin();
i != oldReferences.end(); ++i)
if (references.find(*i) == references.end())
nixDB.delPair(txn, dbReferrers, addPrefix(*i, storePath));
}
void queryReferences(const Transaction & txn,
const Path & storePath, PathSet & references)
{
Paths references2;
if (!isValidPathTxn(txn, storePath))
throw Error(format("path `%1%' is not valid") % storePath);
nixDB.queryStrings(txn, dbReferences, storePath, references2);
references.insert(references2.begin(), references2.end());
}
void LocalStore::queryReferences(const Path & storePath,
PathSet & references)
{
nix::queryReferences(noTxn, storePath, references);
}
void queryReferrers(const Transaction & txn,
const Path & storePath, PathSet & referrers)
{
if (!isValidPathTxn(txn, storePath))
throw Error(format("path `%1%' is not valid") % storePath);
PathSet referrers2 = getReferrers(txn, storePath);
referrers.insert(referrers2.begin(), referrers2.end());
}
void LocalStore::queryReferrers(const Path & storePath,
PathSet & referrers)
{
nix::queryReferrers(noTxn, storePath, referrers);
}
void setDeriver(const Transaction & txn, const Path & storePath,
const Path & deriver)
{
assertStorePath(storePath);
if (deriver == "") return;
assertStorePath(deriver);
if (!isValidPathTxn(txn, storePath))
throw Error(format("path `%1%' is not valid") % storePath);
nixDB.setString(txn, dbDerivers, storePath, deriver);
}
static Path queryDeriver(const Transaction & txn, const Path & storePath)
{
if (!isValidPathTxn(txn, storePath))
throw Error(format("path `%1%' is not valid") % storePath);
Path deriver;
if (nixDB.queryString(txn, dbDerivers, storePath, deriver))
return deriver;
else
return "";
}
Path LocalStore::queryDeriver(const Path & path)
{
return nix::queryDeriver(noTxn, path);
}
PathSet LocalStore::querySubstitutablePaths()
{
if (!substitutablePathsLoaded) {
for (Paths::iterator i = substituters.begin(); i != substituters.end(); ++i) {
debug(format("running `%1%' to find out substitutable paths") % *i);
Strings args;
args.push_back("--query-paths");
Strings ss = tokenizeString(runProgram(*i, false, args), "\n");
for (Strings::iterator j = ss.begin(); j != ss.end(); ++j) {
if (!isStorePath(*j))
throw Error(format("`%1%' returned a bad substitutable path `%2%'")
% *i % *j);
substitutablePaths.insert(*j);
}
}
substitutablePathsLoaded = true;
}
return substitutablePaths;
}
bool LocalStore::hasSubstitutes(const Path & path)
{
if (!substitutablePathsLoaded)
querySubstitutablePaths();
return substitutablePaths.find(path) != substitutablePaths.end();
}
static void setHash(const Transaction & txn, const Path & storePath,
const Hash & hash)
{
assert(hash.type == htSHA256);
nixDB.setString(txn, dbValidPaths, storePath, "sha256:" + printHash(hash));
}
static Hash queryHash(const Transaction & txn, const Path & storePath)
{
string s;
nixDB.queryString(txn, dbValidPaths, storePath, s);
string::size_type colon = s.find(':');
if (colon == string::npos)
throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'")
% s % storePath);
HashType ht = parseHashType(string(s, 0, colon));
if (ht == htUnknown)
throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'")
% string(s, 0, colon) % storePath);
return parseHash(ht, string(s, colon + 1));
}
Hash LocalStore::queryPathHash(const Path & path)
{
if (!isValidPath(path))
throw Error(format("path `%1%' is not valid") % path);
return queryHash(noTxn, path);
}
void registerValidPath(const Transaction & txn,
const Path & path, const Hash & hash, const PathSet & references,
const Path & deriver)
{
ValidPathInfo info;
info.path = path;
info.hash = hash;
info.references = references;
info.deriver = deriver;
ValidPathInfos infos;
infos.push_back(info);
registerValidPaths(txn, infos);
}
void registerValidPaths(const Transaction & txn,
const ValidPathInfos & infos)
{
PathSet newPaths;
for (ValidPathInfos::const_iterator i = infos.begin();
i != infos.end(); ++i)
newPaths.insert(i->path);
for (ValidPathInfos::const_iterator i = infos.begin();
i != infos.end(); ++i)
{
assertStorePath(i->path);
debug(format("registering path `%1%'") % i->path);
setHash(txn, i->path, i->hash);
setReferences(txn, i->path, i->references);
/* Check that all referenced paths are also valid (or about to
become valid). */
for (PathSet::iterator j = i->references.begin();
j != i->references.end(); ++j)
if (!isValidPathTxn(txn, *j) && newPaths.find(*j) == newPaths.end())
throw Error(format("cannot register path `%1%' as valid, since its reference `%2%' is invalid")
% i->path % *j);
setDeriver(txn, i->path, i->deriver);
}
}
/* Invalidate a path. The caller is responsible for checking that
there are no referrers. */
static void invalidatePath(Transaction & txn, const Path & path)
{
debug(format("invalidating path `%1%'") % path);
/* Clear the `references' entry for this path, as well as the
inverse `referrers' entries, and the `derivers' entry. */
setReferences(txn, path, PathSet());
nixDB.delPair(txn, dbDerivers, path);
nixDB.delPair(txn, dbValidPaths, path);
}
Path LocalStore::addToStore(const Path & _srcPath, bool fixed,
bool recursive, string hashAlgo, PathFilter & filter)
{
Path srcPath(absPath(_srcPath));
debug(format("adding `%1%' to the store") % srcPath);
std::pair<Path, Hash> pr =
computeStorePathForPath(srcPath, fixed, recursive, hashAlgo, filter);
Path & dstPath(pr.first);
Hash & h(pr.second);
addTempRoot(dstPath);
if (!isValidPath(dstPath)) {
/* The first check above is an optimisation to prevent
unnecessary lock acquisition. */
2006-06-01 20:13:33 +02:00
PathLocks outputLock(singleton<PathSet, Path>(dstPath));
if (!isValidPath(dstPath)) {
if (pathExists(dstPath)) deletePathWrapped(dstPath);
copyPath(srcPath, dstPath, filter);
Hash h2 = hashPath(htSHA256, dstPath, filter);
if (h != h2)
throw Error(format("contents of `%1%' changed while copying it to `%2%' (%3% -> %4%)")
% srcPath % dstPath % printHash(h) % printHash(h2));
canonicalisePathMetaData(dstPath);
Transaction txn(nixDB);
registerValidPath(txn, dstPath, h, PathSet(), "");
txn.commit();
}
outputLock.setDeletion(true);
}
return dstPath;
}
Path LocalStore::addTextToStore(const string & suffix, const string & s,
const PathSet & references)
{
Path dstPath = computeStorePathForText(suffix, s, references);
addTempRoot(dstPath);
if (!isValidPath(dstPath)) {
2006-06-01 20:13:33 +02:00
PathLocks outputLock(singleton<PathSet, Path>(dstPath));
if (!isValidPath(dstPath)) {
if (pathExists(dstPath)) deletePathWrapped(dstPath);
2003-11-22 16:58:34 +01:00
writeStringToFile(dstPath, s);
canonicalisePathMetaData(dstPath);
Transaction txn(nixDB);
registerValidPath(txn, dstPath,
hashPath(htSHA256, dstPath), references, "");
txn.commit();
}
outputLock.setDeletion(true);
}
return dstPath;
}
struct HashAndWriteSink : Sink
{
Sink & writeSink;
HashSink hashSink;
bool hashing;
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
{
hashing = true;
}
virtual void operator ()
(const unsigned char * data, unsigned int len)
{
writeSink(data, len);
if (hashing) hashSink(data, len);
}
};
#define EXPORT_MAGIC 0x4558494e
static void checkSecrecy(const Path & path)
{
struct stat st;
if (stat(path.c_str(), &st))
throw SysError(format("getting status of `%1%'") % path);
if ((st.st_mode & (S_IRWXG | S_IRWXO)) != 0)
throw Error(format("file `%1%' should be secret (inaccessible to everybody else)!") % path);
}
void LocalStore::exportPath(const Path & path, bool sign,
Sink & sink)
{
assertStorePath(path);
/* Wrap all of this in a transaction to make sure that we export
consistent metadata. */
Transaction txn(nixDB);
addTempRoot(path);
if (!isValidPathTxn(txn, path))
throw Error(format("path `%1%' is not valid") % path);
HashAndWriteSink hashAndWriteSink(sink);
dumpPath(path, hashAndWriteSink);
writeInt(EXPORT_MAGIC, hashAndWriteSink);
writeString(path, hashAndWriteSink);
PathSet references;
nix::queryReferences(txn, path, references);
writeStringSet(references, hashAndWriteSink);
Path deriver = nix::queryDeriver(txn, path);
writeString(deriver, hashAndWriteSink);
if (sign) {
Hash hash = hashAndWriteSink.hashSink.finish();
hashAndWriteSink.hashing = false;
writeInt(1, hashAndWriteSink);
Path tmpDir = createTempDir();
AutoDelete delTmp(tmpDir);
Path hashFile = tmpDir + "/hash";
writeStringToFile(hashFile, printHash(hash));
Path secretKey = nixConfDir + "/signing-key.sec";
checkSecrecy(secretKey);
Strings args;
args.push_back("rsautl");
args.push_back("-sign");
args.push_back("-inkey");
args.push_back(secretKey);
args.push_back("-in");
args.push_back(hashFile);
string signature = runProgram(OPENSSL_PATH, true, args);
writeString(signature, hashAndWriteSink);
} else
writeInt(0, hashAndWriteSink);
txn.commit();
}
struct HashAndReadSource : Source
{
Source & readSource;
HashSink hashSink;
bool hashing;
HashAndReadSource(Source & readSource) : readSource(readSource), hashSink(htSHA256)
{
hashing = true;
}
virtual void operator ()
(unsigned char * data, unsigned int len)
{
readSource(data, len);
if (hashing) hashSink(data, len);
}
};
Path LocalStore::importPath(bool requireSignature, Source & source)
{
HashAndReadSource hashAndReadSource(source);
/* We don't yet know what store path this archive contains (the
store path follows the archive data proper), and besides, we
don't know yet whether the signature is valid. */
Path tmpDir = createTempDir(nixStore);
AutoDelete delTmp(tmpDir);
Path unpacked = tmpDir + "/unpacked";
restorePath(unpacked, hashAndReadSource);
unsigned int magic = readInt(hashAndReadSource);
if (magic != EXPORT_MAGIC)
throw Error("Nix archive cannot be imported; wrong format");
Path dstPath = readStorePath(hashAndReadSource);
PathSet references = readStorePaths(hashAndReadSource);
2007-02-28 00:18:57 +01:00
Path deriver = readString(hashAndReadSource);
if (deriver != "") assertStorePath(deriver);
Hash hash = hashAndReadSource.hashSink.finish();
hashAndReadSource.hashing = false;
bool haveSignature = readInt(hashAndReadSource) == 1;
if (requireSignature && !haveSignature)
throw Error("imported archive lacks a signature");
if (haveSignature) {
string signature = readString(hashAndReadSource);
if (requireSignature) {
Path sigFile = tmpDir + "/sig";
writeStringToFile(sigFile, signature);
Strings args;
args.push_back("rsautl");
args.push_back("-verify");
args.push_back("-inkey");
args.push_back(nixConfDir + "/signing-key.pub");
args.push_back("-pubin");
args.push_back("-in");
args.push_back(sigFile);
string hash2 = runProgram(OPENSSL_PATH, true, args);
/* Note: runProgram() throws an exception if the signature
is invalid. */
if (printHash(hash) != hash2)
throw Error(
"signed hash doesn't match actual contents of imported "
"archive; archive could be corrupt, or someone is trying "
"to import a Trojan horse");
}
}
/* Do the actual import. */
/* !!! way too much code duplication with addTextToStore() etc. */
addTempRoot(dstPath);
if (!isValidPath(dstPath)) {
PathLocks outputLock(singleton<PathSet, Path>(dstPath));
if (!isValidPath(dstPath)) {
if (pathExists(dstPath)) deletePathWrapped(dstPath);
if (rename(unpacked.c_str(), dstPath.c_str()) == -1)
throw SysError(format("cannot move `%1%' to `%2%'")
% unpacked % dstPath);
canonicalisePathMetaData(dstPath);
Transaction txn(nixDB);
/* !!! if we were clever, we could prevent the hashPath()
here. */
if (!isValidPath(deriver)) deriver = "";
registerValidPath(txn, dstPath,
hashPath(htSHA256, dstPath), references, deriver);
txn.commit();
}
outputLock.setDeletion(true);
}
return dstPath;
}
void deleteFromStore(const Path & _path, unsigned long long & bytesFreed)
2003-06-23 16:40:49 +02:00
{
bytesFreed = 0;
Path path(canonPath(_path));
assertStorePath(path);
Transaction txn(nixDB);
if (isValidPathTxn(txn, path)) {
PathSet referrers = getReferrers(txn, path);
for (PathSet::iterator i = referrers.begin();
i != referrers.end(); ++i)
if (*i != path && isValidPathTxn(txn, *i))
throw PathInUse(format("cannot delete path `%1%' because it is in use by path `%2%'") % path % *i);
invalidatePath(txn, path);
}
txn.commit();
deletePathWrapped(path, bytesFreed);
2003-06-23 16:40:49 +02:00
}
void verifyStore(bool checkContents)
{
Transaction txn(nixDB);
printMsg(lvlInfo, "checking path existence");
Paths paths;
PathSet validPaths;
nixDB.enumTable(txn, dbValidPaths, paths);
for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) {
2007-05-01 15:21:05 +02:00
checkInterrupt();
if (!pathExists(*i)) {
printMsg(lvlError, format("path `%1%' disappeared") % *i);
invalidatePath(txn, *i);
} else if (!isStorePath(*i)) {
printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i);
invalidatePath(txn, *i);
} else {
if (checkContents) {
2006-08-01 11:43:41 +02:00
debug(format("checking contents of `%1%'") % *i);
Hash expected = queryHash(txn, *i);
Hash current = hashPath(expected.type, *i);
if (current != expected) {
printMsg(lvlError, format("path `%1%' was modified! "
"expected hash `%2%', got `%3%'")
% *i % printHash(expected) % printHash(current));
}
}
validPaths.insert(*i);
}
}
/* Check the cleanup invariant: only valid paths can have
`references', `referrers', or `derivers' entries. */
/* Check the `derivers' table. */
printMsg(lvlInfo, "checking the derivers table");
Paths deriversKeys;
nixDB.enumTable(txn, dbDerivers, deriversKeys);
for (Paths::iterator i = deriversKeys.begin();
i != deriversKeys.end(); ++i)
{
if (validPaths.find(*i) == validPaths.end()) {
printMsg(lvlError, format("removing deriver entry for invalid path `%1%'")
% *i);
nixDB.delPair(txn, dbDerivers, *i);
}
else {
Path deriver = queryDeriver(txn, *i);
if (!isStorePath(deriver)) {
printMsg(lvlError, format("removing corrupt deriver `%1%' for `%2%'")
% deriver % *i);
nixDB.delPair(txn, dbDerivers, *i);
}
}
}
/* Check the `references' table. */
printMsg(lvlInfo, "checking the references table");
Paths referencesKeys;
nixDB.enumTable(txn, dbReferences, referencesKeys);
for (Paths::iterator i = referencesKeys.begin();
i != referencesKeys.end(); ++i)
{
if (validPaths.find(*i) == validPaths.end()) {
printMsg(lvlError, format("removing references entry for invalid path `%1%'")
% *i);
setReferences(txn, *i, PathSet());
}
else {
PathSet references;
queryReferences(txn, *i, references);
for (PathSet::iterator j = references.begin();
j != references.end(); ++j)
{
string dummy;
if (!nixDB.queryString(txn, dbReferrers, addPrefix(*j, *i), dummy)) {
printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'")
% *j % *i);
nixDB.setString(txn, dbReferrers, addPrefix(*j, *i), "");
}
if (validPaths.find(*j) == validPaths.end()) {
printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'")
% *i % *j);
}
}
}
}
/* Check the `referrers' table. */
printMsg(lvlInfo, "checking the referrers table");
Strings referrers;
nixDB.enumTable(txn, dbReferrers, referrers);
for (Strings::iterator i = referrers.begin(); i != referrers.end(); ++i) {
/* Decode the entry (it's a tuple of paths). */
string::size_type nul = i->find((char) 0);
if (nul == string::npos) {
printMsg(lvlError, format("removing bad referrer table entry `%1%'") % *i);
nixDB.delPair(txn, dbReferrers, *i);
continue;
}
Path to(*i, 0, nul);
Path from(*i, nul + 1);
if (validPaths.find(to) == validPaths.end()) {
printMsg(lvlError, format("removing referrer entry from `%1%' to invalid `%2%'")
% from % to);
nixDB.delPair(txn, dbReferrers, *i);
}
else if (validPaths.find(from) == validPaths.end()) {
printMsg(lvlError, format("removing referrer entry from invalid `%1%' to `%2%'")
% from % to);
nixDB.delPair(txn, dbReferrers, *i);
}
else {
PathSet references;
queryReferences(txn, from, references);
if (find(references.begin(), references.end(), to) == references.end()) {
printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'")
% from % to);
references.insert(to);
setReferences(txn, from, references);
}
}
}
txn.commit();
}
typedef std::map<Hash, std::pair<Path, ino_t> > HashToPath;
static void makeWritable(const Path & path)
{
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
throw SysError(format("changing writability of `%1%'") % path);
}
static void hashAndLink(bool dryRun, HashToPath & hashToPath,
OptimiseStats & stats, const Path & path)
{
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
/* Sometimes SNAFUs can cause files in the Nix store to be
modified, in particular when running programs as root under
NixOS (example: $fontconfig/var/cache being modified). Skip
those files. */
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
return;
}
/* We can hard link regular files and symlinks. */
if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) {
/* Hash the file. Note that hashPath() returns the hash over
the NAR serialisation, which includes the execute bit on
the file. Thus, executable and non-executable files with
the same contents *won't* be linked (which is good because
otherwise the permissions would be screwed up).
Also note that if `path' is a symlink, then we're hashing
the contents of the symlink (i.e. the result of
readlink()), not the contents of the target (which may not
even exist). */
Hash hash = hashPath(htSHA256, path);
stats.totalFiles++;
printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
std::pair<Path, ino_t> prevPath = hashToPath[hash];
if (prevPath.first == "") {
hashToPath[hash] = std::pair<Path, ino_t>(path, st.st_ino);
return;
}
/* Yes! We've seen a file with the same contents. Replace
the current file with a hard link to that file. */
stats.sameContents++;
if (prevPath.second == st.st_ino) {
printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first);
return;
}
if (!dryRun) {
printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first);
Path tempLink = (format("%1%.tmp-%2%-%3%")
% path % getpid() % rand()).str();
/* Make the containing directory writable, but only if
it's not the store itself (we don't want or need to
mess with its permissions). */
bool mustToggle = !isStorePath(path);
if (mustToggle) makeWritable(dirOf(path));
if (link(prevPath.first.c_str(), tempLink.c_str()) == -1)
throw SysError(format("cannot link `%1%' to `%2%'")
% tempLink % prevPath.first);
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1)
throw SysError(format("cannot rename `%1%' to `%2%'")
% tempLink % path);
/* Make the directory read-only again and reset its
timestamp back to 0. */
if (mustToggle) _canonicalisePathMetaData(dirOf(path), false);
} else
printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first);
stats.filesLinked++;
stats.bytesFreed += st.st_size;
}
if (S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path);
for (Strings::iterator i = names.begin(); i != names.end(); ++i)
hashAndLink(dryRun, hashToPath, stats, path + "/" + *i);
}
}
void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
{
HashToPath hashToPath;
Paths paths;
PathSet validPaths;
nixDB.enumTable(noTxn, dbValidPaths, paths);
for (Paths::iterator i = paths.begin(); i != paths.end(); ++i) {
addTempRoot(*i);
if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
hashAndLink(dryRun, hashToPath, stats, *i);
}
}
/* Upgrade from schema 2 (0.8 <= Nix <= 0.9) to schema 3 (Nix >=
0.10). The only thing to do here is to upgrade the old `referer'
table (which causes quadratic complexity in some cases) to the new
(and properly spelled) `referrer' table. */
static void upgradeStore09()
{
/* !!! we should disallow concurrent upgrades */
if (!pathExists(nixDBPath + "/referers")) return;
printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
Transaction txn(nixDB);
std::cerr << "converting referers to referrers...";
TableId dbReferers = nixDB.openTable("referers"); /* sic! */
Paths referersKeys;
nixDB.enumTable(txn, dbReferers, referersKeys);
int n = 0;
for (Paths::iterator i = referersKeys.begin();
i != referersKeys.end(); ++i)
{
Paths referers;
nixDB.queryStrings(txn, dbReferers, *i, referers);
for (Paths::iterator j = referers.begin();
j != referers.end(); ++j)
nixDB.setString(txn, dbReferrers, addPrefix(*i, *j), "");
if (++n % 1000 == 0) {
txn.commit();
txn.begin(nixDB);
std::cerr << "|";
}
std::cerr << ".";
}
txn.commit();
std::cerr << std::endl;
nixDB.closeTable(dbReferers);
nixDB.deleteTable("referers");
}
/* Upgrade from schema 3 (Nix 0.10) to schema 4 (Nix >= 0.11). The
only thing to do here is to delete the substitutes table and get
rid of invalid but substitutable references/referrers. */
static void upgradeStore11()
{
if (!pathExists(nixDBPath + "/substitutes")) return;
printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
Transaction txn(nixDB);
TableId dbSubstitutes = nixDB.openTable("substitutes");
Paths subKeys;
nixDB.enumTable(txn, dbSubstitutes, subKeys);
for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
if (!isValidPathTxn(txn, *i))
invalidatePath(txn, *i);
}
txn.commit();
nixDB.closeTable(dbSubstitutes);
nixDB.deleteTable("substitutes");
}
}