2014-12-14 01:51:14 +01:00
|
|
|
|
#include <cstdlib>
|
2016-11-14 13:37:16 +01:00
|
|
|
|
#include <cstring>
|
2017-07-06 22:42:12 +02:00
|
|
|
|
#include <regex>
|
2020-05-19 16:54:39 +02:00
|
|
|
|
|
2008-06-18 16:13:00 +02:00
|
|
|
|
#include <errno.h>
|
2009-09-24 09:39:55 +02:00
|
|
|
|
#include <stdio.h>
|
2008-06-09 15:52:45 +02:00
|
|
|
|
#include <sys/stat.h>
|
|
|
|
|
#include <sys/types.h>
|
|
|
|
|
#include <unistd.h>
|
2020-05-19 16:54:39 +02:00
|
|
|
|
|
2012-07-23 18:08:34 +02:00
|
|
|
|
#include "globals.hh"
|
2020-05-19 02:02:44 +02:00
|
|
|
|
#include "glog/logging.h"
|
2008-06-09 15:52:45 +02:00
|
|
|
|
#include "local-store.hh"
|
|
|
|
|
#include "util.hh"
|
|
|
|
|
|
|
|
|
|
namespace nix {
|
|
|
|
|
|
|
|
|
|
static void makeWritable(const Path& path) {
|
|
|
|
|
struct stat st;
|
|
|
|
|
if (lstat(path.c_str(), &st)) {
|
2017-07-30 13:27:57 +02:00
|
|
|
|
throw SysError(format("getting attributes of path '%1%'") % path);
|
2020-05-19 21:47:23 +02:00
|
|
|
|
}
|
2008-06-09 15:52:45 +02:00
|
|
|
|
if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) {
|
2017-07-30 13:27:57 +02:00
|
|
|
|
throw SysError(format("changing writability of '%1%'") % path);
|
2020-05-19 21:47:23 +02:00
|
|
|
|
}
|
2008-06-09 15:52:45 +02:00
|
|
|
|
}
|
|
|
|
|
|
2008-06-18 16:13:00 +02:00
|
|
|
|
struct MakeReadOnly {
|
|
|
|
|
Path path;
|
2020-05-19 23:02:23 +02:00
|
|
|
|
explicit MakeReadOnly(const Path& path) : path(path) {}
|
2008-06-18 16:13:00 +02:00
|
|
|
|
~MakeReadOnly() {
|
|
|
|
|
try {
|
2012-09-19 22:17:54 +02:00
|
|
|
|
/* This will make the path read-only. */
|
2013-03-08 01:24:59 +01:00
|
|
|
|
if (path != "") {
|
|
|
|
|
canonicaliseTimestampAndPermissions(path);
|
2020-05-19 19:55:58 +02:00
|
|
|
|
}
|
2008-06-18 16:13:00 +02:00
|
|
|
|
} catch (...) {
|
|
|
|
|
ignoreException();
|
|
|
|
|
}
|
2020-05-17 17:31:57 +02:00
|
|
|
|
}
|
2008-06-18 16:13:00 +02:00
|
|
|
|
};
|
|
|
|
|
|
2014-05-14 22:52:10 +02:00
|
|
|
|
LocalStore::InodeHash LocalStore::loadInodeHash() {
|
2020-05-19 02:02:44 +02:00
|
|
|
|
DLOG(INFO) << "loading hash inodes in memory";
|
2014-05-15 09:02:22 +02:00
|
|
|
|
InodeHash inodeHash;
|
2014-05-15 11:33:46 +02:00
|
|
|
|
|
2017-01-16 22:39:27 +01:00
|
|
|
|
AutoCloseDir dir(opendir(linksDir.c_str()));
|
2020-05-19 02:02:44 +02:00
|
|
|
|
if (!dir) {
|
|
|
|
|
throw SysError(format("opening directory '%1%'") % linksDir);
|
|
|
|
|
}
|
2014-05-14 22:52:10 +02:00
|
|
|
|
|
|
|
|
|
struct dirent* dirent;
|
2017-01-16 22:39:27 +01:00
|
|
|
|
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
|
2014-05-14 22:52:10 +02:00
|
|
|
|
checkInterrupt();
|
|
|
|
|
// We don't care if we hit non-hash files, anything goes
|
2014-05-15 09:02:22 +02:00
|
|
|
|
inodeHash.insert(dirent->d_ino);
|
2014-05-13 23:10:06 +02:00
|
|
|
|
}
|
2020-05-19 02:02:44 +02:00
|
|
|
|
if (errno) {
|
|
|
|
|
throw SysError(format("reading directory '%1%'") % linksDir);
|
|
|
|
|
}
|
2014-05-14 22:52:10 +02:00
|
|
|
|
|
2020-05-19 02:02:44 +02:00
|
|
|
|
DLOG(INFO) << "loaded " << inodeHash.size() << " hash inodes";
|
2014-05-14 22:52:10 +02:00
|
|
|
|
|
2014-05-15 09:02:22 +02:00
|
|
|
|
return inodeHash;
|
2014-05-13 23:10:06 +02:00
|
|
|
|
}
|
2008-06-18 16:13:00 +02:00
|
|
|
|
|
2014-05-15 11:33:46 +02:00
|
|
|
|
Strings LocalStore::readDirectoryIgnoringInodes(const Path& path,
|
|
|
|
|
const InodeHash& inodeHash) {
|
2014-05-15 09:02:22 +02:00
|
|
|
|
Strings names;
|
2014-05-15 11:33:46 +02:00
|
|
|
|
|
2017-01-16 22:39:27 +01:00
|
|
|
|
AutoCloseDir dir(opendir(path.c_str()));
|
2020-05-19 02:02:44 +02:00
|
|
|
|
if (!dir) {
|
|
|
|
|
throw SysError(format("opening directory '%1%'") % path);
|
|
|
|
|
}
|
2014-05-15 09:02:22 +02:00
|
|
|
|
|
|
|
|
|
struct dirent* dirent;
|
2017-01-16 22:39:27 +01:00
|
|
|
|
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
|
2014-05-15 09:02:22 +02:00
|
|
|
|
checkInterrupt();
|
|
|
|
|
|
|
|
|
|
if (inodeHash.count(dirent->d_ino)) {
|
2020-05-19 02:02:44 +02:00
|
|
|
|
DLOG(WARNING) << dirent->d_name << " is already linked";
|
2014-05-15 09:02:22 +02:00
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
2017-07-30 13:27:57 +02:00
|
|
|
|
string name = dirent->d_name;
|
2014-05-15 09:02:22 +02:00
|
|
|
|
if (name == "." || name == "..") {
|
|
|
|
|
continue;
|
2020-05-19 19:55:58 +02:00
|
|
|
|
}
|
2014-05-15 09:02:22 +02:00
|
|
|
|
names.push_back(name);
|
2020-05-17 17:31:57 +02:00
|
|
|
|
}
|
2020-05-19 02:02:44 +02:00
|
|
|
|
if (errno) {
|
|
|
|
|
throw SysError(format("reading directory '%1%'") % path);
|
|
|
|
|
}
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2014-05-15 09:02:22 +02:00
|
|
|
|
return names;
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-19 02:02:44 +02:00
|
|
|
|
void LocalStore::optimisePath_(OptimiseStats& stats, const Path& path,
|
|
|
|
|
InodeHash& inodeHash) {
|
2012-08-01 22:06:49 +02:00
|
|
|
|
checkInterrupt();
|
2014-05-15 11:33:46 +02:00
|
|
|
|
|
2008-06-09 15:52:45 +02:00
|
|
|
|
struct stat st;
|
|
|
|
|
if (lstat(path.c_str(), &st)) {
|
2017-07-30 13:27:57 +02:00
|
|
|
|
throw SysError(format("getting attributes of path '%1%'") % path);
|
2020-05-19 21:47:23 +02:00
|
|
|
|
}
|
2008-06-09 15:52:45 +02:00
|
|
|
|
|
2017-07-06 22:42:12 +02:00
|
|
|
|
#if __APPLE__
|
2017-07-30 12:28:50 +02:00
|
|
|
|
/* HFS/macOS has some undocumented security feature disabling hardlinking for
|
2017-07-06 22:42:12 +02:00
|
|
|
|
special files within .app dirs. *.app/Contents/PkgInfo and
|
|
|
|
|
*.app/Contents/Resources/\*.lproj seem to be the only paths affected. See
|
|
|
|
|
https://github.com/NixOS/nix/issues/1443 for more discussion. */
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2018-06-14 18:34:55 +02:00
|
|
|
|
if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) {
|
2017-07-31 10:31:51 +02:00
|
|
|
|
debug(format("'%1%' is not allowed to be linked in macOS") % path);
|
2017-07-06 22:42:12 +02:00
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2012-07-23 18:08:34 +02:00
|
|
|
|
if (S_ISDIR(st.st_mode)) {
|
2014-05-15 11:33:46 +02:00
|
|
|
|
Strings names = readDirectoryIgnoringInodes(path, inodeHash);
|
2020-05-19 02:02:44 +02:00
|
|
|
|
for (auto& i : names) {
|
|
|
|
|
optimisePath_(stats, path + "/" + i, inodeHash);
|
2020-05-19 20:04:08 +02:00
|
|
|
|
}
|
2012-07-23 18:08:34 +02:00
|
|
|
|
return;
|
|
|
|
|
}
|
2012-07-31 01:55:41 +02:00
|
|
|
|
|
2012-07-24 00:42:18 +02:00
|
|
|
|
/* We can hard link regular files and maybe symlinks. */
|
|
|
|
|
if (!S_ISREG(st.st_mode)
|
|
|
|
|
#if CAN_LINK_SYMLINK
|
|
|
|
|
&& !S_ISLNK(st.st_mode)
|
|
|
|
|
#endif
|
|
|
|
|
)
|
|
|
|
|
return;
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2008-06-09 15:52:45 +02:00
|
|
|
|
/* Sometimes SNAFUs can cause files in the Nix store to be
|
|
|
|
|
modified, in particular when running programs as root under
|
|
|
|
|
NixOS (example: $fontconfig/var/cache being modified). Skip
|
2012-07-23 18:08:34 +02:00
|
|
|
|
those files. FIXME: check the modification time. */
|
2008-06-09 15:52:45 +02:00
|
|
|
|
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
|
2020-05-19 02:02:44 +02:00
|
|
|
|
LOG(WARNING) << "skipping suspicious writable file '" << path << "'";
|
2020-05-17 17:31:57 +02:00
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2015-11-09 20:48:09 +01:00
|
|
|
|
/* This can still happen on top-level files. */
|
2014-05-15 09:02:22 +02:00
|
|
|
|
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
|
2020-05-19 02:02:44 +02:00
|
|
|
|
DLOG(INFO) << path << " is already linked, with " << (st.st_nlink - 2)
|
|
|
|
|
<< " other file(s)";
|
2020-05-17 17:31:57 +02:00
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2012-07-23 18:08:34 +02:00
|
|
|
|
/* Hash the file. Note that hashPath() returns the hash over the
|
|
|
|
|
NAR serialisation, which includes the execute bit on the file.
|
|
|
|
|
Thus, executable and non-executable files with the same
|
|
|
|
|
contents *won't* be linked (which is good because otherwise the
|
|
|
|
|
permissions would be screwed up).
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2012-07-23 18:08:34 +02:00
|
|
|
|
Also note that if `path' is a symlink, then we're hashing the
|
|
|
|
|
contents of the symlink (i.e. the result of readlink()), not
|
|
|
|
|
the contents of the target (which may not even exist). */
|
|
|
|
|
Hash hash = hashPath(htSHA256, path).first;
|
2020-05-19 02:02:44 +02:00
|
|
|
|
LOG(INFO) << path << " has hash " << hash.to_string();
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2012-07-23 18:08:34 +02:00
|
|
|
|
/* Check if this is a known hash. */
|
2017-07-04 14:47:59 +02:00
|
|
|
|
Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
|
|
|
|
retry:
|
2012-07-23 18:08:34 +02:00
|
|
|
|
if (!pathExists(linkPath)) {
|
|
|
|
|
/* Nope, create a hard link in the links directory. */
|
2014-05-13 23:10:06 +02:00
|
|
|
|
if (link(path.c_str(), linkPath.c_str()) == 0) {
|
2014-05-15 09:02:22 +02:00
|
|
|
|
inodeHash.insert(st.st_ino);
|
2008-06-09 15:52:45 +02:00
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-15 09:02:22 +02:00
|
|
|
|
switch (errno) {
|
|
|
|
|
case EEXIST:
|
2017-07-30 13:27:57 +02:00
|
|
|
|
/* Fall through if another process created ‘linkPath’ before
|
2016-11-14 13:33:36 +01:00
|
|
|
|
we did. */
|
2017-07-30 13:27:57 +02:00
|
|
|
|
break;
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2016-11-14 13:33:36 +01:00
|
|
|
|
case ENOSPC:
|
|
|
|
|
/* On ext4, that probably means the directory index is
|
2017-07-30 13:27:57 +02:00
|
|
|
|
full. When that happens, it's fine to ignore it: we
|
|
|
|
|
just effectively disable deduplication of this
|
|
|
|
|
file. */
|
2020-05-19 02:02:44 +02:00
|
|
|
|
LOG(WARNING) << "cannot link '" << linkPath << " to " << path << ": "
|
|
|
|
|
<< strerror(errno);
|
|
|
|
|
|
2014-05-10 15:53:01 +02:00
|
|
|
|
return;
|
|
|
|
|
|
2017-07-30 13:27:57 +02:00
|
|
|
|
default:
|
|
|
|
|
throw SysError("cannot link '%1%' to '%2%'", linkPath, path);
|
2012-07-23 18:08:34 +02:00
|
|
|
|
}
|
2020-05-17 17:31:57 +02:00
|
|
|
|
}
|
|
|
|
|
|
2012-07-23 18:08:34 +02:00
|
|
|
|
/* Yes! We've seen a file with the same contents. Replace the
|
|
|
|
|
current file with a hard link to that file. */
|
|
|
|
|
struct stat stLink;
|
|
|
|
|
if (lstat(linkPath.c_str(), &stLink)) {
|
2017-07-30 13:27:57 +02:00
|
|
|
|
throw SysError(format("getting attributes of path '%1%'") % linkPath);
|
2020-05-19 21:47:23 +02:00
|
|
|
|
}
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2012-07-23 18:08:34 +02:00
|
|
|
|
if (st.st_ino == stLink.st_ino) {
|
2020-05-19 02:02:44 +02:00
|
|
|
|
DLOG(INFO) << path << " is already linked to " << linkPath;
|
2012-07-23 18:08:34 +02:00
|
|
|
|
return;
|
|
|
|
|
}
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2015-11-09 20:48:09 +01:00
|
|
|
|
if (st.st_size != stLink.st_size) {
|
2020-05-19 02:02:44 +02:00
|
|
|
|
LOG(WARNING) << "removing corrupted link '" << linkPath << "'";
|
2015-11-09 20:48:09 +01:00
|
|
|
|
unlink(linkPath.c_str());
|
|
|
|
|
goto retry;
|
|
|
|
|
}
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2020-05-19 02:02:44 +02:00
|
|
|
|
DLOG(INFO) << "linking '" << path << "' to '" << linkPath << "'";
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2012-07-23 18:08:34 +02:00
|
|
|
|
/* Make the containing directory writable, but only if it's not
|
|
|
|
|
the store itself (we don't want or need to mess with its
|
|
|
|
|
permissions). */
|
2016-06-02 15:08:18 +02:00
|
|
|
|
bool mustToggle = dirOf(path) != realStoreDir;
|
2020-05-19 18:38:04 +02:00
|
|
|
|
if (mustToggle) {
|
|
|
|
|
makeWritable(dirOf(path));
|
|
|
|
|
}
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2012-07-23 18:08:34 +02:00
|
|
|
|
/* When we're done, make the directory read-only again and reset
|
|
|
|
|
its timestamp back to 0. */
|
|
|
|
|
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2012-09-19 22:17:54 +02:00
|
|
|
|
Path tempLink =
|
2018-03-07 00:34:44 +01:00
|
|
|
|
(format("%1%/.tmp-link-%2%-%3%") % realStoreDir % getpid() % random())
|
|
|
|
|
.str();
|
2020-05-17 17:31:57 +02:00
|
|
|
|
|
2012-09-19 22:17:54 +02:00
|
|
|
|
if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
|
|
|
|
|
if (errno == EMLINK) {
|
|
|
|
|
/* Too many links to the same file (>= 32000 on most file
|
|
|
|
|
systems). This is likely to happen with empty files.
|
|
|
|
|
Just shrug and ignore. */
|
2020-05-19 02:02:44 +02:00
|
|
|
|
if (st.st_size) {
|
|
|
|
|
LOG(WARNING) << linkPath << " has maximum number of links";
|
|
|
|
|
}
|
2012-09-19 22:17:54 +02:00
|
|
|
|
return;
|
|
|
|
|
}
|
2017-07-30 13:27:57 +02:00
|
|
|
|
throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath);
|
2020-05-17 17:31:57 +02:00
|
|
|
|
}
|
|
|
|
|
|
2012-09-19 22:17:54 +02:00
|
|
|
|
/* Atomically replace the old file with the new hard link. */
|
|
|
|
|
if (rename(tempLink.c_str(), path.c_str()) == -1) {
|
2020-05-19 02:02:44 +02:00
|
|
|
|
if (unlink(tempLink.c_str()) == -1) {
|
|
|
|
|
LOG(ERROR) << "unable to unlink '" << tempLink << "'";
|
|
|
|
|
}
|
2012-09-19 22:17:54 +02:00
|
|
|
|
if (errno == EMLINK) {
|
|
|
|
|
/* Some filesystems generate too many links on the rename,
|
|
|
|
|
rather than on the original link. (Probably it
|
|
|
|
|
temporarily increases the st_nlink field before
|
|
|
|
|
decreasing it again.) */
|
2020-05-19 02:02:44 +02:00
|
|
|
|
DLOG(WARNING) << "'" << linkPath
|
|
|
|
|
<< "' has reached maximum number of links";
|
2012-09-19 22:17:54 +02:00
|
|
|
|
return;
|
2008-06-09 15:52:45 +02:00
|
|
|
|
}
|
2017-07-30 13:27:57 +02:00
|
|
|
|
throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path);
|
2020-05-17 17:31:57 +02:00
|
|
|
|
}
|
2012-07-23 18:08:34 +02:00
|
|
|
|
|
|
|
|
|
stats.filesLinked++;
|
|
|
|
|
stats.bytesFreed += st.st_size;
|
|
|
|
|
stats.blocksFreed += st.st_blocks;
|
2008-06-09 15:52:45 +02:00
|
|
|
|
}
|
|
|
|
|
|
2012-07-23 18:08:34 +02:00
|
|
|
|
void LocalStore::optimiseStore(OptimiseStats& stats) {
|
2012-07-11 16:49:04 +02:00
|
|
|
|
PathSet paths = queryAllValidPaths();
|
2014-05-14 22:52:10 +02:00
|
|
|
|
InodeHash inodeHash = loadInodeHash();
|
2008-06-09 15:52:45 +02:00
|
|
|
|
|
2017-08-16 17:00:24 +02:00
|
|
|
|
uint64_t done = 0;
|
|
|
|
|
|
2015-07-17 19:24:28 +02:00
|
|
|
|
for (auto& i : paths) {
|
|
|
|
|
addTempRoot(i);
|
|
|
|
|
if (!isValidPath(i)) {
|
|
|
|
|
continue;
|
|
|
|
|
} /* path was GC'ed, probably */
|
2017-08-16 17:00:24 +02:00
|
|
|
|
{
|
2020-05-19 02:02:44 +02:00
|
|
|
|
LOG(INFO) << "optimising path '" << i << "'";
|
|
|
|
|
optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash);
|
2012-07-23 21:02:52 +02:00
|
|
|
|
}
|
2020-05-17 17:31:57 +02:00
|
|
|
|
done++;
|
|
|
|
|
}
|
2012-07-23 21:02:52 +02:00
|
|
|
|
}
|
|
|
|
|
|
2014-09-01 22:21:42 +02:00
|
|
|
|
static string showBytes(unsigned long long bytes) {
|
|
|
|
|
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void LocalStore::optimiseStore() {
|
|
|
|
|
OptimiseStats stats;
|
|
|
|
|
|
|
|
|
|
optimiseStore(stats);
|
|
|
|
|
|
2020-05-19 02:02:44 +02:00
|
|
|
|
LOG(INFO) << showBytes(stats.bytesFreed) << " freed by hard-linking "
|
|
|
|
|
<< stats.filesLinked << " files";
|
2014-09-01 22:21:42 +02:00
|
|
|
|
}
|
2012-07-23 21:02:52 +02:00
|
|
|
|
|
|
|
|
|
void LocalStore::optimisePath(const Path& path) {
|
2012-07-31 01:55:41 +02:00
|
|
|
|
OptimiseStats stats;
|
2014-05-14 22:52:10 +02:00
|
|
|
|
InodeHash inodeHash;
|
2014-05-13 23:10:06 +02:00
|
|
|
|
|
2020-05-19 02:02:44 +02:00
|
|
|
|
if (settings.autoOptimiseStore) {
|
|
|
|
|
optimisePath_(stats, path, inodeHash);
|
|
|
|
|
}
|
2008-06-09 15:52:45 +02:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
} // namespace nix
|