BinaryCacheStore: Support local caching of NARs
This speeds up commands like "nix cat-store". For example: $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nar-cache /nix/store/i60yncmq6w9dyv37zd2k454g0fkl3arl-systemd-234/etc/udev/udev.conf real 0m4.336s $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nar-cache /nix/store/i60yncmq6w9dyv37zd2k454g0fkl3arl-systemd-234/etc/udev/udev.conf real 0m0.045s The primary motivation is to allow hydra-server to serve files from S3 binary caches. Previously Hydra had a hack to do "nix-store -r <path>", but that fetches the entire closure so is prohibitively expensive. There is no garbage collection of the NAR cache yet. Also, the entire NAR is read when accessing a single member file. We could generate the NAR listing to provide random access. Note: the NAR cache is indexed by the store path hash, not the content hash, so NAR caches should not be shared between binary caches, unless you're sure that all your builds are binary-reproducible.
This commit is contained in:
parent
11ba4302e3
commit
ca580bec35
4 changed files with 25 additions and 5 deletions
|
@ -319,7 +319,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
|
|||
|
||||
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
|
||||
{
|
||||
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()));
|
||||
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
|
||||
}
|
||||
|
||||
std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path)
|
||||
|
|
|
@ -18,6 +18,7 @@ public:
|
|||
const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"};
|
||||
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
|
||||
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
|
||||
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
|
||||
|
||||
private:
|
||||
|
||||
|
|
|
@ -3,10 +3,12 @@
|
|||
|
||||
namespace nix {
|
||||
|
||||
|
||||
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store)
|
||||
RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
|
||||
: store(store)
|
||||
, cacheDir(cacheDir)
|
||||
{
|
||||
if (cacheDir != "")
|
||||
createDirs(cacheDir);
|
||||
}
|
||||
|
||||
std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
||||
|
@ -23,7 +25,21 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
|
|||
if (i != nars.end()) return {i->second, restPath};
|
||||
|
||||
StringSink sink;
|
||||
store->narFromPath(storePath, sink);
|
||||
|
||||
Path cacheFile = cacheDir != "" ? fmt("%s/%s.nar", cacheDir, storePathToHash(storePath)) : "";
|
||||
|
||||
try {
|
||||
if (cacheFile != "")
|
||||
*sink.s = nix::readFile(cacheFile);
|
||||
} catch (SysError &) { }
|
||||
|
||||
if (sink.s->empty()) {
|
||||
store->narFromPath(storePath, sink);
|
||||
|
||||
if (cacheFile != "")
|
||||
/* FIXME: do this asynchronously. */
|
||||
writeFile(cacheFile, *sink.s);
|
||||
}
|
||||
|
||||
auto accessor = makeNarAccessor(sink.s);
|
||||
nars.emplace(storePath, accessor);
|
||||
|
|
|
@ -12,13 +12,16 @@ class RemoteFSAccessor : public FSAccessor
|
|||
|
||||
std::map<Path, ref<FSAccessor>> nars;
|
||||
|
||||
Path cacheDir;
|
||||
|
||||
std::pair<ref<FSAccessor>, Path> fetch(const Path & path_);
|
||||
|
||||
friend class BinaryCacheStore;
|
||||
|
||||
public:
|
||||
|
||||
RemoteFSAccessor(ref<Store> store);
|
||||
RemoteFSAccessor(ref<Store> store,
|
||||
const /* FIXME: use std::optional */ Path & cacheDir = "");
|
||||
|
||||
Stat stat(const Path & path) override;
|
||||
|
||||
|
|
Loading…
Reference in a new issue