mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

BinaryCacheStore: Optionally write a NAR listing

Browse source 
The store parameter "write-nar-listing=1" will cause BinaryCacheStore
to write a file ‘<store-hash>.ls.xz’ for each ‘<store-hash>.narinfo’
added to the binary cache. This file contains an XZ-compressed JSON
file describing the contents of the NAR, excluding the contents of
regular files.

E.g.

  {
    "version": 1,
    "root": {
      "type": "directory",
      "entries": {
        "lib": {
          "type": "directory",
          "entries": {
            "Mcrt1.o": {
              "type": "regular",
              "size": 1288
            },
            "Scrt1.o": {
              "type": "regular",
              "size": 3920
            },
          }
        }
      }
      ...
    }
  }

(The actual file has no indentation.)

This is intended to speed up the NixOS channels programs index
generator [1], since fetching gazillions of large NARs from
cache.nixos.org is currently a bottleneck for updating the regular
(non-small) channel.

[1] https://github.com/NixOS/nixos-channel-scripts/blob/master/generate-programs-index.cc
This commit is contained in:
Eelco Dolstra 2016-10-21 16:50:28 +02:00
parent 307cc8c33d
commit 542ae5c8f8
10 changed files with 81 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
src/libstore/binary-cache-store.cc
View file

@ -9,6 +9,7 @@
#include "worker-protocol.hh"
#include "nar-accessor.hh"
#include "nar-info-disk-cache.hh"
#include "json.hh"
#include <chrono>
@ -19,6 +20,7 @@ namespace nix {
BinaryCacheStore::BinaryCacheStore(const Params & params)
: Store(params)
, compression(get(params, "compression", "xz"))
, writeNARListing(get(params, "write-nar-listing", "0") == "1")
{
auto secretKeyFile = get(params, "secret-key", "");
if (secretKeyFile != "")
@ -79,7 +81,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
return storePathToHash(storePath) + ".narinfo";
}
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string & nar,
void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs)
{
if (!repair && isValidPath(info.path)) return;
@ -97,20 +99,73 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
auto narInfoFile = narInfoFileFor(info.path);
assert(nar.compare(0, narMagic.size(), narMagic) == 0);
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
auto narInfo = make_ref<NarInfo>(info);
narInfo->narSize = nar.size();
narInfo->narHash = hashString(htSHA256, nar);
narInfo->narSize = nar->size();
narInfo->narHash = hashString(htSHA256, *nar);
if (info.narHash && info.narHash != narInfo->narHash)
throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
if (writeNARListing) {
std::ostringstream jsonOut;
{
JSONObject jsonRoot(jsonOut);
jsonRoot.attr("version", 1);
auto accessor = makeNarAccessor(nar);
std::function<void(const Path &, JSONPlaceholder &)> recurse;
recurse = [&](const Path & path, JSONPlaceholder & res) {
auto st = accessor->stat(path);
auto obj = res.object();
switch (st.type) {
case FSAccessor::Type::tRegular:
obj.attr("type", "regular");
obj.attr("size", st.fileSize);
if (st.isExecutable)
obj.attr("executable", true);
break;
case FSAccessor::Type::tDirectory:
obj.attr("type", "directory");
{
auto res2 = obj.object("entries");
for (auto & name : accessor->readDirectory(path)) {
auto res3 = res2.placeholder(name);
recurse(path + "/" + name, res3);
}
}
break;
case FSAccessor::Type::tSymlink:
obj.attr("type", "symlink");
obj.attr("target", accessor->readLink(path));
break;
default:
abort();
}
};
{
auto res = jsonRoot.placeholder("root");
recurse("", res);
}
}
upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str()));
}
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
auto narCompressed = compress(compression, nar);
auto narCompressed = compress(compression, *nar);
auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
@ -118,7 +173,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
% narInfo->path % narInfo->narSize
% ((1.0 - (double) narCompressed->size() / nar.size()) * 100.0)
% ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0)
% duration);
/* Atomically write the NAR file. */
@ -132,7 +187,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const std::string
} else
stats.narWriteAverted++;
stats.narWriteBytes += nar.size();
stats.narWriteBytes += nar->size();
stats.narWriteCompressedBytes += narCompressed->size();
stats.narWriteCompressionTimeMs += duration;
@ -231,7 +286,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath,
ValidPathInfo info;
info.path = makeFixedOutputPath(recursive, h, name);
addToStore(info, *sink.s, repair);
addToStore(info, sink.s, repair);
return info.path;
}
@ -246,7 +301,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s,
if (repair || !isValidPath(info.path)) {
StringSink sink;
dumpString(s, sink);
addToStore(info, *sink.s, repair);
addToStore(info, sink.s, repair);
}
return info.path;

8
src/libstore/binary-cache-store.hh
View file

@ -19,12 +19,16 @@ private:
std::string compression;
bool writeNARListing;
protected:
BinaryCacheStore(const Params & params);
[[noreturn]] void notImpl();
public:
virtual bool fileExists(const std::string & path) = 0;
virtual void upsertFile(const std::string & path, const std::string & data) = 0;
@ -37,6 +41,8 @@ protected:
std::shared_ptr<std::string> getFile(const std::string & path);
protected:
bool wantMassQuery_ = false;
int priority = 50;
@ -86,7 +92,7 @@ public:
bool wantMassQuery() override { return wantMassQuery_; }
void addToStore(const ValidPathInfo & info, const std::string & nar,
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair = false, bool dontCheckSigs = false) override;
Path addToStore(const string & name, const Path & srcPath,

2
src/libstore/download.cc
View file

@ -558,7 +558,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data);
info.path = store->makeFixedOutputPath(false, hash, name);
info.narHash = hashString(htSHA256, *sink.s);
store->addToStore(info, *sink.s, false, true);
store->addToStore(info, sink.s, false, true);
storePath = info.path;
}

2
src/libstore/export-import.cc
View file

@ -117,7 +117,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
if (readInt(source) == 1)
readString(source);
addToStore(info, *tee.data, false, dontCheckSigs);
addToStore(info, tee.data, false, dontCheckSigs);
if (accessor)
addPathToAccessor(ref<FSAccessor>(accessor), info.path, tee.data);

6
src/libstore/local-store.cc
View file

@ -909,10 +909,10 @@ void LocalStore::invalidatePath(State & state, const Path & path)
}
void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar,
void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs)
{
Hash h = hashString(htSHA256, nar);
Hash h = hashString(htSHA256, *nar);
if (h != info.narHash)
throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") %
info.path % info.narHash.to_string() % h.to_string());
@ -939,7 +939,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar,
deletePath(realPath);
StringSource source(nar);
StringSource source(*nar);
restorePath(realPath, source);
canonicalisePathMetaData(realPath, -1);

2
src/libstore/local-store.hh
View file

@ -125,7 +125,7 @@ public:
void querySubstitutablePathInfos(const PathSet & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, const std::string & nar,
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs) override;
Path addToStore(const string & name, const Path & srcPath,

2
src/libstore/remote-store.cc
View file

@ -332,7 +332,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart)
}
void RemoteStore::addToStore(const ValidPathInfo & info, const std::string & nar,
void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs)
{
throw Error("RemoteStore::addToStore() not implemented");

2
src/libstore/remote-store.hh
View file

@ -53,7 +53,7 @@ public:
void querySubstitutablePathInfos(const PathSet & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, const std::string & nar,
void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair, bool dontCheckSigs) override;
Path addToStore(const string & name, const Path & srcPath,

2
src/libstore/store-api.cc
View file

@ -456,7 +456,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
StringSink sink;
srcStore->narFromPath({storePath}, sink);
dstStore->addToStore(*info, *sink.s, repair);
dstStore->addToStore(*info, sink.s, repair);
}

2
src/libstore/store-api.hh
View file

@ -365,7 +365,7 @@ public:
virtual bool wantMassQuery() { return false; }
/* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, const std::string & nar,
virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar,
bool repair = false, bool dontCheckSigs = false) = 0;
/* Copy the contents of a path to the store and register the