mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

nix-prefetch-url: Download file in constant memory

Browse source 
Before:

  $ command time nix-prefetch-url https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.17.6.tar.xz
  1.19user 1.02system 0:41.96elapsed 5%CPU (0avgtext+0avgdata 182720maxresident)k

After:

  1.38user 1.05system 0:39.73elapsed 6%CPU (0avgtext+0avgdata 16204maxresident)k

Note however that addToStore() can still take a lot of memory
(e.g. RemoteStore::addToStore() is constant space, but
LocalStore::addToStore() isn't; that's fixed by
c94b4fc7ee
though).

Fixes #1400.
This commit is contained in:
Eelco Dolstra 2018-07-12 18:44:37 +02:00
parent c2de2ff385
commit 1b34b69b45
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
1 changed files with 17 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/nix-prefetch-url/nix-prefetch-url.cc
View file

@ -9,6 +9,10 @@
#include <iostream> #include <iostream>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
using namespace nix; using namespace nix;
@ -160,14 +164,20 @@ int main(int argc, char * * argv)
auto actualUri = resolveMirrorUri(*state, uri); auto actualUri = resolveMirrorUri(*state, uri);
/* Download the file. */
DownloadRequest req(actualUri);
req.decompress = false;
auto result = getDownloader()->download(req);
AutoDelete tmpDir(createTempDir(), true); AutoDelete tmpDir(createTempDir(), true);
Path tmpFile = (Path) tmpDir + "/tmp"; Path tmpFile = (Path) tmpDir + "/tmp";
writeFile(tmpFile, *result.data);
/* Download the file. */
{
AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
FdSink sink(fd.get());
DownloadRequest req(actualUri);
req.decompress = false;
getDownloader()->download(std::move(req), sink);
}
/* Optionally unpack the file. */ /* Optionally unpack the file. */
if (unpack) { if (unpack) {
@ -191,7 +201,7 @@ int main(int argc, char * * argv)
/* FIXME: inefficient; addToStore() will also hash /* FIXME: inefficient; addToStore() will also hash
this. */ this. */
hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data); hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
if (expectedHash != Hash(ht) && expectedHash != hash) if (expectedHash != Hash(ht) && expectedHash != hash)
throw Error(format("hash mismatch for '%1%'") % uri); throw Error(format("hash mismatch for '%1%'") % uri);