* Place manifests in /nix/var/nix/manifests.
* Use the new patch downloader.
This commit is contained in:
Eelco Dolstra
parent
7eed57e784
commit
3d1b2101cc
12 changed files with 35 additions and 79 deletions
|
|
@ -4,13 +4,14 @@ bin_SCRIPTS = nix-collect-garbage \
|
|||
|
||||
noinst_SCRIPTS = nix-profile.sh
|
||||
|
||||
nix-pull nix-push: readmanifest.pm
|
||||
nix-pull nix-push: readmanifest.pm download-using-manifests.pl
|
||||
|
||||
install-exec-local: readmanifest.pm
|
||||
install-exec-local: readmanifest.pm download-using-manifests.pl
|
||||
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
|
||||
$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
|
||||
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
|
||||
$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix
|
||||
$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix
|
||||
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
|
||||
|
||||
include ../substitute.mk
|
||||
|
|
@ -20,4 +21,5 @@ EXTRA_DIST = nix-collect-garbage.in \
|
|||
nix-prefetch-url.in nix-install-package.in \
|
||||
nix-channel.in \
|
||||
readmanifest.pm.in \
|
||||
nix-build.in
|
||||
nix-build.in \
|
||||
download-using-manifests.pl.in
|
||||
|
|
|
|||
218
scripts/download-using-manifests.pl.in
Normal file
218
scripts/download-using-manifests.pl.in
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
#! @perl@ -w -I@libexecdir@/nix
|
||||
|
||||
use strict;
|
||||
use readmanifest;
|
||||
|
||||
my $manifestDir = "@localstatedir@/nix/manifests";
|
||||
|
||||
|
||||
# Check the arguments.
|
||||
die unless scalar @ARGV == 1;
|
||||
my $targetPath = $ARGV[0];
|
||||
|
||||
|
||||
# Load all manifests.
|
||||
my %narFiles;
|
||||
my %patches;
|
||||
my %successors;
|
||||
|
||||
for my $manifest (glob "$manifestDir/*.nixmanifest") {
|
||||
print STDERR "reading $manifest\n";
|
||||
readManifest $manifest, \%narFiles, \%patches, \%successors;
|
||||
}
|
||||
|
||||
|
||||
# Build a graph of all store paths that might contribute to the
|
||||
# construction of $targetPath, and the special node "start". The
|
||||
# edges are either patch operations, or downloads of full NAR files.
|
||||
# The latter edges only occur between "start" and a store path.
|
||||
|
||||
my %graph;
|
||||
|
||||
$graph{"start"} = {d => 0, pred => undef, edges => []};
|
||||
|
||||
my @queue = ();
|
||||
my $queueFront = 0;
|
||||
my %done;
|
||||
|
||||
sub addToQueue {
|
||||
my $v = shift;
|
||||
return if defined $done{$v};
|
||||
$done{$v} = 1;
|
||||
push @queue, $v;
|
||||
}
|
||||
|
||||
sub addNode {
|
||||
my $u = shift;
|
||||
$graph{$u} = {d => 999999999999, pred => undef, edges => []}
|
||||
unless defined $graph{$u};
|
||||
}
|
||||
|
||||
sub addEdge {
|
||||
my $u = shift;
|
||||
my $v = shift;
|
||||
my $w = shift;
|
||||
my $type = shift;
|
||||
my $info = shift;
|
||||
addNode $u;
|
||||
push @{$graph{$u}->{edges}},
|
||||
{weight => $w, start => $u, end => $v, type => $type, info => $info};
|
||||
my $n = scalar @{$graph{$u}->{edges}};
|
||||
}
|
||||
|
||||
addToQueue $targetPath;
|
||||
|
||||
while ($queueFront < scalar @queue) {
|
||||
my $u = $queue[$queueFront++];
|
||||
print "$u\n";
|
||||
|
||||
addNode $u;
|
||||
|
||||
# If the path already exists, it has distance 0 from the "start"
|
||||
# node.
|
||||
system "nix-store --isvalid '$u' 2> /dev/null";
|
||||
if ($? == 0) {
|
||||
addEdge "start", $u, 0, "present", undef;
|
||||
}
|
||||
|
||||
else {
|
||||
|
||||
# Add patch edges.
|
||||
my $patchList = $patches{$u};
|
||||
foreach my $patch (@{$patchList}) {
|
||||
# !!! this should be cached
|
||||
my $hash = `nix-hash "$patch->{basePath}"`;
|
||||
chomp $hash;
|
||||
print " MY HASH is $hash\n";
|
||||
if ($hash ne $patch->{baseHash}) {
|
||||
print " REJECTING PATCH from $patch->{basePath}\n";
|
||||
next;
|
||||
}
|
||||
print " PATCH from $patch->{basePath}\n";
|
||||
addToQueue $patch->{basePath};
|
||||
addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
|
||||
}
|
||||
|
||||
# Add NAR file edges to the start node.
|
||||
my $narFileList = $narFiles{$u};
|
||||
foreach my $narFile (@{$narFileList}) {
|
||||
print " NAR from $narFile->{url}\n";
|
||||
addEdge "start", $u, $narFile->{size}, "narfile", $narFile;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Run Dijkstra's shortest path algorithm to determine the shortest
|
||||
# sequence of download and/or patch actions that will produce
|
||||
# $targetPath.
|
||||
|
||||
sub byDistance { # sort by distance, reversed
|
||||
return -($graph{$a}->{d} <=> $graph{$b}->{d});
|
||||
}
|
||||
|
||||
my @todo = keys %graph;
|
||||
|
||||
while (scalar @todo > 0) {
|
||||
|
||||
# Remove the closest element from the todo list.
|
||||
@todo = sort byDistance @todo;
|
||||
my $u = pop @todo;
|
||||
|
||||
my $u_ = $graph{$u};
|
||||
|
||||
print "IN $u $u_->{d}\n";
|
||||
|
||||
foreach my $edge (@{$u_->{edges}}) {
|
||||
my $v_ = $graph{$edge->{end}};
|
||||
if ($v_->{d} > $u_->{d} + $edge->{weight}) {
|
||||
$v_->{d} = $u_->{d} + $edge->{weight};
|
||||
# Store the edge; to edge->start is actually the
|
||||
# predecessor.
|
||||
$v_->{pred} = $edge;
|
||||
print " RELAX $edge->{end} $v_->{d}\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# Retrieve the shortest path from "start" to $targetPath.
|
||||
my @path = ();
|
||||
my $cur = $targetPath;
|
||||
die "don't know how to produce $targetPath\n"
|
||||
unless defined $graph{$targetPath}->{pred};
|
||||
while ($cur ne "start") {
|
||||
push @path, $graph{$cur}->{pred};
|
||||
$cur = $graph{$cur}->{pred}->{start};
|
||||
}
|
||||
|
||||
|
||||
# Traverse the shortest path, perform the actions described by the
|
||||
# edges.
|
||||
my $curStep = 1;
|
||||
my $maxStep = scalar @path;
|
||||
|
||||
sub downloadFile {
|
||||
my $url = shift;
|
||||
my $hash = shift;
|
||||
$ENV{"PRINT_PATH"} = 1;
|
||||
$ENV{"QUIET"} = 1;
|
||||
my ($hash2, $path) = `nix-prefetch-url '$url' '$hash'`;
|
||||
chomp $hash2;
|
||||
chomp $path;
|
||||
die "hash mismatch" if $hash ne $hash2;
|
||||
return $path;
|
||||
}
|
||||
|
||||
while (scalar @path > 0) {
|
||||
my $edge = pop @path;
|
||||
my $u = $edge->{start};
|
||||
my $v = $edge->{end};
|
||||
|
||||
print "\n*** Step $curStep/$maxStep: ";
|
||||
$curStep++;
|
||||
|
||||
if ($edge->{type} eq "present") {
|
||||
print "using already present path `$v'\n";
|
||||
}
|
||||
|
||||
elsif ($edge->{type} eq "patch") {
|
||||
my $patch = $edge->{info};
|
||||
print "applying patch `$patch->{url}' to `$u' to create `$v'\n";
|
||||
|
||||
# Download the patch.
|
||||
print " downloading patch...\n";
|
||||
my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
|
||||
|
||||
# Turn the base path into a NAR archive, to which we can
|
||||
# actually apply the patch.
|
||||
print " packing base path...\n";
|
||||
system "nix-store --dump $patch->{basePath} > /tmp/nar";
|
||||
die "cannot dump `$patch->{basePath}'" if ($? != 0);
|
||||
|
||||
# Apply the patch.
|
||||
print " applying patch...\n";
|
||||
system "bspatch /tmp/nar /tmp/nar2 $patchPath";
|
||||
die "cannot apply patch `$patchPath' to /tmp/nar" if ($? != 0);
|
||||
|
||||
# Unpack the resulting NAR archive into the target path.
|
||||
print " unpacking patched archive...\n";
|
||||
system "nix-store --restore $targetPath < /tmp/nar2";
|
||||
die "cannot unpack /tmp/nar2 into `$targetPath'" if ($? != 0);
|
||||
}
|
||||
|
||||
elsif ($edge->{type} eq "narfile") {
|
||||
my $narFile = $edge->{info};
|
||||
print "downloading `$narFile->{url}' into `$v'\n";
|
||||
|
||||
# Download the archive.
|
||||
print " downloading archive...\n";
|
||||
my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
|
||||
|
||||
# Unpack the archive into the target path.
|
||||
print " unpacking archive...\n";
|
||||
system "bunzip2 < '$narFilePath' | nix-store --restore '$targetPath'";
|
||||
die "cannot unpack `$narFilePath' into `$targetPath'" if ($? != 0);
|
||||
}
|
||||
}
|
||||
0
scripts/nix-build.in
Executable file → Normal file
0
scripts/nix-build.in
Executable file → Normal file
0
scripts/nix-channel.in
Executable file → Normal file
0
scripts/nix-channel.in
Executable file → Normal file
0
scripts/nix-collect-garbage.in
Executable file → Normal file
0
scripts/nix-collect-garbage.in
Executable file → Normal file
|
|
@ -11,7 +11,7 @@ until mkdir $tmpdir, 0777;
|
|||
|
||||
my $manifest = "$tmpdir/manifest";
|
||||
|
||||
#END { unlink $manifest; rmdir $tmpdir; }
|
||||
END { unlink $manifest; rmdir $tmpdir; }
|
||||
|
||||
|
||||
# Obtain URLs either from the command line or from a configuration file.
|
||||
|
|
@ -28,8 +28,22 @@ sub processURL {
|
|||
system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
|
||||
"'$url' > '$manifest'") == 0
|
||||
or die "curl failed: $?";
|
||||
|
||||
|
||||
readManifest $manifest, \%narFiles, \%patches, \%successors;
|
||||
|
||||
my $baseName = "unnamed";
|
||||
if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component
|
||||
$baseName = $1;
|
||||
}
|
||||
|
||||
my $hash = `@bindir@/nix-hash --flat '$manifest'`
|
||||
or die "cannot hash `$manifest'";
|
||||
chomp $hash;
|
||||
|
||||
my $finalPath = "@localstatedir@/nix/manifests/$baseName-$hash.nixmanifest";
|
||||
|
||||
system("mv '$manifest' '$finalPath'") == 0
|
||||
or die "cannot move `$manifest' to `$finalPath";
|
||||
}
|
||||
|
||||
while (@ARGV) {
|
||||
|
|
@ -42,18 +56,6 @@ my $size = scalar (keys %narFiles);
|
|||
print "$size store paths in manifest\n";
|
||||
|
||||
|
||||
# Instantiate a store expression that builds the substitute program
|
||||
# (the program that fetches URLs and unpacks them into the store).
|
||||
my $nixExpr =
|
||||
"(import @datadir@/nix/corepkgs/nix-pull) " .
|
||||
"{system = \"@system@\";}";
|
||||
|
||||
print STDERR "building downloader...\n";
|
||||
my $substProgram = `echo '$nixExpr' | @bindir@/nix-store -qnf \$(@bindir@/nix-instantiate -)`
|
||||
or die "cannot instantiate Nix expression";
|
||||
chomp $substProgram;
|
||||
|
||||
|
||||
# Register all substitutes.
|
||||
print STDERR "registering substitutes...\n";
|
||||
|
||||
|
|
@ -66,10 +68,8 @@ foreach my $storePath (keys %narFiles) {
|
|||
my $narFileList = $narFiles{$storePath};
|
||||
foreach my $narFile (@{$narFileList}) {
|
||||
print WRITE "$storePath\n";
|
||||
print WRITE "$substProgram/fetch\n";
|
||||
print WRITE "2\n";
|
||||
print WRITE "$narFile->{url}\n";
|
||||
print WRITE "$narFile->{hash}\n";
|
||||
print WRITE "@libexecdir@/nix/download-using-manifests.pl\n";
|
||||
print WRITE "0\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ sub readManifest {
|
|||
my $basePath;
|
||||
my $baseHash;
|
||||
my $patchType;
|
||||
my $narHash;
|
||||
|
||||
while (<MANIFEST>) {
|
||||
chomp;
|
||||
|
|
@ -34,6 +35,7 @@ sub readManifest {
|
|||
undef $hash;
|
||||
$size = 999999999;
|
||||
@preds = ();
|
||||
undef $narHash;
|
||||
}
|
||||
elsif (/^patch \{$/) {
|
||||
$type = "patch";
|
||||
|
|
@ -44,6 +46,7 @@ sub readManifest {
|
|||
undef $basePath;
|
||||
undef $baseHash;
|
||||
undef $patchType;
|
||||
undef $narHash;
|
||||
}
|
||||
else { die "bad line: $_"; }
|
||||
} else {
|
||||
|
|
@ -71,7 +74,9 @@ sub readManifest {
|
|||
}
|
||||
if (!$found) {
|
||||
push @{$narFileList},
|
||||
{url => $url, hash => $hash, size => $size};
|
||||
{ url => $url, hash => $hash, size => $size
|
||||
, narHash => $narHash
|
||||
};
|
||||
}
|
||||
|
||||
foreach my $p (@preds) {
|
||||
|
|
@ -102,6 +107,7 @@ sub readManifest {
|
|||
push @{$patchList},
|
||||
{ url => $url, hash => $hash, size => $size
|
||||
, basePath => $basePath, baseHash => $baseHash
|
||||
, narHash => $narHash
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -117,11 +123,12 @@ sub readManifest {
|
|||
elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; }
|
||||
elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
|
||||
elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
|
||||
elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
|
||||
|
||||
# Compatibility;
|
||||
elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
|
||||
elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = $1; }
|
||||
|
||||
|
||||
else { die "bad line: $_"; }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue