From bae75ca5a18edbb9fb959e2e48065a1987ffb61a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Jan 2007 16:50:19 +0000 Subject: [PATCH] * New kind of manifest object: "localPath", which denotes that a store path can be created by copying it from another location in the file system. This is useful in the NixOS installation. --- scripts/download-using-manifests.pl.in | 16 ++++++++++- scripts/generate-patches.pl.in | 6 ++-- scripts/maintenance/gc-releases.pl | 3 +- scripts/maintenance/shrink-manifest.pl | 3 +- scripts/nix-pull.in | 38 ++++++++++++++++++-------- scripts/readmanifest.pm.in | 19 +++++++++++++ scripts/remove-patches.pl | 7 ++--- scripts/update-manifest.pl | 3 +- 8 files changed, 73 insertions(+), 22 deletions(-) diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index e02d82c14..65ab36084 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -34,17 +34,31 @@ print "\n*** Trying to download/patch `$targetPath'\n"; # Load all manifests. my %narFiles; +my %localPaths; my %patches; for my $manifest (glob "$manifestDir/*.nixmanifest") { # print STDERR "reading $manifest\n"; - if (readManifest($manifest, \%narFiles, \%patches) < 3) { + if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) { print STDERR "you have an old-style manifest `$manifest'; please delete it\n"; exit 1; } } +# If we can copy from a local path, do that. +my $localPathList = $localPaths{$targetPath}; +foreach my $localPath (@{$localPathList}) { + my $sourcePath = $localPath->{copyFrom}; + if (-e $sourcePath) { + print "\n*** Step 1/1: copying from $sourcePath\n"; + system("@bindir@/nix-store --dump $sourcePath | @bindir@/nix-store --restore $targetPath") == 0 + or die "cannot copy `$sourcePath' to `$targetPath'"; + exit 0; + } +} + + # Build a graph of all store paths that might contribute to the # construction of $targetPath, and the special node "start". The # edges are either patch operations, or downloads of full NAR files. diff --git a/scripts/generate-patches.pl.in b/scripts/generate-patches.pl.in index 9e355f6ae..9a5c3423f 100755 --- a/scripts/generate-patches.pl.in +++ b/scripts/generate-patches.pl.in @@ -28,16 +28,18 @@ print "TEMP = $tmpDir\n"; #END { rmdir $tmpDir; } my %srcNarFiles; +my %srcLocalPaths; my %srcPatches; my %dstNarFiles; +my %dstLocalPaths; my %dstPatches; readManifest "$srcDir/MANIFEST", - \%srcNarFiles, \%srcPatches; + \%srcNarFiles, \%srcLocalPaths, \%srcPatches; readManifest "$dstDir/MANIFEST", - \%dstNarFiles, \%dstPatches; + \%dstNarFiles, \%dstLocalPaths, \%dstPatches; sub findOutputPaths { diff --git a/scripts/maintenance/gc-releases.pl b/scripts/maintenance/gc-releases.pl index 80b2726c5..b8abb0347 100755 --- a/scripts/maintenance/gc-releases.pl +++ b/scripts/maintenance/gc-releases.pl @@ -7,11 +7,12 @@ use readcache; # Read the manifests. my %narFiles; +my %localPaths; my %patches; foreach my $manifest (@ARGV) { print STDERR "loading $manifest\n"; - if (readManifest($manifest, \%narFiles, \%patches, 1) < 3) { + if (readManifest($manifest, \%narFiles, \%localPaths, \%patches, 1) < 3) { # die "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n"; } } diff --git a/scripts/maintenance/shrink-manifest.pl b/scripts/maintenance/shrink-manifest.pl index 7be581448..60cf9c5b4 100755 --- a/scripts/maintenance/shrink-manifest.pl +++ b/scripts/maintenance/shrink-manifest.pl @@ -6,11 +6,12 @@ use readcache; my %allNarFiles; +my %allLocalPaths; my %allPatches; foreach my $manifest (glob "/data/webserver/dist/*/*/MANIFEST") { print STDERR "loading $manifest\n"; - readManifest($manifest, \%allNarFiles, \%allPatches, 1); + readManifest($manifest, \%allNarFiles, \%allLocalPaths, \%allPatches, 1); } diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index 21fa30c61..46f9f147c 100644 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -28,6 +28,7 @@ umask 0022; # Process the URLs specified on the command line. my %narFiles; +my %localPaths; my %patches; my $skipWrongStore = 0; @@ -42,7 +43,7 @@ sub processURL { "'$url' > '$manifest'") == 0 or die "curl failed: $?"; - if (readManifest($manifest, \%narFiles, \%patches) < 3) { + if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) { die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n"; } @@ -80,7 +81,7 @@ while (@ARGV) { } -my $size = scalar (keys %narFiles); +my $size = scalar (keys %narFiles) + scalar (keys %localPaths); print "$size store paths in manifest\n"; @@ -90,19 +91,32 @@ print STDERR "registering substitutes...\n"; my $pid = open(WRITE, "|$binDir/nix-store --register-substitutes") or die "cannot run nix-store"; +sub writeRegistration { + my $storePath = shift; + my $object = shift; + print WRITE "$storePath\n"; + print WRITE "$object->{deriver}\n"; + print WRITE "$libexecDir/nix/download-using-manifests.pl\n"; + print WRITE "0\n"; + my @references = split " ", $object->{references}; + my $count = scalar @references; + print WRITE "$count\n"; + foreach my $reference (@references) { + print WRITE "$reference\n"; + } +} + foreach my $storePath (keys %narFiles) { my $narFileList = $narFiles{$storePath}; foreach my $narFile (@{$narFileList}) { - print WRITE "$storePath\n"; - print WRITE "$narFile->{deriver}\n"; - print WRITE "$libexecDir/nix/download-using-manifests.pl\n"; - print WRITE "0\n"; - my @references = split " ", $narFile->{references}; - my $count = scalar @references; - print WRITE "$count\n"; - foreach my $reference (@references) { - print WRITE "$reference\n"; - } + writeRegistration $storePath, $narFile; + } +} + +foreach my $storePath (keys %localPaths) { + my $localPathList = $localPaths{$storePath}; + foreach my $localPath (@{$localPathList}) { + writeRegistration $storePath, $localPath; } } diff --git a/scripts/readmanifest.pm.in b/scripts/readmanifest.pm.in index 0e15133cf..1d4444470 100644 --- a/scripts/readmanifest.pm.in +++ b/scripts/readmanifest.pm.in @@ -34,6 +34,7 @@ sub addPatch { sub readManifest { my $manifest = shift; my $narFiles = shift; + my $localPaths = shift; my $patches = shift; my $allowConflicts = shift; $allowConflicts = 0 unless defined $allowConflicts; @@ -57,6 +58,7 @@ sub readManifest { my $references; my $deriver; my $hashAlgo; + my $copyFrom; while () { chomp; @@ -125,9 +127,25 @@ sub readManifest { }, $allowConflicts; } + elsif ($type eq "localPath") { + + $$localPaths{$storePath} = [] + unless defined $$localPaths{$storePath}; + + my $localPathsList = $$localPaths{$storePath}; + + # !!! remove duplicates + + push @{$localPathsList}, + { copyFrom => $copyFrom, references => $references + , deriver => "" + }; + } + } elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; } + elsif (/^\s*CopyFrom:\s*(\/\S+)\s*$/) { $copyFrom = $1; } elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; } elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; } elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; } @@ -158,6 +176,7 @@ sub writeManifest my $manifest = shift; my $narFiles = shift; my $patches = shift; + my $copySources = shift; open MANIFEST, ">$manifest.tmp"; # !!! check exclusive diff --git a/scripts/remove-patches.pl b/scripts/remove-patches.pl index 75eb0ce77..1a94dfa16 100755 --- a/scripts/remove-patches.pl +++ b/scripts/remove-patches.pl @@ -6,13 +6,12 @@ use readmanifest; for my $p (@ARGV) { my %narFiles; + my %localPaths; my %patches; - readManifest $p, - \%narFiles, \%patches; + readManifest $p, \%narFiles, \%localPaths, \%patches; %patches = (); - writeManifest $p, - \%narFiles, \%patches; + writeManifest $p, \%narFiles, \%patches; } diff --git a/scripts/update-manifest.pl b/scripts/update-manifest.pl index 566f64673..ac0ce749d 100755 --- a/scripts/update-manifest.pl +++ b/scripts/update-manifest.pl @@ -8,9 +8,10 @@ die unless scalar @ARGV == 2; my $cache = $ARGV[0]; my $manifest = $ARGV[1]; my %narFiles; +my %localPaths; my %patches; -readManifest $manifest, \%narFiles, \%patches; +readManifest $manifest, \%narFiles, \%localPaths, \%patches; foreach my $storePath (keys %narFiles) { my $narFileList = $narFiles{$storePath};