mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support xz compression in the download-using-manifests substituter

Browse source
This commit is contained in:
Eelco Dolstra 2012-09-19 17:33:42 -04:00
parent 00092b2d35
commit 20582e9ae3
3 changed files with 21 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
perl/lib/Nix/Manifest.pm
View file

@ -68,7 +68,7 @@ sub readManifest_ {
my $manifestVersion = 2; my $manifestVersion = 2;
my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType); my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType);
my ($narHash, $narSize, $references, $deriver, $copyFrom, $system); my ($narHash, $narSize, $references, $deriver, $copyFrom, $system, $compressionType);
while (<MANIFEST>) { while (<MANIFEST>) {
chomp; chomp;
@ -93,6 +93,7 @@ sub readManifest_ {
undef $system; undef $system;
$references = ""; $references = "";
$deriver = ""; $deriver = "";
$compressionType = "bzip2";
} }
} else { } else {
@ -107,6 +108,7 @@ sub readManifest_ {
, references => $references , references => $references
, deriver => $deriver , deriver => $deriver
, system => $system , system => $system
, compressionType => $compressionType
}); });
} }
@ -125,6 +127,7 @@ sub readManifest_ {
elsif (/^\s*CopyFrom:\s*(\/\S+)\s*$/) { $copyFrom = $1; } elsif (/^\s*CopyFrom:\s*(\/\S+)\s*$/) { $copyFrom = $1; }
elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; } elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; }
elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; } elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; }
elsif (/^\s*Compression:\s*(\S+)\s*$/) { $compressionType = $1; }
elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; } elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; }
elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; } elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; }
elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; } elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
@ -172,6 +175,7 @@ sub writeManifest {
print MANIFEST "{\n"; print MANIFEST "{\n";
print MANIFEST " StorePath: $storePath\n"; print MANIFEST " StorePath: $storePath\n";
print MANIFEST " NarURL: $narFile->{url}\n"; print MANIFEST " NarURL: $narFile->{url}\n";
print MANIFEST " Compression: $narFile->{compressionType}\n";
print MANIFEST " Hash: $narFile->{hash}\n" if defined $narFile->{hash}; print MANIFEST " Hash: $narFile->{hash}\n" if defined $narFile->{hash};
print MANIFEST " Size: $narFile->{size}\n" if defined $narFile->{size}; print MANIFEST " Size: $narFile->{size}\n" if defined $narFile->{size};
print MANIFEST " NarHash: $narFile->{narHash}\n"; print MANIFEST " NarHash: $narFile->{narHash}\n";
@ -225,7 +229,8 @@ sub updateManifestDB {
mkpath($manifestDir); mkpath($manifestDir);
my $dbPath = "$manifestDir/cache.sqlite"; unlink "$manifestDir/cache.sqlite"; # remove obsolete cache
my $dbPath = "$manifestDir/cache-v2.sqlite";
# Open/create the database. # Open/create the database.
our $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "") our $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "")
@ -252,6 +257,7 @@ EOF
manifest integer not null, manifest integer not null,
storePath text not null, storePath text not null,
url text not null, url text not null,
compressionType text not null,
hash text, hash text,
size integer, size integer,
narHash text, narHash text,
@ -292,8 +298,8 @@ EOF
flock(MAINLOCK, LOCK_EX) or die; flock(MAINLOCK, LOCK_EX) or die;
our $insertNAR = $dbh->prepare( our $insertNAR = $dbh->prepare(
"insert into NARs(manifest, storePath, url, hash, size, narHash, " . "insert into NARs(manifest, storePath, url, compressionType, hash, size, narHash, " .
"narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die; "narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die;
our $insertPatch = $dbh->prepare( our $insertPatch = $dbh->prepare(
"insert into Patches(manifest, storePath, basePath, baseHash, url, hash, " . "insert into Patches(manifest, storePath, basePath, baseHash, url, hash, " .
@ -327,8 +333,8 @@ EOF
sub addNARToDB { sub addNARToDB {
my ($storePath, $narFile) = @_; my ($storePath, $narFile) = @_;
$insertNAR->execute( $insertNAR->execute(
$id, $storePath, $narFile->{url}, $narFile->{hash}, $narFile->{size}, $id, $storePath, $narFile->{url}, $narFile->{compressionType}, $narFile->{hash},
$narFile->{narHash}, $narFile->{narSize}, $narFile->{references}, $narFile->{size}, $narFile->{narHash}, $narFile->{narSize}, $narFile->{references},
$narFile->{deriver}, $narFile->{system}); $narFile->{deriver}, $narFile->{system});
}; };

10
scripts/download-using-manifests.pl.in
View file

@ -339,13 +339,19 @@ while (scalar @path > 0) {
print LOGFILE "$$ narfile $narFile->{url} $size $v\n"; print LOGFILE "$$ narfile $narFile->{url} $size $v\n";
Nix::Utils::checkURL $narFile->{url}; Nix::Utils::checkURL $narFile->{url};
my $decompressor =
$narFile->{compressionType} eq "bzip2" ? "$Nix::Config::bzip2 -d" :
$narFile->{compressionType} eq "xz" ? "$Nix::Config::xz -d" :
die "unknown compression type `$narFile->{compressionType}'";
if ($curStep < $maxStep) { if ($curStep < $maxStep) {
# The archive will be used a base to a patch. # The archive will be used a base to a patch.
system("$curl '$narFile->{url}' | $Nix::Config::bzip2 -d > $tmpNar") == 0 system("$curl '$narFile->{url}' | $decompressor > $tmpNar") == 0
or die "cannot download and unpack `$narFile->{url}' into `$v'\n"; or die "cannot download and unpack `$narFile->{url}' into `$v'\n";
} else { } else {
# Unpack the archive into the target path. # Unpack the archive into the target path.
system("$curl '$narFile->{url}' | $Nix::Config::bzip2 -d | $Nix::Config::binDir/nix-store --restore '$v'") == 0 system("$curl '$narFile->{url}' | $decompressor | $Nix::Config::binDir/nix-store --restore '$v'") == 0
or die "cannot download and unpack `$narFile->{url}' into `$v'\n"; or die "cannot download and unpack `$narFile->{url}' into `$v'\n";
} }

1
scripts/nix-push.in
View file

@ -227,6 +227,7 @@ for (my $n = 0; $n < scalar @storePaths; $n++) {
{ url => "$archivesURL/$narName" { url => "$archivesURL/$narName"
, hash => "sha256:$compressedHash" , hash => "sha256:$compressedHash"
, size => $compressedSize , size => $compressedSize
, compressionType => $compressionType
, narHash => "$narHash" , narHash => "$narHash"
, narSize => $narSize , narSize => $narSize
, references => join(" ", @{$refs}) , references => join(" ", @{$refs})