Remove manifest support
Manifests have been superseded by binary caches for years. This also gets rid of nix-pull, nix-generate-patches and bsdiff/bspatch.
This commit is contained in:
Eelco Dolstra
parent
af4fb6ef61
commit
867967265b
37 changed files with 20 additions and 2143 deletions
8
.gitignore
vendored
8
.gitignore
vendored
|
|
@ -34,7 +34,6 @@ Makefile.config
|
||||||
|
|
||||||
# /scripts/
|
# /scripts/
|
||||||
/scripts/nix-profile.sh
|
/scripts/nix-profile.sh
|
||||||
/scripts/nix-pull
|
|
||||||
/scripts/nix-push
|
/scripts/nix-push
|
||||||
/scripts/nix-switch
|
/scripts/nix-switch
|
||||||
/scripts/nix-collect-garbage
|
/scripts/nix-collect-garbage
|
||||||
|
|
@ -43,11 +42,8 @@ Makefile.config
|
||||||
/scripts/nix-channel
|
/scripts/nix-channel
|
||||||
/scripts/nix-build
|
/scripts/nix-build
|
||||||
/scripts/nix-copy-closure
|
/scripts/nix-copy-closure
|
||||||
/scripts/nix-generate-patches
|
|
||||||
/scripts/NixConfig.pm
|
/scripts/NixConfig.pm
|
||||||
/scripts/NixManifest.pm
|
/scripts/NixManifest.pm
|
||||||
/scripts/GeneratePatches.pm
|
|
||||||
/scripts/download-using-manifests.pl
|
|
||||||
/scripts/copy-from-other-stores.pl
|
/scripts/copy-from-other-stores.pl
|
||||||
/scripts/download-from-binary-cache.pl
|
/scripts/download-from-binary-cache.pl
|
||||||
/scripts/find-runtime-roots.pl
|
/scripts/find-runtime-roots.pl
|
||||||
|
|
@ -55,10 +51,6 @@ Makefile.config
|
||||||
/scripts/nix-reduce-build
|
/scripts/nix-reduce-build
|
||||||
/scripts/nix-http-export.cgi
|
/scripts/nix-http-export.cgi
|
||||||
|
|
||||||
# /src/bsdiff-4.3/
|
|
||||||
/src/bsdiff-4.3/bsdiff
|
|
||||||
/src/bsdiff-4.3/bspatch
|
|
||||||
|
|
||||||
# /src/libexpr/
|
# /src/libexpr/
|
||||||
/src/libexpr/lexer-tab.cc
|
/src/libexpr/lexer-tab.cc
|
||||||
/src/libexpr/lexer-tab.hh
|
/src/libexpr/lexer-tab.hh
|
||||||
|
|
|
||||||
1
Makefile
1
Makefile
|
|
@ -14,7 +14,6 @@ makefiles = \
|
||||||
src/download-via-ssh/local.mk \
|
src/download-via-ssh/local.mk \
|
||||||
src/nix-log2xml/local.mk \
|
src/nix-log2xml/local.mk \
|
||||||
src/nix-prefetch-url/local.mk \
|
src/nix-prefetch-url/local.mk \
|
||||||
src/bsdiff-4.3/local.mk \
|
|
||||||
perl/local.mk \
|
perl/local.mk \
|
||||||
scripts/local.mk \
|
scripts/local.mk \
|
||||||
corepkgs/local.mk \
|
corepkgs/local.mk \
|
||||||
|
|
|
||||||
|
|
@ -435,18 +435,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para>
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
|
|
||||||
<varlistentry><term><literal>force-manifest</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>If this option is set to <literal>false</literal>
|
|
||||||
(default) and a Nix channel provides both a manifest and a binary
|
|
||||||
cache, only the binary cache will be used. If set to
|
|
||||||
<literal>true</literal>, the manifest will be fetched as well.
|
|
||||||
This is useful if you want to use binary patches (which are
|
|
||||||
currently not supported by binary caches).</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
|
|
||||||
<varlistentry><term><literal>system</literal></term>
|
<varlistentry><term><literal>system</literal></term>
|
||||||
|
|
||||||
<listitem><para>This option specifies the canonical Nix system
|
<listitem><para>This option specifies the canonical Nix system
|
||||||
|
|
|
||||||
|
|
@ -73,11 +73,10 @@ condition="manual">See also <xref linkend="sec-channels"
|
||||||
|
|
||||||
<listitem><para>Downloads the Nix expressions of all subscribed
|
<listitem><para>Downloads the Nix expressions of all subscribed
|
||||||
channels (or only those included in
|
channels (or only those included in
|
||||||
<replaceable>names</replaceable> if specified), makes them the
|
<replaceable>names</replaceable> if specified) and makes them the
|
||||||
default for <command>nix-env</command> operations (by symlinking
|
default for <command>nix-env</command> operations (by symlinking
|
||||||
them from the directory <filename>~/.nix-defexpr</filename>), and
|
them from the directory
|
||||||
performs a <command>nix-pull</command> on the manifests of all
|
<filename>~/.nix-defexpr</filename>).</para></listitem>
|
||||||
channels to make pre-built binaries available.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
|
|
@ -187,16 +186,6 @@ following files:</para>
|
||||||
|
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
<varlistentry><term><filename>MANIFEST.bz2</filename></term>
|
|
||||||
|
|
||||||
<listitem><para>(Deprecated in favour of binary caches.) A
|
|
||||||
manifest as created by <command>nix-push</command>. Only used if
|
|
||||||
<filename>binary-cache-url</filename> is not present or if the
|
|
||||||
<filename>nix.conf</filename> option
|
|
||||||
<option>force-manifest</option> is set.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
</variablelist>
|
</variablelist>
|
||||||
|
|
||||||
</refsection>
|
</refsection>
|
||||||
|
|
|
||||||
|
|
@ -1,44 +0,0 @@
|
||||||
<refentry xmlns="http://docbook.org/ns/docbook"
|
|
||||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
|
||||||
xmlns:xi="http://www.w3.org/2001/XInclude"
|
|
||||||
version="5.0"
|
|
||||||
xml:id="sec-nix-generate-patches">
|
|
||||||
|
|
||||||
<refmeta>
|
|
||||||
<refentrytitle>nix-generate-patches</refentrytitle>
|
|
||||||
<manvolnum>1</manvolnum>
|
|
||||||
<refmiscinfo class="source">Nix</refmiscinfo>
|
|
||||||
<refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
|
|
||||||
</refmeta>
|
|
||||||
|
|
||||||
<refnamediv>
|
|
||||||
<refname>nix-generate-patches</refname>
|
|
||||||
<refpurpose>generates binary patches between NAR files</refpurpose>
|
|
||||||
</refnamediv>
|
|
||||||
|
|
||||||
<refsynopsisdiv>
|
|
||||||
<cmdsynopsis>
|
|
||||||
<command>nix-generate-patches</command>
|
|
||||||
<arg choice='plain'><replaceable>NAR-DIR</replaceable></arg>
|
|
||||||
<arg choice='plain'><replaceable>PATCH-DIR</replaceable></arg>
|
|
||||||
<arg choice='plain'><replaceable>PATCH-URI</replaceable></arg>
|
|
||||||
<arg choice='plain'><replaceable>OLD-MANIFEST</replaceable></arg>
|
|
||||||
<arg choice='plain'><replaceable>NEW-MANIFEST</replaceable></arg>
|
|
||||||
</cmdsynopsis>
|
|
||||||
</refsynopsisdiv>
|
|
||||||
|
|
||||||
|
|
||||||
<refsection><title>Description</title>
|
|
||||||
|
|
||||||
<para>The command <command>nix-generate-patches</command> generates
|
|
||||||
binary patches between NAR files listed in OLD-MANIFEST and NEW-MANIFEST.
|
|
||||||
The patches are written to the directory PATCH-DIR, and the prefix
|
|
||||||
PATCH-URI is used to generate URIs for the patches. The patches are
|
|
||||||
added to NEW-MANIFEST. All NARs are required to exist in NAR-DIR.
|
|
||||||
Patches are generated between succeeding versions of packages with
|
|
||||||
the same name.</para>
|
|
||||||
|
|
||||||
</refsection>
|
|
||||||
|
|
||||||
|
|
||||||
</refentry>
|
|
||||||
|
|
@ -146,9 +146,7 @@ The elements are as follows:
|
||||||
|
|
||||||
<varlistentry><term><replaceable>manifestURL</replaceable></term>
|
<varlistentry><term><replaceable>manifestURL</replaceable></term>
|
||||||
|
|
||||||
<listitem><para>The manifest to be pulled by
|
<listitem><para>Obsolete.</para></listitem>
|
||||||
<command>nix-pull</command>. The manifest must contain
|
|
||||||
<replaceable>outPath</replaceable>.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,54 +0,0 @@
|
||||||
<refentry xmlns="http://docbook.org/ns/docbook"
|
|
||||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
|
||||||
xmlns:xi="http://www.w3.org/2001/XInclude"
|
|
||||||
version="5.0"
|
|
||||||
xml:id="sec-nix-pull">
|
|
||||||
|
|
||||||
<refmeta>
|
|
||||||
<refentrytitle>nix-pull</refentrytitle>
|
|
||||||
<manvolnum>1</manvolnum>
|
|
||||||
<refmiscinfo class="source">Nix</refmiscinfo>
|
|
||||||
<refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
|
|
||||||
</refmeta>
|
|
||||||
|
|
||||||
<refnamediv>
|
|
||||||
<refname>nix-pull</refname>
|
|
||||||
<refpurpose>register availability of pre-built binaries (deprecated)</refpurpose>
|
|
||||||
</refnamediv>
|
|
||||||
|
|
||||||
<refsynopsisdiv>
|
|
||||||
<cmdsynopsis>
|
|
||||||
<command>nix-pull</command>
|
|
||||||
<arg choice='plain'><replaceable>url</replaceable></arg>
|
|
||||||
</cmdsynopsis>
|
|
||||||
</refsynopsisdiv>
|
|
||||||
|
|
||||||
|
|
||||||
<refsection><title>Description</title>
|
|
||||||
|
|
||||||
<note><para>This command and the use of manifests is deprecated. It is
|
|
||||||
better to use binary caches.</para></note>
|
|
||||||
|
|
||||||
<para>The command <command>nix-pull</command> obtains a list of
|
|
||||||
pre-built store paths from the URL <replaceable>url</replaceable>, and
|
|
||||||
for each of these store paths, registers a substitute derivation that
|
|
||||||
downloads and unpacks it into the Nix store. This is used to speed up
|
|
||||||
installations: if you attempt to install something that has already
|
|
||||||
been built and stored into the network cache, Nix can transparently
|
|
||||||
re-use the pre-built store paths.</para>
|
|
||||||
|
|
||||||
<para>The file at <replaceable>url</replaceable> must be compatible
|
|
||||||
with the files created by <replaceable>nix-push</replaceable>.</para>
|
|
||||||
|
|
||||||
</refsection>
|
|
||||||
|
|
||||||
|
|
||||||
<refsection><title>Examples</title>
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-pull https://nixos.org/releases/nixpkgs/nixpkgs-15.05pre54468.69858d7/MANIFEST</screen>
|
|
||||||
|
|
||||||
</refsection>
|
|
||||||
|
|
||||||
|
|
||||||
</refentry>
|
|
||||||
|
|
@ -73,8 +73,7 @@ automatically.</para>
|
||||||
<listitem><para>Optionally, a single <emphasis>manifest</emphasis>
|
<listitem><para>Optionally, a single <emphasis>manifest</emphasis>
|
||||||
file is created that contains the same metadata as the
|
file is created that contains the same metadata as the
|
||||||
<filename>.narinfo</filename> files. This is for compatibility with
|
<filename>.narinfo</filename> files. This is for compatibility with
|
||||||
Nix versions prior to 1.2 (see <command>nix-pull</command> for
|
Nix versions prior to 1.2.</para></listitem>
|
||||||
details).</para></listitem>
|
|
||||||
|
|
||||||
<listitem><para>A file named <option>nix-cache-info</option> is
|
<listitem><para>A file named <option>nix-cache-info</option> is
|
||||||
placed in the destination directory. The existence of this file
|
placed in the destination directory. The existence of this file
|
||||||
|
|
@ -135,7 +134,7 @@ automatically.</para>
|
||||||
<varlistentry><term><option>--manifest</option></term>
|
<varlistentry><term><option>--manifest</option></term>
|
||||||
|
|
||||||
<listitem><para>Force the generation of a manifest suitable for
|
<listitem><para>Force the generation of a manifest suitable for
|
||||||
use by <command>nix-pull</command>. The manifest is stored as
|
use by old versions of Nix. The manifest is stored as
|
||||||
<filename><replaceable>dest-dir</replaceable>/MANIFEST</filename>.</para></listitem>
|
<filename><replaceable>dest-dir</replaceable>/MANIFEST</filename>.</para></listitem>
|
||||||
|
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
@ -203,20 +202,6 @@ $ nix-push --dest /tmp/cache $(nix-instantiate -A thunderbird)
|
||||||
|
|
||||||
</para>
|
</para>
|
||||||
|
|
||||||
<para>To generate a manifest suitable for <command>nix-pull</command>:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-push --dest /tmp/cache $(nix-build -A thunderbird) --manifest
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
On another machine you can then do:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-pull http://example.org/cache
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
to cause the binaries to be used by subsequent Nix operations.</para>
|
|
||||||
|
|
||||||
<para>To generate a signed binary cache, you must first generate a key
|
<para>To generate a signed binary cache, you must first generate a key
|
||||||
pair, in this example called <literal>cache.example.org-1</literal>,
|
pair, in this example called <literal>cache.example.org-1</literal>,
|
||||||
storing the secret key in <filename>./sk</filename> and the public key
|
storing the secret key in <filename>./sk</filename> and the public key
|
||||||
|
|
|
||||||
|
|
@ -13,14 +13,10 @@ work with Nix.</para>
|
||||||
<xi:include href="nix-collect-garbage.xml" />
|
<xi:include href="nix-collect-garbage.xml" />
|
||||||
<xi:include href="nix-copy-closure.xml" />
|
<xi:include href="nix-copy-closure.xml" />
|
||||||
<xi:include href="nix-daemon.xml" />
|
<xi:include href="nix-daemon.xml" />
|
||||||
<!--
|
|
||||||
<xi:include href="nix-generate-patches.xml" />
|
|
||||||
-->
|
|
||||||
<xi:include href="nix-hash.xml" />
|
<xi:include href="nix-hash.xml" />
|
||||||
<xi:include href="nix-install-package.xml" />
|
<xi:include href="nix-install-package.xml" />
|
||||||
<xi:include href="nix-instantiate.xml" />
|
<xi:include href="nix-instantiate.xml" />
|
||||||
<xi:include href="nix-prefetch-url.xml" />
|
<xi:include href="nix-prefetch-url.xml" />
|
||||||
<xi:include href="nix-pull.xml" />
|
|
||||||
<xi:include href="nix-push.xml" />
|
<xi:include href="nix-push.xml" />
|
||||||
|
|
||||||
</chapter>
|
</chapter>
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,7 @@ dist-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid
|
||||||
# Generate man pages.
|
# Generate man pages.
|
||||||
man-pages := $(foreach n, \
|
man-pages := $(foreach n, \
|
||||||
nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
|
nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
|
||||||
nix-collect-garbage.1 nix-push.1 nix-pull.1 \
|
nix-collect-garbage.1 nix-push.1 \
|
||||||
nix-prefetch-url.1 nix-channel.1 \
|
nix-prefetch-url.1 nix-channel.1 \
|
||||||
nix-install-package.1 nix-hash.1 nix-copy-closure.1 \
|
nix-install-package.1 nix-hash.1 nix-copy-closure.1 \
|
||||||
nix.conf.5 nix-daemon.8, \
|
nix.conf.5 nix-daemon.8, \
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,6 @@ $version = "@PACKAGE_VERSION@";
|
||||||
$binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
|
$binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@";
|
||||||
$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@";
|
$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@";
|
||||||
$stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix";
|
$stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix";
|
||||||
$manifestDir = $ENV{"NIX_MANIFESTS_DIR"} || "@localstatedir@/nix/manifests";
|
|
||||||
$logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix";
|
$logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix";
|
||||||
$confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix";
|
$confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix";
|
||||||
$storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@";
|
$storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@";
|
||||||
|
|
|
||||||
|
|
@ -1,340 +0,0 @@
|
||||||
package Nix::GeneratePatches;
|
|
||||||
|
|
||||||
use strict;
|
|
||||||
use File::Temp qw(tempdir);
|
|
||||||
use File::stat;
|
|
||||||
use Nix::Config;
|
|
||||||
use Nix::Manifest;
|
|
||||||
|
|
||||||
our @ISA = qw(Exporter);
|
|
||||||
our @EXPORT = qw(generatePatches propagatePatches copyPatches);
|
|
||||||
|
|
||||||
|
|
||||||
# Some patch generations options.
|
|
||||||
|
|
||||||
# Max size of NAR archives to generate patches for.
|
|
||||||
my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"};
|
|
||||||
$maxNarSize = 160 * 1024 * 1024 if !defined $maxNarSize;
|
|
||||||
|
|
||||||
# If patch is bigger than this fraction of full archive, reject.
|
|
||||||
my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"};
|
|
||||||
$maxPatchFraction = 0.60 if !defined $maxPatchFraction;
|
|
||||||
|
|
||||||
my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"};
|
|
||||||
$timeLimit = 180 if !defined $timeLimit;
|
|
||||||
|
|
||||||
my $hashAlgo = "sha256";
|
|
||||||
|
|
||||||
|
|
||||||
sub findOutputPaths {
|
|
||||||
my $narFiles = shift;
|
|
||||||
|
|
||||||
my %outPaths;
|
|
||||||
|
|
||||||
foreach my $p (keys %{$narFiles}) {
|
|
||||||
|
|
||||||
# Ignore derivations.
|
|
||||||
next if ($p =~ /\.drv$/);
|
|
||||||
|
|
||||||
# Ignore builders (too much ambiguity -- they're all called
|
|
||||||
# `builder.sh').
|
|
||||||
next if ($p =~ /\.sh$/);
|
|
||||||
next if ($p =~ /\.patch$/);
|
|
||||||
|
|
||||||
# Don't bother including tar files etc.
|
|
||||||
next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/);
|
|
||||||
|
|
||||||
$outPaths{$p} = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
return %outPaths;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
sub getNameVersion {
|
|
||||||
my $p = shift;
|
|
||||||
$p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
|
|
||||||
my $name = $1;
|
|
||||||
my $version = $2;
|
|
||||||
return undef unless defined $name && defined $version;
|
|
||||||
$name =~ s/^-//;
|
|
||||||
$version =~ s/^-//;
|
|
||||||
return ($name, $version);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# A quick hack to get a measure of the `distance' between two
|
|
||||||
# versions: it's just the position of the first character that differs
|
|
||||||
# (or 999 if they are the same).
|
|
||||||
sub versionDiff {
|
|
||||||
my $s = shift;
|
|
||||||
my $t = shift;
|
|
||||||
my $i;
|
|
||||||
return 999 if $s eq $t;
|
|
||||||
for ($i = 0; $i < length $s; $i++) {
|
|
||||||
return $i if $i >= length $t or
|
|
||||||
substr($s, $i, 1) ne substr($t, $i, 1);
|
|
||||||
}
|
|
||||||
return $i;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
sub getNarBz2 {
|
|
||||||
my $narPath = shift;
|
|
||||||
my $narFiles = shift;
|
|
||||||
my $storePath = shift;
|
|
||||||
|
|
||||||
my $narFileList = $$narFiles{$storePath};
|
|
||||||
die "missing path $storePath" unless defined $narFileList;
|
|
||||||
|
|
||||||
my $narFile = @{$narFileList}[0];
|
|
||||||
die unless defined $narFile;
|
|
||||||
|
|
||||||
$narFile->{url} =~ /\/([^\/]+)$/;
|
|
||||||
die unless defined $1;
|
|
||||||
return "$narPath/$1";
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
sub containsPatch {
|
|
||||||
my $patches = shift;
|
|
||||||
my $storePath = shift;
|
|
||||||
my $basePath = shift;
|
|
||||||
my $patchList = $$patches{$storePath};
|
|
||||||
return 0 if !defined $patchList;
|
|
||||||
my $found = 0;
|
|
||||||
foreach my $patch (@{$patchList}) {
|
|
||||||
# !!! baseHash might differ
|
|
||||||
return 1 if $patch->{basePath} eq $basePath;
|
|
||||||
}
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
sub generatePatches {
|
|
||||||
my ($srcNarFiles, $dstNarFiles, $srcPatches, $dstPatches, $narPath, $patchesPath, $patchesURL, $tmpDir) = @_;
|
|
||||||
|
|
||||||
my %srcOutPaths = findOutputPaths $srcNarFiles;
|
|
||||||
my %dstOutPaths = findOutputPaths $dstNarFiles;
|
|
||||||
|
|
||||||
# For each output path in the destination, see if we need to / can
|
|
||||||
# create a patch.
|
|
||||||
|
|
||||||
print STDERR "creating patches...\n";
|
|
||||||
|
|
||||||
foreach my $p (keys %dstOutPaths) {
|
|
||||||
|
|
||||||
# If exactly the same path already exists in the source, skip it.
|
|
||||||
next if defined $srcOutPaths{$p};
|
|
||||||
|
|
||||||
print " $p\n";
|
|
||||||
|
|
||||||
# If not, then we should find the paths in the source that are
|
|
||||||
# `most' likely to be present on a system that wants to
|
|
||||||
# install this path.
|
|
||||||
|
|
||||||
(my $name, my $version) = getNameVersion $p;
|
|
||||||
next unless defined $name && defined $version;
|
|
||||||
|
|
||||||
my @closest = ();
|
|
||||||
my $closestVersion;
|
|
||||||
my $minDist = -1; # actually, larger means closer
|
|
||||||
|
|
||||||
# Find all source paths with the same name.
|
|
||||||
|
|
||||||
foreach my $q (keys %srcOutPaths) {
|
|
||||||
(my $name2, my $version2) = getNameVersion $q;
|
|
||||||
next unless defined $name2 && defined $version2;
|
|
||||||
|
|
||||||
if ($name eq $name2) {
|
|
||||||
|
|
||||||
my $srcSystem = @{$$dstNarFiles{$p}}[0]->{system};
|
|
||||||
my $dstSystem = @{$$srcNarFiles{$q}}[0]->{system};
|
|
||||||
if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) {
|
|
||||||
print " SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n";
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
|
|
||||||
# If the sizes differ too much, then skip. This
|
|
||||||
# disambiguates between, e.g., a real component and a
|
|
||||||
# wrapper component (cf. Firefox in Nixpkgs).
|
|
||||||
my $srcSize = @{$$srcNarFiles{$q}}[0]->{size};
|
|
||||||
my $dstSize = @{$$dstNarFiles{$p}}[0]->{size};
|
|
||||||
my $ratio = $srcSize / $dstSize;
|
|
||||||
$ratio = 1 / $ratio if $ratio < 1;
|
|
||||||
# print " SIZE $srcSize $dstSize $ratio $q\n";
|
|
||||||
|
|
||||||
if ($ratio >= 3) {
|
|
||||||
print " SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n";
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
|
|
||||||
# If there are multiple matching names, include the
|
|
||||||
# ones with the closest version numbers.
|
|
||||||
my $dist = versionDiff $version, $version2;
|
|
||||||
if ($dist > $minDist) {
|
|
||||||
$minDist = $dist;
|
|
||||||
@closest = ($q);
|
|
||||||
$closestVersion = $version2;
|
|
||||||
} elsif ($dist == $minDist) {
|
|
||||||
push @closest, $q;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (scalar(@closest) == 0) {
|
|
||||||
print " NO BASE: $p\n";
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach my $closest (@closest) {
|
|
||||||
|
|
||||||
# Generate a patch between $closest and $p.
|
|
||||||
print STDERR " $p <- $closest\n";
|
|
||||||
|
|
||||||
# If the patch already exists, skip it.
|
|
||||||
if (containsPatch($srcPatches, $p, $closest) ||
|
|
||||||
containsPatch($dstPatches, $p, $closest))
|
|
||||||
{
|
|
||||||
print " skipping, already exists\n";
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
|
|
||||||
my $srcNarBz2 = getNarBz2 $narPath, $srcNarFiles, $closest;
|
|
||||||
my $dstNarBz2 = getNarBz2 $narPath, $dstNarFiles, $p;
|
|
||||||
|
|
||||||
if (! -f $srcNarBz2) {
|
|
||||||
warn "patch source archive $srcNarBz2 is missing\n";
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
|
|
||||||
system("$Nix::Config::bzip2 -d < $srcNarBz2 > $tmpDir/A") == 0
|
|
||||||
or die "cannot unpack $srcNarBz2";
|
|
||||||
|
|
||||||
if (stat("$tmpDir/A")->size >= $maxNarSize) {
|
|
||||||
print " skipping, source is too large\n";
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
|
|
||||||
system("$Nix::Config::bzip2 -d < $dstNarBz2 > $tmpDir/B") == 0
|
|
||||||
or die "cannot unpack $dstNarBz2";
|
|
||||||
|
|
||||||
if (stat("$tmpDir/B")->size >= $maxNarSize) {
|
|
||||||
print " skipping, destination is too large\n";
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
|
|
||||||
my $time1 = time();
|
|
||||||
my $res = system("ulimit -t $timeLimit; $Nix::Config::libexecDir/nix/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF");
|
|
||||||
my $time2 = time();
|
|
||||||
if ($res) {
|
|
||||||
warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n";
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
|
|
||||||
my $baseHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die;
|
|
||||||
chomp $baseHash;
|
|
||||||
|
|
||||||
my $narHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die;
|
|
||||||
chomp $narHash;
|
|
||||||
|
|
||||||
my $narDiffHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die;
|
|
||||||
chomp $narDiffHash;
|
|
||||||
|
|
||||||
my $narDiffSize = stat("$tmpDir/DIFF")->size;
|
|
||||||
my $dstNarBz2Size = stat($dstNarBz2)->size;
|
|
||||||
|
|
||||||
print " size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n";
|
|
||||||
|
|
||||||
if ($narDiffSize >= $dstNarBz2Size) {
|
|
||||||
print " rejecting; patch bigger than full archive\n";
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) {
|
|
||||||
print " rejecting; patch too large relative to full archive\n";
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
|
|
||||||
my $finalName = "$narDiffHash.nar-bsdiff";
|
|
||||||
|
|
||||||
if (-e "$patchesPath/$finalName") {
|
|
||||||
print " not copying, already exists\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
else {
|
|
||||||
system("cp '$tmpDir/DIFF' '$patchesPath/$finalName.tmp'") == 0
|
|
||||||
or die "cannot copy diff";
|
|
||||||
rename("$patchesPath/$finalName.tmp", "$patchesPath/$finalName")
|
|
||||||
or die "cannot rename $patchesPath/$finalName.tmp";
|
|
||||||
}
|
|
||||||
|
|
||||||
# Add the patch to the manifest.
|
|
||||||
addPatch $dstPatches, $p,
|
|
||||||
{ url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
|
|
||||||
, size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
|
|
||||||
, narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Propagate useful patches from $srcPatches to $dstPatches. A patch
|
|
||||||
# is useful if it produces either paths in the $dstNarFiles or paths
|
|
||||||
# that can be used as the base for other useful patches.
|
|
||||||
sub propagatePatches {
|
|
||||||
my ($srcPatches, $dstNarFiles, $dstPatches) = @_;
|
|
||||||
|
|
||||||
print STDERR "propagating patches...\n";
|
|
||||||
|
|
||||||
my $changed;
|
|
||||||
do {
|
|
||||||
# !!! we repeat this to reach the transitive closure; inefficient
|
|
||||||
$changed = 0;
|
|
||||||
|
|
||||||
print STDERR "loop\n";
|
|
||||||
|
|
||||||
my %dstBasePaths;
|
|
||||||
foreach my $q (keys %{$dstPatches}) {
|
|
||||||
foreach my $patch (@{$$dstPatches{$q}}) {
|
|
||||||
$dstBasePaths{$patch->{basePath}} = 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach my $p (keys %{$srcPatches}) {
|
|
||||||
my $patchList = $$srcPatches{$p};
|
|
||||||
|
|
||||||
my $include = 0;
|
|
||||||
|
|
||||||
# Is path $p included in the destination? If so, include
|
|
||||||
# patches that produce it.
|
|
||||||
$include = 1 if defined $$dstNarFiles{$p};
|
|
||||||
|
|
||||||
# Is path $p a path that serves as a base for paths in the
|
|
||||||
# destination? If so, include patches that produce it.
|
|
||||||
# !!! check baseHash
|
|
||||||
$include = 1 if defined $dstBasePaths{$p};
|
|
||||||
|
|
||||||
if ($include) {
|
|
||||||
foreach my $patch (@{$patchList}) {
|
|
||||||
$changed = 1 if addPatch $dstPatches, $p, $patch;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
} while $changed;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Add all new patches in $srcPatches to $dstPatches.
|
|
||||||
sub copyPatches {
|
|
||||||
my ($srcPatches, $dstPatches) = @_;
|
|
||||||
foreach my $p (keys %{$srcPatches}) {
|
|
||||||
addPatch $dstPatches, $p, $_ foreach @{$$srcPatches{$p}};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
return 1;
|
|
||||||
|
|
@ -13,7 +13,7 @@ use Nix::Config;
|
||||||
use Nix::Store;
|
use Nix::Store;
|
||||||
|
|
||||||
our @ISA = qw(Exporter);
|
our @ISA = qw(Exporter);
|
||||||
our @EXPORT = qw(readManifest writeManifest updateManifestDB addPatch deleteOldManifests parseNARInfo fingerprintPath);
|
our @EXPORT = qw(readManifest writeManifest addPatch parseNARInfo fingerprintPath);
|
||||||
|
|
||||||
|
|
||||||
sub addNAR {
|
sub addNAR {
|
||||||
|
|
@ -228,172 +228,6 @@ sub writeManifest {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
sub updateManifestDB {
|
|
||||||
my $manifestDir = $Nix::Config::manifestDir;
|
|
||||||
|
|
||||||
my @manifests = glob "$manifestDir/*.nixmanifest";
|
|
||||||
return undef if scalar @manifests == 0;
|
|
||||||
|
|
||||||
mkpath($manifestDir);
|
|
||||||
|
|
||||||
unlink "$manifestDir/cache.sqlite"; # remove obsolete cache
|
|
||||||
my $dbPath = "$manifestDir/cache-v2.sqlite";
|
|
||||||
|
|
||||||
# Open/create the database.
|
|
||||||
our $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "")
|
|
||||||
or die "cannot open database ‘$dbPath’";
|
|
||||||
$dbh->{RaiseError} = 1;
|
|
||||||
$dbh->{PrintError} = 0;
|
|
||||||
|
|
||||||
$dbh->do("pragma foreign_keys = on");
|
|
||||||
$dbh->do("pragma synchronous = off"); # we can always reproduce the cache
|
|
||||||
$dbh->do("pragma journal_mode = truncate");
|
|
||||||
|
|
||||||
# Initialise the database schema, if necessary.
|
|
||||||
$dbh->do(<<EOF);
|
|
||||||
create table if not exists Manifests (
|
|
||||||
id integer primary key autoincrement not null,
|
|
||||||
path text unique not null,
|
|
||||||
timestamp integer not null
|
|
||||||
);
|
|
||||||
EOF
|
|
||||||
|
|
||||||
$dbh->do(<<EOF);
|
|
||||||
create table if not exists NARs (
|
|
||||||
id integer primary key autoincrement not null,
|
|
||||||
manifest integer not null,
|
|
||||||
storePath text not null,
|
|
||||||
url text not null,
|
|
||||||
compressionType text not null,
|
|
||||||
hash text,
|
|
||||||
size integer,
|
|
||||||
narHash text,
|
|
||||||
narSize integer,
|
|
||||||
refs text,
|
|
||||||
deriver text,
|
|
||||||
system text,
|
|
||||||
foreign key (manifest) references Manifests(id) on delete cascade
|
|
||||||
);
|
|
||||||
EOF
|
|
||||||
|
|
||||||
$dbh->do("create index if not exists NARs_storePath on NARs(storePath)");
|
|
||||||
|
|
||||||
$dbh->do(<<EOF);
|
|
||||||
create table if not exists Patches (
|
|
||||||
id integer primary key autoincrement not null,
|
|
||||||
manifest integer not null,
|
|
||||||
storePath text not null,
|
|
||||||
basePath text not null,
|
|
||||||
baseHash text not null,
|
|
||||||
url text not null,
|
|
||||||
hash text,
|
|
||||||
size integer,
|
|
||||||
narHash text,
|
|
||||||
narSize integer,
|
|
||||||
patchType text not null,
|
|
||||||
foreign key (manifest) references Manifests(id) on delete cascade
|
|
||||||
);
|
|
||||||
EOF
|
|
||||||
|
|
||||||
$dbh->do("create index if not exists Patches_storePath on Patches(storePath)");
|
|
||||||
|
|
||||||
# Acquire an exclusive lock to ensure that only one process
|
|
||||||
# updates the DB at the same time. This isn't really necessary,
|
|
||||||
# but it prevents work duplication and lock contention in SQLite.
|
|
||||||
my $lockFile = "$manifestDir/cache.lock";
|
|
||||||
open MAINLOCK, ">>$lockFile" or die "unable to acquire lock ‘$lockFile’: $!\n";
|
|
||||||
flock(MAINLOCK, LOCK_EX) or die;
|
|
||||||
|
|
||||||
our $insertNAR = $dbh->prepare(
|
|
||||||
"insert into NARs(manifest, storePath, url, compressionType, hash, size, narHash, " .
|
|
||||||
"narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die;
|
|
||||||
|
|
||||||
our $insertPatch = $dbh->prepare(
|
|
||||||
"insert into Patches(manifest, storePath, basePath, baseHash, url, hash, " .
|
|
||||||
"size, narHash, narSize, patchType) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
|
|
||||||
|
|
||||||
$dbh->begin_work;
|
|
||||||
|
|
||||||
# Read each manifest in $manifestDir and add it to the database,
|
|
||||||
# unless we've already done so on a previous run.
|
|
||||||
my %seen;
|
|
||||||
|
|
||||||
for my $manifestLink (@manifests) {
|
|
||||||
my $manifest = Cwd::abs_path($manifestLink);
|
|
||||||
next unless -f $manifest;
|
|
||||||
my $timestamp = lstat($manifest)->mtime;
|
|
||||||
$seen{$manifest} = 1;
|
|
||||||
|
|
||||||
next if scalar @{$dbh->selectcol_arrayref(
|
|
||||||
"select 1 from Manifests where path = ? and timestamp = ?",
|
|
||||||
{}, $manifest, $timestamp)} == 1;
|
|
||||||
|
|
||||||
print STDERR "caching $manifest...\n";
|
|
||||||
|
|
||||||
$dbh->do("delete from Manifests where path = ?", {}, $manifest);
|
|
||||||
|
|
||||||
$dbh->do("insert into Manifests(path, timestamp) values (?, ?)",
|
|
||||||
{}, $manifest, $timestamp);
|
|
||||||
|
|
||||||
our $id = $dbh->last_insert_id("", "", "", "");
|
|
||||||
|
|
||||||
sub addNARToDB {
|
|
||||||
my ($storePath, $narFile) = @_;
|
|
||||||
$insertNAR->execute(
|
|
||||||
$id, $storePath, $narFile->{url}, $narFile->{compressionType}, $narFile->{hash},
|
|
||||||
$narFile->{size}, $narFile->{narHash}, $narFile->{narSize}, $narFile->{references},
|
|
||||||
$narFile->{deriver}, $narFile->{system});
|
|
||||||
};
|
|
||||||
|
|
||||||
sub addPatchToDB {
|
|
||||||
my ($storePath, $patch) = @_;
|
|
||||||
$insertPatch->execute(
|
|
||||||
$id, $storePath, $patch->{basePath}, $patch->{baseHash}, $patch->{url},
|
|
||||||
$patch->{hash}, $patch->{size}, $patch->{narHash}, $patch->{narSize},
|
|
||||||
$patch->{patchType});
|
|
||||||
};
|
|
||||||
|
|
||||||
my $version = readManifest_($manifest, \&addNARToDB, \&addPatchToDB);
|
|
||||||
|
|
||||||
if ($version < 3) {
|
|
||||||
die "you have an old-style or corrupt manifest ‘$manifestLink’; please delete it\n";
|
|
||||||
}
|
|
||||||
if ($version >= 10) {
|
|
||||||
die "manifest ‘$manifestLink’ is too new; please delete it or upgrade Nix\n";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Removed cached information for removed manifests from the DB.
|
|
||||||
foreach my $manifest (@{$dbh->selectcol_arrayref("select path from Manifests")}) {
|
|
||||||
next if defined $seen{$manifest};
|
|
||||||
$dbh->do("delete from Manifests where path = ?", {}, $manifest);
|
|
||||||
}
|
|
||||||
|
|
||||||
$dbh->commit;
|
|
||||||
|
|
||||||
close MAINLOCK;
|
|
||||||
|
|
||||||
return $dbh;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Delete all old manifests downloaded from a given URL.
|
|
||||||
sub deleteOldManifests {
|
|
||||||
my ($url, $curUrlFile) = @_;
|
|
||||||
for my $urlFile (glob "$Nix::Config::manifestDir/*.url") {
|
|
||||||
next if defined $curUrlFile && $urlFile eq $curUrlFile;
|
|
||||||
open URL, "<$urlFile" or die;
|
|
||||||
my $url2 = <URL>;
|
|
||||||
chomp $url2;
|
|
||||||
close URL;
|
|
||||||
next unless $url eq $url2;
|
|
||||||
my $base = $urlFile; $base =~ s/.url$//;
|
|
||||||
unlink "${base}.url";
|
|
||||||
unlink "${base}.nixmanifest";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Return a fingerprint of a store path to be used in binary cache
|
# Return a fingerprint of a store path to be used in binary cache
|
||||||
# signatures. It contains the store path, the base-32 SHA-256 hash of
|
# signatures. It contains the store path, the base-32 SHA-256 hash of
|
||||||
# the contents of the path, and the references.
|
# the contents of the path, and the references.
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
nix_perl_sources := \
|
nix_perl_sources := \
|
||||||
$(d)/lib/Nix/Store.pm \
|
$(d)/lib/Nix/Store.pm \
|
||||||
$(d)/lib/Nix/Manifest.pm \
|
$(d)/lib/Nix/Manifest.pm \
|
||||||
$(d)/lib/Nix/GeneratePatches.pm \
|
|
||||||
$(d)/lib/Nix/SSH.pm \
|
$(d)/lib/Nix/SSH.pm \
|
||||||
$(d)/lib/Nix/CopyClosure.pm \
|
$(d)/lib/Nix/CopyClosure.pm \
|
||||||
$(d)/lib/Nix/Config.pm.in \
|
$(d)/lib/Nix/Config.pm.in \
|
||||||
|
|
|
||||||
|
|
@ -1,376 +0,0 @@
|
||||||
#! @perl@ -w @perlFlags@
|
|
||||||
|
|
||||||
use utf8;
|
|
||||||
use strict;
|
|
||||||
use Nix::Config;
|
|
||||||
use Nix::Manifest;
|
|
||||||
use Nix::Store;
|
|
||||||
use Nix::Utils;
|
|
||||||
use POSIX qw(strftime);
|
|
||||||
|
|
||||||
STDOUT->autoflush(1);
|
|
||||||
binmode STDERR, ":encoding(utf8)";
|
|
||||||
|
|
||||||
my $logFile = "$Nix::Config::logDir/downloads";
|
|
||||||
|
|
||||||
# For queries, skip expensive calls to nix-hash etc. We're just
|
|
||||||
# estimating the expected download size.
|
|
||||||
my $fast = 1;
|
|
||||||
|
|
||||||
my $curl = "$Nix::Config::curl --fail --location";
|
|
||||||
|
|
||||||
|
|
||||||
# Open the manifest cache and update it if necessary.
|
|
||||||
my $dbh = updateManifestDB();
|
|
||||||
exit 0 unless defined $dbh; # exit if there are no manifests
|
|
||||||
print "\n";
|
|
||||||
|
|
||||||
|
|
||||||
# $hashCache->{$algo}->{$path} yields the $algo-hash of $path.
|
|
||||||
my $hashCache;
|
|
||||||
|
|
||||||
|
|
||||||
sub parseHash {
|
|
||||||
my $hash = shift;
|
|
||||||
if ($hash =~ /^(.+):(.+)$/) {
|
|
||||||
return ($1, $2);
|
|
||||||
} else {
|
|
||||||
return ("md5", $hash);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Compute the most efficient sequence of downloads to produce the
|
|
||||||
# given path.
|
|
||||||
sub computeSmallestDownload {
|
|
||||||
my $targetPath = shift;
|
|
||||||
|
|
||||||
# Build a graph of all store paths that might contribute to the
|
|
||||||
# construction of $targetPath, and the special node "start". The
|
|
||||||
# edges are either patch operations, or downloads of full NAR
|
|
||||||
# files. The latter edges only occur between "start" and a store
|
|
||||||
# path.
|
|
||||||
my %graph;
|
|
||||||
|
|
||||||
$graph{"start"} = {d => 0, pred => undef, edges => []};
|
|
||||||
|
|
||||||
my @queue = ();
|
|
||||||
my $queueFront = 0;
|
|
||||||
my %done;
|
|
||||||
|
|
||||||
sub addNode {
|
|
||||||
my $graph = shift;
|
|
||||||
my $u = shift;
|
|
||||||
$$graph{$u} = {d => 999999999999, pred => undef, edges => []}
|
|
||||||
unless defined $$graph{$u};
|
|
||||||
}
|
|
||||||
|
|
||||||
sub addEdge {
|
|
||||||
my $graph = shift;
|
|
||||||
my $u = shift;
|
|
||||||
my $v = shift;
|
|
||||||
my $w = shift;
|
|
||||||
my $type = shift;
|
|
||||||
my $info = shift;
|
|
||||||
addNode $graph, $u;
|
|
||||||
push @{$$graph{$u}->{edges}},
|
|
||||||
{weight => $w, start => $u, end => $v, type => $type, info => $info};
|
|
||||||
my $n = scalar @{$$graph{$u}->{edges}};
|
|
||||||
}
|
|
||||||
|
|
||||||
push @queue, $targetPath;
|
|
||||||
|
|
||||||
while ($queueFront < scalar @queue) {
|
|
||||||
my $u = $queue[$queueFront++];
|
|
||||||
next if defined $done{$u};
|
|
||||||
$done{$u} = 1;
|
|
||||||
|
|
||||||
addNode \%graph, $u;
|
|
||||||
|
|
||||||
# If the path already exists, it has distance 0 from the
|
|
||||||
# "start" node.
|
|
||||||
if (isValidPath($u)) {
|
|
||||||
addEdge \%graph, "start", $u, 0, "present", undef;
|
|
||||||
}
|
|
||||||
|
|
||||||
else {
|
|
||||||
|
|
||||||
# Add patch edges.
|
|
||||||
my $patchList = $dbh->selectall_arrayref(
|
|
||||||
"select * from Patches where storePath = ?",
|
|
||||||
{ Slice => {} }, $u);
|
|
||||||
|
|
||||||
foreach my $patch (@{$patchList}) {
|
|
||||||
if (isValidPath($patch->{basePath})) {
|
|
||||||
my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
|
|
||||||
|
|
||||||
my $hash = $hashCache->{$baseHashAlgo}->{$patch->{basePath}};
|
|
||||||
if (!defined $hash) {
|
|
||||||
$hash = $fast && $baseHashAlgo eq "sha256"
|
|
||||||
? queryPathHash($patch->{basePath})
|
|
||||||
: hashPath($baseHashAlgo, $baseHashAlgo ne "md5", $patch->{basePath});
|
|
||||||
$hash =~ s/.*://;
|
|
||||||
$hashCache->{$baseHashAlgo}->{$patch->{basePath}} = $hash;
|
|
||||||
}
|
|
||||||
|
|
||||||
next if $hash ne $baseHash;
|
|
||||||
}
|
|
||||||
push @queue, $patch->{basePath};
|
|
||||||
addEdge \%graph, $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
|
|
||||||
}
|
|
||||||
|
|
||||||
# Add NAR file edges to the start node.
|
|
||||||
my $narFileList = $dbh->selectall_arrayref(
|
|
||||||
"select * from NARs where storePath = ?",
|
|
||||||
{ Slice => {} }, $u);
|
|
||||||
|
|
||||||
foreach my $narFile (@{$narFileList}) {
|
|
||||||
# !!! how to handle files whose size is not known in advance?
|
|
||||||
# For now, assume some arbitrary size (1 GB).
|
|
||||||
# This has the side-effect of preferring non-Hydra downloads.
|
|
||||||
addEdge \%graph, "start", $u, ($narFile->{size} || 1000000000), "narfile", $narFile;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Run Dijkstra's shortest path algorithm to determine the shortest
|
|
||||||
# sequence of download and/or patch actions that will produce
|
|
||||||
# $targetPath.
|
|
||||||
|
|
||||||
my @todo = keys %graph;
|
|
||||||
|
|
||||||
while (scalar @todo > 0) {
|
|
||||||
|
|
||||||
# Remove the closest element from the todo list.
|
|
||||||
# !!! inefficient, use a priority queue
|
|
||||||
@todo = sort { -($graph{$a}->{d} <=> $graph{$b}->{d}) } @todo;
|
|
||||||
my $u = pop @todo;
|
|
||||||
|
|
||||||
my $u_ = $graph{$u};
|
|
||||||
|
|
||||||
foreach my $edge (@{$u_->{edges}}) {
|
|
||||||
my $v_ = $graph{$edge->{end}};
|
|
||||||
if ($v_->{d} > $u_->{d} + $edge->{weight}) {
|
|
||||||
$v_->{d} = $u_->{d} + $edge->{weight};
|
|
||||||
# Store the edge; to edge->start is actually the
|
|
||||||
# predecessor.
|
|
||||||
$v_->{pred} = $edge;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Retrieve the shortest path from "start" to $targetPath.
|
|
||||||
my @path = ();
|
|
||||||
my $cur = $targetPath;
|
|
||||||
return () unless defined $graph{$targetPath}->{pred};
|
|
||||||
while ($cur ne "start") {
|
|
||||||
push @path, $graph{$cur}->{pred};
|
|
||||||
$cur = $graph{$cur}->{pred}->{start};
|
|
||||||
}
|
|
||||||
|
|
||||||
return @path;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Parse the arguments.
|
|
||||||
|
|
||||||
if ($ARGV[0] eq "--query") {
|
|
||||||
|
|
||||||
while (<STDIN>) {
|
|
||||||
chomp;
|
|
||||||
my ($cmd, @args) = split " ", $_;
|
|
||||||
|
|
||||||
if ($cmd eq "have") {
|
|
||||||
foreach my $storePath (@args) {
|
|
||||||
print "$storePath\n" if scalar @{$dbh->selectcol_arrayref("select 1 from NARs where storePath = ?", {}, $storePath)} > 0;
|
|
||||||
}
|
|
||||||
print "\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
elsif ($cmd eq "info") {
|
|
||||||
foreach my $storePath (@args) {
|
|
||||||
|
|
||||||
my $infos = $dbh->selectall_arrayref(
|
|
||||||
"select * from NARs where storePath = ?",
|
|
||||||
{ Slice => {} }, $storePath);
|
|
||||||
|
|
||||||
next unless scalar @{$infos} > 0;
|
|
||||||
my $info = @{$infos}[0];
|
|
||||||
|
|
||||||
print "$storePath\n";
|
|
||||||
print "$info->{deriver}\n";
|
|
||||||
my @references = split " ", $info->{refs};
|
|
||||||
print scalar @references, "\n";
|
|
||||||
print "$_\n" foreach @references;
|
|
||||||
|
|
||||||
my @path = computeSmallestDownload $storePath;
|
|
||||||
|
|
||||||
my $downloadSize = 0;
|
|
||||||
while (scalar @path > 0) {
|
|
||||||
my $edge = pop @path;
|
|
||||||
my $u = $edge->{start};
|
|
||||||
my $v = $edge->{end};
|
|
||||||
if ($edge->{type} eq "patch") {
|
|
||||||
$downloadSize += $edge->{info}->{size} || 0;
|
|
||||||
}
|
|
||||||
elsif ($edge->{type} eq "narfile") {
|
|
||||||
$downloadSize += $edge->{info}->{size} || 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
print "$downloadSize\n";
|
|
||||||
|
|
||||||
my $narSize = $info->{narSize} || 0;
|
|
||||||
print "$narSize\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
print "\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
else { die "unknown command ‘$cmd’"; }
|
|
||||||
}
|
|
||||||
|
|
||||||
exit 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
elsif ($ARGV[0] ne "--substitute") {
|
|
||||||
die;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
die unless scalar @ARGV == 3;
|
|
||||||
my $targetPath = $ARGV[1];
|
|
||||||
my $destPath = $ARGV[2];
|
|
||||||
$fast = 0;
|
|
||||||
|
|
||||||
|
|
||||||
# Create a temporary directory.
|
|
||||||
my $tmpDir = mkTempDir("nix-download");
|
|
||||||
|
|
||||||
my $tmpNar = "$tmpDir/nar";
|
|
||||||
my $tmpNar2 = "$tmpDir/nar2";
|
|
||||||
|
|
||||||
|
|
||||||
open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
|
|
||||||
|
|
||||||
my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
|
|
||||||
print LOGFILE "$$ get $targetPath $date\n";
|
|
||||||
|
|
||||||
print STDERR "\n*** Trying to download/patch ‘$targetPath’\n";
|
|
||||||
|
|
||||||
|
|
||||||
# Compute the shortest path.
|
|
||||||
my @path = computeSmallestDownload $targetPath;
|
|
||||||
die "don't know how to produce $targetPath\n" if scalar @path == 0;
|
|
||||||
|
|
||||||
|
|
||||||
# We don't need the manifest anymore, so close it as an optimisation:
|
|
||||||
# if we still have SQLite locks blocking other processes (we
|
|
||||||
# shouldn't), this gets rid of them.
|
|
||||||
$dbh->disconnect;
|
|
||||||
|
|
||||||
|
|
||||||
# Traverse the shortest path, perform the actions described by the
|
|
||||||
# edges.
|
|
||||||
my $curStep = 1;
|
|
||||||
my $maxStep = scalar @path;
|
|
||||||
|
|
||||||
my $finalNarHash;
|
|
||||||
|
|
||||||
while (scalar @path > 0) {
|
|
||||||
my $edge = pop @path;
|
|
||||||
my $u = $edge->{start};
|
|
||||||
my $v = $edge->{end};
|
|
||||||
|
|
||||||
print STDERR "\n*** Step $curStep/$maxStep: ";
|
|
||||||
|
|
||||||
if ($edge->{type} eq "present") {
|
|
||||||
print STDERR "using already present path ‘$v’\n";
|
|
||||||
print LOGFILE "$$ present $v\n";
|
|
||||||
|
|
||||||
if ($curStep < $maxStep) {
|
|
||||||
# Since this is not the last step, the path will be used
|
|
||||||
# as a base to one or more patches. So turn the base path
|
|
||||||
# into a NAR archive, to which we can apply the patch.
|
|
||||||
print STDERR " packing base path...\n";
|
|
||||||
system("$Nix::Config::binDir/nix-store --dump $v > $tmpNar") == 0
|
|
||||||
or die "cannot dump ‘$v’";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
elsif ($edge->{type} eq "patch") {
|
|
||||||
my $patch = $edge->{info};
|
|
||||||
print STDERR "applying patch ‘$patch->{url}’ to ‘$u’ to create ‘$v’\n";
|
|
||||||
|
|
||||||
print LOGFILE "$$ patch $patch->{url} $patch->{size} $patch->{baseHash} $u $v\n";
|
|
||||||
|
|
||||||
# Download the patch.
|
|
||||||
print STDERR " downloading patch...\n";
|
|
||||||
my $patchPath = "$tmpDir/patch";
|
|
||||||
checkURL $patch->{url};
|
|
||||||
system("$curl '$patch->{url}' -o $patchPath") == 0
|
|
||||||
or die "cannot download patch ‘$patch->{url}’\n";
|
|
||||||
|
|
||||||
# Apply the patch to the NAR archive produced in step 1 (for
|
|
||||||
# the already present path) or a later step (for patch sequences).
|
|
||||||
print STDERR " applying patch...\n";
|
|
||||||
system("$Nix::Config::libexecDir/nix/bspatch $tmpNar $tmpNar2 $patchPath") == 0
|
|
||||||
or die "cannot apply patch ‘$patchPath’ to $tmpNar\n";
|
|
||||||
|
|
||||||
if ($curStep < $maxStep) {
|
|
||||||
# The archive will be used as the base of the next patch.
|
|
||||||
rename "$tmpNar2", "$tmpNar" or die "cannot rename NAR archive: $!";
|
|
||||||
} else {
|
|
||||||
# This was the last patch. Unpack the final NAR archive
|
|
||||||
# into the target path.
|
|
||||||
print STDERR " unpacking patched archive...\n";
|
|
||||||
system("$Nix::Config::binDir/nix-store --restore $destPath < $tmpNar2") == 0
|
|
||||||
or die "cannot unpack $tmpNar2 to ‘$v’\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
$finalNarHash = $patch->{narHash};
|
|
||||||
}
|
|
||||||
|
|
||||||
elsif ($edge->{type} eq "narfile") {
|
|
||||||
my $narFile = $edge->{info};
|
|
||||||
print STDERR "downloading ‘$narFile->{url}’ to ‘$v’\n";
|
|
||||||
|
|
||||||
my $size = $narFile->{size} || -1;
|
|
||||||
print LOGFILE "$$ narfile $narFile->{url} $size $v\n";
|
|
||||||
|
|
||||||
checkURL $narFile->{url};
|
|
||||||
|
|
||||||
my $decompressor =
|
|
||||||
$narFile->{compressionType} eq "bzip2" ? "| $Nix::Config::bzip2 -d" :
|
|
||||||
$narFile->{compressionType} eq "xz" ? "| $Nix::Config::xz -d" :
|
|
||||||
$narFile->{compressionType} eq "none" ? "" :
|
|
||||||
die "unknown compression type ‘$narFile->{compressionType}’";
|
|
||||||
|
|
||||||
if ($curStep < $maxStep) {
|
|
||||||
# The archive will be used a base to a patch.
|
|
||||||
system("$curl '$narFile->{url}' $decompressor > $tmpNar") == 0
|
|
||||||
or die "cannot download and unpack ‘$narFile->{url}’ to ‘$v’\n";
|
|
||||||
} else {
|
|
||||||
# Unpack the archive to the target path.
|
|
||||||
system("$curl '$narFile->{url}' $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0
|
|
||||||
or die "cannot download and unpack ‘$narFile->{url}’ to ‘$v’\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
$finalNarHash = $narFile->{narHash};
|
|
||||||
}
|
|
||||||
|
|
||||||
$curStep++;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Tell Nix about the expected hash so it can verify it.
|
|
||||||
die "cannot check integrity of the downloaded path since its hash is not known\n"
|
|
||||||
unless defined $finalNarHash;
|
|
||||||
print "$finalNarHash\n";
|
|
||||||
|
|
||||||
|
|
||||||
print STDERR "\n";
|
|
||||||
print LOGFILE "$$ success\n";
|
|
||||||
close LOGFILE;
|
|
||||||
|
|
@ -2,17 +2,14 @@ nix_bin_scripts := \
|
||||||
$(d)/nix-build \
|
$(d)/nix-build \
|
||||||
$(d)/nix-channel \
|
$(d)/nix-channel \
|
||||||
$(d)/nix-copy-closure \
|
$(d)/nix-copy-closure \
|
||||||
$(d)/nix-generate-patches \
|
|
||||||
$(d)/nix-install-package \
|
$(d)/nix-install-package \
|
||||||
$(d)/nix-pull \
|
|
||||||
$(d)/nix-push
|
$(d)/nix-push
|
||||||
|
|
||||||
bin-scripts += $(nix_bin_scripts)
|
bin-scripts += $(nix_bin_scripts)
|
||||||
|
|
||||||
nix_substituters := \
|
nix_substituters := \
|
||||||
$(d)/copy-from-other-stores.pl \
|
$(d)/copy-from-other-stores.pl \
|
||||||
$(d)/download-from-binary-cache.pl \
|
$(d)/download-from-binary-cache.pl
|
||||||
$(d)/download-using-manifests.pl
|
|
||||||
|
|
||||||
nix_noinst_scripts := \
|
nix_noinst_scripts := \
|
||||||
$(d)/build-remote.pl \
|
$(d)/build-remote.pl \
|
||||||
|
|
|
||||||
|
|
@ -12,8 +12,6 @@ binmode STDERR, ":encoding(utf8)";
|
||||||
|
|
||||||
Nix::Config::readConfig;
|
Nix::Config::readConfig;
|
||||||
|
|
||||||
my $manifestDir = $Nix::Config::manifestDir;
|
|
||||||
|
|
||||||
|
|
||||||
# Turn on caching in nix-prefetch-url.
|
# Turn on caching in nix-prefetch-url.
|
||||||
my $channelCache = "$Nix::Config::stateDir/channel-cache";
|
my $channelCache = "$Nix::Config::stateDir/channel-cache";
|
||||||
|
|
@ -75,7 +73,6 @@ sub removeChannel {
|
||||||
my ($name) = @_;
|
my ($name) = @_;
|
||||||
readChannels;
|
readChannels;
|
||||||
my $url = $channels{$name};
|
my $url = $channels{$name};
|
||||||
deleteOldManifests($url . "/MANIFEST", undef) if defined $url;
|
|
||||||
delete $channels{$name};
|
delete $channels{$name};
|
||||||
writeChannels;
|
writeChannels;
|
||||||
|
|
||||||
|
|
@ -84,8 +81,7 @@ sub removeChannel {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# Fetch Nix expressions and pull manifests from the subscribed
|
# Fetch Nix expressions and binary cache URLs from the subscribed channels.
|
||||||
# channels.
|
|
||||||
sub update {
|
sub update {
|
||||||
my @channelNames = @_;
|
my @channelNames = @_;
|
||||||
|
|
||||||
|
|
@ -97,7 +93,6 @@ sub update {
|
||||||
next if scalar @channelNames > 0 && ! grep { $_ eq $name } @{channelNames};
|
next if scalar @channelNames > 0 && ! grep { $_ eq $name } @{channelNames};
|
||||||
|
|
||||||
my $url = $channels{$name};
|
my $url = $channels{$name};
|
||||||
my $origUrl = "$url/MANIFEST";
|
|
||||||
|
|
||||||
# We want to download the url to a file to see if it's a tarball while also checking if we
|
# We want to download the url to a file to see if it's a tarball while also checking if we
|
||||||
# got redirected in the process, so that we can grab the various parts of a nix channel
|
# got redirected in the process, so that we can grab the various parts of a nix channel
|
||||||
|
|
@ -132,22 +127,8 @@ sub update {
|
||||||
if ($ret != 0) {
|
if ($ret != 0) {
|
||||||
# Check if the channel advertises a binary cache.
|
# Check if the channel advertises a binary cache.
|
||||||
my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`;
|
my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`;
|
||||||
my $getManifest = ($Nix::Config::config{"force-manifest"} // "false") eq "true";
|
$extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; "
|
||||||
if ($? == 0 && $binaryCacheURL ne "") {
|
if $? == 0 && $binaryCacheURL ne "";
|
||||||
$extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; ";
|
|
||||||
deleteOldManifests($origUrl, undef);
|
|
||||||
} else {
|
|
||||||
$getManifest = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($getManifest) {
|
|
||||||
# No binary cache, so pull the channel manifest.
|
|
||||||
mkdir $manifestDir, 0755 unless -e $manifestDir;
|
|
||||||
die "$0: you do not have write permission to ‘$manifestDir’!\n" unless -W $manifestDir;
|
|
||||||
$ENV{'NIX_ORIG_URL'} = $origUrl;
|
|
||||||
system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0
|
|
||||||
or die "cannot pull manifest from ‘$url’\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
# Download the channel tarball.
|
# Download the channel tarball.
|
||||||
my $fullURL = "$url/nixexprs.tar.xz";
|
my $fullURL = "$url/nixexprs.tar.xz";
|
||||||
|
|
|
||||||
|
|
@ -1,51 +0,0 @@
|
||||||
#! @perl@ -w @perlFlags@
|
|
||||||
|
|
||||||
use strict;
|
|
||||||
use Nix::Manifest;
|
|
||||||
use Nix::GeneratePatches;
|
|
||||||
use Nix::Utils;
|
|
||||||
|
|
||||||
if (scalar @ARGV != 5) {
|
|
||||||
print STDERR <<EOF;
|
|
||||||
Usage: nix-generate-patches NAR-DIR PATCH-DIR PATCH-URI OLD-MANIFEST NEW-MANIFEST
|
|
||||||
|
|
||||||
This command generates binary patches between NAR files listed in
|
|
||||||
OLD-MANIFEST and NEW-MANIFEST. The patches are written to the
|
|
||||||
directory PATCH-DIR, and the prefix PATCH-URI is used to generate URIs
|
|
||||||
for the patches. The patches are added to NEW-MANIFEST. All NARs are
|
|
||||||
required to exist in NAR-DIR. Patches are generated between
|
|
||||||
succeeding versions of packages with the same name.
|
|
||||||
EOF
|
|
||||||
exit 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
my $narPath = $ARGV[0];
|
|
||||||
my $patchesPath = $ARGV[1];
|
|
||||||
my $patchesURL = $ARGV[2];
|
|
||||||
my $srcManifest = $ARGV[3];
|
|
||||||
my $dstManifest = $ARGV[4];
|
|
||||||
|
|
||||||
my (%srcNarFiles, %srcLocalPaths, %srcPatches);
|
|
||||||
readManifest $srcManifest, \%srcNarFiles, \%srcPatches;
|
|
||||||
|
|
||||||
my (%dstNarFiles, %dstLocalPaths, %dstPatches);
|
|
||||||
readManifest $dstManifest, \%dstNarFiles, \%dstPatches;
|
|
||||||
|
|
||||||
my $tmpDir = mkTempDir("nix-generate-patches");
|
|
||||||
|
|
||||||
generatePatches \%srcNarFiles, \%dstNarFiles, \%srcPatches, \%dstPatches,
|
|
||||||
$narPath, $patchesPath, $patchesURL, $tmpDir;
|
|
||||||
|
|
||||||
propagatePatches \%srcPatches, \%dstNarFiles, \%dstPatches;
|
|
||||||
|
|
||||||
# Optionally add all new patches to the manifest in $NIX_ALL_PATCHES.
|
|
||||||
my $allPatchesFile = $ENV{"NIX_ALL_PATCHES"};
|
|
||||||
if (defined $allPatchesFile) {
|
|
||||||
my (%dummy, %allPatches);
|
|
||||||
readManifest("$patchesPath/all-patches", \%dummy, \%allPatches)
|
|
||||||
if -f $allPatchesFile;
|
|
||||||
copyPatches \%dstPatches, \%allPatches;
|
|
||||||
writeManifest($allPatchesFile, {}, \%allPatches, 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
writeManifest $dstManifest, \%dstNarFiles, \%dstPatches;
|
|
||||||
|
|
@ -89,7 +89,7 @@ my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )";
|
||||||
# store path. We'll let nix-env do that.
|
# store path. We'll let nix-env do that.
|
||||||
|
|
||||||
$contents =~
|
$contents =~
|
||||||
/ ^ \s* (\S+) \s+ ($Nix::Utils::urlRE) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )? /x
|
/ ^ \s* (\S+) \s+ (\S+) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )? /x
|
||||||
or barf "invalid package contents";
|
or barf "invalid package contents";
|
||||||
my $version = $1;
|
my $version = $1;
|
||||||
my $manifestURL = $2;
|
my $manifestURL = $2;
|
||||||
|
|
@ -111,25 +111,9 @@ if ($interactive) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
if (defined $binaryCacheURL) {
|
die "$0: package does not supply a binary cache\n" unless defined $binaryCacheURL;
|
||||||
|
|
||||||
push @extraNixEnvArgs, "--option", "extra-binary-caches", $binaryCacheURL;
|
push @extraNixEnvArgs, "--option", "extra-binary-caches", $binaryCacheURL;
|
||||||
|
|
||||||
} else {
|
|
||||||
|
|
||||||
# Store the manifest in the temporary directory so that we don't
|
|
||||||
# pollute /nix/var/nix/manifests. This also requires that we
|
|
||||||
# don't use the Nix daemon (because otherwise
|
|
||||||
# download-using-manifests won't see our NIX_MANIFESTS_DIRS
|
|
||||||
# environment variable).
|
|
||||||
$ENV{NIX_MANIFESTS_DIR} = $tmpDir;
|
|
||||||
$ENV{NIX_REMOTE} = "";
|
|
||||||
|
|
||||||
print "\nPulling manifests...\n";
|
|
||||||
system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0
|
|
||||||
or barf "nix-pull failed: $?";
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
print "\nInstalling package...\n";
|
print "\nInstalling package...\n";
|
||||||
|
|
|
||||||
|
|
@ -1,102 +0,0 @@
|
||||||
#! @perl@ -w @perlFlags@
|
|
||||||
|
|
||||||
use utf8;
|
|
||||||
use strict;
|
|
||||||
use Nix::Config;
|
|
||||||
use Nix::Manifest;
|
|
||||||
|
|
||||||
binmode STDERR, ":encoding(utf8)";
|
|
||||||
|
|
||||||
my $manifestDir = $Nix::Config::manifestDir;
|
|
||||||
|
|
||||||
|
|
||||||
# Prevent access problems in shared-stored installations.
|
|
||||||
umask 0022;
|
|
||||||
|
|
||||||
|
|
||||||
# Create the manifests directory if it doesn't exist.
|
|
||||||
if (! -e $manifestDir) {
|
|
||||||
mkdir $manifestDir, 0755 or die "cannot create directory ‘$manifestDir’";
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Make sure that the manifests directory is scanned for GC roots.
|
|
||||||
my $gcRootsDir = "$Nix::Config::stateDir/gcroots";
|
|
||||||
my $manifestDirLink = "$gcRootsDir/manifests";
|
|
||||||
if (! -l $manifestDirLink) {
|
|
||||||
symlink($manifestDir, $manifestDirLink) or die "cannot create symlink ‘$manifestDirLink’";
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Process the URLs specified on the command line.
|
|
||||||
|
|
||||||
sub downloadFile {
|
|
||||||
my $url = shift;
|
|
||||||
$ENV{"PRINT_PATH"} = 1;
|
|
||||||
$ENV{"QUIET"} = 1;
|
|
||||||
my ($dummy, $path) = `$Nix::Config::binDir/nix-prefetch-url '$url'`;
|
|
||||||
die "cannot fetch ‘$url’" if $? != 0;
|
|
||||||
die "nix-prefetch-url did not return a path" unless defined $path;
|
|
||||||
chomp $path;
|
|
||||||
return $path;
|
|
||||||
}
|
|
||||||
|
|
||||||
sub processURL {
|
|
||||||
my $url = shift;
|
|
||||||
|
|
||||||
$url =~ s/\/$//;
|
|
||||||
|
|
||||||
my $manifest;
|
|
||||||
|
|
||||||
my $origUrl = $ENV{'NIX_ORIG_URL'} || $url;
|
|
||||||
|
|
||||||
# First see if a bzipped manifest is available.
|
|
||||||
if (system("$Nix::Config::curl --fail --silent --location --head '$url'.bz2 > /dev/null") == 0) {
|
|
||||||
print "fetching list of Nix archives at ‘$url.bz2’...\n";
|
|
||||||
$manifest = downloadFile "$url.bz2";
|
|
||||||
}
|
|
||||||
|
|
||||||
# Otherwise, just get the uncompressed manifest.
|
|
||||||
else {
|
|
||||||
print "fetching list of Nix archives at ‘$url’...\n";
|
|
||||||
$manifest = downloadFile $url;
|
|
||||||
}
|
|
||||||
|
|
||||||
my $baseName = "unnamed";
|
|
||||||
if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component
|
|
||||||
$baseName = $1;
|
|
||||||
}
|
|
||||||
|
|
||||||
my $hash = `$Nix::Config::binDir/nix-hash --flat '$manifest'`
|
|
||||||
or die "cannot hash ‘$manifest’";
|
|
||||||
chomp $hash;
|
|
||||||
|
|
||||||
my $urlFile = "$manifestDir/$baseName-$hash.url";
|
|
||||||
open URL, ">$urlFile" or die "cannot create ‘$urlFile’";
|
|
||||||
print URL $origUrl;
|
|
||||||
close URL;
|
|
||||||
|
|
||||||
my $finalPath = "$manifestDir/$baseName-$hash.nixmanifest";
|
|
||||||
|
|
||||||
unlink $finalPath if -e $finalPath;
|
|
||||||
|
|
||||||
symlink("$manifest", "$finalPath")
|
|
||||||
or die "cannot link ‘$finalPath’ to ‘$manifest’";
|
|
||||||
|
|
||||||
deleteOldManifests($origUrl, $urlFile);
|
|
||||||
}
|
|
||||||
|
|
||||||
while (@ARGV) {
|
|
||||||
my $url = shift @ARGV;
|
|
||||||
if ($url eq "--help") {
|
|
||||||
exec "man nix-pull" or die;
|
|
||||||
} elsif ($url eq "--skip-wrong-store") {
|
|
||||||
# No-op, no longer supported.
|
|
||||||
} else {
|
|
||||||
processURL $url;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Update the cache.
|
|
||||||
updateManifestDB();
|
|
||||||
|
|
@ -1,63 +0,0 @@
|
||||||
.\"-
|
|
||||||
.\" Copyright 2003-2005 Colin Percival
|
|
||||||
.\" All rights reserved
|
|
||||||
.\"
|
|
||||||
.\" Redistribution and use in source and binary forms, with or without
|
|
||||||
.\" modification, are permitted providing that the following conditions
|
|
||||||
.\" are met:
|
|
||||||
.\" 1. Redistributions of source code must retain the above copyright
|
|
||||||
.\" notice, this list of conditions and the following disclaimer.
|
|
||||||
.\" 2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
.\" notice, this list of conditions and the following disclaimer in the
|
|
||||||
.\" documentation and/or other materials provided with the distribution.
|
|
||||||
.\"
|
|
||||||
.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
||||||
.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
||||||
.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
|
|
||||||
.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
||||||
.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
||||||
.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
|
||||||
.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
|
|
||||||
.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
||||||
.\" POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
.\"
|
|
||||||
.\" $FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.1,v 1.1 2005/08/06 01:59:05 cperciva Exp $
|
|
||||||
.\"
|
|
||||||
.Dd May 18, 2003
|
|
||||||
.Dt BSDIFF 1
|
|
||||||
.Os FreeBSD
|
|
||||||
.Sh NAME
|
|
||||||
.Nm bsdiff
|
|
||||||
.Nd generate a patch between two binary files
|
|
||||||
.Sh SYNOPSIS
|
|
||||||
.Nm
|
|
||||||
.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac
|
|
||||||
.Sh DESCRIPTION
|
|
||||||
.Nm
|
|
||||||
compares
|
|
||||||
.Ao Ar oldfile Ac
|
|
||||||
to
|
|
||||||
.Ao Ar newfile Ac
|
|
||||||
and writes to
|
|
||||||
.Ao Ar patchfile Ac
|
|
||||||
a binary patch suitable for use by bspatch(1).
|
|
||||||
When
|
|
||||||
.Ao Ar oldfile Ac
|
|
||||||
and
|
|
||||||
.Ao Ar newfile Ac
|
|
||||||
are two versions of an executable program, the
|
|
||||||
patches produced are on average a factor of five smaller
|
|
||||||
than those produced by any other binary patch tool known
|
|
||||||
to the author.
|
|
||||||
.Pp
|
|
||||||
.Nm
|
|
||||||
uses memory equal to 17 times the size of
|
|
||||||
.Ao Ar oldfile Ac ,
|
|
||||||
and requires
|
|
||||||
an absolute minimum working set size of 8 times the size of oldfile.
|
|
||||||
.Sh SEE ALSO
|
|
||||||
.Xr bspatch 1
|
|
||||||
.Sh AUTHORS
|
|
||||||
.An Colin Percival Aq cperciva@freebsd.org
|
|
||||||
|
|
@ -1,405 +0,0 @@
|
||||||
/*-
|
|
||||||
* Copyright 2003-2005 Colin Percival
|
|
||||||
* All rights reserved
|
|
||||||
*
|
|
||||||
* Redistribution and use in source and binary forms, with or without
|
|
||||||
* modification, are permitted providing that the following conditions
|
|
||||||
* are met:
|
|
||||||
* 1. Redistributions of source code must retain the above copyright
|
|
||||||
* notice, this list of conditions and the following disclaimer.
|
|
||||||
* 2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
* notice, this list of conditions and the following disclaimer in the
|
|
||||||
* documentation and/or other materials provided with the distribution.
|
|
||||||
*
|
|
||||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
||||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
||||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
|
|
||||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
||||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
||||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
|
||||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
|
|
||||||
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
||||||
* POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#if 0
|
|
||||||
__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05 cperciva Exp $");
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#include <sys/types.h>
|
|
||||||
|
|
||||||
#include <bzlib.h>
|
|
||||||
#include <err.h>
|
|
||||||
#include <fcntl.h>
|
|
||||||
#include <stdio.h>
|
|
||||||
#include <stdlib.h>
|
|
||||||
#include <string.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
|
|
||||||
#define MIN(x,y) (((x)<(y)) ? (x) : (y))
|
|
||||||
|
|
||||||
static void split(off_t *I,off_t *V,off_t start,off_t len,off_t h)
|
|
||||||
{
|
|
||||||
off_t i,j,k,x,tmp,jj,kk;
|
|
||||||
|
|
||||||
if(len<16) {
|
|
||||||
for(k=start;k<start+len;k+=j) {
|
|
||||||
j=1;x=V[I[k]+h];
|
|
||||||
for(i=1;k+i<start+len;i++) {
|
|
||||||
if(V[I[k+i]+h]<x) {
|
|
||||||
x=V[I[k+i]+h];
|
|
||||||
j=0;
|
|
||||||
};
|
|
||||||
if(V[I[k+i]+h]==x) {
|
|
||||||
tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp;
|
|
||||||
j++;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
for(i=0;i<j;i++) V[I[k+i]]=k+j-1;
|
|
||||||
if(j==1) I[k]=-1;
|
|
||||||
};
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
x=V[I[start+len/2]+h];
|
|
||||||
jj=0;kk=0;
|
|
||||||
for(i=start;i<start+len;i++) {
|
|
||||||
if(V[I[i]+h]<x) jj++;
|
|
||||||
if(V[I[i]+h]==x) kk++;
|
|
||||||
};
|
|
||||||
jj+=start;kk+=jj;
|
|
||||||
|
|
||||||
i=start;j=0;k=0;
|
|
||||||
while(i<jj) {
|
|
||||||
if(V[I[i]+h]<x) {
|
|
||||||
i++;
|
|
||||||
} else if(V[I[i]+h]==x) {
|
|
||||||
tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp;
|
|
||||||
j++;
|
|
||||||
} else {
|
|
||||||
tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp;
|
|
||||||
k++;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
while(jj+j<kk) {
|
|
||||||
if(V[I[jj+j]+h]==x) {
|
|
||||||
j++;
|
|
||||||
} else {
|
|
||||||
tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp;
|
|
||||||
k++;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
if(jj>start) split(I,V,start,jj-start,h);
|
|
||||||
|
|
||||||
for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1;
|
|
||||||
if(jj==kk-1) I[jj]=-1;
|
|
||||||
|
|
||||||
if(start+len>kk) split(I,V,kk,start+len-kk,h);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize)
|
|
||||||
{
|
|
||||||
off_t buckets[256];
|
|
||||||
off_t i,h,len;
|
|
||||||
|
|
||||||
for(i=0;i<256;i++) buckets[i]=0;
|
|
||||||
for(i=0;i<oldsize;i++) buckets[old[i]]++;
|
|
||||||
for(i=1;i<256;i++) buckets[i]+=buckets[i-1];
|
|
||||||
for(i=255;i>0;i--) buckets[i]=buckets[i-1];
|
|
||||||
buckets[0]=0;
|
|
||||||
|
|
||||||
for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i;
|
|
||||||
I[0]=oldsize;
|
|
||||||
for(i=0;i<oldsize;i++) V[i]=buckets[old[i]];
|
|
||||||
V[oldsize]=0;
|
|
||||||
for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1;
|
|
||||||
I[0]=-1;
|
|
||||||
|
|
||||||
for(h=1;I[0]!=-(oldsize+1);h+=h) {
|
|
||||||
len=0;
|
|
||||||
for(i=0;i<oldsize+1;) {
|
|
||||||
if(I[i]<0) {
|
|
||||||
len-=I[i];
|
|
||||||
i-=I[i];
|
|
||||||
} else {
|
|
||||||
if(len) I[i-len]=-len;
|
|
||||||
len=V[I[i]]+1-i;
|
|
||||||
split(I,V,i,len,h);
|
|
||||||
i+=len;
|
|
||||||
len=0;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
if(len) I[i-len]=-len;
|
|
||||||
};
|
|
||||||
|
|
||||||
for(i=0;i<oldsize+1;i++) I[V[i]]=i;
|
|
||||||
}
|
|
||||||
|
|
||||||
static off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize)
|
|
||||||
{
|
|
||||||
off_t i;
|
|
||||||
|
|
||||||
for(i=0;(i<oldsize)&&(i<newsize);i++)
|
|
||||||
if(old[i]!=new[i]) break;
|
|
||||||
|
|
||||||
return i;
|
|
||||||
}
|
|
||||||
|
|
||||||
static off_t search(off_t *I,u_char *old,off_t oldsize,
|
|
||||||
u_char *new,off_t newsize,off_t st,off_t en,off_t *pos)
|
|
||||||
{
|
|
||||||
off_t x,y;
|
|
||||||
|
|
||||||
if(en-st<2) {
|
|
||||||
x=matchlen(old+I[st],oldsize-I[st],new,newsize);
|
|
||||||
y=matchlen(old+I[en],oldsize-I[en],new,newsize);
|
|
||||||
|
|
||||||
if(x>y) {
|
|
||||||
*pos=I[st];
|
|
||||||
return x;
|
|
||||||
} else {
|
|
||||||
*pos=I[en];
|
|
||||||
return y;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
x=st+(en-st)/2;
|
|
||||||
if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) {
|
|
||||||
return search(I,old,oldsize,new,newsize,x,en,pos);
|
|
||||||
} else {
|
|
||||||
return search(I,old,oldsize,new,newsize,st,x,pos);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
static void offtout(off_t x,u_char *buf)
|
|
||||||
{
|
|
||||||
off_t y;
|
|
||||||
|
|
||||||
if(x<0) y=-x; else y=x;
|
|
||||||
|
|
||||||
buf[0]=y%256;y-=buf[0];
|
|
||||||
y=y/256;buf[1]=y%256;y-=buf[1];
|
|
||||||
y=y/256;buf[2]=y%256;y-=buf[2];
|
|
||||||
y=y/256;buf[3]=y%256;y-=buf[3];
|
|
||||||
y=y/256;buf[4]=y%256;y-=buf[4];
|
|
||||||
y=y/256;buf[5]=y%256;y-=buf[5];
|
|
||||||
y=y/256;buf[6]=y%256;y-=buf[6];
|
|
||||||
y=y/256;buf[7]=y%256;
|
|
||||||
|
|
||||||
if(x<0) buf[7]|=0x80;
|
|
||||||
}
|
|
||||||
|
|
||||||
int main(int argc,char *argv[])
|
|
||||||
{
|
|
||||||
int fd;
|
|
||||||
u_char *old,*new;
|
|
||||||
off_t oldsize,newsize;
|
|
||||||
off_t *I,*V;
|
|
||||||
off_t scan,pos,len;
|
|
||||||
off_t lastscan,lastpos,lastoffset;
|
|
||||||
off_t oldscore,scsc;
|
|
||||||
off_t s,Sf,lenf,Sb,lenb;
|
|
||||||
off_t overlap,Ss,lens;
|
|
||||||
off_t i;
|
|
||||||
off_t dblen,eblen;
|
|
||||||
u_char *db,*eb;
|
|
||||||
u_char buf[8];
|
|
||||||
u_char header[32];
|
|
||||||
FILE * pf;
|
|
||||||
BZFILE * pfbz2;
|
|
||||||
int bz2err;
|
|
||||||
|
|
||||||
if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
|
|
||||||
|
|
||||||
/* Allocate oldsize+1 bytes instead of oldsize bytes to ensure
|
|
||||||
that we never try to malloc(0) and get a NULL pointer */
|
|
||||||
if(((fd=open(argv[1],O_RDONLY,0))<0) ||
|
|
||||||
((oldsize=lseek(fd,0,SEEK_END))==-1) ||
|
|
||||||
((old=malloc(oldsize+1))==NULL) ||
|
|
||||||
(lseek(fd,0,SEEK_SET)!=0) ||
|
|
||||||
(read(fd,old,oldsize)!=oldsize) ||
|
|
||||||
(close(fd)==-1)) err(1,"%s",argv[1]);
|
|
||||||
|
|
||||||
if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) ||
|
|
||||||
((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL);
|
|
||||||
|
|
||||||
qsufsort(I,V,old,oldsize);
|
|
||||||
|
|
||||||
free(V);
|
|
||||||
|
|
||||||
/* Allocate newsize+1 bytes instead of newsize bytes to ensure
|
|
||||||
that we never try to malloc(0) and get a NULL pointer */
|
|
||||||
if(((fd=open(argv[2],O_RDONLY,0))<0) ||
|
|
||||||
((newsize=lseek(fd,0,SEEK_END))==-1) ||
|
|
||||||
((new=malloc(newsize+1))==NULL) ||
|
|
||||||
(lseek(fd,0,SEEK_SET)!=0) ||
|
|
||||||
(read(fd,new,newsize)!=newsize) ||
|
|
||||||
(close(fd)==-1)) err(1,"%s",argv[2]);
|
|
||||||
|
|
||||||
if(((db=malloc(newsize+1))==NULL) ||
|
|
||||||
((eb=malloc(newsize+1))==NULL)) err(1,NULL);
|
|
||||||
dblen=0;
|
|
||||||
eblen=0;
|
|
||||||
|
|
||||||
/* Create the patch file */
|
|
||||||
if ((pf = fopen(argv[3], "w")) == NULL)
|
|
||||||
err(1, "%s", argv[3]);
|
|
||||||
|
|
||||||
/* Header is
|
|
||||||
0 8 "BSDIFF40"
|
|
||||||
8 8 length of bzip2ed ctrl block
|
|
||||||
16 8 length of bzip2ed diff block
|
|
||||||
24 8 length of new file */
|
|
||||||
/* File is
|
|
||||||
0 32 Header
|
|
||||||
32 ?? Bzip2ed ctrl block
|
|
||||||
?? ?? Bzip2ed diff block
|
|
||||||
?? ?? Bzip2ed extra block */
|
|
||||||
memcpy(header,"BSDIFF40",8);
|
|
||||||
offtout(0, header + 8);
|
|
||||||
offtout(0, header + 16);
|
|
||||||
offtout(newsize, header + 24);
|
|
||||||
if (fwrite(header, 32, 1, pf) != 1)
|
|
||||||
err(1, "fwrite(%s)", argv[3]);
|
|
||||||
|
|
||||||
/* Compute the differences, writing ctrl as we go */
|
|
||||||
if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL)
|
|
||||||
errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err);
|
|
||||||
scan=0;len=0;
|
|
||||||
lastscan=0;lastpos=0;lastoffset=0;
|
|
||||||
while(scan<newsize) {
|
|
||||||
oldscore=0;
|
|
||||||
|
|
||||||
for(scsc=scan+=len;scan<newsize;scan++) {
|
|
||||||
len=search(I,old,oldsize,new+scan,newsize-scan,
|
|
||||||
0,oldsize,&pos);
|
|
||||||
if (len > 64 * 1024) break;
|
|
||||||
|
|
||||||
for(;scsc<scan+len;scsc++)
|
|
||||||
if((scsc+lastoffset<oldsize) &&
|
|
||||||
(old[scsc+lastoffset] == new[scsc]))
|
|
||||||
oldscore++;
|
|
||||||
|
|
||||||
if(((len==oldscore) && (len!=0)) ||
|
|
||||||
(len>oldscore+8)) break;
|
|
||||||
|
|
||||||
if((scan+lastoffset<oldsize) &&
|
|
||||||
(old[scan+lastoffset] == new[scan]))
|
|
||||||
oldscore--;
|
|
||||||
};
|
|
||||||
|
|
||||||
if((len!=oldscore) || (scan==newsize)) {
|
|
||||||
s=0;Sf=0;lenf=0;
|
|
||||||
for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) {
|
|
||||||
if(old[lastpos+i]==new[lastscan+i]) s++;
|
|
||||||
i++;
|
|
||||||
if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; };
|
|
||||||
};
|
|
||||||
|
|
||||||
lenb=0;
|
|
||||||
if(scan<newsize) {
|
|
||||||
s=0;Sb=0;
|
|
||||||
for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) {
|
|
||||||
if(old[pos-i]==new[scan-i]) s++;
|
|
||||||
if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; };
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
if(lastscan+lenf>scan-lenb) {
|
|
||||||
overlap=(lastscan+lenf)-(scan-lenb);
|
|
||||||
s=0;Ss=0;lens=0;
|
|
||||||
for(i=0;i<overlap;i++) {
|
|
||||||
if(new[lastscan+lenf-overlap+i]==
|
|
||||||
old[lastpos+lenf-overlap+i]) s++;
|
|
||||||
if(new[scan-lenb+i]==
|
|
||||||
old[pos-lenb+i]) s--;
|
|
||||||
if(s>Ss) { Ss=s; lens=i+1; };
|
|
||||||
};
|
|
||||||
|
|
||||||
lenf+=lens-overlap;
|
|
||||||
lenb-=lens;
|
|
||||||
};
|
|
||||||
|
|
||||||
for(i=0;i<lenf;i++)
|
|
||||||
db[dblen+i]=new[lastscan+i]-old[lastpos+i];
|
|
||||||
for(i=0;i<(scan-lenb)-(lastscan+lenf);i++)
|
|
||||||
eb[eblen+i]=new[lastscan+lenf+i];
|
|
||||||
|
|
||||||
dblen+=lenf;
|
|
||||||
eblen+=(scan-lenb)-(lastscan+lenf);
|
|
||||||
|
|
||||||
offtout(lenf,buf);
|
|
||||||
BZ2_bzWrite(&bz2err, pfbz2, buf, 8);
|
|
||||||
if (bz2err != BZ_OK)
|
|
||||||
errx(1, "BZ2_bzWrite, bz2err = %d", bz2err);
|
|
||||||
|
|
||||||
offtout((scan-lenb)-(lastscan+lenf),buf);
|
|
||||||
BZ2_bzWrite(&bz2err, pfbz2, buf, 8);
|
|
||||||
if (bz2err != BZ_OK)
|
|
||||||
errx(1, "BZ2_bzWrite, bz2err = %d", bz2err);
|
|
||||||
|
|
||||||
offtout((pos-lenb)-(lastpos+lenf),buf);
|
|
||||||
BZ2_bzWrite(&bz2err, pfbz2, buf, 8);
|
|
||||||
if (bz2err != BZ_OK)
|
|
||||||
errx(1, "BZ2_bzWrite, bz2err = %d", bz2err);
|
|
||||||
|
|
||||||
lastscan=scan-lenb;
|
|
||||||
lastpos=pos-lenb;
|
|
||||||
lastoffset=pos-scan;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL);
|
|
||||||
if (bz2err != BZ_OK)
|
|
||||||
errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err);
|
|
||||||
|
|
||||||
/* Compute size of compressed ctrl data */
|
|
||||||
if ((len = ftello(pf)) == -1)
|
|
||||||
err(1, "ftello");
|
|
||||||
offtout(len-32, header + 8);
|
|
||||||
|
|
||||||
/* Write compressed diff data */
|
|
||||||
if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL)
|
|
||||||
errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err);
|
|
||||||
BZ2_bzWrite(&bz2err, pfbz2, db, dblen);
|
|
||||||
if (bz2err != BZ_OK)
|
|
||||||
errx(1, "BZ2_bzWrite, bz2err = %d", bz2err);
|
|
||||||
BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL);
|
|
||||||
if (bz2err != BZ_OK)
|
|
||||||
errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err);
|
|
||||||
|
|
||||||
/* Compute size of compressed diff data */
|
|
||||||
if ((newsize = ftello(pf)) == -1)
|
|
||||||
err(1, "ftello");
|
|
||||||
offtout(newsize - len, header + 16);
|
|
||||||
|
|
||||||
/* Write compressed extra data */
|
|
||||||
if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL)
|
|
||||||
errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err);
|
|
||||||
BZ2_bzWrite(&bz2err, pfbz2, eb, eblen);
|
|
||||||
if (bz2err != BZ_OK)
|
|
||||||
errx(1, "BZ2_bzWrite, bz2err = %d", bz2err);
|
|
||||||
BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL);
|
|
||||||
if (bz2err != BZ_OK)
|
|
||||||
errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err);
|
|
||||||
|
|
||||||
/* Seek to the beginning, write the header, and close the file */
|
|
||||||
if (fseeko(pf, 0, SEEK_SET))
|
|
||||||
err(1, "fseeko");
|
|
||||||
if (fwrite(header, 32, 1, pf) != 1)
|
|
||||||
err(1, "fwrite(%s)", argv[3]);
|
|
||||||
if (fclose(pf))
|
|
||||||
err(1, "fclose");
|
|
||||||
|
|
||||||
/* Free the memory we used */
|
|
||||||
free(db);
|
|
||||||
free(eb);
|
|
||||||
free(I);
|
|
||||||
free(old);
|
|
||||||
free(new);
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
@ -1,59 +0,0 @@
|
||||||
.\"-
|
|
||||||
.\" Copyright 2003-2005 Colin Percival
|
|
||||||
.\" All rights reserved
|
|
||||||
.\"
|
|
||||||
.\" Redistribution and use in source and binary forms, with or without
|
|
||||||
.\" modification, are permitted providing that the following conditions
|
|
||||||
.\" are met:
|
|
||||||
.\" 1. Redistributions of source code must retain the above copyright
|
|
||||||
.\" notice, this list of conditions and the following disclaimer.
|
|
||||||
.\" 2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
.\" notice, this list of conditions and the following disclaimer in the
|
|
||||||
.\" documentation and/or other materials provided with the distribution.
|
|
||||||
.\"
|
|
||||||
.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
||||||
.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
||||||
.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
|
|
||||||
.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
||||||
.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
||||||
.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
|
||||||
.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
|
|
||||||
.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
||||||
.\" POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
.\"
|
|
||||||
.\" $FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.1,v 1.1 2005/08/06 01:59:06 cperciva Exp $
|
|
||||||
.\"
|
|
||||||
.Dd May 18, 2003
|
|
||||||
.Dt BSPATCH 1
|
|
||||||
.Os FreeBSD
|
|
||||||
.Sh NAME
|
|
||||||
.Nm bspatch
|
|
||||||
.Nd apply a patch built with bsdiff(1)
|
|
||||||
.Sh SYNOPSIS
|
|
||||||
.Nm
|
|
||||||
.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac
|
|
||||||
.Sh DESCRIPTION
|
|
||||||
.Nm
|
|
||||||
generates
|
|
||||||
.Ao Ar newfile Ac
|
|
||||||
from
|
|
||||||
.Ao Ar oldfile Ac
|
|
||||||
and
|
|
||||||
.Ao Ar patchfile Ac
|
|
||||||
where
|
|
||||||
.Ao Ar patchfile Ac
|
|
||||||
is a binary patch built by bsdiff(1).
|
|
||||||
.Pp
|
|
||||||
.Nm
|
|
||||||
uses memory equal to the size of
|
|
||||||
.Ao Ar oldfile Ac
|
|
||||||
plus the size of
|
|
||||||
.Ao Ar newfile Ac ,
|
|
||||||
but can tolerate a very small working set without a dramatic loss
|
|
||||||
of performance.
|
|
||||||
.Sh SEE ALSO
|
|
||||||
.Xr bsdiff 1
|
|
||||||
.Sh AUTHORS
|
|
||||||
.An Colin Percival Aq cperciva@freebsd.org
|
|
||||||
|
|
@ -1,224 +0,0 @@
|
||||||
/*-
|
|
||||||
* Copyright 2003-2005 Colin Percival
|
|
||||||
* All rights reserved
|
|
||||||
*
|
|
||||||
* Redistribution and use in source and binary forms, with or without
|
|
||||||
* modification, are permitted providing that the following conditions
|
|
||||||
* are met:
|
|
||||||
* 1. Redistributions of source code must retain the above copyright
|
|
||||||
* notice, this list of conditions and the following disclaimer.
|
|
||||||
* 2. Redistributions in binary form must reproduce the above copyright
|
|
||||||
* notice, this list of conditions and the following disclaimer in the
|
|
||||||
* documentation and/or other materials provided with the distribution.
|
|
||||||
*
|
|
||||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
||||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
|
||||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
||||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
|
|
||||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
||||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
||||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
||||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
|
||||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
|
|
||||||
* IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
||||||
* POSSIBILITY OF SUCH DAMAGE.
|
|
||||||
*/
|
|
||||||
|
|
||||||
#if 0
|
|
||||||
__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.c,v 1.1 2005/08/06 01:59:06 cperciva Exp $");
|
|
||||||
#endif
|
|
||||||
|
|
||||||
#include <bzlib.h>
|
|
||||||
#include <stdlib.h>
|
|
||||||
#include <stdio.h>
|
|
||||||
#include <string.h>
|
|
||||||
#include <err.h>
|
|
||||||
#include <errno.h>
|
|
||||||
#include <unistd.h>
|
|
||||||
#include <fcntl.h>
|
|
||||||
#include <sys/types.h>
|
|
||||||
|
|
||||||
static off_t offtin(u_char *buf)
|
|
||||||
{
|
|
||||||
off_t y;
|
|
||||||
|
|
||||||
y=buf[7]&0x7F;
|
|
||||||
y=y*256;y+=buf[6];
|
|
||||||
y=y*256;y+=buf[5];
|
|
||||||
y=y*256;y+=buf[4];
|
|
||||||
y=y*256;y+=buf[3];
|
|
||||||
y=y*256;y+=buf[2];
|
|
||||||
y=y*256;y+=buf[1];
|
|
||||||
y=y*256;y+=buf[0];
|
|
||||||
|
|
||||||
if(buf[7]&0x80) y=-y;
|
|
||||||
|
|
||||||
return y;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void writeFull(const char * name, int fd,
|
|
||||||
const unsigned char * buf, size_t count)
|
|
||||||
{
|
|
||||||
while (count) {
|
|
||||||
ssize_t res = write(fd, (char *) buf, count);
|
|
||||||
if (res == -1) {
|
|
||||||
if (errno == EINTR) continue;
|
|
||||||
err(1,"writing to %s",name);
|
|
||||||
}
|
|
||||||
count -= res;
|
|
||||||
buf += res;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
int main(int argc,char * argv[])
|
|
||||||
{
|
|
||||||
FILE * f, * cpf, * dpf, * epf;
|
|
||||||
BZFILE * cpfbz2, * dpfbz2, * epfbz2;
|
|
||||||
int cbz2err, dbz2err, ebz2err;
|
|
||||||
int fd;
|
|
||||||
ssize_t oldsize,newsize;
|
|
||||||
ssize_t bzctrllen,bzdatalen;
|
|
||||||
u_char header[32],buf[8];
|
|
||||||
u_char *old, *new;
|
|
||||||
off_t oldpos,newpos;
|
|
||||||
off_t ctrl[3];
|
|
||||||
off_t lenread;
|
|
||||||
off_t i;
|
|
||||||
|
|
||||||
if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
|
|
||||||
|
|
||||||
/* Open patch file */
|
|
||||||
if ((f = fopen(argv[3], "r")) == NULL)
|
|
||||||
err(1, "fopen(%s)", argv[3]);
|
|
||||||
|
|
||||||
/*
|
|
||||||
File format:
|
|
||||||
0 8 "BSDIFF40"
|
|
||||||
8 8 X
|
|
||||||
16 8 Y
|
|
||||||
24 8 sizeof(newfile)
|
|
||||||
32 X bzip2(control block)
|
|
||||||
32+X Y bzip2(diff block)
|
|
||||||
32+X+Y ??? bzip2(extra block)
|
|
||||||
with control block a set of triples (x,y,z) meaning "add x bytes
|
|
||||||
from oldfile to x bytes from the diff block; copy y bytes from the
|
|
||||||
extra block; seek forwards in oldfile by z bytes".
|
|
||||||
*/
|
|
||||||
|
|
||||||
/* Read header */
|
|
||||||
if (fread(header, 1, 32, f) < 32) {
|
|
||||||
if (feof(f))
|
|
||||||
errx(1, "Corrupt patch\n");
|
|
||||||
err(1, "fread(%s)", argv[3]);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Check for appropriate magic */
|
|
||||||
if (memcmp(header, "BSDIFF40", 8) != 0)
|
|
||||||
errx(1, "Corrupt patch\n");
|
|
||||||
|
|
||||||
/* Read lengths from header */
|
|
||||||
bzctrllen=offtin(header+8);
|
|
||||||
bzdatalen=offtin(header+16);
|
|
||||||
newsize=offtin(header+24);
|
|
||||||
if((bzctrllen<0) || (bzdatalen<0) || (newsize<0))
|
|
||||||
errx(1,"Corrupt patch\n");
|
|
||||||
|
|
||||||
/* Close patch file and re-open it via libbzip2 at the right places */
|
|
||||||
if (fclose(f))
|
|
||||||
err(1, "fclose(%s)", argv[3]);
|
|
||||||
if ((cpf = fopen(argv[3], "r")) == NULL)
|
|
||||||
err(1, "fopen(%s)", argv[3]);
|
|
||||||
if (fseeko(cpf, 32, SEEK_SET))
|
|
||||||
err(1, "fseeko(%s, %lld)", argv[3],
|
|
||||||
(long long)32);
|
|
||||||
if ((cpfbz2 = BZ2_bzReadOpen(&cbz2err, cpf, 0, 0, NULL, 0)) == NULL)
|
|
||||||
errx(1, "BZ2_bzReadOpen, bz2err = %d", cbz2err);
|
|
||||||
if ((dpf = fopen(argv[3], "r")) == NULL)
|
|
||||||
err(1, "fopen(%s)", argv[3]);
|
|
||||||
if (fseeko(dpf, 32 + bzctrllen, SEEK_SET))
|
|
||||||
err(1, "fseeko(%s, %lld)", argv[3],
|
|
||||||
(long long)(32 + bzctrllen));
|
|
||||||
if ((dpfbz2 = BZ2_bzReadOpen(&dbz2err, dpf, 0, 0, NULL, 0)) == NULL)
|
|
||||||
errx(1, "BZ2_bzReadOpen, bz2err = %d", dbz2err);
|
|
||||||
if ((epf = fopen(argv[3], "r")) == NULL)
|
|
||||||
err(1, "fopen(%s)", argv[3]);
|
|
||||||
if (fseeko(epf, 32 + bzctrllen + bzdatalen, SEEK_SET))
|
|
||||||
err(1, "fseeko(%s, %lld)", argv[3],
|
|
||||||
(long long)(32 + bzctrllen + bzdatalen));
|
|
||||||
if ((epfbz2 = BZ2_bzReadOpen(&ebz2err, epf, 0, 0, NULL, 0)) == NULL)
|
|
||||||
errx(1, "BZ2_bzReadOpen, bz2err = %d", ebz2err);
|
|
||||||
|
|
||||||
if(((fd=open(argv[1],O_RDONLY,0))<0) ||
|
|
||||||
((oldsize=lseek(fd,0,SEEK_END))==-1) ||
|
|
||||||
((old=malloc(oldsize+1))==NULL) ||
|
|
||||||
(lseek(fd,0,SEEK_SET)!=0) ||
|
|
||||||
(read(fd,old,oldsize)!=oldsize) ||
|
|
||||||
(close(fd)==-1)) err(1,"%s",argv[1]);
|
|
||||||
if((new=malloc(newsize+1))==NULL) err(1,NULL);
|
|
||||||
|
|
||||||
oldpos=0;newpos=0;
|
|
||||||
while(newpos<newsize) {
|
|
||||||
/* Read control data */
|
|
||||||
for(i=0;i<=2;i++) {
|
|
||||||
lenread = BZ2_bzRead(&cbz2err, cpfbz2, buf, 8);
|
|
||||||
if ((lenread < 8) || ((cbz2err != BZ_OK) &&
|
|
||||||
(cbz2err != BZ_STREAM_END)))
|
|
||||||
errx(1, "Corrupt patch\n");
|
|
||||||
ctrl[i]=offtin(buf);
|
|
||||||
};
|
|
||||||
|
|
||||||
/* Sanity-check */
|
|
||||||
if(newpos+ctrl[0]>newsize)
|
|
||||||
errx(1,"Corrupt patch\n");
|
|
||||||
|
|
||||||
/* Read diff string */
|
|
||||||
lenread = BZ2_bzRead(&dbz2err, dpfbz2, new + newpos, ctrl[0]);
|
|
||||||
if ((lenread < ctrl[0]) ||
|
|
||||||
((dbz2err != BZ_OK) && (dbz2err != BZ_STREAM_END)))
|
|
||||||
errx(1, "Corrupt patch\n");
|
|
||||||
|
|
||||||
/* Add old data to diff string */
|
|
||||||
for(i=0;i<ctrl[0];i++)
|
|
||||||
if((oldpos+i>=0) && (oldpos+i<oldsize))
|
|
||||||
new[newpos+i]+=old[oldpos+i];
|
|
||||||
|
|
||||||
/* Adjust pointers */
|
|
||||||
newpos+=ctrl[0];
|
|
||||||
oldpos+=ctrl[0];
|
|
||||||
|
|
||||||
/* Sanity-check */
|
|
||||||
if(newpos+ctrl[1]>newsize)
|
|
||||||
errx(1,"Corrupt patch\n");
|
|
||||||
|
|
||||||
/* Read extra string */
|
|
||||||
lenread = BZ2_bzRead(&ebz2err, epfbz2, new + newpos, ctrl[1]);
|
|
||||||
if ((lenread < ctrl[1]) ||
|
|
||||||
((ebz2err != BZ_OK) && (ebz2err != BZ_STREAM_END)))
|
|
||||||
errx(1, "Corrupt patch\n");
|
|
||||||
|
|
||||||
/* Adjust pointers */
|
|
||||||
newpos+=ctrl[1];
|
|
||||||
oldpos+=ctrl[2];
|
|
||||||
};
|
|
||||||
|
|
||||||
/* Clean up the bzip2 reads */
|
|
||||||
BZ2_bzReadClose(&cbz2err, cpfbz2);
|
|
||||||
BZ2_bzReadClose(&dbz2err, dpfbz2);
|
|
||||||
BZ2_bzReadClose(&ebz2err, epfbz2);
|
|
||||||
if (fclose(cpf) || fclose(dpf) || fclose(epf))
|
|
||||||
err(1, "fclose(%s)", argv[3]);
|
|
||||||
|
|
||||||
/* Write the new file */
|
|
||||||
if((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0)
|
|
||||||
err(1,"%s",argv[2]);
|
|
||||||
writeFull(argv[2], fd, new, newsize);
|
|
||||||
if(close(fd)==-1)
|
|
||||||
err(1,"%s",argv[2]);
|
|
||||||
|
|
||||||
free(new);
|
|
||||||
free(old);
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
/* Simulate BSD's <err.h> functionality. */
|
|
||||||
|
|
||||||
#ifndef COMPAT_ERR_H_INCLUDED
|
|
||||||
#define COMPAT_ERR_H_INCLUDED 1
|
|
||||||
|
|
||||||
#include <stdio.h>
|
|
||||||
#include <stdlib.h>
|
|
||||||
|
|
||||||
#define err(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0)
|
|
||||||
#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0)
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
@ -1,11 +0,0 @@
|
||||||
programs += bsdiff bspatch
|
|
||||||
|
|
||||||
bsdiff_DIR := $(d)
|
|
||||||
bsdiff_SOURCES := $(d)/bsdiff.c
|
|
||||||
bsdiff_LDFLAGS = -lbz2 $(bsddiff_compat_include)
|
|
||||||
bsdiff_INSTALL_DIR = $(libexecdir)/nix
|
|
||||||
|
|
||||||
bspatch_DIR := $(d)
|
|
||||||
bspatch_SOURCES := $(d)/bspatch.c
|
|
||||||
bspatch_LDFLAGS = -lbz2 $(bsddiff_compat_include)
|
|
||||||
bspatch_INSTALL_DIR = $(libexecdir)/nix
|
|
||||||
|
|
@ -194,7 +194,6 @@ void Settings::update()
|
||||||
if (getEnv("NIX_OTHER_STORES") != "")
|
if (getEnv("NIX_OTHER_STORES") != "")
|
||||||
substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
|
substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
|
||||||
#endif
|
#endif
|
||||||
substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
|
|
||||||
substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
|
substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
|
||||||
if (useSshSubstituter && !sshSubstituterHosts.empty())
|
if (useSshSubstituter && !sshSubstituterHosts.empty())
|
||||||
substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh");
|
substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh");
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
source common.sh
|
source common.sh
|
||||||
|
|
||||||
clearStore
|
clearStore
|
||||||
clearManifests
|
|
||||||
clearCache
|
clearCache
|
||||||
|
|
||||||
# Create the binary cache.
|
# Create the binary cache.
|
||||||
|
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
{ version }:
|
|
||||||
|
|
||||||
with import ./config.nix;
|
|
||||||
|
|
||||||
mkDerivation {
|
|
||||||
name = "foo-${toString version}";
|
|
||||||
builder = builtins.toFile "builder.sh"
|
|
||||||
''
|
|
||||||
mkdir $out
|
|
||||||
(for ((n = 1; n < 100000; n++)); do echo $n; done) > $out/foo
|
|
||||||
${if version != 1 then ''
|
|
||||||
(for ((n = 100000; n < 110000; n++)); do echo $n; done) >> $out/foo
|
|
||||||
'' else ""}
|
|
||||||
${if version == 3 then ''
|
|
||||||
echo foobar >> $out/foo
|
|
||||||
'' else ""}
|
|
||||||
'';
|
|
||||||
}
|
|
||||||
|
|
@ -1,61 +0,0 @@
|
||||||
source common.sh
|
|
||||||
|
|
||||||
clearManifests
|
|
||||||
|
|
||||||
mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches
|
|
||||||
|
|
||||||
RESULT=$TEST_ROOT/result
|
|
||||||
|
|
||||||
# Build version 1 and 2 of the "foo" package.
|
|
||||||
nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 \
|
|
||||||
$(nix-build -o $RESULT binary-patching.nix --arg version 1)
|
|
||||||
mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest1
|
|
||||||
|
|
||||||
out2=$(nix-build -o $RESULT binary-patching.nix --arg version 2)
|
|
||||||
nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out2
|
|
||||||
mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest2
|
|
||||||
|
|
||||||
out3=$(nix-build -o $RESULT binary-patching.nix --arg version 3)
|
|
||||||
nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out3
|
|
||||||
mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest3
|
|
||||||
|
|
||||||
rm $RESULT
|
|
||||||
|
|
||||||
# Generate binary patches.
|
|
||||||
nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \
|
|
||||||
file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest2
|
|
||||||
|
|
||||||
nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \
|
|
||||||
file://$TEST_ROOT/patches $TEST_ROOT/manifest2 $TEST_ROOT/manifest3
|
|
||||||
|
|
||||||
grep -q "patch {" $TEST_ROOT/manifest3
|
|
||||||
|
|
||||||
# Get rid of versions 2 and 3.
|
|
||||||
nix-store --delete $out2 $out3
|
|
||||||
|
|
||||||
# Pull the manifest containing the patches.
|
|
||||||
clearManifests
|
|
||||||
nix-pull file://$TEST_ROOT/manifest3
|
|
||||||
|
|
||||||
# Make sure that the download size prediction uses the patches rather
|
|
||||||
# than the full download.
|
|
||||||
nix-build -o $RESULT binary-patching.nix --arg version 3 --dry-run 2>&1 | grep -q "0.01 MiB"
|
|
||||||
|
|
||||||
# Now rebuild it. This should use the two patches generated above.
|
|
||||||
rm -f $TEST_ROOT/var/log/nix/downloads
|
|
||||||
nix-build -o $RESULT binary-patching.nix --arg version 3
|
|
||||||
rm $RESULT
|
|
||||||
[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 2 ]
|
|
||||||
|
|
||||||
# Add a patch from version 1 directly to version 3.
|
|
||||||
nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \
|
|
||||||
file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest3
|
|
||||||
|
|
||||||
# Rebuild version 3. This should use the direct patch rather than the
|
|
||||||
# sequence of two patches.
|
|
||||||
nix-store --delete $out2 $out3
|
|
||||||
clearManifests
|
|
||||||
rm $TEST_ROOT/var/log/nix/downloads
|
|
||||||
nix-pull file://$TEST_ROOT/manifest3
|
|
||||||
nix-build -o $RESULT binary-patching.nix --arg version 3
|
|
||||||
[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 1 ]
|
|
||||||
|
|
@ -54,10 +54,6 @@ clearStore() {
|
||||||
rm -f "$NIX_STATE_DIR"/gcroots/ref
|
rm -f "$NIX_STATE_DIR"/gcroots/ref
|
||||||
}
|
}
|
||||||
|
|
||||||
clearManifests() {
|
|
||||||
rm -f $NIX_STATE_DIR/manifests/*
|
|
||||||
}
|
|
||||||
|
|
||||||
clearCache() {
|
clearCache() {
|
||||||
rm -rf "$cacheDir"
|
rm -rf "$cacheDir"
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,14 @@
|
||||||
source common.sh
|
source common.sh
|
||||||
|
|
||||||
# Note: this test expects to be run *after* nix-push.sh.
|
|
||||||
|
|
||||||
drvPath=$(nix-instantiate ./dependencies.nix)
|
drvPath=$(nix-instantiate ./dependencies.nix)
|
||||||
outPath=$(nix-store -q $drvPath)
|
outPath=$(nix-store -r $drvPath)
|
||||||
|
nix-push --dest $cacheDir $outPath
|
||||||
|
|
||||||
clearStore
|
clearStore
|
||||||
clearProfiles
|
clearProfiles
|
||||||
|
|
||||||
cat > $TEST_ROOT/foo.nixpkg <<EOF
|
cat > $TEST_ROOT/foo.nixpkg <<EOF
|
||||||
NIXPKG1 file://$TEST_ROOT/cache/MANIFEST simple $system $drvPath $outPath
|
NIXPKG1 - simple $system $drvPath $outPath file://$cacheDir
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
nix-install-package --non-interactive -p $profiles/test $TEST_ROOT/foo.nixpkg
|
nix-install-package --non-interactive -p $profiles/test $TEST_ROOT/foo.nixpkg
|
||||||
|
|
|
||||||
|
|
@ -4,11 +4,11 @@ check:
|
||||||
nix_tests = \
|
nix_tests = \
|
||||||
init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
|
init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
|
||||||
build-hook.sh substitutes.sh substitutes2.sh \
|
build-hook.sh substitutes.sh substitutes2.sh \
|
||||||
fallback.sh nix-push.sh gc.sh gc-concurrent.sh nix-pull.sh \
|
fallback.sh nix-push.sh gc.sh gc-concurrent.sh \
|
||||||
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
|
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
|
||||||
gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \
|
gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \
|
||||||
remote-store.sh export.sh export-graph.sh \
|
remote-store.sh export.sh export-graph.sh \
|
||||||
binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \
|
timeout.sh secure-drv-outputs.sh nix-channel.sh \
|
||||||
multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
|
multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
|
||||||
binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
|
binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
|
||||||
check-reqs.sh pass-as-file.sh tarball.sh
|
check-reqs.sh pass-as-file.sh tarball.sh
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
source common.sh
|
source common.sh
|
||||||
|
|
||||||
clearProfiles
|
clearProfiles
|
||||||
clearManifests
|
|
||||||
|
|
||||||
rm -f $TEST_ROOT/.nix-channels
|
rm -f $TEST_ROOT/.nix-channels
|
||||||
|
|
||||||
|
|
@ -45,7 +44,6 @@ nix-env -i dependencies
|
||||||
|
|
||||||
|
|
||||||
clearProfiles
|
clearProfiles
|
||||||
clearManifests
|
|
||||||
rm -f $TEST_ROOT/.nix-channels
|
rm -f $TEST_ROOT/.nix-channels
|
||||||
|
|
||||||
# Test updating from a tarball
|
# Test updating from a tarball
|
||||||
|
|
|
||||||
|
|
@ -1,33 +0,0 @@
|
||||||
source common.sh
|
|
||||||
|
|
||||||
pullCache () {
|
|
||||||
echo "pulling cache..."
|
|
||||||
nix-pull file://$TEST_ROOT/cache/MANIFEST
|
|
||||||
}
|
|
||||||
|
|
||||||
clearStore
|
|
||||||
clearManifests
|
|
||||||
pullCache
|
|
||||||
|
|
||||||
drvPath=$(nix-instantiate dependencies.nix)
|
|
||||||
outPath=$(nix-store -q $drvPath)
|
|
||||||
|
|
||||||
echo "building $outPath using substitutes..."
|
|
||||||
nix-store -r $outPath
|
|
||||||
|
|
||||||
cat $outPath/input-2/bar
|
|
||||||
|
|
||||||
clearStore
|
|
||||||
clearManifests
|
|
||||||
pullCache
|
|
||||||
|
|
||||||
echo "building $drvPath using substitutes..."
|
|
||||||
nix-store -r $drvPath
|
|
||||||
|
|
||||||
cat $outPath/input-2/bar
|
|
||||||
|
|
||||||
# Check that the derivers are set properly.
|
|
||||||
test $(nix-store -q --deriver "$outPath") = "$drvPath"
|
|
||||||
nix-store -q --deriver $(readLink $outPath/input-2) | grep -q -- "-input-2.drv"
|
|
||||||
|
|
||||||
clearManifests
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
source common.sh
|
source common.sh
|
||||||
|
|
||||||
clearStore
|
clearStore
|
||||||
clearManifests
|
|
||||||
|
|
||||||
startDaemon
|
startDaemon
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,6 @@
|
||||||
source common.sh
|
source common.sh
|
||||||
|
|
||||||
clearStore
|
clearStore
|
||||||
clearManifests
|
|
||||||
|
|
||||||
startDaemon
|
startDaemon
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue