nix-copy-closure: Implement in C++.
Tests fail currently because the database is not given proper hashes in the VM
This commit is contained in:
parent
8af062f372
commit
bfa41eb671
11 changed files with 100 additions and 115 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -81,6 +81,8 @@ Makefile.config
|
||||||
# /src/nix-build/
|
# /src/nix-build/
|
||||||
/src/nix-build/nix-build
|
/src/nix-build/nix-build
|
||||||
|
|
||||||
|
/src/nix-copy-closure/nix-copy-closure
|
||||||
|
|
||||||
/src/build-remote/build-remote
|
/src/build-remote/build-remote
|
||||||
|
|
||||||
# /tests/
|
# /tests/
|
||||||
|
|
1
Makefile
1
Makefile
|
@ -11,6 +11,7 @@ makefiles = \
|
||||||
src/nix-env/local.mk \
|
src/nix-env/local.mk \
|
||||||
src/nix-daemon/local.mk \
|
src/nix-daemon/local.mk \
|
||||||
src/nix-collect-garbage/local.mk \
|
src/nix-collect-garbage/local.mk \
|
||||||
|
src/nix-copy-closure/local.mk \
|
||||||
src/nix-prefetch-url/local.mk \
|
src/nix-prefetch-url/local.mk \
|
||||||
src/buildenv/local.mk \
|
src/buildenv/local.mk \
|
||||||
src/resolve-system-dependencies/local.mk \
|
src/resolve-system-dependencies/local.mk \
|
||||||
|
|
|
@ -1,8 +1,3 @@
|
||||||
nix_bin_scripts := \
|
|
||||||
$(d)/nix-copy-closure \
|
|
||||||
|
|
||||||
bin-scripts += $(nix_bin_scripts)
|
|
||||||
|
|
||||||
nix_noinst_scripts := \
|
nix_noinst_scripts := \
|
||||||
$(d)/build-remote.pl \
|
$(d)/build-remote.pl \
|
||||||
$(d)/nix-http-export.cgi \
|
$(d)/nix-http-export.cgi \
|
||||||
|
@ -16,4 +11,4 @@ profiledir = $(sysconfdir)/profile.d
|
||||||
$(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
|
$(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644))
|
||||||
$(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix))
|
$(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix))
|
||||||
|
|
||||||
clean-files += $(nix_bin_scripts) $(nix_noinst_scripts)
|
clean-files += $(nix_noinst_scripts)
|
||||||
|
|
|
@ -1,103 +0,0 @@
|
||||||
#! @perl@ -w @perlFlags@
|
|
||||||
|
|
||||||
use utf8;
|
|
||||||
use strict;
|
|
||||||
use Nix::SSH;
|
|
||||||
use Nix::Config;
|
|
||||||
use Nix::Store;
|
|
||||||
use Nix::CopyClosure;
|
|
||||||
use List::Util qw(sum);
|
|
||||||
|
|
||||||
binmode STDERR, ":encoding(utf8)";
|
|
||||||
|
|
||||||
if (scalar @ARGV < 1) {
|
|
||||||
print STDERR <<EOF
|
|
||||||
Usage: nix-copy-closure [--from | --to] HOSTNAME [--gzip] [--bzip2] [--xz] PATHS...
|
|
||||||
EOF
|
|
||||||
;
|
|
||||||
exit 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Get the target host.
|
|
||||||
my $sshHost;
|
|
||||||
my $toMode = 1;
|
|
||||||
my $includeOutputs = 0;
|
|
||||||
my $dryRun = 0;
|
|
||||||
my $useSubstitutes = 0;
|
|
||||||
my $verbosity = 1;
|
|
||||||
|
|
||||||
|
|
||||||
# !!! Copied from nix-pack-closure, should put this in a module.
|
|
||||||
my @storePaths = ();
|
|
||||||
|
|
||||||
while (@ARGV) {
|
|
||||||
my $arg = shift @ARGV;
|
|
||||||
|
|
||||||
if ($arg eq "--help") {
|
|
||||||
exec "man nix-copy-closure" or die;
|
|
||||||
}
|
|
||||||
elsif ($arg eq "--gzip" || $arg eq "--bzip2" || $arg eq "--xz") {
|
|
||||||
warn "$0: ‘$arg’ is not implemented\n" if $arg ne "--gzip";
|
|
||||||
push @globalSshOpts, "-C";
|
|
||||||
}
|
|
||||||
elsif ($arg eq "--from") {
|
|
||||||
$toMode = 0;
|
|
||||||
}
|
|
||||||
elsif ($arg eq "--to") {
|
|
||||||
$toMode = 1;
|
|
||||||
}
|
|
||||||
elsif ($arg eq "--include-outputs") {
|
|
||||||
$includeOutputs = 1;
|
|
||||||
}
|
|
||||||
elsif ($arg eq "--show-progress") {
|
|
||||||
warn "$0: ‘$arg’ is not implemented\n";
|
|
||||||
}
|
|
||||||
elsif ($arg eq "--dry-run") {
|
|
||||||
$dryRun = 1;
|
|
||||||
}
|
|
||||||
elsif ($arg eq "--use-substitutes" || $arg eq "-s") {
|
|
||||||
$useSubstitutes = 1;
|
|
||||||
}
|
|
||||||
elsif ($arg eq "-v") {
|
|
||||||
$verbosity++;
|
|
||||||
setVerbosity($verbosity);
|
|
||||||
}
|
|
||||||
elsif (!defined $sshHost) {
|
|
||||||
$sshHost = $arg;
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
push @storePaths, $arg;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
die "$0: you did not specify a host name\n" unless defined $sshHost;
|
|
||||||
|
|
||||||
|
|
||||||
if ($toMode) { # Copy TO the remote machine.
|
|
||||||
Nix::CopyClosure::copyTo(
|
|
||||||
$sshHost, [ @storePaths ],
|
|
||||||
$includeOutputs, $dryRun, $useSubstitutes);
|
|
||||||
}
|
|
||||||
|
|
||||||
else { # Copy FROM the remote machine.
|
|
||||||
|
|
||||||
my ($from, $to) = connectToRemoteNix($sshHost, []);
|
|
||||||
|
|
||||||
# Query the closure of the given store paths on the remote
|
|
||||||
# machine. Paths are assumed to be store paths; there is no
|
|
||||||
# resolution (following of symlinks).
|
|
||||||
syswrite($to, pack("L<x4L<x4", 7, $includeOutputs ? 1 : 0)) or die;
|
|
||||||
writeStrings(\@storePaths, $to);
|
|
||||||
my @missing = grep { !isValidPath($_) } readStrings($from);
|
|
||||||
|
|
||||||
# Export the store paths on the remote machine and import them locally.
|
|
||||||
if (scalar @missing > 0) {
|
|
||||||
print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n";
|
|
||||||
writeInt(5, $to); # == cmdExportPaths
|
|
||||||
writeInt(0, $to); # obsolete
|
|
||||||
writeStrings(\@missing, $to);
|
|
||||||
importPaths(fileno($from), 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -9,7 +9,6 @@
|
||||||
#include "shared.hh"
|
#include "shared.hh"
|
||||||
#include "pathlocks.hh"
|
#include "pathlocks.hh"
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
#include "serve-protocol.hh"
|
|
||||||
#include "serialise.hh"
|
#include "serialise.hh"
|
||||||
#include "store-api.hh"
|
#include "store-api.hh"
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
|
|
|
@ -8,4 +8,4 @@ build-remote_LIBS = libmain libutil libformat libstore
|
||||||
|
|
||||||
build-remote_SOURCES := $(d)/build-remote.cc
|
build-remote_SOURCES := $(d)/build-remote.cc
|
||||||
|
|
||||||
build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\"" -Isrc/nix-store
|
build-remote_CXXFLAGS = -DSYSCONFDIR="\"$(sysconfdir)\""
|
||||||
|
|
|
@ -39,6 +39,8 @@ private:
|
||||||
string uri;
|
string uri;
|
||||||
|
|
||||||
Path key;
|
Path key;
|
||||||
|
|
||||||
|
bool compress;
|
||||||
};
|
};
|
||||||
|
|
||||||
SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
|
SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
|
||||||
|
@ -48,6 +50,7 @@ SSHStore::SSHStore(string uri, const Params & params, size_t maxConnections)
|
||||||
, socketPath((Path) tmpDir + "/ssh.sock")
|
, socketPath((Path) tmpDir + "/ssh.sock")
|
||||||
, uri(std::move(uri))
|
, uri(std::move(uri))
|
||||||
, key(get(params, "ssh-key", ""))
|
, key(get(params, "ssh-key", ""))
|
||||||
|
, compress(get(params, "compress", "") == "true")
|
||||||
{
|
{
|
||||||
/* open a connection and perform the handshake to verify all is well */
|
/* open a connection and perform the handshake to verify all is well */
|
||||||
connections->get();
|
connections->get();
|
||||||
|
@ -90,11 +93,12 @@ ref<FSAccessor> SSHStore::getFSAccessor()
|
||||||
ref<RemoteStore::Connection> SSHStore::openConnection()
|
ref<RemoteStore::Connection> SSHStore::openConnection()
|
||||||
{
|
{
|
||||||
if ((pid_t) sshMaster == -1) {
|
if ((pid_t) sshMaster == -1) {
|
||||||
|
auto flags = compress ? "-NMCS" : "-NMS";
|
||||||
sshMaster = startProcess([&]() {
|
sshMaster = startProcess([&]() {
|
||||||
if (key.empty())
|
if (key.empty())
|
||||||
execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), uri.c_str(), NULL);
|
execlp("ssh", "ssh", flags, socketPath.c_str(), uri.c_str(), NULL);
|
||||||
else
|
else
|
||||||
execlp("ssh", "ssh", "-N", "-M", "-S", socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
|
execlp("ssh", "ssh", flags, socketPath.c_str(), "-i", key.c_str(), uri.c_str(), NULL);
|
||||||
throw SysError("starting ssh master");
|
throw SysError("starting ssh master");
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,7 @@
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
#include "nar-info-disk-cache.hh"
|
#include "nar-info-disk-cache.hh"
|
||||||
#include "thread-pool.hh"
|
#include "thread-pool.hh"
|
||||||
|
#include "derivations.hh"
|
||||||
|
|
||||||
#include <future>
|
#include <future>
|
||||||
|
|
||||||
|
@ -699,8 +700,27 @@ std::list<ref<Store>> getDefaultSubstituters()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths)
|
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute)
|
||||||
{
|
{
|
||||||
|
if (substitute) {
|
||||||
|
/* Filter out .drv files (we don't want to build anything). */
|
||||||
|
PathSet paths2;
|
||||||
|
for (auto & path : storePaths)
|
||||||
|
if (!isDerivation(path)) paths2.insert(path);
|
||||||
|
unsigned long long downloadSize, narSize;
|
||||||
|
PathSet willBuild, willSubstitute, unknown;
|
||||||
|
to->queryMissing(PathSet(paths2.begin(), paths2.end()),
|
||||||
|
willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||||
|
/* FIXME: should use ensurePath(), but it only
|
||||||
|
does one path at a time. */
|
||||||
|
if (!willSubstitute.empty())
|
||||||
|
try {
|
||||||
|
to->buildPaths(willSubstitute);
|
||||||
|
} catch (Error & e) {
|
||||||
|
printMsg(lvlError, format("warning: %1%") % e.msg());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
std::string copiedLabel = "copied";
|
std::string copiedLabel = "copied";
|
||||||
|
|
||||||
logger->setExpected(copiedLabel, storePaths.size());
|
logger->setExpected(copiedLabel, storePaths.size());
|
||||||
|
|
|
@ -625,7 +625,7 @@ void removeTempRoots();
|
||||||
ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
|
ref<Store> openStore(const std::string & uri = getEnv("NIX_REMOTE"));
|
||||||
|
|
||||||
|
|
||||||
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths);
|
void copyPaths(ref<Store> from, ref<Store> to, const Paths & storePaths, bool substitute = false);
|
||||||
|
|
||||||
enum StoreType {
|
enum StoreType {
|
||||||
tDaemon,
|
tDaemon,
|
||||||
|
|
7
src/nix-copy-closure/local.mk
Normal file
7
src/nix-copy-closure/local.mk
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
programs += nix-copy-closure
|
||||||
|
|
||||||
|
nix-copy-closure_DIR := $(d)
|
||||||
|
|
||||||
|
nix-copy-closure_LIBS = libmain libutil libformat libstore
|
||||||
|
|
||||||
|
nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc
|
60
src/nix-copy-closure/nix-copy-closure.cc
Executable file
60
src/nix-copy-closure/nix-copy-closure.cc
Executable file
|
@ -0,0 +1,60 @@
|
||||||
|
#include "shared.hh"
|
||||||
|
#include "store-api.hh"
|
||||||
|
|
||||||
|
using namespace nix;
|
||||||
|
|
||||||
|
int main(int argc, char ** argv)
|
||||||
|
{
|
||||||
|
return handleExceptions(argv[0], [&]() {
|
||||||
|
initNix();
|
||||||
|
auto gzip = false;
|
||||||
|
auto toMode = true;
|
||||||
|
auto includeOutputs = false;
|
||||||
|
auto dryRun = false;
|
||||||
|
auto useSubstitutes = false;
|
||||||
|
auto sshHost = string{};
|
||||||
|
auto storePaths = PathSet{};
|
||||||
|
parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
|
||||||
|
if (*arg == "--help")
|
||||||
|
showManPage("nix-copy-closure");
|
||||||
|
else if (*arg == "--version")
|
||||||
|
printVersion("nix-copy-closure");
|
||||||
|
else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") {
|
||||||
|
if (*arg != "--gzip")
|
||||||
|
printMsg(lvlError, format("Warning: ‘%1%’ is not implemented, falling back to gzip") % *arg);
|
||||||
|
gzip = true;
|
||||||
|
} else if (*arg == "--from")
|
||||||
|
toMode = false;
|
||||||
|
else if (*arg == "--to")
|
||||||
|
toMode = true;
|
||||||
|
else if (*arg == "--include-outputs")
|
||||||
|
includeOutputs = true;
|
||||||
|
else if (*arg == "--show-progress")
|
||||||
|
printMsg(lvlError, "Warning: ‘--show-progress’ is not implemented");
|
||||||
|
else if (*arg == "--dry-run")
|
||||||
|
dryRun = true;
|
||||||
|
else if (*arg == "--use-substitutes" || *arg == "-s")
|
||||||
|
useSubstitutes = true;
|
||||||
|
else if (sshHost.empty())
|
||||||
|
sshHost = *arg;
|
||||||
|
else
|
||||||
|
storePaths.insert(*arg);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
if (sshHost.empty())
|
||||||
|
throw UsageError("no host name specified");
|
||||||
|
|
||||||
|
auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : "");
|
||||||
|
auto to = toMode ? openStore(remoteUri) : openStore();
|
||||||
|
auto from = toMode ? openStore() : openStore(remoteUri);
|
||||||
|
if (includeOutputs) {
|
||||||
|
auto newPaths = PathSet{};
|
||||||
|
for (const auto & p : storePaths) {
|
||||||
|
auto outputs = from->queryDerivationOutputs(p);
|
||||||
|
newPaths.insert(outputs.begin(), outputs.end());
|
||||||
|
}
|
||||||
|
storePaths.insert(newPaths.begin(), newPaths.end());
|
||||||
|
}
|
||||||
|
copyPaths(from, to, Paths(storePaths.begin(), storePaths.end()), useSubstitutes);
|
||||||
|
});
|
||||||
|
}
|
Loading…
Reference in a new issue