Run the daemon worker on the same CPU as the client
On a system with multiple CPUs, running Nix operations through the daemon is significantly slower than "direct" mode: $ NIX_REMOTE= nix-instantiate '<nixos>' -A system real 0m0.974s user 0m0.875s sys 0m0.088s $ NIX_REMOTE=daemon nix-instantiate '<nixos>' -A system real 0m2.118s user 0m1.463s sys 0m0.218s The main reason seems to be that the client and the worker get moved to a different CPU after every call to the worker. This patch adds a hack to lock them to the same CPU. With this, the overhead of going through the daemon is very small: $ NIX_REMOTE=daemon nix-instantiate '<nixos>' -A system real 0m1.074s user 0m0.809s sys 0m0.098s
This commit is contained in:
parent
263d668222
commit
a583a2bc59
9 changed files with 92 additions and 4 deletions
|
@ -127,6 +127,10 @@ AC_CHECK_HEADERS([sys/mount.h], [], [],
|
||||||
AC_CHECK_FUNCS([lutimes])
|
AC_CHECK_FUNCS([lutimes])
|
||||||
|
|
||||||
|
|
||||||
|
# Check for sched_setaffinity.
|
||||||
|
AC_CHECK_FUNCS([sched_setaffinity])
|
||||||
|
|
||||||
|
|
||||||
# Check whether the store optimiser can optimise symlinks.
|
# Check whether the store optimiser can optimise symlinks.
|
||||||
AC_MSG_CHECKING([whether it is possible to create a link to a symlink])
|
AC_MSG_CHECKING([whether it is possible to create a link to a symlink])
|
||||||
ln -s bla tmp_link
|
ln -s bla tmp_link
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
#include "local-store.hh"
|
#include "local-store.hh"
|
||||||
#include "util.hh"
|
#include "util.hh"
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
|
#include "affinity.hh"
|
||||||
|
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
|
@ -366,6 +367,8 @@ void Goal::trace(const format & f)
|
||||||
/* Common initialisation performed in child processes. */
|
/* Common initialisation performed in child processes. */
|
||||||
static void commonChildInit(Pipe & logPipe)
|
static void commonChildInit(Pipe & logPipe)
|
||||||
{
|
{
|
||||||
|
restoreAffinity();
|
||||||
|
|
||||||
/* Put the child in a separate session (and thus a separate
|
/* Put the child in a separate session (and thus a separate
|
||||||
process group) so that it has no controlling terminal (meaning
|
process group) so that it has no controlling terminal (meaning
|
||||||
that e.g. ssh cannot open /dev/tty) and it doesn't receive
|
that e.g. ssh cannot open /dev/tty) and it doesn't receive
|
||||||
|
@ -568,6 +571,7 @@ static void runSetuidHelper(const string & command,
|
||||||
args.push_back(0);
|
args.push_back(0);
|
||||||
|
|
||||||
restoreSIGPIPE();
|
restoreSIGPIPE();
|
||||||
|
restoreAffinity();
|
||||||
|
|
||||||
execve(program.c_str(), (char * *) &args[0], 0);
|
execve(program.c_str(), (char * *) &args[0], 0);
|
||||||
throw SysError(format("executing `%1%'") % program);
|
throw SysError(format("executing `%1%'") % program);
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
#include "pathlocks.hh"
|
#include "pathlocks.hh"
|
||||||
#include "worker-protocol.hh"
|
#include "worker-protocol.hh"
|
||||||
#include "derivations.hh"
|
#include "derivations.hh"
|
||||||
|
#include "affinity.hh"
|
||||||
|
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
|
@ -1021,6 +1022,7 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
|
||||||
|
|
||||||
case 0: /* child */
|
case 0: /* child */
|
||||||
try {
|
try {
|
||||||
|
restoreAffinity();
|
||||||
if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
|
if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
|
||||||
throw SysError("dupping stdin");
|
throw SysError("dupping stdin");
|
||||||
if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
|
if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
#include "remote-store.hh"
|
#include "remote-store.hh"
|
||||||
#include "worker-protocol.hh"
|
#include "worker-protocol.hh"
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
|
#include "affinity.hh"
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
|
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
|
@ -15,7 +16,6 @@
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
|
|
||||||
|
|
||||||
namespace nix {
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
|
@ -71,8 +71,19 @@ void RemoteStore::openConnection(bool reserveSpace)
|
||||||
if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
|
if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
|
||||||
throw Error("Nix daemon protocol version not supported");
|
throw Error("Nix daemon protocol version not supported");
|
||||||
writeInt(PROTOCOL_VERSION, to);
|
writeInt(PROTOCOL_VERSION, to);
|
||||||
|
|
||||||
|
if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) {
|
||||||
|
int cpu = lockToCurrentCPU();
|
||||||
|
if (cpu != -1) {
|
||||||
|
writeInt(1, to);
|
||||||
|
writeInt(cpu, to);
|
||||||
|
} else
|
||||||
|
writeInt(0, to);
|
||||||
|
}
|
||||||
|
|
||||||
if (GET_PROTOCOL_MINOR(daemonVersion) >= 11)
|
if (GET_PROTOCOL_MINOR(daemonVersion) >= 11)
|
||||||
writeInt(reserveSpace, to);
|
writeInt(reserveSpace, to);
|
||||||
|
|
||||||
processStderr();
|
processStderr();
|
||||||
}
|
}
|
||||||
catch (Error & e) {
|
catch (Error & e) {
|
||||||
|
|
|
@ -6,7 +6,7 @@ namespace nix {
|
||||||
#define WORKER_MAGIC_1 0x6e697863
|
#define WORKER_MAGIC_1 0x6e697863
|
||||||
#define WORKER_MAGIC_2 0x6478696f
|
#define WORKER_MAGIC_2 0x6478696f
|
||||||
|
|
||||||
#define PROTOCOL_VERSION 0x10d
|
#define PROTOCOL_VERSION 0x10e
|
||||||
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
|
||||||
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
|
||||||
|
|
||||||
|
|
|
@ -1,12 +1,12 @@
|
||||||
pkglib_LTLIBRARIES = libutil.la
|
pkglib_LTLIBRARIES = libutil.la
|
||||||
|
|
||||||
libutil_la_SOURCES = util.cc hash.cc serialise.cc \
|
libutil_la_SOURCES = util.cc hash.cc serialise.cc \
|
||||||
archive.cc xml-writer.cc
|
archive.cc xml-writer.cc affinity.cc
|
||||||
|
|
||||||
libutil_la_LIBADD = ../boost/format/libformat.la
|
libutil_la_LIBADD = ../boost/format/libformat.la
|
||||||
|
|
||||||
pkginclude_HEADERS = util.hh hash.hh serialise.hh \
|
pkginclude_HEADERS = util.hh hash.hh serialise.hh \
|
||||||
archive.hh xml-writer.hh types.hh
|
archive.hh xml-writer.hh types.hh affinity.hh
|
||||||
|
|
||||||
if !HAVE_OPENSSL
|
if !HAVE_OPENSSL
|
||||||
libutil_la_SOURCES += \
|
libutil_la_SOURCES += \
|
||||||
|
|
54
src/libutil/affinity.cc
Normal file
54
src/libutil/affinity.cc
Normal file
|
@ -0,0 +1,54 @@
|
||||||
|
#include "types.hh"
|
||||||
|
#include "util.hh"
|
||||||
|
#include "affinity.hh"
|
||||||
|
|
||||||
|
#if HAVE_SCHED_H
|
||||||
|
#include <sched.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
|
||||||
|
static bool didSaveAffinity = false;
|
||||||
|
static cpu_set_t savedAffinity;
|
||||||
|
|
||||||
|
|
||||||
|
void setAffinityTo(int cpu)
|
||||||
|
{
|
||||||
|
#if HAVE_SCHED_SETAFFINITY
|
||||||
|
if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return;
|
||||||
|
didSaveAffinity = true;
|
||||||
|
printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu);
|
||||||
|
cpu_set_t newAffinity;
|
||||||
|
CPU_ZERO(&newAffinity);
|
||||||
|
CPU_SET(cpu, &newAffinity);
|
||||||
|
if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1)
|
||||||
|
printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
int lockToCurrentCPU()
|
||||||
|
{
|
||||||
|
#if HAVE_SCHED_SETAFFINITY
|
||||||
|
if (getEnv("NIX_AFFINITY_HACK", "1") == "1") {
|
||||||
|
int cpu = sched_getcpu();
|
||||||
|
if (cpu != -1) setAffinityTo(cpu);
|
||||||
|
return cpu;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void restoreAffinity()
|
||||||
|
{
|
||||||
|
#if HAVE_SCHED_SETAFFINITY
|
||||||
|
if (!didSaveAffinity) return;
|
||||||
|
if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1)
|
||||||
|
printMsg(lvlError, "failed to restore affinity %1%");
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
9
src/libutil/affinity.hh
Normal file
9
src/libutil/affinity.hh
Normal file
|
@ -0,0 +1,9 @@
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
namespace nix {
|
||||||
|
|
||||||
|
void setAffinityTo(int cpu);
|
||||||
|
int lockToCurrentCPU();
|
||||||
|
void restoreAffinity();
|
||||||
|
|
||||||
|
}
|
|
@ -4,6 +4,7 @@
|
||||||
#include "serialise.hh"
|
#include "serialise.hh"
|
||||||
#include "worker-protocol.hh"
|
#include "worker-protocol.hh"
|
||||||
#include "archive.hh"
|
#include "archive.hh"
|
||||||
|
#include "affinity.hh"
|
||||||
#include "globals.hh"
|
#include "globals.hh"
|
||||||
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
|
@ -671,6 +672,9 @@ static void processConnection(bool trusted)
|
||||||
to.flush();
|
to.flush();
|
||||||
unsigned int clientVersion = readInt(from);
|
unsigned int clientVersion = readInt(from);
|
||||||
|
|
||||||
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from))
|
||||||
|
setAffinityTo(readInt(from));
|
||||||
|
|
||||||
bool reserveSpace = true;
|
bool reserveSpace = true;
|
||||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
|
if (GET_PROTOCOL_MINOR(clientVersion) >= 11)
|
||||||
reserveSpace = readInt(from) != 0;
|
reserveSpace = readInt(from) != 0;
|
||||||
|
|
Loading…
Reference in a new issue