* Buffer writes in FdSink. This significantly reduces the number of
system calls / context switches when dumping a NAR and in the worker protocol.
This commit is contained in:
parent
893cac1402
commit
3a48282b06
4 changed files with 51 additions and 15 deletions
|
@ -65,6 +65,7 @@ void RemoteStore::openConnection()
|
||||||
/* Send the magic greeting, check for the reply. */
|
/* Send the magic greeting, check for the reply. */
|
||||||
try {
|
try {
|
||||||
writeInt(WORKER_MAGIC_1, to);
|
writeInt(WORKER_MAGIC_1, to);
|
||||||
|
to.flush();
|
||||||
unsigned int magic = readInt(from);
|
unsigned int magic = readInt(from);
|
||||||
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
|
if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch");
|
||||||
|
|
||||||
|
@ -166,6 +167,7 @@ void RemoteStore::connectToDaemon()
|
||||||
RemoteStore::~RemoteStore()
|
RemoteStore::~RemoteStore()
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
|
to.flush();
|
||||||
fdSocket.close();
|
fdSocket.close();
|
||||||
if (child != -1)
|
if (child != -1)
|
||||||
child.wait(true);
|
child.wait(true);
|
||||||
|
@ -488,6 +490,7 @@ void RemoteStore::clearFailedPaths(const PathSet & paths)
|
||||||
|
|
||||||
void RemoteStore::processStderr(Sink * sink, Source * source)
|
void RemoteStore::processStderr(Sink * sink, Source * source)
|
||||||
{
|
{
|
||||||
|
to.flush();
|
||||||
unsigned int msg;
|
unsigned int msg;
|
||||||
while ((msg = readInt(from)) == STDERR_NEXT
|
while ((msg = readInt(from)) == STDERR_NEXT
|
||||||
|| msg == STDERR_READ || msg == STDERR_WRITE) {
|
|| msg == STDERR_READ || msg == STDERR_WRITE) {
|
||||||
|
@ -503,6 +506,7 @@ void RemoteStore::processStderr(Sink * sink, Source * source)
|
||||||
AutoDeleteArray<unsigned char> d(buf);
|
AutoDeleteArray<unsigned char> d(buf);
|
||||||
(*source)(buf, len);
|
(*source)(buf, len);
|
||||||
writeString(string((const char *) buf, len), to);
|
writeString(string((const char *) buf, len), to);
|
||||||
|
to.flush();
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
string s = readString(from);
|
string s = readString(from);
|
||||||
|
|
|
@ -9,7 +9,30 @@ namespace nix {
|
||||||
|
|
||||||
void FdSink::operator () (const unsigned char * data, unsigned int len)
|
void FdSink::operator () (const unsigned char * data, unsigned int len)
|
||||||
{
|
{
|
||||||
writeFull(fd, data, len);
|
if (!buffer) buffer = new unsigned char[bufSize];
|
||||||
|
|
||||||
|
while (len) {
|
||||||
|
/* Optimisation: bypass the buffer if the data exceeds the
|
||||||
|
buffer size and there is no unflushed data. */
|
||||||
|
if (bufPos == 0 && len >= bufSize) {
|
||||||
|
writeFull(fd, data, len);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
/* Otherwise, copy the bytes to the buffer. Flush the buffer
|
||||||
|
when it's full. */
|
||||||
|
size_t n = bufPos + len > bufSize ? bufSize - bufPos : len;
|
||||||
|
memcpy(buffer + bufPos, data, n);
|
||||||
|
data += n; bufPos += n; len -= n;
|
||||||
|
if (bufPos == bufSize) flush();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
void FdSink::flush()
|
||||||
|
{
|
||||||
|
if (fd == -1 || bufPos == 0) return;
|
||||||
|
writeFull(fd, buffer, bufPos);
|
||||||
|
bufPos = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -28,22 +28,29 @@ struct Source
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/* A sink that writes data to a file descriptor. */
|
/* A sink that writes data to a file descriptor (using a buffer). */
|
||||||
struct FdSink : Sink
|
struct FdSink : Sink
|
||||||
{
|
{
|
||||||
int fd;
|
int fd;
|
||||||
|
unsigned int bufSize, bufPos;
|
||||||
|
unsigned char * buffer;
|
||||||
|
|
||||||
FdSink()
|
FdSink() : fd(-1), bufSize(32 * 1024), bufPos(0), buffer(0) { }
|
||||||
{
|
|
||||||
fd = -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
FdSink(int fd)
|
FdSink(int fd, unsigned int bufSize = 32 * 1024)
|
||||||
|
: fd(fd), bufSize(bufSize), bufPos(0), buffer(0)
|
||||||
{
|
{
|
||||||
this->fd = fd;
|
}
|
||||||
|
|
||||||
|
~FdSink()
|
||||||
|
{
|
||||||
|
flush();
|
||||||
|
if (buffer) delete[] buffer;
|
||||||
}
|
}
|
||||||
|
|
||||||
void operator () (const unsigned char * data, unsigned int len);
|
void operator () (const unsigned char * data, unsigned int len);
|
||||||
|
|
||||||
|
void flush();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -57,6 +57,7 @@ static void tunnelStderr(const unsigned char * buf, size_t count)
|
||||||
try {
|
try {
|
||||||
writeInt(STDERR_NEXT, to);
|
writeInt(STDERR_NEXT, to);
|
||||||
writeString(string((char *) buf, count), to);
|
writeString(string((char *) buf, count), to);
|
||||||
|
to.flush();
|
||||||
} catch (...) {
|
} catch (...) {
|
||||||
/* Write failed; that means that the other side is
|
/* Write failed; that means that the other side is
|
||||||
gone. */
|
gone. */
|
||||||
|
@ -200,9 +201,7 @@ static void stopWork(bool success = true, const string & msg = "", unsigned int
|
||||||
struct TunnelSink : Sink
|
struct TunnelSink : Sink
|
||||||
{
|
{
|
||||||
Sink & to;
|
Sink & to;
|
||||||
TunnelSink(Sink & to) : to(to)
|
TunnelSink(Sink & to) : to(to) { }
|
||||||
{
|
|
||||||
}
|
|
||||||
virtual void operator ()
|
virtual void operator ()
|
||||||
(const unsigned char * data, unsigned int len)
|
(const unsigned char * data, unsigned int len)
|
||||||
{
|
{
|
||||||
|
@ -215,9 +214,7 @@ struct TunnelSink : Sink
|
||||||
struct TunnelSource : Source
|
struct TunnelSource : Source
|
||||||
{
|
{
|
||||||
Source & from;
|
Source & from;
|
||||||
TunnelSource(Source & from) : from(from)
|
TunnelSource(Source & from) : from(from) { }
|
||||||
{
|
|
||||||
}
|
|
||||||
virtual void operator ()
|
virtual void operator ()
|
||||||
(unsigned char * data, unsigned int len)
|
(unsigned char * data, unsigned int len)
|
||||||
{
|
{
|
||||||
|
@ -228,6 +225,7 @@ struct TunnelSource : Source
|
||||||
|
|
||||||
writeInt(STDERR_READ, to);
|
writeInt(STDERR_READ, to);
|
||||||
writeInt(len, to);
|
writeInt(len, to);
|
||||||
|
to.flush();
|
||||||
string s = readString(from);
|
string s = readString(from);
|
||||||
if (s.size() != len) throw Error("not enough data");
|
if (s.size() != len) throw Error("not enough data");
|
||||||
memcpy(data, (const unsigned char *) s.c_str(), len);
|
memcpy(data, (const unsigned char *) s.c_str(), len);
|
||||||
|
@ -596,8 +594,8 @@ static void processConnection()
|
||||||
unsigned int magic = readInt(from);
|
unsigned int magic = readInt(from);
|
||||||
if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
|
if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
|
||||||
writeInt(WORKER_MAGIC_2, to);
|
writeInt(WORKER_MAGIC_2, to);
|
||||||
|
|
||||||
writeInt(PROTOCOL_VERSION, to);
|
writeInt(PROTOCOL_VERSION, to);
|
||||||
|
to.flush();
|
||||||
unsigned int clientVersion = readInt(from);
|
unsigned int clientVersion = readInt(from);
|
||||||
|
|
||||||
/* Send startup error messages to the client. */
|
/* Send startup error messages to the client. */
|
||||||
|
@ -619,9 +617,11 @@ static void processConnection()
|
||||||
store = boost::shared_ptr<StoreAPI>(new LocalStore());
|
store = boost::shared_ptr<StoreAPI>(new LocalStore());
|
||||||
|
|
||||||
stopWork();
|
stopWork();
|
||||||
|
to.flush();
|
||||||
|
|
||||||
} catch (Error & e) {
|
} catch (Error & e) {
|
||||||
stopWork(false, e.msg());
|
stopWork(false, e.msg());
|
||||||
|
to.flush();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -652,6 +652,8 @@ static void processConnection()
|
||||||
if (!errorAllowed) break;
|
if (!errorAllowed) break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
to.flush();
|
||||||
|
|
||||||
assert(!canSendStderr);
|
assert(!canSendStderr);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue