mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

S3BinaryCacheStore: Share TransferManager

Browse source 
TransferManager allocates a lot of memory (50 MiB by default), and it
might leak but I'm not sure about that. In any case it was causing
OOMs in hydra-queue-runner. So allocate only one TransferManager per
S3BinaryCacheStore.

Hopefully fixes https://github.com/NixOS/hydra/issues/586.
This commit is contained in:
Eelco Dolstra 2018-08-09 20:44:18 +02:00
parent c87f4b9324
commit ada4e90267
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
1 changed files with 22 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
src/libstore/s3-binary-cache-store.cc
View file

@ -275,6 +275,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
return true; return true;
} }
std::shared_ptr<TransferManager> transferManager;
std::once_flag transferManagerCreated;
void uploadFile(const std::string & path, const std::string & data, void uploadFile(const std::string & path, const std::string & data,
const std::string & mimeType, const std::string & mimeType,
const std::string & contentEncoding) const std::string & contentEncoding)
@ -286,6 +289,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads); executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads);
std::call_once(transferManagerCreated, [&]() {
TransferManagerConfiguration transferConfig(executor.get()); TransferManagerConfiguration transferConfig(executor.get());
transferConfig.s3Client = s3Helper.client; transferConfig.s3Client = s3Helper.client;
@ -294,7 +299,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
transferConfig.uploadProgressCallback = transferConfig.uploadProgressCallback =
[&](const TransferManager *transferManager, [&](const TransferManager *transferManager,
const std::shared_ptr<const TransferHandle> const std::shared_ptr<const TransferHandle>
&transferHandle) { &transferHandle)
{
//FIXME: find a way to properly abort the multipart upload. //FIXME: find a way to properly abort the multipart upload.
//checkInterrupt(); //checkInterrupt();
debug("upload progress ('%s'): '%d' of '%d' bytes", debug("upload progress ('%s'): '%d' of '%d' bytes",
@ -303,8 +309,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
transferHandle->GetBytesTotalSize()); transferHandle->GetBytesTotalSize());
}; };
std::shared_ptr<TransferManager> transferManager = transferManager = TransferManager::Create(transferConfig);
TransferManager::Create(transferConfig); });
auto now1 = std::chrono::steady_clock::now(); auto now1 = std::chrono::steady_clock::now();