mdebray/tvl-depot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Resume NAR downloads

Browse source 
This is a much simpler fix to the 'error 9 while decompressing xz
file' problem than 78fa47a7f0. We just
do a ranged HTTP request starting after the data that we previously
wrote into the sink.

Fixes #2952, #379.
This commit is contained in:
Eelco Dolstra 2019-07-10 23:12:17 +02:00
parent 00f6fafad6
commit 53247d6b11
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
1 changed files with 28 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
src/libstore/download.cc
View file

@ -71,6 +71,10 @@ struct CurlDownloader : public Downloader
std::string encoding; std::string encoding;
bool acceptRanges = false;
curl_off_t writtenToSink = 0;
DownloadItem(CurlDownloader & downloader, DownloadItem(CurlDownloader & downloader,
const DownloadRequest & request, const DownloadRequest & request,
Callback<DownloadResult> callback) Callback<DownloadResult> callback)
@ -81,9 +85,10 @@ struct CurlDownloader : public Downloader
{request.uri}, request.parentAct) {request.uri}, request.parentAct)
, callback(callback) , callback(callback)
, finalSink([this](const unsigned char * data, size_t len) { , finalSink([this](const unsigned char * data, size_t len) {
if (this->request.dataCallback) if (this->request.dataCallback) {
writtenToSink += len;
this->request.dataCallback((char *) data, len); this->request.dataCallback((char *) data, len);
else } else
this->result.data->append((char *) data, len); this->result.data->append((char *) data, len);
}) })
{ {
@ -161,6 +166,7 @@ struct CurlDownloader : public Downloader
status = ss.size() >= 2 ? ss[1] : ""; status = ss.size() >= 2 ? ss[1] : "";
result.data = std::make_shared<std::string>(); result.data = std::make_shared<std::string>();
result.bodySize = 0; result.bodySize = 0;
acceptRanges = false;
encoding = ""; encoding = "";
} else { } else {
auto i = line.find(':'); auto i = line.find(':');
@ -178,7 +184,9 @@ struct CurlDownloader : public Downloader
return 0; return 0;
} }
} else if (name == "content-encoding") } else if (name == "content-encoding")
encoding = trim(string(line, i + 1));; encoding = trim(string(line, i + 1));
else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes")
acceptRanges = true;
} }
} }
return realSize; return realSize;
@ -296,6 +304,9 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str());
curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
if (writtenToSink)
curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink);
result.data = std::make_shared<std::string>(); result.data = std::make_shared<std::string>();
result.bodySize = 0; result.bodySize = 0;
} }
@ -330,7 +341,7 @@ struct CurlDownloader : public Downloader
failEx(writeException); failEx(writeException);
else if (code == CURLE_OK && else if (code == CURLE_OK &&
(httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */))
{ {
result.cached = httpStatus == 304; result.cached = httpStatus == 304;
done = true; done = true;
@ -403,9 +414,19 @@ struct CurlDownloader : public Downloader
request.verb(), request.uri, curl_easy_strerror(code), code)); request.verb(), request.uri, curl_easy_strerror(code), code));
/* If this is a transient error, then maybe retry the /* If this is a transient error, then maybe retry the
download after a while. */ download after a while. If we're writing to a
if (err == Transient && attempt < request.tries) { sink, we can only retry if the server supports
ranged requests. */
if (err == Transient
&& attempt < request.tries
&& (!this->request.dataCallback
|| writtenToSink == 0
|| (acceptRanges && encoding.empty())))
{
int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
if (writtenToSink)
warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
else
warn("%s; retrying in %d ms", exc.what(), ms); warn("%s; retrying in %d ms", exc.what(), ms);
embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
downloader.enqueueItem(shared_from_this()); downloader.enqueueItem(shared_from_this());