17 CurlMultiPartSetoptError ( CURLcode code ) :
_code(code){};
18 CURLcode code()
const;
23 CURLcode CurlMultiPartSetoptError::code()
const {
30 CurlMultInitRangeError ( std::string what ) :
zypp::Exception(
std::move(what) ){}
66 ._digest = _digest ? _digest->clone() : std::optional<zypp::Digest>{},
67 ._checksum = _checksum,
68 ._relevantDigestLen = _relevantDigestLen,
69 ._chksumPad = _chksumPad,
71 ._rangeState = _rangeState
81 ._digest = std::move( digest ),
82 ._checksum = std::move( expectedChkSum ),
83 ._relevantDigestLen = std::move( digestCompareLen ),
84 ._chksumPad = std::move( dataBlockPadding ),
85 .userData = std::move( userData ),
86 ._rangeState = State::Pending
98 WAR <<
"!!!! Downloading ranges without HTTP might be slow !!!!" << std::endl;
106 curl_easy_setopt(
_easyHandle, CURLOPT_HEADERFUNCTION,
nullptr );
107 curl_easy_setopt(
_easyHandle, CURLOPT_HEADERDATA,
nullptr );
108 curl_easy_setopt(
_easyHandle, CURLOPT_WRITEFUNCTION,
nullptr );
109 curl_easy_setopt(
_easyHandle, CURLOPT_WRITEDATA,
nullptr );
117 if ( size * nmemb == 0)
122 std::string_view hdr( ptr, size*nmemb );
124 hdr.remove_prefix( std::min( hdr.find_first_not_of(
" \t\r\n"), hdr.size() ) );
125 const auto lastNonWhitespace = hdr.find_last_not_of(
" \t\r\n");
126 if ( lastNonWhitespace != hdr.npos )
127 hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
129 hdr = std::string_view();
133 return ( size * nmemb );
139 (void)curl_easy_getinfo(
_easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
141 const auto &doRangeFail = [&](){
142 WAR <<
_easyHandle <<
" " <<
"Range FAIL, trying with a smaller batch" << std::endl;
144 setCode ( Code::RangeFail,
"Expected range status code 206, but got none." );
148 if ( range._rangeState ==
Running )
155 if ( ( statuscode >= 200 && statuscode <= 299 && statuscode != 206 )
156 || statuscode == 416 ) {
157 return doRangeFail();
171 WAR <<
"Setting request filesize to " << fileLen << std::endl;
194 const auto max = ( size * nmemb );
201 size_t bytesConsumedSoFar = 0;
202 while ( bytesConsumedSoFar != max ) {
204 std::optional<off_t> seekTo;
213 setCode( Code::ServerReturnedError,
"Invalid data from server, range respone was announced but there was no range definiton." );
231 std::optional<uint> foundRange;
234 auto currDist = ULONG_MAX;
239 if ( currR._rangeState ==
Finished || currR._rangeState ==
Error )
243 if ( currR.len && currR.len == currR.bytesWritten )
246 const auto currRBegin = currR.start + currR.bytesWritten;
247 if ( !( beginSrvRange <= currRBegin && endSrvRange >= currRBegin ) )
251 const auto newDist = currRBegin - beginSrvRange;
264 if ( newDist < currDist ) {
284 const auto skipBytes = *seekTo - beginSrvRange;
285 bytesConsumedSoFar += skipBytes;
288 std::string errBuf =
"Receiver cancelled starting the current range.";
318 auto availableData = max - bytesConsumedSoFar;
326 const auto bytesToWrite = rng.len > 0 ? std::min( rng.len - rng.bytesWritten, availableData ) : availableData;
332 if ( rng._digest && rng._checksum.size() ) {
333 if ( !rng._digest->update( ptr + bytesConsumedSoFar, written ) )
337 rng.bytesWritten += written;
341 if ( rng.len > 0 && rng.bytesWritten >= rng.len ) {
342 std::string errBuf =
"Receiver cancelled after finishing the current range.";
359 bytesConsumedSoFar += written;
362 if ( bytesConsumedSoFar == max )
371 std::string_view incoming( ptr + bytesConsumedSoFar, max - bytesConsumedSoFar );
372 auto hdrEnd = incoming.find(
"\r\n\r\n");
373 if ( hdrEnd == incoming.npos ) {
381 bytesConsumedSoFar += ( hdrEnd + 4 );
385 if ( sepStrIndex == data.npos ) {
391 std::vector<std::string_view>
lines;
393 for (
const auto &hdrLine :
lines ) {
420 return bytesConsumedSoFar;
465 setCode ( Code::RangeFail,
"No more range batch sizes available",
true );
473 setCode ( Code::NoError,
"Request has no more work",
true );
496 if ( r.len > 0 && r.bytesWritten != r.len )
497 setCode( Code::MissingData, (
zypp::str::Format(
"Did not receive all requested data from the server ( off: %1%, req: %2%, recv: %3% ).") % r.start % r.len % r.bytesWritten ) );
522 setCode(
Code::InternalError,
"Calling the CurlMultiPartHandler::prepare function without a range to download is not supported.");
526 const auto setCurlOption = [&]( CURLoption opt,
auto &&data )
528 auto ret = curl_easy_setopt(
_easyHandle, opt, data );
530 throw CurlMultiPartSetoptError(ret);
536 setCurlOption( CURLOPT_HEADERDATA,
this );
538 setCurlOption( CURLOPT_WRITEDATA,
this );
540 std::string rangeDesc;
541 uint rangesAdded = 0;
545 auto addRangeString = [ &rangeDesc, &rangesAdded ](
const std::pair<size_t, size_t> &range ) {
546 std::string rangeD =
zypp::str::form(
"%llu-", static_cast<unsigned long long>( range.first ) );
547 if( range.second > 0 )
548 rangeD.append(
zypp::str::form(
"%llu", static_cast<unsigned long long>( range.second ) ) );
550 if ( rangeDesc.size() )
551 rangeDesc.append(
",").append( rangeD );
553 rangeDesc = std::move( rangeD );
558 std::optional<std::pair<size_t, size_t>> currentZippedRange;
559 bool closedRange =
true;
562 if ( range._rangeState !=
Pending )
566 range.bytesWritten = 0;
571 throw CurlMultInitRangeError(
"It is not supported to request more ranges after a open range.");
573 const auto rangeEnd = range.len > 0 ? range.start + range.len - 1 : 0;
574 closedRange = (rangeEnd > 0);
580 if ( !currentZippedRange ) {
582 currentZippedRange = std::make_pair( range.start, rangeEnd );
585 if ( currentZippedRange->second + 1 == range.start ) {
587 currentZippedRange->second = rangeEnd;
590 if ( rangesAdded +1 >= maxRanges )
break;
592 addRangeString( *currentZippedRange );
593 currentZippedRange = std::make_pair( range.start, rangeEnd );
600 range.bytesWritten = 0;
602 range._digest->reset();
605 if ( rangesAdded >= maxRanges ) {
606 MIL <<
_easyHandle <<
" " <<
"Reached max nr of ranges (" << maxRanges <<
"), batching the request to not break the server" << std::endl;
612 if ( currentZippedRange )
613 addRangeString( *currentZippedRange );
615 MIL <<
_easyHandle <<
" " <<
"Requesting Ranges: " << rangeDesc << std::endl;
617 setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() );
619 }
catch(
const CurlMultiPartSetoptError &err ) {
621 }
catch(
const CurlMultInitRangeError &err ) {
625 }
catch(
const std::exception &err ) {
634 if (
_lastCode != Code::NoError && !force )
649 DBG <<
_easyHandle <<
" " <<
"Invalid Content-Range Header format: '" << std::string(line) << std::endl;
653 size_t s = zypp::str::strtonum<size_t>( what[1]);
654 size_t e = zypp::str::strtonum<size_t>( what[2]);
655 fileLen = zypp::str::strtonum<size_t>( what[3]);
667 if ( what.
size() >= 2 ) {
694 auto bytesHashed = rng.
_digest->bytesHashed ();
698 rng.
_digest->update( padding.data(), padding.size() );
700 auto digVec = rng.
_digest->digestVector();
std::optional< size_t > reportedFileSize() const
CurlMultiPartDataReceiver & _receiver
std::optional< size_t > _chksumPad
std::string _seperatorString
The seperator string for multipart responses as defined in RFC 7233 Section 4.1.
The CurlMultiPartHandler class.
bool hasPrefixCI(const C_Str &str_r, const C_Str &prefix_r)
static Range make(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
size_t hdrcallback(char *ptr, size_t size, size_t nmemb)
virtual void notifyErrorCodeChanged()
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
void setRangeState(Range &rng, State state)
~CurlMultiPartHandler() override
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
static const char * lines[][3]
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Do not differentiate case.
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
const std::string & lastErrorMessage() const
std::string trim(const std::string &s, const Trim trim_r)
std::optional< Range > _currentSrvRange
void setCode(Code c, std::string msg, bool force=false)
std::string asUserString() const
Translated error message as string suitable for the user.
virtual size_t headerfunction(char *ptr, size_t bytes)=0
CheckSumBytes _checksum
Enables automated checking of downloaded contents against a checksum.
virtual size_t writefunction(char *ptr, std::optional< off_t > offset, size_t bytes)=0
std::vector< Range > & _requestedRanges
the requested ranges that need to be downloaded
std::string _lastErrorMsg
static constexpr unsigned _rangeAttemptSize
CurlMultiPartHandler(ProtocolMode mode, void *easyHandle, std::vector< Range > &ranges, CurlMultiPartDataReceiver &receiver)
virtual bool finishedRange(off_t range, bool validated, std::string &cancelReason)
static size_t curl_wrtcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
std::optional< off_t > currentRange() const
Regular expression match result.
bool checkIfRangeChkSumIsValid(Range &rng)
Base class for Exception.
bool any_of(const Container &c, Fnc &&cb)
std::optional< size_t > _relevantDigestLen
virtual bool beginRange(off_t range, std::string &cancelReason)
bool _gotContentRangeInfo
bool validateRange(Range &rng)
bool regex_match(const std::string &s, smatch &matches, const regex ®ex)
regex ZYPP_STR_REGEX regex ZYPP_STR_REGEX
unsigned _rangeAttemptIdx
std::vector< char > _rangePrefaceBuffer
Here we buffer.
These are enforced even if you don't pass them as flag argument.
std::optional< zypp::Digest > _digest
ProtocolMode _protocolMode
Easy-to use interface to the ZYPP dependency resolver.
std::optional< off_t > _currentRange
std::optional< size_t > _reportedFileSize
Filesize as reported by the content range or byte range headers.
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len, size_t &fileLen)
static constexpr unsigned _rangeAttempt[]
static size_t curl_hdrcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
void * easyHandle() const
size_t wrtcallback(char *ptr, size_t size, size_t nmemb)