4 #include <zypp-core/base/StringV.h> 15 CurlMultiPartSetoptError ( CURLcode code ) :
_code(code){};
16 CURLcode code()
const;
21 CURLcode CurlMultiPartSetoptError::code()
const {
28 CurlMultInitRangeError ( std::string what ) :
zypp::Exception(
std::move(what) ){}
64 ._digest = _digest ? _digest->clone() : std::optional<zypp::Digest>{},
65 ._checksum = _checksum,
66 ._relevantDigestLen = _relevantDigestLen,
67 ._chksumPad = _chksumPad,
69 ._rangeState = _rangeState
79 ._digest = std::move( digest ),
80 ._checksum = std::move( expectedChkSum ),
81 ._relevantDigestLen = std::move( digestCompareLen ),
82 ._chksumPad = std::move( dataBlockPadding ),
83 .userData = std::move( userData ),
84 ._rangeState = State::Pending
96 WAR <<
"!!!! Downloading ranges without HTTP might be slow !!!!" << std::endl;
104 curl_easy_setopt(
_easyHandle, CURLOPT_HEADERFUNCTION,
nullptr );
105 curl_easy_setopt(
_easyHandle, CURLOPT_HEADERDATA,
nullptr );
106 curl_easy_setopt(
_easyHandle, CURLOPT_WRITEFUNCTION,
nullptr );
107 curl_easy_setopt(
_easyHandle, CURLOPT_WRITEDATA,
nullptr );
115 if ( size * nmemb == 0)
120 std::string_view hdr( ptr, size*nmemb );
122 hdr.remove_prefix( std::min( hdr.find_first_not_of(
" \t\r\n"), hdr.size() ) );
123 const auto lastNonWhitespace = hdr.find_last_not_of(
" \t\r\n");
124 if ( lastNonWhitespace != hdr.npos )
125 hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
127 hdr = std::string_view();
131 return ( size * nmemb );
137 (void)curl_easy_getinfo(
_easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
139 const auto &doRangeFail = [&](){
140 WAR <<
_easyHandle <<
" " <<
"Range FAIL, trying with a smaller batch" << std::endl;
142 setCode ( Code::RangeFail,
"Expected range status code 206, but got none." );
146 if ( range._rangeState ==
Running )
153 if ( ( statuscode >= 200 && statuscode <= 299 && statuscode != 206 )
154 || statuscode == 416 ) {
155 return doRangeFail();
164 setCode( Code::InternalError,
"Invalid Content-Range header format." );
169 WAR <<
"Setting request filesize to " << fileLen << std::endl;
192 const auto max = ( size * nmemb );
199 size_t bytesConsumedSoFar = 0;
200 while ( bytesConsumedSoFar != max ) {
202 std::optional<off_t> seekTo;
211 setCode( Code::ServerReturnedError,
"Invalid data from server, range respone was announced but there was no range definiton." );
229 std::optional<uint> foundRange;
232 auto currDist = ULONG_MAX;
237 if ( currR._rangeState ==
Finished || currR._rangeState ==
Error )
241 if ( currR.len && currR.len == currR.bytesWritten )
244 const auto currRBegin = currR.start + currR.bytesWritten;
245 if ( !( beginSrvRange <= currRBegin && endSrvRange >= currRBegin ) )
249 const auto newDist = currRBegin - beginSrvRange;
262 if ( newDist < currDist ) {
271 setCode( Code::InternalError,
"Unable to find a matching range for data returned by the server.");
282 const auto skipBytes = *seekTo - beginSrvRange;
283 bytesConsumedSoFar += skipBytes;
286 std::string errBuf =
"Receiver cancelled starting the current range.";
288 setCode( Code::InternalError,
"Receiver cancelled starting the current range.");
299 setCode( Code::InternalError,
"Received data but no range was marked as requested." );
316 auto availableData = max - bytesConsumedSoFar;
324 const auto bytesToWrite = rng.len > 0 ? std::min( rng.len - rng.bytesWritten, availableData ) : availableData;
330 if ( rng._digest && rng._checksum.size() ) {
331 if ( !rng._digest->update( ptr + bytesConsumedSoFar, written ) )
335 rng.bytesWritten += written;
339 if ( rng.len > 0 && rng.bytesWritten >= rng.len ) {
340 std::string errBuf =
"Receiver cancelled after finishing the current range.";
343 setCode( Code::InternalError,
"Receiver cancelled starting the current range.");
357 bytesConsumedSoFar += written;
360 if ( bytesConsumedSoFar == max )
369 std::string_view incoming( ptr + bytesConsumedSoFar, max - bytesConsumedSoFar );
370 auto hdrEnd = incoming.find(
"\r\n\r\n");
371 if ( hdrEnd == incoming.npos ) {
379 bytesConsumedSoFar += ( hdrEnd + 4 );
383 if ( sepStrIndex == data.npos ) {
384 setCode( Code::InternalError,
"Invalid multirange header format, seperator string missing." );
389 std::vector<std::string_view> lines;
391 for (
const auto &hdrLine : lines ) {
397 setCode( Code::InternalError,
"Invalid Content-Range header format." );
409 setCode( Code::InternalError,
"Could not read a server range from the response." );
418 return bytesConsumedSoFar;
463 setCode ( Code::RangeFail,
"No more range batch sizes available",
true );
471 setCode ( Code::NoError,
"Request has no more work",
true );
494 if ( r.len > 0 && r.bytesWritten != r.len )
495 setCode( Code::MissingData, (
zypp::str::Format(
"Did not receive all requested data from the server ( off: %1%, req: %2%, recv: %3% ).") % r.start % r.len % r.bytesWritten ) );
520 setCode( Code::InternalError,
"Calling the CurlMultiPartHandler::prepare function without a range to download is not supported.");
524 const auto setCurlOption = [&]( CURLoption opt,
auto &&data )
526 auto ret = curl_easy_setopt(
_easyHandle, opt, data );
528 throw CurlMultiPartSetoptError(ret);
534 setCurlOption( CURLOPT_HEADERDATA,
this );
536 setCurlOption( CURLOPT_WRITEDATA,
this );
538 std::string rangeDesc;
539 uint rangesAdded = 0;
543 auto addRangeString = [ &rangeDesc, &rangesAdded ](
const std::pair<size_t, size_t> &range ) {
544 std::string rangeD =
zypp::str::form(
"%llu-", static_cast<unsigned long long>( range.first ) );
545 if( range.second > 0 )
546 rangeD.append(
zypp::str::form(
"%llu", static_cast<unsigned long long>( range.second ) ) );
548 if ( rangeDesc.size() )
549 rangeDesc.append(
",").append( rangeD );
551 rangeDesc = std::move( rangeD );
556 std::optional<std::pair<size_t, size_t>> currentZippedRange;
557 bool closedRange =
true;
560 if ( range._rangeState !=
Pending )
564 range.bytesWritten = 0;
569 throw CurlMultInitRangeError(
"It is not supported to request more ranges after a open range.");
571 const auto rangeEnd = range.len > 0 ? range.start + range.len - 1 : 0;
572 closedRange = (rangeEnd > 0);
578 if ( !currentZippedRange ) {
580 currentZippedRange = std::make_pair( range.start, rangeEnd );
583 if ( currentZippedRange->second + 1 == range.start ) {
585 currentZippedRange->second = rangeEnd;
588 if ( rangesAdded +1 >= maxRanges )
break;
590 addRangeString( *currentZippedRange );
591 currentZippedRange = std::make_pair( range.start, rangeEnd );
598 range.bytesWritten = 0;
600 range._digest->reset();
603 if ( rangesAdded >= maxRanges ) {
604 MIL <<
_easyHandle <<
" " <<
"Reached max nr of ranges (" << maxRanges <<
"), batching the request to not break the server" << std::endl;
610 if ( currentZippedRange )
611 addRangeString( *currentZippedRange );
613 MIL <<
_easyHandle <<
" " <<
"Requesting Ranges: " << rangeDesc << std::endl;
615 setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() );
617 }
catch(
const CurlMultiPartSetoptError &err ) {
618 setCode( Code::InternalError,
"" );
619 }
catch(
const CurlMultInitRangeError &err ) {
620 setCode( Code::InternalError, err.asUserString() );
623 }
catch(
const std::exception &err ) {
632 if (
_lastCode != Code::NoError && !force )
646 DBG <<
_easyHandle <<
" " <<
"Invalid Content-Range Header format: '" << std::string(line) << std::endl;
650 size_t s = zypp::str::strtonum<size_t>( what[1]);
651 size_t e = zypp::str::strtonum<size_t>( what[2]);
652 fileLen = zypp::str::strtonum<size_t>( what[3]);
653 start = std::move(s);
664 if ( what.
size() >= 2 ) {
691 auto bytesHashed = rng.
_digest->bytesHashed ();
695 rng.
_digest->update( padding.data(), padding.size() );
697 auto digVec = rng.
_digest->digestVector();
std::optional< size_t > reportedFileSize() const
CurlMultiPartDataReceiver & _receiver
std::optional< size_t > _chksumPad
std::string _seperatorString
The seperator string for multipart responses as defined in RFC 7233 Section 4.1.
The CurlMultiPartHandler class.
bool hasPrefixCI(const C_Str &str_r, const C_Str &prefix_r)
static Range make(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
size_t hdrcallback(char *ptr, size_t size, size_t nmemb)
const std::string & asString(const std::string &t)
Global asString() that works with std::string too.
void setRangeState(Range &rng, State state)
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Do not differentiate case.
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
const std::string & lastErrorMessage() const
std::string trim(const std::string &s, const Trim trim_r)
std::optional< Range > _currentSrvRange
void setCode(Code c, std::string msg, bool force=false)
std::string asUserString() const
Translated error message as string suitable for the user.
virtual size_t headerfunction(char *ptr, size_t bytes)=0
CheckSumBytes _checksum
Enables automated checking of downloaded contents against a checksum.
virtual size_t writefunction(char *ptr, std::optional< off_t > offset, size_t bytes)=0
std::vector< Range > & _requestedRanges
the requested ranges that need to be downloaded
std::string _lastErrorMsg
static constexpr unsigned _rangeAttemptSize
CurlMultiPartHandler(ProtocolMode mode, void *easyHandle, std::vector< Range > &ranges, CurlMultiPartDataReceiver &receiver)
virtual bool finishedRange(off_t range, bool validated, std::string &cancelReason)
static size_t curl_wrtcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
std::optional< off_t > currentRange() const
Regular expression match result.
bool checkIfRangeChkSumIsValid(Range &rng)
Base class for Exception.
bool any_of(const Container &c, Fnc &&cb)
std::optional< size_t > _relevantDigestLen
virtual bool beginRange(off_t range, std::string &cancelReason)
bool _gotContentRangeInfo
bool validateRange(Range &rng)
bool regex_match(const std::string &s, smatch &matches, const regex ®ex)
regex ZYPP_STR_REGEX regex ZYPP_STR_REGEX
unsigned _rangeAttemptIdx
std::vector< char > _rangePrefaceBuffer
Here we buffer.
These are enforced even if you don't pass them as flag argument.
std::optional< zypp::Digest > _digest
ProtocolMode _protocolMode
Easy-to use interface to the ZYPP dependency resolver.
std::optional< off_t > _currentRange
std::optional< size_t > _reportedFileSize
Filesize as reported by the content range or byte range headers.
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len, size_t &fileLen)
static constexpr unsigned _rangeAttempt[]
static size_t curl_hdrcallback(char *ptr, size_t size, size_t nmemb, void *userdata)
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
void * easyHandle() const
size_t wrtcallback(char *ptr, size_t size, size_t nmemb)