13 #include <zypp-core/zyppng/base/EventDispatcher> 14 #include <zypp-core/zyppng/base/private/linuxhelpers_p.h> 15 #include <zypp-core/zyppng/core/String> 16 #include <zypp-core/fs/PathInfo.h> 18 #include <zypp-curl/CurlConfig> 19 #include <zypp-curl/auth/CurlAuthData> 20 #include <zypp-media/MediaConfig> 21 #include <zypp-core/base/String.h> 22 #include <zypp-core/base/StringV.h> 23 #include <zypp-core/base/IOTools.h> 24 #include <zypp-core/Pathname.h> 25 #include <curl/curl.h> 31 #include <boost/variant.hpp> 32 #include <boost/variant/polymorphic_get.hpp> 38 static size_t nwr_headerCallback (
char *ptr,
size_t size,
size_t nmemb,
void *userdata ) {
42 NetworkRequestPrivate *that =
reinterpret_cast<NetworkRequestPrivate *
>( userdata );
43 return that->headerfunction( ptr, size * nmemb );
45 static size_t nwr_writeCallback (
char *ptr,
size_t size,
size_t nmemb,
void *userdata ) {
49 NetworkRequestPrivate *that =
reinterpret_cast<NetworkRequestPrivate *
>( userdata );
50 return that->writefunction( ptr, {}, size * nmemb );
54 template<
class T>
struct always_false : std::false_type {};
58 : _outFile(
std::move(prevState._outFile) )
59 , _downloaded( prevState._downloaded )
60 , _partialHelper(
std::move(prevState._partialHelper) )
64 : _partialHelper(
std::move(prevState._partialHelper) )
68 : _outFile(
std::move(prevState._outFile) )
69 , _partialHelper(
std::move(prevState._partialHelper) )
70 , _downloaded( prevState._downloaded )
78 ,
_headers(
std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >(
nullptr, &curl_slist_free_all ) )
108 const std::string urlScheme =
_url.getScheme();
109 if ( urlScheme ==
"http" || urlScheme ==
"https" )
121 std::string urlBuffer(
_url.asString() );
150 const auto &cHeaders =
_dispatcher->hostSpecificHeaders();
151 if (
auto i = cHeaders.find(
_url.getHost()); i != cHeaders.end() ) {
152 for (
const auto &[key, value] : i->second ) {
154 "%s: %s", key.c_str(), value.c_str() )
165 case 4:
setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 );
break;
166 case 6:
setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 );
break;
170 setCurlOption( CURLOPT_HEADERFUNCTION, &nwr_headerCallback );
185 if ( urlScheme ==
"https" )
206 #ifdef CURLSSLOPT_ALLOW_BEAST 208 setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
235 if (use_auth.empty())
236 use_auth =
"digest,basic";
238 if( auth != CURLAUTH_NONE)
240 DBG <<
_easyHandle <<
" " <<
"Enabling HTTP authentication methods: " << use_auth
241 <<
" (CURLOPT_HTTPAUTH=" << auth <<
")" << std::endl;
250 setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
261 if ( proxyuserpwd.empty() )
266 DBG <<
_easyHandle <<
" " <<
"Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
270 DBG <<
_easyHandle <<
" " <<
"Proxy: using proxy-user from ~/.curlrc" << std::endl;
278 if ( ! proxyuserpwd.empty() )
283 #if CURLVERSION_AT_LEAST(7,19,4) 288 DBG <<
_easyHandle <<
" " <<
"Proxy: explicitly NOPROXY" << std::endl;
303 #if CURLVERSION_AT_LEAST(7,15,5) 315 #if CURLVERSION_AT_LEAST(7,18,0) 322 for (
const auto &header : locSet.
headers() ) {
323 if ( !z_func()->addRequestHeader( header.c_str() ) )
335 return ( elem1.start < elem2.start );
339 if (
auto initState = std::get_if<pending_t>(&
_runningMode) ) {
342 initState->_partialHelper = std::make_unique<CurlMultiPartHandler>(
348 helper = initState->_partialHelper.get();
350 }
else if (
auto pendingState = std::get_if<prepareNextRangeBatch_t>(&
_runningMode) ) {
351 helper = pendingState->_partialHelper.get();
353 errBuf =
"Request is in invalid state to call setupHandle";
357 if ( !helper->prepare () ) {
358 errBuf = helper->lastErrorMessage ();
375 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
377 DBG <<
_easyHandle <<
"Can only create output file in running mode" << std::endl;
381 if ( !rmode->_outFile ) {
382 std::string openMode =
"w+b";
393 if ( !rmode->_outFile ) {
406 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
407 if ( rmode->_partialHelper )
return rmode->_partialHelper->canRecover();
413 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
415 errBuf =
"NetworkRequestPrivate::prepareToContinue called in invalid state";
419 if ( rmode->_partialHelper && rmode->_partialHelper->hasMoreWork() ) {
425 auto &prepMode = std::get<prepareNextRangeBatch_t>(
_runningMode);
426 if ( !prepMode._partialHelper->prepareToContinue() ) {
427 errBuf = prepMode._partialHelper->lastErrorMessage();
431 if ( hadRangeFail ) {
444 errBuf =
"Request has no more work";
457 bool isRangeContinuation = std::holds_alternative<prepareNextRangeBatch_t>(
_runningMode );
458 if ( isRangeContinuation ) {
459 MIL <<
_easyHandle <<
" " <<
"Continuing a previously started range batch." << std::endl;
467 if ( m._activityTimer ) {
470 m._activityTimer->start( static_cast<uint64_t>(
_settings.
timeout() * 1000 ) );
473 if ( !isRangeContinuation )
479 if ( std::holds_alternative<running_t>(
_runningMode) ) {
481 if ( rmode._partialHelper )
482 rmode._partialHelper->finalize();
489 resState.
_result = std::move(err);
491 if ( std::holds_alternative<running_t>(
_runningMode) ) {
500 if ( !rmode._partialHelper->verifyData() ){
507 if ( fseek( rmode._outFile, 0, SEEK_SET ) != 0 ) {
510 constexpr
size_t bufSize = 4096;
512 while(
auto cnt = fread(buf, 1, bufSize, rmode._outFile ) > 0 ) {
523 const UByteArray &expSum = zypp::Digest::hexStringToUByteArray(
_fileVerification->_fileChecksum.checksum () );
524 if ( calcSum != expSum ) {
533 rmode._outFile.reset();
557 MIL_MEDIA <<
_easyHandle <<
" Request timeout interval: " << t.interval()<<
" remaining: " << t.remaining() << std::endl;
558 std::map<std::string, boost::any> extraInfo;
559 extraInfo.insert( {
"requestUrl",
_url } );
571 if ( std::holds_alternative<running_t>(
_runningMode ) ){
573 if ( rmode._activityTimer && rmode._activityTimer->isRunning() )
574 rmode._activityTimer->start();
584 if ( !std::holds_alternative<running_t>(that->
_runningMode) ){
585 DBG << that->
_easyHandle <<
" " <<
"Curl progress callback was called in invalid state "<< that->z_func()->state() << std::endl;
589 auto &rmode = std::get<running_t>( that->
_runningMode );
594 rmode._isInCallback =
true;
595 if ( rmode._lastProgressNow != dlnow ) {
596 rmode._lastProgressNow = dlnow;
597 that->
_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
599 rmode._isInCallback =
false;
601 return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
614 std::string_view hdr( ptr, bytes );
616 hdr.remove_prefix( std::min( hdr.find_first_not_of(
" \t\r\n"), hdr.size() ) );
617 const auto lastNonWhitespace = hdr.find_last_not_of(
" \t\r\n");
618 if ( lastNonWhitespace != hdr.npos )
619 hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
621 hdr = std::string_view();
628 const auto &repSize = rmode._partialHelper->reportedFileSize ();
637 (void)curl_easy_getinfo(
_easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
649 auto str = std::string ( lenStr.data(), lenStr.length() );
650 auto len = zypp::str::strtonum<typename zypp::ByteCount::SizeType>(
str.data() );
652 DBG <<
_easyHandle <<
" " <<
"Got Content-Length Header: " << len << std::endl;
682 if ( fseek( rmode._outFile, *offset, SEEK_SET ) != 0 ) {
684 "Unable to set output file pointer." );
687 rmode._currentFileOffset = *offset;
691 const auto &repSize = rmode._partialHelper->reportedFileSize ();
704 auto written = fwrite( data, 1, max, rmode._outFile );
713 rmode._currentFileOffset += written;
716 rmode._downloaded += written;
733 if ( d->_dispatcher )
734 d->_dispatcher->cancel( *
this,
"Request destroyed while still running" );
739 d_func()->_expectedFileSize = std::move( expectedFileSize );
746 if (
state() ==
Pending && triggerReschedule && d->_dispatcher )
747 d->_dispatcher->reschedule();
752 return d_func()->_priority;
757 d_func()->_options = opt;
762 return d_func()->_options;
771 d->_requestedRanges.push_back(
Range::make( start, len, std::move(digest), std::move( expectedChkSum ), std::move( userData ), digestCompareLen, chksumpad ) );
780 d->_requestedRanges.push_back( std::move(range) );
781 auto &rng = d->_requestedRanges.back();
783 rng.bytesWritten = 0;
785 rng._digest->reset();
800 ._fileChecksum = expected
810 d->_requestedRanges.clear();
815 const auto mystate =
state();
821 std::vector<Range> failed;
822 for (
auto &r : d->_requestedRanges ) {
824 failed.push_back( r.clone() );
832 return d_func()->_requestedRanges;
837 return d_func()->_lastRedirect;
842 return d_func()->_easyHandle;
847 const auto myerr =
error();
848 const auto mystate =
state();
854 auto getMeasurement = [ this ](
const CURLINFO info, std::chrono::microseconds &target ){
855 using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
857 const auto res = curl_easy_getinfo( d_func()->_easyHandle, info, &val );
858 if ( CURLE_OK == res ) {
859 target = std::chrono::duration_cast<std::chrono::microseconds>( FPSeconds(val) );
863 getMeasurement( CURLINFO_NAMELOOKUP_TIME, t.
namelookup );
864 getMeasurement( CURLINFO_CONNECT_TIME, t.
connect);
865 getMeasurement( CURLINFO_APPCONNECT_TIME, t.
appconnect);
866 getMeasurement( CURLINFO_PRETRANSFER_TIME , t.
pretransfer);
867 getMeasurement( CURLINFO_TOTAL_TIME, t.
total);
868 getMeasurement( CURLINFO_REDIRECT_TIME, t.
redirect);
877 if ( !std::holds_alternative<NetworkRequestPrivate::running_t>( d->_runningMode) )
880 const auto &rmode = std::get<NetworkRequestPrivate::running_t>( d->_runningMode );
886 return d_func()->_url;
900 return d_func()->_targetFile;
908 d->_targetFile = path;
913 return d_func()->_fMode;
921 d->_fMode = std::move( mode );
927 if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
928 return std::string(ptr);
929 return std::string();
935 using T = std::decay_t<decltype(arg)>;
936 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
938 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
939 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
940 return arg._contentLenght;
942 static_assert(always_false<T>::value,
"Unhandled state type");
943 }, d_func()->_runningMode);
949 using T = std::decay_t<decltype(arg)>;
950 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
952 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
953 || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
954 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
955 return arg._downloaded;
957 static_assert(always_false<T>::value,
"Unhandled state type");
958 }, d_func()->_runningMode);
963 return d_func()->_settings;
968 return std::visit([
this](
auto& arg) {
969 using T = std::decay_t<decltype(arg)>;
970 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
972 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
974 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
975 if ( std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode ).
_result.isError() )
981 static_assert(always_false<T>::value,
"Unhandled state type");
982 }, d_func()->_runningMode);
987 const auto s =
state();
990 return std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode).
_result;
996 return std::string();
1010 curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() :
nullptr, header.c_str() );
1015 d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
1022 return d_func()->_sigStarted;
1027 return d_func()->_sigBytesDownloaded;
1032 return d_func()->_sigProgress;
1037 return d_func()->_sigFinished;
Signal< void(NetworkRequest &req)> _sigStarted
std::string errorMessage() const
bool isError() const
isError Will return true if this is a actual error
void setCurlOption(CURLoption opt, T data)
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
void * nativeHandle() const
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size...
const std::vector< Range > & requestedRanges() const
std::chrono::microseconds connect
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
std::optional< FileVerifyInfo > _fileVerification
The digest for the full file.
The CurlMultiPartHandler class.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
ZYPP_IMPL_PRIVATE(Provide)
void addRequestRange(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
bool hasPrefixCI(const C_Str &str_r, const C_Str &prefix_r)
Compute Message Digests (MD5, SHA1 etc)
NetworkRequest::FileMode _fMode
Store and operate with byte count.
const std::string & lastRedirectInfo() const
const std::string _currentCookieFile
static Range make(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
std::chrono::microseconds pretransfer
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
NetworkRequest::Options _options
const char * c_str() const
String representation.
void resetActivityTimer()
String related utilities and Regular expression matching.
std::chrono::microseconds appconnect
constexpr bool always_false
running_t(pending_t &&prevState)
std::string nativeErrorString() const
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
void setOptions(Options opt)
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
void resetRequestRanges()
TransferSettings & transferSettings()
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
bool hasError() const
Checks if there was a error with the request.
void onActivityTimeout(Timer &)
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
zypp::Pathname _targetFile
virtual ~NetworkRequestPrivate()
bool empty() const
Test for an empty path.
void setUrl(const Url &url)
This will change the URL of the request.
std::chrono::microseconds namelookup
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
bool addRequestHeader(const std::string &header)
std::string trim(const std::string &s, const Trim trim_r)
const std::string & asString() const
String representation.
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
std::string asString() const
Error message provided by dumpOn as string.
bool initialize(std::string &errBuf)
The NetworkRequestError class Represents a error that occured in.
std::vector< char > peekData(off_t offset, size_t count) const
NetworkRequestError error() const
Returns the last set Error.
zypp::ByteCount _expectedFileSize
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Priority priority() const
bool setupHandle(std::string &errBuf)
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
bool create(const std::string &name)
initialize creation of a new message digest
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
size_t writefunction(char *ptr, std::optional< off_t > offset, size_t bytes) override
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
std::string contentType() const
Returns the content type as reported from the server.
static const Unit B
1 Byte
bool setExpectedFileChecksum(const zypp::CheckSum &expected)
Base class for Exception.
std::string _lastRedirect
to log/report redirections
std::chrono::microseconds total
bool any_of(const Container &c, Fnc &&cb)
std::string curlUnEscape(std::string text_r)
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
void setPriority(Priority prio, bool triggerReschedule=true)
State state() const
Returns the current state the HttpDownloadRequest is in.
TransferSettings _settings
NetworkRequestDispatcher * _dispatcher
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
virtual ~NetworkRequest()
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
std::chrono::microseconds redirect
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
size_t headerfunction(char *ptr, size_t bytes) override
Type type() const
type Returns the type of the error
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
zypp::ByteCount _contentLenght
#define EXPLICITLY_NO_PROXY
FileMode fileOpenMode() const
Returns the currently configured file open mode.
std::vector< Range > failedRanges() const
Easy-to use interface to the ZYPP dependency resolver.
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
CURLcode setCurlRedirProtocols(CURL *curl)
void setResult(NetworkRequestError &&err)
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> sigProgress()
Signals if there was data read from the download.
bool prepareToContinue(std::string &errBuf)
NetworkRequestError _result
ZYppCommitResult & _result
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode