libzypp  17.31.11
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <chrono>
15 #include <list>
16 
17 #include <zypp/base/Logger.h>
18 #include <zypp/ExternalProgram.h>
19 #include <zypp/base/String.h>
20 #include <zypp/base/Gettext.h>
21 #include <zypp-core/parser/Sysconfig>
22 #include <zypp/base/Gettext.h>
23 
24 #include <zypp/media/MediaCurl.h>
25 #include <zypp-curl/ProxyInfo>
26 #include <zypp-curl/auth/CurlAuthData>
27 #include <zypp-media/auth/CredentialManager>
28 #include <zypp-curl/CurlConfig>
30 #include <zypp/Target.h>
31 #include <zypp/ZYppFactory.h>
32 #include <zypp/ZConfig.h>
33 
34 #include <cstdlib>
35 #include <sys/types.h>
36 #include <sys/stat.h>
37 #include <sys/mount.h>
38 #include <errno.h>
39 #include <dirent.h>
40 #include <unistd.h>
41 
42 using std::endl;
43 
44 namespace internal {
45  using namespace zypp;
49  struct OptionalDownloadProgressReport : public callback::ReceiveReport<media::DownloadProgressReport>
50  {
51  using TimePoint = std::chrono::steady_clock::time_point;
52 
53  OptionalDownloadProgressReport( bool isOptional=false )
54  : _oldRec { Distributor::instance().getReceiver() }
55  , _isOptional { isOptional }
56  { connect(); }
57 
59  { if ( _oldRec ) Distributor::instance().setReceiver( *_oldRec ); else Distributor::instance().noReceiver(); }
60 
61  void reportbegin() override
62  { if ( _oldRec ) _oldRec->reportbegin(); }
63 
64  void reportend() override
65  { if ( _oldRec ) _oldRec->reportend(); }
66 
67  void report( const UserData & userData_r = UserData() ) override
68  { if ( _oldRec ) _oldRec->report( userData_r ); }
69 
70 
71  void start( const Url & file_r, Pathname localfile_r ) override
72  {
73  if ( not _oldRec ) return;
74  if ( _isOptional ) {
75  // delay start until first data are received.
76  _startFile = file_r;
77  _startLocalfile = std::move(localfile_r);
78  return;
79  }
80  _oldRec->start( file_r, localfile_r );
81  }
82 
83  bool progress( int value_r, const Url & file_r, double dbps_avg_r = -1, double dbps_current_r = -1 ) override
84  {
85  if ( not _oldRec ) return true;
86  if ( notStarted() ) {
87  if ( not ( value_r || dbps_avg_r || dbps_current_r ) )
88  return true;
89  sendStart();
90  }
91 
92  //static constexpr std::chrono::milliseconds minfequency { 1000 }; only needed if we'd avoid sending reports without change
93  static constexpr std::chrono::milliseconds maxfequency { 100 };
94  TimePoint now { TimePoint::clock::now() };
95  TimePoint::duration elapsed { now - _lastProgressSent };
96  if ( elapsed < maxfequency )
97  return true; // continue
98  _lastProgressSent = now;
99  return _oldRec->progress( value_r, file_r, dbps_avg_r, dbps_current_r );
100  }
101 
102  Action problem( const Url & file_r, Error error_r, const std::string & description_r ) override
103  {
104  if ( not _oldRec || notStarted() ) return ABORT;
105  return _oldRec->problem( file_r, error_r, description_r );
106  }
107 
108  void finish( const Url & file_r, Error error_r, const std::string & reason_r ) override
109  {
110  if ( not _oldRec || notStarted() ) return;
111  _oldRec->finish( file_r, error_r, reason_r );
112  }
113 
114  private:
115  // _isOptional also indicates the delayed start
116  bool notStarted() const
117  { return _isOptional; }
118 
119  void sendStart()
120  {
121  if ( _isOptional ) {
122  // we know _oldRec is valid...
123  _oldRec->start( std::move(_startFile), std::move(_startLocalfile) );
124  _isOptional = false;
125  }
126  }
127 
128  private:
129  Receiver *const _oldRec;
132  Pathname _startLocalfile;
134  };
135 
137  {
138  ProgressData( CURL *curl, time_t timeout = 0, const zypp::Url & url = zypp::Url(),
139  zypp::ByteCount expectedFileSize_r = 0,
141 
142  void updateStats( double dltotal = 0.0, double dlnow = 0.0 );
143 
144  int reportProgress() const;
145 
146  CURL * curl()
147  { return _curl; }
148 
149  bool timeoutReached() const
150  { return _timeoutReached; }
151 
152  bool fileSizeExceeded() const
153  { return _fileSizeExceeded; }
154 
156  { return _expectedFileSize; }
157 
158  void expectedFileSize( ByteCount newval_r )
159  { _expectedFileSize = newval_r; }
160 
161  private:
162  CURL * _curl;
164  time_t _timeout;
169 
170  time_t _timeStart = 0;
171  time_t _timeLast = 0;
172  time_t _timeRcv = 0;
173  time_t _timeNow = 0;
174 
175  double _dnlTotal = 0.0;
176  double _dnlLast = 0.0;
177  double _dnlNow = 0.0;
178 
179  int _dnlPercent= 0;
180 
181  double _drateTotal= 0.0;
182  double _drateLast = 0.0;
183  };
184 
185 
186 
187  ProgressData::ProgressData(CURL *curl, time_t timeout, const Url &url, ByteCount expectedFileSize_r, zypp::callback::SendReport< zypp::media::DownloadProgressReport> *_report)
188  : _curl( curl )
189  , _url( url )
190  , _timeout( timeout )
191  , _timeoutReached( false )
192  , _fileSizeExceeded ( false )
193  , _expectedFileSize( expectedFileSize_r )
194  , report( _report )
195  {}
196 
197  void ProgressData::updateStats(double dltotal, double dlnow)
198  {
199  time_t now = _timeNow = time(0);
200 
201  // If called without args (0.0), recompute based on the last values seen
202  if ( dltotal && dltotal != _dnlTotal )
203  _dnlTotal = dltotal;
204 
205  if ( dlnow && dlnow != _dnlNow )
206  {
207  _timeRcv = now;
208  _dnlNow = dlnow;
209  }
210 
211  // init or reset if time jumps back
212  if ( !_timeStart || _timeStart > now )
213  _timeStart = _timeLast = _timeRcv = now;
214 
215  // timeout condition
216  if ( _timeout )
217  _timeoutReached = ( (now - _timeRcv) > _timeout );
218 
219  // check if the downloaded data is already bigger than what we expected
220  _fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
221 
222  // percentage:
223  if ( _dnlTotal )
224  _dnlPercent = int(_dnlNow * 100 / _dnlTotal);
225 
226  // download rates:
227  _drateTotal = _dnlNow / std::max( int(now - _timeStart), 1 );
228 
229  if ( _timeLast < now )
230  {
231  _drateLast = (_dnlNow - _dnlLast) / int(now - _timeLast);
232  // start new period
233  _timeLast = now;
234  _dnlLast = _dnlNow;
235  }
236  else if ( _timeStart == _timeLast )
238  }
239 
241  {
242  if ( _fileSizeExceeded )
243  return 1;
244  if ( _timeoutReached )
245  return 1; // no-data timeout
246  if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
247  return 1; // user requested abort
248  return 0;
249  }
250 
251  const char * anonymousIdHeader()
252  {
253  // we need to add the release and identifier to the
254  // agent string.
255  // The target could be not initialized, and then this information
256  // is guessed.
257  static const std::string _value(
259  "X-ZYpp-AnonymousId: %s",
260  Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str() ) )
261  );
262  return _value.c_str();
263  }
264 
266  {
267  // we need to add the release and identifier to the
268  // agent string.
269  // The target could be not initialized, and then this information
270  // is guessed.
271  static const std::string _value(
273  "X-ZYpp-DistributionFlavor: %s",
274  Target::distributionFlavor( Pathname()/*guess root*/ ).c_str() ) )
275  );
276  return _value.c_str();
277  }
278 
279  const char * agentString()
280  {
281  // we need to add the release and identifier to the
282  // agent string.
283  // The target could be not initialized, and then this information
284  // is guessed.
285  static const std::string _value(
286  str::form(
287  "ZYpp " LIBZYPP_VERSION_STRING " (curl %s) %s"
288  , curl_version_info(CURLVERSION_NOW)->version
289  , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
290  )
291  );
292  return _value.c_str();
293  }
294 
298  {
299  public:
301  const std::string & err_r,
302  const std::string & msg_r )
303  : media::MediaCurlException( url_r, err_r, msg_r )
304  {}
305  //~MediaCurlExceptionMayRetryInternaly() noexcept {}
306  };
307 
308 }
309 
310 
311 using namespace internal;
312 using namespace zypp::base;
313 
314 namespace zypp {
315 
316  namespace media {
317 
318 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
319 
320 // we use this define to unbloat code as this C setting option
321 // and catching exception is done frequently.
323 #define SET_OPTION(opt,val) do { \
324  ret = curl_easy_setopt ( _curl, opt, val ); \
325  if ( ret != 0) { \
326  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
327  } \
328  } while ( false )
329 
330 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
331 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
332 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
333 
334 MediaCurl::MediaCurl( const Url & url_r,
335  const Pathname & attach_point_hint_r )
336  : MediaNetworkCommonHandler( url_r, attach_point_hint_r,
337  "/", // urlpath at attachpoint
338  true ), // does_download
339  _curl( NULL ),
340  _customHeaders(0L)
341 {
342  _curlError[0] = '\0';
343 
344  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
345 
347 
348  if( !attachPoint().empty())
349  {
350  PathInfo ainfo(attachPoint());
351  Pathname apath(attachPoint() + "XXXXXX");
352  char *atemp = ::strdup( apath.asString().c_str());
353  char *atest = NULL;
354  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
355  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
356  {
357  WAR << "attach point " << ainfo.path()
358  << " is not useable for " << url_r.getScheme() << endl;
359  setAttachPoint("", true);
360  }
361  else if( atest != NULL)
362  ::rmdir(atest);
363 
364  if( atemp != NULL)
365  ::free(atemp);
366  }
367 }
368 
370 {
372 }
373 
374 void MediaCurl::setCookieFile( const Pathname &fileName )
375 {
376  _cookieFile = fileName;
377 }
378 
380 
381 void MediaCurl::checkProtocol(const Url &url) const
382 {
383  curl_version_info_data *curl_info = NULL;
384  curl_info = curl_version_info(CURLVERSION_NOW);
385  // curl_info does not need any free (is static)
386  if (curl_info->protocols)
387  {
388  const char * const *proto;
389  std::string scheme( url.getScheme());
390  bool found = false;
391  for(proto=curl_info->protocols; !found && *proto; ++proto)
392  {
393  if( scheme == std::string((const char *)*proto))
394  found = true;
395  }
396  if( !found)
397  {
398  std::string msg("Unsupported protocol '");
399  msg += scheme;
400  msg += "'";
402  }
403  }
404 }
405 
407 {
409 
410  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
411  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
412  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
413  if ( ret != 0 ) {
414  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
415  }
416 
417  SET_OPTION(CURLOPT_FAILONERROR, 1L);
418  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
419 
420  // create non persistant settings
421  // so that we don't add headers twice
422  TransferSettings vol_settings(_settings);
423 
424  // add custom headers for download.opensuse.org (bsc#955801)
425  if ( _url.getHost() == "download.opensuse.org" )
426  {
427  vol_settings.addHeader(anonymousIdHeader());
428  vol_settings.addHeader(distributionFlavorHeader());
429  }
430  vol_settings.addHeader("Pragma:");
431 
433 
434  // fill some settings from url query parameters
435  try
436  {
438  }
439  catch ( const MediaException &e )
440  {
441  disconnectFrom();
442  ZYPP_RETHROW(e);
443  }
444  // if the proxy was not set (or explicitly unset) by url, then look...
445  if ( _settings.proxy().empty() )
446  {
447  // ...at the system proxy settings
449  }
450 
452  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
453  {
454  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
455  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
456  }
457 
461  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
462  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
463  // just in case curl does not trigger its progress callback frequently
464  // enough.
465  if ( _settings.timeout() )
466  {
467  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
468  }
469 
470  // follow any Location: header that the server sends as part of
471  // an HTTP header (#113275)
472  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
473  // 3 redirects seem to be too few in some cases (bnc #465532)
474  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
475 
476  if ( _url.getScheme() == "https" )
477  {
478 #if CURLVERSION_AT_LEAST(7,19,4)
479  // restrict following of redirections from https to https only
480  if ( _url.getHost() == "download.opensuse.org" )
481  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
482  else
483  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
484 #endif
485 
488  {
490  }
491 
493  {
494  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
495  }
496  if( ! _settings.clientKeyPath().empty() )
497  {
498  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
499  }
500 
501 #ifdef CURLSSLOPT_ALLOW_BEAST
502  // see bnc#779177
503  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
504  if ( ret != 0 ) {
505  disconnectFrom();
507  }
508 #endif
509  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
510  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
511  // bnc#903405 - POODLE: libzypp should only talk TLS
512  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
513  }
514 
515  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
516 
517  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
518  * We should proactively add the password to the request if basic auth is configured
519  * and a password is available in the credentials but not in the URL.
520  *
521  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
522  * and ask the server first about the auth method
523  */
524  if ( _settings.authType() == "basic"
525  && _settings.username().size()
526  && !_settings.password().size() ) {
527 
528  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
529  const auto cred = cm.getCred( _url );
530  if ( cred && cred->valid() ) {
531  if ( !_settings.username().size() )
532  _settings.setUsername(cred->username());
533  _settings.setPassword(cred->password());
534  }
535  }
536 
537  /*---------------------------------------------------------------*
538  CURLOPT_USERPWD: [user name]:[password]
539 
540  Url::username/password -> CURLOPT_USERPWD
541  If not provided, anonymous FTP identification
542  *---------------------------------------------------------------*/
543 
544  if ( _settings.userPassword().size() )
545  {
546  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
547  std::string use_auth = _settings.authType();
548  if (use_auth.empty())
549  use_auth = "digest,basic"; // our default
550  long auth = CurlAuthData::auth_type_str2long(use_auth);
551  if( auth != CURLAUTH_NONE)
552  {
553  DBG << "Enabling HTTP authentication methods: " << use_auth
554  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
555  SET_OPTION(CURLOPT_HTTPAUTH, auth);
556  }
557  }
558 
559  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
560  {
561  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
562  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
563  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
564  /*---------------------------------------------------------------*
565  * CURLOPT_PROXYUSERPWD: [user name]:[password]
566  *
567  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
568  * If not provided, $HOME/.curlrc is evaluated
569  *---------------------------------------------------------------*/
570 
571  std::string proxyuserpwd = _settings.proxyUserPassword();
572 
573  if ( proxyuserpwd.empty() )
574  {
575  CurlConfig curlconf;
576  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
577  if ( curlconf.proxyuserpwd.empty() )
578  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
579  else
580  {
581  proxyuserpwd = curlconf.proxyuserpwd;
582  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
583  }
584  }
585  else
586  {
587  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
588  }
589 
590  if ( ! proxyuserpwd.empty() )
591  {
592  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
593  }
594  }
595 #if CURLVERSION_AT_LEAST(7,19,4)
596  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
597  {
598  // Explicitly disabled in URL (see fillSettingsFromUrl()).
599  // This should also prevent libcurl from looking into the environment.
600  DBG << "Proxy: explicitly NOPROXY" << endl;
601  SET_OPTION(CURLOPT_NOPROXY, "*");
602  }
603 #endif
604  else
605  {
606  DBG << "Proxy: not explicitly set" << endl;
607  DBG << "Proxy: libcurl may look into the environment" << endl;
608  }
609 
611  if ( _settings.minDownloadSpeed() != 0 )
612  {
613  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
614  // default to 10 seconds at low speed
615  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
616  }
617 
618 #if CURLVERSION_AT_LEAST(7,15,5)
619  if ( _settings.maxDownloadSpeed() != 0 )
620  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
621 #endif
622 
623  /*---------------------------------------------------------------*
624  *---------------------------------------------------------------*/
625 
628  if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
629  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
630  else
631  MIL << "No cookies requested" << endl;
632  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
633  SET_OPTION(CURLOPT_PROGRESSFUNCTION, &progressCallback );
634  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
635 
636 #if CURLVERSION_AT_LEAST(7,18,0)
637  // bnc #306272
638  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
639 #endif
640  // append settings custom headers to curl
641  for ( const auto &header : vol_settings.headers() )
642  {
643  // MIL << "HEADER " << *it << std::endl;
644 
645  _customHeaders = curl_slist_append(_customHeaders, header.c_str());
646  if ( !_customHeaders )
648  }
649 
650  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
651 }
652 
654 
655 
656 void MediaCurl::attachTo (bool next)
657 {
658  if ( next )
660 
661  if ( !_url.isValid() )
663 
666  {
668  }
669 
670  disconnectFrom(); // clean _curl if needed
671  _curl = curl_easy_init();
672  if ( !_curl ) {
674  }
675  try
676  {
677  setupEasy();
678  }
679  catch (Exception & ex)
680  {
681  disconnectFrom();
682  ZYPP_RETHROW(ex);
683  }
684 
685  // FIXME: need a derived class to propelly compare url's
687  setMediaSource(media);
688 }
689 
690 bool
692 {
693  return MediaHandler::checkAttachPoint( apoint, true, true);
694 }
695 
697 
699 {
700  if ( _customHeaders )
701  {
702  curl_slist_free_all(_customHeaders);
703  _customHeaders = 0L;
704  }
705 
706  if ( _curl )
707  {
708  // bsc#1201092: Strange but within global_dtors we may exceptions here.
709  try { curl_easy_cleanup( _curl ); }
710  catch (...) { ; }
711  _curl = NULL;
712  }
713 }
714 
716 
717 void MediaCurl::releaseFrom( const std::string & ejectDev )
718 {
719  disconnect();
720 }
721 
723 
724 void MediaCurl::getFile( const OnMediaLocation &file ) const
725 {
726  // Use absolute file name to prevent access of files outside of the
727  // hierarchy below the attach point.
728  getFileCopy( file, localPath(file.filename()).absolutename() );
729 }
730 
732 
733 void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
734 {
735 
736  const auto &filename = srcFile.filename();
737 
738  // Optional files will send no report until data are actually received (we know it exists).
739  OptionalDownloadProgressReport reportfilter( srcFile.optional() );
741 
742  Url fileurl(getFileUrl(filename));
743 
744  bool firstAuth = true; // bsc#1210870: authenticate must not return stored credentials more than once.
745  unsigned internalTry = 0;
746  static constexpr unsigned maxInternalTry = 3;
747 
748  do
749  {
750  try
751  {
752  doGetFileCopy( srcFile, target, report );
753  break; // success!
754  }
755  // retry with proper authentication data
756  catch (MediaUnauthorizedException & ex_r)
757  {
758  if ( authenticate(ex_r.hint(), firstAuth) ) {
759  firstAuth = false; // must not return stored credentials again
760  continue; // retry
761  }
762 
763  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
764  ZYPP_RETHROW(ex_r);
765  }
766  // unexpected exception
767  catch (MediaException & excpt_r)
768  {
769  if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
770  ++internalTry;
771  if ( internalTry < maxInternalTry ) {
772  // just report (NO_ERROR); no interactive request to the user
773  report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
774  continue; // retry
775  }
776  excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
777  }
778 
780  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
781  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
782  {
784  }
785  report->finish(fileurl, reason, excpt_r.asUserHistory());
786  ZYPP_RETHROW(excpt_r);
787  }
788  }
789  while ( true );
790  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
791 }
792 
794 
795 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
796 {
797  bool retry = false;
798 
799  do
800  {
801  try
802  {
803  return doGetDoesFileExist( filename );
804  }
805  // authentication problem, retry with proper authentication data
806  catch (MediaUnauthorizedException & ex_r)
807  {
808  if(authenticate(ex_r.hint(), !retry))
809  retry = true;
810  else
811  ZYPP_RETHROW(ex_r);
812  }
813  // unexpected exception
814  catch (MediaException & excpt_r)
815  {
816  ZYPP_RETHROW(excpt_r);
817  }
818  }
819  while (retry);
820 
821  return false;
822 }
823 
825 
827  CURLcode code,
828  bool timeout_reached) const
829 {
830  if ( code != 0 )
831  {
832  Url url;
833  if (filename.empty())
834  url = _url;
835  else
836  url = getFileUrl(filename);
837 
838  std::string err;
839  {
840  switch ( code )
841  {
842  case CURLE_UNSUPPORTED_PROTOCOL:
843  err = " Unsupported protocol";
844  if ( !_lastRedirect.empty() )
845  {
846  err += " or redirect (";
847  err += _lastRedirect;
848  err += ")";
849  }
850  break;
851  case CURLE_URL_MALFORMAT:
852  case CURLE_URL_MALFORMAT_USER:
853  err = " Bad URL";
854  break;
855  case CURLE_LOGIN_DENIED:
856  ZYPP_THROW(
857  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
858  break;
859  case CURLE_HTTP_RETURNED_ERROR:
860  {
861  long httpReturnCode = 0;
862  CURLcode infoRet = curl_easy_getinfo( _curl,
863  CURLINFO_RESPONSE_CODE,
864  &httpReturnCode );
865  if ( infoRet == CURLE_OK )
866  {
867  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
868  switch ( httpReturnCode )
869  {
870  case 401:
871  {
872  std::string auth_hint = getAuthHint();
873 
874  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
875  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
876 
878  url, "Login failed.", _curlError, auth_hint
879  ));
880  }
881 
882  case 502: // bad gateway (bnc #1070851)
883  case 503: // service temporarily unavailable (bnc #462545)
885  case 504: // gateway timeout
887  case 403:
888  {
889  std::string msg403;
890  if ( url.getHost().find(".suse.com") != std::string::npos )
891  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
892  else if (url.asString().find("novell.com") != std::string::npos)
893  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
895  }
896  case 404:
897  case 410:
899  }
900 
901  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
903  }
904  else
905  {
906  std::string msg = "Unable to retrieve HTTP response:";
907  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
909  }
910  }
911  break;
912  case CURLE_FTP_COULDNT_RETR_FILE:
913 #if CURLVERSION_AT_LEAST(7,16,0)
914  case CURLE_REMOTE_FILE_NOT_FOUND:
915 #endif
916  case CURLE_FTP_ACCESS_DENIED:
917  case CURLE_TFTP_NOTFOUND:
918  err = "File not found";
920  break;
921  case CURLE_BAD_PASSWORD_ENTERED:
922  case CURLE_FTP_USER_PASSWORD_INCORRECT:
923  err = "Login failed";
924  break;
925  case CURLE_COULDNT_RESOLVE_PROXY:
926  case CURLE_COULDNT_RESOLVE_HOST:
927  case CURLE_COULDNT_CONNECT:
928  case CURLE_FTP_CANT_GET_HOST:
929  err = "Connection failed";
930  break;
931  case CURLE_WRITE_ERROR:
932  err = "Write error";
933  break;
934  case CURLE_PARTIAL_FILE:
935  case CURLE_OPERATION_TIMEDOUT:
936  timeout_reached = true; // fall though to TimeoutException
937  // fall though...
938  case CURLE_ABORTED_BY_CALLBACK:
939  if( timeout_reached )
940  {
941  err = "Timeout reached";
943  }
944  else
945  {
946  err = "User abort";
947  }
948  break;
949 
950  // Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy
951  case CURLE_HTTP2:
952  case CURLE_HTTP2_STREAM:
953  err = "Curl error " + str::numstring( code );
955  break;
956 
957  default:
958  err = "Curl error " + str::numstring( code );
959  break;
960  }
961 
962  // uhm, no 0 code but unknown curl exception
964  }
965  }
966  else
967  {
968  // actually the code is 0, nothing happened
969  }
970 }
971 
973 
974 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
975 {
976  DBG << filename.asString() << endl;
977 
978  if(!_url.isValid())
980 
981  if(_url.getHost().empty())
983 
984  Url url(getFileUrl(filename));
985 
986  DBG << "URL: " << url.asString() << endl;
987  // Use URL without options and without username and passwd
988  // (some proxies dislike them in the URL).
989  // Curl seems to need the just scheme, hostname and a path;
990  // the rest was already passed as curl options (in attachTo).
991  Url curlUrl( clearQueryString(url) );
992 
993  //
994  // See also Bug #154197 and ftp url definition in RFC 1738:
995  // The url "ftp://user@host/foo/bar/file" contains a path,
996  // that is relative to the user's home.
997  // The url "ftp://user@host//foo/bar/file" (or also with
998  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
999  // contains an absolute path.
1000  //
1001  _lastRedirect.clear();
1002  std::string urlBuffer( curlUrl.asString());
1003  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1004  urlBuffer.c_str() );
1005  if ( ret != 0 ) {
1007  }
1008 
1009  // If no head requests allowed (?head_requests=no):
1010  // Instead of returning no data with NOBODY, we return
1011  // little data, that works with broken servers, and
1012  // works for ftp as well, because retrieving only headers
1013  // ftp will return always OK code ?
1014  // See http://curl.haxx.se/docs/knownbugs.html #58
1016  struct TempSetHeadRequest
1017  {
1018  TempSetHeadRequest( CURL * curl_r, bool doHttpHeadRequest_r )
1019  : _curl { curl_r }
1020  , _doHttpHeadRequest { doHttpHeadRequest_r }
1021  {
1022  if ( _doHttpHeadRequest ) {
1023  curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
1024  } else {
1025  curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
1026  }
1027  }
1028  ~TempSetHeadRequest() {
1029  if ( _doHttpHeadRequest ) {
1030  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1031  /* yes, this is why we never got to get NOBODY working before,
1032  because setting it changes this option too, and we also*
1033  need to reset it
1034  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1035  */
1036  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1037  } else {
1038  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1039  }
1040  }
1041  private:
1042  CURL * _curl;
1043  bool _doHttpHeadRequest;
1044  } _guard( _curl, (_url.getScheme() == "http" || _url.getScheme() == "https") && _settings.headRequestsAllowed() );
1045 
1046 
1047  AutoFILE file { ::fopen( "/dev/null", "w" ) };
1048  if ( !file ) {
1049  ERR << "fopen failed for /dev/null" << endl;
1050  ZYPP_THROW(MediaWriteException("/dev/null"));
1051  }
1052 
1053  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
1054  if ( ret != 0 ) {
1056  }
1057 
1058  CURLcode ok = curl_easy_perform( _curl );
1059  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1060 
1061  // as we are not having user interaction, the user can't cancel
1062  // the file existence checking, a callback or timeout return code
1063  // will be always a timeout.
1064  try {
1065  evaluateCurlCode( filename, ok, true /* timeout */);
1066  }
1067  catch ( const MediaFileNotFoundException &e ) {
1068  // if the file did not exist then we can return false
1069  return false;
1070  }
1071  catch ( const MediaException &e ) {
1072  // some error, we are not sure about file existence, rethrw
1073  ZYPP_RETHROW(e);
1074  }
1075  // exists
1076  return ( ok == CURLE_OK );
1077 }
1078 
1080 
1081 void MediaCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1082 {
1083  Pathname dest = target.absolutename();
1084  if( assert_dir( dest.dirname() ) )
1085  {
1086  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1087  ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
1088  }
1089 
1090  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1091  AutoFILE file;
1092  {
1093  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1094  if( ! buf )
1095  {
1096  ERR << "out of memory for temp file name" << endl;
1097  ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
1098  }
1099 
1100  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1101  if( tmp_fd == -1 )
1102  {
1103  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1104  ZYPP_THROW(MediaWriteException(destNew));
1105  }
1106  destNew = ManagedFile( (*buf), filesystem::unlink );
1107 
1108  file = ::fdopen( tmp_fd, "we" );
1109  if ( ! file )
1110  {
1111  ERR << "fopen failed for file '" << destNew << "'" << endl;
1112  ZYPP_THROW(MediaWriteException(destNew));
1113  }
1114  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1115  }
1116 
1117  DBG << "dest: " << dest << endl;
1118  DBG << "temp: " << destNew << endl;
1119 
1120  // set IFMODSINCE time condition (no download if not modified)
1121  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1122  {
1123  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1124  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1125  }
1126  else
1127  {
1128  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1129  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1130  }
1131  try
1132  {
1133  doGetFileCopyFile( srcFile, dest, file, report, options);
1134  }
1135  catch (Exception &e)
1136  {
1137  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1138  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1139  ZYPP_RETHROW(e);
1140  }
1141 
1142  long httpReturnCode = 0;
1143  CURLcode infoRet = curl_easy_getinfo(_curl,
1144  CURLINFO_RESPONSE_CODE,
1145  &httpReturnCode);
1146  bool modified = true;
1147  if (infoRet == CURLE_OK)
1148  {
1149  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1150  if ( httpReturnCode == 304
1151  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1152  {
1153  DBG << " Not modified.";
1154  modified = false;
1155  }
1156  DBG << endl;
1157  }
1158  else
1159  {
1160  WAR << "Could not get the response code." << endl;
1161  }
1162 
1163  if (modified || infoRet != CURLE_OK)
1164  {
1165  // apply umask
1166  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1167  {
1168  ERR << "Failed to chmod file " << destNew << endl;
1169  }
1170 
1171  file.resetDispose(); // we're going to close it manually here
1172  if ( ::fclose( file ) )
1173  {
1174  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1175  ZYPP_THROW(MediaWriteException(destNew));
1176  }
1177 
1178  // move the temp file into dest
1179  if ( rename( destNew, dest ) != 0 ) {
1180  ERR << "Rename failed" << endl;
1182  }
1183  destNew.resetDispose(); // no more need to unlink it
1184  }
1185 
1186  DBG << "done: " << PathInfo(dest) << endl;
1187 }
1188 
1190 
1191 void MediaCurl::doGetFileCopyFile( const OnMediaLocation & srcFile, const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1192 {
1193  DBG << srcFile.filename().asString() << endl;
1194 
1195  if(!_url.isValid())
1197 
1198  if(_url.getHost().empty())
1200 
1201  Url url(getFileUrl(srcFile.filename()));
1202 
1203  DBG << "URL: " << url.asString() << endl;
1204  // Use URL without options and without username and passwd
1205  // (some proxies dislike them in the URL).
1206  // Curl seems to need the just scheme, hostname and a path;
1207  // the rest was already passed as curl options (in attachTo).
1208  Url curlUrl( clearQueryString(url) );
1209 
1210  //
1211  // See also Bug #154197 and ftp url definition in RFC 1738:
1212  // The url "ftp://user@host/foo/bar/file" contains a path,
1213  // that is relative to the user's home.
1214  // The url "ftp://user@host//foo/bar/file" (or also with
1215  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1216  // contains an absolute path.
1217  //
1218  _lastRedirect.clear();
1219  std::string urlBuffer( curlUrl.asString());
1220  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1221  urlBuffer.c_str() );
1222  if ( ret != 0 ) {
1224  }
1225 
1226  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1227  if ( ret != 0 ) {
1229  }
1230 
1231  // Set callback and perform.
1232  internal::ProgressData progressData(_curl, _settings.timeout(), url, srcFile.downloadSize(), &report);
1233  if (!(options & OPTION_NO_REPORT_START))
1234  report->start(url, dest);
1235  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1236  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1237  }
1238 
1239  ret = curl_easy_perform( _curl );
1240 #if CURLVERSION_AT_LEAST(7,19,4)
1241  // bnc#692260: If the client sends a request with an If-Modified-Since header
1242  // with a future date for the server, the server may respond 200 sending a
1243  // zero size file.
1244  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1245  if ( ftell(file) == 0 && ret == 0 )
1246  {
1247  long httpReturnCode = 33;
1248  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1249  {
1250  long conditionUnmet = 33;
1251  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1252  {
1253  WAR << "TIMECONDITION unmet - retry without." << endl;
1254  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1255  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1256  ret = curl_easy_perform( _curl );
1257  }
1258  }
1259  }
1260 #endif
1261 
1262  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1263  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1264  }
1265 
1266  if ( ret != 0 )
1267  {
1268  ERR << "curl error: " << ret << ": " << _curlError
1269  << ", temp file size " << ftell(file)
1270  << " bytes." << endl;
1271 
1272  // the timeout is determined by the progress data object
1273  // which holds whether the timeout was reached or not,
1274  // otherwise it would be a user cancel
1275  try {
1276 
1277  if ( progressData.fileSizeExceeded() )
1278  ZYPP_THROW(MediaFileSizeExceededException(url, progressData.expectedFileSize()));
1279 
1280  evaluateCurlCode( srcFile.filename(), ret, progressData.timeoutReached() );
1281  }
1282  catch ( const MediaException &e ) {
1283  // some error, we are not sure about file existence, rethrw
1284  ZYPP_RETHROW(e);
1285  }
1286  }
1287 }
1288 
1290 
1291 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1292 {
1293  filesystem::DirContent content;
1294  getDirInfo( content, dirname, /*dots*/false );
1295 
1296  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1297  Pathname filename = dirname + it->name;
1298  int res = 0;
1299 
1300  switch ( it->type ) {
1301  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1302  case filesystem::FT_FILE:
1303  getFile( OnMediaLocation( filename ) );
1304  break;
1305  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1306  if ( recurse_r ) {
1307  getDir( filename, recurse_r );
1308  } else {
1309  res = assert_dir( localPath( filename ) );
1310  if ( res ) {
1311  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1312  }
1313  }
1314  break;
1315  default:
1316  // don't provide devices, sockets, etc.
1317  break;
1318  }
1319  }
1320 }
1321 
1323 
1324 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1325  const Pathname & dirname, bool dots ) const
1326 {
1327  getDirectoryYast( retlist, dirname, dots );
1328 }
1329 
1331 
1333  const Pathname & dirname, bool dots ) const
1334 {
1335  getDirectoryYast( retlist, dirname, dots );
1336 }
1337 
1339 //
1340 int MediaCurl::aliveCallback( void *clientp, double /*dltotal*/, double dlnow, double /*ultotal*/, double /*ulnow*/ )
1341 {
1342  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1343  if( pdata )
1344  {
1345  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1346  // prevent a percentage raise while downloading a metalink file. Download
1347  // activity however is indicated by propagating the download rate (via dlnow).
1348  pdata->updateStats( 0.0, dlnow );
1349  return pdata->reportProgress();
1350  }
1351  return 0;
1352 }
1353 
1354 int MediaCurl::progressCallback( void *clientp, double dltotal, double dlnow, double ultotal, double ulnow )
1355 {
1356  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1357  if( pdata )
1358  {
1359  // work around curl bug that gives us old data
1360  long httpReturnCode = 0;
1361  if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1362  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1363 
1364  pdata->updateStats( dltotal, dlnow );
1365  return pdata->reportProgress();
1366  }
1367  return 0;
1368 }
1369 
1371 {
1372  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>(clientp);
1373  return pdata ? pdata->curl() : 0;
1374 }
1375 
1377 
1378 std::string MediaCurl::getAuthHint() const
1379 {
1380  long auth_info = CURLAUTH_NONE;
1381 
1382  CURLcode infoRet =
1383  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1384 
1385  if(infoRet == CURLE_OK)
1386  {
1387  return CurlAuthData::auth_type_long2str(auth_info);
1388  }
1389 
1390  return "";
1391 }
1392 
1397 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1398 {
1399  internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1400  if ( data ) {
1401  data->expectedFileSize( expectedFileSize );
1402  }
1403 }
1404 
1406 
1407 bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1408 {
1410  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
1411  CurlAuthData_Ptr credentials;
1412 
1413  // get stored credentials
1414  AuthData_Ptr cmcred = cm.getCred(_url);
1415 
1416  if (cmcred && firstTry)
1417  {
1418  credentials.reset(new CurlAuthData(*cmcred));
1419  DBG << "got stored credentials:" << endl << *credentials << endl;
1420  }
1421  // if not found, ask user
1422  else
1423  {
1424 
1425  CurlAuthData_Ptr curlcred;
1426  curlcred.reset(new CurlAuthData());
1428 
1429  // preset the username if present in current url
1430  if (!_url.getUsername().empty() && firstTry)
1431  curlcred->setUsername(_url.getUsername());
1432  // if CM has found some credentials, preset the username from there
1433  else if (cmcred)
1434  curlcred->setUsername(cmcred->username());
1435 
1436  // indicate we have no good credentials from CM
1437  cmcred.reset();
1438 
1439  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1440 
1441  // set available authentication types from the exception
1442  // might be needed in prompt
1443  curlcred->setAuthType(availAuthTypes);
1444 
1445  // ask user
1446  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1447  {
1448  DBG << "callback answer: retry" << endl
1449  << "CurlAuthData: " << *curlcred << endl;
1450 
1451  if (curlcred->valid())
1452  {
1453  credentials = curlcred;
1454  // if (credentials->username() != _url.getUsername())
1455  // _url.setUsername(credentials->username());
1463  }
1464  }
1465  else
1466  {
1467  DBG << "callback answer: cancel" << endl;
1468  }
1469  }
1470 
1471  // set username and password
1472  if (credentials)
1473  {
1474  // HACK, why is this const?
1475  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1476  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1477 
1478  // set username and password
1479  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1481 
1482  // set available authentication types from the exception
1483  if (credentials->authType() == CURLAUTH_NONE)
1484  credentials->setAuthType(availAuthTypes);
1485 
1486  // set auth type (seems this must be set _after_ setting the userpwd)
1487  if (credentials->authType() != CURLAUTH_NONE)
1488  {
1489  // FIXME: only overwrite if not empty?
1490  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1491  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1493  }
1494 
1495  if (!cmcred)
1496  {
1497  credentials->setUrl(_url);
1498  cm.addCred(*credentials);
1499  cm.save();
1500  }
1501 
1502  return true;
1503  }
1504 
1505  return false;
1506 }
1507 
1508 //need a out of line definiton, otherwise vtable is emitted for every translation unit
1510 
1511  } // namespace media
1512 } // namespace zypp
1513 //
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:533
long timeout() const
transfer timeout
void globalInitCurlOnce()
Definition: curlhelper.cc:64
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
virtual bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:691
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:319
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:300
std::string targetDistribution() const
This is register.target attribute of the installed base product.
Definition: Target.cc:102
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:330
#define MIL
Definition: Logger.h:96
const Pathname & clientCertificatePath() const
SSL client certificate file.
bool progress(int value_r, const Url &file_r, double dbps_avg_r=-1, double dbps_current_r=-1) override
Download progress.
Definition: MediaCurl.cc:83
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar"
#define _(MSG)
Definition: Gettext.h:37
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:136
ProgressData()
Ctor no range [0,0](0).
Definition: progressdata.h:174
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:246
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:922
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:381
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1407
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:31
void setPassword(const std::string &val_r)
sets the auth password
std::chrono::steady_clock::time_point TimePoint
Definition: MediaCurl.cc:51
ByteCount _expectedFileSize
Definition: MediaCurl.cc:167
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
const char * distributionFlavorHeader()
Definition: MediaCurl.cc:265
Store and operate with byte count.
Definition: ByteCount.h:30
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
Holds transfer setting.
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
const std::string & authType() const
get the allowed authentication types
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
static int progressCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1354
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:173
int reportProgress() const
Definition: MediaCurl.cc:240
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:366
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
void start(const Url &file_r, Pathname localfile_r) override
Definition: MediaCurl.cc:71
const char * anonymousIdHeader()
Definition: MediaCurl.cc:251
const char * c_str() const
String representation.
Definition: Pathname.h:110
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:140
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
void setUsername(const std::string &val_r)
sets the auth username
Action problem(const Url &file_r, Error error_r, const std::string &description_r) override
Definition: MediaCurl.cc:102
virtual void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
Definition: MediaCurl.cc:733
static int aliveCallback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1340
time_t _timeNow
Now.
Definition: MediaCurl.cc:173
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:406
bool timeoutReached() const
Definition: MediaCurl.cc:149
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
Edition * _value
Definition: SysContent.cc:311
AutoDispose<int> calling ::close
Definition: AutoDispose.h:300
std::string _currentCookieFile
Definition: MediaCurl.h:160
const std::string & password() const
auth password
virtual void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1291
virtual void getFile(const OnMediaLocation &file) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:724
#define ERR
Definition: Logger.h:98
const std::string & username() const
auth username
bool fileSizeExceeded() const
Definition: MediaCurl.cc:152
Pathname localPath(const Pathname &pathname) const
Files provided will be available at &#39;localPath(filename)&#39;.
const Headers & headers() const
returns a list of all added headers
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:374
virtual void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:717
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1397
const std::string & hint() const
comma separated list of available authentication types
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:440
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1202
double _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:175
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:497
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:660
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
Definition: MediaCurl.cc:49
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:369
void setAuthType(const std::string &val_r)
set the allowed authentication types
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
void finish(const Url &file_r, Error error_r, const std::string &reason_r) override
Definition: MediaCurl.cc:108
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:700
const Url _url
Url to handle.
Definition: MediaHandler.h:113
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
const std::string & asString() const
String representation.
Definition: Pathname.h:91
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:742
Just inherits Exception to separate media exceptions.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:826
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:91
const ByteCount & downloadSize() const
The size of the resource on the server.
void disconnect()
Use concrete handler to isconnect media.
long connectTimeout() const
connection timeout
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
do not send a start ProgressReport
Definition: MediaCurl.h:45
#define WAR
Definition: Logger.h:97
double _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:176
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:518
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:176
Receiver * _oldRec
std::string proxyuserpwd
Definition: curlconfig.h:49
bool isValid() const
Verifies the Url.
Definition: Url.cc:489
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:974
const Pathname & clientKeyPath() const
SSL client key file.
void expectedFileSize(ByteCount newval_r)
Definition: MediaCurl.cc:158
shared_ptr< CurlAuthData > CurlAuthData_Ptr
Definition: curlauthdata.h:102
const Pathname & filename() const
The path to the resource on the medium.
virtual void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:656
std::string numstring(char n, int w=0)
Definition: String.h:289
Common baseclass for MediaCurl and MediaNetwork.
virtual bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:795
const char * agentString()
Definition: MediaCurl.cc:279
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:180
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1191
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:36
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:179
curl_slist * _customHeaders
Definition: MediaCurl.h:168
bool proxyEnabled() const
proxy is enabled
shared_ptr< AuthData > AuthData_Ptr
Definition: authdata.h:79
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:366
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:323
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:139
Base class for Exception.
Definition: Exception.h:145
Pathname attachPoint() const
Return the currently used attach point.
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:163
Url url() const
Url used.
Definition: MediaHandler.h:503
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:182
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:359
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:588
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:170
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1324
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:123
virtual void disconnectFrom() override
Definition: MediaCurl.cc:698
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1370
Typesafe passing of user data via callbacks.
Definition: UserData.h:38
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
double _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:177
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3"
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
void report(const UserData &userData_r=UserData()) override
The most generic way of sending/receiving data.
Definition: MediaCurl.cc:67
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:311
void updateStats(double dltotal=0.0, double dlnow=0.0)
Definition: MediaCurl.cc:197
static Pathname _cookieFile
Definition: MediaCurl.h:161
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:324
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:181
std::string anonymousUniqueId() const
anonymous unique id
Definition: Target.cc:132
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:789
std::string userPassword() const
returns the user and password as a user:pass string
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1378
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:172
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1081
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
Curl HTTP authentication data.
Definition: curlauthdata.h:22
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
Definition: Target.cc:127
OptionalDownloadProgressReport(bool isOptional=false)
Definition: MediaCurl.cc:53
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:167
ByteCount expectedFileSize() const
Definition: MediaCurl.cc:155
const std::string & proxy() const
proxy host
MediaCurlException(const Url &url_r, const std::string &err_r, const std::string &msg_r)
bool optional() const
Whether this is an optional resource.
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:171
const char * c_str() const
Definition: IdStringType.h:105
TrueBool _guard
Definition: TargetImpl.cc:1595
bool userMayRWX() const
Definition: PathInfo.h:353
const std::string & userAgentString() const
user agent string
Url manipulation class.
Definition: Url.h:91
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:95
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g. ...
Definition: MediaCurl.cc:297
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: MediaCurl.cc:168
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:572