libzypp  17.31.27
MediaMultiCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <ctype.h>
14 #include <sys/types.h>
15 #include <signal.h>
16 #include <sys/wait.h>
17 #include <netdb.h>
18 #include <arpa/inet.h>
19 #include <glib.h>
20 
21 #include <vector>
22 #include <iostream>
23 #include <algorithm>
24 
25 
26 #include <zypp/ZConfig.h>
27 #include <zypp/base/Logger.h>
29 #include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
30 #include <zypp-curl/parser/MetaLinkParser>
32 #include <zypp/ManagedFile.h>
34 #include <zypp-curl/auth/CurlAuthData>
37 
38 using std::endl;
39 using namespace zypp::base;
40 
41 #undef CURLVERSION_AT_LEAST
42 #define CURLVERSION_AT_LEAST(M,N,O) LIBCURL_VERSION_NUM >= ((((M)<<8)+(N))<<8)+(O)
43 
44 namespace zypp {
45  namespace media {
46 
48 
49 
112 class multifetchrequest;
113 
114 struct Stripe {
115 
116  enum RState {
117  PENDING, //< Pending Range
118  FETCH, //< Fetch is running!
119  COMPETING, //< Competing workers, needs checksum recheck
120  FINALIZED, //< Done, don't write to it anymore
121  REFETCH //< This block needs a refetch
122  };
123 
124  std::vector<off_t> blocks; //< required block numbers from blocklist
125  std::vector<RState> blockStates; //< current state of each block in blocks
126 };
127 
136 };
137 
138 // Hack: we derive from MediaCurl just to get the storage space for
139 // settings, url, curlerrors and the like
141  friend class multifetchrequest;
142 
143 public:
144  multifetchworker(int no, multifetchrequest &request, const Url &url);
145  ~multifetchworker();
146 
151  void nextjob();
152 
157  void runjob();
158 
164  bool continueJob();
165 
170  bool recheckChecksum( off_t blockIdx );
171 
175  void disableCompetition();
176 
180  void checkdns();
181  void adddnsfd( std::vector<GPollFD> &waitFds );
182  void dnsevent(const std::vector<GPollFD> &waitFds );
183 
184  const int _workerno;
185 
187  bool _competing = false;
188 
189  std::vector<MultiByteHandler::Range> _blocks;
190  std::vector<off_t> _rangeToStripeBlock;
191 
192  MultiByteHandler::ProtocolMode _protocolMode = MultiByteHandler::ProtocolMode::Basic;
193  std::unique_ptr<MultiByteHandler> _multiByteHandler;
194 
195  off_t _stripe = 0; //< The current stripe we are downloading
196  size_t _datasize = 0; //< The nr of bytes we need to download overall
197 
198  double _starttime = 0; //< When was the current job started
199  size_t _datareceived = 0; //< Data downloaded in the current job only
200  off_t _received = 0; //< Overall data"MultiByteHandler::prepare failed" fetched by this worker
201 
202  double _avgspeed = 0;
203  double _maxspeed = 0;
204 
205  double _sleepuntil = 0;
206 
207 private:
208  void run();
209  void stealjob();
210  bool setupHandle();
211  MultiByteHandler::Range rangeFromBlock( off_t blockNo ) const;
212 
213  size_t writefunction ( char *ptr, std::optional<off_t> offset, size_t bytes ) override;
214  size_t headerfunction ( char *ptr, size_t bytes ) override;
215  bool beginRange ( off_t range, std::string &cancelReason ) override;
216  bool finishedRange ( off_t range, bool validated, std::string &cancelReason ) override;
217 
218  multifetchrequest *_request = nullptr;
219  int _pass = 0;
220  std::string _urlbuf;
221 
222  pid_t _pid = 0;
223  int _dnspipe = -1;
224 };
225 
227 public:
228  multifetchrequest(const MediaMultiCurl *context, const Pathname &filename, const Url &baseurl, CURLM *multi, FILE *fp, callback::SendReport<DownloadProgressReport> *report, MediaBlockList &&blklist, off_t filesize);
230 
231  void run(std::vector<Url> &urllist);
232  static ByteCount makeBlksize( uint maxConns, size_t filesize );
233 
235  return _blklist;
236  }
237 
238 protected:
239  friend class multifetchworker;
240 
244 
245  FILE *_fp = nullptr;
248 
249  std::vector<Stripe> _requiredStripes; // all the data we need
250 
251  off_t _filesize = 0; //< size of the file we want to download
252 
253  CURLM *_multi = nullptr;
254 
255  std::list< std::unique_ptr<multifetchworker> > _workers;
256  bool _stealing = false;
257  bool _havenewjob = false;
258 
259  zypp::ByteCount _defaultBlksize = 0; //< The blocksize to use if the metalink file does not specify one
260  off_t _stripeNo = 0; //< next stripe to download
261 
262  size_t _activeworkers = 0;
263  size_t _lookupworkers = 0;
264  size_t _sleepworkers = 0;
265  double _minsleepuntil = 0;
266  bool _finished = false;
267 
268  off_t _totalsize = 0; //< nr of bytes we need to download ( e.g. filesize - ( bytes reused from deltafile ) )
269  off_t _fetchedsize = 0;
270  off_t _fetchedgoodsize = 0;
271 
272  double _starttime = 0;
273  double _lastprogress = 0;
274 
275  double _lastperiodstart = 0;
276  double _lastperiodfetched = 0;
277  double _periodavg = 0;
278 
279 public:
280  double _timeout = 0;
281  double _connect_timeout = 0;
282  double _maxspeed = 0;
283  int _maxworkers = 0;
284 };
285 
286 constexpr auto MIN_REQ_MIRRS = 4;
287 constexpr auto MAXURLS = 10;
288 
290 
291 static double
293 {
294 #if _POSIX_C_SOURCE >= 199309L
295  struct timespec ts;
296  if ( clock_gettime( CLOCK_MONOTONIC, &ts) )
297  return 0;
298  return ts.tv_sec + ts.tv_nsec / 1000000000.;
299 #else
300  struct timeval tv;
301  if (gettimeofday(&tv, NULL))
302  return 0;
303  return tv.tv_sec + tv.tv_usec / 1000000.;
304 #endif
305 }
306 
307 size_t
308 multifetchworker::writefunction(char *ptr, std::optional<off_t> offset, size_t bytes)
309 {
310  if ( _state == WORKER_BROKEN || _state == WORKER_DISCARD )
311  return bytes ? 0 : 1;
312 
313  double now = currentTime();
314 
315  // update stats of overall data
316  _datareceived += bytes;
317  _received += bytes;
318  _request->_lastprogress = now;
319 
320  const auto &currRange = _multiByteHandler->currentRange();
321  if (!currRange)
322  return 0; // we always write to a range
323 
324  auto &stripeDesc = _request->_requiredStripes[_stripe];
325  if ( !_request->_fp || stripeDesc.blockStates[ _rangeToStripeBlock[*currRange] ] == Stripe::FINALIZED ) {
326  // someone else finished our block first!
327  // we stop here and fetch new jobs if there are still some
329  _competing = false;
330  return 0;
331  }
332 
333  const auto &blk = _blocks[*currRange];
334  off_t seekTo = blk.start + blk.bytesWritten;
335 
336  if ( ftell( _request->_fp ) != seekTo ) {
337  // if we can't seek the file there is no purpose in trying again
338  if (fseeko(_request->_fp, seekTo, SEEK_SET))
339  return bytes ? 0 : 1;
340  }
341 
342  size_t cnt = fwrite(ptr, 1, bytes, _request->_fp);
343  _request->_fetchedsize += cnt;
344  return cnt;
345 }
346 
347 bool multifetchworker::beginRange ( off_t workerRangeOff, std::string &cancelReason )
348 {
349  auto &stripeDesc = _request->_requiredStripes[_stripe];
350  auto stripeRangeOff = _rangeToStripeBlock[workerRangeOff];
351  const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
352 
353  if ( currRangeState == Stripe::FINALIZED ){
354  cancelReason = "Cancelled because stripe block is already finalized";
356  WAR << "#" << _workerno << ": trying to start a range ("<<stripeRangeOff<<"["<< _blocks[workerRangeOff].start <<" : "<<_blocks[workerRangeOff].len<<"]) that was already finalized, cancelling. Stealing was: " << _request->_stealing << endl;
357  return false;
358  }
359  stripeDesc.blockStates[stripeRangeOff] = currRangeState == Stripe::PENDING ? Stripe::FETCH : Stripe::COMPETING;
360  return true;
361 }
362 
363 bool multifetchworker::finishedRange ( off_t workerRangeOff, bool validated, std::string &cancelReason )
364 {
365  auto &stripeDesc = _request->_requiredStripes[_stripe];
366  auto stripeRangeOff = _rangeToStripeBlock[workerRangeOff];
367  const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
368 
369  if ( !validated ) {
370  // fail, worker will go into WORKER_BROKEN
371  cancelReason = "Block failed to validate";
372  return false;
373  }
374 
375  if ( currRangeState == Stripe::FETCH ) {
376  // only us who wrote here, block is finalized
377  stripeDesc.blockStates[stripeRangeOff] = Stripe::FINALIZED;
378  _request->_fetchedgoodsize += _blocks[workerRangeOff].len;
379  } else {
380  // others wrote here, we need to check the full checksum
381  if ( recheckChecksum ( workerRangeOff ) ) {
382  stripeDesc.blockStates[stripeRangeOff] = Stripe::FINALIZED;
383  _request->_fetchedgoodsize += _blocks[workerRangeOff].len;
384  } else {
385  // someone messed that block up, set it to refetch but continue since our
386  // data is valid
387  WAR << "#" << _workerno << ": Broken data in COMPETING block, requesting refetch. Stealing is: " << _request->_stealing << endl;
388  stripeDesc.blockStates[stripeRangeOff] = Stripe::REFETCH;
389  }
390  }
391  return true;
392 }
393 
394 size_t
395 multifetchworker::headerfunction( char *p, size_t bytes )
396 {
397  size_t l = bytes;
398  if (l > 9 && !strncasecmp(p, "Location:", 9)) {
399  std::string line(p + 9, l - 9);
400  if (line[l - 10] == '\r')
401  line.erase(l - 10, 1);
402  XXX << "#" << _workerno << ": redirecting to" << line << endl;
403  return bytes;
404  }
405 
406  const auto &repSize = _multiByteHandler->reportedFileSize ();
407  if ( repSize && *repSize != _request->_filesize ) {
408  XXX << "#" << _workerno << ": filesize mismatch" << endl;
410  strncpy(_curlError, "filesize mismatch", CURL_ERROR_SIZE);
411  return 0;
412  }
413 
414  return bytes;
415 }
416 
417 multifetchworker::multifetchworker(int no, multifetchrequest &request, const Url &url)
418 : MediaCurl(url, Pathname())
419 , _workerno( no )
420 , _maxspeed( request._maxspeed )
421 , _request ( &request )
422 {
423  Url curlUrl( clearQueryString(url) );
424  _urlbuf = curlUrl.asString();
426  if (_curl)
427  XXX << "reused worker from pool" << endl;
428  if (!_curl && !(_curl = curl_easy_init()))
429  {
431  strncpy(_curlError, "curl_easy_init failed", CURL_ERROR_SIZE);
432  return;
433  }
434 
435  if ( url.getScheme() == "http" || url.getScheme() == "https" )
437 
438  setupHandle();
439  checkdns();
440 }
441 
443 {
444  try {
445  setupEasy();
446  } catch (Exception &ex) {
447  curl_easy_cleanup(_curl);
448  _curl = 0;
450  strncpy(_curlError, "curl_easy_setopt failed", CURL_ERROR_SIZE);
451  return false;
452  }
453  curl_easy_setopt(_curl, CURLOPT_PRIVATE, this);
454  curl_easy_setopt(_curl, CURLOPT_URL, _urlbuf.c_str());
455 
456  // if this is the same host copy authorization
457  // (the host check is also what curl does when doing a redirect)
458  // (note also that unauthorized exceptions are thrown with the request host)
459  if ( _url.getHost() == _request->_context->_url.getHost()) {
463  if ( _settings.userPassword().size() ) {
464  curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
465  std::string use_auth = _settings.authType();
466  if (use_auth.empty())
467  use_auth = "digest,basic"; // our default
468  long auth = CurlAuthData::auth_type_str2long(use_auth);
469  if( auth != CURLAUTH_NONE)
470  {
471  XXX << "#" << _workerno << ": Enabling HTTP authentication methods: " << use_auth
472  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
473  curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, auth);
474  }
475  }
476  }
477  return true;
478 }
479 
481 {
482  if (_curl)
483  {
485  curl_multi_remove_handle(_request->_multi, _curl);
486  if (_state == WORKER_DONE || _state == WORKER_SLEEP)
487  {
488 #if CURLVERSION_AT_LEAST(7,15,5)
489  curl_easy_setopt(_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)0);
490 #endif
491  curl_easy_setopt(_curl, CURLOPT_PRIVATE, (void *)0);
492  curl_easy_setopt(_curl, CURLOPT_WRITEFUNCTION, (void *)0);
493  curl_easy_setopt(_curl, CURLOPT_WRITEDATA, (void *)0);
494  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, (void *)0);
495  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, (void *)0);
497  }
498  else
499  curl_easy_cleanup(_curl);
500  _curl = 0;
501  }
502  if (_pid)
503  {
504  kill(_pid, SIGKILL);
505  int status;
506  while (waitpid(_pid, &status, 0) == -1)
507  if (errno != EINTR)
508  break;
509  _pid = 0;
510  }
511  if (_dnspipe != -1)
512  {
513  close(_dnspipe);
514  _dnspipe = -1;
515  }
516  // the destructor in MediaCurl doesn't call disconnect() if
517  // the media is not attached, so we do it here manually
518  disconnectFrom();
519 }
520 
521 static inline bool env_isset(std::string name)
522 {
523  const char *s = getenv(name.c_str());
524  return s && *s ? true : false;
525 }
526 
527 void
529 {
530  std::string host = _url.getHost();
531 
532  if (host.empty())
533  return;
534 
535  if (_request->_context->isDNSok(host))
536  return;
537 
538  // no need to do dns checking for numeric hosts
539  char addrbuf[128];
540  if (inet_pton(AF_INET, host.c_str(), addrbuf) == 1)
541  return;
542  if (inet_pton(AF_INET6, host.c_str(), addrbuf) == 1)
543  return;
544 
545  // no need to do dns checking if we use a proxy
546  if (!_settings.proxy().empty())
547  return;
548  if (env_isset("all_proxy") || env_isset("ALL_PROXY"))
549  return;
550  std::string schemeproxy = _url.getScheme() + "_proxy";
551  if (env_isset(schemeproxy))
552  return;
553  if (schemeproxy != "http_proxy")
554  {
555  std::transform(schemeproxy.begin(), schemeproxy.end(), schemeproxy.begin(), ::toupper);
556  if (env_isset(schemeproxy))
557  return;
558  }
559 
560  XXX << "checking DNS lookup of " << host << endl;
561  int pipefds[2];
562  if (pipe(pipefds))
563  {
565  strncpy(_curlError, "DNS pipe creation failed", CURL_ERROR_SIZE);
566  return;
567  }
568  _pid = fork();
569  if (_pid == pid_t(-1))
570  {
571  close(pipefds[0]);
572  close(pipefds[1]);
573  _pid = 0;
575  strncpy(_curlError, "DNS checker fork failed", CURL_ERROR_SIZE);
576  return;
577  }
578  else if (_pid == 0)
579  {
580  close(pipefds[0]);
581  // XXX: close all other file descriptors
582  struct addrinfo *ai, aihints;
583  memset(&aihints, 0, sizeof(aihints));
584  aihints.ai_family = PF_UNSPEC;
585  int tstsock = socket(PF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
586  if (tstsock == -1)
587  aihints.ai_family = PF_INET;
588  else
589  close(tstsock);
590  aihints.ai_socktype = SOCK_STREAM;
591  aihints.ai_flags = AI_CANONNAME;
592  unsigned int connecttimeout = _request->_connect_timeout;
593  if (connecttimeout)
594  alarm(connecttimeout);
595  signal(SIGALRM, SIG_DFL);
596  if (getaddrinfo(host.c_str(), NULL, &aihints, &ai))
597  _exit(1);
598  _exit(0);
599  }
600  close(pipefds[1]);
601  _dnspipe = pipefds[0];
603 }
604 
605 void
606 multifetchworker::adddnsfd(std::vector<GPollFD> &waitFds)
607 {
608  if (_state != WORKER_LOOKUP)
609  return;
610 
611  waitFds.push_back (
612  GPollFD {
613  .fd = _dnspipe,
614  .events = G_IO_IN | G_IO_HUP | G_IO_ERR,
615  .revents = 0
616  });
617 }
618 
619 void
620 multifetchworker::dnsevent( const std::vector<GPollFD> &waitFds )
621 {
622  bool hasEvent = std::any_of( waitFds.begin (), waitFds.end(),[this]( const GPollFD &waitfd ){
623  return ( waitfd.fd == _dnspipe && waitfd.revents != 0 );
624  });
625 
626  if (_state != WORKER_LOOKUP || !hasEvent)
627  return;
628  int status;
629  while (waitpid(_pid, &status, 0) == -1)
630  {
631  if (errno != EINTR)
632  return;
633  }
634  _pid = 0;
635  if (_dnspipe != -1)
636  {
637  close(_dnspipe);
638  _dnspipe = -1;
639  }
640  if (!WIFEXITED(status))
641  {
643  strncpy(_curlError, "DNS lookup failed", CURL_ERROR_SIZE);
645  return;
646  }
647  int exitcode = WEXITSTATUS(status);
648  XXX << "#" << _workerno << ": DNS lookup returned " << exitcode << endl;
649  if (exitcode != 0)
650  {
652  strncpy(_curlError, "DNS lookup failed", CURL_ERROR_SIZE);
654  return;
655  }
657  nextjob();
658 }
659 
660 bool multifetchworker::recheckChecksum( off_t workerRangeIdx )
661 {
662  // XXX << "recheckChecksum block " << _blkno << endl;
663  if (!_request->_fp || !_datasize || !_blocks.size() )
664  return true;
665 
666  auto &blk = _blocks[workerRangeIdx];
667  if ( !blk._digest )
668  return true;
669 
670  const auto currOf = ftell( _request->_fp );
671  if ( currOf == -1 )
672  return false;
673 
674  if (fseeko(_request->_fp, blk.start, SEEK_SET))
675  return false;
676 
677  zypp::Digest newDig = blk._digest->clone();
678 
679  char buf[4096];
680  size_t l = blk.len;
681  while (l) {
682  size_t cnt = l > sizeof(buf) ? sizeof(buf) : l;
683  if (fread(buf, cnt, 1, _request->_fp) != 1)
684  return false;
685  newDig.update(buf, cnt);
686  l -= cnt;
687  }
688 
689  if (fseeko(_request->_fp, currOf, SEEK_SET))
690  return false;
691 
692  blk._digest = std::move(newDig);
693  if (!_multiByteHandler->validateRange(blk)) {
694  WAR << "#" << _workerno << " Stripe: " << _stripe << ": Stripe-Block: " << _rangeToStripeBlock[workerRangeIdx] << " failed to validate" << endl;
695  return false;
696  }
697 
698  return true;
699 }
700 
705 {
706  UByteArray sum;
707  std::optional<zypp::Digest> digest;
708  std::optional<size_t> relDigLen;
709  std::optional<size_t> blkSumPad;
710 
711  const auto &blk = _request->_blklist.getBlock( blkNo );
712  if ( _request->_blklist.haveChecksum ( blkNo ) ) {
713  sum = _request->_blklist.getChecksum( blkNo );
714  relDigLen = sum.size( );
715  blkSumPad = _request->_blklist.checksumPad( );
716  digest = zypp::Digest();
717  digest->create( _request->_blklist.getChecksumType() );
718  }
719 
721  blk.off,
722  blk.size,
723  std::move(digest),
724  std::move(sum),
725  {}, // empty user data
726  std::move(relDigLen),
727  std::move(blkSumPad) );
728 }
729 
731 {
732  if (!_request->_stealing)
733  {
734  XXX << "start stealing!" << endl;
735  _request->_stealing = true;
736  }
737 
738  multifetchworker *best = 0; // best choice for the worker we want to compete with
739  double now = 0;
740 
741  // look through all currently running workers to find the best candidate we
742  // could steal from
743  for (auto workeriter = _request->_workers.begin(); workeriter != _request->_workers.end(); ++workeriter)
744  {
745  multifetchworker *worker = workeriter->get();
746  if (worker == this)
747  continue;
748  if (worker->_pass == -1)
749  continue; // do not steal!
750  if (worker->_state == WORKER_DISCARD || worker->_state == WORKER_DONE || worker->_state == WORKER_SLEEP || !worker->_datasize)
751  continue; // do not steal finished jobs
752  if (!worker->_avgspeed && worker->_datareceived)
753  {
754  // calculate avg speed for the worker if that was not done yet
755  if (!now)
756  now = currentTime();
757  if (now > worker->_starttime)
758  worker->_avgspeed = worker->_datareceived / (now - worker->_starttime);
759  }
760  // only consider worker who still have work
761  if ( worker->_datasize - worker->_datareceived <= 0 )
762  continue;
763  if (!best || best->_pass > worker->_pass)
764  {
765  best = worker;
766  continue;
767  }
768  if (best->_pass < worker->_pass)
769  continue;
770  // if it is the same stripe, our current best choice is competing with the worker we are looking at
771  // we need to check if we are faster than the fastest one competing for this stripe, so we want the best.
772  // Otherwise the worst.
773  if (worker->_stripe == best->_stripe)
774  {
775  if ((worker->_datasize - worker->_datareceived) * best->_avgspeed < (best->_datasize - best->_datareceived) * worker->_avgspeed)
776  best = worker;
777  }
778  else
779  {
780  if ((worker->_datasize - worker->_datareceived) * best->_avgspeed > (best->_datasize - best->_datareceived) * worker->_avgspeed)
781  best = worker;
782  }
783  }
784  if (!best)
785  {
788  _request->_finished = true;
789  return;
790  }
791  // do not sleep twice
792  if (_state != WORKER_SLEEP)
793  {
794  if (!_avgspeed && _datareceived)
795  {
796  if (!now)
797  now = currentTime();
798  if (now > _starttime)
799  _avgspeed = _datareceived / (now - _starttime);
800  }
801 
802  // lets see if we should sleep a bit
803  XXX << "me #" << _workerno << ": " << _avgspeed << ", size " << best->_datasize << endl;
804  XXX << "best #" << best->_workerno << ": " << best->_avgspeed << ", size " << (best->_datasize - best->_datareceived) << endl;
805 
806  // check if we could download the full data from best faster than best could download its remaining data
807  if ( _avgspeed && best->_avgspeed // we and best have average speed information
808  && _avgspeed <= best->_avgspeed ) // and we are not faster than best
809  {
810  if (!now)
811  now = currentTime();
812  double sl = (best->_datasize - best->_datareceived) / best->_avgspeed * 2;
813  if (sl > 1)
814  sl = 1;
815  XXX << "#" << _workerno << ": going to sleep for " << sl * 1000 << " ms" << endl;
816  _sleepuntil = now + sl;
819  return;
820  }
821  }
822 
823  _competing = true;
824  best->_competing = true;
825  _stripe = best->_stripe;
826 
827  best->_pass++;
828  _pass = best->_pass;
829 
830  runjob();
831 }
832 
833 void
835 {
836  for ( auto workeriter = _request->_workers.begin(); workeriter != _request->_workers.end(); ++workeriter)
837  {
838  multifetchworker *worker = workeriter->get();
839  if (worker == this)
840  continue;
841  if (worker->_stripe == _stripe)
842  {
843  if (worker->_state == WORKER_FETCH)
844  worker->_state = WORKER_DISCARD;
845  worker->_pass = -1; /* do not steal this one, we already have it */
846  }
847  }
848 }
849 
851 {
852  _datasize = 0;
853  _blocks.clear();
854 
855  // claim next stripe for us, or steal if there nothing left to claim
857  stealjob();
858  return;
859  }
860 
862  runjob();
863 }
864 
866 {
867  _datasize = 0;
868  _blocks.clear ();
869  _rangeToStripeBlock.clear ();
870 
871  auto &stripeDesc = _request->_requiredStripes[_stripe];
872  for ( uint i = 0; i < stripeDesc.blocks.size(); i++ ) {
873  // ignore verified and finalized ranges
874  if( stripeDesc.blockStates[i] == Stripe::FINALIZED ) {
875  continue;
876  } else {
877  _blocks.push_back( rangeFromBlock(stripeDesc.blocks[i]) );
878  _rangeToStripeBlock.push_back( i );
879  _datasize += _blocks.back().len;
880  }
881  }
882 
883  if ( _datasize == 0 ) {
884  // no blocks left in this stripe
887  if ( !_request->_activeworkers )
888  _request->_finished = true;
889  return;
890  }
891 
892  DBG << "#" << _workerno << "Done adding blocks to download, going to download: " << _blocks.size() << " nr of block with " << _datasize << " nr of bytes" << std::endl;
893 
894  _multiByteHandler.reset( nullptr );
895  _multiByteHandler = std::make_unique<MultiByteHandler>(_protocolMode, _curl, _blocks, *this );
897  _datareceived = 0;
898  run();
899 }
900 
902 {
903  bool hadRangeFail = _multiByteHandler->lastError() == MultiByteHandler::Code::RangeFail;
904  if ( !_multiByteHandler->prepareToContinue() ) {
905  strncpy( _curlError, _multiByteHandler->lastErrorMessage().c_str(), CURL_ERROR_SIZE );
906  return false;
907  }
908 
909  if ( hadRangeFail ) {
910  // we reset the handle to default values. We do this to not run into
911  // "transfer closed with outstanding read data remaining" error CURL sometimes returns when
912  // we cancel a connection because of a range error to request a smaller batch.
913  // The error will still happen but much less frequently than without resetting the handle.
914  //
915  // Note: Even creating a new handle will NOT fix the issue
916  curl_easy_reset( _curl );
917  if ( !setupHandle())
918  return false;
919  }
920 
921  run();
922  return true;
923 }
924 
925 void
927 {
928  if (_state == WORKER_BROKEN || _state == WORKER_DONE)
929  return; // just in case...
930 
931  if ( !_multiByteHandler->prepare() ) {
934  strncpy( _curlError, _multiByteHandler->lastErrorMessage ().c_str() , CURL_ERROR_SIZE );
935  return;
936  }
937 
938  if (curl_multi_add_handle(_request->_multi, _curl) != CURLM_OK) {
941  strncpy( _curlError, "curl_multi_add_handle failed", CURL_ERROR_SIZE );
942  return;
943  }
944 
945  _request->_havenewjob = true;
947 }
948 
949 
951 
952 
953 multifetchrequest::multifetchrequest(const MediaMultiCurl *context, const Pathname &filename, const Url &baseurl, CURLM *multi, FILE *fp, callback::SendReport<DownloadProgressReport> *report, MediaBlockList &&blklist, off_t filesize)
954  : _context(context)
955  , _filename(filename)
956  , _baseurl(baseurl)
957  , _fp(fp)
958  , _report(report)
959  , _blklist(std::move(blklist))
960  , _filesize(filesize)
961  , _multi(multi)
962  , _starttime(currentTime())
963  , _timeout(context->_settings.timeout())
964  , _connect_timeout(context->_settings.connectTimeout())
965  , _maxspeed(context->_settings.maxDownloadSpeed())
966  , _maxworkers(context->_settings.maxConcurrentConnections())
967  {
969  if (_maxworkers > MAXURLS)
971  if (_maxworkers <= 0)
972  _maxworkers = 1;
973 
974  // calculate the total size of our download
975  for (size_t blkno = 0; blkno < _blklist.numBlocks(); blkno++)
976  _totalsize += _blklist.getBlock(blkno).size;
977 
978  // equally distribute the data we want to download over all workers
980 
981  // lets build stripe informations
982  zypp::ByteCount currStripeSize = 0;
983  for (size_t blkno = 0; blkno < _blklist.numBlocks(); blkno++) {
984 
985  const MediaBlock &blk = _blklist.getBlock(blkno);
986  if ( _requiredStripes.empty() || currStripeSize >= _defaultBlksize ) {
987  _requiredStripes.push_back( Stripe{} );
988  currStripeSize = 0;
989  }
990 
991  _requiredStripes.back().blocks.push_back(blkno);
992  _requiredStripes.back().blockStates.push_back(Stripe::PENDING);
993  currStripeSize += blk.size;
994  }
995 
996  MIL << "Downloading " << _blklist.numBlocks() << " blocks via " << _requiredStripes.size() << " stripes on " << _maxworkers << " connections." << endl;
997 }
998 
1000 {
1001  _workers.clear();
1002 }
1003 
1004 void
1005 multifetchrequest::run(std::vector<Url> &urllist)
1006 {
1007  int workerno = 0;
1008  std::vector<Url>::iterator urliter = urllist.begin();
1009 
1010  struct CurlMuliSockHelper {
1011  CurlMuliSockHelper( multifetchrequest &p ) : _parent(p) {
1012  curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETFUNCTION, socketcb );
1013  curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETDATA, this );
1014  curl_multi_setopt( _parent._multi, CURLMOPT_TIMERFUNCTION, timercb );
1015  curl_multi_setopt( _parent._multi, CURLMOPT_TIMERDATA, this );
1016  }
1017 
1018  ~CurlMuliSockHelper() {
1019  curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETFUNCTION, nullptr );
1020  curl_multi_setopt( _parent._multi, CURLMOPT_SOCKETDATA, nullptr );
1021  curl_multi_setopt( _parent._multi, CURLMOPT_TIMERFUNCTION, nullptr );
1022  curl_multi_setopt( _parent._multi, CURLMOPT_TIMERDATA, nullptr );
1023  }
1024 
1025  static int socketcb (CURL * easy, curl_socket_t s, int what, CurlMuliSockHelper *userp, void *sockp ) {
1026  auto it = std::find_if( userp->socks.begin(), userp->socks.end(), [&]( const GPollFD &fd){ return fd.fd == s; });
1027  gushort events = 0;
1028  if ( what == CURL_POLL_REMOVE ) {
1029  if ( it == userp->socks.end() ) {
1030  WAR << "Ignoring unknown socket in static_socketcb" << std::endl;
1031  return 0;
1032  }
1033  userp->socks.erase(it);
1034  return 0;
1035  } else if ( what == CURL_POLL_IN ) {
1036  events = G_IO_IN | G_IO_HUP | G_IO_ERR;
1037  } else if ( what == CURL_POLL_OUT ) {
1038  events = G_IO_OUT | G_IO_ERR;
1039  } else if ( what == CURL_POLL_INOUT ) {
1040  events = G_IO_IN | G_IO_OUT | G_IO_HUP | G_IO_ERR;
1041  }
1042 
1043  if ( it != userp->socks.end() ) {
1044  it->events = events;
1045  it->revents = 0;
1046  } else {
1047  userp->socks.push_back(
1048  GPollFD{
1049  .fd = s,
1050  .events = events,
1051  .revents = 0
1052  }
1053  );
1054  }
1055 
1056  return 0;
1057  }
1058 
1059  //called by curl to setup a timer
1060  static int timercb( CURLM *, long timeout_ms, CurlMuliSockHelper *thatPtr ) {
1061  if ( !thatPtr )
1062  return 0;
1063  if ( timeout_ms == -1 )
1064  thatPtr->timeout_ms.reset(); // curl wants to delete its timer
1065  else
1066  thatPtr->timeout_ms = timeout_ms; // maximum time curl wants us to sleep
1067  return 0;
1068  }
1069 
1070  multifetchrequest &_parent;
1071  std::vector<GPollFD> socks;
1072  std::optional<long> timeout_ms = 0; //if set curl wants a timeout
1073  } _curlHelper(*this) ;
1074 
1075  // kickstart curl
1076  int handles = 0;
1077  CURLMcode mcode = curl_multi_socket_action( _multi, CURL_SOCKET_TIMEOUT, 0, &handles );
1078  if (mcode != CURLM_OK)
1079  ZYPP_THROW(MediaCurlException(_baseurl, "curl_multi_socket_action", "unknown error"));
1080 
1081  for (;;)
1082  {
1083  // list of all fds we want to poll
1084  std::vector<GPollFD> waitFds;
1085  int dnsFdCount = 0;
1086 
1087  if (_finished)
1088  {
1089  XXX << "finished!" << endl;
1090  break;
1091  }
1092 
1093  if ((int)_activeworkers < _maxworkers && urliter != urllist.end() && _workers.size() < MAXURLS)
1094  {
1095  // spawn another worker!
1096  _workers.push_back(std::make_unique<multifetchworker>(workerno++, *this, *urliter));
1097  auto &worker = _workers.back();
1098  if (worker->_state != WORKER_BROKEN)
1099  {
1100  _activeworkers++;
1101  if (worker->_state != WORKER_LOOKUP)
1102  {
1103  worker->nextjob();
1104  }
1105  else
1106  _lookupworkers++;
1107  }
1108  ++urliter;
1109  continue;
1110  }
1111  if (!_activeworkers)
1112  {
1113  WAR << "No more active workers!" << endl;
1114  // show the first worker error we find
1115  for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1116  {
1117  if ((*workeriter)->_state != WORKER_BROKEN)
1118  continue;
1119  ZYPP_THROW(MediaCurlException(_baseurl, "Server error", (*workeriter)->_curlError));
1120  }
1121  break;
1122  }
1123 
1124  if (_lookupworkers)
1125  for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1126  (*workeriter)->adddnsfd( waitFds );
1127 
1128  // if we added a new job we have to call multi_perform once
1129  // to make it show up in the fd set. do not sleep in this case.
1130  int timeoutMs = _havenewjob ? 0 : 200;
1131  if ( _sleepworkers && !_havenewjob ) {
1132  if (_minsleepuntil == 0) {
1133  for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter) {
1134  multifetchworker *worker = workeriter->get();
1135  if (worker->_state != WORKER_SLEEP)
1136  continue;
1137  if (!_minsleepuntil || _minsleepuntil > worker->_sleepuntil)
1138  _minsleepuntil = worker->_sleepuntil;
1139  }
1140  }
1141  double sl = _minsleepuntil - currentTime();
1142  if (sl < 0) {
1143  sl = 0;
1144  _minsleepuntil = 0;
1145  }
1146  if (sl < .2)
1147  timeoutMs = sl * 1000;
1148  }
1149 
1150  if ( _curlHelper.timeout_ms.has_value() )
1151  timeoutMs = std::min<long>( timeoutMs, _curlHelper.timeout_ms.value() );
1152 
1153  dnsFdCount = waitFds.size(); // remember how many dns fd's we have
1154  waitFds.insert( waitFds.end(), _curlHelper.socks.begin(), _curlHelper.socks.end() ); // add the curl fd's to the poll data
1155 
1156  int r = zyppng::eintrSafeCall( g_poll, waitFds.data(), waitFds.size(), timeoutMs );
1157  if ( r == -1 )
1158  ZYPP_THROW(MediaCurlException(_baseurl, "g_poll() failed", "unknown error"));
1159  if ( r != 0 && _lookupworkers ) {
1160  for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1161  {
1162  multifetchworker *worker = workeriter->get();
1163  if (worker->_state != WORKER_LOOKUP)
1164  continue;
1165  (*workeriter)->dnsevent( waitFds );
1166  if (worker->_state != WORKER_LOOKUP)
1167  _lookupworkers--;
1168  }
1169  }
1170  _havenewjob = false;
1171 
1172  // run curl
1173  if ( r == 0 ) {
1174  int handles = 0;
1175  CURLMcode mcode = curl_multi_socket_action( _multi, CURL_SOCKET_TIMEOUT, 0, &handles );
1176  if (mcode != CURLM_OK)
1177  ZYPP_THROW(MediaCurlException(_baseurl, "curl_multi_socket_action", "unknown error"));
1178  } else {
1179  for ( int sock = dnsFdCount; sock < waitFds.size(); sock++ ) {
1180  const auto &waitFd = waitFds[sock];
1181  if ( waitFd.revents == 0 )
1182  continue;
1183 
1184  int ev = 0;
1185  if ( (waitFd.revents & G_IO_HUP) == G_IO_HUP
1186  || (waitFd.revents & G_IO_IN) == G_IO_IN ) {
1187  ev = CURL_CSELECT_IN;
1188  }
1189  if ( (waitFd.revents & G_IO_OUT) == G_IO_OUT ) {
1190  ev |= CURL_CSELECT_OUT;
1191  }
1192  if ( (waitFd.revents & G_IO_ERR) == G_IO_ERR ) {
1193  ev |= CURL_CSELECT_ERR;
1194  }
1195 
1196  int runn = 0;
1197  CURLMcode mcode = curl_multi_socket_action( _multi, waitFd.fd, ev, &runn );
1198  if (mcode != CURLM_OK)
1199  ZYPP_THROW(MediaCurlException(_baseurl, "curl_multi_socket_action", "unknown error"));
1200  }
1201  }
1202 
1203  double now = currentTime();
1204 
1205  // update periodavg
1206  if (now > _lastperiodstart + .5)
1207  {
1208  if (!_periodavg)
1210  else
1213  _lastperiodstart = now;
1214  }
1215 
1216  // wake up sleepers
1217  if (_sleepworkers)
1218  {
1219  for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1220  {
1221  multifetchworker *worker = workeriter->get();
1222  if (worker->_state != WORKER_SLEEP)
1223  continue;
1224  if (worker->_sleepuntil > now)
1225  continue;
1226  if (_minsleepuntil == worker->_sleepuntil)
1227  _minsleepuntil = 0;
1228  XXX << "#" << worker->_workerno << ": sleep done, wake up" << endl;
1229  _sleepworkers--;
1230  // nextjob changes the state
1231  worker->nextjob();
1232  }
1233  }
1234 
1235  // collect all curl results, (re)schedule jobs
1236  CURLMsg *msg;
1237  int nqueue;
1238  while ((msg = curl_multi_info_read(_multi, &nqueue)) != 0)
1239  {
1240  if (msg->msg != CURLMSG_DONE)
1241  continue;
1242  CURL *easy = msg->easy_handle;
1243  CURLcode cc = msg->data.result;
1244  multifetchworker *worker;
1245 
1246  if (curl_easy_getinfo(easy, CURLINFO_PRIVATE, &worker) != CURLE_OK)
1247  ZYPP_THROW(MediaCurlException(_baseurl, "curl_easy_getinfo", "unknown error"));
1248 
1249  if (worker->_datareceived && now > worker->_starttime) {
1250  if (worker->_avgspeed)
1251  worker->_avgspeed = (worker->_avgspeed + worker->_datareceived / (now - worker->_starttime)) / 2;
1252  else
1253  worker->_avgspeed = worker->_datareceived / (now - worker->_starttime);
1254  }
1255 
1256  XXX << "#" << worker->_workerno << " done code " << cc << " speed " << worker->_avgspeed << endl;
1257  curl_multi_remove_handle(_multi, easy);
1258 
1259  const auto &setWorkerBroken = [&]( const std::string &str = {} ){
1260  worker->_state = WORKER_BROKEN;
1261  if ( !str.empty () )
1262  strncpy(worker->_curlError, str.c_str(), CURL_ERROR_SIZE);
1263  _activeworkers--;
1264 
1265  if (!_activeworkers && !(urliter != urllist.end() && _workers.size() < MAXURLS)) {
1266  // end of workers reached! goodbye!
1267  worker->evaluateCurlCode(Pathname(), cc, false);
1268  }
1269  };
1270 
1271  if ( !worker->_multiByteHandler ) {
1272  WAR << "#" << worker->_workerno << ": has no multibyte handler, this is a bug" << endl;
1273  setWorkerBroken("Multibyte handler error");
1274  continue;
1275  }
1276 
1277  // tell the worker to finalize the current block
1278  worker->_multiByteHandler->finalize();
1279 
1280  if ( worker->_multiByteHandler->hasMoreWork() && ( cc == CURLE_OK || worker->_multiByteHandler->canRecover() ) ) {
1281 
1282  WAR << "#" << worker->_workerno << ": still has work to do or can recover from a error, continuing the job!" << endl;
1283  // the current job is not done, or we failed and need to try more, enqueue and start again
1284  if ( !worker->continueJob() ) {
1285  WAR << "#" << worker->_workerno << ": failed to continue (" << worker->_multiByteHandler->lastErrorMessage() << ")" << endl;
1286  setWorkerBroken( worker->_multiByteHandler->lastErrorMessage() );
1287  }
1288  continue;
1289  }
1290 
1291  // --- from here on worker has no more ranges in its current job, or had a error it can't recover from ---
1292 
1293  if ( cc != CURLE_OK ) {
1294  if ( worker->_state != WORKER_DISCARD ) {
1295  // something went wrong and we can not recover, broken worker!
1296  setWorkerBroken();
1297  continue;
1298  } else {
1299  WAR << "#" << worker->_workerno << ": failed, but was set to discard, reusing for new requests" << endl;
1300  }
1301  } else {
1302 
1303  // we got what we asked for, maybe. Lets see if all ranges have been marked as finalized
1304  if( !worker->_multiByteHandler->verifyData() ) {
1305  WAR << "#" << worker->_workerno << ": error: " << worker->_multiByteHandler->lastErrorMessage() << ", disable worker" << endl;
1306  setWorkerBroken();
1307  continue;
1308  }
1309 
1310  // from here on we know THIS worker only got data that verified
1311  // now lets see if the stripe was finished too
1312  // stripe blocks can now be only in FINALIZED or ERROR states
1313  if (worker->_state == WORKER_FETCH ) {
1314  if ( worker->_competing ) {
1315  worker->disableCompetition ();
1316  }
1317  auto &wrkerStripe = _requiredStripes[worker->_stripe];
1318  bool done = std::all_of( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), []( const Stripe::RState s ) { return s == Stripe::FINALIZED; } );
1319  if ( !done ) {
1320  // all ranges that are not finalized are in a bogus state, refetch them
1321  std::for_each( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), []( Stripe::RState &s ) {
1322  if ( s != Stripe::FINALIZED)
1323  s = Stripe::PENDING;
1324  });
1325 
1326  _finished = false; //reset finished flag
1327  worker->runjob();
1328  continue;
1329  }
1330  }
1331 
1332  // make bad workers ( bad as in really slow ) sleep a little
1333  double maxavg = 0;
1334  int maxworkerno = 0;
1335  int numbetter = 0;
1336  for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1337  {
1338  multifetchworker *oworker = workeriter->get();
1339  if (oworker->_state == WORKER_BROKEN)
1340  continue;
1341  if (oworker->_avgspeed > maxavg)
1342  {
1343  maxavg = oworker->_avgspeed;
1344  maxworkerno = oworker->_workerno;
1345  }
1346  if (oworker->_avgspeed > worker->_avgspeed)
1347  numbetter++;
1348  }
1349  if (maxavg && !_stealing)
1350  {
1351  double ratio = worker->_avgspeed / maxavg;
1352  ratio = 1 - ratio;
1353  if (numbetter < 3) // don't sleep that much if we're in the top two
1354  ratio = ratio * ratio;
1355  if (ratio > .01)
1356  {
1357  XXX << "#" << worker->_workerno << ": too slow ("<< ratio << ", " << worker->_avgspeed << ", #" << maxworkerno << ": " << maxavg << "), going to sleep for " << ratio * 1000 << " ms" << endl;
1358  worker->_sleepuntil = now + ratio;
1359  worker->_state = WORKER_SLEEP;
1360  _sleepworkers++;
1361  continue;
1362  }
1363  }
1364 
1365  // do rate control (if requested)
1366  // should use periodavg, but that's not what libcurl does
1367  if (_maxspeed && now > _starttime)
1368  {
1369  double avg = _fetchedsize / (now - _starttime);
1370  avg = worker->_maxspeed * _maxspeed / avg;
1371  if (avg < _maxspeed / _maxworkers)
1372  avg = _maxspeed / _maxworkers;
1373  if (avg > _maxspeed)
1374  avg = _maxspeed;
1375  if (avg < 1024)
1376  avg = 1024;
1377  worker->_maxspeed = avg;
1378 #if CURLVERSION_AT_LEAST(7,15,5)
1379  curl_easy_setopt(worker->_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)(avg));
1380 #endif
1381  }
1382 
1383  worker->nextjob();
1384  }
1385 
1386  if ( _filesize > 0 && _fetchedgoodsize > _filesize ) {
1388  }
1389  }
1390 
1391  // send report
1392  if (_report)
1393  {
1394  int percent = _totalsize ? (100 * (_fetchedgoodsize + _fetchedsize)) / (_totalsize + _fetchedsize) : 0;
1395 
1396  double avg = 0;
1397  if (now > _starttime)
1398  avg = _fetchedsize / (now - _starttime);
1399  if (!(*(_report))->progress(percent, _baseurl, avg, _lastperiodstart == _starttime ? avg : _periodavg))
1400  ZYPP_THROW(MediaCurlException(_baseurl, "User abort", "cancelled"));
1401  }
1402 
1403  if (_timeout && now - _lastprogress > _timeout)
1404  break;
1405  }
1406 
1407  if (!_finished)
1409 
1410  // print some download stats
1411  WAR << "overall result" << endl;
1412  for (auto workeriter = _workers.begin(); workeriter != _workers.end(); ++workeriter)
1413  {
1414  multifetchworker *worker = workeriter->get();
1415  WAR << "#" << worker->_workerno << ": state: " << worker->_state << " received: " << worker->_received << " url: " << worker->_url << endl;
1416  }
1417 }
1418 
1419 inline zypp::ByteCount multifetchrequest::makeBlksize ( uint maxConns, size_t filesize )
1420 {
1421  return std::max<zypp::ByteCount>( filesize / std::min( std::max<int>( 1, maxConns ) , MAXURLS ), zypp::ByteCount(4, zypp::ByteCount::K) );
1422 }
1423 
1425 
1426 
1427 MediaMultiCurl::MediaMultiCurl(const Url &url_r, const Pathname & attach_point_hint_r)
1428  : MediaCurl(url_r, attach_point_hint_r)
1429 {
1430  MIL << "MediaMultiCurl::MediaMultiCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
1431  _multi = 0;
1433 }
1434 
1436 {
1438  {
1439  curl_slist_free_all(_customHeadersMetalink);
1441  }
1442  if (_multi)
1443  {
1444  curl_multi_cleanup(_multi);
1445  _multi = 0;
1446  }
1447  std::map<std::string, CURL *>::iterator it;
1448  for (it = _easypool.begin(); it != _easypool.end(); it++)
1449  {
1450  CURL *easy = it->second;
1451  if (easy)
1452  {
1453  curl_easy_cleanup(easy);
1454  it->second = NULL;
1455  }
1456  }
1457 }
1458 
1460 {
1462 
1464  {
1465  curl_slist_free_all(_customHeadersMetalink);
1467  }
1468  struct curl_slist *sl = _customHeaders;
1469  for (; sl; sl = sl->next)
1470  _customHeadersMetalink = curl_slist_append(_customHeadersMetalink, sl->data);
1471  //, application/x-zsync
1472  _customHeadersMetalink = curl_slist_append(_customHeadersMetalink, "Accept: */*, application/x-zsync, application/metalink+xml, application/metalink4+xml");
1473 }
1474 
1475 // here we try to suppress all progress coming from a metalink download
1476 // bsc#1021291: Nevertheless send alive trigger (without stats), so UIs
1477 // are able to abort a hanging metalink download via callback response.
1478 int MediaMultiCurl::progressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
1479 {
1480  CURL *_curl = MediaCurl::progressCallback_getcurl(clientp);
1481  if (!_curl)
1482  return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
1483 
1484  // bsc#408814: Don't report any sizes before we don't have data on disk. Data reported
1485  // due to redirection etc. are not interesting, but may disturb filesize checks.
1486  FILE *fp = 0;
1487  if ( curl_easy_getinfo( _curl, CURLINFO_PRIVATE, &fp ) != CURLE_OK || !fp )
1488  return MediaCurl::aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1489  if ( ftell( fp ) == 0 )
1490  return MediaCurl::aliveCallback( clientp, dltotal, 0.0, ultotal, ulnow );
1491 
1492  // (no longer needed due to the filesize check above?)
1493  // work around curl bug that gives us old data
1494  long httpReturnCode = 0;
1495  if (curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0)
1496  return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
1497 
1498  char *ptr = NULL;
1499  bool ismetalink = false;
1500  if (curl_easy_getinfo(_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1501  {
1502  std::string ct = std::string(ptr);
1503  if (ct.find("application/x-zsync") == 0 || ct.find("application/metalink+xml") == 0 || ct.find("application/metalink4+xml") == 0)
1504  ismetalink = true;
1505  }
1506  if (!ismetalink && dlnow < 256)
1507  {
1508  // can't tell yet, ...
1509  return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
1510  }
1511  if (!ismetalink)
1512  {
1513  fflush(fp);
1514  ismetalink = looks_like_meta_file(fp) != MetaDataType::None;
1515  DBG << "looks_like_meta_file: " << ismetalink << endl;
1516  }
1517  if (ismetalink)
1518  {
1519  // this is a metalink file change the expected filesize
1521  // we're downloading the metalink file. Just trigger aliveCallbacks
1522  curl_easy_setopt(_curl, CURLOPT_XFERINFOFUNCTION, &MediaCurl::aliveCallback);
1523  return MediaCurl::aliveCallback(clientp, dltotal, dlnow, ultotal, ulnow);
1524  }
1525  curl_easy_setopt(_curl, CURLOPT_XFERINFOFUNCTION, &MediaCurl::progressCallback);
1526  return MediaCurl::progressCallback(clientp, dltotal, dlnow, ultotal, ulnow);
1527 }
1528 
1529 void MediaMultiCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1530 {
1531  Pathname dest = target.absolutename();
1532  if( assert_dir( dest.dirname() ) )
1533  {
1534  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1535  ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
1536  }
1537 
1538  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1539  AutoFILE file;
1540  {
1541  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1542  if( ! buf )
1543  {
1544  ERR << "out of memory for temp file name" << endl;
1545  ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
1546  }
1547 
1548  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1549  if( tmp_fd == -1 )
1550  {
1551  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1552  ZYPP_THROW(MediaWriteException(destNew));
1553  }
1554  destNew = ManagedFile( (*buf), filesystem::unlink );
1555 
1556  file = ::fdopen( tmp_fd, "we" );
1557  if ( ! file )
1558  {
1559  ERR << "fopen failed for file '" << destNew << "'" << endl;
1560  ZYPP_THROW(MediaWriteException(destNew));
1561  }
1562  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1563  }
1564 
1565  DBG << "dest: " << dest << endl;
1566  DBG << "temp: " << destNew << endl;
1567 
1568  // set IFMODSINCE time condition (no download if not modified)
1569  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1570  {
1571  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1572  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1573  }
1574  else
1575  {
1576  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1577  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1578  }
1579  // change header to include Accept: metalink
1580  curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _customHeadersMetalink);
1581  // change to our own progress funcion
1582  curl_easy_setopt(_curl, CURLOPT_XFERINFOFUNCTION, &progressCallback);
1583  curl_easy_setopt(_curl, CURLOPT_PRIVATE, (*file) ); // important to pass the FILE* explicitly (passing through varargs)
1584  try
1585  {
1586  MediaCurl::doGetFileCopyFile( srcFile, dest, file, report, options );
1587  }
1588  catch (Exception &ex)
1589  {
1590  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1591  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1592  curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _customHeaders);
1593  curl_easy_setopt(_curl, CURLOPT_PRIVATE, (void *)0);
1594  ZYPP_RETHROW(ex);
1595  }
1596  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1597  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1598  curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _customHeaders);
1599  curl_easy_setopt(_curl, CURLOPT_PRIVATE, (void *)0);
1600  long httpReturnCode = 0;
1601  CURLcode infoRet = curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode);
1602  if (infoRet == CURLE_OK)
1603  {
1604  DBG << "HTTP response: " + str::numstring(httpReturnCode) << endl;
1605  if ( httpReturnCode == 304
1606  || ( httpReturnCode == 213 && _url.getScheme() == "ftp" ) ) // not modified
1607  {
1608  DBG << "not modified: " << PathInfo(dest) << endl;
1609  return;
1610  }
1611  }
1612  else
1613  {
1614  WAR << "Could not get the response code." << endl;
1615  }
1616 
1617  MetaDataType ismetalink = MetaDataType::None;
1618 
1619  char *ptr = NULL;
1620  if (curl_easy_getinfo(_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1621  {
1622  std::string ct = std::string(ptr);
1623  if (ct.find("application/x-zsync") == 0 )
1624  ismetalink = MetaDataType::Zsync;
1625  else if (ct.find("application/metalink+xml") == 0 || ct.find("application/metalink4+xml") == 0)
1626  ismetalink = MetaDataType::MetaLink;
1627  }
1628 
1629  if ( ismetalink == MetaDataType::None )
1630  {
1631  // some proxies do not store the content type, so also look at the file to find
1632  // out if we received a metalink (bnc#649925)
1633  fflush(file);
1634  ismetalink = looks_like_meta_file(destNew);
1635  }
1636 
1637  if ( ismetalink != MetaDataType::None )
1638  {
1639  bool userabort = false;
1640  Pathname failedFile = ZConfig::instance().repoCachePath() / "MultiCurl.failed";
1641  file = nullptr; // explicitly close destNew before the parser reads it.
1642  try
1643  {
1644  MediaBlockList bl;
1645  std::vector<Url> urls;
1646  if ( ismetalink == MetaDataType::Zsync ) {
1647  ZsyncParser parser;
1648  parser.parse( destNew );
1649  bl = parser.getBlockList();
1650  urls = parser.getUrls();
1651 
1652  XXX << getFileUrl(srcFile.filename()) << " returned zsync meta data." << std::endl;
1653  } else {
1654  MetaLinkParser mlp;
1655  mlp.parse(destNew);
1656  bl = mlp.getBlockList();
1657  urls = mlp.getUrls();
1658 
1659  XXX << getFileUrl(srcFile.filename()) << " returned metalink meta data." << std::endl;
1660  }
1661 
1662  if ( bl.numBlocks() )
1663  XXX << "With " << bl.numBlocks() << " nr of blocks and a blocksize of " << bl.getBlock(0).size << std::endl;
1664  else
1665  XXX << "With no blocks" << std::endl;
1666 
1667  /*
1668  * gihub issue libzipp:#277 Multicurl backend breaks with MirrorCache and Metalink with unknown filesize.
1669  * Fall back to a normal download if we have no knowledge about the filesize we want to download.
1670  */
1671  if ( !bl.haveFilesize() && ! srcFile.downloadSize() ) {
1672  XXX << "No filesize in metalink file and no expected filesize, aborting multicurl." << std::endl;
1673  ZYPP_THROW( MediaException("Multicurl requires filesize but none was provided.") );
1674  }
1675 
1676 #if 0
1677  Disabling this workaround for now, since we now do zip ranges into bigger requests
1678  /*
1679  * bsc#1191609 In certain locations we do not receive a suitable number of metalink mirrors, and might even
1680  * download chunks serially from one and the same server. In those cases we need to fall back to a normal download.
1681  */
1682  if ( urls.size() < MIN_REQ_MIRRS ) {
1683  ZYPP_THROW( MediaException("Multicurl enabled but not enough mirrors provided") );
1684  }
1685 #endif
1686 
1687  // XXX << bl << endl;
1688  file = fopen((*destNew).c_str(), "w+e");
1689  if (!file)
1690  ZYPP_THROW(MediaWriteException(destNew));
1691  if (PathInfo(target).isExist())
1692  {
1693  XXX << "reusing blocks from file " << target << endl;
1694  bl.reuseBlocks(file, target.asString());
1695  XXX << bl << endl;
1696  }
1697  if (bl.haveChecksum(1) && PathInfo(failedFile).isExist())
1698  {
1699  XXX << "reusing blocks from file " << failedFile << endl;
1700  bl.reuseBlocks(file, failedFile.asString());
1701  XXX << bl << endl;
1702  filesystem::unlink(failedFile);
1703  }
1704  Pathname df = srcFile.deltafile();
1705  if (!df.empty())
1706  {
1707  XXX << "reusing blocks from file " << df << endl;
1708  bl.reuseBlocks(file, df.asString());
1709  XXX << bl << endl;
1710  }
1711  try
1712  {
1713  multifetch(srcFile.filename(), file, &urls, &report, std::move(bl), srcFile.downloadSize());
1714  }
1715  catch (MediaCurlException &ex)
1716  {
1717  userabort = ex.errstr() == "User abort";
1718  ZYPP_RETHROW(ex);
1719  }
1720  }
1721  catch (MediaFileSizeExceededException &ex) {
1722  ZYPP_RETHROW(ex);
1723  }
1724  catch (Exception &ex)
1725  {
1726  // something went wrong. fall back to normal download
1727  file = nullptr; // explicitly close destNew before moving it
1728  WAR<< "Failed to multifetch file " << ex << " falling back to single Curl download!" << std::endl;
1729  if (PathInfo(destNew).size() >= 63336)
1730  {
1731  ::unlink(failedFile.asString().c_str());
1732  filesystem::hardlinkCopy(destNew, failedFile);
1733  }
1734  if (userabort)
1735  {
1736  ZYPP_RETHROW(ex);
1737  }
1738  file = fopen((*destNew).c_str(), "w+e");
1739  if (!file)
1740  ZYPP_THROW(MediaWriteException(destNew));
1741 
1742  // use the default progressCallback
1743  curl_easy_setopt(_curl, CURLOPT_XFERINFOFUNCTION, &MediaCurl::progressCallback);
1744  MediaCurl::doGetFileCopyFile(srcFile, dest, file, report, options | OPTION_NO_REPORT_START);
1745  }
1746  }
1747 
1748  if (::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 )))
1749  {
1750  ERR << "Failed to chmod file " << destNew << endl;
1751  }
1752 
1753  file.resetDispose(); // we're going to close it manually here
1754  if (::fclose(file))
1755  {
1756  filesystem::unlink(destNew);
1757  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1758  ZYPP_THROW(MediaWriteException(destNew));
1759  }
1760 
1761  if ( rename( destNew, dest ) != 0 )
1762  {
1763  ERR << "Rename failed" << endl;
1765  }
1766  destNew.resetDispose(); // no more need to unlink it
1767 
1768  DBG << "done: " << PathInfo(dest) << endl;
1769 }
1770 
1771 void MediaMultiCurl::multifetch(const Pathname & filename, FILE *fp, std::vector<Url> *urllist, MediaBlockList &&blklist, callback::SendReport<DownloadProgressReport> *report, off_t filesize) const
1772 {
1773  Url baseurl(getFileUrl(filename));
1774  if (filesize == off_t(-1) && blklist.haveFilesize())
1775  filesize = blklist.getFilesize();
1776  if (!blklist.haveBlocks() && filesize != 0) {
1777  if ( filesize == -1 ) {
1778  ZYPP_THROW(MediaException("No filesize and no blocklist, falling back to normal download."));
1779  }
1780 
1781  // build a blocklist on demand just so that we have ranges
1782  MIL << "Generate blocklist, since there was none in the metalink file." << std::endl;
1783 
1784  off_t currOff = 0;
1785  const auto prefSize = multifetchrequest::makeBlksize( _settings.maxConcurrentConnections(), filesize );
1786 
1787  while ( currOff < filesize ) {
1788 
1789  auto blksize = filesize - currOff ;
1790  if ( blksize > prefSize )
1791  blksize = prefSize;
1792 
1793  blklist.addBlock( currOff, blksize );
1794  currOff += blksize;
1795  }
1796 
1797  XXX << "Generated blocklist: " << std::endl << blklist << std::endl << " End blocklist " << std::endl;
1798 
1799  }
1800  if (filesize == 0 || !blklist.numBlocks()) {
1801  checkFileDigest(baseurl, fp, blklist);
1802  return;
1803  }
1804  if (filesize == 0)
1805  return;
1806 
1807  if (!_multi)
1808  {
1809  _multi = curl_multi_init();
1810  if (!_multi)
1812  }
1813 
1814  multifetchrequest req(this, filename, baseurl, _multi, fp, report, std::move(blklist), filesize);
1815  std::vector<Url> myurllist;
1816  for (std::vector<Url>::iterator urliter = urllist->begin(); urliter != urllist->end(); ++urliter)
1817  {
1818  try
1819  {
1820  std::string scheme = urliter->getScheme();
1821  if (scheme == "http" || scheme == "https" || scheme == "ftp" || scheme == "tftp")
1822  {
1823  checkProtocol(*urliter);
1824  myurllist.push_back(internal::propagateQueryParams(*urliter, _url));
1825  }
1826  }
1827  catch (...)
1828  {
1829  }
1830  }
1831  if (!myurllist.size())
1832  myurllist.push_back(baseurl);
1833  req.run(myurllist);
1834  checkFileDigest(baseurl, fp, req.blockList() );
1835 }
1836 
1837 void MediaMultiCurl::checkFileDigest(Url &url, FILE *fp, MediaBlockList &blklist) const
1838 {
1839  if ( !blklist.haveFileChecksum() )
1840  return;
1841  if (fseeko(fp, off_t(0), SEEK_SET))
1842  ZYPP_THROW(MediaCurlException(url, "fseeko", "seek error"));
1843  Digest dig;
1844  blklist.createFileDigest(dig);
1845  char buf[4096];
1846  size_t l;
1847  while ((l = fread(buf, 1, sizeof(buf), fp)) > 0)
1848  dig.update(buf, l);
1849  if (!blklist.verifyFileDigest(dig))
1850  ZYPP_THROW(MediaCurlException(url, "file verification failed", "checksum error"));
1851 }
1852 
1853 bool MediaMultiCurl::isDNSok(const std::string &host) const
1854 {
1855  return _dnsok.find(host) == _dnsok.end() ? false : true;
1856 }
1857 
1858 void MediaMultiCurl::setDNSok(const std::string &host) const
1859 {
1860  _dnsok.insert(host);
1861 }
1862 
1863 CURL *MediaMultiCurl::fromEasyPool(const std::string &host) const
1864 {
1865  if (_easypool.find(host) == _easypool.end())
1866  return 0;
1867  CURL *ret = _easypool[host];
1868  _easypool.erase(host);
1869  return ret;
1870 }
1871 
1872 void MediaMultiCurl::toEasyPool(const std::string &host, CURL *easy) const
1873 {
1874  CURL *oldeasy = _easypool[host];
1875  _easypool[host] = easy;
1876  if (oldeasy)
1877  curl_easy_cleanup(oldeasy);
1878 }
1879 
1880  } // namespace media
1881 } // namespace zypp
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:533
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
MediaBlockList getBlockList() const
return the block list from the parsed metalink data
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:319
#define MIL
Definition: Logger.h:96
constexpr auto MAXURLS
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const override
bool recheckChecksum(off_t blockIdx)
std::vector< Url > getUrls()
return the download urls from the parsed metalink data
Definition: zsyncparser.cc:187
The CurlMultiPartHandler class.
std::set< std::string > _dnsok
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Describes a resource file located on a medium.
void dnsevent(const std::vector< GPollFD > &waitFds)
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:922
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:381
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:31
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
Definition: curlhelper.cc:394
void setPassword(const std::string &val_r)
sets the auth password
Compute Message Digests (MD5, SHA1 etc)
Definition: Digest.h:37
Store and operate with byte count.
Definition: ByteCount.h:30
static Range make(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
void reuseBlocks(FILE *wfp, std::string filename)
void run(std::vector< Url > &urllist)
const std::string & authType() const
get the allowed authentication types
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:173
std::map< std::string, CURL * > _easypool
callback::SendReport< DownloadProgressReport > * _report
String related utilities and Regular expression matching.
void setUsername(const std::string &val_r)
sets the auth username
static double currentTime()
Definition: Arch.h:363
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
#define XXX
Definition: Logger.h:94
static const Unit MB
1000^2 Byte
Definition: ByteCount.h:60
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:406
std::vector< MultiByteHandler::Range > _blocks
void parse(const Pathname &filename)
parse a file consisting of zlink data
Definition: zsyncparser.cc:77
void toEasyPool(const std::string &host, CURL *easy) const
AutoDispose<int> calling ::close
Definition: AutoDispose.h:301
const std::string & password() const
auth password
void parse(const Pathname &filename)
parse a file consisting of metalink xml data
#define ERR
Definition: Logger.h:98
MediaBlockList getBlockList()
return the block list from the parsed metalink data
Definition: zsyncparser.cc:197
const std::string & username() const
auth username
std::optional< KeyManagerCtx > _context
Definition: KeyRing.cc:157
static ByteCount makeBlksize(uint maxConns, size_t filesize)
multifetchrequest(const MediaMultiCurl *context, const Pathname &filename, const Url &baseurl, CURLM *multi, FILE *fp, callback::SendReport< DownloadProgressReport > *report, MediaBlockList &&blklist, off_t filesize)
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1404
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:440
const MediaMultiCurl * _context
virtual void setupEasy() override
initializes the curl easy handle with the data from the url
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:497
bool verifyFileDigest(Digest &digest) const
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:369
void setAuthType(const std::string &val_r)
set the allowed authentication types
CURL * fromEasyPool(const std::string &host) const
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:700
multifetchrequest * _request
const Url _url
Url to handle.
Definition: MediaHandler.h:113
bool isDNSok(const std::string &host) const
const std::string & asString() const
String representation.
Definition: Pathname.h:91
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:742
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:281
std::vector< Stripe > _requiredStripes
Just inherits Exception to separate media exceptions.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:833
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Pathname repoCachePath() const
Path where the caches are kept (/var/cache/zypp)
Definition: ZConfig.cc:1039
std::string getChecksumType() const
const ByteCount & downloadSize() const
The size of the resource on the server.
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
do not send a start ProgressReport
Definition: MediaCurl.h:45
#define WAR
Definition: Logger.h:97
MetaDataType looks_like_meta_file(const Pathname &file)
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
Definition: PathInfo.cc:883
a single block from the blocklist, consisting of an offset and a size
bool createFileDigest(Digest &digest) const
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1361
void setDNSok(const std::string &host) const
size_t numBlocks() const
return the number of blocks in the blocklist
void multifetch(const Pathname &filename, FILE *fp, std::vector< Url > *urllist, MediaBlockList &&blklist, callback::SendReport< DownloadProgressReport > *report=0, off_t filesize=off_t(-1)) const
const Pathname & filename() const
The path to the resource on the medium.
std::string numstring(char n, int w=0)
Definition: String.h:289
std::string asString(unsigned field_width_r=0, unsigned unit_width_r=1) const
Auto selected Unit and precision.
Definition: ByteCount.h:133
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:180
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1198
curl_slist * _customHeaders
Definition: MediaCurl.h:168
MultiByteHandler::Range rangeFromBlock(off_t blockNo) const
const Pathname & deltafile() const
The existing deltafile that can be used to reduce download size ( zchunk or metalink ) ...
MediaMultiCurl(const Url &url_r, const Pathname &attach_point_hint_r)
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:139
UByteArray getChecksum(size_t blkno) const
Base class for Exception.
Definition: Exception.h:145
Url url() const
Url used.
Definition: MediaHandler.h:503
std::unique_ptr< MultiByteHandler > _multiByteHandler
bool any_of(const Container &c, Fnc &&cb)
Definition: Algorithm.h:76
const MediaBlock & getBlock(size_t blkno) const
return the offset/size of a block with number blkno
long maxConcurrentConnections() const
Maximum number of concurrent connections for a single transfer.
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:588
curl_slist * _customHeadersMetalink
virtual void disconnectFrom() override
Definition: MediaCurl.cc:705
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1377
MultiFetchWorkerState _state
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
bool haveChecksum(size_t blkno) const
std::list< std::unique_ptr< multifetchworker > > _workers
static const Unit K
1024 Byte
Definition: ByteCount.h:45
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
constexpr auto MIN_REQ_MIRRS
std::vector< off_t > _rangeToStripeBlock
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:312
AutoDispose< void * > _state
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:789
std::vector< Url > getUrls() const
return the download urls from the parsed metalink data
MultiByteHandler::ProtocolMode _protocolMode
std::string userPassword() const
returns the user and password as a user:pass string
Digest clone() const
Returns a clone of the current Digest and returns it.
Definition: Digest.cc:228
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:167
static bool env_isset(std::string name)
const std::string & proxy() const
proxy host
bool update(const char *bytes, size_t len)
feed data into digest computation algorithm
Definition: Digest.cc:309
std::vector< off_t > blocks
static int aliveCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1347
void adddnsfd(std::vector< GPollFD > &waitFds)
Url manipulation class.
Definition: Url.h:91
#define DBG
Definition: Logger.h:95
ByteCount df(const Pathname &path_r)
Report free disk space on a mounted file system.
Definition: PathInfo.cc:1155
void checkFileDigest(Url &url, FILE *fp, MediaBlockList &blklist) const
std::vector< RState > blockStates