libzypp  17.35.14
downloader.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 ----------------------------------------------------------------------*/
13 #include <utility>
14 #include <zypp-curl/TransferSettings>
16 #include <zypp-media/MediaException>
17 #include <zypp-core/base/String.h>
18 
19 namespace zyppng {
20 
21  DownloadPrivateBase::DownloadPrivateBase(Downloader &parent, std::shared_ptr<NetworkRequestDispatcher> requestDispatcher, std::shared_ptr<MirrorControl> mirrors, DownloadSpec &&spec, Download &p)
22  : BasePrivate(p)
23  , _requestDispatcher ( std::move(requestDispatcher) )
24  , _mirrorControl( std::move(mirrors) )
25  , _spec( std::move(spec) )
26  , _parent( &parent )
27  {}
28 
30  { }
31 
32  bool DownloadPrivateBase::handleRequestAuthError( const std::shared_ptr<Request>& req, const zyppng::NetworkRequestError &err )
33  {
34  //Handle the auth errors explicitly, we need to give the user a way to put in new credentials
35  //if we get valid new credentials we can retry the request
36  bool retry = false;
38 
39  MIL << "Authentication failed for " << req->url() << " trying to recover." << std::endl;
40 
41  TransferSettings &ts = req->transferSettings();
42  const auto &applyCredToSettings = [&ts]( const AuthData_Ptr& auth, const std::string &authHint ) {
43  ts.setUsername( auth->username() );
44  ts.setPassword( auth->password() );
45  auto nwCred = dynamic_cast<NetworkAuthData *>( auth.get() );
46  if ( nwCred ) {
47  // set available authentication types from the error
48  if ( nwCred->authType() == CURLAUTH_NONE )
49  nwCred->setAuthType( authHint );
50 
51  // set auth type (seems this must be set _after_ setting the userpwd)
52  if ( nwCred->authType() != CURLAUTH_NONE ) {
53  // FIXME: only overwrite if not empty?
54  ts.setAuthType(nwCred->authTypeAsString());
55  }
56  }
57  };
58 
59  // try to find one in the cache
61  vopt = vopt
62  - zypp::url::ViewOption::WITH_USERNAME
63  - zypp::url::ViewOption::WITH_PASSWORD
64  - zypp::url::ViewOption::WITH_QUERY_STR;
65 
66  auto cachedCred = zypp::media::CredentialManager::findIn( _credCache, req->url(), vopt );
67 
68  // only consider a cache entry if its newer than what we tried last time
69  if ( cachedCred && cachedCred->lastDatabaseUpdate() > req->_authTimestamp ) {
70  MIL << "Found a credential match in the cache!" << std::endl;
71  applyCredToSettings( cachedCred, "" );
72  _lastTriedAuthTime = req->_authTimestamp = cachedCred->lastDatabaseUpdate();
73  retry = true;
74  } else {
75 
77  credFromUser->setUrl( req->url() );
78  credFromUser->setLastDatabaseUpdate ( req->_authTimestamp );
79 
80  //in case we got a auth hint from the server the error object will contain it
81  std::string authHint = err.extraInfoValue("authHint", std::string());
82 
83  _sigAuthRequired.emit( *z_func(), *credFromUser, authHint );
84  if ( credFromUser->valid() ) {
85  // remember for next time , we don't want to ask the user again for the same URL set
86  _credCache.insert( credFromUser );
87  applyCredToSettings( credFromUser, authHint );
88  _lastTriedAuthTime = req->_authTimestamp = credFromUser->lastDatabaseUpdate();
89  retry = true;
90  }
91  }
92  }
93  return retry;
94  }
95 
96 #if ENABLE_ZCHUNK_COMPRESSION
97  bool DownloadPrivateBase::hasZckInfo() const
98  {
99  if ( zypp::indeterminate(_specHasZckInfo) )
100  _specHasZckInfo = ( _spec.headerSize() > 0 && isZchunkFile( _spec.deltaFile() ) );
101  return bool(_specHasZckInfo);
102  }
103 #endif
104 
106  {
107  _sigStartedConn.disconnect();
108  _sigProgressConn.disconnect();
109  _sigFinishedConn.disconnect();
110  }
111 
112  DownloadPrivate::DownloadPrivate(Downloader &parent, std::shared_ptr<NetworkRequestDispatcher> requestDispatcher, std::shared_ptr<MirrorControl> mirrors, DownloadSpec &&spec, Download &p)
113  : DownloadPrivateBase( parent, std::move(requestDispatcher), std::move(mirrors), std::move(spec), p )
114  { }
115 
117  {
119  DownloadPrivateBase::_sigFinished.emit( *z_func() );
120  } );
121 
123  DownloadPrivateBase::_sigStateChanged.emit( *z_func(), state );
124  } );
125  }
126 
128  {
129  auto cState = currentState();
130  if ( !cState )
132 
133  cState = currentState();
134  if ( *cState != Download::InitialState && *cState != Download::Finished ) {
135  // the state machine has advaned already, we can only restart it in a finished state
136  return;
137  }
138 
139  //reset state variables
140  _specHasZckInfo = zypp::indeterminate;
141  _emittedSigStart = false;
142  _stoppedOnMetalink = false;
143  _lastTriedAuthTime = 0;
144 
145  // restart the statemachine
146  if ( cState == Download::Finished )
148 
149  //jumpstart the process
150  state<InitialState>()->initiate();
151  }
152 
153 
155  {
156  auto buildExtraInfo = [this, &url](){
157  std::map<std::string, boost::any> extraInfo;
158  extraInfo.insert( {"requestUrl", url } );
159  extraInfo.insert( {"filepath", _spec.targetPath() } );
160  return extraInfo;
161  };
162 
164  try {
166  if ( _spec.settings().proxy().empty() )
167  ::internal::fillSettingsSystemProxy( url, set );
168 
169 #if 0
170  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
171  * We should proactively add the password to the request if basic auth is configured
172  * and a password is available in the credentials but not in the URL.
173  *
174  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
175  * and ask the server first about the auth method
176  */
177  if ( set.authType() == "basic"
178  && set.username().size()
179  && !set.password().size() ) {
181  const auto cred = cm.getCred( url );
182  if ( cred && cred->valid() ) {
183  if ( !set.username().size() )
184  set.setUsername(cred->username());
185  set.setPassword(cred->password());
186  }
187  }
188 #endif
189 
190  } catch ( const zypp::media::MediaBadUrlException & e ) {
192  } catch ( const zypp::media::MediaUnauthorizedException & e ) {
194  } catch ( const zypp::Exception & e ) {
196  }
197  return res;
198  }
199 
200  Download::Download(zyppng::Downloader &parent, std::shared_ptr<zyppng::NetworkRequestDispatcher> requestDispatcher, std::shared_ptr<zyppng::MirrorControl> mirrors, zyppng::DownloadSpec &&spec)
201  : Base( *new DownloadPrivate( parent, std::move(requestDispatcher), std::move(mirrors), std::move(spec), *this ) )
202  { }
203 
205 
207  {
208  if ( state() != InitialState && state() != Finished )
209  cancel();
210  }
211 
213  {
214  const auto &s = d_func()->currentState();
215  if ( !s )
216  return Download::InitialState;
217  return *s;
218  }
219 
221  {
222  if ( state() == Finished ) {
223  return d_func()->state<FinishedState>()->_error;
224  }
225  return NetworkRequestError();
226  }
227 
228  bool Download::hasError() const
229  {
230  return lastRequestError().isError();
231  }
232 
233  std::string Download::errorString() const
234  {
235  const auto &lReq = lastRequestError();
236  if (! lReq.isError() ) {
237  return {};
238  }
239 
240  return ( zypp::str::Format("%1%(%2%)") % lReq.toString() % lReq.nativeErrorString() );
241  }
242 
244  {
245  d_func()->start();
246  }
247 
249  {
250  Z_D();
251 
252  if ( !d->_requestDispatcher )
253  return;
254 
255  d->_defaultSubRequestPriority = NetworkRequest::Critical;
256 
257  // we only reschedule requests when we are in a state that downloads in blocks
258  d->visitState( []( auto &s ){
259  using T = std::decay_t<decltype (s)>;
260  if constexpr ( std::is_same_v<T, DlMetalinkState>
261 #if ENABLE_ZCHUNK_COMPRESSION
262  || std::is_same_v<T, DLZckState>
263 #endif
264  ) {
265  s.reschedule();
266  }
267  });
268  }
269 
271  {
272  Z_D();
273  d->forceState ( std::make_unique<FinishedState>( NetworkRequestErrorPrivate::customError( NetworkRequestError::Cancelled, "Download was cancelled explicitly" ), *d_func() ) );
274  }
275 
276  void Download::setStopOnMetalink(const bool set)
277  {
278  d_func()->_stopOnMetalink = set;
279  }
280 
282  {
283  return d_func()->_stoppedOnMetalink;
284  }
285 
287  {
288  return d_func()->_spec;
289  }
290 
292  {
293  return d_func()->_spec;
294  }
295 
297  {
298  return d_func()->_lastTriedAuthTime;
299  }
300 
301  zyppng::NetworkRequestDispatcher &Download::dispatcher() const
302  {
303  return *d_func()->_requestDispatcher;
304  }
305 
307  {
308  return d_func()->_sigStarted;
309  }
310 
312  {
313  return d_func()->DownloadPrivateBase::_sigStateChanged;
314  }
315 
317  {
318  return d_func()->_sigAlive;
319  }
320 
322  {
323  return d_func()->_sigProgress;
324  }
325 
327  {
328  return d_func()->DownloadPrivateBase::_sigFinished;
329  }
330 
332  {
333  return d_func()->_sigAuthRequired;
334  }
335 
336  DownloaderPrivate::DownloaderPrivate(std::shared_ptr<MirrorControl> mc, Downloader &p)
337  : BasePrivate(p)
338  , _mirrors( std::move(mc) )
339  {
340  _requestDispatcher = std::make_shared<NetworkRequestDispatcher>( );
341  if ( !_mirrors ) {
343  }
344  }
345 
347  {
348  _sigStarted.emit( *z_func(), download );
349  }
350 
352  {
353  _sigFinished.emit( *z_func(), download );
354 
355  auto it = std::find_if( _runningDownloads.begin(), _runningDownloads.end(), [ &download ]( const std::shared_ptr<Download> &dl){
356  return dl.get() == &download;
357  });
358 
359  if ( it != _runningDownloads.end() ) {
360  //make sure this is not deleted before all user code was done
361  _runningDownloads.erase( it );
362  }
363 
364  if ( _runningDownloads.empty() )
365  _queueEmpty.emit( *z_func() );
366  }
367 
369 
371  : Base ( *new DownloaderPrivate( {}, *this ) )
372  {
373 
374  }
375 
376  Downloader::Downloader( std::shared_ptr<MirrorControl> mc )
377  : Base ( *new DownloaderPrivate( std::move(mc), *this ) )
378  { }
379 
381  {
382  Z_D();
383  while ( d->_runningDownloads.size() ) {
384  d->_runningDownloads.back()->cancel();
385  d->_runningDownloads.pop_back();
386  }
387  }
388 
389  std::shared_ptr<Download> Downloader::downloadFile(const zyppng::DownloadSpec &spec )
390  {
391  Z_D();
392  std::shared_ptr<Download> dl ( new Download ( *this, d->_requestDispatcher, d->_mirrors, DownloadSpec(spec) ) );
393 
394  d->_runningDownloads.push_back( dl );
396  d->_requestDispatcher->run();
397 
398  return dl;
399  }
400 
401  std::shared_ptr<NetworkRequestDispatcher> Downloader::requestDispatcher() const
402  {
403  return d_func()->_requestDispatcher;
404  }
405 
407  {
408  return d_func()->_sigStarted;
409  }
410 
412  {
413  return d_func()->_sigFinished;
414  }
415 
417  {
418  return d_func()->_queueEmpty;
419  }
420 
421 }
bool isError() const
isError Will return true if this is a actual error
#define MIL
Definition: Logger.h:100
zypp::media::AuthData_Ptr AuthData_Ptr
Definition: authdata.h:22
SignalProxy< void(Download &req, off_t dlnow)> sigAlive()
Definition: downloader.cc:316
The Downloader class.
Definition: downloader.h:38
DownloaderPrivate(std::shared_ptr< MirrorControl > mc, Downloader &p)
Definition: downloader.cc:336
Downloader * _parent
Definition: base_p.h:101
void setPassword(const std::string &val_r)
sets the auth password
void init() override
Definition: downloader.cc:116
DownloadSpec _spec
Definition: base_p.h:98
void setAuthType(std::string auth_type)
Set HTTP authentication type(s) to use.
Definition: curlauthdata.h:55
bool handleRequestAuthError(const std::shared_ptr< Request > &req, const zyppng::NetworkRequestError &err)
Definition: downloader.cc:32
Holds transfer setting.
void setStopOnMetalink(const bool set=true)
Definition: downloader.cc:276
SignalProxy< void(Download &req, NetworkAuthData &auth, const std::string &availAuth)> sigAuthRequired()
Definition: downloader.cc:331
std::shared_ptr< NetworkRequestDispatcher > _requestDispatcher
Definition: downloader_p.h:104
DownloadPrivateBase(Downloader &parent, std::shared_ptr< NetworkRequestDispatcher > requestDispatcher, std::shared_ptr< MirrorControl > mirrors, DownloadSpec &&spec, Download &p)
Definition: downloader.cc:21
void setUsername(const std::string &val_r)
sets the auth username
zypp::TriBool _specHasZckInfo
Definition: base_p.h:99
Definition: Arch.h:363
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
Signal< void(Download &req, Download::State state)> _sigStateChanged
Definition: base_p.h:109
std::vector< std::shared_ptr< Download > > _runningDownloads
Definition: downloader_p.h:103
uint64_t lastAuthTimestamp() const
Definition: downloader.cc:296
Convenient building of std::string with boost::format.
Definition: String.h:252
SignalProxy< void(Downloader &parent, Download &download)> sigStarted()
Definition: downloader.cc:406
SignalProxy< void(Download &req)> sigFinished()
Definition: downloader.cc:326
Url::asString() view options.
Definition: UrlBase.h:39
zypp::filesystem::Pathname deltaFile() const
Definition: downloadspec.cc:98
Signal< void(Downloader &parent, Download &download)> _sigFinished
Definition: downloader_p.h:110
#define Z_D()
Definition: zyppglobal.h:105
void onDownloadFinished(Download &download)
Definition: downloader.cc:351
SignalProxy< void(Downloader &parent, Download &download)> sigFinished()
Definition: downloader.cc:411
Signal< void(Downloader &parent)> _queueEmpty
Definition: downloader_p.h:111
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition: plaindir.cc:88
zypp::repo::RepoException _error
std::shared_ptr< T > state()
Definition: statemachine.h:434
State state() const
Definition: downloader.cc:212
std::shared_ptr< MirrorControl > _mirrors
Definition: downloader_p.h:112
void setAuthType(const std::string &val_r)
set the allowed authentication types
T extraInfoValue(const std::string &key, T &&defaultVal=T()) const
std::string asString() const
Error message provided by dumpOn as string.
Definition: Exception.cc:111
Download(Downloader &parent, std::shared_ptr< NetworkRequestDispatcher > requestDispatcher, std::shared_ptr< MirrorControl > mirrors, DownloadSpec &&spec)
Definition: downloader.cc:200
bool stoppedOnMetalink() const
Definition: downloader.cc:281
The NetworkRequestError class Represents a error that occured in.
std::shared_ptr< NetworkRequestDispatcher > requestDispatcher() const
Definition: downloader.cc:401
zypp::media::CurlAuthData_Ptr NetworkAuthData_Ptr
Definition: authdata.h:25
Base::WeakPtr parent
Definition: base_p.h:22
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:183
std::shared_ptr< Download > downloadFile(const DownloadSpec &spec)
Definition: downloader.cc:389
NetworkRequestDispatcher & dispatcher() const
Definition: downloader.cc:301
const zypp::Pathname & targetPath() const
Definition: downloadspec.cc:59
NetworkRequestError lastRequestError() const
Definition: downloader.cc:220
zypp::media::CurlAuthData NetworkAuthData
Definition: authdata.h:24
SignalProxy< void(Downloader &parent)> queueEmpty()
Definition: downloader.cc:416
NetworkRequestError safeFillSettingsFromURL(const Url &url, TransferSettings &set)
Definition: downloader.cc:154
DownloadPrivate(Downloader &parent, std::shared_ptr< NetworkRequestDispatcher > requestDispatcher, std::shared_ptr< MirrorControl > mirrors, DownloadSpec &&spec, Download &p)
Definition: downloader.cc:112
DownloadSpec & spec()
Definition: downloader.cc:286
SignalProxy< void(Download &req)> sigStarted()
Definition: downloader.cc:306
const TransferSettings & settings() const
Base class for Exception.
Definition: Exception.h:146
static auto connectFunc(typename internal::MemberFunction< SenderFunc >::ClassType &s, SenderFunc &&sFun, ReceiverFunc &&rFunc, const Tracker &...trackers)
Definition: base.h:163
~Downloader() override
Definition: downloader.cc:380
void onDownloadStarted(Download &download)
Definition: downloader.cc:346
ZYPP_IMPL_PRIVATE(UnixSignalSource)
Signal< void(zyppng::Download &req, zyppng::NetworkAuthData &auth, const std::string &availAuth)> _sigAuthRequired
Definition: base_p.h:113
SignalProxy< void(Download &req, off_t dltotal, off_t dlnow)> sigProgress()
Definition: downloader.cc:321
std::optional< StateId > currentState() const
Definition: statemachine.h:410
static AuthData_Ptr findIn(const CredentialManager::CredentialSet &set, const Url &url, url::ViewOption vopt)
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:331
This defines the actual StateMachine.
Definition: statemachine.h:364
typename decay< T >::type decay_t
Definition: TypeTraits.h:42
Type type() const
type Returns the type of the error
Curl HTTP authentication data.
Definition: curlauthdata.h:22
std::string errorString() const
Definition: downloader.cc:233
zypp::ByteCount headerSize() const
SignalProxy< void(Download &req, State state)> sigStateChanged()
Definition: downloader.cc:311
const std::string & proxy() const
proxy host
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
Signal< void(Download &req)> _sigFinished
Definition: base_p.h:112
Signal< void(Downloader &parent, Download &download)> _sigStarted
Definition: downloader_p.h:109
Url manipulation class.
Definition: Url.h:91
bool hasError() const
Definition: downloader.cc:228
zypp::media::CredentialManager::CredentialSet _credCache
Definition: base_p.h:96