32 #undef ZYPP_BASE_LOGGER_LOGGROUP 33 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp:fetcher" 69 if ( lhs->location.medianr() == rhs->location.medianr() )
70 return lhs->location.filename() < rhs->location.filename();
72 return lhs->location.medianr() < rhs->location.medianr();
87 RecursiveDirectory = Directory | Recursive,
90 AlwaysVerifyChecksum = 0x0004,
120 return str << obj->location;
137 void setOptions( Fetcher::Options options );
138 Fetcher::Options options()
const;
147 void addCachePath(
const Pathname &cache_dir );
149 void start(
const Pathname &dest_dir,
156 static shared_ptr<Impl> _nullimpl(
new Impl );
183 void readContentFileIndex(
const Pathname &index,
const Pathname &basedir );
205 const Pathname &dest_dir, FetcherJob::Flags flags );
220 friend Impl * rwcowClone<Impl>(
const Impl * rhs );
223 {
return new Impl( *
this ); }
226 std::set<FetcherIndex_Ptr,SameFetcherIndex>
_indexes;
240 job.reset(
new FetcherJob(resource, deltafile));
241 job->flags |= FetcherJob:: AlwaysVerifyChecksum;
242 _resources.push_back(job);
245 Fetcher::Impl::Impl()
263 job->checkers.push_back(checker);
268 _resources.push_back(job);
278 job->checkers.push_back(checker);
284 _resources.push_back(job);
293 job->checkers.push_back(checker);
294 _resources.push_back(job);
299 MIL <<
"adding index " << resource << endl;
319 DBG <<
"Adding fetcher cache: '" << cache_dir <<
"'." << endl;
320 _caches.insert(cache_dir);
325 ERR <<
"Not adding cache: '" << cache_dir <<
"'. Not a directory." << endl;
330 ERR <<
"Not adding cache '" << cache_dir <<
"'. Path does not exists." << endl;
343 if (
PathInfo(dest_full_path).isExist() )
350 MIL <<
"start fetcher with " << _caches.size() <<
" cache directories." << endl;
351 for_ ( it_cache, _caches.begin(), _caches.end() )
355 if (
PathInfo( cached_file ).isExist() )
357 DBG <<
"File '" << cached_file <<
"' exist, testing checksum " << resource.
checksum() << endl;
362 MIL <<
"file " << resource.
filename() <<
" found in previous cache. Using cached copy." << endl;
366 if( dest_full_path != cached_file )
373 ERR <<
"Can't hardlink/copy " << cached_file +
" to " + dest_dir << endl;
392 MIL <<
"Checking job [" << localfile <<
"] (" << checkers.size() <<
" checkers )" << endl;
394 for ( list<FileChecker>::const_iterator it = checkers.begin();
395 it != checkers.end();
404 ERR <<
"Invalid checker for '" << localfile <<
"'" << endl;
428 auto fnc_addIfInContent( [&](
const std::string & index_r ) ->
bool 437 downloadAndReadIndexList( media, dest_dir );
443 fnc_addIfInContent(
"CHECKSUMS" ) || fnc_addIfInContent(
"SHA1SUMS" );
447 fnc_addIfInContent(
"content" );
456 != _dircontent.end() )
460 std::copy(filled.begin(), filled.end(), std::back_inserter(content));
469 std::copy(tofill.begin(), tofill.end(), std::back_inserter(content));
476 const Pathname &dest_dir, FetcherJob::Flags flags )
480 MIL <<
"Adding directory " << resource.
filename() << endl;
483 getDirectoryContent(media, resource, content);
488 WAR <<
"Skiping subtree hidden at " << resource.
filename() << endl;
494 autoaddIndexes(content, media, resource, dest_dir);
496 for ( filesystem::DirContent::const_iterator it = content.begin();
512 if ( _checksums.find(filename.
asString()) != _checksums.end() )
515 chksm = _checksums[filename.
asString()];
519 WAR <<
"Resource " << filename <<
" has no checksum in the index either." << endl;
529 addDirJobs(media, filename, dest_dir, flags);
540 bool got_from_cache =
false;
543 got_from_cache = provideFromCache(resource, dest_dir);
545 if ( ! got_from_cache )
547 MIL <<
"Not found in cache, downloading" << endl;
560 if ( !
PathInfo(tmp_file).isExist() )
561 ERR << tmp_file <<
" does not exist" << endl;
563 ERR << dest_full_path.
dirname() <<
" does not exist" << endl;
576 WAR <<
"optional resource " << resource <<
" could not be transferred" << endl;
603 { _repoindex = data_r; }
612 readChecksumsIndex(index, basedir);
613 else if ( index.
basename() ==
"content" )
614 readContentFileIndex(index, basedir);
616 WAR << index <<
": index file format not known" << endl;
624 MIL << index <<
" contains " << reader.
_repoindex->mediaFileChecksums.size() <<
" checksums." << endl;
628 _checksums[(basedir + it->first).
asString()] = it->second;
635 std::ifstream in( index.
c_str() );
639 while (
getline( in, buffer ) )
642 if ( buffer[0] ==
'#' )
649 if ( buffer.empty() )
651 WAR <<
"Missing filename in CHECKSUMS file: " << index.
asString() <<
" (" <<
checksum <<
")" << endl;
664 MIL <<
"downloading index " << resource << endl;
690 fetcher.
start( dest_dir, media );
699 fetcher.
start( dest_dir, media );
706 WAR <<
"No public key specified by user for index '" << keyloc.
filename() <<
"'"<< endl;
710 fetcher.
start( dest_dir, media );
721 if ( _indexes.empty() )
723 MIL <<
"No indexes to read." << endl;
727 for_( it_idx, _indexes.begin(), _indexes.end() )
729 if ( (*it_idx)->read )
731 DBG <<
"Already read index " <<
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
736 downloadIndex( media, (*it_idx)->location, dest_dir );
738 readIndex( dest_dir + (*it_idx)->location.filename(), (*it_idx)->location.filename().
dirname() );
740 MIL <<
"Remember read index " <<
PathInfo(dest_dir + (*it_idx)->location.filename()) << endl;
741 (*it_idx)->read =
true;
744 MIL <<
"done reading indexes" << endl;
754 progress.
sendTo(progress_receiver);
756 downloadAndReadIndexList(media, dest_dir);
758 for ( list<FetcherJob_Ptr>::const_iterator it_res = _resources.begin(); it_res != _resources.end(); ++it_res )
764 addDirJobs(media, location, dest_dir, (*it_res)->flags);
778 getDirectoryContent(media, (*it_res)->location.filename().dirname(), content);
781 MIL <<
"Autodiscovering signed indexes on '" 782 << (*it_res)->location.filename().dirname() <<
"' for '" 783 << (*it_res)->location.filename() <<
"'" << endl;
785 autoaddIndexes(content, media, (*it_res)->location.filename().dirname(), dest_dir);
789 getDirectoryContent(media,
Pathname(
"/"), content);
792 MIL <<
"Autodiscovering signed indexes on '" 794 << (*it_res)->location.filename() <<
"'" << endl;
796 autoaddIndexes(content, media,
Pathname(
"/"), dest_dir);
799 provideToDest(media, (*it_res)->location, dest_dir, (*it_res)->deltafile);
803 if ( !
PathInfo(dest_dir + (*it_res)->location.filename()).isExist() )
808 if ( (*it_res)->location.checksum().empty() )
810 if ( _checksums.find((*it_res)->location.filename().asString())
811 != _checksums.end() )
813 CheckSum chksm = _checksums[(*it_res)->location.filename().asString()];
815 (*it_res)->checkers.push_back(digest_check);
825 (*it_res)->checkers.push_back(digest_check);
833 (*it_res)->checkers.push_back(digest_check);
837 validate((*it_res)->location, dest_dir, (*it_res)->checkers);
839 if ( ! progress.incr() )
847 for ( list<FetcherJob_Ptr>::const_iterator it_res = obj.
_resources.begin(); it_res != obj.
_resources.end(); ++it_res )
bool provideFromCache(const OnMediaLocation &resource, const Pathname &dest_dir)
tries to provide the file represented by job into dest_dir by looking at the cache.
int assert_dir(const Pathname &path, unsigned mode)
Like 'mkdir -p'.
void consumeIndex(const parser::susetags::RepoIndex_Ptr &data_r)
void readContentFileIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for content file
shared_ptr< FetcherIndex > FetcherIndex_Ptr
void setOptions(Options options)
Set the Fetcher options.
Listentry returned by readdir.
std::string asString(const DefaultIntegral< Tp, TInitial > &obj)
void sendTo(const ReceiverFnc &fnc_r)
Set ReceiverFnc.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Checks for the validity of a signature.
void addIndex(const OnMediaLocation &resource)
Adds an index containing metadata (for example checksums ) that will be retrieved and read before the...
parser::susetags::RepoIndex_Ptr _repoindex
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
list< FetcherJob_Ptr > _resources
const char * c_str() const
String representation.
String related utilities and Regular expression matching.
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
void addCachePath(const Pathname &cache_dir)
adds a directory to the list of directories where to look for cached files
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
If a content file is found, it is downloaded and read.
std::string basename() const
Return the last component of this path.
void enqueueDigestedDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory and always check for checksums.
void addCachePath(const Pathname &cache_dir)
OnMediaLocation location
Index localtion.
function< bool(const ProgressData &)> ReceiverFnc
Most simple version of progress reporting The percentage in most cases.
void provideToDest(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, const Pathname &deltafile)
Provide the resource to dest_dir.
static shared_ptr< Impl > nullimpl()
Offer default Impl.
RWCOW_pointer< Impl > _pimpl
Pointer to implementation.
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
Enqueue a object for transferal, they will not be transferred until start() is called.
void remember(const Exception &old_r)
Store an other Exception as history.
void validate(const OnMediaLocation &resource, const Pathname &dest_dir, const list< FileChecker > &checkers)
Validates the job against is checkers, by using the file instance on dest_dir.
std::set< FetcherIndex_Ptr, SameFetcherIndex > _indexes
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
std::string getline(std::istream &str)
Read one line from stream.
std::string stripFirstWord(std::string &line, const bool ltrim_first)
Fetcher::Options _options
std::ostream & operator<<(std::ostream &str, const Exception &obj)
const std::string & asString() const
String representation.
bool isExist() const
Return whether valid stat info exists.
void reset()
Reset the transfer (jobs) list.
Pathname dirname() const
Return all but the last component od this path.
list< FileChecker > checkers
void readIndex(const Pathname &index, const Pathname &basedir)
reads a downloaded index file and updates internal attributes table
Parse repoindex part from a content file.
std::set ordering (less semantic)
Fetcher::Options options() const
Maintain [min,max] and counter (value) for progress counting.
void enqueueDir(const OnMediaLocation &resource, bool recursive, const FileChecker &checker=FileChecker())
std::list< DirEntry > DirContent
Returned by readdir.
DefaultIntegral< bool, false > read
Whether we read this index.
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
void downloadIndex(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
download the indexes and reads them
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress=ProgressData::ReceiverFnc())
start the transfer to a destination directory dest_dir You have to provde a media set access media to...
void enqueueDigested(const OnMediaLocation &resource, const FileChecker &checker=FileChecker(), const Pathname &deltafile=Pathname())
void enqueueDir(const OnMediaLocation &resource, bool recursive=false, const FileChecker &checker=FileChecker())
Enqueue a directory.
std::ostream & operator<<(std::ostream &str, const zypp::shared_ptr< void > &obj)
std::ostream & copy(std::istream &from_r, std::ostream &to_r)
Copy istream to ostream.
map< string, filesystem::DirContent > _dircontent
void getDirectoryContent(MediaSetAccess &media, const OnMediaLocation &resource, filesystem::DirContent &content)
reads the content of a directory but keeps a cache
If a CHECKSUMS file is found, it is downloaded and read.
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Class to encapsulate the OnMediaLocation object and the FileChecker together.
void setOptions(Fetcher::Options options)
FetcherIndex(const OnMediaLocation &loc)
Class representing one GPG Public Key (PublicKeyData + ASCII armored in a tempfile).
void readChecksumsIndex(const Pathname &index, const Pathname &basedir)
specific version of readIndex for CHECKSUMS file
void downloadAndReadIndexList(MediaSetAccess &media, const Pathname &dest_dir)
download the indexes and reads them
#define ZYPP_DECLARE_OPERATORS_FOR_FLAGS(Name)
Base class for Exception.
#define ZYPP_DECLARE_FLAGS(Name, Enum)
FetcherJob(const OnMediaLocation &loc, const Pathname dfile=Pathname())
std::string checksum(const Pathname &file, const std::string &algorithm)
Compute a files checksum.
Wrapper class for ::stat/::lstat.
class that represents indexes which add metadata to fetcher jobs and therefore need to be retrieved i...
std::set< Pathname > _caches
void start(const Pathname &dest_dir, MediaSetAccess &media, const ProgressData::ReceiverFnc &progress_receiver)
function< void(const Pathname &file)> FileChecker
Functor signature used to check files.
std::ostream & operator<<(std::ostream &str, const Fetcher::Impl &obj)
Options options() const
Get current options.
shared_ptr< FetcherJob > FetcherJob_Ptr
void enqueue(const OnMediaLocation &resource, const FileChecker &checker=FileChecker())
Enqueue a object for transferal, they will not be transferred until start() is called.
bool operator()(const FetcherIndex_Ptr &lhs, const FetcherIndex_Ptr &rhs)
Easy-to use interface to the ZYPP dependency resolver.
void autoaddIndexes(const filesystem::DirContent &content, MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir)
auto discovery and reading of indexes
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
map< string, CheckSum > _checksums
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
This class allows to retrieve a group of files in a confortable way, providing some smartness that do...
void addIndex(const OnMediaLocation &resource)
bool is_checksum(const Pathname &file, const CheckSum &checksum)
check files checksum
void addDirJobs(MediaSetAccess &media, const OnMediaLocation &resource, const Pathname &dest_dir, FetcherJob::Flags flags)
scan the directory and adds the individual jobs
Impl * clone() const
clone for RWCOW_pointer