libzypp  17.36.3
susetags.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
9 #include "susetags.h"
10 #include "zypp-core/base/Regex.h"
11 #include <zypp-core/zyppng/ui/ProgressObserver>
12 #include <zypp-media/ng/ProvideSpec>
13 #include <zypp/ng/Context>
14 
15 #include <zypp-core/parser/ParseException>
16 
19 
24 
25 #undef ZYPP_BASE_LOGGER_LOGGROUP
26 #define ZYPP_BASE_LOGGER_LOGGROUP "zypp::repomanager"
27 
29 
30  namespace {
31 
32  using namespace zyppng::operators;
33 
39  template<class Executor, class OpType>
40  struct StatusLogic : public LogicBase<Executor, OpType>{
41  ZYPP_ENABLE_LOGIC_BASE(Executor, OpType);
42 
43  public:
44 
45  using DlContextRefType = std::conditional_t<zyppng::detail::is_async_op_v<OpType>, repo::AsyncDownloadContextRef, repo::SyncDownloadContextRef>;
46  using ZyppContextType = typename DlContextRefType::element_type::ContextType;
47  using ProvideType = typename ZyppContextType::ProvideType;
48  using MediaHandle = typename ProvideType::MediaHandle;
49  using ProvideRes = typename ProvideType::Res;
50 
51  StatusLogic( DlContextRefType ctx, MediaHandle &&media )
52  : _ctx(std::move(ctx))
53  , _handle(std::move(media))
54  {}
55 
56  MaybeAsyncRef<expected<zypp::RepoStatus>> execute() {
57  return _ctx->zyppContext()->provider()->provide( _handle, _ctx->repoInfo().path() / "content" , ProvideFileSpec() )
58  | [this]( expected<ProvideRes> contentFile ) {
59 
60  // mandatory master index is missing -> stay empty
61  if ( !contentFile )
63 
64  zypp::RepoStatus status ( contentFile->file() );
65 
66  if ( !status.empty() /* && _ctx->repoInfo().requireStatusWithMediaFile() */ ) {
67  return _ctx->zyppContext()->provider()->provide( _handle, "/media.1/media" , ProvideFileSpec())
68  | [status = std::move(status)]( expected<ProvideRes> mediaFile ) mutable {
69  if ( mediaFile ) {
70  return make_expected_success(status && zypp::RepoStatus( mediaFile->file()) );
71  }
72  return make_expected_success( std::move(status) );
73  };
74  }
75  return makeReadyResult( make_expected_success(std::move(status)) );
76  };
77  }
78 
79  DlContextRefType _ctx;
80  MediaHandle _handle;
81  };
82  }
83 
84  AsyncOpRef<expected<zypp::RepoStatus> > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
85  {
86  return SimpleExecutor< StatusLogic, AsyncOp<expected<zypp::RepoStatus>> >::run( std::move(dl), std::move(mediaHandle) );
87  }
88 
89  expected<zypp::RepoStatus> repoStatus(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle)
90  {
91  return SimpleExecutor< StatusLogic, SyncOp<expected<zypp::RepoStatus>> >::run( std::move(dl), std::move(mediaHandle) );
92  }
93 
94 
95 
96  namespace {
97 
98  using namespace zyppng::operators;
99 
100  // search old repository file file to run the delta algorithm on
101  static zypp::Pathname search_deltafile( const zypp::Pathname &dir, const zypp::Pathname &file )
102  {
103  zypp::Pathname deltafile(dir + file.basename());
105  return deltafile;
106  return zypp::Pathname();
107  }
108 
109 
110  template<class Executor, class OpType>
111  struct DlLogic : public LogicBase<Executor, OpType> {
112 
113  ZYPP_ENABLE_LOGIC_BASE(Executor, OpType);
114  public:
115 
116  using DlContextRefType = std::conditional_t<zyppng::detail::is_async_op_v<OpType>, repo::AsyncDownloadContextRef, repo::SyncDownloadContextRef>;
117  using ZyppContextType = typename DlContextRefType::element_type::ContextType;
118  using ProvideType = typename ZyppContextType::ProvideType;
119  using MediaHandle = typename ProvideType::MediaHandle;
120  using ProvideRes = typename ProvideType::Res;
121 
122  DlLogic( DlContextRefType ctx, MediaHandle &&mediaHandle, ProgressObserverRef &&progressObserver )
123  : _ctx( std::move(ctx))
124  , _mediaHandle(std::move(mediaHandle))
125  , _progressObserver(std::move(progressObserver))
126  {}
127 
128  auto execute() {
129  // download media file here
131  | [this]( expected<zypp::ManagedFile> &&mediaInfo ) {
132 
133  // remember the media info if we had one
134  if ( mediaInfo ) _ctx->files().push_back ( std::move(mediaInfo.get()) );
135 
136  if ( _progressObserver ) _progressObserver->inc();
137 
138  return RepoDownloaderWorkflow::downloadMasterIndex( _ctx, _mediaHandle, _ctx->repoInfo().path() / "content" )
140  | and_then( [this] ( DlContextRefType && ) {
141 
142  zypp::Pathname contentPath = _ctx->files().front();
143  std::vector<zypp::OnMediaLocation> requiredFiles;
144 
145  // Content file first to get the repoindex
146  try {
147  const zypp::Pathname &inputfile = contentPath;
149  content.setRepoIndexConsumer( [this]( const auto & data_r ) {
150  MIL << "Consuming repo index" << std::endl;
151  _repoindex = data_r;
152  });
153  content.parse( inputfile );
154 
155  if ( ! _repoindex ) {
156  ZYPP_THROW( zypp::parser::ParseException( ( _ctx->destDir() / _ctx->repoInfo().path() ).asString() + ": " + "No repository index in content file." ) );
157  }
158 
159  MIL << "RepoIndex: " << _repoindex << std::endl;
160  if ( _repoindex->metaFileChecksums.empty() ) {
161  ZYPP_THROW( zypp::parser::ParseException( ( _ctx->destDir() / _ctx->repoInfo().path() ).asString() + ": " + "No metadata checksums in content file." ) );
162  }
163 
164  if ( _repoindex->signingKeys.empty() ) {
165  WAR << "No signing keys defined." << std::endl;
166  }
167 
168  // Prepare parsing
169  zypp::Pathname descr_dir = _repoindex->descrdir; // path below reporoot
170  //_datadir = _repoIndex->datadir; // path below reporoot
171 
172  std::map<std::string,zypp::parser::susetags::RepoIndex::FileChecksumMap::const_iterator> availablePackageTranslations;
173 
174  for_( it, _repoindex->metaFileChecksums.begin(), _repoindex->metaFileChecksums.end() )
175  {
176  // omit unwanted translations
177  if ( zypp::str::hasPrefix( it->first, "packages" ) )
178  {
179  static const zypp::str::regex rx_packages( "^packages((.gz)?|(.([^.]*))(.gz)?)$" );
180  zypp::str::smatch what;
181 
182  if ( zypp::str::regex_match( it->first, what, rx_packages ) ) {
183  if ( what[4].empty() // packages(.gz)?
184  || what[4] == "DU"
185  || what[4] == "en" )
186  { ; /* always downloaded */ }
187  else if ( what[4] == "FL" )
188  { continue; /* never downloaded */ }
189  else
190  {
191  // remember and decide later
192  availablePackageTranslations[what[4]] = it;
193  continue;
194  }
195  }
196  else
197  continue; // discard
198 
199  } else if ( it->first == "patterns.pat"
200  || it->first == "patterns.pat.gz" ) {
201  // take all patterns in one go
202 
203  } else if ( zypp::str::endsWith( it->first, ".pat" )
204  || zypp::str::endsWith( it->first, ".pat.gz" ) ) {
205 
206  // *** see also zypp/parser/susetags/RepoParser.cc ***
207 
208  // omit unwanted patterns, see https://bugzilla.novell.com/show_bug.cgi?id=298716
209  // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
210  // split at dots, take .pat or .pat.gz into account
211 
212  std::vector<std::string> patparts;
213  unsigned archpos = 2;
214  // expect "<name>.<arch>.pat[.gz]", <name> might contain additional dots
215  unsigned count = zypp::str::split( it->first, std::back_inserter(patparts), "." );
216  if ( patparts[count-1] == "gz" )
217  archpos++;
218 
219  if ( count > archpos ) {
220  try { // might by an invalid architecture
221  zypp::Arch patarch( patparts[count-archpos] );
222  if ( !patarch.compatibleWith( zConfig().systemArchitecture() ) ) {
223  // discard, if not compatible
224  MIL << "Discarding pattern " << it->first << std::endl;
225  continue;
226  }
227 
228  } catch ( const zypp::Exception & excpt ) {
229  WAR << "Pattern file name does not contain recognizable architecture: " << it->first << std::endl;
230  // keep .pat file if it doesn't contain an recognizable arch
231  }
232  }
233  }
234 
235  MIL << "adding job " << it->first << std::endl;
236  auto location = zypp::OnMediaLocation( repoInfo().path() + descr_dir + it->first, 1 )
237  .setChecksum( it->second )
238  .setDeltafile( search_deltafile( _ctx->deltaDir() + descr_dir, it->first) );
239 
240  requiredFiles.push_back( std::move(location) );
241  }
242 
243 
244  // check whether to download more package translations:
245  {
246  auto fnc_checkTransaltions( [&]( const zypp::Locale & locale_r ) {
247  for ( zypp::Locale toGet( locale_r ); toGet; toGet = toGet.fallback() ) {
248  auto it( availablePackageTranslations.find( toGet.code() ) );
249  if ( it != availablePackageTranslations.end() ) {
250  auto mit( it->second );
251  MIL << "adding job " << mit->first << std::endl;
252  requiredFiles.push_back( zypp::OnMediaLocation( repoInfo().path() + descr_dir + mit->first, 1 )
253  .setChecksum( mit->second )
254  .setDeltafile( search_deltafile( deltaDir() + descr_dir, mit->first) ));
255  break;
256  }
257  }
258  });
259 
260  for ( const zypp::Locale & it : zConfig().repoRefreshLocales() ) {
261  fnc_checkTransaltions( it );
262  }
263  fnc_checkTransaltions( zConfig().textLocale() );
264  }
265 
266  for( const auto &it : _repoindex->mediaFileChecksums ) {
267  // Repo adopts license files listed in HASH
268  if ( it.first != "license.tar.gz" )
269  continue;
270 
271  MIL << "adding job " << it.first << std::endl;
272  requiredFiles.push_back( zypp::OnMediaLocation ( repoInfo().path() + it.first, 1 )
273  .setChecksum( it.second )
274  .setDeltafile( search_deltafile( deltaDir(), it.first ) ));
275  }
276 
277  for( const auto &it : _repoindex->signingKeys ) {
278  MIL << "adding job " << it.first << std::endl;
279  zypp::OnMediaLocation location( repoInfo().path() + it.first, 1 );
280  location.setChecksum( it.second );
281  requiredFiles.push_back( std::move(location) );
282  }
283 
284  } catch ( const zypp::Exception &e ) {
285  ZYPP_CAUGHT( e );
287  } catch ( ... ) {
289  }
290 
291  // add the required files to the base steps
292  if ( _progressObserver ) _progressObserver->setBaseSteps ( _progressObserver->baseSteps () + requiredFiles.size() );
293 
294  return transform_collect ( std::move(requiredFiles), [this]( zypp::OnMediaLocation file ) {
295 
296  return DownloadWorkflow::provideToCacheDir( _ctx, _mediaHandle, file.filename(), ProvideFileSpec(file) )
298 
299  }) | and_then ( [this]( std::vector<zypp::ManagedFile> &&dlFiles ) {
300  auto &downloadedFiles = _ctx->files();
301  downloadedFiles.insert( downloadedFiles.end(), std::make_move_iterator(dlFiles.begin()), std::make_move_iterator(dlFiles.end()) );
302  return expected<DlContextRefType>::success( std::move(_ctx) );
303  });
304  });
305 
307  }
308 
309  private:
310 
311  const zypp::RepoInfo &repoInfo() const {
312  return _ctx->repoInfo();
313  }
314 
315  const zypp::filesystem::Pathname &deltaDir() const {
316  return _ctx->deltaDir();
317  }
318 
319  zypp::ZConfig &zConfig() {
320  return _ctx->zyppContext()->config();
321  }
322 
323  DlContextRefType _ctx;
324  zypp::parser::susetags::RepoIndex_Ptr _repoindex;
325  MediaHandle _mediaHandle;
326  ProgressObserverRef _progressObserver;
327  };
328  }
329 
330  AsyncOpRef<expected<repo::AsyncDownloadContextRef> > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
331  {
332  return SimpleExecutor< DlLogic, AsyncOp<expected<repo::AsyncDownloadContextRef>> >::run( std::move(dl), std::move(mediaHandle), std::move(progressObserver) );
333  }
334 
335  expected<repo::SyncDownloadContextRef> download(repo::SyncDownloadContextRef dl, SyncMediaHandle mediaHandle, ProgressObserverRef progressObserver)
336  {
337  return SimpleExecutor< DlLogic, SyncOp<expected<repo::SyncDownloadContextRef>> >::run( std::move(dl), std::move(mediaHandle), std::move(progressObserver) );
338  }
339 
340 }
#define MIL
Definition: Logger.h:100
Pathname deltafile
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:424
Describes a resource file located on a medium.
Regular expression.
Definition: Regex.h:94
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > download(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, ProgressObserverRef progressObserver)
Definition: susetags.cc:330
expected< T, E > inspect(expected< T, E > exp, Function &&f)
Definition: expected.h:531
Architecture.
Definition: Arch.h:36
auto finishProgress(ProgressObserverRef progressObserver, ProgressObserver::FinishResult result=ProgressObserver::Success)
A ProvideRes object is a reference counted ownership of a resource in the cache provided by a Provide...
Definition: provideres.h:35
Locale fallback() const
Return the fallback locale for this locale, if no fallback exists the empty Locale::noCode.
Definition: Locale.cc:208
#define for_(IT, BEG, END)
Convenient for-loops using iterator.
Definition: Easy.h:27
OnMediaLocation & setChecksum(CheckSum val_r)
Set the checksum.
Definition: Arch.h:363
What is known about a repository.
Definition: RepoInfo.h:71
static expected< std::decay_t< Type >, Err > make_expected_success(Type &&t)
Definition: expected.h:397
ProgressObserverRef _progressObserver
Definition: susetags.cc:326
DlContextRefType _ctx
Definition: susetags.cc:79
std::string basename() const
Return the last component of this path.
Definition: Pathname.h:130
MediaHandle _handle
Definition: susetags.cc:80
std::string asString(TInt val, char zero='0', char one='1')
For printing bits.
Definition: Bit.h:57
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
auto transform_collect(Container< Msg, CArgs... > &&in, Transformation &&f)
Definition: expected.h:666
bool isExist() const
Return whether valid stat info exists.
Definition: PathInfo.h:286
AsyncOpRef< expected< zypp::ManagedFile > > provideToCacheDir(AsyncCacheProviderContextRef cacheContext, ProvideMediaHandle medium, zypp::Pathname file, ProvideFileSpec filespec)
Definition: downloadwf.cc:209
Interim helper class to collect global options and settings.
Definition: ZConfig.h:68
#define WAR
Definition: Logger.h:101
Parse repoindex part from a content file.
#define ZYPP_ENABLE_LOGIC_BASE(Executor, OpType)
Definition: logichelpers.h:223
AsyncOpRef< expected< repo::AsyncDownloadContextRef > > downloadMasterIndex(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle, zypp::filesystem::Pathname masterIndex_r)
void setRepoIndexConsumer(const RepoIndexConsumer &fnc_r)
Consumer to call when repo index was parsed.
typename conditional< B, T, F >::type conditional_t
Definition: TypeTraits.h:39
std::conditional_t< isAsync, AsyncOpRef< T >, T > makeReadyResult(T &&result)
Definition: asyncop.h:297
const Pathname & filename() const
The path to the resource on the medium.
bool endsWith(const C_Str &str_r, const C_Str &prefix_r)
alias for hasSuffix
Definition: String.h:1092
static expected success(ConsParams &&...params)
Definition: expected.h:115
OnMediaLocation & setDeltafile(Pathname path)
Set the deltafile.
&#39;Language[_Country]&#39; codes.
Definition: Locale.h:50
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:440
std::shared_ptr< AsyncOp< T > > AsyncOpRef
Definition: asyncop.h:255
Regular expression match result.
Definition: Regex.h:167
zypp::parser::susetags::RepoIndex_Ptr _repoindex
Definition: susetags.cc:324
Base class for Exception.
Definition: Exception.h:146
AsyncOpRef< expected< zypp::RepoStatus > > repoStatus(repo::AsyncDownloadContextRef dl, ProvideMediaHandle mediaHandle)
Definition: susetags.cc:84
MediaHandle _mediaHandle
Definition: susetags.cc:325
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:225
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
regex ZYPP_STR_REGEX regex ZYPP_STR_REGEX
Definition: Regex.h:70
ResultType and_then(const expected< T, E > &exp, Function &&f)
Definition: expected.h:423
Track changing files or directories.
Definition: RepoStatus.h:40
Download workflow namespace for SUSETags (YaST2) repositories Encapsulates all the knowledge of which...
Definition: susetags.cc:28
#define ZYPP_FWD_CURRENT_EXCPT()
Drops a logline and returns the current Exception as a std::exception_ptr.
Definition: Exception.h:436
auto downloadMediaInfo(MediaHandle &&mediaHandle, const zypp::filesystem::Pathname &destdir)
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1027
virtual void parse(const InputStream &imput_r, const ProgressData::ReceiverFnc &fnc_r=ProgressData::ReceiverFnc())
Parse the stream.
auto incProgress(ProgressObserverRef progressObserver, double progrIncrease=1.0, std::optional< std::string > newStr={})