1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 parsing of manager configuration files
24 """
25
26 import warnings
27
28 from flumotion.common import log, errors, common, registry
29 from flumotion.common import config as fluconfig
30 from flumotion.common.xmlwriter import cmpComponentType, XMLWriter
31 from flumotion.configure import configure
32
33 __version__ = "$Rev: 7669 $"
34
35
38
39
41
42 def parseFeedId(feedId):
43 if feedId.find(':') == -1:
44 return "%s:default" % feedId
45 else:
46 return feedId
47
48 eaterConfig = conf.get('eater', {})
49 sourceConfig = conf.get('source', [])
50 if eaterConfig == {} and sourceConfig != []:
51 eaters = registry.getRegistry().getComponent(
52 conf.get('type')).getEaters()
53 eatersDict = {}
54 eatersTuple = [(None, parseFeedId(s)) for s in sourceConfig]
55 eatersDict = buildEatersDict(eatersTuple, eaters)
56 conf['eater'] = eatersDict
57
58 if sourceConfig:
59 sources = []
60 for s in sourceConfig:
61 sources.append(parseFeedId(s))
62 conf['source'] = sources
63
64
66 eaters = dict(conf.get('eater', {}))
67 concat = lambda lists: reduce(list.__add__, lists, [])
68 if not reduce(lambda x, y: y and isinstance(x, tuple),
69 concat(eaters.values()),
70 True):
71 for eater in eaters:
72 aliases = []
73 feeders = eaters[eater]
74 for i in range(len(feeders)):
75 val = feeders[i]
76 if isinstance(val, tuple):
77 feedId, alias = val
78 aliases.append(val[1])
79 else:
80 feedId = val
81 alias = eater
82 while alias in aliases:
83 log.warning('config', "Duplicate alias %s for "
84 "eater %s, uniquifying", alias, eater)
85 alias += '-bis'
86 aliases.append(alias)
87 feeders[i] = (feedId, val)
88 conf['eater'] = eaters
89
90 UPGRADERS = [upgradeEaters, upgradeAliases]
91 CURRENT_VERSION = len(UPGRADERS)
92
93
95 """Build a eaters dict suitable for forming part of a component
96 config.
97
98 @param eatersList: List of eaters. For example,
99 [('default', 'othercomp:feeder', 'foo')] says
100 that our eater 'default' will be fed by the feed
101 identified by the feedId 'othercomp:feeder', and
102 that it has the alias 'foo'. Alias is optional.
103 @type eatersList: List of (eaterName, feedId, eaterAlias?)
104 @param eaterDefs: The set of allowed and required eaters
105 @type eaterDefs: List of
106 L{flumotion.common.registry.RegistryEntryEater}
107 @returns: Dict of eaterName => [(feedId, eaterAlias)]
108 """
109
110 def parseEaterTuple(tup):
111
112 def parse(eaterName, feedId, eaterAlias=None):
113 if eaterAlias is None:
114 eaterAlias = eaterName
115 return (eaterName, feedId, eaterAlias)
116 return parse(*tup)
117
118 eaters = {}
119 for eater, feedId, alias in [parseEaterTuple(t) for t in eatersList]:
120 if eater is None:
121 if not eaterDefs:
122 raise errors.ConfigError(
123 "Feed %r cannot be connected, component has no eaters" %
124 (feedId, ))
125
126 eater = eaterDefs[0].getName()
127 if alias is None:
128 alias = eater
129 feeders = eaters.get(eater, [])
130 if feedId in feeders:
131 raise errors.ConfigError(
132 "Already have a feedId %s eating from %s" %
133 (feedId, eater))
134 while alias in [a for f, a in feeders]:
135 log.debug('config', "Duplicate alias %s for eater %s, "
136 "uniquifying", alias, eater)
137 alias += '-bis'
138
139 feeders.append((feedId, alias))
140 eaters[eater] = feeders
141 for e in eaterDefs:
142 eater = e.getName()
143 if e.getRequired() and not eater in eaters:
144 raise errors.ConfigError("Component wants to eat on %s,"
145 " but no feeders specified."
146 % (e.getName(), ))
147 if not e.getMultiple() and len(eaters.get(eater, [])) > 1:
148 raise errors.ConfigError("Component does not support multiple "
149 "sources feeding %s (%r)"
150 % (eater, eaters[eater]))
151 aliases = reduce(list.__add__,
152 [[x[1] for x in tups] for tups in eaters.values()],
153 [])
154
155
156 while aliases:
157 alias = aliases.pop()
158 if alias in aliases:
159 raise errors.ConfigError("Duplicate alias: %s" % (alias, ))
160
161 return eaters
162
163
165 """Build a virtual feeds dict suitable for forming part of a
166 component config.
167
168 @param feedPairs: List of virtual feeds, as name-feederName pairs. For
169 example, [('bar:baz', 'qux')] defines one
170 virtual feed 'bar:baz', which is provided by
171 the component's 'qux' feed.
172 @type feedPairs: List of (feedId, feedName) -- both strings.
173 @param feeders: The feeders exported by this component, from the
174 registry.
175 @type feeders: List of str.
176 """
177 ret = {}
178 for virtual, real in feedPairs:
179 if real not in feeders:
180 raise errors.ConfigError('virtual feed maps to unknown feeder: '
181 '%s -> %s' % (virtual, real))
182 try:
183 common.parseFeedId(virtual)
184 except:
185 raise errors.ConfigError('virtual feed name not a valid feedId: %s'
186 % (virtual, ))
187 ret[virtual] = real
188 return ret
189
190
191 -def dictDiff(old, new, onlyOld=None, onlyNew=None, diff=None,
192 keyBase=None):
193 """Compute the difference between two config dicts.
194
195 @returns: 3 tuple: (onlyOld, onlyNew, diff) where:
196 onlyOld is a list of (key, value), representing key-value
197 pairs that are only in old;
198 onlyNew is a list of (key, value), representing key-value
199 pairs that are only in new;
200 diff is a list of (key, oldValue, newValue), representing
201 keys with different values in old and new; and
202 key is a tuple of strings representing the recursive key
203 to get to a value. For example, ('foo', 'bar') represents
204 the value d['foo']['bar'] on a dict d.
205 """
206
207
208 if onlyOld is None:
209 onlyOld = []
210 onlyNew = []
211 diff = []
212 keyBase = ()
213
214 for k in old:
215 key = (keyBase + (k, ))
216 if k not in new:
217 onlyOld.append((key, old[k]))
218 elif old[k] != new[k]:
219 if isinstance(old[k], dict) and isinstance(new[k], dict):
220 dictDiff(old[k], new[k], onlyOld, onlyNew, diff, key)
221 else:
222 diff.append((key, old[k], new[k]))
223
224 for k in new:
225 key = (keyBase + (k, ))
226 if k not in old:
227 onlyNew.append((key, new[k]))
228
229 return onlyOld, onlyNew, diff
230
231
234
235 def ref(label, k):
236 return "%s%s: '%s'" % (label,
237 ''.join(["[%r]" % (subk, )
238 for subk in k[:-1]]),
239 k[-1])
240
241 out = []
242 for k, v in old:
243 out.append('Only in %s = %r' % (ref(oldLabel, k), v))
244 for k, v in new:
245 out.append('Only in %s = %r' % (ref(newLabel, k), v))
246 for k, oldv, newv in diff:
247 out.append('Value mismatch:')
248 out.append(' %s = %r' % (ref(oldLabel, k), oldv))
249 out.append(' %s = %r' % (ref(newLabel, k), newv))
250 return '\n'.join(out)
251
252
253 -class ConfigEntryComponent(log.Loggable):
254 "I represent a <component> entry in a planet config file"
255 nice = 0
256 logCategory = 'config'
257
258 __pychecker__ = 'maxargs=13'
259
260 - def __init__(self, name, parent, type, label, propertyList, plugList,
261 worker, eatersList, isClockMaster, project, version,
262 virtualFeeds=None):
263 self.name = name
264 self.parent = parent
265 self.type = type
266 self.label = label
267 self.worker = worker
268 self.defs = registry.getRegistry().getComponent(self.type)
269 try:
270 self.config = self._buildConfig(propertyList, plugList,
271 eatersList, isClockMaster,
272 project, version,
273 virtualFeeds)
274 except errors.ConfigError, e:
275
276 e.args = ("While parsing component %s: %s"
277 % (name, log.getExceptionMessage(e)), )
278 raise
279
280 - def _buildVersionTuple(self, version):
281 if version is None:
282 return configure.versionTuple
283 elif isinstance(version, tuple):
284 assert len(version) == 4
285 return version
286 elif isinstance(version, str):
287 try:
288
289 def parse(maj, min, mic, nan=0):
290 return maj, min, mic, nan
291 return parse(*map(int, version.split('.')))
292 except:
293 raise errors.ConfigError("<component> version not parseable")
294 raise errors.ConfigError("<component> version not parseable")
295
296 - def _buildConfig(self, propertyList, plugsList, eatersList,
297 isClockMaster, project, version, virtualFeeds):
298 """
299 Build a component configuration dictionary.
300 """
301
302
303
304 config = {'name': self.name,
305 'parent': self.parent,
306 'type': self.type,
307 'config-version': CURRENT_VERSION,
308 'avatarId': common.componentId(self.parent, self.name),
309 'project': project or configure.PACKAGE,
310 'version': self._buildVersionTuple(version),
311 'clock-master': isClockMaster or None,
312 'feed': self.defs.getFeeders(),
313 'properties': fluconfig.buildPropertyDict(propertyList,
314 self.defs.getProperties()),
315 'plugs': fluconfig.buildPlugsSet(plugsList,
316 self.defs.getSockets()),
317 'eater': buildEatersDict(eatersList,
318 self.defs.getEaters()),
319 'source': [tup[1] for tup in eatersList],
320 'virtual-feeds': buildVirtualFeeds(virtualFeeds or [],
321 self.defs.getFeeders())}
322
323 if self.label:
324
325 config['label'] = self.label
326
327 if not config['source']:
328
329 del config['source']
330
331
332 return config
333
336
337 - def getLabel(self):
339
342
343 - def getParent(self):
345
346 - def getConfigDict(self):
348
349 - def getWorker(self):
351
352
354 "I represent a <flow> entry in a planet config file"
355
356 - def __init__(self, name, components):
357 self.name = name
358 self.components = {}
359 for c in components:
360 if c.name in self.components:
361 raise errors.ConfigError(
362 'flow %s already has component named %s' % (name, c.name))
363 self.components[c.name] = c
364
365
367 "I represent a <manager> entry in a planet config file"
368
369 - def __init__(self, name, host, port, transport, certificate, bouncer,
370 fludebug, plugs):
371 self.name = name
372 self.host = host
373 self.port = port
374 self.transport = transport
375 self.certificate = certificate
376 self.bouncer = bouncer
377 self.fludebug = fludebug
378 self.plugs = plugs
379
380
382 "I represent a <atmosphere> entry in a planet config file"
383
384 - def __init__(self):
386
388 return len(self.components)
389
390
392 """
393 This is a base class for parsing planet configuration files (both manager
394 and flow files).
395 """
396 logCategory = 'config'
397
399 if feedId.find(':') == -1:
400 return "%s:default" % feedId
401 else:
402 return feedId
403
410
411 - def parseComponent(self, node, parent, isFeedComponent,
412 needsWorker):
413 """
414 Parse a <component></component> block.
415
416 @rtype: L{ConfigEntryComponent}
417 """
418
419
420
421
422
423
424
425
426
427
428
429
430 attrs = self.parseAttributes(node, ('name', 'type'),
431 ('label', 'worker', 'project', 'version', ))
432 name, componentType, label, worker, project, version = attrs
433 if needsWorker and not worker:
434 raise errors.ConfigError(
435 'component %s does not specify the worker '
436 'that it is to run on' % (name, ))
437 elif worker and not needsWorker:
438 raise errors.ConfigError('component %s specifies a worker to run '
439 'on, but does not need a worker' % (name, ))
440
441 properties = []
442 plugs = []
443 eaters = []
444 clockmasters = []
445 sources = []
446 virtual_feeds = []
447
448 def parseBool(node):
449 return self.parseTextNode(node, common.strToBool)
450 parsers = {'property': (self._parseProperty, properties.append),
451 'compound-property': (self._parseCompoundProperty,
452 properties.append),
453 'plugs': (self.parsePlugs, plugs.extend)}
454
455 if isFeedComponent:
456 parsers.update({'eater': (self._parseEater, eaters.extend),
457 'clock-master': (parseBool, clockmasters.append),
458 'source': (self._parseSource, sources.append),
459 'virtual-feed': (self._parseVirtualFeed,
460 virtual_feeds.append)})
461
462 self.parseFromTable(node, parsers)
463
464 if len(clockmasters) == 0:
465 isClockMaster = None
466 elif len(clockmasters) == 1:
467 isClockMaster = clockmasters[0]
468 else:
469 raise errors.ConfigError("Only one <clock-master> node allowed")
470
471 if sources:
472 msg = ('"source" tag has been deprecated in favor of "eater",'
473 ' please update your configuration file (found in'
474 ' component %r)' % name)
475 warnings.warn(msg, DeprecationWarning)
476
477 for feedId in sources:
478
479 eaters.append((None, feedId))
480
481 return ConfigEntryComponent(name, parent, componentType, label,
482 properties, plugs, worker, eaters,
483 isClockMaster, project, version,
484 virtual_feeds)
485
488
493
495
496
497
498 name, = self.parseAttributes(node, ('name', ))
499 feeds = []
500 parsers = {'feed': (self._parseFeed, feeds.append)}
501 self.parseFromTable(node, parsers)
502 if len(feeds) == 0:
503
504 raise errors.ConfigError(
505 "Eater node %s with no <feed> nodes, is not allowed" % (
506 name, ))
507 return [(name, feedId, alias) for feedId, alias in feeds]
508
509
511 """
512 I represent a planet configuration file for Flumotion.
513
514 @ivar atmosphere: A L{ConfigEntryAtmosphere}, filled in when parse() is
515 called.
516 @ivar flows: A list of L{ConfigEntryFlow}, filled in when parse() is
517 called.
518 """
519 logCategory = 'config'
520
526
546
548
549
550
551
552 ret = {}
553
554 def parseComponent(node):
555 return self.parseComponent(node, 'atmosphere', False, True)
556
557 def gotComponent(comp):
558 ret[comp.name] = comp
559 parsers = {'component': (parseComponent, gotComponent)}
560 self.parseFromTable(node, parsers)
561 return ret
562
579 parsers = {'component': (parseComponent, components.append)}
580 self.parseFromTable(node, parsers)
581
582
583
584
585 masters = [x for x in components if x.config['clock-master']]
586 if len(masters) > 1:
587 raise errors.ConfigError("Multiple clock masters in flow %s: %s"
588 % (name, ', '.join([m.name for m in masters])))
589
590 need_sync = [(x.defs.getClockPriority(), x) for x in components
591 if x.defs.getNeedsSynchronization()]
592 need_sync.sort()
593 need_sync = [x[1] for x in need_sync]
594
595 if need_sync:
596 if masters:
597 master = masters[0]
598 else:
599 master = need_sync[-1]
600
601 masterAvatarId = master.config['avatarId']
602 self.info("Setting %s as clock master" % masterAvatarId)
603
604 for c in need_sync:
605 c.config['clock-master'] = masterAvatarId
606 elif masters:
607 self.info('master clock specified, but no synchronization '
608 'necessary -- ignoring')
609 masters[0].config['clock-master'] = None
610
611 return ConfigEntryFlow(name, components)
612
613
614
616 """
617 Get all component entries from both atmosphere and all flows
618 from the configuration.
619
620 @rtype: dictionary of /parent/name -> L{ConfigEntryComponent}
621 """
622 entries = {}
623 if self.atmosphere and self.atmosphere.components:
624 for c in self.atmosphere.components.values():
625 path = common.componentId('atmosphere', c.name)
626 entries[path] = c
627
628 for flowEntry in self.flows:
629 for c in flowEntry.components.values():
630 path = common.componentId(c.parent, c.name)
631 entries[path] = c
632
633 return entries
634
635
636
637
638
639
641 """
642 I parse manager configuration out of a planet configuration file.
643
644 @ivar manager: A L{ConfigEntryManager} containing options for
645 the manager section, filled in at construction time.
646 """
647 logCategory = 'config'
648
649 MANAGER_SOCKETS = \
650 ['flumotion.component.plugs.adminaction.AdminActionPlug',
651 'flumotion.component.plugs.base.ManagerPlug',
652 'flumotion.component.plugs.identity.IdentityProviderPlug']
653
668
680
682
683
684 name, = self.parseAttributes(node, (), ('name', ))
685 ret = ConfigEntryManager(name, None, None, None, None, None,
686 None, self.plugs)
687
688 def simpleparse(proc):
689 return lambda node: self.parseTextNode(node, proc)
690
691 def recordval(k):
692
693 def record(v):
694 if getattr(ret, k):
695 raise errors.ConfigError('duplicate %s: %s'
696 % (k, getattr(ret, k)))
697 setattr(ret, k, v)
698 return record
699
700 def enum(*allowed):
701
702 def eparse(v):
703 v = str(v)
704 if v not in allowed:
705 raise errors.ConfigError('unknown value %s (should be '
706 'one of %r)' % (v, allowed))
707 return v
708 return eparse
709
710 parsers = {'host': (simpleparse(str), recordval('host')),
711 'port': (simpleparse(int), recordval('port')),
712 'transport': (simpleparse(enum('tcp', 'ssl')),
713 recordval('transport')),
714 'certificate': (simpleparse(str), recordval('certificate')),
715 'component': (_ignore, _ignore),
716 'plugs': (_ignore, _ignore),
717 'debug': (simpleparse(str), recordval('fludebug'))}
718 self.parseFromTable(node, parsers)
719 return ret
720
722
723 def parsecomponent(node):
724 return self.parseComponent(node, 'manager', False, False)
725
726 def gotcomponent(val):
727 if self.bouncer is not None:
728 raise errors.ConfigError('can only have one bouncer '
729 '(%s is superfluous)' % (val.name, ))
730
731 self.bouncer = val
732
733 def parseplugs(node):
734 return fluconfig.buildPlugsSet(self.parsePlugs(node),
735 self.MANAGER_SOCKETS)
736
737 def gotplugs(newplugs):
738 for socket in self.plugs:
739 self.plugs[socket].extend(newplugs[socket])
740
741 parsers = {'host': (_ignore, _ignore),
742 'port': (_ignore, _ignore),
743 'transport': (_ignore, _ignore),
744 'certificate': (_ignore, _ignore),
745 'component': (parsecomponent, gotcomponent),
746 'plugs': (parseplugs, gotplugs),
747 'debug': (_ignore, _ignore)}
748 self.parseFromTable(node, parsers)
749
765
767 self.doc.unlink()
768 self.doc = None
769
770
811
813 config = component.get('config')
814 attrs = [('name', component.get('name')),
815 ('type', component.get('type')),
816 ('label', config.get('label', component.get('name'))),
817 ('worker', component.get('workerRequested')),
818 ('project', config['project']),
819 ('version', common.versionTupleToString(config['version']))]
820 self.pushTag('component', attrs)
821 for name, feeders in config['eater'].items():
822 self._writeEater(name, feeders)
823 self._writeProperties(config['properties'].items())
824 if isFeedComponent:
825 if config['clock-master'] == config['avatarId']:
826 value = 'true'
827 else:
828 value = 'false'
829 self.writeTag('clock-master', data=value)
830 self._writePlugs(config['plugs'].items())
831 self._writeVirtualFeeds(config['virtual-feeds'].items())
832 self.popTag()
833 self.writeLine()
834
836 attrs = [('name', name)]
837 self.pushTag('eater', attrs)
838 for feedId, alias in feeders:
839 attrs = [('alias', alias)]
840 self.writeTag('feed', attrs, feedId)
841 self.popTag()
842
844
845 def serialise(propVal):
846 if isinstance(propVal, tuple):
847 return ["%d/%d" % propVal]
848 elif isinstance(propVal, list):
849 return propVal
850 else:
851 return [propVal]
852 for name, value in properties:
853 attrs = [('name', name)]
854 for value in serialise(value):
855 self.writeTag('property', attrs, value)
856
865
872
874 for name, real in virtualfeeds:
875 attrs = [('name', name),
876 ('real', real)]
877 self.writeTag('virtual-feed', attrs)
878
879
883