jhbuild/moduleset.py

   1 # jhbuild - a build script for GNOME 1.x and 2.x
   2 # Copyright (C) 2001-2006  James Henstridge
   3 #
   4 #   moduleset.py: logic for running the build.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 from __future__ import generators
  21
  22 import os
  23 import sys
  24 import urlparse
  25 import logging
  26
  27 from jhbuild.errors import UsageError, FatalError, DependencyCycleError, CommandError
  28
  29 try:
  30     import xml.dom.minidom
  31 except ImportError:
  32     raise FatalError(_('Python xml packages are required but could not be found'))
  33
  34 from jhbuild import modtypes
  35 from jhbuild.versioncontrol import get_repo_type
  36 from jhbuild.utils import httpcache
  37 from jhbuild.utils.cmds import get_output
  38
  39 __all__ = ['load', 'load_tests']
  40
  41 class ModuleSet:
  42     def __init__(self, config = None):
  43         self.config = config
  44         self.modules = {}
  45     def add(self, module):
  46         '''add a Module object to this set of modules'''
  47         self.modules[module.name] = module
  48
  49     def get_module(self, module_name, ignore_case = False):
  50         if self.modules.has_key(module_name) or not ignore_case:
  51             return self.modules[module_name]
  52         module_name_lower = module_name.lower()
  53         for module in self.modules.keys():
  54             if module.lower() == module_name_lower:
  55                 if self.config is None or not self.config.quiet_mode:
  56                     logging.info(_('fixed case of module \'%(orig)s\' to \'%(new)s\'') % {
  57                             'orig': module_name, 'new': module})
  58                 return self.modules[module]
  59         print "Couldn't find the specified module: %s" % module_name
  60         sys.exit(2)
  61
  62     def get_module_list(self, seed, skip=[], tags=[], ignore_cycles=False,
  63                 ignore_suggests=False, include_optional_modules=False,
  64                 ignore_missing=False):
  65         '''gets a list of module objects (in correct dependency order)
  66         needed to build the modules in the seed list'''
  67
  68         if seed == 'all': seed = self.modules.keys()
  69         try:
  70             all_modules = [self.get_module(mod, ignore_case = True) for mod in seed if mod not in skip]
  71         except KeyError, e:
  72             raise UsageError(_('module "%s" not found') % e)
  73
  74         asked_modules = all_modules[:]
  75
  76         # 1st: get all modules that will be needed
  77         # note this is only needed to skip "after" modules that would not
  78         # otherwise be built
  79         i = 0
  80         while i < len(all_modules):
  81             for modname in all_modules[i].dependencies:
  82                 depmod = self.modules.get(modname)
  83                 if not depmod:
  84                     if not ignore_missing:
  85                         raise UsageError(_(
  86                                 '%(module)s has a dependency on unknown "%(invalid)s" module') % {
  87                                     'module': all_modules[i].name,
  88                                     'invalid': modname})
  89                     del all_modules[i]
  90                     continue
  91                 if not depmod in all_modules:
  92                     all_modules.append(depmod)
  93
  94             if not ignore_suggests:
  95                 # suggests can be ignored if not in moduleset
  96                 for modname in all_modules[i].suggests:
  97                     depmod = self.modules.get(modname)
  98                     if not depmod:
  99                         continue
 100                     if not depmod in all_modules:
 101                         all_modules.append(depmod)
 102             i += 1
 103
 104         # 2nd: order them, raise an exception on hard dependency cycle, ignore
 105         # them for soft dependencies
 106         self._ordered = []
 107         self._state = {}
 108
 109         for modname in skip:
 110             # mark skipped modules as already processed
 111             self._state[self.modules.get(modname)] = 'processed'
 112
 113         if tags:
 114             for modname in self.modules:
 115                 for tag in tags:
 116                     if tag in self.modules[modname].tags:
 117                         break
 118                 else:
 119                     # no tag matched, mark module as processed
 120                     self._state[self.modules[modname]] = 'processed'
 121
 122         def order(modules, module, mode = 'dependencies'):
 123             if self._state.get(module, 'clean') == 'processed':
 124                 # already seen
 125                 return
 126             if self._state.get(module, 'clean') == 'in-progress':
 127                 # dependency circle, abort when processing hard dependencies
 128                 if not ignore_cycles:
 129                     raise DependencyCycleError()
 130                 else:
 131                     self._state[module] = 'in-progress'
 132                     return
 133             self._state[module] = 'in-progress'
 134             for modname in module.dependencies:
 135                 depmod = self.modules[modname]
 136                 order([self.modules[x] for x in depmod.dependencies], depmod, 'dependencies')
 137             if not ignore_suggests:
 138                 for modname in module.suggests:
 139                     depmod = self.modules.get(modname)
 140                     if not depmod:
 141                         continue
 142                     save_state, save_ordered = self._state.copy(), self._ordered[:]
 143                     try:
 144                         order([self.modules[x] for x in depmod.dependencies], depmod, 'suggests')
 145                     except DependencyCycleError:
 146                         self._state, self._ordered = save_state, save_ordered
 147
 148             extra_afters = []
 149             for modname in module.after:
 150                 depmod = self.modules.get(modname)
 151                 if not depmod:
 152                     # this module doesn't exist, skip.
 153                     continue
 154                 if not depmod in all_modules and not include_optional_modules:
 155                     # skip modules that would not be built otherwise
 156                     # (build_optional_modules being the argument to force them
 157                     # to be included nevertheless)
 158
 159                     if not depmod.dependencies:
 160                         # depmod itself has no dependencies, skip.
 161                         continue
 162
 163                     # more expensive, if depmod has dependencies, compute its
 164                     # full list of hard dependencies, getting it into
 165                     # extra_afters, so they are also evaluated.
 166                     # <http://bugzilla.gnome.org/show_bug.cgi?id=546640>
 167                     t_ms = ModuleSet(self.config)
 168                     t_ms.modules = self.modules.copy()
 169                     dep_modules = t_ms.get_module_list(seed=[depmod.name])
 170                     for m in dep_modules[:-1]:
 171                         if m in all_modules:
 172                             extra_afters.append(m)
 173                     continue
 174                 save_state, save_ordered = self._state.copy(), self._ordered[:]
 175                 try:
 176                     order([self.modules[x] for x in depmod.dependencies], depmod, 'after')
 177                 except DependencyCycleError:
 178                     self._state, self._ordered = save_state, save_ordered
 179             for depmod in extra_afters:
 180                 save_state, save_ordered = self._state.copy(), self._ordered[:]
 181                 try:
 182                     order([self.modules[x] for x in depmod.dependencies], depmod, 'after')
 183                 except DependencyCycleError:
 184                     self._state, self._ordered = save_state, save_ordered
 185             self._state[module] = 'processed'
 186             self._ordered.append(module)
 187
 188         for i, module in enumerate(all_modules):
 189             order([], module)
 190             if i+1 == len(asked_modules):
 191                 break
 192
 193         ordered = self._ordered[:]
 194         del self._ordered
 195         del self._state
 196         return ordered
 197
 198     def get_full_module_list(self, skip=[], ignore_cycles=False):
 199         return self.get_module_list(self.modules.keys(), skip=skip,
 200                 ignore_cycles=ignore_cycles, ignore_missing=True)
 201
 202     def get_test_module_list (self, seed, skip=[]):
 203         test_modules = []
 204         if seed == []:
 205             return
 206         for mod in self.modules.values():
 207             for test_app in seed:
 208                 if test_app in mod.tested_pkgs:
 209                     test_modules.append(mod)
 210         return test_modules
 211
 212     def write_dot(self, modules=None, fp=sys.stdout, suggests=False, clusters=False):
 213         from jhbuild.modtypes import MetaModule
 214         from jhbuild.modtypes.autotools import AutogenModule
 215         from jhbuild.versioncontrol.tarball import TarballBranch
 216
 217         if modules is None:
 218             modules = self.modules.keys()
 219         inlist = {}
 220         for module in modules:
 221             inlist[module] = None
 222
 223         fp.write('digraph "G" {\n'
 224                  '  fontsize = 8;\n'
 225                  '  ratio = auto;\n')
 226         while modules:
 227             modname = modules[0]
 228             try:
 229                 mod = self.modules[modname]
 230             except KeyError:
 231                 logging.warning(_('Unknown module:') + ' '+ modname)
 232                 del modules[0]
 233                 continue
 234             if isinstance(mod, MetaModule):
 235                 attrs = '[color="lightcoral",style="filled",' \
 236                         'label="%s"]' % mod.name
 237             else:
 238                 label = mod.name
 239                 color = 'lightskyblue'
 240                 if mod.branch.branchname:
 241                     label += '\\n(%s)' % mod.branch.branchname
 242                 if isinstance(mod.branch, TarballBranch):
 243                     color = 'lightgoldenrod'
 244                 attrs = '[color="%s",style="filled",label="%s"]' % (color, label)
 245             fp.write('  "%s" %s;\n' % (modname, attrs))
 246             del modules[0]
 247
 248             for dep in self.modules[modname].dependencies:
 249                 fp.write('  "%s" -> "%s";\n' % (modname, dep))
 250                 if not inlist.has_key(dep):
 251                     modules.append(dep)
 252                 inlist[dep] = None
 253
 254             if suggests:
 255                 for dep in self.modules[modname].after + self.modules[modname].suggests:
 256                     if self.modules.has_key(dep):
 257                         fp.write('  "%s" -> "%s" [style=dotted];\n' % (modname, dep))
 258                         if not inlist.has_key(dep):
 259                             modules.append(dep)
 260                         inlist[dep] = None
 261
 262         if clusters:
 263             # create clusters for MetaModules
 264             for modname in inlist.keys():
 265                 mod = self.modules.get(modname)
 266                 if isinstance(mod, MetaModule):
 267                     fp.write('  subgraph "cluster_%s" {\n' % mod.name)
 268                     fp.write('     label="%s";\n' % mod.name)
 269                     fp.write('     style="filled";bgcolor="honeydew2";\n')
 270
 271                     for dep in mod.dependencies:
 272                         fp.write('    "%s";\n' % dep)
 273                     fp.write('  }\n')
 274
 275         fp.write('}\n')
 276
 277 def load(config, uri=None):
 278     if uri is not None:
 279         modulesets = [ uri ]
 280     elif type(config.moduleset) in (list, tuple):
 281         modulesets = config.moduleset
 282     else:
 283         modulesets = [ config.moduleset ]
 284     ms = ModuleSet(config = config)
 285     for uri in modulesets:
 286         if '/' not in uri and not os.path.exists(uri):
 287             if config.modulesets_dir and config.nonetwork or config.use_local_modulesets:
 288                 uri = os.path.join(config.modulesets_dir, uri + '.modules')
 289             else:
 290                 uri = 'http://git.gnome.org/browse/jhbuild/plain/modulesets/%s.modules' % uri
 291         try:
 292             ms.modules.update(_parse_module_set(config, uri).modules)
 293         except xml.parsers.expat.ExpatError, e:
 294             raise FatalError(_('failed to parse %s: %s') % (uri, e))
 295     return ms
 296
 297 def load_tests (config, uri=None):
 298     ms = load (config, uri)
 299     ms_tests = ModuleSet(config = config)
 300     for app, module in ms.modules.iteritems():
 301         if module.__class__ == testmodule.TestModule:
 302             ms_tests.modules[app] = module
 303     return ms_tests
 304
 305 def _child_elements(parent):
 306     for node in parent.childNodes:
 307         if node.nodeType == node.ELEMENT_NODE:
 308             yield node
 309
 310 def _child_elements_matching(parent, names):
 311     for node in parent.childNodes:
 312         if node.nodeType == node.ELEMENT_NODE and node.nodeName in names:
 313             yield node
 314
 315 def _parse_module_set(config, uri):
 316     try:
 317         filename = httpcache.load(uri, nonetwork=config.nonetwork, age=0)
 318     except Exception, e:
 319         raise FatalError(_('could not download %s: %s') % (uri, e))
 320     filename = os.path.normpath(filename)
 321     try:
 322         document = xml.dom.minidom.parse(filename)
 323     except IOError, e:
 324         raise FatalError(_('failed to parse %s: %s') % (filename, e))
 325
 326     assert document.documentElement.nodeName == 'moduleset'
 327     moduleset = ModuleSet(config = config)
 328     moduleset_name = document.documentElement.getAttribute('name')
 329     if not moduleset_name and uri.endswith('.modules'):
 330         moduleset_name = os.path.basename(uri)[:-len('.modules')]
 331
 332     # load up list of repositories
 333     repositories = {}
 334     default_repo = None
 335     for node in _child_elements_matching(
 336             document.documentElement, ['repository', 'cvsroot', 'svnroot',
 337                                        'arch-archive']):
 338         name = node.getAttribute('name')
 339         if node.getAttribute('default') == 'yes':
 340             default_repo = name
 341         if node.nodeName == 'repository':
 342             repo_type = node.getAttribute('type')
 343             repo_class = get_repo_type(repo_type)
 344             kws = {}
 345             for attr in repo_class.init_xml_attrs:
 346                 if node.hasAttribute(attr):
 347                     kws[attr.replace('-', '_')] = node.getAttribute(attr)
 348             repositories[name] = repo_class(config, name, **kws)
 349             repositories[name].moduleset_uri = uri
 350             mirrors = {}
 351             for mirror in _child_elements_matching(node, ['mirror']):
 352                 mirror_type = mirror.getAttribute('type')
 353                 mirror_class = get_repo_type(mirror_type)
 354                 kws = {}
 355                 for attr in mirror_class.init_xml_attrs:
 356                     if mirror.hasAttribute(attr):
 357                         kws[attr.replace('-','_')] = mirror.getAttribute(attr)
 358                 mirrors[mirror_type] = mirror_class(config, name, **kws)
 359                 #mirrors[mirror_type].moduleset_uri = uri
 360             setattr(repositories[name], "mirrors", mirrors)
 361         if node.nodeName == 'cvsroot':
 362             cvsroot = node.getAttribute('root')
 363             if node.hasAttribute('password'):
 364                 password = node.getAttribute('password')
 365             else:
 366                 password = None
 367             repo_type = get_repo_type('cvs')
 368             repositories[name] = repo_type(config, name,
 369                                            cvsroot=cvsroot, password=password)
 370         elif node.nodeName == 'svnroot':
 371             svnroot = node.getAttribute('href')
 372             repo_type = get_repo_type('svn')
 373             repositories[name] = repo_type(config, name, href=svnroot)
 374         elif node.nodeName == 'arch-archive':
 375             archive_uri = node.getAttribute('href')
 376             repo_type = get_repo_type('arch')
 377             repositories[name] = repo_type(config, name,
 378                                            archive=name, href=archive_uri)
 379
 380     # and now module definitions
 381     for node in _child_elements(document.documentElement):
 382         if node.nodeName == 'include':
 383             href = node.getAttribute('href')
 384             inc_uri = urlparse.urljoin(uri, href)
 385             try:
 386                 inc_moduleset = _parse_module_set(config, inc_uri)
 387             except FatalError, e:
 388                 if inc_uri[0] == '/':
 389                     raise e
 390                 # look up in local modulesets
 391                 inc_uri = os.path.join(os.path.dirname(__file__), '..', 'modulesets',
 392                                    href)
 393                 inc_moduleset = _parse_module_set(config, inc_uri)
 394
 395             moduleset.modules.update(inc_moduleset.modules)
 396         elif node.nodeName in ['repository', 'cvsroot', 'svnroot',
 397                                'arch-archive']:
 398             pass
 399         else:
 400             module = modtypes.parse_xml_node(node, config, uri,
 401                     repositories, default_repo)
 402             if moduleset_name:
 403                 module.tags.append(moduleset_name)
 404             module.moduleset_name = moduleset_name
 405             module.config = config
 406             moduleset.add(module)
 407
 408     return moduleset
 409
 410 def warn_local_modulesets(config):
 411     if config.use_local_modulesets:
 412         return
 413
 414     moduleset_local_path = os.path.join(SRCDIR, 'modulesets')
 415     if not os.path.exists(moduleset_local_path):
 416         # moduleset-less checkout
 417         return
 418
 419     if not os.path.exists(os.path.join(moduleset_local_path, '..', '.git')):
 420         # checkout was not done via git
 421         return
 422
 423     if type(config.moduleset) == type([]):
 424         modulesets = config.moduleset
 425     else:
 426         modulesets = [ config.moduleset ]
 427
 428     if not [x for x in modulesets if x.find('/') == -1]:
 429         # all modulesets have a slash; they are URI
 430         return
 431
 432     try:
 433         git_diff = get_output(['git', 'diff', 'origin/master', '--', '.'],
 434                 cwd=moduleset_local_path).strip()
 435     except CommandError:
 436         # git error, ignore
 437         return
 438
 439     if not git_diff:
 440         # no locally modified moduleset
 441         return
 442
 443     logging.info(
 444             _('Modulesets were edited locally but JHBuild is configured '\
 445               'to get them from the network, perhaps you need to add '\
 446               'use_local_modulesets = True to your .jhbuildrc.'))
 447