jhbuild/moduleset.py

   1 # jhbuild - a build script for GNOME 1.x and 2.x
   2 # Copyright (C) 2001-2006  James Henstridge
   3 #
   4 #   moduleset.py: logic for running the build.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 from __future__ import generators
  21
  22 import os
  23 import sys
  24 import urlparse
  25 import logging
  26
  27 from jhbuild.errors import UsageError, FatalError, DependencyCycleError, CommandError
  28
  29 try:
  30     import xml.dom.minidom
  31 except ImportError:
  32     raise FatalError(_('Python xml packages are required but could not be found'))
  33
  34 from jhbuild import modtypes
  35 from jhbuild.versioncontrol import get_repo_type
  36 from jhbuild.utils import httpcache
  37 from jhbuild.utils.cmds import get_output
  38 from jhbuild.modtypes.testmodule import TestModule
  39
  40 __all__ = ['load', 'load_tests']
  41
  42 class ModuleSet:
  43     def __init__(self, config = None):
  44         self.config = config
  45         self.modules = {}
  46     def add(self, module):
  47         '''add a Module object to this set of modules'''
  48         self.modules[module.name] = module
  49
  50     def get_module(self, module_name, ignore_case = False):
  51         if self.modules.has_key(module_name) or not ignore_case:
  52             return self.modules[module_name]
  53         module_name_lower = module_name.lower()
  54         for module in self.modules.keys():
  55             if module.lower() == module_name_lower:
  56                 if self.config is None or not self.config.quiet_mode:
  57                     logging.info(_('fixed case of module \'%(orig)s\' to \'%(new)s\'') % {
  58                             'orig': module_name, 'new': module})
  59                 return self.modules[module]
  60         print "Couldn't find the specified module: %s" % module_name
  61         sys.exit(2)
  62
  63     def get_module_list(self, seed, skip=[], tags=[], ignore_cycles=False,
  64                 ignore_suggests=False, include_optional_modules=False,
  65                 ignore_missing=False):
  66         '''gets a list of module objects (in correct dependency order)
  67         needed to build the modules in the seed list'''
  68
  69         if seed == 'all': seed = self.modules.keys()
  70         try:
  71             all_modules = [self.get_module(mod, ignore_case = True) for mod in seed if mod not in skip]
  72         except KeyError, e:
  73             raise UsageError(_('module "%s" not found') % e)
  74
  75         asked_modules = all_modules[:]
  76
  77         # 1st: get all modules that will be needed
  78         # note this is only needed to skip "after" modules that would not
  79         # otherwise be built
  80         i = 0
  81         while i < len(all_modules):
  82             for modname in all_modules[i].dependencies:
  83                 depmod = self.modules.get(modname)
  84                 if not depmod:
  85                     if not ignore_missing:
  86                         raise UsageError(_(
  87                                 '%(module)s has a dependency on unknown "%(invalid)s" module') % {
  88                                     'module': all_modules[i].name,
  89                                     'invalid': modname})
  90                     del all_modules[i]
  91                     continue
  92                 if not depmod in all_modules:
  93                     all_modules.append(depmod)
  94
  95             if not ignore_suggests:
  96                 # suggests can be ignored if not in moduleset
  97                 for modname in all_modules[i].suggests:
  98                     depmod = self.modules.get(modname)
  99                     if not depmod:
 100                         continue
 101                     if not depmod in all_modules:
 102                         all_modules.append(depmod)
 103             i += 1
 104
 105         # 2nd: order them, raise an exception on hard dependency cycle, ignore
 106         # them for soft dependencies
 107         self._ordered = []
 108         self._state = {}
 109
 110         for modname in skip:
 111             # mark skipped modules as already processed
 112             self._state[self.modules.get(modname)] = 'processed'
 113
 114         if tags:
 115             for modname in self.modules:
 116                 for tag in tags:
 117                     if tag in self.modules[modname].tags:
 118                         break
 119                 else:
 120                     # no tag matched, mark module as processed
 121                     self._state[self.modules[modname]] = 'processed'
 122
 123         def order(modules, module, mode = 'dependencies'):
 124             if self._state.get(module, 'clean') == 'processed':
 125                 # already seen
 126                 return
 127             if self._state.get(module, 'clean') == 'in-progress':
 128                 # dependency circle, abort when processing hard dependencies
 129                 if not ignore_cycles:
 130                     raise DependencyCycleError()
 131                 else:
 132                     self._state[module] = 'in-progress'
 133                     return
 134             self._state[module] = 'in-progress'
 135             for modname in module.dependencies:
 136                 depmod = self.modules[modname]
 137                 order([self.modules[x] for x in depmod.dependencies], depmod, 'dependencies')
 138             if not ignore_suggests:
 139                 for modname in module.suggests:
 140                     depmod = self.modules.get(modname)
 141                     if not depmod:
 142                         continue
 143                     save_state, save_ordered = self._state.copy(), self._ordered[:]
 144                     try:
 145                         order([self.modules[x] for x in depmod.dependencies], depmod, 'suggests')
 146                     except DependencyCycleError:
 147                         self._state, self._ordered = save_state, save_ordered
 148
 149             extra_afters = []
 150             for modname in module.after:
 151                 depmod = self.modules.get(modname)
 152                 if not depmod:
 153                     # this module doesn't exist, skip.
 154                     continue
 155                 if not depmod in all_modules and not include_optional_modules:
 156                     # skip modules that would not be built otherwise
 157                     # (build_optional_modules being the argument to force them
 158                     # to be included nevertheless)
 159
 160                     if not depmod.dependencies:
 161                         # depmod itself has no dependencies, skip.
 162                         continue
 163
 164                     # more expensive, if depmod has dependencies, compute its
 165                     # full list of hard dependencies, getting it into
 166                     # extra_afters, so they are also evaluated.
 167                     # <http://bugzilla.gnome.org/show_bug.cgi?id=546640>
 168                     t_ms = ModuleSet(self.config)
 169                     t_ms.modules = self.modules.copy()
 170                     dep_modules = t_ms.get_module_list(seed=[depmod.name])
 171                     for m in dep_modules[:-1]:
 172                         if m in all_modules:
 173                             extra_afters.append(m)
 174                     continue
 175                 save_state, save_ordered = self._state.copy(), self._ordered[:]
 176                 try:
 177                     order([self.modules[x] for x in depmod.dependencies], depmod, 'after')
 178                 except DependencyCycleError:
 179                     self._state, self._ordered = save_state, save_ordered
 180             for depmod in extra_afters:
 181                 save_state, save_ordered = self._state.copy(), self._ordered[:]
 182                 try:
 183                     order([self.modules[x] for x in depmod.dependencies], depmod, 'after')
 184                 except DependencyCycleError:
 185                     self._state, self._ordered = save_state, save_ordered
 186             self._state[module] = 'processed'
 187             self._ordered.append(module)
 188
 189         for i, module in enumerate(all_modules):
 190             order([], module)
 191             if i+1 == len(asked_modules):
 192                 break
 193
 194         ordered = self._ordered[:]
 195         del self._ordered
 196         del self._state
 197         return ordered
 198
 199     def get_full_module_list(self, skip=[], ignore_cycles=False):
 200         return self.get_module_list(self.modules.keys(), skip=skip,
 201                 ignore_cycles=ignore_cycles, ignore_missing=True)
 202
 203     def get_test_module_list (self, seed, skip=[]):
 204         test_modules = []
 205         if seed == []:
 206             return
 207         for mod in self.modules.values():
 208             for test_app in seed:
 209                 if test_app in mod.tested_pkgs:
 210                     test_modules.append(mod)
 211         return test_modules
 212
 213     def write_dot(self, modules=None, fp=sys.stdout, suggests=False, clusters=False):
 214         from jhbuild.modtypes import MetaModule
 215         from jhbuild.modtypes.autotools import AutogenModule
 216         from jhbuild.versioncontrol.tarball import TarballBranch
 217
 218         if modules is None:
 219             modules = self.modules.keys()
 220         inlist = {}
 221         for module in modules:
 222             inlist[module] = None
 223
 224         fp.write('digraph "G" {\n'
 225                  '  fontsize = 8;\n'
 226                  '  ratio = auto;\n')
 227         while modules:
 228             modname = modules[0]
 229             try:
 230                 mod = self.modules[modname]
 231             except KeyError:
 232                 logging.warning(_('Unknown module:') + ' '+ modname)
 233                 del modules[0]
 234                 continue
 235             if isinstance(mod, MetaModule):
 236                 attrs = '[color="lightcoral",style="filled",' \
 237                         'label="%s"]' % mod.name
 238             else:
 239                 label = mod.name
 240                 color = 'lightskyblue'
 241                 if mod.branch.branchname:
 242                     label += '\\n(%s)' % mod.branch.branchname
 243                 if isinstance(mod.branch, TarballBranch):
 244                     color = 'lightgoldenrod'
 245                 attrs = '[color="%s",style="filled",label="%s"]' % (color, label)
 246             fp.write('  "%s" %s;\n' % (modname, attrs))
 247             del modules[0]
 248
 249             for dep in self.modules[modname].dependencies:
 250                 fp.write('  "%s" -> "%s";\n' % (modname, dep))
 251                 if not inlist.has_key(dep):
 252                     modules.append(dep)
 253                 inlist[dep] = None
 254
 255             if suggests:
 256                 for dep in self.modules[modname].after + self.modules[modname].suggests:
 257                     if self.modules.has_key(dep):
 258                         fp.write('  "%s" -> "%s" [style=dotted];\n' % (modname, dep))
 259                         if not inlist.has_key(dep):
 260                             modules.append(dep)
 261                         inlist[dep] = None
 262
 263         if clusters:
 264             # create clusters for MetaModules
 265             for modname in inlist.keys():
 266                 mod = self.modules.get(modname)
 267                 if isinstance(mod, MetaModule):
 268                     fp.write('  subgraph "cluster_%s" {\n' % mod.name)
 269                     fp.write('     label="%s";\n' % mod.name)
 270                     fp.write('     style="filled";bgcolor="honeydew2";\n')
 271
 272                     for dep in mod.dependencies:
 273                         fp.write('    "%s";\n' % dep)
 274                     fp.write('  }\n')
 275
 276         fp.write('}\n')
 277
 278 def load(config, uri=None):
 279     if uri is not None:
 280         modulesets = [ uri ]
 281     elif type(config.moduleset) in (list, tuple):
 282         modulesets = config.moduleset
 283     else:
 284         modulesets = [ config.moduleset ]
 285     ms = ModuleSet(config = config)
 286     for uri in modulesets:
 287         if '/' not in uri and not os.path.exists(uri):
 288             if config.modulesets_dir and config.nonetwork or config.use_local_modulesets:
 289                 uri = os.path.join(config.modulesets_dir, uri + '.modules')
 290             else:
 291                 uri = 'http://git.gnome.org/browse/jhbuild/plain/modulesets/%s.modules' % uri
 292         try:
 293             ms.modules.update(_parse_module_set(config, uri).modules)
 294         except xml.parsers.expat.ExpatError, e:
 295             raise FatalError(_('failed to parse %s: %s') % (uri, e))
 296     return ms
 297
 298 def load_tests (config, uri=None):
 299     ms = load (config, uri)
 300     ms_tests = ModuleSet(config = config)
 301     for app, module in ms.modules.iteritems():
 302         if module.__class__ == TestModule:
 303             ms_tests.modules[app] = module
 304     return ms_tests
 305
 306 def _child_elements(parent):
 307     for node in parent.childNodes:
 308         if node.nodeType == node.ELEMENT_NODE:
 309             yield node
 310
 311 def _child_elements_matching(parent, names):
 312     for node in parent.childNodes:
 313         if node.nodeType == node.ELEMENT_NODE and node.nodeName in names:
 314             yield node
 315
 316 def _parse_module_set(config, uri):
 317     try:
 318         filename = httpcache.load(uri, nonetwork=config.nonetwork, age=0)
 319     except Exception, e:
 320         raise FatalError(_('could not download %s: %s') % (uri, e))
 321     filename = os.path.normpath(filename)
 322     try:
 323         document = xml.dom.minidom.parse(filename)
 324     except IOError, e:
 325         raise FatalError(_('failed to parse %s: %s') % (filename, e))
 326
 327     assert document.documentElement.nodeName == 'moduleset'
 328     moduleset = ModuleSet(config = config)
 329     moduleset_name = document.documentElement.getAttribute('name')
 330     if not moduleset_name and uri.endswith('.modules'):
 331         moduleset_name = os.path.basename(uri)[:-len('.modules')]
 332
 333     # load up list of repositories
 334     repositories = {}
 335     default_repo = None
 336     for node in _child_elements_matching(
 337             document.documentElement, ['repository', 'cvsroot', 'svnroot',
 338                                        'arch-archive']):
 339         name = node.getAttribute('name')
 340         if node.getAttribute('default') == 'yes':
 341             default_repo = name
 342         if node.nodeName == 'repository':
 343             repo_type = node.getAttribute('type')
 344             repo_class = get_repo_type(repo_type)
 345             kws = {}
 346             for attr in repo_class.init_xml_attrs:
 347                 if node.hasAttribute(attr):
 348                     kws[attr.replace('-', '_')] = node.getAttribute(attr)
 349             repositories[name] = repo_class(config, name, **kws)
 350             repositories[name].moduleset_uri = uri
 351             mirrors = {}
 352             for mirror in _child_elements_matching(node, ['mirror']):
 353                 mirror_type = mirror.getAttribute('type')
 354                 mirror_class = get_repo_type(mirror_type)
 355                 kws = {}
 356                 for attr in mirror_class.init_xml_attrs:
 357                     if mirror.hasAttribute(attr):
 358                         kws[attr.replace('-','_')] = mirror.getAttribute(attr)
 359                 mirrors[mirror_type] = mirror_class(config, name, **kws)
 360                 #mirrors[mirror_type].moduleset_uri = uri
 361             setattr(repositories[name], "mirrors", mirrors)
 362         if node.nodeName == 'cvsroot':
 363             cvsroot = node.getAttribute('root')
 364             if node.hasAttribute('password'):
 365                 password = node.getAttribute('password')
 366             else:
 367                 password = None
 368             repo_type = get_repo_type('cvs')
 369             repositories[name] = repo_type(config, name,
 370                                            cvsroot=cvsroot, password=password)
 371         elif node.nodeName == 'svnroot':
 372             svnroot = node.getAttribute('href')
 373             repo_type = get_repo_type('svn')
 374             repositories[name] = repo_type(config, name, href=svnroot)
 375         elif node.nodeName == 'arch-archive':
 376             archive_uri = node.getAttribute('href')
 377             repo_type = get_repo_type('arch')
 378             repositories[name] = repo_type(config, name,
 379                                            archive=name, href=archive_uri)
 380
 381     # and now module definitions
 382     for node in _child_elements(document.documentElement):
 383         if node.nodeName == 'include':
 384             href = node.getAttribute('href')
 385             inc_uri = urlparse.urljoin(uri, href)
 386             try:
 387                 inc_moduleset = _parse_module_set(config, inc_uri)
 388             except FatalError, e:
 389                 if inc_uri[0] == '/':
 390                     raise e
 391                 # look up in local modulesets
 392                 inc_uri = os.path.join(os.path.dirname(__file__), '..', 'modulesets',
 393                                    href)
 394                 inc_moduleset = _parse_module_set(config, inc_uri)
 395
 396             moduleset.modules.update(inc_moduleset.modules)
 397         elif node.nodeName in ['repository', 'cvsroot', 'svnroot',
 398                                'arch-archive']:
 399             pass
 400         else:
 401             module = modtypes.parse_xml_node(node, config, uri,
 402                     repositories, default_repo)
 403             if moduleset_name:
 404                 module.tags.append(moduleset_name)
 405             module.moduleset_name = moduleset_name
 406             module.config = config
 407             moduleset.add(module)
 408
 409     return moduleset
 410
 411 def warn_local_modulesets(config):
 412     if config.use_local_modulesets:
 413         return
 414
 415     moduleset_local_path = os.path.join(SRCDIR, 'modulesets')
 416     if not os.path.exists(moduleset_local_path):
 417         # moduleset-less checkout
 418         return
 419
 420     if not os.path.exists(os.path.join(moduleset_local_path, '..', '.git')):
 421         # checkout was not done via git
 422         return
 423
 424     if type(config.moduleset) == type([]):
 425         modulesets = config.moduleset
 426     else:
 427         modulesets = [ config.moduleset ]
 428
 429     if not [x for x in modulesets if x.find('/') == -1]:
 430         # all modulesets have a slash; they are URI
 431         return
 432
 433     try:
 434         git_diff = get_output(['git', 'diff', 'origin/master', '--', '.'],
 435                 cwd=moduleset_local_path).strip()
 436     except CommandError:
 437         # git error, ignore
 438         return
 439
 440     if not git_diff:
 441         # no locally modified moduleset
 442         return
 443
 444     logging.info(
 445             _('Modulesets were edited locally but JHBuild is configured '\
 446               'to get them from the network, perhaps you need to add '\
 447               'use_local_modulesets = True to your .jhbuildrc.'))
 448