jhbuild/moduleset.py

   1 # jhbuild - a build script for GNOME 1.x and 2.x
   2 # Copyright (C) 2001-2006  James Henstridge
   3 #
   4 #   moduleset.py: logic for running the build.
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software
  18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19
  20 from __future__ import generators
  21
  22 import os
  23 import sys
  24 import urlparse
  25 import logging
  26
  27 from jhbuild.errors import UsageError, FatalError, DependencyCycleError, CommandError
  28
  29 try:
  30     import xml.dom.minidom
  31 except ImportError:
  32     raise FatalError(_('Python xml packages are required but could not be found'))
  33
  34 from jhbuild import modtypes
  35 from jhbuild.versioncontrol import get_repo_type
  36 from jhbuild.utils import httpcache
  37 from jhbuild.utils.cmds import get_output
  38
  39 __all__ = ['load', 'load_tests']
  40
  41 class ModuleSet:
  42     def __init__(self, config = None):
  43         self.config = config
  44         self.modules = {}
  45     def add(self, module):
  46         '''add a Module object to this set of modules'''
  47         self.modules[module.name] = module
  48
  49     def get_module(self, module_name, ignore_case = False):
  50         if self.modules.has_key(module_name) or not ignore_case:
  51             return self.modules[module_name]
  52         module_name = module_name.lower()
  53         for module in self.modules.keys():
  54             if module.lower() == module_name:
  55                 if self.config is None or not self.config.quiet_mode:
  56                     logging.info(_('fixed case of module \'%(orig)s\' to \'%(new)s\'') % {
  57                             'orig': module_name, 'new': module})
  58                 return self.modules[module]
  59         raise KeyError(module_name)
  60
  61     def get_module_list(self, seed, skip=[], tags=[], ignore_cycles=False,
  62                 ignore_suggests=False, include_optional_modules=False,
  63                 ignore_missing=False):
  64         '''gets a list of module objects (in correct dependency order)
  65         needed to build the modules in the seed list'''
  66
  67         if seed == 'all': seed = self.modules.keys()
  68         try:
  69             all_modules = [self.get_module(mod, ignore_case = True) for mod in seed if mod not in skip]
  70         except KeyError, e:
  71             raise UsageError(_('module "%s" not found') % e)
  72
  73         asked_modules = all_modules[:]
  74
  75         # 1st: get all modules that will be needed
  76         # note this is only needed to skip "after" modules that would not
  77         # otherwise be built
  78         i = 0
  79         while i < len(all_modules):
  80             for modname in all_modules[i].dependencies:
  81                 depmod = self.modules.get(modname)
  82                 if not depmod:
  83                     if not ignore_missing:
  84                         raise UsageError(_(
  85                                 '%(module)s has a dependency on unknown "%(invalid)s" module') % {
  86                                     'module': all_modules[i].name,
  87                                     'invalid': modname})
  88                     del all_modules[i]
  89                     continue
  90                 if not depmod in all_modules:
  91                     all_modules.append(depmod)
  92
  93             if not ignore_suggests:
  94                 # suggests can be ignored if not in moduleset
  95                 for modname in all_modules[i].suggests:
  96                     depmod = self.modules.get(modname)
  97                     if not depmod:
  98                         continue
  99                     if not depmod in all_modules:
 100                         all_modules.append(depmod)
 101             i += 1
 102
 103         # 2nd: order them, raise an exception on hard dependency cycle, ignore
 104         # them for soft dependencies
 105         self._ordered = []
 106         self._state = {}
 107
 108         for modname in skip:
 109             # mark skipped modules as already processed
 110             self._state[self.modules.get(modname)] = 'processed'
 111
 112         if tags:
 113             for modname in self.modules:
 114                 for tag in tags:
 115                     if tag in self.modules[modname].tags:
 116                         break
 117                 else:
 118                     # no tag matched, mark module as processed
 119                     self._state[self.modules[modname]] = 'processed'
 120
 121         def order(modules, module, mode = 'dependencies'):
 122             if self._state.get(module, 'clean') == 'processed':
 123                 # already seen
 124                 return
 125             if self._state.get(module, 'clean') == 'in-progress':
 126                 # dependency circle, abort when processing hard dependencies
 127                 if not ignore_cycles:
 128                     raise DependencyCycleError()
 129                 else:
 130                     self._state[module] = 'in-progress'
 131                     return
 132             self._state[module] = 'in-progress'
 133             for modname in module.dependencies:
 134                 depmod = self.modules[modname]
 135                 order([self.modules[x] for x in depmod.dependencies], depmod, 'dependencies')
 136             if not ignore_suggests:
 137                 for modname in module.suggests:
 138                     depmod = self.modules.get(modname)
 139                     if not depmod:
 140                         continue
 141                     save_state, save_ordered = self._state.copy(), self._ordered[:]
 142                     try:
 143                         order([self.modules[x] for x in depmod.dependencies], depmod, 'suggests')
 144                     except DependencyCycleError:
 145                         self._state, self._ordered = save_state, save_ordered
 146
 147             extra_afters = []
 148             for modname in module.after:
 149                 depmod = self.modules.get(modname)
 150                 if not depmod:
 151                     # this module doesn't exist, skip.
 152                     continue
 153                 if not depmod in all_modules and not include_optional_modules:
 154                     # skip modules that would not be built otherwise
 155                     # (build_optional_modules being the argument to force them
 156                     # to be included nevertheless)
 157
 158                     if not depmod.dependencies:
 159                         # depmod itself has no dependencies, skip.
 160                         continue
 161
 162                     # more expensive, if depmod has dependencies, compute its
 163                     # full list of hard dependencies, getting it into
 164                     # extra_afters, so they are also evaluated.
 165                     # <http://bugzilla.gnome.org/show_bug.cgi?id=546640>
 166                     t_ms = ModuleSet(self.config)
 167                     t_ms.modules = self.modules.copy()
 168                     dep_modules = t_ms.get_module_list(seed=[depmod.name])
 169                     for m in dep_modules[:-1]:
 170                         if m in all_modules:
 171                             extra_afters.append(m)
 172                     continue
 173                 save_state, save_ordered = self._state.copy(), self._ordered[:]
 174                 try:
 175                     order([self.modules[x] for x in depmod.dependencies], depmod, 'after')
 176                 except DependencyCycleError:
 177                     self._state, self._ordered = save_state, save_ordered
 178             for depmod in extra_afters:
 179                 save_state, save_ordered = self._state.copy(), self._ordered[:]
 180                 try:
 181                     order([self.modules[x] for x in depmod.dependencies], depmod, 'after')
 182                 except DependencyCycleError:
 183                     self._state, self._ordered = save_state, save_ordered
 184             self._state[module] = 'processed'
 185             self._ordered.append(module)
 186
 187         for i, module in enumerate(all_modules):
 188             order([], module)
 189             if i+1 == len(asked_modules):
 190                 break
 191
 192         ordered = self._ordered[:]
 193         del self._ordered
 194         del self._state
 195         return ordered
 196
 197     def get_full_module_list(self, skip=[], ignore_cycles=False):
 198         return self.get_module_list(self.modules.keys(), skip=skip,
 199                 ignore_cycles=ignore_cycles, ignore_missing=True)
 200
 201     def get_test_module_list (self, seed, skip=[]):
 202         test_modules = []
 203         if seed == []:
 204             return
 205         for mod in self.modules.values():
 206             for test_app in seed:
 207                 if test_app in mod.tested_pkgs:
 208                     test_modules.append(mod)
 209         return test_modules
 210
 211     def write_dot(self, modules=None, fp=sys.stdout, suggests=False, clusters=False):
 212         from jhbuild.modtypes import MetaModule
 213         from jhbuild.modtypes.autotools import AutogenModule
 214         from jhbuild.versioncontrol.tarball import TarballBranch
 215
 216         if modules is None:
 217             modules = self.modules.keys()
 218         inlist = {}
 219         for module in modules:
 220             inlist[module] = None
 221
 222         fp.write('digraph "G" {\n'
 223                  '  fontsize = 8;\n'
 224                  '  ratio = auto;\n')
 225         while modules:
 226             modname = modules[0]
 227             try:
 228                 mod = self.modules[modname]
 229             except KeyError:
 230                 logging.warning(_('Unknown module:') + ' '+ modname)
 231                 del modules[0]
 232                 continue
 233             if isinstance(mod, MetaModule):
 234                 attrs = '[color="lightcoral",style="filled",' \
 235                         'label="%s"]' % mod.name
 236             else:
 237                 label = mod.name
 238                 color = 'lightskyblue'
 239                 if mod.branch.branchname:
 240                     label += '\\n(%s)' % mod.branch.branchname
 241                 if isinstance(mod.branch, TarballBranch):
 242                     color = 'lightgoldenrod'
 243                 attrs = '[color="%s",style="filled",label="%s"]' % (color, label)
 244             fp.write('  "%s" %s;\n' % (modname, attrs))
 245             del modules[0]
 246
 247             for dep in self.modules[modname].dependencies:
 248                 fp.write('  "%s" -> "%s";\n' % (modname, dep))
 249                 if not inlist.has_key(dep):
 250                     modules.append(dep)
 251                 inlist[dep] = None
 252
 253             if suggests:
 254                 for dep in self.modules[modname].after + self.modules[modname].suggests:
 255                     if self.modules.has_key(dep):
 256                         fp.write('  "%s" -> "%s" [style=dotted];\n' % (modname, dep))
 257                         if not inlist.has_key(dep):
 258                             modules.append(dep)
 259                         inlist[dep] = None
 260
 261         if clusters:
 262             # create clusters for MetaModules
 263             for modname in inlist.keys():
 264                 mod = self.modules.get(modname)
 265                 if isinstance(mod, MetaModule):
 266                     fp.write('  subgraph "cluster_%s" {\n' % mod.name)
 267                     fp.write('     label="%s";\n' % mod.name)
 268                     fp.write('     style="filled";bgcolor="honeydew2";\n')
 269
 270                     for dep in mod.dependencies:
 271                         fp.write('    "%s";\n' % dep)
 272                     fp.write('  }\n')
 273
 274         fp.write('}\n')
 275
 276 def load(config, uri=None):
 277     if uri is not None:
 278         modulesets = [ uri ]
 279     elif type(config.moduleset) in (list, tuple):
 280         modulesets = config.moduleset
 281     else:
 282         modulesets = [ config.moduleset ]
 283     ms = ModuleSet(config = config)
 284     for uri in modulesets:
 285         if '/' not in uri:
 286             if config.modulesets_dir and config.nonetwork or config.use_local_modulesets:
 287                 uri = os.path.join(config.modulesets_dir, uri + '.modules')
 288             else:
 289                 uri = 'http://git.gnome.org/cgit/jhbuild/plain/modulesets/%s.modules' % uri
 290         try:
 291             ms.modules.update(_parse_module_set(config, uri).modules)
 292         except xml.parsers.expat.ExpatError, e:
 293             raise FatalError(_('failed to parse %s: %s') % (uri, e))
 294     return ms
 295
 296 def load_tests (config, uri=None):
 297     ms = load (config, uri)
 298     ms_tests = ModuleSet(config = config)
 299     for app, module in ms.modules.iteritems():
 300         if module.__class__ == testmodule.TestModule:
 301             ms_tests.modules[app] = module
 302     return ms_tests
 303
 304 def _child_elements(parent):
 305     for node in parent.childNodes:
 306         if node.nodeType == node.ELEMENT_NODE:
 307             yield node
 308
 309 def _child_elements_matching(parent, names):
 310     for node in parent.childNodes:
 311         if node.nodeType == node.ELEMENT_NODE and node.nodeName in names:
 312             yield node
 313
 314 def _parse_module_set(config, uri):
 315     try:
 316         filename = httpcache.load(uri, nonetwork=config.nonetwork, age=0)
 317     except Exception, e:
 318         raise FatalError(_('could not download %s: %s') % (uri, e))
 319     filename = os.path.normpath(filename)
 320     try:
 321         document = xml.dom.minidom.parse(filename)
 322     except IOError, e:
 323         raise FatalError(_('failed to parse %s: %s') % (filename, e))
 324
 325     assert document.documentElement.nodeName == 'moduleset'
 326     moduleset = ModuleSet(config = config)
 327     moduleset_name = document.documentElement.getAttribute('name')
 328     if not moduleset_name and uri.endswith('.modules'):
 329         moduleset_name = os.path.basename(uri)[:-len('.modules')]
 330
 331     # load up list of repositories
 332     repositories = {}
 333     default_repo = None
 334     for node in _child_elements_matching(
 335             document.documentElement, ['repository', 'cvsroot', 'svnroot',
 336                                        'arch-archive']):
 337         name = node.getAttribute('name')
 338         if node.getAttribute('default') == 'yes':
 339             default_repo = name
 340         if node.nodeName == 'repository':
 341             repo_type = node.getAttribute('type')
 342             repo_class = get_repo_type(repo_type)
 343             kws = {}
 344             for attr in repo_class.init_xml_attrs:
 345                 if node.hasAttribute(attr):
 346                     kws[attr.replace('-', '_')] = node.getAttribute(attr)
 347             repositories[name] = repo_class(config, name, **kws)
 348             repositories[name].moduleset_uri = uri
 349             mirrors = {}
 350             for mirror in _child_elements_matching(node, ['mirror']):
 351                 mirror_type = mirror.getAttribute('type')
 352                 mirror_class = get_repo_type(mirror_type)
 353                 kws = {}
 354                 for attr in mirror_class.init_xml_attrs:
 355                     if mirror.hasAttribute(attr):
 356                         kws[attr.replace('-','_')] = mirror.getAttribute(attr)
 357                 mirrors[mirror_type] = mirror_class(config, name, **kws)
 358                 #mirrors[mirror_type].moduleset_uri = uri
 359             setattr(repositories[name], "mirrors", mirrors)
 360         if node.nodeName == 'cvsroot':
 361             cvsroot = node.getAttribute('root')
 362             if node.hasAttribute('password'):
 363                 password = node.getAttribute('password')
 364             else:
 365                 password = None
 366             repo_type = get_repo_type('cvs')
 367             repositories[name] = repo_type(config, name,
 368                                            cvsroot=cvsroot, password=password)
 369         elif node.nodeName == 'svnroot':
 370             svnroot = node.getAttribute('href')
 371             repo_type = get_repo_type('svn')
 372             repositories[name] = repo_type(config, name, href=svnroot)
 373         elif node.nodeName == 'arch-archive':
 374             archive_uri = node.getAttribute('href')
 375             repo_type = get_repo_type('arch')
 376             repositories[name] = repo_type(config, name,
 377                                            archive=name, href=archive_uri)
 378
 379     # and now module definitions
 380     for node in _child_elements(document.documentElement):
 381         if node.nodeName == 'include':
 382             href = node.getAttribute('href')
 383             inc_uri = urlparse.urljoin(uri, href)
 384             try:
 385                 inc_moduleset = _parse_module_set(config, inc_uri)
 386             except FatalError, e:
 387                 if inc_uri[0] == '/':
 388                     raise e
 389                 # look up in local modulesets
 390                 inc_uri = os.path.join(os.path.dirname(__file__), '..', 'modulesets',
 391                                    href)
 392                 inc_moduleset = _parse_module_set(config, inc_uri)
 393
 394             moduleset.modules.update(inc_moduleset.modules)
 395         elif node.nodeName in ['repository', 'cvsroot', 'svnroot',
 396                                'arch-archive']:
 397             pass
 398         else:
 399             module = modtypes.parse_xml_node(node, config, uri,
 400                     repositories, default_repo)
 401             if moduleset_name:
 402                 module.tags.append(moduleset_name)
 403             module.moduleset_name = moduleset_name
 404             module.config = config
 405             moduleset.add(module)
 406
 407     return moduleset
 408
 409 def warn_local_modulesets(config):
 410     if config.use_local_modulesets:
 411         return
 412
 413     moduleset_local_path = os.path.join(SRCDIR, 'modulesets')
 414     if not os.path.exists(moduleset_local_path):
 415         # moduleset-less checkout
 416         return
 417
 418     if not os.path.exists(os.path.join(moduleset_local_path, '..', '.git')):
 419         # checkout was not done via git
 420         return
 421
 422     if type(config.moduleset) == type([]):
 423         modulesets = config.moduleset
 424     else:
 425         modulesets = [ config.moduleset ]
 426
 427     if not [x for x in modulesets if x.find('/') == -1]:
 428         # all modulesets have a slash; they are URI
 429         return
 430
 431     try:
 432         git_diff = get_output(['git', 'diff', 'origin/master', '--', '.'],
 433                 cwd=moduleset_local_path).strip()
 434     except CommandError:
 435         # git error, ignore
 436         return
 437
 438     if not git_diff:
 439         # no locally modified moduleset
 440         return
 441
 442     logging.info(
 443             _('Modulesets were edited locally but JHBuild is configured '\
 444               'to get them from the network, perhaps you need to add '\
 445               'use_local_modulesets = True to your .jhbuildrc.'))
 446