1 # jhbuild - a build script for GNOME 1.x and 2.x
2 # Copyright (C) 2001-2006 James Henstridge
4 # moduleset.py: logic for running the build.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 from __future__
import generators
27 from jhbuild
.errors
import UsageError
, FatalError
, DependencyCycleError
, CommandError
30 import xml
.dom
.minidom
32 raise FatalError(_('Python xml packages are required but could not be found'))
34 from jhbuild
import modtypes
35 from jhbuild
.versioncontrol
import get_repo_type
36 from jhbuild
.utils
import httpcache
37 from jhbuild
.utils
.cmds
import get_output
38 from jhbuild
.modtypes
.testmodule
import TestModule
40 __all__
= ['load', 'load_tests']
43 def __init__(self
, config
= None):
46 def add(self
, module
):
47 '''add a Module object to this set of modules'''
48 self
.modules
[module
.name
] = module
50 def get_module(self
, module_name
, ignore_case
= False):
51 if self
.modules
.has_key(module_name
) or not ignore_case
:
52 return self
.modules
[module_name
]
53 module_name_lower
= module_name
.lower()
54 for module
in self
.modules
.keys():
55 if module
.lower() == module_name_lower
:
56 if self
.config
is None or not self
.config
.quiet_mode
:
57 logging
.info(_('fixed case of module \'%(orig)s\' to \'%(new)s\'') % {
58 'orig': module_name
, 'new': module
})
59 return self
.modules
[module
]
60 print "Couldn't find the specified module: %s" % module_name
63 def get_module_list(self
, seed
, skip
=[], tags
=[], ignore_cycles
=False,
64 ignore_suggests
=False, include_optional_modules
=False,
65 ignore_missing
=False):
66 '''gets a list of module objects (in correct dependency order)
67 needed to build the modules in the seed list'''
69 if seed
== 'all': seed
= self
.modules
.keys()
71 all_modules
= [self
.get_module(mod
, ignore_case
= True) for mod
in seed
if mod
not in skip
]
73 raise UsageError(_('module "%s" not found') % e
)
75 asked_modules
= all_modules
[:]
77 # 1st: get all modules that will be needed
78 # note this is only needed to skip "after" modules that would not
81 while i
< len(all_modules
):
82 for modname
in all_modules
[i
].dependencies
:
83 depmod
= self
.modules
.get(modname
)
85 if not ignore_missing
:
87 '%(module)s has a dependency on unknown "%(invalid)s" module') % {
88 'module': all_modules
[i
].name
,
92 if not depmod
in all_modules
:
93 all_modules
.append(depmod
)
95 if not ignore_suggests
:
96 # suggests can be ignored if not in moduleset
97 for modname
in all_modules
[i
].suggests
:
98 depmod
= self
.modules
.get(modname
)
101 if not depmod
in all_modules
:
102 all_modules
.append(depmod
)
105 # 2nd: order them, raise an exception on hard dependency cycle, ignore
106 # them for soft dependencies
111 # mark skipped modules as already processed
112 self
._state
[self
.modules
.get(modname
)] = 'processed'
115 for modname
in self
.modules
:
117 if tag
in self
.modules
[modname
].tags
:
120 # no tag matched, mark module as processed
121 self
._state
[self
.modules
[modname
]] = 'processed'
123 def order(modules
, module
, mode
= 'dependencies'):
124 if self
._state
.get(module
, 'clean') == 'processed':
127 if self
._state
.get(module
, 'clean') == 'in-progress':
128 # dependency circle, abort when processing hard dependencies
129 if not ignore_cycles
:
130 raise DependencyCycleError()
132 self
._state
[module
] = 'in-progress'
134 self
._state
[module
] = 'in-progress'
135 for modname
in module
.dependencies
:
136 depmod
= self
.modules
[modname
]
137 order([self
.modules
[x
] for x
in depmod
.dependencies
], depmod
, 'dependencies')
138 if not ignore_suggests
:
139 for modname
in module
.suggests
:
140 depmod
= self
.modules
.get(modname
)
143 save_state
, save_ordered
= self
._state
.copy(), self
._ordered
[:]
145 order([self
.modules
[x
] for x
in depmod
.dependencies
], depmod
, 'suggests')
146 except DependencyCycleError
:
147 self
._state
, self
._ordered
= save_state
, save_ordered
150 for modname
in module
.after
:
151 depmod
= self
.modules
.get(modname
)
153 # this module doesn't exist, skip.
155 if not depmod
in all_modules
and not include_optional_modules
:
156 # skip modules that would not be built otherwise
157 # (build_optional_modules being the argument to force them
158 # to be included nevertheless)
160 if not depmod
.dependencies
:
161 # depmod itself has no dependencies, skip.
164 # more expensive, if depmod has dependencies, compute its
165 # full list of hard dependencies, getting it into
166 # extra_afters, so they are also evaluated.
167 # <http://bugzilla.gnome.org/show_bug.cgi?id=546640>
168 t_ms
= ModuleSet(self
.config
)
169 t_ms
.modules
= self
.modules
.copy()
170 dep_modules
= t_ms
.get_module_list(seed
=[depmod
.name
])
171 for m
in dep_modules
[:-1]:
173 extra_afters
.append(m
)
175 save_state
, save_ordered
= self
._state
.copy(), self
._ordered
[:]
177 order([self
.modules
[x
] for x
in depmod
.dependencies
], depmod
, 'after')
178 except DependencyCycleError
:
179 self
._state
, self
._ordered
= save_state
, save_ordered
180 for depmod
in extra_afters
:
181 save_state
, save_ordered
= self
._state
.copy(), self
._ordered
[:]
183 order([self
.modules
[x
] for x
in depmod
.dependencies
], depmod
, 'after')
184 except DependencyCycleError
:
185 self
._state
, self
._ordered
= save_state
, save_ordered
186 self
._state
[module
] = 'processed'
187 self
._ordered
.append(module
)
189 for i
, module
in enumerate(all_modules
):
191 if i
+1 == len(asked_modules
):
194 ordered
= self
._ordered
[:]
199 def get_full_module_list(self
, skip
=[], ignore_cycles
=False):
200 return self
.get_module_list(self
.modules
.keys(), skip
=skip
,
201 ignore_cycles
=ignore_cycles
, ignore_missing
=True)
203 def get_test_module_list (self
, seed
, skip
=[]):
207 for mod
in self
.modules
.values():
208 for test_app
in seed
:
209 if test_app
in mod
.tested_pkgs
:
210 test_modules
.append(mod
)
213 def write_dot(self
, modules
=None, fp
=sys
.stdout
, suggests
=False, clusters
=False):
214 from jhbuild
.modtypes
import MetaModule
215 from jhbuild
.modtypes
.autotools
import AutogenModule
216 from jhbuild
.versioncontrol
.tarball
import TarballBranch
219 modules
= self
.modules
.keys()
221 for module
in modules
:
222 inlist
[module
] = None
224 fp
.write('digraph "G" {\n'
230 mod
= self
.modules
[modname
]
232 logging
.warning(_('Unknown module:') + ' '+ modname
)
235 if isinstance(mod
, MetaModule
):
236 attrs
= '[color="lightcoral",style="filled",' \
237 'label="%s"]' % mod
.name
240 color
= 'lightskyblue'
241 if mod
.branch
.branchname
:
242 label
+= '\\n(%s)' % mod
.branch
.branchname
243 if isinstance(mod
.branch
, TarballBranch
):
244 color
= 'lightgoldenrod'
245 attrs
= '[color="%s",style="filled",label="%s"]' % (color
, label
)
246 fp
.write(' "%s" %s;\n' % (modname
, attrs
))
249 for dep
in self
.modules
[modname
].dependencies
:
250 fp
.write(' "%s" -> "%s";\n' % (modname
, dep
))
251 if not inlist
.has_key(dep
):
256 for dep
in self
.modules
[modname
].after
+ self
.modules
[modname
].suggests
:
257 if self
.modules
.has_key(dep
):
258 fp
.write(' "%s" -> "%s" [style=dotted];\n' % (modname
, dep
))
259 if not inlist
.has_key(dep
):
264 # create clusters for MetaModules
265 for modname
in inlist
.keys():
266 mod
= self
.modules
.get(modname
)
267 if isinstance(mod
, MetaModule
):
268 fp
.write(' subgraph "cluster_%s" {\n' % mod
.name
)
269 fp
.write(' label="%s";\n' % mod
.name
)
270 fp
.write(' style="filled";bgcolor="honeydew2";\n')
272 for dep
in mod
.dependencies
:
273 fp
.write(' "%s";\n' % dep
)
278 def load(config
, uri
=None):
281 elif type(config
.moduleset
) in (list, tuple):
282 modulesets
= config
.moduleset
284 modulesets
= [ config
.moduleset
]
285 ms
= ModuleSet(config
= config
)
286 for uri
in modulesets
:
287 if '/' not in uri
and not os
.path
.exists(uri
):
288 if config
.modulesets_dir
and config
.nonetwork
or config
.use_local_modulesets
:
289 uri
= os
.path
.join(config
.modulesets_dir
, uri
+ '.modules')
291 uri
= 'http://git.gnome.org/browse/jhbuild/plain/modulesets/%s.modules' % uri
293 ms
.modules
.update(_parse_module_set(config
, uri
).modules
)
294 except xml
.parsers
.expat
.ExpatError
, e
:
295 raise FatalError(_('failed to parse %s: %s') % (uri
, e
))
298 def load_tests (config
, uri
=None):
299 ms
= load (config
, uri
)
300 ms_tests
= ModuleSet(config
= config
)
301 for app
, module
in ms
.modules
.iteritems():
302 if module
.__class
__ == TestModule
:
303 ms_tests
.modules
[app
] = module
306 def _child_elements(parent
):
307 for node
in parent
.childNodes
:
308 if node
.nodeType
== node
.ELEMENT_NODE
:
311 def _child_elements_matching(parent
, names
):
312 for node
in parent
.childNodes
:
313 if node
.nodeType
== node
.ELEMENT_NODE
and node
.nodeName
in names
:
316 def _parse_module_set(config
, uri
):
318 filename
= httpcache
.load(uri
, nonetwork
=config
.nonetwork
, age
=0)
320 raise FatalError(_('could not download %s: %s') % (uri
, e
))
321 filename
= os
.path
.normpath(filename
)
323 document
= xml
.dom
.minidom
.parse(filename
)
325 raise FatalError(_('failed to parse %s: %s') % (filename
, e
))
327 assert document
.documentElement
.nodeName
== 'moduleset'
328 moduleset
= ModuleSet(config
= config
)
329 moduleset_name
= document
.documentElement
.getAttribute('name')
330 if not moduleset_name
and uri
.endswith('.modules'):
331 moduleset_name
= os
.path
.basename(uri
)[:-len('.modules')]
333 # load up list of repositories
336 for node
in _child_elements_matching(
337 document
.documentElement
, ['repository', 'cvsroot', 'svnroot',
339 name
= node
.getAttribute('name')
340 if node
.getAttribute('default') == 'yes':
342 if node
.nodeName
== 'repository':
343 repo_type
= node
.getAttribute('type')
344 repo_class
= get_repo_type(repo_type
)
346 for attr
in repo_class
.init_xml_attrs
:
347 if node
.hasAttribute(attr
):
348 kws
[attr
.replace('-', '_')] = node
.getAttribute(attr
)
349 repositories
[name
] = repo_class(config
, name
, **kws
)
350 repositories
[name
].moduleset_uri
= uri
352 for mirror
in _child_elements_matching(node
, ['mirror']):
353 mirror_type
= mirror
.getAttribute('type')
354 mirror_class
= get_repo_type(mirror_type
)
356 for attr
in mirror_class
.init_xml_attrs
:
357 if mirror
.hasAttribute(attr
):
358 kws
[attr
.replace('-','_')] = mirror
.getAttribute(attr
)
359 mirrors
[mirror_type
] = mirror_class(config
, name
, **kws
)
360 #mirrors[mirror_type].moduleset_uri = uri
361 setattr(repositories
[name
], "mirrors", mirrors
)
362 if node
.nodeName
== 'cvsroot':
363 cvsroot
= node
.getAttribute('root')
364 if node
.hasAttribute('password'):
365 password
= node
.getAttribute('password')
368 repo_type
= get_repo_type('cvs')
369 repositories
[name
] = repo_type(config
, name
,
370 cvsroot
=cvsroot
, password
=password
)
371 elif node
.nodeName
== 'svnroot':
372 svnroot
= node
.getAttribute('href')
373 repo_type
= get_repo_type('svn')
374 repositories
[name
] = repo_type(config
, name
, href
=svnroot
)
375 elif node
.nodeName
== 'arch-archive':
376 archive_uri
= node
.getAttribute('href')
377 repo_type
= get_repo_type('arch')
378 repositories
[name
] = repo_type(config
, name
,
379 archive
=name
, href
=archive_uri
)
381 # and now module definitions
382 for node
in _child_elements(document
.documentElement
):
383 if node
.nodeName
== 'include':
384 href
= node
.getAttribute('href')
385 inc_uri
= urlparse
.urljoin(uri
, href
)
387 inc_moduleset
= _parse_module_set(config
, inc_uri
)
388 except FatalError
, e
:
389 if inc_uri
[0] == '/':
391 # look up in local modulesets
392 inc_uri
= os
.path
.join(os
.path
.dirname(__file__
), '..', 'modulesets',
394 inc_moduleset
= _parse_module_set(config
, inc_uri
)
396 moduleset
.modules
.update(inc_moduleset
.modules
)
397 elif node
.nodeName
in ['repository', 'cvsroot', 'svnroot',
401 module
= modtypes
.parse_xml_node(node
, config
, uri
,
402 repositories
, default_repo
)
404 module
.tags
.append(moduleset_name
)
405 module
.moduleset_name
= moduleset_name
406 module
.config
= config
407 moduleset
.add(module
)
411 def warn_local_modulesets(config
):
412 if config
.use_local_modulesets
:
415 moduleset_local_path
= os
.path
.join(SRCDIR
, 'modulesets')
416 if not os
.path
.exists(moduleset_local_path
):
417 # moduleset-less checkout
420 if not os
.path
.exists(os
.path
.join(moduleset_local_path
, '..', '.git')):
421 # checkout was not done via git
424 if type(config
.moduleset
) == type([]):
425 modulesets
= config
.moduleset
427 modulesets
= [ config
.moduleset
]
429 if not [x
for x
in modulesets
if x
.find('/') == -1]:
430 # all modulesets have a slash; they are URI
434 git_diff
= get_output(['git', 'diff', 'origin/master', '--', '.'],
435 cwd
=moduleset_local_path
).strip()
441 # no locally modified moduleset
445 _('Modulesets were edited locally but JHBuild is configured '\
446 'to get them from the network, perhaps you need to add '\
447 'use_local_modulesets = True to your .jhbuildrc.'))