Convert mtime from a time_t to a struct timespec.
[Samba.git] / buildtools / wafsamba / samba_deps.py
blobadeb3645ce16b58815463eebe08eed958be133d0
1 # Samba automatic dependency handling and project rules
3 import Build, os, sys, re, Environment, Logs, time
4 from samba_utils import *
5 from samba_autoconf import *
6 from samba_bundled import BUILTIN_LIBRARY
8 @conf
9 def ADD_GLOBAL_DEPENDENCY(ctx, dep):
10 '''add a dependency for all binaries and libraries'''
11 if not 'GLOBAL_DEPENDENCIES' in ctx.env:
12 ctx.env.GLOBAL_DEPENDENCIES = []
13 ctx.env.GLOBAL_DEPENDENCIES.append(dep)
16 @conf
17 def BREAK_CIRCULAR_LIBRARY_DEPENDENCIES(ctx):
18 '''indicate that circular dependencies between libraries should be broken.'''
19 ctx.env.ALLOW_CIRCULAR_LIB_DEPENDENCIES = True
22 @conf
23 def SET_SYSLIB_DEPS(conf, target, deps):
24 '''setup some implied dependencies for a SYSLIB'''
25 cache = LOCAL_CACHE(conf, 'SYSLIB_DEPS')
26 cache[target] = deps
29 def expand_subsystem_deps(bld):
30 '''expand the reverse dependencies resulting from subsystem
31 attributes of modules. This is walking over the complete list
32 of declared subsystems, and expands the samba_deps_extended list for any
33 module<->subsystem dependencies'''
35 subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
36 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
38 for subsystem_name in subsystem_list:
39 bld.ASSERT(subsystem_name in targets, "Subsystem target %s not declared" % subsystem_name)
40 type = targets[subsystem_name]
41 if type == 'DISABLED' or type == 'EMPTY':
42 continue
44 # for example,
45 # subsystem_name = dcerpc_server (a subsystem)
46 # subsystem = dcerpc_server (a subsystem object)
47 # module_name = rpc_epmapper (a module within the dcerpc_server subsystem)
48 # module = rpc_epmapper (a module object within the dcerpc_server subsystem)
50 subsystem = bld.name_to_obj(subsystem_name, bld.env)
51 bld.ASSERT(subsystem is not None, "Unable to find subsystem %s" % subsystem_name)
52 for d in subsystem_list[subsystem_name]:
53 module_name = d['TARGET']
54 module_type = targets[module_name]
55 if module_type in ['DISABLED', 'EMPTY']:
56 continue
57 bld.ASSERT(subsystem is not None,
58 "Subsystem target %s for %s (%s) not found" % (subsystem_name, module_name, module_type))
59 if module_type in ['SUBSYSTEM']:
60 # if a module is a plain object type (not a library) then the
61 # subsystem it is part of needs to have it as a dependency, so targets
62 # that depend on this subsystem get the modules of that subsystem
63 subsystem.samba_deps_extended.append(module_name)
64 subsystem.samba_deps_extended = unique_list(subsystem.samba_deps_extended)
68 def build_dependencies(self):
69 '''This builds the dependency list for a target. It runs after all the targets are declared
71 The reason this is not just done in the SAMBA_*() rules is that we have no way of knowing
72 the full dependency list for a target until we have all of the targets declared.
73 '''
75 if self.samba_type in ['LIBRARY', 'BINARY', 'PYTHON']:
76 self.uselib = list(self.final_syslibs)
77 self.uselib_local = list(self.final_libs)
78 self.add_objects = list(self.final_objects)
80 # extra link flags from pkg_config
81 libs = self.final_syslibs.copy()
83 (ccflags, ldflags) = library_flags(self, list(libs))
84 new_ldflags = getattr(self, 'samba_ldflags', [])[:]
85 new_ldflags.extend(ldflags)
86 self.ldflags = new_ldflags
88 if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ldflags:
89 for f in self.env.undefined_ldflags:
90 self.ldflags.remove(f)
92 debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
93 self.sname, self.uselib, self.uselib_local, self.add_objects)
95 if self.samba_type in ['SUBSYSTEM']:
96 # this is needed for the ccflags of libs that come from pkg_config
97 self.uselib = list(self.final_syslibs)
98 self.uselib.extend(list(self.direct_syslibs))
99 for lib in self.final_libs:
100 t = self.bld.name_to_obj(lib, self.bld.env)
101 self.uselib.extend(list(t.final_syslibs))
102 self.uselib = unique_list(self.uselib)
104 if getattr(self, 'uselib', None):
105 up_list = []
106 for l in self.uselib:
107 up_list.append(l.upper())
108 self.uselib = up_list
111 def build_includes(self):
112 '''This builds the right set of includes for a target.
114 One tricky part of this is that the includes= attribute for a
115 target needs to use paths which are relative to that targets
116 declaration directory (which we can get at via t.path).
118 The way this works is the includes list gets added as
119 samba_includes in the main build task declaration. Then this
120 function runs after all of the tasks are declared, and it
121 processes the samba_includes attribute to produce a includes=
122 attribute
125 if getattr(self, 'samba_includes', None) is None:
126 return
128 bld = self.bld
130 inc_deps = includes_objects(bld, self, set(), {})
132 includes = []
134 # maybe add local includes
135 if getattr(self, 'local_include', True) == True and getattr(self, 'local_include_first', True):
136 includes.append('.')
138 includes.extend(self.samba_includes_extended)
140 if 'EXTRA_INCLUDES' in bld.env and getattr(self, 'global_include', True):
141 includes.extend(bld.env['EXTRA_INCLUDES'])
143 includes.append('#')
145 inc_set = set()
146 inc_abs = []
148 for d in inc_deps:
149 t = bld.name_to_obj(d, bld.env)
150 bld.ASSERT(t is not None, "Unable to find dependency %s for %s" % (d, self.sname))
151 inclist = getattr(t, 'samba_includes_extended', [])[:]
152 if getattr(t, 'local_include', True) == True:
153 inclist.append('.')
154 if inclist == []:
155 continue
156 tpath = t.samba_abspath
157 for inc in inclist:
158 npath = tpath + '/' + inc
159 if not npath in inc_set:
160 inc_abs.append(npath)
161 inc_set.add(npath)
163 mypath = self.path.abspath(bld.env)
164 for inc in inc_abs:
165 relpath = os_path_relpath(inc, mypath)
166 includes.append(relpath)
168 if getattr(self, 'local_include', True) == True and not getattr(self, 'local_include_first', True):
169 includes.append('.')
171 # now transform the includes list to be relative to the top directory
172 # which is represented by '#' in waf. This allows waf to cache the
173 # includes lists more efficiently
174 includes_top = []
175 for i in includes:
176 if i[0] == '#':
177 # some are already top based
178 includes_top.append(i)
179 continue
180 absinc = os.path.join(self.path.abspath(), i)
181 relinc = os_path_relpath(absinc, self.bld.srcnode.abspath())
182 includes_top.append('#' + relinc)
184 self.includes = unique_list(includes_top)
185 debug('deps: includes for target %s: includes=%s',
186 self.sname, self.includes)
191 def add_init_functions(self):
192 '''This builds the right set of init functions'''
194 bld = self.bld
196 subsystems = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
198 # cope with the separated object lists from BINARY and LIBRARY targets
199 sname = self.sname
200 if sname.endswith('.objlist'):
201 sname = sname[0:-8]
203 modules = []
204 if sname in subsystems:
205 modules.append(sname)
207 m = getattr(self, 'samba_modules', None)
208 if m is not None:
209 modules.extend(TO_LIST(m))
211 m = getattr(self, 'samba_subsystem', None)
212 if m is not None:
213 modules.append(m)
215 sentinal = getattr(self, 'init_function_sentinal', 'NULL')
217 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
218 cflags = getattr(self, 'samba_cflags', [])[:]
220 if modules == []:
221 sname = sname.replace('-','_')
222 sname = sname.replace('/','_')
223 cflags.append('-DSTATIC_%s_MODULES=%s' % (sname, sentinal))
224 if sentinal == 'NULL':
225 cflags.append('-DSTATIC_%s_MODULES_PROTO' % sname)
226 self.ccflags = cflags
227 return
229 for m in modules:
230 bld.ASSERT(m in subsystems,
231 "No init_function defined for module '%s' in target '%s'" % (m, self.sname))
232 init_fn_list = []
233 for d in subsystems[m]:
234 if targets[d['TARGET']] != 'DISABLED':
235 init_fn_list.append(d['INIT_FUNCTION'])
236 if init_fn_list == []:
237 cflags.append('-DSTATIC_%s_MODULES=%s' % (m, sentinal))
238 if sentinal == 'NULL':
239 cflags.append('-DSTATIC_%s_MODULES_PROTO' % m)
240 else:
241 cflags.append('-DSTATIC_%s_MODULES=%s' % (m, ','.join(init_fn_list) + ',' + sentinal))
242 proto=''
243 for f in init_fn_list:
244 proto = proto + '_MODULE_PROTO(%s)' % f
245 cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
246 self.ccflags = cflags
250 def check_duplicate_sources(bld, tgt_list):
251 '''see if we are compiling the same source file more than once
252 without an allow_duplicates attribute'''
254 debug('deps: checking for duplicate sources')
256 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
257 ret = True
259 global tstart
261 for t in tgt_list:
262 source_list = TO_LIST(getattr(t, 'source', ''))
263 tpath = os.path.normpath(os_path_relpath(t.path.abspath(bld.env), t.env.BUILD_DIRECTORY + '/default'))
264 obj_sources = set()
265 for s in source_list:
266 p = os.path.normpath(os.path.join(tpath, s))
267 if p in obj_sources:
268 Logs.error("ERROR: source %s appears twice in target '%s'" % (p, t.sname))
269 sys.exit(1)
270 obj_sources.add(p)
271 t.samba_source_set = obj_sources
273 subsystems = {}
275 # build a list of targets that each source file is part of
276 for t in tgt_list:
277 sources = []
278 if not targets[t.sname] in [ 'LIBRARY', 'BINARY', 'PYTHON' ]:
279 continue
280 for obj in t.add_objects:
281 t2 = t.bld.name_to_obj(obj, bld.env)
282 source_set = getattr(t2, 'samba_source_set', set())
283 for s in source_set:
284 if not s in subsystems:
285 subsystems[s] = {}
286 if not t.sname in subsystems[s]:
287 subsystems[s][t.sname] = []
288 subsystems[s][t.sname].append(t2.sname)
290 for s in subsystems:
291 if len(subsystems[s]) > 1 and Options.options.SHOW_DUPLICATES:
292 Logs.warn("WARNING: source %s is in more than one target: %s" % (s, subsystems[s].keys()))
293 for tname in subsystems[s]:
294 if len(subsystems[s][tname]) > 1:
295 raise Utils.WafError("ERROR: source %s is in more than one subsystem of target '%s': %s" % (s, tname, subsystems[s][tname]))
297 return ret
300 def check_orpaned_targets(bld, tgt_list):
301 '''check if any build targets are orphaned'''
303 target_dict = LOCAL_CACHE(bld, 'TARGET_TYPE')
305 debug('deps: checking for orphaned targets')
307 for t in tgt_list:
308 if getattr(t, 'samba_used', False) == True:
309 continue
310 type = target_dict[t.sname]
311 if not type in ['BINARY', 'LIBRARY', 'MODULE', 'ET', 'PYTHON']:
312 if re.search('^PIDL_', t.sname) is None:
313 Logs.warn("Target %s of type %s is unused by any other target" % (t.sname, type))
316 def check_group_ordering(bld, tgt_list):
317 '''see if we have any dependencies that violate the group ordering
319 It is an error for a target to depend on a target from a later
320 build group
323 def group_name(g):
324 tm = bld.task_manager
325 return [x for x in tm.groups_names if id(tm.groups_names[x]) == id(g)][0]
327 for g in bld.task_manager.groups:
328 gname = group_name(g)
329 for t in g.tasks_gen:
330 t.samba_group = gname
332 grp_map = {}
333 idx = 0
334 for g in bld.task_manager.groups:
335 name = group_name(g)
336 grp_map[name] = idx
337 idx += 1
339 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
341 ret = True
342 for t in tgt_list:
343 tdeps = getattr(t, 'add_objects', []) + getattr(t, 'uselib_local', [])
344 for d in tdeps:
345 t2 = bld.name_to_obj(d, bld.env)
346 if t2 is None:
347 continue
348 map1 = grp_map[t.samba_group]
349 map2 = grp_map[t2.samba_group]
351 if map2 > map1:
352 Logs.error("Target %r in build group %r depends on target %r from later build group %r" % (
353 t.sname, t.samba_group, t2.sname, t2.samba_group))
354 ret = False
356 return ret
359 def show_final_deps(bld, tgt_list):
360 '''show the final dependencies for all targets'''
362 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
364 for t in tgt_list:
365 if not targets[t.sname] in ['LIBRARY', 'BINARY', 'PYTHON', 'SUBSYSTEM']:
366 continue
367 debug('deps: final dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
368 t.sname, t.uselib, getattr(t, 'uselib_local', []), getattr(t, 'add_objects', []))
371 def add_samba_attributes(bld, tgt_list):
372 '''ensure a target has a the required samba attributes'''
374 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
376 for t in tgt_list:
377 if t.name != '':
378 t.sname = t.name
379 else:
380 t.sname = t.target
381 t.samba_type = targets[t.sname]
382 t.samba_abspath = t.path.abspath(bld.env)
383 t.samba_deps_extended = t.samba_deps[:]
384 t.samba_includes_extended = TO_LIST(t.samba_includes)[:]
385 t.ccflags = getattr(t, 'samba_cflags', '')
387 def replace_grouping_libraries(bld, tgt_list):
388 '''replace dependencies based on grouping libraries
390 If a library is marked as a grouping library, then any target that
391 depends on a subsystem that is part of that grouping library gets
392 that dependency replaced with a dependency on the grouping library
395 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
397 grouping = {}
399 # find our list of grouping libraries, mapped from the subsystems they depend on
400 for t in tgt_list:
401 if not getattr(t, 'grouping_library', False):
402 continue
403 for dep in t.samba_deps_extended:
404 bld.ASSERT(dep in targets, "grouping library target %s not declared in %s" % (dep, t.sname))
405 if targets[dep] == 'SUBSYSTEM':
406 grouping[dep] = t.sname
408 # now replace any dependencies on elements of grouping libraries
409 for t in tgt_list:
410 for i in range(len(t.samba_deps_extended)):
411 dep = t.samba_deps_extended[i]
412 if dep in grouping:
413 if t.sname != grouping[dep]:
414 debug("deps: target %s: replacing dependency %s with grouping library %s" % (t.sname, dep, grouping[dep]))
415 t.samba_deps_extended[i] = grouping[dep]
419 def build_direct_deps(bld, tgt_list):
420 '''build the direct_objects and direct_libs sets for each target'''
422 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
423 syslib_deps = LOCAL_CACHE(bld, 'SYSLIB_DEPS')
425 global_deps = bld.env.GLOBAL_DEPENDENCIES
426 global_deps_exclude = set()
427 for dep in global_deps:
428 t = bld.name_to_obj(dep, bld.env)
429 for d in t.samba_deps:
430 # prevent loops from the global dependencies list
431 global_deps_exclude.add(d)
432 global_deps_exclude.add(d + '.objlist')
434 for t in tgt_list:
435 t.direct_objects = set()
436 t.direct_libs = set()
437 t.direct_syslibs = set()
438 deps = t.samba_deps_extended[:]
439 if getattr(t, 'samba_use_global_deps', False) and not t.sname in global_deps_exclude:
440 deps.extend(global_deps)
441 for d in deps:
442 if d == t.sname: continue
443 if not d in targets:
444 Logs.error("Unknown dependency '%s' in '%s'" % (d, t.sname))
445 sys.exit(1)
446 if targets[d] in [ 'EMPTY', 'DISABLED' ]:
447 continue
448 if targets[d] == 'PYTHON' and targets[t.sname] != 'PYTHON' and t.sname.find('.objlist') == -1:
449 # this check should be more restrictive, but for now we have pidl-generated python
450 # code that directly depends on other python modules
451 Logs.error('ERROR: Target %s has dependency on python module %s' % (t.sname, d))
452 sys.exit(1)
453 if targets[d] == 'SYSLIB':
454 t.direct_syslibs.add(d)
455 if d in syslib_deps:
456 for implied in TO_LIST(syslib_deps[d]):
457 if BUILTIN_LIBRARY(bld, implied):
458 t.direct_objects.add(implied)
459 elif targets[implied] == 'SYSLIB':
460 t.direct_syslibs.add(implied)
461 elif targets[implied] in ['LIBRARY', 'MODULE']:
462 t.direct_libs.add(implied)
463 else:
464 Logs.error('Implied dependency %s in %s is of type %s' % (
465 implied, t.sname, targets[implied]))
466 sys.exit(1)
467 continue
468 t2 = bld.name_to_obj(d, bld.env)
469 if t2 is None:
470 Logs.error("no task %s of type %s in %s" % (d, targets[d], t.sname))
471 sys.exit(1)
472 if t2.samba_type in [ 'LIBRARY', 'MODULE' ]:
473 t.direct_libs.add(d)
474 elif t2.samba_type in [ 'SUBSYSTEM', 'ASN1', 'PYTHON' ]:
475 t.direct_objects.add(d)
476 debug('deps: built direct dependencies')
479 def dependency_loop(loops, t, target):
480 '''add a dependency loop to the loops dictionary'''
481 if t.sname == target:
482 return
483 if not target in loops:
484 loops[target] = set()
485 if not t.sname in loops[target]:
486 loops[target].add(t.sname)
489 def indirect_libs(bld, t, chain, loops):
490 '''recursively calculate the indirect library dependencies for a target
492 An indirect library is a library that results from a dependency on
493 a subsystem
496 ret = getattr(t, 'indirect_libs', None)
497 if ret is not None:
498 return ret
500 ret = set()
501 for obj in t.direct_objects:
502 if obj in chain:
503 dependency_loop(loops, t, obj)
504 continue
505 chain.add(obj)
506 t2 = bld.name_to_obj(obj, bld.env)
507 r2 = indirect_libs(bld, t2, chain, loops)
508 chain.remove(obj)
509 ret = ret.union(t2.direct_libs)
510 ret = ret.union(r2)
512 for obj in indirect_objects(bld, t, set(), loops):
513 if obj in chain:
514 dependency_loop(loops, t, obj)
515 continue
516 chain.add(obj)
517 t2 = bld.name_to_obj(obj, bld.env)
518 r2 = indirect_libs(bld, t2, chain, loops)
519 chain.remove(obj)
520 ret = ret.union(t2.direct_libs)
521 ret = ret.union(r2)
523 t.indirect_libs = ret
525 return ret
528 def indirect_objects(bld, t, chain, loops):
529 '''recursively calculate the indirect object dependencies for a target
531 indirect objects are the set of objects from expanding the
532 subsystem dependencies
535 ret = getattr(t, 'indirect_objects', None)
536 if ret is not None: return ret
538 ret = set()
539 for lib in t.direct_objects:
540 if lib in chain:
541 dependency_loop(loops, t, lib)
542 continue
543 chain.add(lib)
544 t2 = bld.name_to_obj(lib, bld.env)
545 r2 = indirect_objects(bld, t2, chain, loops)
546 chain.remove(lib)
547 ret = ret.union(t2.direct_objects)
548 ret = ret.union(r2)
550 t.indirect_objects = ret
551 return ret
554 def extended_objects(bld, t, chain):
555 '''recursively calculate the extended object dependencies for a target
557 extended objects are the union of:
558 - direct objects
559 - indirect objects
560 - direct and indirect objects of all direct and indirect libraries
563 ret = getattr(t, 'extended_objects', None)
564 if ret is not None: return ret
566 ret = set()
567 ret = ret.union(t.final_objects)
569 for lib in t.final_libs:
570 if lib in chain:
571 continue
572 t2 = bld.name_to_obj(lib, bld.env)
573 chain.add(lib)
574 r2 = extended_objects(bld, t2, chain)
575 chain.remove(lib)
576 ret = ret.union(t2.final_objects)
577 ret = ret.union(r2)
579 t.extended_objects = ret
580 return ret
583 def includes_objects(bld, t, chain, inc_loops):
584 '''recursively calculate the includes object dependencies for a target
586 includes dependencies come from either library or object dependencies
588 ret = getattr(t, 'includes_objects', None)
589 if ret is not None:
590 return ret
592 ret = t.direct_objects.copy()
593 ret = ret.union(t.direct_libs)
595 for obj in t.direct_objects:
596 if obj in chain:
597 dependency_loop(inc_loops, t, obj)
598 continue
599 chain.add(obj)
600 t2 = bld.name_to_obj(obj, bld.env)
601 r2 = includes_objects(bld, t2, chain, inc_loops)
602 chain.remove(obj)
603 ret = ret.union(t2.direct_objects)
604 ret = ret.union(r2)
606 for lib in t.direct_libs:
607 if lib in chain:
608 dependency_loop(inc_loops, t, lib)
609 continue
610 chain.add(lib)
611 t2 = bld.name_to_obj(lib, bld.env)
612 if t2 is None:
613 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
614 Logs.error('Target %s of type %s not found in direct_libs for %s' % (
615 lib, targets[lib], t.sname))
616 sys.exit(1)
617 r2 = includes_objects(bld, t2, chain, inc_loops)
618 chain.remove(lib)
619 ret = ret.union(t2.direct_objects)
620 ret = ret.union(r2)
622 t.includes_objects = ret
623 return ret
626 def break_dependency_loops(bld, tgt_list):
627 '''find and break dependency loops'''
628 loops = {}
629 inc_loops = {}
631 # build up the list of loops
632 for t in tgt_list:
633 indirect_objects(bld, t, set(), loops)
634 indirect_libs(bld, t, set(), loops)
635 includes_objects(bld, t, set(), inc_loops)
637 # break the loops
638 for t in tgt_list:
639 if t.sname in loops:
640 for attr in ['direct_objects', 'indirect_objects', 'direct_libs', 'indirect_libs']:
641 objs = getattr(t, attr, set())
642 setattr(t, attr, objs.difference(loops[t.sname]))
644 for loop in loops:
645 debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
647 for loop in inc_loops:
648 debug('deps: Found include loops for target %s : %s', loop, inc_loops[loop])
650 # expand the loops mapping by one level
651 for loop in loops.copy():
652 for tgt in loops[loop]:
653 if tgt in loops:
654 loops[loop] = loops[loop].union(loops[tgt])
656 for loop in inc_loops.copy():
657 for tgt in inc_loops[loop]:
658 if tgt in inc_loops:
659 inc_loops[loop] = inc_loops[loop].union(inc_loops[tgt])
662 # expand indirect subsystem and library loops
663 for loop in loops.copy():
664 t = bld.name_to_obj(loop, bld.env)
665 if t.samba_type in ['SUBSYSTEM']:
666 loops[loop] = loops[loop].union(t.indirect_objects)
667 loops[loop] = loops[loop].union(t.direct_objects)
668 if t.samba_type in ['LIBRARY','PYTHON']:
669 loops[loop] = loops[loop].union(t.indirect_libs)
670 loops[loop] = loops[loop].union(t.direct_libs)
671 if loop in loops[loop]:
672 loops[loop].remove(loop)
674 # expand indirect includes loops
675 for loop in inc_loops.copy():
676 t = bld.name_to_obj(loop, bld.env)
677 inc_loops[loop] = inc_loops[loop].union(t.includes_objects)
678 if loop in inc_loops[loop]:
679 inc_loops[loop].remove(loop)
681 # add in the replacement dependencies
682 for t in tgt_list:
683 for loop in loops:
684 for attr in ['indirect_objects', 'indirect_libs']:
685 objs = getattr(t, attr, set())
686 if loop in objs:
687 diff = loops[loop].difference(objs)
688 if t.sname in diff:
689 diff.remove(t.sname)
690 if diff:
691 debug('deps: Expanded target %s of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
692 objs = objs.union(diff)
693 setattr(t, attr, objs)
695 for loop in inc_loops:
696 objs = getattr(t, 'includes_objects', set())
697 if loop in objs:
698 diff = inc_loops[loop].difference(objs)
699 if t.sname in diff:
700 diff.remove(t.sname)
701 if diff:
702 debug('deps: Expanded target %s includes of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
703 objs = objs.union(diff)
704 setattr(t, 'includes_objects', objs)
707 def reduce_objects(bld, tgt_list):
708 '''reduce objects by looking for indirect object dependencies'''
709 rely_on = {}
711 for t in tgt_list:
712 t.extended_objects = None
714 changed = False
716 for type in ['BINARY', 'PYTHON', 'LIBRARY']:
717 for t in tgt_list:
718 if t.samba_type != type: continue
719 # if we will indirectly link to a target then we don't need it
720 new = t.final_objects.copy()
721 for l in t.final_libs:
722 t2 = bld.name_to_obj(l, bld.env)
723 t2_obj = extended_objects(bld, t2, set())
724 dup = new.intersection(t2_obj)
725 if t.sname in rely_on:
726 dup = dup.difference(rely_on[t.sname])
727 if dup:
728 debug('deps: removing dups from %s of type %s: %s also in %s %s',
729 t.sname, t.samba_type, dup, t2.samba_type, l)
730 new = new.difference(dup)
731 changed = True
732 if not l in rely_on:
733 rely_on[l] = set()
734 rely_on[l] = rely_on[l].union(dup)
735 t.final_objects = new
737 if not changed:
738 return False
740 # add back in any objects that were relied upon by the reduction rules
741 for r in rely_on:
742 t = bld.name_to_obj(r, bld.env)
743 t.final_objects = t.final_objects.union(rely_on[r])
745 return True
748 def show_library_loop(bld, lib1, lib2, path, seen):
749 '''show the detailed path of a library loop between lib1 and lib2'''
751 t = bld.name_to_obj(lib1, bld.env)
752 if not lib2 in getattr(t, 'final_libs', set()):
753 return
755 for d in t.samba_deps_extended:
756 if d in seen:
757 continue
758 seen.add(d)
759 path2 = path + '=>' + d
760 if d == lib2:
761 Logs.warn('library loop path: ' + path2)
762 return
763 show_library_loop(bld, d, lib2, path2, seen)
764 seen.remove(d)
767 def calculate_final_deps(bld, tgt_list, loops):
768 '''calculate the final library and object dependencies'''
769 for t in tgt_list:
770 # start with the maximum possible list
771 t.final_libs = t.direct_libs.union(indirect_libs(bld, t, set(), loops))
772 t.final_objects = t.direct_objects.union(indirect_objects(bld, t, set(), loops))
774 for t in tgt_list:
775 # don't depend on ourselves
776 if t.sname in t.final_libs:
777 t.final_libs.remove(t.sname)
778 if t.sname in t.final_objects:
779 t.final_objects.remove(t.sname)
781 # handle any non-shared binaries
782 for t in tgt_list:
783 if t.samba_type == 'BINARY' and bld.NONSHARED_BINARY(t.sname):
784 subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
785 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
787 # replace lib deps with objlist deps
788 for l in t.final_libs:
789 objname = l + '.objlist'
790 t2 = bld.name_to_obj(objname, bld.env)
791 if t2 is None:
792 Logs.error('ERROR: subsystem %s not found' % objname)
793 sys.exit(1)
794 t.final_objects.add(objname)
795 t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
796 if l in subsystem_list:
797 # its a subsystem - we also need the contents of any modules
798 for d in subsystem_list[l]:
799 module_name = d['TARGET']
800 if targets[module_name] == 'LIBRARY':
801 objname = module_name + '.objlist'
802 elif targets[module_name] == 'SUBSYSTEM':
803 objname = module_name
804 else:
805 continue
806 t2 = bld.name_to_obj(objname, bld.env)
807 if t2 is None:
808 Logs.error('ERROR: subsystem %s not found' % objname)
809 sys.exit(1)
810 t.final_objects.add(objname)
811 t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
812 t.final_libs = set()
814 # find any library loops
815 for t in tgt_list:
816 if t.samba_type in ['LIBRARY', 'PYTHON']:
817 for l in t.final_libs.copy():
818 t2 = bld.name_to_obj(l, bld.env)
819 if t.sname in t2.final_libs:
820 if getattr(bld.env, "ALLOW_CIRCULAR_LIB_DEPENDENCIES", False):
821 # we could break this in either direction. If one of the libraries
822 # has a version number, and will this be distributed publicly, then
823 # we should make it the lower level library in the DAG
824 Logs.warn('deps: removing library loop %s from %s' % (t.sname, t2.sname))
825 dependency_loop(loops, t, t2.sname)
826 t2.final_libs.remove(t.sname)
827 else:
828 Logs.error('ERROR: circular library dependency between %s and %s'
829 % (t.sname, t2.sname))
830 show_library_loop(bld, t.sname, t2.sname, t.sname, set())
831 show_library_loop(bld, t2.sname, t.sname, t2.sname, set())
832 sys.exit(1)
834 for loop in loops:
835 debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
837 # we now need to make corrections for any library loops we broke up
838 # any target that depended on the target of the loop and doesn't
839 # depend on the source of the loop needs to get the loop source added
840 for type in ['BINARY','PYTHON','LIBRARY','BINARY']:
841 for t in tgt_list:
842 if t.samba_type != type: continue
843 for loop in loops:
844 if loop in t.final_libs:
845 diff = loops[loop].difference(t.final_libs)
846 if t.sname in diff:
847 diff.remove(t.sname)
848 if t.sname in diff:
849 diff.remove(t.sname)
850 # make sure we don't recreate the loop again!
851 for d in diff.copy():
852 t2 = bld.name_to_obj(d, bld.env)
853 if t2.samba_type == 'LIBRARY':
854 if t.sname in t2.final_libs:
855 debug('deps: removing expansion %s from %s', d, t.sname)
856 diff.remove(d)
857 if diff:
858 debug('deps: Expanded target %s by loop %s libraries (loop %s) %s', t.sname, loop,
859 loops[loop], diff)
860 t.final_libs = t.final_libs.union(diff)
862 # remove objects that are also available in linked libs
863 count = 0
864 while reduce_objects(bld, tgt_list):
865 count += 1
866 if count > 100:
867 Logs.warn("WARNING: Unable to remove all inter-target object duplicates")
868 break
869 debug('deps: Object reduction took %u iterations', count)
871 # add in any syslib dependencies
872 for t in tgt_list:
873 if not t.samba_type in ['BINARY','PYTHON','LIBRARY','SUBSYSTEM']:
874 continue
875 syslibs = set()
876 for d in t.final_objects:
877 t2 = bld.name_to_obj(d, bld.env)
878 syslibs = syslibs.union(t2.direct_syslibs)
879 # this adds the indirect syslibs as well, which may not be needed
880 # depending on the linker flags
881 for d in t.final_libs:
882 t2 = bld.name_to_obj(d, bld.env)
883 syslibs = syslibs.union(t2.direct_syslibs)
884 t.final_syslibs = syslibs
887 # find any unresolved library loops
888 lib_loop_error = False
889 for t in tgt_list:
890 if t.samba_type in ['LIBRARY', 'PYTHON']:
891 for l in t.final_libs.copy():
892 t2 = bld.name_to_obj(l, bld.env)
893 if t.sname in t2.final_libs:
894 Logs.error('ERROR: Unresolved library loop %s from %s' % (t.sname, t2.sname))
895 lib_loop_error = True
896 if lib_loop_error:
897 sys.exit(1)
899 debug('deps: removed duplicate dependencies')
902 def show_dependencies(bld, target, seen):
903 '''recursively show the dependencies of target'''
905 if target in seen:
906 return
908 t = bld.name_to_obj(target, bld.env)
909 if t is None:
910 Logs.error("ERROR: Unable to find target '%s'" % target)
911 sys.exit(1)
913 Logs.info('%s(OBJECTS): %s' % (target, t.direct_objects))
914 Logs.info('%s(LIBS): %s' % (target, t.direct_libs))
915 Logs.info('%s(SYSLIBS): %s' % (target, t.direct_syslibs))
917 seen.add(target)
919 for t2 in t.direct_objects:
920 show_dependencies(bld, t2, seen)
923 def show_object_duplicates(bld, tgt_list):
924 '''show a list of object files that are included in more than
925 one library or binary'''
927 targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
929 used_by = {}
931 Logs.info("showing duplicate objects")
933 for t in tgt_list:
934 if not targets[t.sname] in [ 'LIBRARY', 'PYTHON' ]:
935 continue
936 for n in getattr(t, 'final_objects', set()):
937 t2 = bld.name_to_obj(n, bld.env)
938 if not n in used_by:
939 used_by[n] = set()
940 used_by[n].add(t.sname)
942 for n in used_by:
943 if len(used_by[n]) > 1:
944 Logs.info("target '%s' is used by %s" % (n, used_by[n]))
946 Logs.info("showing indirect dependency counts (sorted by count)")
948 def indirect_count(t1, t2):
949 return len(t2.indirect_objects) - len(t1.indirect_objects)
951 sorted_list = sorted(tgt_list, cmp=indirect_count)
952 for t in sorted_list:
953 if len(t.indirect_objects) > 1:
954 Logs.info("%s depends on %u indirect objects" % (t.sname, len(t.indirect_objects)))
957 ######################################################################
958 # this provides a way to save our dependency calculations between runs
959 savedeps_version = 3
960 savedeps_inputs = ['samba_deps', 'samba_includes', 'local_include', 'local_include_first', 'samba_cflags',
961 'source', 'grouping_library', 'samba_ldflags', 'allow_undefined_symbols',
962 'use_global_deps', 'global_include' ]
963 savedeps_outputs = ['uselib', 'uselib_local', 'add_objects', 'includes', 'ccflags', 'ldflags', 'samba_deps_extended']
964 savedeps_outenv = ['INC_PATHS']
965 savedeps_envvars = ['NONSHARED_BINARIES', 'GLOBAL_DEPENDENCIES', 'EXTRA_CFLAGS', 'EXTRA_LDFLAGS', 'EXTRA_INCLUDES' ]
966 savedeps_caches = ['GLOBAL_DEPENDENCIES', 'TARGET_TYPE', 'INIT_FUNCTIONS', 'SYSLIB_DEPS']
967 savedeps_files = ['buildtools/wafsamba/samba_deps.py']
969 def save_samba_deps(bld, tgt_list):
970 '''save the dependency calculations between builds, to make
971 further builds faster'''
972 denv = Environment.Environment()
974 denv.version = savedeps_version
975 denv.savedeps_inputs = savedeps_inputs
976 denv.savedeps_outputs = savedeps_outputs
977 denv.input = {}
978 denv.output = {}
979 denv.outenv = {}
980 denv.caches = {}
981 denv.envvar = {}
982 denv.files = {}
984 for f in savedeps_files:
985 denv.files[f] = os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime
987 for c in savedeps_caches:
988 denv.caches[c] = LOCAL_CACHE(bld, c)
990 for e in savedeps_envvars:
991 denv.envvar[e] = bld.env[e]
993 for t in tgt_list:
994 # save all the input attributes for each target
995 tdeps = {}
996 for attr in savedeps_inputs:
997 v = getattr(t, attr, None)
998 if v is not None:
999 tdeps[attr] = v
1000 if tdeps != {}:
1001 denv.input[t.sname] = tdeps
1003 # save all the output attributes for each target
1004 tdeps = {}
1005 for attr in savedeps_outputs:
1006 v = getattr(t, attr, None)
1007 if v is not None:
1008 tdeps[attr] = v
1009 if tdeps != {}:
1010 denv.output[t.sname] = tdeps
1012 tdeps = {}
1013 for attr in savedeps_outenv:
1014 if attr in t.env:
1015 tdeps[attr] = t.env[attr]
1016 if tdeps != {}:
1017 denv.outenv[t.sname] = tdeps
1019 depsfile = os.path.join(bld.bdir, "sambadeps")
1020 denv.store(depsfile)
1024 def load_samba_deps(bld, tgt_list):
1025 '''load a previous set of build dependencies if possible'''
1026 depsfile = os.path.join(bld.bdir, "sambadeps")
1027 denv = Environment.Environment()
1028 try:
1029 debug('deps: checking saved dependencies')
1030 denv.load(depsfile)
1031 if (denv.version != savedeps_version or
1032 denv.savedeps_inputs != savedeps_inputs or
1033 denv.savedeps_outputs != savedeps_outputs):
1034 return False
1035 except:
1036 return False
1038 # check if critical files have changed
1039 for f in savedeps_files:
1040 if f not in denv.files:
1041 return False
1042 if denv.files[f] != os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime:
1043 return False
1045 # check if caches are the same
1046 for c in savedeps_caches:
1047 if c not in denv.caches or denv.caches[c] != LOCAL_CACHE(bld, c):
1048 return False
1050 # check if caches are the same
1051 for e in savedeps_envvars:
1052 if e not in denv.envvar or denv.envvar[e] != bld.env[e]:
1053 return False
1055 # check inputs are the same
1056 for t in tgt_list:
1057 tdeps = {}
1058 for attr in savedeps_inputs:
1059 v = getattr(t, attr, None)
1060 if v is not None:
1061 tdeps[attr] = v
1062 if t.sname in denv.input:
1063 olddeps = denv.input[t.sname]
1064 else:
1065 olddeps = {}
1066 if tdeps != olddeps:
1067 #print '%s: \ntdeps=%s \nodeps=%s' % (t.sname, tdeps, olddeps)
1068 return False
1070 # put outputs in place
1071 for t in tgt_list:
1072 if not t.sname in denv.output: continue
1073 tdeps = denv.output[t.sname]
1074 for a in tdeps:
1075 setattr(t, a, tdeps[a])
1077 # put output env vars in place
1078 for t in tgt_list:
1079 if not t.sname in denv.outenv: continue
1080 tdeps = denv.outenv[t.sname]
1081 for a in tdeps:
1082 t.env[a] = tdeps[a]
1084 debug('deps: loaded saved dependencies')
1085 return True
1089 def check_project_rules(bld):
1090 '''check the project rules - ensuring the targets are sane'''
1092 loops = {}
1093 inc_loops = {}
1095 tgt_list = get_tgt_list(bld)
1097 add_samba_attributes(bld, tgt_list)
1099 force_project_rules = (Options.options.SHOWDEPS or
1100 Options.options.SHOW_DUPLICATES)
1102 if not force_project_rules and load_samba_deps(bld, tgt_list):
1103 return
1105 global tstart
1106 tstart = time.clock()
1108 bld.new_rules = True
1109 Logs.info("Checking project rules ...")
1111 debug('deps: project rules checking started')
1113 expand_subsystem_deps(bld)
1115 debug("deps: expand_subsystem_deps: %f" % (time.clock() - tstart))
1117 replace_grouping_libraries(bld, tgt_list)
1119 debug("deps: replace_grouping_libraries: %f" % (time.clock() - tstart))
1121 build_direct_deps(bld, tgt_list)
1123 debug("deps: build_direct_deps: %f" % (time.clock() - tstart))
1125 break_dependency_loops(bld, tgt_list)
1127 debug("deps: break_dependency_loops: %f" % (time.clock() - tstart))
1129 if Options.options.SHOWDEPS:
1130 show_dependencies(bld, Options.options.SHOWDEPS, set())
1132 calculate_final_deps(bld, tgt_list, loops)
1134 debug("deps: calculate_final_deps: %f" % (time.clock() - tstart))
1136 if Options.options.SHOW_DUPLICATES:
1137 show_object_duplicates(bld, tgt_list)
1139 # run the various attribute generators
1140 for f in [ build_dependencies, build_includes, add_init_functions ]:
1141 debug('deps: project rules checking %s', f)
1142 for t in tgt_list: f(t)
1143 debug("deps: %s: %f" % (f, time.clock() - tstart))
1145 debug('deps: project rules stage1 completed')
1147 #check_orpaned_targets(bld, tgt_list)
1149 if not check_duplicate_sources(bld, tgt_list):
1150 Logs.error("Duplicate sources present - aborting")
1151 sys.exit(1)
1153 debug("deps: check_duplicate_sources: %f" % (time.clock() - tstart))
1155 if not check_group_ordering(bld, tgt_list):
1156 Logs.error("Bad group ordering - aborting")
1157 sys.exit(1)
1159 debug("deps: check_group_ordering: %f" % (time.clock() - tstart))
1161 show_final_deps(bld, tgt_list)
1163 debug("deps: show_final_deps: %f" % (time.clock() - tstart))
1165 debug('deps: project rules checking completed - %u targets checked',
1166 len(tgt_list))
1168 if not bld.is_install:
1169 save_samba_deps(bld, tgt_list)
1171 debug("deps: save_samba_deps: %f" % (time.clock() - tstart))
1173 Logs.info("Project rules pass")
1176 def CHECK_PROJECT_RULES(bld):
1177 '''enable checking of project targets for sanity'''
1178 if bld.env.added_project_rules:
1179 return
1180 bld.env.added_project_rules = True
1181 bld.add_pre_fun(check_project_rules)
1182 Build.BuildContext.CHECK_PROJECT_RULES = CHECK_PROJECT_RULES