buildtools/wafsamba/samba_deps.py

   1 # Samba automatic dependency handling and project rules
   2
   3 import os, sys, re, time
   4
   5 import Build, Environment, Options, Logs, Utils
   6 from Logs import debug
   7 from Configure import conf
   8
   9 from samba_bundled import BUILTIN_LIBRARY
  10 from samba_utils import LOCAL_CACHE, TO_LIST, get_tgt_list, unique_list, os_path_relpath
  11 from samba_autoconf import library_flags
  12
  13 @conf
  14 def ADD_GLOBAL_DEPENDENCY(ctx, dep):
  15     '''add a dependency for all binaries and libraries'''
  16     if not 'GLOBAL_DEPENDENCIES' in ctx.env:
  17         ctx.env.GLOBAL_DEPENDENCIES = []
  18     ctx.env.GLOBAL_DEPENDENCIES.append(dep)
  19
  20
  21 @conf
  22 def BREAK_CIRCULAR_LIBRARY_DEPENDENCIES(ctx):
  23     '''indicate that circular dependencies between libraries should be broken.'''
  24     ctx.env.ALLOW_CIRCULAR_LIB_DEPENDENCIES = True
  25
  26
  27 @conf
  28 def SET_SYSLIB_DEPS(conf, target, deps):
  29     '''setup some implied dependencies for a SYSLIB'''
  30     cache = LOCAL_CACHE(conf, 'SYSLIB_DEPS')
  31     cache[target] = deps
  32
  33
  34 def expand_subsystem_deps(bld):
  35     '''expand the reverse dependencies resulting from subsystem
  36        attributes of modules. This is walking over the complete list
  37        of declared subsystems, and expands the samba_deps_extended list for any
  38        module<->subsystem dependencies'''
  39
  40     subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
  41     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
  42
  43     for subsystem_name in subsystem_list:
  44         bld.ASSERT(subsystem_name in targets, "Subsystem target %s not declared" % subsystem_name)
  45         type = targets[subsystem_name]
  46         if type == 'DISABLED' or type == 'EMPTY':
  47             continue
  48
  49         # for example,
  50         #    subsystem_name = dcerpc_server (a subsystem)
  51         #    subsystem      = dcerpc_server (a subsystem object)
  52         #    module_name    = rpc_epmapper (a module within the dcerpc_server subsystem)
  53         #    module         = rpc_epmapper (a module object within the dcerpc_server subsystem)
  54
  55         subsystem = bld.get_tgen_by_name(subsystem_name)
  56         bld.ASSERT(subsystem is not None, "Unable to find subsystem %s" % subsystem_name)
  57         for d in subsystem_list[subsystem_name]:
  58             module_name = d['TARGET']
  59             module_type = targets[module_name]
  60             if module_type in ['DISABLED', 'EMPTY']:
  61                 continue
  62             bld.ASSERT(subsystem is not None,
  63                        "Subsystem target %s for %s (%s) not found" % (subsystem_name, module_name, module_type))
  64             if module_type in ['SUBSYSTEM']:
  65                 # if a module is a plain object type (not a library) then the
  66                 # subsystem it is part of needs to have it as a dependency, so targets
  67                 # that depend on this subsystem get the modules of that subsystem
  68                 subsystem.samba_deps_extended.append(module_name)
  69         subsystem.samba_deps_extended = unique_list(subsystem.samba_deps_extended)
  70
  71
  72
  73 def build_dependencies(self):
  74     '''This builds the dependency list for a target. It runs after all the targets are declared
  75
  76     The reason this is not just done in the SAMBA_*() rules is that we have no way of knowing
  77     the full dependency list for a target until we have all of the targets declared.
  78     '''
  79
  80     if self.samba_type in ['LIBRARY', 'BINARY', 'PYTHON']:
  81         self.uselib        = list(self.final_syslibs)
  82         self.uselib_local  = list(self.final_libs)
  83         self.add_objects   = list(self.final_objects)
  84
  85         # extra link flags from pkg_config
  86         libs = self.final_syslibs.copy()
  87
  88         (ccflags, ldflags, cpppath) = library_flags(self, list(libs))
  89         new_ldflags        = getattr(self, 'samba_ldflags', [])[:]
  90         new_ldflags.extend(ldflags)
  91         self.ldflags       = new_ldflags
  92
  93         if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ldflags:
  94             for f in self.env.undefined_ldflags:
  95                 self.ldflags.remove(f)
  96
  97         if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ignore_ldflags:
  98             for f in self.env.undefined_ignore_ldflags:
  99                 self.ldflags.append(f)
 100
 101         debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
 102               self.sname, self.uselib, self.uselib_local, self.add_objects)
 103
 104     if self.samba_type in ['SUBSYSTEM']:
 105         # this is needed for the ccflags of libs that come from pkg_config
 106         self.uselib = list(self.final_syslibs)
 107         self.uselib.extend(list(self.direct_syslibs))
 108         for lib in self.final_libs:
 109             t = self.bld.get_tgen_by_name(lib)
 110             self.uselib.extend(list(t.final_syslibs))
 111         self.uselib = unique_list(self.uselib)
 112
 113     if getattr(self, 'uselib', None):
 114         up_list = []
 115         for l in self.uselib:
 116            up_list.append(l.upper())
 117         self.uselib = up_list
 118
 119
 120 def build_includes(self):
 121     '''This builds the right set of includes for a target.
 122
 123     One tricky part of this is that the includes= attribute for a
 124     target needs to use paths which are relative to that targets
 125     declaration directory (which we can get at via t.path).
 126
 127     The way this works is the includes list gets added as
 128     samba_includes in the main build task declaration. Then this
 129     function runs after all of the tasks are declared, and it
 130     processes the samba_includes attribute to produce a includes=
 131     attribute
 132     '''
 133
 134     if getattr(self, 'samba_includes', None) is None:
 135         return
 136
 137     bld = self.bld
 138
 139     inc_deps = includes_objects(bld, self, set(), {})
 140
 141     includes = []
 142
 143     # maybe add local includes
 144     if getattr(self, 'local_include', True) and getattr(self, 'local_include_first', True):
 145         includes.append('.')
 146
 147     includes.extend(self.samba_includes_extended)
 148
 149     if 'EXTRA_INCLUDES' in bld.env and getattr(self, 'global_include', True):
 150         includes.extend(bld.env['EXTRA_INCLUDES'])
 151
 152     includes.append('#')
 153
 154     inc_set = set()
 155     inc_abs = []
 156
 157     for d in inc_deps:
 158         t = bld.get_tgen_by_name(d)
 159         bld.ASSERT(t is not None, "Unable to find dependency %s for %s" % (d, self.sname))
 160         inclist = getattr(t, 'samba_includes_extended', [])[:]
 161         if getattr(t, 'local_include', True):
 162             inclist.append('.')
 163         if inclist == []:
 164             continue
 165         tpath = t.samba_abspath
 166         for inc in inclist:
 167             npath = tpath + '/' + inc
 168             if not npath in inc_set:
 169                 inc_abs.append(npath)
 170                 inc_set.add(npath)
 171
 172     mypath = self.path.abspath(bld.env)
 173     for inc in inc_abs:
 174         relpath = os_path_relpath(inc, mypath)
 175         includes.append(relpath)
 176
 177     if getattr(self, 'local_include', True) and not getattr(self, 'local_include_first', True):
 178         includes.append('.')
 179
 180     # now transform the includes list to be relative to the top directory
 181     # which is represented by '#' in waf. This allows waf to cache the
 182     # includes lists more efficiently
 183     includes_top = []
 184     for i in includes:
 185         if i[0] == '#':
 186             # some are already top based
 187             includes_top.append(i)
 188             continue
 189         absinc = os.path.join(self.path.abspath(), i)
 190         relinc = os_path_relpath(absinc, self.bld.srcnode.abspath())
 191         includes_top.append('#' + relinc)
 192
 193     self.includes = unique_list(includes_top)
 194     debug('deps: includes for target %s: includes=%s',
 195           self.sname, self.includes)
 196
 197
 198 def add_init_functions(self):
 199     '''This builds the right set of init functions'''
 200
 201     bld = self.bld
 202
 203     subsystems = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 204
 205     # cope with the separated object lists from BINARY and LIBRARY targets
 206     sname = self.sname
 207     if sname.endswith('.objlist'):
 208         sname = sname[0:-8]
 209
 210     modules = []
 211     if sname in subsystems:
 212         modules.append(sname)
 213
 214     m = getattr(self, 'samba_modules', None)
 215     if m is not None:
 216         modules.extend(TO_LIST(m))
 217
 218     m = getattr(self, 'samba_subsystem', None)
 219     if m is not None:
 220         modules.append(m)
 221
 222     if 'pyembed' in self.features:
 223         return
 224
 225     sentinel = getattr(self, 'init_function_sentinel', 'NULL')
 226
 227     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
 228     cflags = getattr(self, 'samba_cflags', [])[:]
 229
 230     if modules == []:
 231         sname = sname.replace('-','_')
 232         sname = sname.replace('/','_')
 233         cflags.append('-DSTATIC_%s_MODULES=%s' % (sname, sentinel))
 234         if sentinel == 'NULL':
 235             proto = "extern void __%s_dummy_module_proto(void)" % (sname)
 236             cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (sname, proto))
 237         self.ccflags = cflags
 238         return
 239
 240     for m in modules:
 241         bld.ASSERT(m in subsystems,
 242                    "No init_function defined for module '%s' in target '%s'" % (m, self.sname))
 243         init_fn_list = []
 244         for d in subsystems[m]:
 245             if targets[d['TARGET']] != 'DISABLED':
 246                 init_fn_list.append(d['INIT_FUNCTION'])
 247         if init_fn_list == []:
 248             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, sentinel))
 249             if sentinel == 'NULL':
 250                 proto = "extern void __%s_dummy_module_proto(void)" % (m)
 251                 cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
 252         else:
 253             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, ','.join(init_fn_list) + ',' + sentinel))
 254             proto=''
 255             for f in init_fn_list:
 256                 proto += '_MODULE_PROTO(%s)' % f
 257             proto += "extern void __%s_dummy_module_proto(void)" % (m)
 258             cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
 259     self.ccflags = cflags
 260
 261
 262 def check_duplicate_sources(bld, tgt_list):
 263     '''see if we are compiling the same source file more than once'''
 264
 265     debug('deps: checking for duplicate sources')
 266     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 267
 268     for t in tgt_list:
 269         source_list = TO_LIST(getattr(t, 'source', ''))
 270         tpath = os.path.normpath(os_path_relpath(t.path.abspath(bld.env), t.env.BUILD_DIRECTORY + '/default'))
 271         obj_sources = set()
 272         for s in source_list:
 273             p = os.path.normpath(os.path.join(tpath, s))
 274             if p in obj_sources:
 275                 Logs.error("ERROR: source %s appears twice in target '%s'" % (p, t.sname))
 276                 sys.exit(1)
 277             obj_sources.add(p)
 278         t.samba_source_set = obj_sources
 279
 280     subsystems = {}
 281
 282     # build a list of targets that each source file is part of
 283     for t in tgt_list:
 284         if not targets[t.sname] in [ 'LIBRARY', 'BINARY', 'PYTHON' ]:
 285             continue
 286         for obj in t.add_objects:
 287             t2 = t.bld.get_tgen_by_name(obj)
 288             source_set = getattr(t2, 'samba_source_set', set())
 289             for s in source_set:
 290                 if not s in subsystems:
 291                     subsystems[s] = {}
 292                 if not t.sname in subsystems[s]:
 293                     subsystems[s][t.sname] = []
 294                 subsystems[s][t.sname].append(t2.sname)
 295
 296     for s in subsystems:
 297         if len(subsystems[s]) > 1 and Options.options.SHOW_DUPLICATES:
 298             Logs.warn("WARNING: source %s is in more than one target: %s" % (s, subsystems[s].keys()))
 299         for tname in subsystems[s]:
 300             if len(subsystems[s][tname]) > 1:
 301                 raise Utils.WafError("ERROR: source %s is in more than one subsystem of target '%s': %s" % (s, tname, subsystems[s][tname]))
 302
 303     return True
 304
 305 def check_group_ordering(bld, tgt_list):
 306     '''see if we have any dependencies that violate the group ordering
 307
 308     It is an error for a target to depend on a target from a later
 309     build group
 310     '''
 311
 312     def group_name(g):
 313         tm = bld.task_manager
 314         return [x for x in tm.groups_names if id(tm.groups_names[x]) == id(g)][0]
 315
 316     for g in bld.task_manager.groups:
 317         gname = group_name(g)
 318         for t in g.tasks_gen:
 319             t.samba_group = gname
 320
 321     grp_map = {}
 322     idx = 0
 323     for g in bld.task_manager.groups:
 324         name = group_name(g)
 325         grp_map[name] = idx
 326         idx += 1
 327
 328     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 329
 330     ret = True
 331     for t in tgt_list:
 332         tdeps = getattr(t, 'add_objects', []) + getattr(t, 'uselib_local', [])
 333         for d in tdeps:
 334             t2 = bld.get_tgen_by_name(d)
 335             if t2 is None:
 336                 continue
 337             map1 = grp_map[t.samba_group]
 338             map2 = grp_map[t2.samba_group]
 339
 340             if map2 > map1:
 341                 Logs.error("Target %r in build group %r depends on target %r from later build group %r" % (
 342                            t.sname, t.samba_group, t2.sname, t2.samba_group))
 343                 ret = False
 344
 345     return ret
 346 Build.BuildContext.check_group_ordering = check_group_ordering
 347
 348 def show_final_deps(bld, tgt_list):
 349     '''show the final dependencies for all targets'''
 350
 351     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 352
 353     for t in tgt_list:
 354         if not targets[t.sname] in ['LIBRARY', 'BINARY', 'PYTHON', 'SUBSYSTEM']:
 355             continue
 356         debug('deps: final dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
 357               t.sname, t.uselib, getattr(t, 'uselib_local', []), getattr(t, 'add_objects', []))
 358
 359
 360 def add_samba_attributes(bld, tgt_list):
 361     '''ensure a target has a the required samba attributes'''
 362
 363     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 364
 365     for t in tgt_list:
 366         if t.name != '':
 367             t.sname = t.name
 368         else:
 369             t.sname = t.target
 370         t.samba_type = targets[t.sname]
 371         t.samba_abspath = t.path.abspath(bld.env)
 372         t.samba_deps_extended = t.samba_deps[:]
 373         t.samba_includes_extended = TO_LIST(t.samba_includes)[:]
 374         t.ccflags = getattr(t, 'samba_cflags', '')
 375
 376 def replace_grouping_libraries(bld, tgt_list):
 377     '''replace dependencies based on grouping libraries
 378
 379     If a library is marked as a grouping library, then any target that
 380     depends on a subsystem that is part of that grouping library gets
 381     that dependency replaced with a dependency on the grouping library
 382     '''
 383
 384     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 385
 386     grouping = {}
 387
 388     # find our list of grouping libraries, mapped from the subsystems they depend on
 389     for t in tgt_list:
 390         if not getattr(t, 'grouping_library', False):
 391             continue
 392         for dep in t.samba_deps_extended:
 393             bld.ASSERT(dep in targets, "grouping library target %s not declared in %s" % (dep, t.sname))
 394             if targets[dep] == 'SUBSYSTEM':
 395                 grouping[dep] = t.sname
 396
 397     # now replace any dependencies on elements of grouping libraries
 398     for t in tgt_list:
 399         for i in range(len(t.samba_deps_extended)):
 400             dep = t.samba_deps_extended[i]
 401             if dep in grouping:
 402                 if t.sname != grouping[dep]:
 403                     debug("deps: target %s: replacing dependency %s with grouping library %s" % (t.sname, dep, grouping[dep]))
 404                     t.samba_deps_extended[i] = grouping[dep]
 405
 406
 407
 408 def build_direct_deps(bld, tgt_list):
 409     '''build the direct_objects and direct_libs sets for each target'''
 410
 411     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 412     syslib_deps  = LOCAL_CACHE(bld, 'SYSLIB_DEPS')
 413
 414     global_deps = bld.env.GLOBAL_DEPENDENCIES
 415     global_deps_exclude = set()
 416     for dep in global_deps:
 417         t = bld.get_tgen_by_name(dep)
 418         for d in t.samba_deps:
 419             # prevent loops from the global dependencies list
 420             global_deps_exclude.add(d)
 421             global_deps_exclude.add(d + '.objlist')
 422
 423     for t in tgt_list:
 424         t.direct_objects = set()
 425         t.direct_libs = set()
 426         t.direct_syslibs = set()
 427         deps = t.samba_deps_extended[:]
 428         if getattr(t, 'samba_use_global_deps', False) and not t.sname in global_deps_exclude:
 429             deps.extend(global_deps)
 430         for d in deps:
 431             if d == t.sname: continue
 432             if not d in targets:
 433                 Logs.error("Unknown dependency '%s' in '%s'" % (d, t.sname))
 434                 sys.exit(1)
 435             if targets[d] in [ 'EMPTY', 'DISABLED' ]:
 436                 continue
 437             if targets[d] == 'PYTHON' and targets[t.sname] != 'PYTHON' and t.sname.find('.objlist') == -1:
 438                 # this check should be more restrictive, but for now we have pidl-generated python
 439                 # code that directly depends on other python modules
 440                 Logs.error('ERROR: Target %s has dependency on python module %s' % (t.sname, d))
 441                 sys.exit(1)
 442             if targets[d] == 'SYSLIB':
 443                 t.direct_syslibs.add(d)
 444                 if d in syslib_deps:
 445                     for implied in TO_LIST(syslib_deps[d]):
 446                         if BUILTIN_LIBRARY(bld, implied):
 447                             t.direct_objects.add(implied)
 448                         elif targets[implied] == 'SYSLIB':
 449                             t.direct_syslibs.add(implied)
 450                         elif targets[implied] in ['LIBRARY', 'MODULE']:
 451                             t.direct_libs.add(implied)
 452                         else:
 453                             Logs.error('Implied dependency %s in %s is of type %s' % (
 454                                 implied, t.sname, targets[implied]))
 455                             sys.exit(1)
 456                 continue
 457             t2 = bld.get_tgen_by_name(d)
 458             if t2 is None:
 459                 Logs.error("no task %s of type %s in %s" % (d, targets[d], t.sname))
 460                 sys.exit(1)
 461             if t2.samba_type in [ 'LIBRARY', 'MODULE' ]:
 462                 t.direct_libs.add(d)
 463             elif t2.samba_type in [ 'SUBSYSTEM', 'ASN1', 'PYTHON' ]:
 464                 t.direct_objects.add(d)
 465     debug('deps: built direct dependencies')
 466
 467
 468 def dependency_loop(loops, t, target):
 469     '''add a dependency loop to the loops dictionary'''
 470     if t.sname == target:
 471         return
 472     if not target in loops:
 473         loops[target] = set()
 474     if not t.sname in loops[target]:
 475         loops[target].add(t.sname)
 476
 477
 478 def indirect_libs(bld, t, chain, loops):
 479     '''recursively calculate the indirect library dependencies for a target
 480
 481     An indirect library is a library that results from a dependency on
 482     a subsystem
 483     '''
 484
 485     ret = getattr(t, 'indirect_libs', None)
 486     if ret is not None:
 487         return ret
 488
 489     ret = set()
 490     for obj in t.direct_objects:
 491         if obj in chain:
 492             dependency_loop(loops, t, obj)
 493             continue
 494         chain.add(obj)
 495         t2 = bld.get_tgen_by_name(obj)
 496         r2 = indirect_libs(bld, t2, chain, loops)
 497         chain.remove(obj)
 498         ret = ret.union(t2.direct_libs)
 499         ret = ret.union(r2)
 500
 501     for obj in indirect_objects(bld, t, set(), loops):
 502         if obj in chain:
 503             dependency_loop(loops, t, obj)
 504             continue
 505         chain.add(obj)
 506         t2 = bld.get_tgen_by_name(obj)
 507         r2 = indirect_libs(bld, t2, chain, loops)
 508         chain.remove(obj)
 509         ret = ret.union(t2.direct_libs)
 510         ret = ret.union(r2)
 511
 512     t.indirect_libs = ret
 513
 514     return ret
 515
 516
 517 def indirect_objects(bld, t, chain, loops):
 518     '''recursively calculate the indirect object dependencies for a target
 519
 520     indirect objects are the set of objects from expanding the
 521     subsystem dependencies
 522     '''
 523
 524     ret = getattr(t, 'indirect_objects', None)
 525     if ret is not None: return ret
 526
 527     ret = set()
 528     for lib in t.direct_objects:
 529         if lib in chain:
 530             dependency_loop(loops, t, lib)
 531             continue
 532         chain.add(lib)
 533         t2 = bld.get_tgen_by_name(lib)
 534         r2 = indirect_objects(bld, t2, chain, loops)
 535         chain.remove(lib)
 536         ret = ret.union(t2.direct_objects)
 537         ret = ret.union(r2)
 538
 539     t.indirect_objects = ret
 540     return ret
 541
 542
 543 def extended_objects(bld, t, chain):
 544     '''recursively calculate the extended object dependencies for a target
 545
 546     extended objects are the union of:
 547        - direct objects
 548        - indirect objects
 549        - direct and indirect objects of all direct and indirect libraries
 550     '''
 551
 552     ret = getattr(t, 'extended_objects', None)
 553     if ret is not None: return ret
 554
 555     ret = set()
 556     ret = ret.union(t.final_objects)
 557
 558     for lib in t.final_libs:
 559         if lib in chain:
 560             continue
 561         t2 = bld.get_tgen_by_name(lib)
 562         chain.add(lib)
 563         r2 = extended_objects(bld, t2, chain)
 564         chain.remove(lib)
 565         ret = ret.union(t2.final_objects)
 566         ret = ret.union(r2)
 567
 568     t.extended_objects = ret
 569     return ret
 570
 571
 572 def includes_objects(bld, t, chain, inc_loops):
 573     '''recursively calculate the includes object dependencies for a target
 574
 575     includes dependencies come from either library or object dependencies
 576     '''
 577     ret = getattr(t, 'includes_objects', None)
 578     if ret is not None:
 579         return ret
 580
 581     ret = t.direct_objects.copy()
 582     ret = ret.union(t.direct_libs)
 583
 584     for obj in t.direct_objects:
 585         if obj in chain:
 586             dependency_loop(inc_loops, t, obj)
 587             continue
 588         chain.add(obj)
 589         t2 = bld.get_tgen_by_name(obj)
 590         r2 = includes_objects(bld, t2, chain, inc_loops)
 591         chain.remove(obj)
 592         ret = ret.union(t2.direct_objects)
 593         ret = ret.union(r2)
 594
 595     for lib in t.direct_libs:
 596         if lib in chain:
 597             dependency_loop(inc_loops, t, lib)
 598             continue
 599         chain.add(lib)
 600         t2 = bld.get_tgen_by_name(lib)
 601         if t2 is None:
 602             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 603             Logs.error('Target %s of type %s not found in direct_libs for %s' % (
 604                 lib, targets[lib], t.sname))
 605             sys.exit(1)
 606         r2 = includes_objects(bld, t2, chain, inc_loops)
 607         chain.remove(lib)
 608         ret = ret.union(t2.direct_objects)
 609         ret = ret.union(r2)
 610
 611     t.includes_objects = ret
 612     return ret
 613
 614
 615 def break_dependency_loops(bld, tgt_list):
 616     '''find and break dependency loops'''
 617     loops = {}
 618     inc_loops = {}
 619
 620     # build up the list of loops
 621     for t in tgt_list:
 622         indirect_objects(bld, t, set(), loops)
 623         indirect_libs(bld, t, set(), loops)
 624         includes_objects(bld, t, set(), inc_loops)
 625
 626     # break the loops
 627     for t in tgt_list:
 628         if t.sname in loops:
 629             for attr in ['direct_objects', 'indirect_objects', 'direct_libs', 'indirect_libs']:
 630                 objs = getattr(t, attr, set())
 631                 setattr(t, attr, objs.difference(loops[t.sname]))
 632
 633     for loop in loops:
 634         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 635
 636     for loop in inc_loops:
 637         debug('deps: Found include loops for target %s : %s', loop, inc_loops[loop])
 638
 639     # expand the loops mapping by one level
 640     for loop in loops.copy():
 641         for tgt in loops[loop]:
 642             if tgt in loops:
 643                 loops[loop] = loops[loop].union(loops[tgt])
 644
 645     for loop in inc_loops.copy():
 646         for tgt in inc_loops[loop]:
 647             if tgt in inc_loops:
 648                 inc_loops[loop] = inc_loops[loop].union(inc_loops[tgt])
 649
 650
 651     # expand indirect subsystem and library loops
 652     for loop in loops.copy():
 653         t = bld.get_tgen_by_name(loop)
 654         if t.samba_type in ['SUBSYSTEM']:
 655             loops[loop] = loops[loop].union(t.indirect_objects)
 656             loops[loop] = loops[loop].union(t.direct_objects)
 657         if t.samba_type in ['LIBRARY','PYTHON']:
 658             loops[loop] = loops[loop].union(t.indirect_libs)
 659             loops[loop] = loops[loop].union(t.direct_libs)
 660         if loop in loops[loop]:
 661             loops[loop].remove(loop)
 662
 663     # expand indirect includes loops
 664     for loop in inc_loops.copy():
 665         t = bld.get_tgen_by_name(loop)
 666         inc_loops[loop] = inc_loops[loop].union(t.includes_objects)
 667         if loop in inc_loops[loop]:
 668             inc_loops[loop].remove(loop)
 669
 670     # add in the replacement dependencies
 671     for t in tgt_list:
 672         for loop in loops:
 673             for attr in ['indirect_objects', 'indirect_libs']:
 674                 objs = getattr(t, attr, set())
 675                 if loop in objs:
 676                     diff = loops[loop].difference(objs)
 677                     if t.sname in diff:
 678                         diff.remove(t.sname)
 679                     if diff:
 680                         debug('deps: Expanded target %s of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 681                         objs = objs.union(diff)
 682                 setattr(t, attr, objs)
 683
 684         for loop in inc_loops:
 685             objs = getattr(t, 'includes_objects', set())
 686             if loop in objs:
 687                 diff = inc_loops[loop].difference(objs)
 688                 if t.sname in diff:
 689                     diff.remove(t.sname)
 690                 if diff:
 691                     debug('deps: Expanded target %s includes of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 692                     objs = objs.union(diff)
 693             setattr(t, 'includes_objects', objs)
 694
 695
 696 def reduce_objects(bld, tgt_list):
 697     '''reduce objects by looking for indirect object dependencies'''
 698     rely_on = {}
 699
 700     for t in tgt_list:
 701         t.extended_objects = None
 702
 703     changed = False
 704
 705     for type in ['BINARY', 'PYTHON', 'LIBRARY']:
 706         for t in tgt_list:
 707             if t.samba_type != type: continue
 708             # if we will indirectly link to a target then we don't need it
 709             new = t.final_objects.copy()
 710             for l in t.final_libs:
 711                 t2 = bld.get_tgen_by_name(l)
 712                 t2_obj = extended_objects(bld, t2, set())
 713                 dup = new.intersection(t2_obj)
 714                 if t.sname in rely_on:
 715                     dup = dup.difference(rely_on[t.sname])
 716                 if dup:
 717                     debug('deps: removing dups from %s of type %s: %s also in %s %s',
 718                           t.sname, t.samba_type, dup, t2.samba_type, l)
 719                     new = new.difference(dup)
 720                     changed = True
 721                     if not l in rely_on:
 722                         rely_on[l] = set()
 723                     rely_on[l] = rely_on[l].union(dup)
 724             t.final_objects = new
 725
 726     if not changed:
 727         return False
 728
 729     # add back in any objects that were relied upon by the reduction rules
 730     for r in rely_on:
 731         t = bld.get_tgen_by_name(r)
 732         t.final_objects = t.final_objects.union(rely_on[r])
 733
 734     return True
 735
 736
 737 def show_library_loop(bld, lib1, lib2, path, seen):
 738     '''show the detailed path of a library loop between lib1 and lib2'''
 739
 740     t = bld.get_tgen_by_name(lib1)
 741     if not lib2 in getattr(t, 'final_libs', set()):
 742         return
 743
 744     for d in t.samba_deps_extended:
 745         if d in seen:
 746             continue
 747         seen.add(d)
 748         path2 = path + '=>' + d
 749         if d == lib2:
 750             Logs.warn('library loop path: ' + path2)
 751             return
 752         show_library_loop(bld, d, lib2, path2, seen)
 753         seen.remove(d)
 754
 755
 756 def calculate_final_deps(bld, tgt_list, loops):
 757     '''calculate the final library and object dependencies'''
 758     for t in tgt_list:
 759         # start with the maximum possible list
 760         t.final_libs    = t.direct_libs.union(indirect_libs(bld, t, set(), loops))
 761         t.final_objects = t.direct_objects.union(indirect_objects(bld, t, set(), loops))
 762
 763     for t in tgt_list:
 764         # don't depend on ourselves
 765         if t.sname in t.final_libs:
 766             t.final_libs.remove(t.sname)
 767         if t.sname in t.final_objects:
 768             t.final_objects.remove(t.sname)
 769
 770     # handle any non-shared binaries
 771     for t in tgt_list:
 772         if t.samba_type == 'BINARY' and bld.NONSHARED_BINARY(t.sname):
 773             subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 774             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 775
 776             # replace lib deps with objlist deps
 777             for l in t.final_libs:
 778                 objname = l + '.objlist'
 779                 t2 = bld.get_tgen_by_name(objname)
 780                 if t2 is None:
 781                     Logs.error('ERROR: subsystem %s not found' % objname)
 782                     sys.exit(1)
 783                 t.final_objects.add(objname)
 784                 t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 785                 if l in subsystem_list:
 786                     # its a subsystem - we also need the contents of any modules
 787                     for d in subsystem_list[l]:
 788                         module_name = d['TARGET']
 789                         if targets[module_name] == 'LIBRARY':
 790                             objname = module_name + '.objlist'
 791                         elif targets[module_name] == 'SUBSYSTEM':
 792                             objname = module_name
 793                         else:
 794                             continue
 795                         t2 = bld.get_tgen_by_name(objname)
 796                         if t2 is None:
 797                             Logs.error('ERROR: subsystem %s not found' % objname)
 798                             sys.exit(1)
 799                         t.final_objects.add(objname)
 800                         t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 801             t.final_libs = set()
 802
 803     # find any library loops
 804     for t in tgt_list:
 805         if t.samba_type in ['LIBRARY', 'PYTHON']:
 806             for l in t.final_libs.copy():
 807                 t2 = bld.get_tgen_by_name(l)
 808                 if t.sname in t2.final_libs:
 809                     if getattr(bld.env, "ALLOW_CIRCULAR_LIB_DEPENDENCIES", False):
 810                         # we could break this in either direction. If one of the libraries
 811                         # has a version number, and will this be distributed publicly, then
 812                         # we should make it the lower level library in the DAG
 813                         Logs.warn('deps: removing library loop %s from %s' % (t.sname, t2.sname))
 814                         dependency_loop(loops, t, t2.sname)
 815                         t2.final_libs.remove(t.sname)
 816                     else:
 817                         Logs.error('ERROR: circular library dependency between %s and %s'
 818                             % (t.sname, t2.sname))
 819                         show_library_loop(bld, t.sname, t2.sname, t.sname, set())
 820                         show_library_loop(bld, t2.sname, t.sname, t2.sname, set())
 821                         sys.exit(1)
 822
 823     for loop in loops:
 824         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 825
 826     # we now need to make corrections for any library loops we broke up
 827     # any target that depended on the target of the loop and doesn't
 828     # depend on the source of the loop needs to get the loop source added
 829     for type in ['BINARY','PYTHON','LIBRARY','BINARY']:
 830         for t in tgt_list:
 831             if t.samba_type != type: continue
 832             for loop in loops:
 833                 if loop in t.final_libs:
 834                     diff = loops[loop].difference(t.final_libs)
 835                     if t.sname in diff:
 836                         diff.remove(t.sname)
 837                     if t.sname in diff:
 838                         diff.remove(t.sname)
 839                     # make sure we don't recreate the loop again!
 840                     for d in diff.copy():
 841                         t2 = bld.get_tgen_by_name(d)
 842                         if t2.samba_type == 'LIBRARY':
 843                             if t.sname in t2.final_libs:
 844                                 debug('deps: removing expansion %s from %s', d, t.sname)
 845                                 diff.remove(d)
 846                     if diff:
 847                         debug('deps: Expanded target %s by loop %s libraries (loop %s) %s', t.sname, loop,
 848                               loops[loop], diff)
 849                         t.final_libs = t.final_libs.union(diff)
 850
 851     # remove objects that are also available in linked libs
 852     count = 0
 853     while reduce_objects(bld, tgt_list):
 854         count += 1
 855         if count > 100:
 856             Logs.warn("WARNING: Unable to remove all inter-target object duplicates")
 857             break
 858     debug('deps: Object reduction took %u iterations', count)
 859
 860     # add in any syslib dependencies
 861     for t in tgt_list:
 862         if not t.samba_type in ['BINARY','PYTHON','LIBRARY','SUBSYSTEM']:
 863             continue
 864         syslibs = set()
 865         for d in t.final_objects:
 866             t2 = bld.get_tgen_by_name(d)
 867             syslibs = syslibs.union(t2.direct_syslibs)
 868         # this adds the indirect syslibs as well, which may not be needed
 869         # depending on the linker flags
 870         for d in t.final_libs:
 871             t2 = bld.get_tgen_by_name(d)
 872             syslibs = syslibs.union(t2.direct_syslibs)
 873         t.final_syslibs = syslibs
 874
 875
 876     # find any unresolved library loops
 877     lib_loop_error = False
 878     for t in tgt_list:
 879         if t.samba_type in ['LIBRARY', 'PYTHON']:
 880             for l in t.final_libs.copy():
 881                 t2 = bld.get_tgen_by_name(l)
 882                 if t.sname in t2.final_libs:
 883                     Logs.error('ERROR: Unresolved library loop %s from %s' % (t.sname, t2.sname))
 884                     lib_loop_error = True
 885     if lib_loop_error:
 886         sys.exit(1)
 887
 888     debug('deps: removed duplicate dependencies')
 889
 890
 891 def show_dependencies(bld, target, seen):
 892     '''recursively show the dependencies of target'''
 893
 894     if target in seen:
 895         return
 896
 897     t = bld.get_tgen_by_name(target)
 898     if t is None:
 899         Logs.error("ERROR: Unable to find target '%s'" % target)
 900         sys.exit(1)
 901
 902     Logs.info('%s(OBJECTS): %s' % (target, t.direct_objects))
 903     Logs.info('%s(LIBS): %s' % (target, t.direct_libs))
 904     Logs.info('%s(SYSLIBS): %s' % (target, t.direct_syslibs))
 905
 906     seen.add(target)
 907
 908     for t2 in t.direct_objects:
 909         show_dependencies(bld, t2, seen)
 910
 911
 912 def show_object_duplicates(bld, tgt_list):
 913     '''show a list of object files that are included in more than
 914     one library or binary'''
 915
 916     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 917
 918     used_by = {}
 919
 920     Logs.info("showing duplicate objects")
 921
 922     for t in tgt_list:
 923         if not targets[t.sname] in [ 'LIBRARY', 'PYTHON' ]:
 924             continue
 925         for n in getattr(t, 'final_objects', set()):
 926             t2 = bld.get_tgen_by_name(n)
 927             if not n in used_by:
 928                 used_by[n] = set()
 929             used_by[n].add(t.sname)
 930
 931     for n in used_by:
 932         if len(used_by[n]) > 1:
 933             Logs.info("target '%s' is used by %s" % (n, used_by[n]))
 934
 935     Logs.info("showing indirect dependency counts (sorted by count)")
 936
 937     def indirect_count(t1, t2):
 938         return len(t2.indirect_objects) - len(t1.indirect_objects)
 939
 940     sorted_list = sorted(tgt_list, cmp=indirect_count)
 941     for t in sorted_list:
 942         if len(t.indirect_objects) > 1:
 943             Logs.info("%s depends on %u indirect objects" % (t.sname, len(t.indirect_objects)))
 944
 945
 946 ######################################################################
 947 # this provides a way to save our dependency calculations between runs
 948 savedeps_version = 3
 949 savedeps_inputs  = ['samba_deps', 'samba_includes', 'local_include', 'local_include_first', 'samba_cflags',
 950                     'source', 'grouping_library', 'samba_ldflags', 'allow_undefined_symbols',
 951                     'use_global_deps', 'global_include' ]
 952 savedeps_outputs = ['uselib', 'uselib_local', 'add_objects', 'includes',
 953                     'ccflags', 'ldflags', 'samba_deps_extended', 'final_libs']
 954 savedeps_outenv  = ['INC_PATHS']
 955 savedeps_envvars = ['NONSHARED_BINARIES', 'GLOBAL_DEPENDENCIES', 'EXTRA_CFLAGS', 'EXTRA_LDFLAGS', 'EXTRA_INCLUDES' ]
 956 savedeps_caches  = ['GLOBAL_DEPENDENCIES', 'TARGET_TYPE', 'INIT_FUNCTIONS', 'SYSLIB_DEPS']
 957 savedeps_files   = ['buildtools/wafsamba/samba_deps.py']
 958
 959 def save_samba_deps(bld, tgt_list):
 960     '''save the dependency calculations between builds, to make
 961        further builds faster'''
 962     denv = Environment.Environment()
 963
 964     denv.version = savedeps_version
 965     denv.savedeps_inputs = savedeps_inputs
 966     denv.savedeps_outputs = savedeps_outputs
 967     denv.input = {}
 968     denv.output = {}
 969     denv.outenv = {}
 970     denv.caches = {}
 971     denv.envvar = {}
 972     denv.files  = {}
 973
 974     for f in savedeps_files:
 975         denv.files[f] = os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime
 976
 977     for c in savedeps_caches:
 978         denv.caches[c] = LOCAL_CACHE(bld, c)
 979
 980     for e in savedeps_envvars:
 981         denv.envvar[e] = bld.env[e]
 982
 983     for t in tgt_list:
 984         # save all the input attributes for each target
 985         tdeps = {}
 986         for attr in savedeps_inputs:
 987             v = getattr(t, attr, None)
 988             if v is not None:
 989                 tdeps[attr] = v
 990         if tdeps != {}:
 991             denv.input[t.sname] = tdeps
 992
 993         # save all the output attributes for each target
 994         tdeps = {}
 995         for attr in savedeps_outputs:
 996             v = getattr(t, attr, None)
 997             if v is not None:
 998                 tdeps[attr] = v
 999         if tdeps != {}:
1000             denv.output[t.sname] = tdeps
1001
1002         tdeps = {}
1003         for attr in savedeps_outenv:
1004             if attr in t.env:
1005                 tdeps[attr] = t.env[attr]
1006         if tdeps != {}:
1007             denv.outenv[t.sname] = tdeps
1008
1009     depsfile = os.path.join(bld.bdir, "sambadeps")
1010     denv.store_fast(depsfile)
1011
1012
1013
1014 def load_samba_deps(bld, tgt_list):
1015     '''load a previous set of build dependencies if possible'''
1016     depsfile = os.path.join(bld.bdir, "sambadeps")
1017     denv = Environment.Environment()
1018     try:
1019         debug('deps: checking saved dependencies')
1020         denv.load_fast(depsfile)
1021         if (denv.version != savedeps_version or
1022             denv.savedeps_inputs != savedeps_inputs or
1023             denv.savedeps_outputs != savedeps_outputs):
1024             return False
1025     except Exception:
1026         return False
1027
1028     # check if critical files have changed
1029     for f in savedeps_files:
1030         if f not in denv.files:
1031             return False
1032         if denv.files[f] != os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime:
1033             return False
1034
1035     # check if caches are the same
1036     for c in savedeps_caches:
1037         if c not in denv.caches or denv.caches[c] != LOCAL_CACHE(bld, c):
1038             return False
1039
1040     # check if caches are the same
1041     for e in savedeps_envvars:
1042         if e not in denv.envvar or denv.envvar[e] != bld.env[e]:
1043             return False
1044
1045     # check inputs are the same
1046     for t in tgt_list:
1047         tdeps = {}
1048         for attr in savedeps_inputs:
1049             v = getattr(t, attr, None)
1050             if v is not None:
1051                 tdeps[attr] = v
1052         if t.sname in denv.input:
1053             olddeps = denv.input[t.sname]
1054         else:
1055             olddeps = {}
1056         if tdeps != olddeps:
1057             #print '%s: \ntdeps=%s \nodeps=%s' % (t.sname, tdeps, olddeps)
1058             return False
1059
1060     # put outputs in place
1061     for t in tgt_list:
1062         if not t.sname in denv.output: continue
1063         tdeps = denv.output[t.sname]
1064         for a in tdeps:
1065             setattr(t, a, tdeps[a])
1066
1067     # put output env vars in place
1068     for t in tgt_list:
1069         if not t.sname in denv.outenv: continue
1070         tdeps = denv.outenv[t.sname]
1071         for a in tdeps:
1072             t.env[a] = tdeps[a]
1073
1074     debug('deps: loaded saved dependencies')
1075     return True
1076
1077
1078
1079 def check_project_rules(bld):
1080     '''check the project rules - ensuring the targets are sane'''
1081
1082     loops = {}
1083     inc_loops = {}
1084
1085     tgt_list = get_tgt_list(bld)
1086
1087     add_samba_attributes(bld, tgt_list)
1088
1089     force_project_rules = (Options.options.SHOWDEPS or
1090                            Options.options.SHOW_DUPLICATES)
1091
1092     if not force_project_rules and load_samba_deps(bld, tgt_list):
1093         return
1094
1095     global tstart
1096     tstart = time.clock()
1097
1098     bld.new_rules = True
1099     Logs.info("Checking project rules ...")
1100
1101     debug('deps: project rules checking started')
1102
1103     expand_subsystem_deps(bld)
1104
1105     debug("deps: expand_subsystem_deps: %f" % (time.clock() - tstart))
1106
1107     replace_grouping_libraries(bld, tgt_list)
1108
1109     debug("deps: replace_grouping_libraries: %f" % (time.clock() - tstart))
1110
1111     build_direct_deps(bld, tgt_list)
1112
1113     debug("deps: build_direct_deps: %f" % (time.clock() - tstart))
1114
1115     break_dependency_loops(bld, tgt_list)
1116
1117     debug("deps: break_dependency_loops: %f" % (time.clock() - tstart))
1118
1119     if Options.options.SHOWDEPS:
1120             show_dependencies(bld, Options.options.SHOWDEPS, set())
1121
1122     calculate_final_deps(bld, tgt_list, loops)
1123
1124     debug("deps: calculate_final_deps: %f" % (time.clock() - tstart))
1125
1126     if Options.options.SHOW_DUPLICATES:
1127             show_object_duplicates(bld, tgt_list)
1128
1129     # run the various attribute generators
1130     for f in [ build_dependencies, build_includes, add_init_functions ]:
1131         debug('deps: project rules checking %s', f)
1132         for t in tgt_list: f(t)
1133         debug("deps: %s: %f" % (f, time.clock() - tstart))
1134
1135     debug('deps: project rules stage1 completed')
1136
1137     if not check_duplicate_sources(bld, tgt_list):
1138         Logs.error("Duplicate sources present - aborting")
1139         sys.exit(1)
1140
1141     debug("deps: check_duplicate_sources: %f" % (time.clock() - tstart))
1142
1143     if not bld.check_group_ordering(tgt_list):
1144         Logs.error("Bad group ordering - aborting")
1145         sys.exit(1)
1146
1147     debug("deps: check_group_ordering: %f" % (time.clock() - tstart))
1148
1149     show_final_deps(bld, tgt_list)
1150
1151     debug("deps: show_final_deps: %f" % (time.clock() - tstart))
1152
1153     debug('deps: project rules checking completed - %u targets checked',
1154           len(tgt_list))
1155
1156     if not bld.is_install:
1157         save_samba_deps(bld, tgt_list)
1158
1159     debug("deps: save_samba_deps: %f" % (time.clock() - tstart))
1160
1161     Logs.info("Project rules pass")
1162
1163
1164 def CHECK_PROJECT_RULES(bld):
1165     '''enable checking of project targets for sanity'''
1166     if bld.env.added_project_rules:
1167         return
1168     bld.env.added_project_rules = True
1169     bld.add_pre_fun(check_project_rules)
1170 Build.BuildContext.CHECK_PROJECT_RULES = CHECK_PROJECT_RULES
1171
1172