buildtools/wafsamba/samba_deps.py

   1 # Samba automatic dependency handling and project rules
   2
   3 import os, sys, re, time
   4
   5 from waflib import Build, Options, Logs, Utils, Errors
   6 from waflib.Logs import debug
   7 from waflib.Configure import conf
   8 from waflib import ConfigSet
   9
  10 from samba_bundled import BUILTIN_LIBRARY
  11 from samba_utils import LOCAL_CACHE, TO_LIST, get_tgt_list, unique_list, os_path_relpath
  12 from samba_autoconf import library_flags
  13
  14 @conf
  15 def ADD_GLOBAL_DEPENDENCY(ctx, dep):
  16     '''add a dependency for all binaries and libraries'''
  17     if not 'GLOBAL_DEPENDENCIES' in ctx.env:
  18         ctx.env.GLOBAL_DEPENDENCIES = []
  19     ctx.env.GLOBAL_DEPENDENCIES.append(dep)
  20
  21
  22 @conf
  23 def BREAK_CIRCULAR_LIBRARY_DEPENDENCIES(ctx):
  24     '''indicate that circular dependencies between libraries should be broken.'''
  25     ctx.env.ALLOW_CIRCULAR_LIB_DEPENDENCIES = True
  26
  27
  28 @conf
  29 def SET_SYSLIB_DEPS(conf, target, deps):
  30     '''setup some implied dependencies for a SYSLIB'''
  31     cache = LOCAL_CACHE(conf, 'SYSLIB_DEPS')
  32     cache[target] = deps
  33
  34
  35 def expand_subsystem_deps(bld):
  36     '''expand the reverse dependencies resulting from subsystem
  37        attributes of modules. This is walking over the complete list
  38        of declared subsystems, and expands the samba_deps_extended list for any
  39        module<->subsystem dependencies'''
  40
  41     subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
  42     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
  43
  44     for subsystem_name in subsystem_list:
  45         bld.ASSERT(subsystem_name in targets, "Subsystem target %s not declared" % subsystem_name)
  46         type = targets[subsystem_name]
  47         if type == 'DISABLED' or type == 'EMPTY':
  48             continue
  49
  50         # for example,
  51         #    subsystem_name = dcerpc_server (a subsystem)
  52         #    subsystem      = dcerpc_server (a subsystem object)
  53         #    module_name    = rpc_epmapper (a module within the dcerpc_server subsystem)
  54         #    module         = rpc_epmapper (a module object within the dcerpc_server subsystem)
  55
  56         subsystem = bld.get_tgen_by_name(subsystem_name)
  57         bld.ASSERT(subsystem is not None, "Unable to find subsystem %s" % subsystem_name)
  58         for d in subsystem_list[subsystem_name]:
  59             module_name = d['TARGET']
  60             module_type = targets[module_name]
  61             if module_type in ['DISABLED', 'EMPTY']:
  62                 continue
  63             bld.ASSERT(subsystem is not None,
  64                        "Subsystem target %s for %s (%s) not found" % (subsystem_name, module_name, module_type))
  65             if module_type in ['SUBSYSTEM']:
  66                 # if a module is a plain object type (not a library) then the
  67                 # subsystem it is part of needs to have it as a dependency, so targets
  68                 # that depend on this subsystem get the modules of that subsystem
  69                 subsystem.samba_deps_extended.append(module_name)
  70         subsystem.samba_deps_extended = unique_list(subsystem.samba_deps_extended)
  71
  72
  73
  74 def build_dependencies(self):
  75     '''This builds the dependency list for a target. It runs after all the targets are declared
  76
  77     The reason this is not just done in the SAMBA_*() rules is that we have no way of knowing
  78     the full dependency list for a target until we have all of the targets declared.
  79     '''
  80
  81     if self.samba_type in ['LIBRARY', 'BINARY', 'PYTHON']:
  82         self.uselib        = list(self.final_syslibs)
  83         self.uselib_local  = list(self.final_libs)
  84         self.add_objects   = list(self.final_objects)
  85
  86         # extra link flags from pkg_config
  87         libs = self.final_syslibs.copy()
  88
  89         (cflags, ldflags, cpppath) = library_flags(self, list(libs))
  90         new_ldflags        = getattr(self, 'samba_ldflags', [])[:]
  91         new_ldflags.extend(ldflags)
  92         self.ldflags       = new_ldflags
  93
  94         if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ldflags:
  95             for f in self.env.undefined_ldflags:
  96                 self.ldflags.remove(f)
  97
  98         if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ignore_ldflags:
  99             for f in self.env.undefined_ignore_ldflags:
 100                 self.ldflags.append(f)
 101
 102         debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
 103               self.sname, self.uselib, self.uselib_local, self.add_objects)
 104
 105     if self.samba_type in ['SUBSYSTEM']:
 106         # this is needed for the cflags of libs that come from pkg_config
 107         self.uselib = list(self.final_syslibs)
 108         self.uselib.extend(list(self.direct_syslibs))
 109         for lib in self.final_libs:
 110             t = self.bld.get_tgen_by_name(lib)
 111             self.uselib.extend(list(t.final_syslibs))
 112         self.uselib = unique_list(self.uselib)
 113
 114     if getattr(self, 'uselib', None):
 115         up_list = []
 116         for l in self.uselib:
 117            up_list.append(l.upper())
 118         self.uselib = up_list
 119
 120
 121 def build_includes(self):
 122     '''This builds the right set of includes for a target.
 123
 124     One tricky part of this is that the includes= attribute for a
 125     target needs to use paths which are relative to that targets
 126     declaration directory (which we can get at via t.path).
 127
 128     The way this works is the includes list gets added as
 129     samba_includes in the main build task declaration. Then this
 130     function runs after all of the tasks are declared, and it
 131     processes the samba_includes attribute to produce a includes=
 132     attribute
 133     '''
 134
 135     if getattr(self, 'samba_includes', None) is None:
 136         return
 137
 138     bld = self.bld
 139
 140     inc_deps = includes_objects(bld, self, set(), {})
 141
 142     includes = []
 143
 144     # maybe add local includes
 145     if getattr(self, 'local_include', True) and getattr(self, 'local_include_first', True):
 146         includes.append('.')
 147
 148     includes.extend(self.samba_includes_extended)
 149
 150     if 'EXTRA_INCLUDES' in bld.env and getattr(self, 'global_include', True):
 151         includes.extend(bld.env['EXTRA_INCLUDES'])
 152
 153     includes.append('#')
 154
 155     inc_set = set()
 156     inc_abs = []
 157
 158     for d in inc_deps:
 159         t = bld.get_tgen_by_name(d)
 160         bld.ASSERT(t is not None, "Unable to find dependency %s for %s" % (d, self.sname))
 161         inclist = getattr(t, 'samba_includes_extended', [])[:]
 162         if getattr(t, 'local_include', True):
 163             inclist.append('.')
 164         if inclist == []:
 165             continue
 166         tpath = t.samba_abspath
 167         for inc in inclist:
 168             npath = tpath + '/' + inc
 169             if not npath in inc_set:
 170                 inc_abs.append(npath)
 171                 inc_set.add(npath)
 172
 173     mypath = self.path.abspath(bld.env)
 174     for inc in inc_abs:
 175         relpath = os_path_relpath(inc, mypath)
 176         includes.append(relpath)
 177
 178     if getattr(self, 'local_include', True) and not getattr(self, 'local_include_first', True):
 179         includes.append('.')
 180
 181     # now transform the includes list to be relative to the top directory
 182     # which is represented by '#' in waf. This allows waf to cache the
 183     # includes lists more efficiently
 184     includes_top = []
 185     for i in includes:
 186         if i[0] == '#':
 187             # some are already top based
 188             includes_top.append(i)
 189             continue
 190         absinc = os.path.join(self.path.abspath(), i)
 191         relinc = os_path_relpath(absinc, self.bld.srcnode.abspath())
 192         includes_top.append('#' + relinc)
 193
 194     self.includes = unique_list(includes_top)
 195     debug('deps: includes for target %s: includes=%s',
 196           self.sname, self.includes)
 197
 198
 199 def add_init_functions(self):
 200     '''This builds the right set of init functions'''
 201
 202     bld = self.bld
 203
 204     subsystems = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 205
 206     # cope with the separated object lists from BINARY and LIBRARY targets
 207     sname = self.sname
 208     if sname.endswith('.objlist'):
 209         sname = sname[0:-8]
 210
 211     modules = []
 212     if sname in subsystems:
 213         modules.append(sname)
 214
 215     m = getattr(self, 'samba_modules', None)
 216     if m is not None:
 217         modules.extend(TO_LIST(m))
 218
 219     m = getattr(self, 'samba_subsystem', None)
 220     if m is not None:
 221         modules.append(m)
 222
 223     if 'pyembed' in self.features:
 224         return
 225
 226     sentinel = getattr(self, 'init_function_sentinel', 'NULL')
 227
 228     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
 229     cflags = getattr(self, 'samba_cflags', [])[:]
 230
 231     if modules == []:
 232         sname = sname.replace('-','_')
 233         sname = sname.replace('.','_')
 234         sname = sname.replace('/','_')
 235         cflags.append('-DSTATIC_%s_MODULES=%s' % (sname, sentinel))
 236         if sentinel == 'NULL':
 237             proto = "extern void __%s_dummy_module_proto(void)" % (sname)
 238             cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (sname, proto))
 239         self.cflags = cflags
 240         return
 241
 242     for m in modules:
 243         bld.ASSERT(m in subsystems,
 244                    "No init_function defined for module '%s' in target '%s'" % (m, self.sname))
 245         init_fn_list = []
 246         for d in subsystems[m]:
 247             if targets[d['TARGET']] != 'DISABLED':
 248                 init_fn_list.append(d['INIT_FUNCTION'])
 249         if init_fn_list == []:
 250             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, sentinel))
 251             if sentinel == 'NULL':
 252                 proto = "extern void __%s_dummy_module_proto(void)" % (m)
 253                 cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
 254         else:
 255             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, ','.join(init_fn_list) + ',' + sentinel))
 256             proto=''
 257             for f in init_fn_list:
 258                 proto += '_MODULE_PROTO(%s)' % f
 259             proto += "extern void __%s_dummy_module_proto(void)" % (m)
 260             cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
 261     self.cflags = cflags
 262
 263
 264 def check_duplicate_sources(bld, tgt_list):
 265     '''see if we are compiling the same source file more than once'''
 266
 267     debug('deps: checking for duplicate sources')
 268     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 269
 270     for t in tgt_list:
 271         source_list = TO_LIST(getattr(t, 'source', ''))
 272         tpath = os.path.normpath(os_path_relpath(t.path.abspath(bld.env), t.env.BUILD_DIRECTORY + '/default'))
 273         obj_sources = set()
 274         for s in source_list:
 275             if not isinstance(s, str):
 276                 print('strange path in check_duplicate_sources %r' % s)
 277                 s = s.abspath()
 278             p = os.path.normpath(os.path.join(tpath, s))
 279             if p in obj_sources:
 280                 Logs.error("ERROR: source %s appears twice in target '%s'" % (p, t.sname))
 281                 sys.exit(1)
 282             obj_sources.add(p)
 283         t.samba_source_set = obj_sources
 284
 285     subsystems = {}
 286
 287     # build a list of targets that each source file is part of
 288     for t in tgt_list:
 289         if not targets[t.sname] in [ 'LIBRARY', 'BINARY', 'PYTHON' ]:
 290             continue
 291         for obj in t.add_objects:
 292             t2 = t.bld.get_tgen_by_name(obj)
 293             source_set = getattr(t2, 'samba_source_set', set())
 294             for s in source_set:
 295                 if not s in subsystems:
 296                     subsystems[s] = {}
 297                 if not t.sname in subsystems[s]:
 298                     subsystems[s][t.sname] = []
 299                 subsystems[s][t.sname].append(t2.sname)
 300
 301     for s in subsystems:
 302         if len(subsystems[s]) > 1 and Options.options.SHOW_DUPLICATES:
 303             Logs.warn("WARNING: source %s is in more than one target: %s" % (s, subsystems[s].keys()))
 304         for tname in subsystems[s]:
 305             if len(subsystems[s][tname]) > 1:
 306                 raise Errors.WafError("ERROR: source %s is in more than one subsystem of target '%s': %s" % (s, tname, subsystems[s][tname]))
 307
 308     return True
 309
 310 def check_group_ordering(bld, tgt_list):
 311     '''see if we have any dependencies that violate the group ordering
 312
 313     It is an error for a target to depend on a target from a later
 314     build group
 315     '''
 316
 317     def group_name(g):
 318         tm = bld.task_manager
 319         return [x for x in tm.groups_names if id(tm.groups_names[x]) == id(g)][0]
 320
 321     for g in bld.task_manager.groups:
 322         gname = group_name(g)
 323         for t in g.tasks_gen:
 324             t.samba_group = gname
 325
 326     grp_map = {}
 327     idx = 0
 328     for g in bld.task_manager.groups:
 329         name = group_name(g)
 330         grp_map[name] = idx
 331         idx += 1
 332
 333     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 334
 335     ret = True
 336     for t in tgt_list:
 337         tdeps = getattr(t, 'add_objects', []) + getattr(t, 'uselib_local', [])
 338         for d in tdeps:
 339             t2 = bld.get_tgen_by_name(d)
 340             if t2 is None:
 341                 continue
 342             map1 = grp_map[t.samba_group]
 343             map2 = grp_map[t2.samba_group]
 344
 345             if map2 > map1:
 346                 Logs.error("Target %r in build group %r depends on target %r from later build group %r" % (
 347                            t.sname, t.samba_group, t2.sname, t2.samba_group))
 348                 ret = False
 349
 350     return ret
 351 Build.BuildContext.check_group_ordering = check_group_ordering
 352
 353 def show_final_deps(bld, tgt_list):
 354     '''show the final dependencies for all targets'''
 355
 356     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 357
 358     for t in tgt_list:
 359         if not targets[t.sname] in ['LIBRARY', 'BINARY', 'PYTHON', 'SUBSYSTEM']:
 360             continue
 361         debug('deps: final dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
 362               t.sname, t.uselib, getattr(t, 'uselib_local', []), getattr(t, 'add_objects', []))
 363
 364
 365 def add_samba_attributes(bld, tgt_list):
 366     '''ensure a target has a the required samba attributes'''
 367
 368     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 369
 370     for t in tgt_list:
 371         if t.name != '':
 372             t.sname = t.name
 373         else:
 374             t.sname = t.target
 375         t.samba_type = targets[t.sname]
 376         t.samba_abspath = t.path.abspath(bld.env)
 377         t.samba_deps_extended = t.samba_deps[:]
 378         t.samba_includes_extended = TO_LIST(t.samba_includes)[:]
 379         t.cflags = getattr(t, 'samba_cflags', '')
 380
 381 def replace_grouping_libraries(bld, tgt_list):
 382     '''replace dependencies based on grouping libraries
 383
 384     If a library is marked as a grouping library, then any target that
 385     depends on a subsystem that is part of that grouping library gets
 386     that dependency replaced with a dependency on the grouping library
 387     '''
 388
 389     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 390
 391     grouping = {}
 392
 393     # find our list of grouping libraries, mapped from the subsystems they depend on
 394     for t in tgt_list:
 395         if not getattr(t, 'grouping_library', False):
 396             continue
 397         for dep in t.samba_deps_extended:
 398             bld.ASSERT(dep in targets, "grouping library target %s not declared in %s" % (dep, t.sname))
 399             if targets[dep] == 'SUBSYSTEM':
 400                 grouping[dep] = t.sname
 401
 402     # now replace any dependencies on elements of grouping libraries
 403     for t in tgt_list:
 404         for i in range(len(t.samba_deps_extended)):
 405             dep = t.samba_deps_extended[i]
 406             if dep in grouping:
 407                 if t.sname != grouping[dep]:
 408                     debug("deps: target %s: replacing dependency %s with grouping library %s" % (t.sname, dep, grouping[dep]))
 409                     t.samba_deps_extended[i] = grouping[dep]
 410
 411
 412
 413 def build_direct_deps(bld, tgt_list):
 414     '''build the direct_objects and direct_libs sets for each target'''
 415
 416     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 417     syslib_deps  = LOCAL_CACHE(bld, 'SYSLIB_DEPS')
 418
 419     global_deps = bld.env.GLOBAL_DEPENDENCIES
 420     global_deps_exclude = set()
 421     for dep in global_deps:
 422         t = bld.get_tgen_by_name(dep)
 423         for d in t.samba_deps:
 424             # prevent loops from the global dependencies list
 425             global_deps_exclude.add(d)
 426             global_deps_exclude.add(d + '.objlist')
 427
 428     for t in tgt_list:
 429         t.direct_objects = set()
 430         t.direct_libs = set()
 431         t.direct_syslibs = set()
 432         deps = t.samba_deps_extended[:]
 433         if getattr(t, 'samba_use_global_deps', False) and not t.sname in global_deps_exclude:
 434             deps.extend(global_deps)
 435         for d in deps:
 436             if d == t.sname: continue
 437             if not d in targets:
 438                 Logs.error("Unknown dependency '%s' in '%s'" % (d, t.sname))
 439                 sys.exit(1)
 440             if targets[d] in [ 'EMPTY', 'DISABLED' ]:
 441                 continue
 442             if targets[d] == 'PYTHON' and targets[t.sname] != 'PYTHON' and t.sname.find('.objlist') == -1:
 443                 # this check should be more restrictive, but for now we have pidl-generated python
 444                 # code that directly depends on other python modules
 445                 Logs.error('ERROR: Target %s has dependency on python module %s' % (t.sname, d))
 446                 sys.exit(1)
 447             if targets[d] == 'SYSLIB':
 448                 t.direct_syslibs.add(d)
 449                 if d in syslib_deps:
 450                     for implied in TO_LIST(syslib_deps[d]):
 451                         if BUILTIN_LIBRARY(bld, implied):
 452                             t.direct_objects.add(implied)
 453                         elif targets[implied] == 'SYSLIB':
 454                             t.direct_syslibs.add(implied)
 455                         elif targets[implied] in ['LIBRARY', 'MODULE']:
 456                             t.direct_libs.add(implied)
 457                         else:
 458                             Logs.error('Implied dependency %s in %s is of type %s' % (
 459                                 implied, t.sname, targets[implied]))
 460                             sys.exit(1)
 461                 continue
 462             t2 = bld.get_tgen_by_name(d)
 463             if t2 is None:
 464                 Logs.error("no task %s of type %s in %s" % (d, targets[d], t.sname))
 465                 sys.exit(1)
 466             if t2.samba_type in [ 'LIBRARY', 'MODULE' ]:
 467                 t.direct_libs.add(d)
 468             elif t2.samba_type in [ 'SUBSYSTEM', 'ASN1', 'PYTHON' ]:
 469                 t.direct_objects.add(d)
 470     debug('deps: built direct dependencies')
 471
 472
 473 def dependency_loop(loops, t, target):
 474     '''add a dependency loop to the loops dictionary'''
 475     if t.sname == target:
 476         return
 477     if not target in loops:
 478         loops[target] = set()
 479     if not t.sname in loops[target]:
 480         loops[target].add(t.sname)
 481
 482
 483 def indirect_libs(bld, t, chain, loops):
 484     '''recursively calculate the indirect library dependencies for a target
 485
 486     An indirect library is a library that results from a dependency on
 487     a subsystem
 488     '''
 489
 490     ret = getattr(t, 'indirect_libs', None)
 491     if ret is not None:
 492         return ret
 493
 494     ret = set()
 495     for obj in t.direct_objects:
 496         if obj in chain:
 497             dependency_loop(loops, t, obj)
 498             continue
 499         chain.add(obj)
 500         t2 = bld.get_tgen_by_name(obj)
 501         r2 = indirect_libs(bld, t2, chain, loops)
 502         chain.remove(obj)
 503         ret = ret.union(t2.direct_libs)
 504         ret = ret.union(r2)
 505
 506     for obj in indirect_objects(bld, t, set(), loops):
 507         if obj in chain:
 508             dependency_loop(loops, t, obj)
 509             continue
 510         chain.add(obj)
 511         t2 = bld.get_tgen_by_name(obj)
 512         r2 = indirect_libs(bld, t2, chain, loops)
 513         chain.remove(obj)
 514         ret = ret.union(t2.direct_libs)
 515         ret = ret.union(r2)
 516
 517     t.indirect_libs = ret
 518
 519     return ret
 520
 521
 522 def indirect_objects(bld, t, chain, loops):
 523     '''recursively calculate the indirect object dependencies for a target
 524
 525     indirect objects are the set of objects from expanding the
 526     subsystem dependencies
 527     '''
 528
 529     ret = getattr(t, 'indirect_objects', None)
 530     if ret is not None: return ret
 531
 532     ret = set()
 533     for lib in t.direct_objects:
 534         if lib in chain:
 535             dependency_loop(loops, t, lib)
 536             continue
 537         chain.add(lib)
 538         t2 = bld.get_tgen_by_name(lib)
 539         r2 = indirect_objects(bld, t2, chain, loops)
 540         chain.remove(lib)
 541         ret = ret.union(t2.direct_objects)
 542         ret = ret.union(r2)
 543
 544     t.indirect_objects = ret
 545     return ret
 546
 547
 548 def extended_objects(bld, t, chain):
 549     '''recursively calculate the extended object dependencies for a target
 550
 551     extended objects are the union of:
 552        - direct objects
 553        - indirect objects
 554        - direct and indirect objects of all direct and indirect libraries
 555     '''
 556
 557     ret = getattr(t, 'extended_objects', None)
 558     if ret is not None: return ret
 559
 560     ret = set()
 561     ret = ret.union(t.final_objects)
 562
 563     for lib in t.final_libs:
 564         if lib in chain:
 565             continue
 566         t2 = bld.get_tgen_by_name(lib)
 567         chain.add(lib)
 568         r2 = extended_objects(bld, t2, chain)
 569         chain.remove(lib)
 570         ret = ret.union(t2.final_objects)
 571         ret = ret.union(r2)
 572
 573     t.extended_objects = ret
 574     return ret
 575
 576
 577 def includes_objects(bld, t, chain, inc_loops):
 578     '''recursively calculate the includes object dependencies for a target
 579
 580     includes dependencies come from either library or object dependencies
 581     '''
 582     ret = getattr(t, 'includes_objects', None)
 583     if ret is not None:
 584         return ret
 585
 586     ret = t.direct_objects.copy()
 587     ret = ret.union(t.direct_libs)
 588
 589     for obj in t.direct_objects:
 590         if obj in chain:
 591             dependency_loop(inc_loops, t, obj)
 592             continue
 593         chain.add(obj)
 594         t2 = bld.get_tgen_by_name(obj)
 595         r2 = includes_objects(bld, t2, chain, inc_loops)
 596         chain.remove(obj)
 597         ret = ret.union(t2.direct_objects)
 598         ret = ret.union(r2)
 599
 600     for lib in t.direct_libs:
 601         if lib in chain:
 602             dependency_loop(inc_loops, t, lib)
 603             continue
 604         chain.add(lib)
 605         t2 = bld.get_tgen_by_name(lib)
 606         if t2 is None:
 607             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 608             Logs.error('Target %s of type %s not found in direct_libs for %s' % (
 609                 lib, targets[lib], t.sname))
 610             sys.exit(1)
 611         r2 = includes_objects(bld, t2, chain, inc_loops)
 612         chain.remove(lib)
 613         ret = ret.union(t2.direct_objects)
 614         ret = ret.union(r2)
 615
 616     t.includes_objects = ret
 617     return ret
 618
 619
 620 def break_dependency_loops(bld, tgt_list):
 621     '''find and break dependency loops'''
 622     loops = {}
 623     inc_loops = {}
 624
 625     # build up the list of loops
 626     for t in tgt_list:
 627         indirect_objects(bld, t, set(), loops)
 628         indirect_libs(bld, t, set(), loops)
 629         includes_objects(bld, t, set(), inc_loops)
 630
 631     # break the loops
 632     for t in tgt_list:
 633         if t.sname in loops:
 634             for attr in ['direct_objects', 'indirect_objects', 'direct_libs', 'indirect_libs']:
 635                 objs = getattr(t, attr, set())
 636                 setattr(t, attr, objs.difference(loops[t.sname]))
 637
 638     for loop in loops:
 639         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 640
 641     for loop in inc_loops:
 642         debug('deps: Found include loops for target %s : %s', loop, inc_loops[loop])
 643
 644     # expand the loops mapping by one level
 645     for loop in loops.copy():
 646         for tgt in loops[loop]:
 647             if tgt in loops:
 648                 loops[loop] = loops[loop].union(loops[tgt])
 649
 650     for loop in inc_loops.copy():
 651         for tgt in inc_loops[loop]:
 652             if tgt in inc_loops:
 653                 inc_loops[loop] = inc_loops[loop].union(inc_loops[tgt])
 654
 655
 656     # expand indirect subsystem and library loops
 657     for loop in loops.copy():
 658         t = bld.get_tgen_by_name(loop)
 659         if t.samba_type in ['SUBSYSTEM']:
 660             loops[loop] = loops[loop].union(t.indirect_objects)
 661             loops[loop] = loops[loop].union(t.direct_objects)
 662         if t.samba_type in ['LIBRARY','PYTHON']:
 663             loops[loop] = loops[loop].union(t.indirect_libs)
 664             loops[loop] = loops[loop].union(t.direct_libs)
 665         if loop in loops[loop]:
 666             loops[loop].remove(loop)
 667
 668     # expand indirect includes loops
 669     for loop in inc_loops.copy():
 670         t = bld.get_tgen_by_name(loop)
 671         inc_loops[loop] = inc_loops[loop].union(t.includes_objects)
 672         if loop in inc_loops[loop]:
 673             inc_loops[loop].remove(loop)
 674
 675     # add in the replacement dependencies
 676     for t in tgt_list:
 677         for loop in loops:
 678             for attr in ['indirect_objects', 'indirect_libs']:
 679                 objs = getattr(t, attr, set())
 680                 if loop in objs:
 681                     diff = loops[loop].difference(objs)
 682                     if t.sname in diff:
 683                         diff.remove(t.sname)
 684                     if diff:
 685                         debug('deps: Expanded target %s of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 686                         objs = objs.union(diff)
 687                 setattr(t, attr, objs)
 688
 689         for loop in inc_loops:
 690             objs = getattr(t, 'includes_objects', set())
 691             if loop in objs:
 692                 diff = inc_loops[loop].difference(objs)
 693                 if t.sname in diff:
 694                     diff.remove(t.sname)
 695                 if diff:
 696                     debug('deps: Expanded target %s includes of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 697                     objs = objs.union(diff)
 698             setattr(t, 'includes_objects', objs)
 699
 700
 701 def reduce_objects(bld, tgt_list):
 702     '''reduce objects by looking for indirect object dependencies'''
 703     rely_on = {}
 704
 705     for t in tgt_list:
 706         t.extended_objects = None
 707
 708     changed = False
 709
 710     for type in ['BINARY', 'PYTHON', 'LIBRARY']:
 711         for t in tgt_list:
 712             if t.samba_type != type: continue
 713             # if we will indirectly link to a target then we don't need it
 714             new = t.final_objects.copy()
 715             for l in t.final_libs:
 716                 t2 = bld.get_tgen_by_name(l)
 717                 t2_obj = extended_objects(bld, t2, set())
 718                 dup = new.intersection(t2_obj)
 719                 if t.sname in rely_on:
 720                     dup = dup.difference(rely_on[t.sname])
 721                 if dup:
 722                     debug('deps: removing dups from %s of type %s: %s also in %s %s',
 723                           t.sname, t.samba_type, dup, t2.samba_type, l)
 724                     new = new.difference(dup)
 725                     changed = True
 726                     if not l in rely_on:
 727                         rely_on[l] = set()
 728                     rely_on[l] = rely_on[l].union(dup)
 729             t.final_objects = new
 730
 731     if not changed:
 732         return False
 733
 734     # add back in any objects that were relied upon by the reduction rules
 735     for r in rely_on:
 736         t = bld.get_tgen_by_name(r)
 737         t.final_objects = t.final_objects.union(rely_on[r])
 738
 739     return True
 740
 741
 742 def show_library_loop(bld, lib1, lib2, path, seen):
 743     '''show the detailed path of a library loop between lib1 and lib2'''
 744
 745     t = bld.get_tgen_by_name(lib1)
 746     if not lib2 in getattr(t, 'final_libs', set()):
 747         return
 748
 749     for d in t.samba_deps_extended:
 750         if d in seen:
 751             continue
 752         seen.add(d)
 753         path2 = path + '=>' + d
 754         if d == lib2:
 755             Logs.warn('library loop path: ' + path2)
 756             return
 757         show_library_loop(bld, d, lib2, path2, seen)
 758         seen.remove(d)
 759
 760
 761 def calculate_final_deps(bld, tgt_list, loops):
 762     '''calculate the final library and object dependencies'''
 763     for t in tgt_list:
 764         # start with the maximum possible list
 765         t.final_libs    = t.direct_libs.union(indirect_libs(bld, t, set(), loops))
 766         t.final_objects = t.direct_objects.union(indirect_objects(bld, t, set(), loops))
 767
 768     for t in tgt_list:
 769         # don't depend on ourselves
 770         if t.sname in t.final_libs:
 771             t.final_libs.remove(t.sname)
 772         if t.sname in t.final_objects:
 773             t.final_objects.remove(t.sname)
 774
 775     # handle any non-shared binaries
 776     for t in tgt_list:
 777         if t.samba_type == 'BINARY' and bld.NONSHARED_BINARY(t.sname):
 778             subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 779             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 780
 781             # replace lib deps with objlist deps
 782             for l in t.final_libs:
 783                 objname = l + '.objlist'
 784                 t2 = bld.get_tgen_by_name(objname)
 785                 if t2 is None:
 786                     Logs.error('ERROR: subsystem %s not found' % objname)
 787                     sys.exit(1)
 788                 t.final_objects.add(objname)
 789                 t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 790                 if l in subsystem_list:
 791                     # its a subsystem - we also need the contents of any modules
 792                     for d in subsystem_list[l]:
 793                         module_name = d['TARGET']
 794                         if targets[module_name] == 'LIBRARY':
 795                             objname = module_name + '.objlist'
 796                         elif targets[module_name] == 'SUBSYSTEM':
 797                             objname = module_name
 798                         else:
 799                             continue
 800                         t2 = bld.get_tgen_by_name(objname)
 801                         if t2 is None:
 802                             Logs.error('ERROR: subsystem %s not found' % objname)
 803                             sys.exit(1)
 804                         t.final_objects.add(objname)
 805                         t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 806             t.final_libs = set()
 807
 808     # find any library loops
 809     for t in tgt_list:
 810         if t.samba_type in ['LIBRARY', 'PYTHON']:
 811             for l in t.final_libs.copy():
 812                 t2 = bld.get_tgen_by_name(l)
 813                 if t.sname in t2.final_libs:
 814                     if getattr(bld.env, "ALLOW_CIRCULAR_LIB_DEPENDENCIES", False):
 815                         # we could break this in either direction. If one of the libraries
 816                         # has a version number, and will this be distributed publicly, then
 817                         # we should make it the lower level library in the DAG
 818                         Logs.warn('deps: removing library loop %s from %s' % (t.sname, t2.sname))
 819                         dependency_loop(loops, t, t2.sname)
 820                         t2.final_libs.remove(t.sname)
 821                     else:
 822                         Logs.error('ERROR: circular library dependency between %s and %s'
 823                             % (t.sname, t2.sname))
 824                         show_library_loop(bld, t.sname, t2.sname, t.sname, set())
 825                         show_library_loop(bld, t2.sname, t.sname, t2.sname, set())
 826                         sys.exit(1)
 827
 828     for loop in loops:
 829         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 830
 831     # we now need to make corrections for any library loops we broke up
 832     # any target that depended on the target of the loop and doesn't
 833     # depend on the source of the loop needs to get the loop source added
 834     for type in ['BINARY','PYTHON','LIBRARY','BINARY']:
 835         for t in tgt_list:
 836             if t.samba_type != type: continue
 837             for loop in loops:
 838                 if loop in t.final_libs:
 839                     diff = loops[loop].difference(t.final_libs)
 840                     if t.sname in diff:
 841                         diff.remove(t.sname)
 842                     if t.sname in diff:
 843                         diff.remove(t.sname)
 844                     # make sure we don't recreate the loop again!
 845                     for d in diff.copy():
 846                         t2 = bld.get_tgen_by_name(d)
 847                         if t2.samba_type == 'LIBRARY':
 848                             if t.sname in t2.final_libs:
 849                                 debug('deps: removing expansion %s from %s', d, t.sname)
 850                                 diff.remove(d)
 851                     if diff:
 852                         debug('deps: Expanded target %s by loop %s libraries (loop %s) %s', t.sname, loop,
 853                               loops[loop], diff)
 854                         t.final_libs = t.final_libs.union(diff)
 855
 856     # remove objects that are also available in linked libs
 857     count = 0
 858     while reduce_objects(bld, tgt_list):
 859         count += 1
 860         if count > 100:
 861             Logs.warn("WARNING: Unable to remove all inter-target object duplicates")
 862             break
 863     debug('deps: Object reduction took %u iterations', count)
 864
 865     # add in any syslib dependencies
 866     for t in tgt_list:
 867         if not t.samba_type in ['BINARY','PYTHON','LIBRARY','SUBSYSTEM']:
 868             continue
 869         syslibs = set()
 870         for d in t.final_objects:
 871             t2 = bld.get_tgen_by_name(d)
 872             syslibs = syslibs.union(t2.direct_syslibs)
 873         # this adds the indirect syslibs as well, which may not be needed
 874         # depending on the linker flags
 875         for d in t.final_libs:
 876             t2 = bld.get_tgen_by_name(d)
 877             syslibs = syslibs.union(t2.direct_syslibs)
 878         t.final_syslibs = syslibs
 879
 880
 881     # find any unresolved library loops
 882     lib_loop_error = False
 883     for t in tgt_list:
 884         if t.samba_type in ['LIBRARY', 'PYTHON']:
 885             for l in t.final_libs.copy():
 886                 t2 = bld.get_tgen_by_name(l)
 887                 if t.sname in t2.final_libs:
 888                     Logs.error('ERROR: Unresolved library loop %s from %s' % (t.sname, t2.sname))
 889                     lib_loop_error = True
 890     if lib_loop_error:
 891         sys.exit(1)
 892
 893     debug('deps: removed duplicate dependencies')
 894
 895
 896 def show_dependencies(bld, target, seen):
 897     '''recursively show the dependencies of target'''
 898
 899     if target in seen:
 900         return
 901
 902     t = bld.get_tgen_by_name(target)
 903     if t is None:
 904         Logs.error("ERROR: Unable to find target '%s'" % target)
 905         sys.exit(1)
 906
 907     Logs.info('%s(OBJECTS): %s' % (target, t.direct_objects))
 908     Logs.info('%s(LIBS): %s' % (target, t.direct_libs))
 909     Logs.info('%s(SYSLIBS): %s' % (target, t.direct_syslibs))
 910
 911     seen.add(target)
 912
 913     for t2 in t.direct_objects:
 914         show_dependencies(bld, t2, seen)
 915
 916
 917 def show_object_duplicates(bld, tgt_list):
 918     '''show a list of object files that are included in more than
 919     one library or binary'''
 920
 921     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 922
 923     used_by = {}
 924
 925     Logs.info("showing duplicate objects")
 926
 927     for t in tgt_list:
 928         if not targets[t.sname] in [ 'LIBRARY', 'PYTHON' ]:
 929             continue
 930         for n in getattr(t, 'final_objects', set()):
 931             t2 = bld.get_tgen_by_name(n)
 932             if not n in used_by:
 933                 used_by[n] = set()
 934             used_by[n].add(t.sname)
 935
 936     for n in used_by:
 937         if len(used_by[n]) > 1:
 938             Logs.info("target '%s' is used by %s" % (n, used_by[n]))
 939
 940     Logs.info("showing indirect dependency counts (sorted by count)")
 941
 942     def indirect_count(t1, t2):
 943         return len(t2.indirect_objects) - len(t1.indirect_objects)
 944
 945     sorted_list = sorted(tgt_list, cmp=indirect_count)
 946     for t in sorted_list:
 947         if len(t.indirect_objects) > 1:
 948             Logs.info("%s depends on %u indirect objects" % (t.sname, len(t.indirect_objects)))
 949
 950
 951 ######################################################################
 952 # this provides a way to save our dependency calculations between runs
 953 savedeps_version = 3
 954 savedeps_inputs  = ['samba_deps', 'samba_includes', 'local_include', 'local_include_first', 'samba_cflags',
 955                     'source', 'grouping_library', 'samba_ldflags', 'allow_undefined_symbols',
 956                     'use_global_deps', 'global_include' ]
 957 savedeps_outputs = ['uselib', 'uselib_local', 'add_objects', 'includes',
 958                     'cflags', 'ldflags', 'samba_deps_extended', 'final_libs']
 959 savedeps_outenv  = ['INC_PATHS']
 960 savedeps_envvars = ['NONSHARED_BINARIES', 'GLOBAL_DEPENDENCIES', 'EXTRA_CFLAGS', 'EXTRA_LDFLAGS', 'EXTRA_INCLUDES' ]
 961 savedeps_caches  = ['GLOBAL_DEPENDENCIES', 'TARGET_TYPE', 'INIT_FUNCTIONS', 'SYSLIB_DEPS']
 962 savedeps_files   = ['buildtools/wafsamba/samba_deps.py']
 963
 964 def save_samba_deps(bld, tgt_list):
 965     '''save the dependency calculations between builds, to make
 966        further builds faster'''
 967     denv = ConfigSet.ConfigSet()
 968
 969     denv.version = savedeps_version
 970     denv.savedeps_inputs = savedeps_inputs
 971     denv.savedeps_outputs = savedeps_outputs
 972     denv.input = {}
 973     denv.output = {}
 974     denv.outenv = {}
 975     denv.caches = {}
 976     denv.envvar = {}
 977     denv.files  = {}
 978
 979     for f in savedeps_files:
 980         denv.files[f] = os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime
 981
 982     for c in savedeps_caches:
 983         denv.caches[c] = LOCAL_CACHE(bld, c)
 984
 985     for e in savedeps_envvars:
 986         denv.envvar[e] = bld.env[e]
 987
 988     for t in tgt_list:
 989         # save all the input attributes for each target
 990         tdeps = {}
 991         for attr in savedeps_inputs:
 992             v = getattr(t, attr, None)
 993             if v is not None:
 994                 tdeps[attr] = v
 995         if tdeps != {}:
 996             denv.input[t.sname] = tdeps
 997
 998         # save all the output attributes for each target
 999         tdeps = {}
1000         for attr in savedeps_outputs:
1001             v = getattr(t, attr, None)
1002             if v is not None:
1003                 tdeps[attr] = v
1004         if tdeps != {}:
1005             denv.output[t.sname] = tdeps
1006
1007         tdeps = {}
1008         for attr in savedeps_outenv:
1009             if attr in t.env:
1010                 tdeps[attr] = t.env[attr]
1011         if tdeps != {}:
1012             denv.outenv[t.sname] = tdeps
1013
1014     depsfile = os.path.join(bld.cache_dir, "sambadeps")
1015     denv.store_fast(depsfile)
1016
1017
1018
1019 def load_samba_deps(bld, tgt_list):
1020     '''load a previous set of build dependencies if possible'''
1021     depsfile = os.path.join(bld.cache_dir, "sambadeps")
1022     denv = ConfigSet.ConfigSet()
1023     try:
1024         debug('deps: checking saved dependencies')
1025         denv.load_fast(depsfile)
1026         if (denv.version != savedeps_version or
1027             denv.savedeps_inputs != savedeps_inputs or
1028             denv.savedeps_outputs != savedeps_outputs):
1029             return False
1030     except Exception:
1031         return False
1032
1033     # check if critical files have changed
1034     for f in savedeps_files:
1035         if f not in denv.files:
1036             return False
1037         if denv.files[f] != os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime:
1038             return False
1039
1040     # check if caches are the same
1041     for c in savedeps_caches:
1042         if c not in denv.caches or denv.caches[c] != LOCAL_CACHE(bld, c):
1043             return False
1044
1045     # check if caches are the same
1046     for e in savedeps_envvars:
1047         if e not in denv.envvar or denv.envvar[e] != bld.env[e]:
1048             return False
1049
1050     # check inputs are the same
1051     for t in tgt_list:
1052         tdeps = {}
1053         for attr in savedeps_inputs:
1054             v = getattr(t, attr, None)
1055             if v is not None:
1056                 tdeps[attr] = v
1057         if t.sname in denv.input:
1058             olddeps = denv.input[t.sname]
1059         else:
1060             olddeps = {}
1061         if tdeps != olddeps:
1062             #print '%s: \ntdeps=%s \nodeps=%s' % (t.sname, tdeps, olddeps)
1063             return False
1064
1065     # put outputs in place
1066     for t in tgt_list:
1067         if not t.sname in denv.output: continue
1068         tdeps = denv.output[t.sname]
1069         for a in tdeps:
1070             setattr(t, a, tdeps[a])
1071
1072     # put output env vars in place
1073     for t in tgt_list:
1074         if not t.sname in denv.outenv: continue
1075         tdeps = denv.outenv[t.sname]
1076         for a in tdeps:
1077             t.env[a] = tdeps[a]
1078
1079     debug('deps: loaded saved dependencies')
1080     return True
1081
1082
1083
1084 def check_project_rules(bld):
1085     '''check the project rules - ensuring the targets are sane'''
1086
1087     loops = {}
1088     inc_loops = {}
1089
1090     tgt_list = get_tgt_list(bld)
1091
1092     add_samba_attributes(bld, tgt_list)
1093
1094     force_project_rules = (Options.options.SHOWDEPS or
1095                            Options.options.SHOW_DUPLICATES)
1096
1097     if not force_project_rules and load_samba_deps(bld, tgt_list):
1098         return
1099
1100     global tstart
1101     tstart = time.clock()
1102
1103     bld.new_rules = True
1104     Logs.info("Checking project rules ...")
1105
1106     debug('deps: project rules checking started')
1107
1108     expand_subsystem_deps(bld)
1109
1110     debug("deps: expand_subsystem_deps: %f" % (time.clock() - tstart))
1111
1112     replace_grouping_libraries(bld, tgt_list)
1113
1114     debug("deps: replace_grouping_libraries: %f" % (time.clock() - tstart))
1115
1116     build_direct_deps(bld, tgt_list)
1117
1118     debug("deps: build_direct_deps: %f" % (time.clock() - tstart))
1119
1120     break_dependency_loops(bld, tgt_list)
1121
1122     debug("deps: break_dependency_loops: %f" % (time.clock() - tstart))
1123
1124     if Options.options.SHOWDEPS:
1125             show_dependencies(bld, Options.options.SHOWDEPS, set())
1126
1127     calculate_final_deps(bld, tgt_list, loops)
1128
1129     debug("deps: calculate_final_deps: %f" % (time.clock() - tstart))
1130
1131     if Options.options.SHOW_DUPLICATES:
1132             show_object_duplicates(bld, tgt_list)
1133
1134     # run the various attribute generators
1135     for f in [ build_dependencies, build_includes, add_init_functions ]:
1136         debug('deps: project rules checking %s', f)
1137         for t in tgt_list: f(t)
1138         debug("deps: %s: %f" % (f, time.clock() - tstart))
1139
1140     debug('deps: project rules stage1 completed')
1141
1142     if not check_duplicate_sources(bld, tgt_list):
1143         Logs.error("Duplicate sources present - aborting")
1144         sys.exit(1)
1145
1146     debug("deps: check_duplicate_sources: %f" % (time.clock() - tstart))
1147
1148     if not bld.check_group_ordering(tgt_list):
1149         Logs.error("Bad group ordering - aborting")
1150         sys.exit(1)
1151
1152     debug("deps: check_group_ordering: %f" % (time.clock() - tstart))
1153
1154     show_final_deps(bld, tgt_list)
1155
1156     debug("deps: show_final_deps: %f" % (time.clock() - tstart))
1157
1158     debug('deps: project rules checking completed - %u targets checked',
1159           len(tgt_list))
1160
1161     if not bld.is_install:
1162         save_samba_deps(bld, tgt_list)
1163
1164     debug("deps: save_samba_deps: %f" % (time.clock() - tstart))
1165
1166     Logs.info("Project rules pass")
1167
1168
1169 def CHECK_PROJECT_RULES(bld):
1170     '''enable checking of project targets for sanity'''
1171     if bld.env.added_project_rules:
1172         return
1173     bld.env.added_project_rules = True
1174     bld.add_pre_fun(check_project_rules)
1175 Build.BuildContext.CHECK_PROJECT_RULES = CHECK_PROJECT_RULES
1176
1177