buildtools/wafsamba/samba_deps.py

   1 # Samba automatic dependency handling and project rules
   2
   3 import Build, os, sys, re, Environment, Logs, time
   4 from samba_utils import *
   5 from samba_autoconf import *
   6 from samba_bundled import BUILTIN_LIBRARY
   7
   8 @conf
   9 def ADD_GLOBAL_DEPENDENCY(ctx, dep):
  10     '''add a dependency for all binaries and libraries'''
  11     if not 'GLOBAL_DEPENDENCIES' in ctx.env:
  12         ctx.env.GLOBAL_DEPENDENCIES = []
  13     ctx.env.GLOBAL_DEPENDENCIES.append(dep)
  14
  15
  16 @conf
  17 def BREAK_CIRCULAR_LIBRARY_DEPENDENCIES(ctx):
  18     '''indicate that circular dependencies between libraries should be broken.'''
  19     ctx.env.ALLOW_CIRCULAR_LIB_DEPENDENCIES = True
  20
  21
  22 @conf
  23 def SET_SYSLIB_DEPS(conf, target, deps):
  24     '''setup some implied dependencies for a SYSLIB'''
  25     cache = LOCAL_CACHE(conf, 'SYSLIB_DEPS')
  26     cache[target] = deps
  27
  28
  29 def expand_subsystem_deps(bld):
  30     '''expand the reverse dependencies resulting from subsystem
  31        attributes of modules. This is walking over the complete list
  32        of declared subsystems, and expands the samba_deps_extended list for any
  33        module<->subsystem dependencies'''
  34
  35     subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
  36     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
  37
  38     for subsystem_name in subsystem_list:
  39         bld.ASSERT(subsystem_name in targets, "Subsystem target %s not declared" % subsystem_name)
  40         type = targets[subsystem_name]
  41         if type == 'DISABLED' or type == 'EMPTY':
  42             continue
  43
  44         # for example,
  45         #    subsystem_name = dcerpc_server (a subsystem)
  46         #    subsystem      = dcerpc_server (a subsystem object)
  47         #    module_name    = rpc_epmapper (a module within the dcerpc_server subsystem)
  48         #    module         = rpc_epmapper (a module object within the dcerpc_server subsystem)
  49
  50         subsystem = bld.name_to_obj(subsystem_name, bld.env)
  51         bld.ASSERT(subsystem is not None, "Unable to find subsystem %s" % subsystem_name)
  52         for d in subsystem_list[subsystem_name]:
  53             module_name = d['TARGET']
  54             module_type = targets[module_name]
  55             if module_type in ['DISABLED', 'EMPTY']:
  56                 continue
  57             bld.ASSERT(subsystem is not None,
  58                        "Subsystem target %s for %s (%s) not found" % (subsystem_name, module_name, module_type))
  59             if module_type in ['SUBSYSTEM']:
  60                 # if a module is a plain object type (not a library) then the
  61                 # subsystem it is part of needs to have it as a dependency, so targets
  62                 # that depend on this subsystem get the modules of that subsystem
  63                 subsystem.samba_deps_extended.append(module_name)
  64         subsystem.samba_deps_extended = unique_list(subsystem.samba_deps_extended)
  65
  66
  67
  68 def build_dependencies(self):
  69     '''This builds the dependency list for a target. It runs after all the targets are declared
  70
  71     The reason this is not just done in the SAMBA_*() rules is that we have no way of knowing
  72     the full dependency list for a target until we have all of the targets declared.
  73     '''
  74
  75     if self.samba_type in ['LIBRARY', 'BINARY', 'PYTHON']:
  76         self.uselib        = list(self.final_syslibs)
  77         self.uselib_local  = list(self.final_libs)
  78         self.add_objects   = list(self.final_objects)
  79
  80         # extra link flags from pkg_config
  81         libs = self.final_syslibs.copy()
  82
  83         (ccflags, ldflags, cpppath) = library_flags(self, list(libs))
  84         new_ldflags        = getattr(self, 'samba_ldflags', [])[:]
  85         new_ldflags.extend(ldflags)
  86         self.ldflags       = new_ldflags
  87
  88         if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ldflags:
  89             for f in self.env.undefined_ldflags:
  90                 self.ldflags.remove(f)
  91
  92         if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ignore_ldflags:
  93             for f in self.env.undefined_ignore_ldflags:
  94                 self.ldflags.append(f)
  95
  96         debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
  97               self.sname, self.uselib, self.uselib_local, self.add_objects)
  98
  99     if self.samba_type in ['SUBSYSTEM']:
 100         # this is needed for the ccflags of libs that come from pkg_config
 101         self.uselib = list(self.final_syslibs)
 102         self.uselib.extend(list(self.direct_syslibs))
 103         for lib in self.final_libs:
 104             t = self.bld.name_to_obj(lib, self.bld.env)
 105             self.uselib.extend(list(t.final_syslibs))
 106         self.uselib = unique_list(self.uselib)
 107
 108     if getattr(self, 'uselib', None):
 109         up_list = []
 110         for l in self.uselib:
 111            up_list.append(l.upper())
 112         self.uselib = up_list
 113
 114
 115 def build_includes(self):
 116     '''This builds the right set of includes for a target.
 117
 118     One tricky part of this is that the includes= attribute for a
 119     target needs to use paths which are relative to that targets
 120     declaration directory (which we can get at via t.path).
 121
 122     The way this works is the includes list gets added as
 123     samba_includes in the main build task declaration. Then this
 124     function runs after all of the tasks are declared, and it
 125     processes the samba_includes attribute to produce a includes=
 126     attribute
 127     '''
 128
 129     if getattr(self, 'samba_includes', None) is None:
 130         return
 131
 132     bld = self.bld
 133
 134     inc_deps = includes_objects(bld, self, set(), {})
 135
 136     includes = []
 137
 138     # maybe add local includes
 139     if getattr(self, 'local_include', True) == True and getattr(self, 'local_include_first', True):
 140         includes.append('.')
 141
 142     includes.extend(self.samba_includes_extended)
 143
 144     if 'EXTRA_INCLUDES' in bld.env and getattr(self, 'global_include', True):
 145         includes.extend(bld.env['EXTRA_INCLUDES'])
 146
 147     includes.append('#')
 148
 149     inc_set = set()
 150     inc_abs = []
 151
 152     for d in inc_deps:
 153         t = bld.name_to_obj(d, bld.env)
 154         bld.ASSERT(t is not None, "Unable to find dependency %s for %s" % (d, self.sname))
 155         inclist = getattr(t, 'samba_includes_extended', [])[:]
 156         if getattr(t, 'local_include', True) == True:
 157             inclist.append('.')
 158         if inclist == []:
 159             continue
 160         tpath = t.samba_abspath
 161         for inc in inclist:
 162             npath = tpath + '/' + inc
 163             if not npath in inc_set:
 164                 inc_abs.append(npath)
 165                 inc_set.add(npath)
 166
 167     mypath = self.path.abspath(bld.env)
 168     for inc in inc_abs:
 169         relpath = os_path_relpath(inc, mypath)
 170         includes.append(relpath)
 171
 172     if getattr(self, 'local_include', True) == True and not getattr(self, 'local_include_first', True):
 173         includes.append('.')
 174
 175     # now transform the includes list to be relative to the top directory
 176     # which is represented by '#' in waf. This allows waf to cache the
 177     # includes lists more efficiently
 178     includes_top = []
 179     for i in includes:
 180         if i[0] == '#':
 181             # some are already top based
 182             includes_top.append(i)
 183             continue
 184         absinc = os.path.join(self.path.abspath(), i)
 185         relinc = os_path_relpath(absinc, self.bld.srcnode.abspath())
 186         includes_top.append('#' + relinc)
 187
 188     self.includes = unique_list(includes_top)
 189     debug('deps: includes for target %s: includes=%s',
 190           self.sname, self.includes)
 191
 192
 193 def add_init_functions(self):
 194     '''This builds the right set of init functions'''
 195
 196     bld = self.bld
 197
 198     subsystems = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 199
 200     # cope with the separated object lists from BINARY and LIBRARY targets
 201     sname = self.sname
 202     if sname.endswith('.objlist'):
 203         sname = sname[0:-8]
 204
 205     modules = []
 206     if sname in subsystems:
 207         modules.append(sname)
 208
 209     m = getattr(self, 'samba_modules', None)
 210     if m is not None:
 211         modules.extend(TO_LIST(m))
 212
 213     m = getattr(self, 'samba_subsystem', None)
 214     if m is not None:
 215         modules.append(m)
 216
 217     sentinel = getattr(self, 'init_function_sentinel', 'NULL')
 218
 219     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
 220     cflags = getattr(self, 'samba_cflags', [])[:]
 221
 222     if modules == []:
 223         sname = sname.replace('-','_')
 224         sname = sname.replace('/','_')
 225         cflags.append('-DSTATIC_%s_MODULES=%s' % (sname, sentinel))
 226         if sentinel == 'NULL':
 227             cflags.append('-DSTATIC_%s_MODULES_PROTO' % sname)
 228         self.ccflags = cflags
 229         return
 230
 231     for m in modules:
 232         bld.ASSERT(m in subsystems,
 233                    "No init_function defined for module '%s' in target '%s'" % (m, self.sname))
 234         init_fn_list = []
 235         for d in subsystems[m]:
 236             if targets[d['TARGET']] != 'DISABLED':
 237                 init_fn_list.append(d['INIT_FUNCTION'])
 238         if init_fn_list == []:
 239             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, sentinel))
 240             if sentinel == 'NULL':
 241                 cflags.append('-DSTATIC_%s_MODULES_PROTO' % m)
 242         else:
 243             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, ','.join(init_fn_list) + ',' + sentinel))
 244             proto=''
 245             for f in init_fn_list:
 246                 proto = proto + '_MODULE_PROTO(%s)' % f
 247             cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
 248     self.ccflags = cflags
 249
 250
 251 def check_duplicate_sources(bld, tgt_list):
 252     '''see if we are compiling the same source file more than once
 253        without an allow_duplicates attribute'''
 254
 255     debug('deps: checking for duplicate sources')
 256
 257     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 258     ret = True
 259
 260     global tstart
 261
 262     for t in tgt_list:
 263         source_list = TO_LIST(getattr(t, 'source', ''))
 264         tpath = os.path.normpath(os_path_relpath(t.path.abspath(bld.env), t.env.BUILD_DIRECTORY + '/default'))
 265         obj_sources = set()
 266         for s in source_list:
 267             p = os.path.normpath(os.path.join(tpath, s))
 268             if p in obj_sources:
 269                 Logs.error("ERROR: source %s appears twice in target '%s'" % (p, t.sname))
 270                 sys.exit(1)
 271             obj_sources.add(p)
 272         t.samba_source_set = obj_sources
 273
 274     subsystems = {}
 275
 276     # build a list of targets that each source file is part of
 277     for t in tgt_list:
 278         sources = []
 279         if not targets[t.sname] in [ 'LIBRARY', 'BINARY', 'PYTHON' ]:
 280             continue
 281         for obj in t.add_objects:
 282             t2 = t.bld.name_to_obj(obj, bld.env)
 283             source_set = getattr(t2, 'samba_source_set', set())
 284             for s in source_set:
 285                 if not s in subsystems:
 286                     subsystems[s] = {}
 287                 if not t.sname in subsystems[s]:
 288                     subsystems[s][t.sname] = []
 289                 subsystems[s][t.sname].append(t2.sname)
 290
 291     for s in subsystems:
 292         if len(subsystems[s]) > 1 and Options.options.SHOW_DUPLICATES:
 293             Logs.warn("WARNING: source %s is in more than one target: %s" % (s, subsystems[s].keys()))
 294         for tname in subsystems[s]:
 295             if len(subsystems[s][tname]) > 1:
 296                 raise Utils.WafError("ERROR: source %s is in more than one subsystem of target '%s': %s" % (s, tname, subsystems[s][tname]))
 297
 298     return ret
 299
 300
 301 def check_orphaned_targets(bld, tgt_list):
 302     '''check if any build targets are orphaned'''
 303
 304     target_dict = LOCAL_CACHE(bld, 'TARGET_TYPE')
 305
 306     debug('deps: checking for orphaned targets')
 307
 308     for t in tgt_list:
 309         if getattr(t, 'samba_used', False) == True:
 310             continue
 311         type = target_dict[t.sname]
 312         if not type in ['BINARY', 'LIBRARY', 'MODULE', 'ET', 'PYTHON']:
 313             if re.search('^PIDL_', t.sname) is None:
 314                 Logs.warn("Target %s of type %s is unused by any other target" % (t.sname, type))
 315
 316
 317 def check_group_ordering(bld, tgt_list):
 318     '''see if we have any dependencies that violate the group ordering
 319
 320     It is an error for a target to depend on a target from a later
 321     build group
 322     '''
 323
 324     def group_name(g):
 325         tm = bld.task_manager
 326         return [x for x in tm.groups_names if id(tm.groups_names[x]) == id(g)][0]
 327
 328     for g in bld.task_manager.groups:
 329         gname = group_name(g)
 330         for t in g.tasks_gen:
 331             t.samba_group = gname
 332
 333     grp_map = {}
 334     idx = 0
 335     for g in bld.task_manager.groups:
 336         name = group_name(g)
 337         grp_map[name] = idx
 338         idx += 1
 339
 340     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 341
 342     ret = True
 343     for t in tgt_list:
 344         tdeps = getattr(t, 'add_objects', []) + getattr(t, 'uselib_local', [])
 345         for d in tdeps:
 346             t2 = bld.name_to_obj(d, bld.env)
 347             if t2 is None:
 348                 continue
 349             map1 = grp_map[t.samba_group]
 350             map2 = grp_map[t2.samba_group]
 351
 352             if map2 > map1:
 353                 Logs.error("Target %r in build group %r depends on target %r from later build group %r" % (
 354                            t.sname, t.samba_group, t2.sname, t2.samba_group))
 355                 ret = False
 356
 357     return ret
 358
 359
 360 def show_final_deps(bld, tgt_list):
 361     '''show the final dependencies for all targets'''
 362
 363     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 364
 365     for t in tgt_list:
 366         if not targets[t.sname] in ['LIBRARY', 'BINARY', 'PYTHON', 'SUBSYSTEM']:
 367             continue
 368         debug('deps: final dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
 369               t.sname, t.uselib, getattr(t, 'uselib_local', []), getattr(t, 'add_objects', []))
 370
 371
 372 def add_samba_attributes(bld, tgt_list):
 373     '''ensure a target has a the required samba attributes'''
 374
 375     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 376
 377     for t in tgt_list:
 378         if t.name != '':
 379             t.sname = t.name
 380         else:
 381             t.sname = t.target
 382         t.samba_type = targets[t.sname]
 383         t.samba_abspath = t.path.abspath(bld.env)
 384         t.samba_deps_extended = t.samba_deps[:]
 385         t.samba_includes_extended = TO_LIST(t.samba_includes)[:]
 386         t.ccflags = getattr(t, 'samba_cflags', '')
 387
 388 def replace_grouping_libraries(bld, tgt_list):
 389     '''replace dependencies based on grouping libraries
 390
 391     If a library is marked as a grouping library, then any target that
 392     depends on a subsystem that is part of that grouping library gets
 393     that dependency replaced with a dependency on the grouping library
 394     '''
 395
 396     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 397
 398     grouping = {}
 399
 400     # find our list of grouping libraries, mapped from the subsystems they depend on
 401     for t in tgt_list:
 402         if not getattr(t, 'grouping_library', False):
 403             continue
 404         for dep in t.samba_deps_extended:
 405             bld.ASSERT(dep in targets, "grouping library target %s not declared in %s" % (dep, t.sname))
 406             if targets[dep] == 'SUBSYSTEM':
 407                 grouping[dep] = t.sname
 408
 409     # now replace any dependencies on elements of grouping libraries
 410     for t in tgt_list:
 411         for i in range(len(t.samba_deps_extended)):
 412             dep = t.samba_deps_extended[i]
 413             if dep in grouping:
 414                 if t.sname != grouping[dep]:
 415                     debug("deps: target %s: replacing dependency %s with grouping library %s" % (t.sname, dep, grouping[dep]))
 416                     t.samba_deps_extended[i] = grouping[dep]
 417
 418
 419
 420 def build_direct_deps(bld, tgt_list):
 421     '''build the direct_objects and direct_libs sets for each target'''
 422
 423     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 424     syslib_deps  = LOCAL_CACHE(bld, 'SYSLIB_DEPS')
 425
 426     global_deps = bld.env.GLOBAL_DEPENDENCIES
 427     global_deps_exclude = set()
 428     for dep in global_deps:
 429         t = bld.name_to_obj(dep, bld.env)
 430         for d in t.samba_deps:
 431             # prevent loops from the global dependencies list
 432             global_deps_exclude.add(d)
 433             global_deps_exclude.add(d + '.objlist')
 434
 435     for t in tgt_list:
 436         t.direct_objects = set()
 437         t.direct_libs = set()
 438         t.direct_syslibs = set()
 439         deps = t.samba_deps_extended[:]
 440         if getattr(t, 'samba_use_global_deps', False) and not t.sname in global_deps_exclude:
 441             deps.extend(global_deps)
 442         for d in deps:
 443             if d == t.sname: continue
 444             if not d in targets:
 445                 Logs.error("Unknown dependency '%s' in '%s'" % (d, t.sname))
 446                 sys.exit(1)
 447             if targets[d] in [ 'EMPTY', 'DISABLED' ]:
 448                 continue
 449             if targets[d] == 'PYTHON' and targets[t.sname] != 'PYTHON' and t.sname.find('.objlist') == -1:
 450                 # this check should be more restrictive, but for now we have pidl-generated python
 451                 # code that directly depends on other python modules
 452                 Logs.error('ERROR: Target %s has dependency on python module %s' % (t.sname, d))
 453                 sys.exit(1)
 454             if targets[d] == 'SYSLIB':
 455                 t.direct_syslibs.add(d)
 456                 if d in syslib_deps:
 457                     for implied in TO_LIST(syslib_deps[d]):
 458                         if BUILTIN_LIBRARY(bld, implied):
 459                             t.direct_objects.add(implied)
 460                         elif targets[implied] == 'SYSLIB':
 461                             t.direct_syslibs.add(implied)
 462                         elif targets[implied] in ['LIBRARY', 'MODULE']:
 463                             t.direct_libs.add(implied)
 464                         else:
 465                             Logs.error('Implied dependency %s in %s is of type %s' % (
 466                                 implied, t.sname, targets[implied]))
 467                             sys.exit(1)
 468                 continue
 469             t2 = bld.name_to_obj(d, bld.env)
 470             if t2 is None:
 471                 Logs.error("no task %s of type %s in %s" % (d, targets[d], t.sname))
 472                 sys.exit(1)
 473             if t2.samba_type in [ 'LIBRARY', 'MODULE' ]:
 474                 t.direct_libs.add(d)
 475             elif t2.samba_type in [ 'SUBSYSTEM', 'ASN1', 'PYTHON' ]:
 476                 t.direct_objects.add(d)
 477     debug('deps: built direct dependencies')
 478
 479
 480 def dependency_loop(loops, t, target):
 481     '''add a dependency loop to the loops dictionary'''
 482     if t.sname == target:
 483         return
 484     if not target in loops:
 485         loops[target] = set()
 486     if not t.sname in loops[target]:
 487         loops[target].add(t.sname)
 488
 489
 490 def indirect_libs(bld, t, chain, loops):
 491     '''recursively calculate the indirect library dependencies for a target
 492
 493     An indirect library is a library that results from a dependency on
 494     a subsystem
 495     '''
 496
 497     ret = getattr(t, 'indirect_libs', None)
 498     if ret is not None:
 499         return ret
 500
 501     ret = set()
 502     for obj in t.direct_objects:
 503         if obj in chain:
 504             dependency_loop(loops, t, obj)
 505             continue
 506         chain.add(obj)
 507         t2 = bld.name_to_obj(obj, bld.env)
 508         r2 = indirect_libs(bld, t2, chain, loops)
 509         chain.remove(obj)
 510         ret = ret.union(t2.direct_libs)
 511         ret = ret.union(r2)
 512
 513     for obj in indirect_objects(bld, t, set(), loops):
 514         if obj in chain:
 515             dependency_loop(loops, t, obj)
 516             continue
 517         chain.add(obj)
 518         t2 = bld.name_to_obj(obj, bld.env)
 519         r2 = indirect_libs(bld, t2, chain, loops)
 520         chain.remove(obj)
 521         ret = ret.union(t2.direct_libs)
 522         ret = ret.union(r2)
 523
 524     t.indirect_libs = ret
 525
 526     return ret
 527
 528
 529 def indirect_objects(bld, t, chain, loops):
 530     '''recursively calculate the indirect object dependencies for a target
 531
 532     indirect objects are the set of objects from expanding the
 533     subsystem dependencies
 534     '''
 535
 536     ret = getattr(t, 'indirect_objects', None)
 537     if ret is not None: return ret
 538
 539     ret = set()
 540     for lib in t.direct_objects:
 541         if lib in chain:
 542             dependency_loop(loops, t, lib)
 543             continue
 544         chain.add(lib)
 545         t2 = bld.name_to_obj(lib, bld.env)
 546         r2 = indirect_objects(bld, t2, chain, loops)
 547         chain.remove(lib)
 548         ret = ret.union(t2.direct_objects)
 549         ret = ret.union(r2)
 550
 551     t.indirect_objects = ret
 552     return ret
 553
 554
 555 def extended_objects(bld, t, chain):
 556     '''recursively calculate the extended object dependencies for a target
 557
 558     extended objects are the union of:
 559        - direct objects
 560        - indirect objects
 561        - direct and indirect objects of all direct and indirect libraries
 562     '''
 563
 564     ret = getattr(t, 'extended_objects', None)
 565     if ret is not None: return ret
 566
 567     ret = set()
 568     ret = ret.union(t.final_objects)
 569
 570     for lib in t.final_libs:
 571         if lib in chain:
 572             continue
 573         t2 = bld.name_to_obj(lib, bld.env)
 574         chain.add(lib)
 575         r2 = extended_objects(bld, t2, chain)
 576         chain.remove(lib)
 577         ret = ret.union(t2.final_objects)
 578         ret = ret.union(r2)
 579
 580     t.extended_objects = ret
 581     return ret
 582
 583
 584 def includes_objects(bld, t, chain, inc_loops):
 585     '''recursively calculate the includes object dependencies for a target
 586
 587     includes dependencies come from either library or object dependencies
 588     '''
 589     ret = getattr(t, 'includes_objects', None)
 590     if ret is not None:
 591         return ret
 592
 593     ret = t.direct_objects.copy()
 594     ret = ret.union(t.direct_libs)
 595
 596     for obj in t.direct_objects:
 597         if obj in chain:
 598             dependency_loop(inc_loops, t, obj)
 599             continue
 600         chain.add(obj)
 601         t2 = bld.name_to_obj(obj, bld.env)
 602         r2 = includes_objects(bld, t2, chain, inc_loops)
 603         chain.remove(obj)
 604         ret = ret.union(t2.direct_objects)
 605         ret = ret.union(r2)
 606
 607     for lib in t.direct_libs:
 608         if lib in chain:
 609             dependency_loop(inc_loops, t, lib)
 610             continue
 611         chain.add(lib)
 612         t2 = bld.name_to_obj(lib, bld.env)
 613         if t2 is None:
 614             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 615             Logs.error('Target %s of type %s not found in direct_libs for %s' % (
 616                 lib, targets[lib], t.sname))
 617             sys.exit(1)
 618         r2 = includes_objects(bld, t2, chain, inc_loops)
 619         chain.remove(lib)
 620         ret = ret.union(t2.direct_objects)
 621         ret = ret.union(r2)
 622
 623     t.includes_objects = ret
 624     return ret
 625
 626
 627 def break_dependency_loops(bld, tgt_list):
 628     '''find and break dependency loops'''
 629     loops = {}
 630     inc_loops = {}
 631
 632     # build up the list of loops
 633     for t in tgt_list:
 634         indirect_objects(bld, t, set(), loops)
 635         indirect_libs(bld, t, set(), loops)
 636         includes_objects(bld, t, set(), inc_loops)
 637
 638     # break the loops
 639     for t in tgt_list:
 640         if t.sname in loops:
 641             for attr in ['direct_objects', 'indirect_objects', 'direct_libs', 'indirect_libs']:
 642                 objs = getattr(t, attr, set())
 643                 setattr(t, attr, objs.difference(loops[t.sname]))
 644
 645     for loop in loops:
 646         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 647
 648     for loop in inc_loops:
 649         debug('deps: Found include loops for target %s : %s', loop, inc_loops[loop])
 650
 651     # expand the loops mapping by one level
 652     for loop in loops.copy():
 653         for tgt in loops[loop]:
 654             if tgt in loops:
 655                 loops[loop] = loops[loop].union(loops[tgt])
 656
 657     for loop in inc_loops.copy():
 658         for tgt in inc_loops[loop]:
 659             if tgt in inc_loops:
 660                 inc_loops[loop] = inc_loops[loop].union(inc_loops[tgt])
 661
 662
 663     # expand indirect subsystem and library loops
 664     for loop in loops.copy():
 665         t = bld.name_to_obj(loop, bld.env)
 666         if t.samba_type in ['SUBSYSTEM']:
 667             loops[loop] = loops[loop].union(t.indirect_objects)
 668             loops[loop] = loops[loop].union(t.direct_objects)
 669         if t.samba_type in ['LIBRARY','PYTHON']:
 670             loops[loop] = loops[loop].union(t.indirect_libs)
 671             loops[loop] = loops[loop].union(t.direct_libs)
 672         if loop in loops[loop]:
 673             loops[loop].remove(loop)
 674
 675     # expand indirect includes loops
 676     for loop in inc_loops.copy():
 677         t = bld.name_to_obj(loop, bld.env)
 678         inc_loops[loop] = inc_loops[loop].union(t.includes_objects)
 679         if loop in inc_loops[loop]:
 680             inc_loops[loop].remove(loop)
 681
 682     # add in the replacement dependencies
 683     for t in tgt_list:
 684         for loop in loops:
 685             for attr in ['indirect_objects', 'indirect_libs']:
 686                 objs = getattr(t, attr, set())
 687                 if loop in objs:
 688                     diff = loops[loop].difference(objs)
 689                     if t.sname in diff:
 690                         diff.remove(t.sname)
 691                     if diff:
 692                         debug('deps: Expanded target %s of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 693                         objs = objs.union(diff)
 694                 setattr(t, attr, objs)
 695
 696         for loop in inc_loops:
 697             objs = getattr(t, 'includes_objects', set())
 698             if loop in objs:
 699                 diff = inc_loops[loop].difference(objs)
 700                 if t.sname in diff:
 701                     diff.remove(t.sname)
 702                 if diff:
 703                     debug('deps: Expanded target %s includes of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 704                     objs = objs.union(diff)
 705             setattr(t, 'includes_objects', objs)
 706
 707
 708 def reduce_objects(bld, tgt_list):
 709     '''reduce objects by looking for indirect object dependencies'''
 710     rely_on = {}
 711
 712     for t in tgt_list:
 713         t.extended_objects = None
 714
 715     changed = False
 716
 717     for type in ['BINARY', 'PYTHON', 'LIBRARY']:
 718         for t in tgt_list:
 719             if t.samba_type != type: continue
 720             # if we will indirectly link to a target then we don't need it
 721             new = t.final_objects.copy()
 722             for l in t.final_libs:
 723                 t2 = bld.name_to_obj(l, bld.env)
 724                 t2_obj = extended_objects(bld, t2, set())
 725                 dup = new.intersection(t2_obj)
 726                 if t.sname in rely_on:
 727                     dup = dup.difference(rely_on[t.sname])
 728                 if dup:
 729                     debug('deps: removing dups from %s of type %s: %s also in %s %s',
 730                           t.sname, t.samba_type, dup, t2.samba_type, l)
 731                     new = new.difference(dup)
 732                     changed = True
 733                     if not l in rely_on:
 734                         rely_on[l] = set()
 735                     rely_on[l] = rely_on[l].union(dup)
 736             t.final_objects = new
 737
 738     if not changed:
 739         return False
 740
 741     # add back in any objects that were relied upon by the reduction rules
 742     for r in rely_on:
 743         t = bld.name_to_obj(r, bld.env)
 744         t.final_objects = t.final_objects.union(rely_on[r])
 745
 746     return True
 747
 748
 749 def show_library_loop(bld, lib1, lib2, path, seen):
 750     '''show the detailed path of a library loop between lib1 and lib2'''
 751
 752     t = bld.name_to_obj(lib1, bld.env)
 753     if not lib2 in getattr(t, 'final_libs', set()):
 754         return
 755
 756     for d in t.samba_deps_extended:
 757         if d in seen:
 758             continue
 759         seen.add(d)
 760         path2 = path + '=>' + d
 761         if d == lib2:
 762             Logs.warn('library loop path: ' + path2)
 763             return
 764         show_library_loop(bld, d, lib2, path2, seen)
 765         seen.remove(d)
 766
 767
 768 def calculate_final_deps(bld, tgt_list, loops):
 769     '''calculate the final library and object dependencies'''
 770     for t in tgt_list:
 771         # start with the maximum possible list
 772         t.final_libs    = t.direct_libs.union(indirect_libs(bld, t, set(), loops))
 773         t.final_objects = t.direct_objects.union(indirect_objects(bld, t, set(), loops))
 774
 775     for t in tgt_list:
 776         # don't depend on ourselves
 777         if t.sname in t.final_libs:
 778             t.final_libs.remove(t.sname)
 779         if t.sname in t.final_objects:
 780             t.final_objects.remove(t.sname)
 781
 782     # handle any non-shared binaries
 783     for t in tgt_list:
 784         if t.samba_type == 'BINARY' and bld.NONSHARED_BINARY(t.sname):
 785             subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 786             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 787
 788             # replace lib deps with objlist deps
 789             for l in t.final_libs:
 790                 objname = l + '.objlist'
 791                 t2 = bld.name_to_obj(objname, bld.env)
 792                 if t2 is None:
 793                     Logs.error('ERROR: subsystem %s not found' % objname)
 794                     sys.exit(1)
 795                 t.final_objects.add(objname)
 796                 t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 797                 if l in subsystem_list:
 798                     # its a subsystem - we also need the contents of any modules
 799                     for d in subsystem_list[l]:
 800                         module_name = d['TARGET']
 801                         if targets[module_name] == 'LIBRARY':
 802                             objname = module_name + '.objlist'
 803                         elif targets[module_name] == 'SUBSYSTEM':
 804                             objname = module_name
 805                         else:
 806                             continue
 807                         t2 = bld.name_to_obj(objname, bld.env)
 808                         if t2 is None:
 809                             Logs.error('ERROR: subsystem %s not found' % objname)
 810                             sys.exit(1)
 811                         t.final_objects.add(objname)
 812                         t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 813             t.final_libs = set()
 814
 815     # find any library loops
 816     for t in tgt_list:
 817         if t.samba_type in ['LIBRARY', 'PYTHON']:
 818             for l in t.final_libs.copy():
 819                 t2 = bld.name_to_obj(l, bld.env)
 820                 if t.sname in t2.final_libs:
 821                     if getattr(bld.env, "ALLOW_CIRCULAR_LIB_DEPENDENCIES", False):
 822                         # we could break this in either direction. If one of the libraries
 823                         # has a version number, and will this be distributed publicly, then
 824                         # we should make it the lower level library in the DAG
 825                         Logs.warn('deps: removing library loop %s from %s' % (t.sname, t2.sname))
 826                         dependency_loop(loops, t, t2.sname)
 827                         t2.final_libs.remove(t.sname)
 828                     else:
 829                         Logs.error('ERROR: circular library dependency between %s and %s'
 830                             % (t.sname, t2.sname))
 831                         show_library_loop(bld, t.sname, t2.sname, t.sname, set())
 832                         show_library_loop(bld, t2.sname, t.sname, t2.sname, set())
 833                         sys.exit(1)
 834
 835     for loop in loops:
 836         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 837
 838     # we now need to make corrections for any library loops we broke up
 839     # any target that depended on the target of the loop and doesn't
 840     # depend on the source of the loop needs to get the loop source added
 841     for type in ['BINARY','PYTHON','LIBRARY','BINARY']:
 842         for t in tgt_list:
 843             if t.samba_type != type: continue
 844             for loop in loops:
 845                 if loop in t.final_libs:
 846                     diff = loops[loop].difference(t.final_libs)
 847                     if t.sname in diff:
 848                         diff.remove(t.sname)
 849                     if t.sname in diff:
 850                         diff.remove(t.sname)
 851                     # make sure we don't recreate the loop again!
 852                     for d in diff.copy():
 853                         t2 = bld.name_to_obj(d, bld.env)
 854                         if t2.samba_type == 'LIBRARY':
 855                             if t.sname in t2.final_libs:
 856                                 debug('deps: removing expansion %s from %s', d, t.sname)
 857                                 diff.remove(d)
 858                     if diff:
 859                         debug('deps: Expanded target %s by loop %s libraries (loop %s) %s', t.sname, loop,
 860                               loops[loop], diff)
 861                         t.final_libs = t.final_libs.union(diff)
 862
 863     # remove objects that are also available in linked libs
 864     count = 0
 865     while reduce_objects(bld, tgt_list):
 866         count += 1
 867         if count > 100:
 868             Logs.warn("WARNING: Unable to remove all inter-target object duplicates")
 869             break
 870     debug('deps: Object reduction took %u iterations', count)
 871
 872     # add in any syslib dependencies
 873     for t in tgt_list:
 874         if not t.samba_type in ['BINARY','PYTHON','LIBRARY','SUBSYSTEM']:
 875             continue
 876         syslibs = set()
 877         for d in t.final_objects:
 878             t2 = bld.name_to_obj(d, bld.env)
 879             syslibs = syslibs.union(t2.direct_syslibs)
 880         # this adds the indirect syslibs as well, which may not be needed
 881         # depending on the linker flags
 882         for d in t.final_libs:
 883             t2 = bld.name_to_obj(d, bld.env)
 884             syslibs = syslibs.union(t2.direct_syslibs)
 885         t.final_syslibs = syslibs
 886
 887
 888     # find any unresolved library loops
 889     lib_loop_error = False
 890     for t in tgt_list:
 891         if t.samba_type in ['LIBRARY', 'PYTHON']:
 892             for l in t.final_libs.copy():
 893                 t2 = bld.name_to_obj(l, bld.env)
 894                 if t.sname in t2.final_libs:
 895                     Logs.error('ERROR: Unresolved library loop %s from %s' % (t.sname, t2.sname))
 896                     lib_loop_error = True
 897     if lib_loop_error:
 898         sys.exit(1)
 899
 900     debug('deps: removed duplicate dependencies')
 901
 902
 903 def show_dependencies(bld, target, seen):
 904     '''recursively show the dependencies of target'''
 905
 906     if target in seen:
 907         return
 908
 909     t = bld.name_to_obj(target, bld.env)
 910     if t is None:
 911         Logs.error("ERROR: Unable to find target '%s'" % target)
 912         sys.exit(1)
 913
 914     Logs.info('%s(OBJECTS): %s' % (target, t.direct_objects))
 915     Logs.info('%s(LIBS): %s' % (target, t.direct_libs))
 916     Logs.info('%s(SYSLIBS): %s' % (target, t.direct_syslibs))
 917
 918     seen.add(target)
 919
 920     for t2 in t.direct_objects:
 921         show_dependencies(bld, t2, seen)
 922
 923
 924 def show_object_duplicates(bld, tgt_list):
 925     '''show a list of object files that are included in more than
 926     one library or binary'''
 927
 928     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 929
 930     used_by = {}
 931
 932     Logs.info("showing duplicate objects")
 933
 934     for t in tgt_list:
 935         if not targets[t.sname] in [ 'LIBRARY', 'PYTHON' ]:
 936             continue
 937         for n in getattr(t, 'final_objects', set()):
 938             t2 = bld.name_to_obj(n, bld.env)
 939             if not n in used_by:
 940                 used_by[n] = set()
 941             used_by[n].add(t.sname)
 942
 943     for n in used_by:
 944         if len(used_by[n]) > 1:
 945             Logs.info("target '%s' is used by %s" % (n, used_by[n]))
 946
 947     Logs.info("showing indirect dependency counts (sorted by count)")
 948
 949     def indirect_count(t1, t2):
 950         return len(t2.indirect_objects) - len(t1.indirect_objects)
 951
 952     sorted_list = sorted(tgt_list, cmp=indirect_count)
 953     for t in sorted_list:
 954         if len(t.indirect_objects) > 1:
 955             Logs.info("%s depends on %u indirect objects" % (t.sname, len(t.indirect_objects)))
 956
 957
 958 ######################################################################
 959 # this provides a way to save our dependency calculations between runs
 960 savedeps_version = 3
 961 savedeps_inputs  = ['samba_deps', 'samba_includes', 'local_include', 'local_include_first', 'samba_cflags',
 962                     'source', 'grouping_library', 'samba_ldflags', 'allow_undefined_symbols',
 963                     'use_global_deps', 'global_include' ]
 964 savedeps_outputs = ['uselib', 'uselib_local', 'add_objects', 'includes', 'ccflags', 'ldflags', 'samba_deps_extended']
 965 savedeps_outenv  = ['INC_PATHS']
 966 savedeps_envvars = ['NONSHARED_BINARIES', 'GLOBAL_DEPENDENCIES', 'EXTRA_CFLAGS', 'EXTRA_LDFLAGS', 'EXTRA_INCLUDES' ]
 967 savedeps_caches  = ['GLOBAL_DEPENDENCIES', 'TARGET_TYPE', 'INIT_FUNCTIONS', 'SYSLIB_DEPS']
 968 savedeps_files   = ['buildtools/wafsamba/samba_deps.py']
 969
 970 def save_samba_deps(bld, tgt_list):
 971     '''save the dependency calculations between builds, to make
 972        further builds faster'''
 973     denv = Environment.Environment()
 974
 975     denv.version = savedeps_version
 976     denv.savedeps_inputs = savedeps_inputs
 977     denv.savedeps_outputs = savedeps_outputs
 978     denv.input = {}
 979     denv.output = {}
 980     denv.outenv = {}
 981     denv.caches = {}
 982     denv.envvar = {}
 983     denv.files  = {}
 984
 985     for f in savedeps_files:
 986         denv.files[f] = os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime
 987
 988     for c in savedeps_caches:
 989         denv.caches[c] = LOCAL_CACHE(bld, c)
 990
 991     for e in savedeps_envvars:
 992         denv.envvar[e] = bld.env[e]
 993
 994     for t in tgt_list:
 995         # save all the input attributes for each target
 996         tdeps = {}
 997         for attr in savedeps_inputs:
 998             v = getattr(t, attr, None)
 999             if v is not None:
1000                 tdeps[attr] = v
1001         if tdeps != {}:
1002             denv.input[t.sname] = tdeps
1003
1004         # save all the output attributes for each target
1005         tdeps = {}
1006         for attr in savedeps_outputs:
1007             v = getattr(t, attr, None)
1008             if v is not None:
1009                 tdeps[attr] = v
1010         if tdeps != {}:
1011             denv.output[t.sname] = tdeps
1012
1013         tdeps = {}
1014         for attr in savedeps_outenv:
1015             if attr in t.env:
1016                 tdeps[attr] = t.env[attr]
1017         if tdeps != {}:
1018             denv.outenv[t.sname] = tdeps
1019
1020     depsfile = os.path.join(bld.bdir, "sambadeps")
1021     denv.store(depsfile)
1022
1023
1024
1025 def load_samba_deps(bld, tgt_list):
1026     '''load a previous set of build dependencies if possible'''
1027     depsfile = os.path.join(bld.bdir, "sambadeps")
1028     denv = Environment.Environment()
1029     try:
1030         debug('deps: checking saved dependencies')
1031         denv.load(depsfile)
1032         if (denv.version != savedeps_version or
1033             denv.savedeps_inputs != savedeps_inputs or
1034             denv.savedeps_outputs != savedeps_outputs):
1035             return False
1036     except:
1037         return False
1038
1039     # check if critical files have changed
1040     for f in savedeps_files:
1041         if f not in denv.files:
1042             return False
1043         if denv.files[f] != os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime:
1044             return False
1045
1046     # check if caches are the same
1047     for c in savedeps_caches:
1048         if c not in denv.caches or denv.caches[c] != LOCAL_CACHE(bld, c):
1049             return False
1050
1051     # check if caches are the same
1052     for e in savedeps_envvars:
1053         if e not in denv.envvar or denv.envvar[e] != bld.env[e]:
1054             return False
1055
1056     # check inputs are the same
1057     for t in tgt_list:
1058         tdeps = {}
1059         for attr in savedeps_inputs:
1060             v = getattr(t, attr, None)
1061             if v is not None:
1062                 tdeps[attr] = v
1063         if t.sname in denv.input:
1064             olddeps = denv.input[t.sname]
1065         else:
1066             olddeps = {}
1067         if tdeps != olddeps:
1068             #print '%s: \ntdeps=%s \nodeps=%s' % (t.sname, tdeps, olddeps)
1069             return False
1070
1071     # put outputs in place
1072     for t in tgt_list:
1073         if not t.sname in denv.output: continue
1074         tdeps = denv.output[t.sname]
1075         for a in tdeps:
1076             setattr(t, a, tdeps[a])
1077
1078     # put output env vars in place
1079     for t in tgt_list:
1080         if not t.sname in denv.outenv: continue
1081         tdeps = denv.outenv[t.sname]
1082         for a in tdeps:
1083             t.env[a] = tdeps[a]
1084
1085     debug('deps: loaded saved dependencies')
1086     return True
1087
1088
1089
1090 def check_project_rules(bld):
1091     '''check the project rules - ensuring the targets are sane'''
1092
1093     loops = {}
1094     inc_loops = {}
1095
1096     tgt_list = get_tgt_list(bld)
1097
1098     add_samba_attributes(bld, tgt_list)
1099
1100     force_project_rules = (Options.options.SHOWDEPS or
1101                            Options.options.SHOW_DUPLICATES)
1102
1103     if not force_project_rules and load_samba_deps(bld, tgt_list):
1104         return
1105
1106     global tstart
1107     tstart = time.clock()
1108
1109     bld.new_rules = True
1110     Logs.info("Checking project rules ...")
1111
1112     debug('deps: project rules checking started')
1113
1114     expand_subsystem_deps(bld)
1115
1116     debug("deps: expand_subsystem_deps: %f" % (time.clock() - tstart))
1117
1118     replace_grouping_libraries(bld, tgt_list)
1119
1120     debug("deps: replace_grouping_libraries: %f" % (time.clock() - tstart))
1121
1122     build_direct_deps(bld, tgt_list)
1123
1124     debug("deps: build_direct_deps: %f" % (time.clock() - tstart))
1125
1126     break_dependency_loops(bld, tgt_list)
1127
1128     debug("deps: break_dependency_loops: %f" % (time.clock() - tstart))
1129
1130     if Options.options.SHOWDEPS:
1131             show_dependencies(bld, Options.options.SHOWDEPS, set())
1132
1133     calculate_final_deps(bld, tgt_list, loops)
1134
1135     debug("deps: calculate_final_deps: %f" % (time.clock() - tstart))
1136
1137     if Options.options.SHOW_DUPLICATES:
1138             show_object_duplicates(bld, tgt_list)
1139
1140     # run the various attribute generators
1141     for f in [ build_dependencies, build_includes, add_init_functions ]:
1142         debug('deps: project rules checking %s', f)
1143         for t in tgt_list: f(t)
1144         debug("deps: %s: %f" % (f, time.clock() - tstart))
1145
1146     debug('deps: project rules stage1 completed')
1147
1148     #check_orphaned_targets(bld, tgt_list)
1149
1150     if not check_duplicate_sources(bld, tgt_list):
1151         Logs.error("Duplicate sources present - aborting")
1152         sys.exit(1)
1153
1154     debug("deps: check_duplicate_sources: %f" % (time.clock() - tstart))
1155
1156     if not check_group_ordering(bld, tgt_list):
1157         Logs.error("Bad group ordering - aborting")
1158         sys.exit(1)
1159
1160     debug("deps: check_group_ordering: %f" % (time.clock() - tstart))
1161
1162     show_final_deps(bld, tgt_list)
1163
1164     debug("deps: show_final_deps: %f" % (time.clock() - tstart))
1165
1166     debug('deps: project rules checking completed - %u targets checked',
1167           len(tgt_list))
1168
1169     if not bld.is_install:
1170         save_samba_deps(bld, tgt_list)
1171
1172     debug("deps: save_samba_deps: %f" % (time.clock() - tstart))
1173
1174     Logs.info("Project rules pass")
1175
1176
1177 def CHECK_PROJECT_RULES(bld):
1178     '''enable checking of project targets for sanity'''
1179     if bld.env.added_project_rules:
1180         return
1181     bld.env.added_project_rules = True
1182     bld.add_pre_fun(check_project_rules)
1183 Build.BuildContext.CHECK_PROJECT_RULES = CHECK_PROJECT_RULES
1184
1185