buildtools/wafsamba/samba_deps.py

   1 # Samba automatic dependency handling and project rules
   2
   3 import Build, os, sys, re, Environment, Logs, time
   4 from samba_utils import *
   5 from samba_autoconf import *
   6 from samba_bundled import BUILTIN_LIBRARY
   7
   8 @conf
   9 def ADD_GLOBAL_DEPENDENCY(ctx, dep):
  10     '''add a dependency for all binaries and libraries'''
  11     if not 'GLOBAL_DEPENDENCIES' in ctx.env:
  12         ctx.env.GLOBAL_DEPENDENCIES = []
  13     ctx.env.GLOBAL_DEPENDENCIES.append(dep)
  14
  15
  16 @conf
  17 def BREAK_CIRCULAR_LIBRARY_DEPENDENCIES(ctx):
  18     '''indicate that circular dependencies between libraries should be broken.'''
  19     ctx.env.ALLOW_CIRCULAR_LIB_DEPENDENCIES = True
  20
  21
  22 @conf
  23 def SET_SYSLIB_DEPS(conf, target, deps):
  24     '''setup some implied dependencies for a SYSLIB'''
  25     cache = LOCAL_CACHE(conf, 'SYSLIB_DEPS')
  26     cache[target] = deps
  27
  28
  29 def expand_subsystem_deps(bld):
  30     '''expand the reverse dependencies resulting from subsystem
  31        attributes of modules. This is walking over the complete list
  32        of declared subsystems, and expands the samba_deps_extended list for any
  33        module<->subsystem dependencies'''
  34
  35     subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
  36     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
  37
  38     for subsystem_name in subsystem_list:
  39         bld.ASSERT(subsystem_name in targets, "Subsystem target %s not declared" % subsystem_name)
  40         type = targets[subsystem_name]
  41         if type == 'DISABLED' or type == 'EMPTY':
  42             continue
  43
  44         # for example,
  45         #    subsystem_name = dcerpc_server (a subsystem)
  46         #    subsystem      = dcerpc_server (a subsystem object)
  47         #    module_name    = rpc_epmapper (a module within the dcerpc_server subsystem)
  48         #    module         = rpc_epmapper (a module object within the dcerpc_server subsystem)
  49
  50         subsystem = bld.get_tgen_by_name(subsystem_name)
  51         bld.ASSERT(subsystem is not None, "Unable to find subsystem %s" % subsystem_name)
  52         for d in subsystem_list[subsystem_name]:
  53             module_name = d['TARGET']
  54             module_type = targets[module_name]
  55             if module_type in ['DISABLED', 'EMPTY']:
  56                 continue
  57             bld.ASSERT(subsystem is not None,
  58                        "Subsystem target %s for %s (%s) not found" % (subsystem_name, module_name, module_type))
  59             if module_type in ['SUBSYSTEM']:
  60                 # if a module is a plain object type (not a library) then the
  61                 # subsystem it is part of needs to have it as a dependency, so targets
  62                 # that depend on this subsystem get the modules of that subsystem
  63                 subsystem.samba_deps_extended.append(module_name)
  64         subsystem.samba_deps_extended = unique_list(subsystem.samba_deps_extended)
  65
  66
  67
  68 def build_dependencies(self):
  69     '''This builds the dependency list for a target. It runs after all the targets are declared
  70
  71     The reason this is not just done in the SAMBA_*() rules is that we have no way of knowing
  72     the full dependency list for a target until we have all of the targets declared.
  73     '''
  74
  75     if self.samba_type in ['LIBRARY', 'BINARY', 'PYTHON']:
  76         self.uselib        = list(self.final_syslibs)
  77         self.uselib_local  = list(self.final_libs)
  78         self.add_objects   = list(self.final_objects)
  79
  80         # extra link flags from pkg_config
  81         libs = self.final_syslibs.copy()
  82
  83         (ccflags, ldflags, cpppath) = library_flags(self, list(libs))
  84         new_ldflags        = getattr(self, 'samba_ldflags', [])[:]
  85         new_ldflags.extend(ldflags)
  86         self.ldflags       = new_ldflags
  87
  88         if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ldflags:
  89             for f in self.env.undefined_ldflags:
  90                 self.ldflags.remove(f)
  91
  92         if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ignore_ldflags:
  93             for f in self.env.undefined_ignore_ldflags:
  94                 self.ldflags.append(f)
  95
  96         debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
  97               self.sname, self.uselib, self.uselib_local, self.add_objects)
  98
  99     if self.samba_type in ['SUBSYSTEM']:
 100         # this is needed for the ccflags of libs that come from pkg_config
 101         self.uselib = list(self.final_syslibs)
 102         self.uselib.extend(list(self.direct_syslibs))
 103         for lib in self.final_libs:
 104             t = self.bld.get_tgen_by_name(lib)
 105             self.uselib.extend(list(t.final_syslibs))
 106         self.uselib = unique_list(self.uselib)
 107
 108     if getattr(self, 'uselib', None):
 109         up_list = []
 110         for l in self.uselib:
 111            up_list.append(l.upper())
 112         self.uselib = up_list
 113
 114
 115 def build_includes(self):
 116     '''This builds the right set of includes for a target.
 117
 118     One tricky part of this is that the includes= attribute for a
 119     target needs to use paths which are relative to that targets
 120     declaration directory (which we can get at via t.path).
 121
 122     The way this works is the includes list gets added as
 123     samba_includes in the main build task declaration. Then this
 124     function runs after all of the tasks are declared, and it
 125     processes the samba_includes attribute to produce a includes=
 126     attribute
 127     '''
 128
 129     if getattr(self, 'samba_includes', None) is None:
 130         return
 131
 132     bld = self.bld
 133
 134     inc_deps = includes_objects(bld, self, set(), {})
 135
 136     includes = []
 137
 138     # maybe add local includes
 139     if getattr(self, 'local_include', True) and getattr(self, 'local_include_first', True):
 140         includes.append('.')
 141
 142     includes.extend(self.samba_includes_extended)
 143
 144     if 'EXTRA_INCLUDES' in bld.env and getattr(self, 'global_include', True):
 145         includes.extend(bld.env['EXTRA_INCLUDES'])
 146
 147     includes.append('#')
 148
 149     inc_set = set()
 150     inc_abs = []
 151
 152     for d in inc_deps:
 153         t = bld.get_tgen_by_name(d)
 154         bld.ASSERT(t is not None, "Unable to find dependency %s for %s" % (d, self.sname))
 155         inclist = getattr(t, 'samba_includes_extended', [])[:]
 156         if getattr(t, 'local_include', True):
 157             inclist.append('.')
 158         if inclist == []:
 159             continue
 160         tpath = t.samba_abspath
 161         for inc in inclist:
 162             npath = tpath + '/' + inc
 163             if not npath in inc_set:
 164                 inc_abs.append(npath)
 165                 inc_set.add(npath)
 166
 167     mypath = self.path.abspath(bld.env)
 168     for inc in inc_abs:
 169         relpath = os_path_relpath(inc, mypath)
 170         includes.append(relpath)
 171
 172     if getattr(self, 'local_include', True) and not getattr(self, 'local_include_first', True):
 173         includes.append('.')
 174
 175     # now transform the includes list to be relative to the top directory
 176     # which is represented by '#' in waf. This allows waf to cache the
 177     # includes lists more efficiently
 178     includes_top = []
 179     for i in includes:
 180         if i[0] == '#':
 181             # some are already top based
 182             includes_top.append(i)
 183             continue
 184         absinc = os.path.join(self.path.abspath(), i)
 185         relinc = os_path_relpath(absinc, self.bld.srcnode.abspath())
 186         includes_top.append('#' + relinc)
 187
 188     self.includes = unique_list(includes_top)
 189     debug('deps: includes for target %s: includes=%s',
 190           self.sname, self.includes)
 191
 192
 193 def add_init_functions(self):
 194     '''This builds the right set of init functions'''
 195
 196     bld = self.bld
 197
 198     subsystems = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 199
 200     # cope with the separated object lists from BINARY and LIBRARY targets
 201     sname = self.sname
 202     if sname.endswith('.objlist'):
 203         sname = sname[0:-8]
 204
 205     modules = []
 206     if sname in subsystems:
 207         modules.append(sname)
 208
 209     m = getattr(self, 'samba_modules', None)
 210     if m is not None:
 211         modules.extend(TO_LIST(m))
 212
 213     m = getattr(self, 'samba_subsystem', None)
 214     if m is not None:
 215         modules.append(m)
 216
 217     if 'pyembed' in self.features:
 218         return
 219
 220     sentinel = getattr(self, 'init_function_sentinel', 'NULL')
 221
 222     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
 223     cflags = getattr(self, 'samba_cflags', [])[:]
 224
 225     if modules == []:
 226         sname = sname.replace('-','_')
 227         sname = sname.replace('/','_')
 228         cflags.append('-DSTATIC_%s_MODULES=%s' % (sname, sentinel))
 229         if sentinel == 'NULL':
 230             proto = "extern void __%s_dummy_module_proto(void)" % (sname)
 231             cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (sname, proto))
 232         self.ccflags = cflags
 233         return
 234
 235     for m in modules:
 236         bld.ASSERT(m in subsystems,
 237                    "No init_function defined for module '%s' in target '%s'" % (m, self.sname))
 238         init_fn_list = []
 239         for d in subsystems[m]:
 240             if targets[d['TARGET']] != 'DISABLED':
 241                 init_fn_list.append(d['INIT_FUNCTION'])
 242         if init_fn_list == []:
 243             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, sentinel))
 244             if sentinel == 'NULL':
 245                 proto = "extern void __%s_dummy_module_proto(void)" % (m)
 246                 cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
 247         else:
 248             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, ','.join(init_fn_list) + ',' + sentinel))
 249             proto=''
 250             for f in init_fn_list:
 251                 proto += '_MODULE_PROTO(%s)' % f
 252             proto += "extern void __%s_dummy_module_proto(void)" % (m)
 253             cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
 254     self.ccflags = cflags
 255
 256
 257 def check_duplicate_sources(bld, tgt_list):
 258     '''see if we are compiling the same source file more than once
 259        without an allow_duplicates attribute'''
 260
 261     debug('deps: checking for duplicate sources')
 262
 263     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 264     ret = True
 265
 266     global tstart
 267
 268     for t in tgt_list:
 269         source_list = TO_LIST(getattr(t, 'source', ''))
 270         tpath = os.path.normpath(os_path_relpath(t.path.abspath(bld.env), t.env.BUILD_DIRECTORY + '/default'))
 271         obj_sources = set()
 272         for s in source_list:
 273             p = os.path.normpath(os.path.join(tpath, s))
 274             if p in obj_sources:
 275                 Logs.error("ERROR: source %s appears twice in target '%s'" % (p, t.sname))
 276                 sys.exit(1)
 277             obj_sources.add(p)
 278         t.samba_source_set = obj_sources
 279
 280     subsystems = {}
 281
 282     # build a list of targets that each source file is part of
 283     for t in tgt_list:
 284         sources = []
 285         if not targets[t.sname] in [ 'LIBRARY', 'BINARY', 'PYTHON' ]:
 286             continue
 287         for obj in t.add_objects:
 288             t2 = t.bld.get_tgen_by_name(obj)
 289             source_set = getattr(t2, 'samba_source_set', set())
 290             for s in source_set:
 291                 if not s in subsystems:
 292                     subsystems[s] = {}
 293                 if not t.sname in subsystems[s]:
 294                     subsystems[s][t.sname] = []
 295                 subsystems[s][t.sname].append(t2.sname)
 296
 297     for s in subsystems:
 298         if len(subsystems[s]) > 1 and Options.options.SHOW_DUPLICATES:
 299             Logs.warn("WARNING: source %s is in more than one target: %s" % (s, subsystems[s].keys()))
 300         for tname in subsystems[s]:
 301             if len(subsystems[s][tname]) > 1:
 302                 raise Utils.WafError("ERROR: source %s is in more than one subsystem of target '%s': %s" % (s, tname, subsystems[s][tname]))
 303
 304     return ret
 305
 306
 307 def check_orphaned_targets(bld, tgt_list):
 308     '''check if any build targets are orphaned'''
 309
 310     target_dict = LOCAL_CACHE(bld, 'TARGET_TYPE')
 311
 312     debug('deps: checking for orphaned targets')
 313
 314     for t in tgt_list:
 315         if getattr(t, 'samba_used', False):
 316             continue
 317         type = target_dict[t.sname]
 318         if not type in ['BINARY', 'LIBRARY', 'MODULE', 'ET', 'PYTHON']:
 319             if re.search('^PIDL_', t.sname) is None:
 320                 Logs.warn("Target %s of type %s is unused by any other target" % (t.sname, type))
 321
 322
 323 def check_group_ordering(bld, tgt_list):
 324     '''see if we have any dependencies that violate the group ordering
 325
 326     It is an error for a target to depend on a target from a later
 327     build group
 328     '''
 329
 330     def group_name(g):
 331         tm = bld.task_manager
 332         return [x for x in tm.groups_names if id(tm.groups_names[x]) == id(g)][0]
 333
 334     for g in bld.task_manager.groups:
 335         gname = group_name(g)
 336         for t in g.tasks_gen:
 337             t.samba_group = gname
 338
 339     grp_map = {}
 340     idx = 0
 341     for g in bld.task_manager.groups:
 342         name = group_name(g)
 343         grp_map[name] = idx
 344         idx += 1
 345
 346     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 347
 348     ret = True
 349     for t in tgt_list:
 350         tdeps = getattr(t, 'add_objects', []) + getattr(t, 'uselib_local', [])
 351         for d in tdeps:
 352             t2 = bld.get_tgen_by_name(d)
 353             if t2 is None:
 354                 continue
 355             map1 = grp_map[t.samba_group]
 356             map2 = grp_map[t2.samba_group]
 357
 358             if map2 > map1:
 359                 Logs.error("Target %r in build group %r depends on target %r from later build group %r" % (
 360                            t.sname, t.samba_group, t2.sname, t2.samba_group))
 361                 ret = False
 362
 363     return ret
 364
 365
 366 def show_final_deps(bld, tgt_list):
 367     '''show the final dependencies for all targets'''
 368
 369     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 370
 371     for t in tgt_list:
 372         if not targets[t.sname] in ['LIBRARY', 'BINARY', 'PYTHON', 'SUBSYSTEM']:
 373             continue
 374         debug('deps: final dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
 375               t.sname, t.uselib, getattr(t, 'uselib_local', []), getattr(t, 'add_objects', []))
 376
 377
 378 def add_samba_attributes(bld, tgt_list):
 379     '''ensure a target has a the required samba attributes'''
 380
 381     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 382
 383     for t in tgt_list:
 384         if t.name != '':
 385             t.sname = t.name
 386         else:
 387             t.sname = t.target
 388         t.samba_type = targets[t.sname]
 389         t.samba_abspath = t.path.abspath(bld.env)
 390         t.samba_deps_extended = t.samba_deps[:]
 391         t.samba_includes_extended = TO_LIST(t.samba_includes)[:]
 392         t.ccflags = getattr(t, 'samba_cflags', '')
 393
 394 def replace_grouping_libraries(bld, tgt_list):
 395     '''replace dependencies based on grouping libraries
 396
 397     If a library is marked as a grouping library, then any target that
 398     depends on a subsystem that is part of that grouping library gets
 399     that dependency replaced with a dependency on the grouping library
 400     '''
 401
 402     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 403
 404     grouping = {}
 405
 406     # find our list of grouping libraries, mapped from the subsystems they depend on
 407     for t in tgt_list:
 408         if not getattr(t, 'grouping_library', False):
 409             continue
 410         for dep in t.samba_deps_extended:
 411             bld.ASSERT(dep in targets, "grouping library target %s not declared in %s" % (dep, t.sname))
 412             if targets[dep] == 'SUBSYSTEM':
 413                 grouping[dep] = t.sname
 414
 415     # now replace any dependencies on elements of grouping libraries
 416     for t in tgt_list:
 417         for i in range(len(t.samba_deps_extended)):
 418             dep = t.samba_deps_extended[i]
 419             if dep in grouping:
 420                 if t.sname != grouping[dep]:
 421                     debug("deps: target %s: replacing dependency %s with grouping library %s" % (t.sname, dep, grouping[dep]))
 422                     t.samba_deps_extended[i] = grouping[dep]
 423
 424
 425
 426 def build_direct_deps(bld, tgt_list):
 427     '''build the direct_objects and direct_libs sets for each target'''
 428
 429     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 430     syslib_deps  = LOCAL_CACHE(bld, 'SYSLIB_DEPS')
 431
 432     global_deps = bld.env.GLOBAL_DEPENDENCIES
 433     global_deps_exclude = set()
 434     for dep in global_deps:
 435         t = bld.get_tgen_by_name(dep)
 436         for d in t.samba_deps:
 437             # prevent loops from the global dependencies list
 438             global_deps_exclude.add(d)
 439             global_deps_exclude.add(d + '.objlist')
 440
 441     for t in tgt_list:
 442         t.direct_objects = set()
 443         t.direct_libs = set()
 444         t.direct_syslibs = set()
 445         deps = t.samba_deps_extended[:]
 446         if getattr(t, 'samba_use_global_deps', False) and not t.sname in global_deps_exclude:
 447             deps.extend(global_deps)
 448         for d in deps:
 449             if d == t.sname: continue
 450             if not d in targets:
 451                 Logs.error("Unknown dependency '%s' in '%s'" % (d, t.sname))
 452                 sys.exit(1)
 453             if targets[d] in [ 'EMPTY', 'DISABLED' ]:
 454                 continue
 455             if targets[d] == 'PYTHON' and targets[t.sname] != 'PYTHON' and t.sname.find('.objlist') == -1:
 456                 # this check should be more restrictive, but for now we have pidl-generated python
 457                 # code that directly depends on other python modules
 458                 Logs.error('ERROR: Target %s has dependency on python module %s' % (t.sname, d))
 459                 sys.exit(1)
 460             if targets[d] == 'SYSLIB':
 461                 t.direct_syslibs.add(d)
 462                 if d in syslib_deps:
 463                     for implied in TO_LIST(syslib_deps[d]):
 464                         if BUILTIN_LIBRARY(bld, implied):
 465                             t.direct_objects.add(implied)
 466                         elif targets[implied] == 'SYSLIB':
 467                             t.direct_syslibs.add(implied)
 468                         elif targets[implied] in ['LIBRARY', 'MODULE']:
 469                             t.direct_libs.add(implied)
 470                         else:
 471                             Logs.error('Implied dependency %s in %s is of type %s' % (
 472                                 implied, t.sname, targets[implied]))
 473                             sys.exit(1)
 474                 continue
 475             t2 = bld.get_tgen_by_name(d)
 476             if t2 is None:
 477                 Logs.error("no task %s of type %s in %s" % (d, targets[d], t.sname))
 478                 sys.exit(1)
 479             if t2.samba_type in [ 'LIBRARY', 'MODULE' ]:
 480                 t.direct_libs.add(d)
 481             elif t2.samba_type in [ 'SUBSYSTEM', 'ASN1', 'PYTHON' ]:
 482                 t.direct_objects.add(d)
 483     debug('deps: built direct dependencies')
 484
 485
 486 def dependency_loop(loops, t, target):
 487     '''add a dependency loop to the loops dictionary'''
 488     if t.sname == target:
 489         return
 490     if not target in loops:
 491         loops[target] = set()
 492     if not t.sname in loops[target]:
 493         loops[target].add(t.sname)
 494
 495
 496 def indirect_libs(bld, t, chain, loops):
 497     '''recursively calculate the indirect library dependencies for a target
 498
 499     An indirect library is a library that results from a dependency on
 500     a subsystem
 501     '''
 502
 503     ret = getattr(t, 'indirect_libs', None)
 504     if ret is not None:
 505         return ret
 506
 507     ret = set()
 508     for obj in t.direct_objects:
 509         if obj in chain:
 510             dependency_loop(loops, t, obj)
 511             continue
 512         chain.add(obj)
 513         t2 = bld.get_tgen_by_name(obj)
 514         r2 = indirect_libs(bld, t2, chain, loops)
 515         chain.remove(obj)
 516         ret = ret.union(t2.direct_libs)
 517         ret = ret.union(r2)
 518
 519     for obj in indirect_objects(bld, t, set(), loops):
 520         if obj in chain:
 521             dependency_loop(loops, t, obj)
 522             continue
 523         chain.add(obj)
 524         t2 = bld.get_tgen_by_name(obj)
 525         r2 = indirect_libs(bld, t2, chain, loops)
 526         chain.remove(obj)
 527         ret = ret.union(t2.direct_libs)
 528         ret = ret.union(r2)
 529
 530     t.indirect_libs = ret
 531
 532     return ret
 533
 534
 535 def indirect_objects(bld, t, chain, loops):
 536     '''recursively calculate the indirect object dependencies for a target
 537
 538     indirect objects are the set of objects from expanding the
 539     subsystem dependencies
 540     '''
 541
 542     ret = getattr(t, 'indirect_objects', None)
 543     if ret is not None: return ret
 544
 545     ret = set()
 546     for lib in t.direct_objects:
 547         if lib in chain:
 548             dependency_loop(loops, t, lib)
 549             continue
 550         chain.add(lib)
 551         t2 = bld.get_tgen_by_name(lib)
 552         r2 = indirect_objects(bld, t2, chain, loops)
 553         chain.remove(lib)
 554         ret = ret.union(t2.direct_objects)
 555         ret = ret.union(r2)
 556
 557     t.indirect_objects = ret
 558     return ret
 559
 560
 561 def extended_objects(bld, t, chain):
 562     '''recursively calculate the extended object dependencies for a target
 563
 564     extended objects are the union of:
 565        - direct objects
 566        - indirect objects
 567        - direct and indirect objects of all direct and indirect libraries
 568     '''
 569
 570     ret = getattr(t, 'extended_objects', None)
 571     if ret is not None: return ret
 572
 573     ret = set()
 574     ret = ret.union(t.final_objects)
 575
 576     for lib in t.final_libs:
 577         if lib in chain:
 578             continue
 579         t2 = bld.get_tgen_by_name(lib)
 580         chain.add(lib)
 581         r2 = extended_objects(bld, t2, chain)
 582         chain.remove(lib)
 583         ret = ret.union(t2.final_objects)
 584         ret = ret.union(r2)
 585
 586     t.extended_objects = ret
 587     return ret
 588
 589
 590 def includes_objects(bld, t, chain, inc_loops):
 591     '''recursively calculate the includes object dependencies for a target
 592
 593     includes dependencies come from either library or object dependencies
 594     '''
 595     ret = getattr(t, 'includes_objects', None)
 596     if ret is not None:
 597         return ret
 598
 599     ret = t.direct_objects.copy()
 600     ret = ret.union(t.direct_libs)
 601
 602     for obj in t.direct_objects:
 603         if obj in chain:
 604             dependency_loop(inc_loops, t, obj)
 605             continue
 606         chain.add(obj)
 607         t2 = bld.get_tgen_by_name(obj)
 608         r2 = includes_objects(bld, t2, chain, inc_loops)
 609         chain.remove(obj)
 610         ret = ret.union(t2.direct_objects)
 611         ret = ret.union(r2)
 612
 613     for lib in t.direct_libs:
 614         if lib in chain:
 615             dependency_loop(inc_loops, t, lib)
 616             continue
 617         chain.add(lib)
 618         t2 = bld.get_tgen_by_name(lib)
 619         if t2 is None:
 620             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 621             Logs.error('Target %s of type %s not found in direct_libs for %s' % (
 622                 lib, targets[lib], t.sname))
 623             sys.exit(1)
 624         r2 = includes_objects(bld, t2, chain, inc_loops)
 625         chain.remove(lib)
 626         ret = ret.union(t2.direct_objects)
 627         ret = ret.union(r2)
 628
 629     t.includes_objects = ret
 630     return ret
 631
 632
 633 def break_dependency_loops(bld, tgt_list):
 634     '''find and break dependency loops'''
 635     loops = {}
 636     inc_loops = {}
 637
 638     # build up the list of loops
 639     for t in tgt_list:
 640         indirect_objects(bld, t, set(), loops)
 641         indirect_libs(bld, t, set(), loops)
 642         includes_objects(bld, t, set(), inc_loops)
 643
 644     # break the loops
 645     for t in tgt_list:
 646         if t.sname in loops:
 647             for attr in ['direct_objects', 'indirect_objects', 'direct_libs', 'indirect_libs']:
 648                 objs = getattr(t, attr, set())
 649                 setattr(t, attr, objs.difference(loops[t.sname]))
 650
 651     for loop in loops:
 652         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 653
 654     for loop in inc_loops:
 655         debug('deps: Found include loops for target %s : %s', loop, inc_loops[loop])
 656
 657     # expand the loops mapping by one level
 658     for loop in loops.copy():
 659         for tgt in loops[loop]:
 660             if tgt in loops:
 661                 loops[loop] = loops[loop].union(loops[tgt])
 662
 663     for loop in inc_loops.copy():
 664         for tgt in inc_loops[loop]:
 665             if tgt in inc_loops:
 666                 inc_loops[loop] = inc_loops[loop].union(inc_loops[tgt])
 667
 668
 669     # expand indirect subsystem and library loops
 670     for loop in loops.copy():
 671         t = bld.get_tgen_by_name(loop)
 672         if t.samba_type in ['SUBSYSTEM']:
 673             loops[loop] = loops[loop].union(t.indirect_objects)
 674             loops[loop] = loops[loop].union(t.direct_objects)
 675         if t.samba_type in ['LIBRARY','PYTHON']:
 676             loops[loop] = loops[loop].union(t.indirect_libs)
 677             loops[loop] = loops[loop].union(t.direct_libs)
 678         if loop in loops[loop]:
 679             loops[loop].remove(loop)
 680
 681     # expand indirect includes loops
 682     for loop in inc_loops.copy():
 683         t = bld.get_tgen_by_name(loop)
 684         inc_loops[loop] = inc_loops[loop].union(t.includes_objects)
 685         if loop in inc_loops[loop]:
 686             inc_loops[loop].remove(loop)
 687
 688     # add in the replacement dependencies
 689     for t in tgt_list:
 690         for loop in loops:
 691             for attr in ['indirect_objects', 'indirect_libs']:
 692                 objs = getattr(t, attr, set())
 693                 if loop in objs:
 694                     diff = loops[loop].difference(objs)
 695                     if t.sname in diff:
 696                         diff.remove(t.sname)
 697                     if diff:
 698                         debug('deps: Expanded target %s of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 699                         objs = objs.union(diff)
 700                 setattr(t, attr, objs)
 701
 702         for loop in inc_loops:
 703             objs = getattr(t, 'includes_objects', set())
 704             if loop in objs:
 705                 diff = inc_loops[loop].difference(objs)
 706                 if t.sname in diff:
 707                     diff.remove(t.sname)
 708                 if diff:
 709                     debug('deps: Expanded target %s includes of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 710                     objs = objs.union(diff)
 711             setattr(t, 'includes_objects', objs)
 712
 713
 714 def reduce_objects(bld, tgt_list):
 715     '''reduce objects by looking for indirect object dependencies'''
 716     rely_on = {}
 717
 718     for t in tgt_list:
 719         t.extended_objects = None
 720
 721     changed = False
 722
 723     for type in ['BINARY', 'PYTHON', 'LIBRARY']:
 724         for t in tgt_list:
 725             if t.samba_type != type: continue
 726             # if we will indirectly link to a target then we don't need it
 727             new = t.final_objects.copy()
 728             for l in t.final_libs:
 729                 t2 = bld.get_tgen_by_name(l)
 730                 t2_obj = extended_objects(bld, t2, set())
 731                 dup = new.intersection(t2_obj)
 732                 if t.sname in rely_on:
 733                     dup = dup.difference(rely_on[t.sname])
 734                 if dup:
 735                     debug('deps: removing dups from %s of type %s: %s also in %s %s',
 736                           t.sname, t.samba_type, dup, t2.samba_type, l)
 737                     new = new.difference(dup)
 738                     changed = True
 739                     if not l in rely_on:
 740                         rely_on[l] = set()
 741                     rely_on[l] = rely_on[l].union(dup)
 742             t.final_objects = new
 743
 744     if not changed:
 745         return False
 746
 747     # add back in any objects that were relied upon by the reduction rules
 748     for r in rely_on:
 749         t = bld.get_tgen_by_name(r)
 750         t.final_objects = t.final_objects.union(rely_on[r])
 751
 752     return True
 753
 754
 755 def show_library_loop(bld, lib1, lib2, path, seen):
 756     '''show the detailed path of a library loop between lib1 and lib2'''
 757
 758     t = bld.get_tgen_by_name(lib1)
 759     if not lib2 in getattr(t, 'final_libs', set()):
 760         return
 761
 762     for d in t.samba_deps_extended:
 763         if d in seen:
 764             continue
 765         seen.add(d)
 766         path2 = path + '=>' + d
 767         if d == lib2:
 768             Logs.warn('library loop path: ' + path2)
 769             return
 770         show_library_loop(bld, d, lib2, path2, seen)
 771         seen.remove(d)
 772
 773
 774 def calculate_final_deps(bld, tgt_list, loops):
 775     '''calculate the final library and object dependencies'''
 776     for t in tgt_list:
 777         # start with the maximum possible list
 778         t.final_libs    = t.direct_libs.union(indirect_libs(bld, t, set(), loops))
 779         t.final_objects = t.direct_objects.union(indirect_objects(bld, t, set(), loops))
 780
 781     for t in tgt_list:
 782         # don't depend on ourselves
 783         if t.sname in t.final_libs:
 784             t.final_libs.remove(t.sname)
 785         if t.sname in t.final_objects:
 786             t.final_objects.remove(t.sname)
 787
 788     # handle any non-shared binaries
 789     for t in tgt_list:
 790         if t.samba_type == 'BINARY' and bld.NONSHARED_BINARY(t.sname):
 791             subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 792             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 793
 794             # replace lib deps with objlist deps
 795             for l in t.final_libs:
 796                 objname = l + '.objlist'
 797                 t2 = bld.get_tgen_by_name(objname)
 798                 if t2 is None:
 799                     Logs.error('ERROR: subsystem %s not found' % objname)
 800                     sys.exit(1)
 801                 t.final_objects.add(objname)
 802                 t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 803                 if l in subsystem_list:
 804                     # its a subsystem - we also need the contents of any modules
 805                     for d in subsystem_list[l]:
 806                         module_name = d['TARGET']
 807                         if targets[module_name] == 'LIBRARY':
 808                             objname = module_name + '.objlist'
 809                         elif targets[module_name] == 'SUBSYSTEM':
 810                             objname = module_name
 811                         else:
 812                             continue
 813                         t2 = bld.get_tgen_by_name(objname)
 814                         if t2 is None:
 815                             Logs.error('ERROR: subsystem %s not found' % objname)
 816                             sys.exit(1)
 817                         t.final_objects.add(objname)
 818                         t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 819             t.final_libs = set()
 820
 821     # find any library loops
 822     for t in tgt_list:
 823         if t.samba_type in ['LIBRARY', 'PYTHON']:
 824             for l in t.final_libs.copy():
 825                 t2 = bld.get_tgen_by_name(l)
 826                 if t.sname in t2.final_libs:
 827                     if getattr(bld.env, "ALLOW_CIRCULAR_LIB_DEPENDENCIES", False):
 828                         # we could break this in either direction. If one of the libraries
 829                         # has a version number, and will this be distributed publicly, then
 830                         # we should make it the lower level library in the DAG
 831                         Logs.warn('deps: removing library loop %s from %s' % (t.sname, t2.sname))
 832                         dependency_loop(loops, t, t2.sname)
 833                         t2.final_libs.remove(t.sname)
 834                     else:
 835                         Logs.error('ERROR: circular library dependency between %s and %s'
 836                             % (t.sname, t2.sname))
 837                         show_library_loop(bld, t.sname, t2.sname, t.sname, set())
 838                         show_library_loop(bld, t2.sname, t.sname, t2.sname, set())
 839                         sys.exit(1)
 840
 841     for loop in loops:
 842         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 843
 844     # we now need to make corrections for any library loops we broke up
 845     # any target that depended on the target of the loop and doesn't
 846     # depend on the source of the loop needs to get the loop source added
 847     for type in ['BINARY','PYTHON','LIBRARY','BINARY']:
 848         for t in tgt_list:
 849             if t.samba_type != type: continue
 850             for loop in loops:
 851                 if loop in t.final_libs:
 852                     diff = loops[loop].difference(t.final_libs)
 853                     if t.sname in diff:
 854                         diff.remove(t.sname)
 855                     if t.sname in diff:
 856                         diff.remove(t.sname)
 857                     # make sure we don't recreate the loop again!
 858                     for d in diff.copy():
 859                         t2 = bld.get_tgen_by_name(d)
 860                         if t2.samba_type == 'LIBRARY':
 861                             if t.sname in t2.final_libs:
 862                                 debug('deps: removing expansion %s from %s', d, t.sname)
 863                                 diff.remove(d)
 864                     if diff:
 865                         debug('deps: Expanded target %s by loop %s libraries (loop %s) %s', t.sname, loop,
 866                               loops[loop], diff)
 867                         t.final_libs = t.final_libs.union(diff)
 868
 869     # remove objects that are also available in linked libs
 870     count = 0
 871     while reduce_objects(bld, tgt_list):
 872         count += 1
 873         if count > 100:
 874             Logs.warn("WARNING: Unable to remove all inter-target object duplicates")
 875             break
 876     debug('deps: Object reduction took %u iterations', count)
 877
 878     # add in any syslib dependencies
 879     for t in tgt_list:
 880         if not t.samba_type in ['BINARY','PYTHON','LIBRARY','SUBSYSTEM']:
 881             continue
 882         syslibs = set()
 883         for d in t.final_objects:
 884             t2 = bld.get_tgen_by_name(d)
 885             syslibs = syslibs.union(t2.direct_syslibs)
 886         # this adds the indirect syslibs as well, which may not be needed
 887         # depending on the linker flags
 888         for d in t.final_libs:
 889             t2 = bld.get_tgen_by_name(d)
 890             syslibs = syslibs.union(t2.direct_syslibs)
 891         t.final_syslibs = syslibs
 892
 893
 894     # find any unresolved library loops
 895     lib_loop_error = False
 896     for t in tgt_list:
 897         if t.samba_type in ['LIBRARY', 'PYTHON']:
 898             for l in t.final_libs.copy():
 899                 t2 = bld.get_tgen_by_name(l)
 900                 if t.sname in t2.final_libs:
 901                     Logs.error('ERROR: Unresolved library loop %s from %s' % (t.sname, t2.sname))
 902                     lib_loop_error = True
 903     if lib_loop_error:
 904         sys.exit(1)
 905
 906     debug('deps: removed duplicate dependencies')
 907
 908
 909 def show_dependencies(bld, target, seen):
 910     '''recursively show the dependencies of target'''
 911
 912     if target in seen:
 913         return
 914
 915     t = bld.get_tgen_by_name(target)
 916     if t is None:
 917         Logs.error("ERROR: Unable to find target '%s'" % target)
 918         sys.exit(1)
 919
 920     Logs.info('%s(OBJECTS): %s' % (target, t.direct_objects))
 921     Logs.info('%s(LIBS): %s' % (target, t.direct_libs))
 922     Logs.info('%s(SYSLIBS): %s' % (target, t.direct_syslibs))
 923
 924     seen.add(target)
 925
 926     for t2 in t.direct_objects:
 927         show_dependencies(bld, t2, seen)
 928
 929
 930 def show_object_duplicates(bld, tgt_list):
 931     '''show a list of object files that are included in more than
 932     one library or binary'''
 933
 934     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 935
 936     used_by = {}
 937
 938     Logs.info("showing duplicate objects")
 939
 940     for t in tgt_list:
 941         if not targets[t.sname] in [ 'LIBRARY', 'PYTHON' ]:
 942             continue
 943         for n in getattr(t, 'final_objects', set()):
 944             t2 = bld.get_tgen_by_name(n)
 945             if not n in used_by:
 946                 used_by[n] = set()
 947             used_by[n].add(t.sname)
 948
 949     for n in used_by:
 950         if len(used_by[n]) > 1:
 951             Logs.info("target '%s' is used by %s" % (n, used_by[n]))
 952
 953     Logs.info("showing indirect dependency counts (sorted by count)")
 954
 955     def indirect_count(t1, t2):
 956         return len(t2.indirect_objects) - len(t1.indirect_objects)
 957
 958     sorted_list = sorted(tgt_list, cmp=indirect_count)
 959     for t in sorted_list:
 960         if len(t.indirect_objects) > 1:
 961             Logs.info("%s depends on %u indirect objects" % (t.sname, len(t.indirect_objects)))
 962
 963
 964 ######################################################################
 965 # this provides a way to save our dependency calculations between runs
 966 savedeps_version = 3
 967 savedeps_inputs  = ['samba_deps', 'samba_includes', 'local_include', 'local_include_first', 'samba_cflags',
 968                     'source', 'grouping_library', 'samba_ldflags', 'allow_undefined_symbols',
 969                     'use_global_deps', 'global_include' ]
 970 savedeps_outputs = ['uselib', 'uselib_local', 'add_objects', 'includes',
 971                     'ccflags', 'ldflags', 'samba_deps_extended', 'final_libs']
 972 savedeps_outenv  = ['INC_PATHS']
 973 savedeps_envvars = ['NONSHARED_BINARIES', 'GLOBAL_DEPENDENCIES', 'EXTRA_CFLAGS', 'EXTRA_LDFLAGS', 'EXTRA_INCLUDES' ]
 974 savedeps_caches  = ['GLOBAL_DEPENDENCIES', 'TARGET_TYPE', 'INIT_FUNCTIONS', 'SYSLIB_DEPS']
 975 savedeps_files   = ['buildtools/wafsamba/samba_deps.py']
 976
 977 def save_samba_deps(bld, tgt_list):
 978     '''save the dependency calculations between builds, to make
 979        further builds faster'''
 980     denv = Environment.Environment()
 981
 982     denv.version = savedeps_version
 983     denv.savedeps_inputs = savedeps_inputs
 984     denv.savedeps_outputs = savedeps_outputs
 985     denv.input = {}
 986     denv.output = {}
 987     denv.outenv = {}
 988     denv.caches = {}
 989     denv.envvar = {}
 990     denv.files  = {}
 991
 992     for f in savedeps_files:
 993         denv.files[f] = os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime
 994
 995     for c in savedeps_caches:
 996         denv.caches[c] = LOCAL_CACHE(bld, c)
 997
 998     for e in savedeps_envvars:
 999         denv.envvar[e] = bld.env[e]
1000
1001     for t in tgt_list:
1002         # save all the input attributes for each target
1003         tdeps = {}
1004         for attr in savedeps_inputs:
1005             v = getattr(t, attr, None)
1006             if v is not None:
1007                 tdeps[attr] = v
1008         if tdeps != {}:
1009             denv.input[t.sname] = tdeps
1010
1011         # save all the output attributes for each target
1012         tdeps = {}
1013         for attr in savedeps_outputs:
1014             v = getattr(t, attr, None)
1015             if v is not None:
1016                 tdeps[attr] = v
1017         if tdeps != {}:
1018             denv.output[t.sname] = tdeps
1019
1020         tdeps = {}
1021         for attr in savedeps_outenv:
1022             if attr in t.env:
1023                 tdeps[attr] = t.env[attr]
1024         if tdeps != {}:
1025             denv.outenv[t.sname] = tdeps
1026
1027     depsfile = os.path.join(bld.bdir, "sambadeps")
1028     denv.store_fast(depsfile)
1029
1030
1031
1032 def load_samba_deps(bld, tgt_list):
1033     '''load a previous set of build dependencies if possible'''
1034     depsfile = os.path.join(bld.bdir, "sambadeps")
1035     denv = Environment.Environment()
1036     try:
1037         debug('deps: checking saved dependencies')
1038         denv.load_fast(depsfile)
1039         if (denv.version != savedeps_version or
1040             denv.savedeps_inputs != savedeps_inputs or
1041             denv.savedeps_outputs != savedeps_outputs):
1042             return False
1043     except Exception:
1044         return False
1045
1046     # check if critical files have changed
1047     for f in savedeps_files:
1048         if f not in denv.files:
1049             return False
1050         if denv.files[f] != os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime:
1051             return False
1052
1053     # check if caches are the same
1054     for c in savedeps_caches:
1055         if c not in denv.caches or denv.caches[c] != LOCAL_CACHE(bld, c):
1056             return False
1057
1058     # check if caches are the same
1059     for e in savedeps_envvars:
1060         if e not in denv.envvar or denv.envvar[e] != bld.env[e]:
1061             return False
1062
1063     # check inputs are the same
1064     for t in tgt_list:
1065         tdeps = {}
1066         for attr in savedeps_inputs:
1067             v = getattr(t, attr, None)
1068             if v is not None:
1069                 tdeps[attr] = v
1070         if t.sname in denv.input:
1071             olddeps = denv.input[t.sname]
1072         else:
1073             olddeps = {}
1074         if tdeps != olddeps:
1075             #print '%s: \ntdeps=%s \nodeps=%s' % (t.sname, tdeps, olddeps)
1076             return False
1077
1078     # put outputs in place
1079     for t in tgt_list:
1080         if not t.sname in denv.output: continue
1081         tdeps = denv.output[t.sname]
1082         for a in tdeps:
1083             setattr(t, a, tdeps[a])
1084
1085     # put output env vars in place
1086     for t in tgt_list:
1087         if not t.sname in denv.outenv: continue
1088         tdeps = denv.outenv[t.sname]
1089         for a in tdeps:
1090             t.env[a] = tdeps[a]
1091
1092     debug('deps: loaded saved dependencies')
1093     return True
1094
1095
1096
1097 def check_project_rules(bld):
1098     '''check the project rules - ensuring the targets are sane'''
1099
1100     loops = {}
1101     inc_loops = {}
1102
1103     tgt_list = get_tgt_list(bld)
1104
1105     add_samba_attributes(bld, tgt_list)
1106
1107     force_project_rules = (Options.options.SHOWDEPS or
1108                            Options.options.SHOW_DUPLICATES)
1109
1110     if not force_project_rules and load_samba_deps(bld, tgt_list):
1111         return
1112
1113     global tstart
1114     tstart = time.clock()
1115
1116     bld.new_rules = True
1117     Logs.info("Checking project rules ...")
1118
1119     debug('deps: project rules checking started')
1120
1121     expand_subsystem_deps(bld)
1122
1123     debug("deps: expand_subsystem_deps: %f" % (time.clock() - tstart))
1124
1125     replace_grouping_libraries(bld, tgt_list)
1126
1127     debug("deps: replace_grouping_libraries: %f" % (time.clock() - tstart))
1128
1129     build_direct_deps(bld, tgt_list)
1130
1131     debug("deps: build_direct_deps: %f" % (time.clock() - tstart))
1132
1133     break_dependency_loops(bld, tgt_list)
1134
1135     debug("deps: break_dependency_loops: %f" % (time.clock() - tstart))
1136
1137     if Options.options.SHOWDEPS:
1138             show_dependencies(bld, Options.options.SHOWDEPS, set())
1139
1140     calculate_final_deps(bld, tgt_list, loops)
1141
1142     debug("deps: calculate_final_deps: %f" % (time.clock() - tstart))
1143
1144     if Options.options.SHOW_DUPLICATES:
1145             show_object_duplicates(bld, tgt_list)
1146
1147     # run the various attribute generators
1148     for f in [ build_dependencies, build_includes, add_init_functions ]:
1149         debug('deps: project rules checking %s', f)
1150         for t in tgt_list: f(t)
1151         debug("deps: %s: %f" % (f, time.clock() - tstart))
1152
1153     debug('deps: project rules stage1 completed')
1154
1155     #check_orphaned_targets(bld, tgt_list)
1156
1157     if not check_duplicate_sources(bld, tgt_list):
1158         Logs.error("Duplicate sources present - aborting")
1159         sys.exit(1)
1160
1161     debug("deps: check_duplicate_sources: %f" % (time.clock() - tstart))
1162
1163     if not check_group_ordering(bld, tgt_list):
1164         Logs.error("Bad group ordering - aborting")
1165         sys.exit(1)
1166
1167     debug("deps: check_group_ordering: %f" % (time.clock() - tstart))
1168
1169     show_final_deps(bld, tgt_list)
1170
1171     debug("deps: show_final_deps: %f" % (time.clock() - tstart))
1172
1173     debug('deps: project rules checking completed - %u targets checked',
1174           len(tgt_list))
1175
1176     if not bld.is_install:
1177         save_samba_deps(bld, tgt_list)
1178
1179     debug("deps: save_samba_deps: %f" % (time.clock() - tstart))
1180
1181     Logs.info("Project rules pass")
1182
1183
1184 def CHECK_PROJECT_RULES(bld):
1185     '''enable checking of project targets for sanity'''
1186     if bld.env.added_project_rules:
1187         return
1188     bld.env.added_project_rules = True
1189     bld.add_pre_fun(check_project_rules)
1190 Build.BuildContext.CHECK_PROJECT_RULES = CHECK_PROJECT_RULES
1191
1192