buildtools/wafsamba/samba_deps.py

   1 # Samba automatic dependency handling and project rules
   2
   3 import Build, os, re, Environment, Logs, time
   4 from samba_utils import *
   5 from samba_autoconf import *
   6 from samba_bundled import BUILTIN_LIBRARY
   7
   8 @conf
   9 def ADD_GLOBAL_DEPENDENCY(ctx, dep):
  10     '''add a dependency for all binaries and libraries'''
  11     if not 'GLOBAL_DEPENDENCIES' in ctx.env:
  12         ctx.env.GLOBAL_DEPENDENCIES = []
  13     ctx.env.GLOBAL_DEPENDENCIES.append(dep)
  14
  15
  16 @conf
  17 def BREAK_CIRCULAR_LIBRARY_DEPENDENCIES(ctx):
  18     '''indicate that circular dependencies between libraries should be broken.'''
  19     ctx.env.ALLOW_CIRCULAR_LIB_DEPENDENCIES = True
  20
  21
  22 @conf
  23 def SET_SYSLIB_DEPS(conf, target, deps):
  24     '''setup some implied dependencies for a SYSLIB'''
  25     cache = LOCAL_CACHE(conf, 'SYSLIB_DEPS')
  26     cache[target] = deps
  27
  28
  29 def expand_subsystem_deps(bld):
  30     '''expand the reverse dependencies resulting from subsystem
  31        attributes of modules. This is walking over the complete list
  32        of declared subsystems, and expands the samba_deps_extended list for any
  33        module<->subsystem dependencies'''
  34
  35     subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
  36     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
  37
  38     for subsystem_name in subsystem_list:
  39         bld.ASSERT(subsystem_name in targets, "Subsystem target %s not declared" % subsystem_name)
  40         type = targets[subsystem_name]
  41         if type == 'DISABLED' or type == 'EMPTY':
  42             continue
  43
  44         # for example,
  45         #    subsystem_name = dcerpc_server (a subsystem)
  46         #    subsystem      = dcerpc_server (a subsystem object)
  47         #    module_name    = rpc_epmapper (a module within the dcerpc_server subsystem)
  48         #    module         = rpc_epmapper (a module object within the dcerpc_server subsystem)
  49
  50         subsystem = bld.name_to_obj(subsystem_name, bld.env)
  51         bld.ASSERT(subsystem is not None, "Unable to find subsystem %s" % subsystem_name)
  52         for d in subsystem_list[subsystem_name]:
  53             module_name = d['TARGET']
  54             module_type = targets[module_name]
  55             if module_type in ['DISABLED', 'EMPTY']:
  56                 continue
  57             bld.ASSERT(subsystem is not None,
  58                        "Subsystem target %s for %s (%s) not found" % (subsystem_name, module_name, module_type))
  59             if module_type in ['SUBSYSTEM']:
  60                 # if a module is a plain object type (not a library) then the
  61                 # subsystem it is part of needs to have it as a dependency, so targets
  62                 # that depend on this subsystem get the modules of that subsystem
  63                 subsystem.samba_deps_extended.append(module_name)
  64         subsystem.samba_deps_extended = unique_list(subsystem.samba_deps_extended)
  65
  66
  67
  68 def build_dependencies(self):
  69     '''This builds the dependency list for a target. It runs after all the targets are declared
  70
  71     The reason this is not just done in the SAMBA_*() rules is that we have no way of knowing
  72     the full dependency list for a target until we have all of the targets declared.
  73     '''
  74
  75     if self.samba_type in ['LIBRARY', 'BINARY', 'PYTHON']:
  76         self.uselib        = list(self.final_syslibs)
  77         self.uselib_local  = list(self.final_libs)
  78         self.add_objects   = list(self.final_objects)
  79
  80         # extra link flags from pkg_config
  81         libs = self.final_syslibs.copy()
  82
  83         (ccflags, ldflags) = library_flags(self, list(libs))
  84         new_ldflags        = getattr(self, 'ldflags', [])
  85         new_ldflags.extend(ldflags)
  86         self.ldflags       = new_ldflags
  87
  88         debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
  89               self.sname, self.uselib, self.uselib_local, self.add_objects)
  90
  91     if self.samba_type in ['SUBSYSTEM']:
  92         # this is needed for the ccflags of libs that come from pkg_config
  93         self.uselib = list(self.direct_syslibs)
  94
  95     if getattr(self, 'uselib', None):
  96         up_list = []
  97         for l in self.uselib:
  98            up_list.append(l.upper())
  99         self.uselib = up_list
 100
 101
 102 def build_includes(self):
 103     '''This builds the right set of includes for a target.
 104
 105     One tricky part of this is that the includes= attribute for a
 106     target needs to use paths which are relative to that targets
 107     declaration directory (which we can get at via t.path).
 108
 109     The way this works is the includes list gets added as
 110     samba_includes in the main build task declaration. Then this
 111     function runs after all of the tasks are declared, and it
 112     processes the samba_includes attribute to produce a includes=
 113     attribute
 114     '''
 115
 116     if getattr(self, 'samba_includes', None) is None:
 117         return
 118
 119     bld = self.bld
 120
 121     inc_deps = includes_objects(bld, self, set(), {})
 122
 123     includes = []
 124
 125     # maybe add local includes
 126     if getattr(self, 'local_include', True) == True and getattr(self, 'local_include_first', True):
 127         includes.append('.')
 128
 129     includes.extend(self.samba_includes_extended)
 130
 131     if 'EXTRA_INCLUDES' in bld.env:
 132         includes.extend(bld.env['EXTRA_INCLUDES'])
 133
 134     includes.append('#')
 135
 136     inc_set = set()
 137     inc_abs = []
 138
 139     for d in inc_deps:
 140         t = bld.name_to_obj(d, bld.env)
 141         bld.ASSERT(t is not None, "Unable to find dependency %s for %s" % (d, self.sname))
 142         inclist = getattr(t, 'samba_includes_extended', [])[:]
 143         if getattr(t, 'local_include', True) == True:
 144             inclist.append('.')
 145         if inclist == []:
 146             continue
 147         tpath = t.samba_abspath
 148         for inc in inclist:
 149             npath = tpath + '/' + inc
 150             if not npath in inc_set:
 151                 inc_abs.append(npath)
 152                 inc_set.add(npath)
 153
 154     mypath = self.path.abspath(bld.env)
 155     for inc in inc_abs:
 156         relpath = os_path_relpath(inc, mypath)
 157         includes.append(relpath)
 158
 159     if getattr(self, 'local_include', True) == True and not getattr(self, 'local_include_first', True):
 160         includes.append('.')
 161
 162     # now transform the includes list to be relative to the top directory
 163     # which is represented by '#' in waf. This allows waf to cache the
 164     # includes lists more efficiently
 165     includes_top = []
 166     for i in includes:
 167         if i[0] == '#':
 168             # some are already top based
 169             includes_top.append(i)
 170             continue
 171         absinc = os.path.join(self.path.abspath(), i)
 172         relinc = os_path_relpath(absinc, self.bld.srcnode.abspath())
 173         includes_top.append('#' + relinc)
 174
 175     self.includes = unique_list(includes_top)
 176     debug('deps: includes for target %s: includes=%s',
 177           self.sname, self.includes)
 178
 179
 180
 181
 182 def add_init_functions(self):
 183     '''This builds the right set of init functions'''
 184
 185     bld = self.bld
 186
 187     subsystems = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 188
 189     # cope with the separated object lists from BINARY and LIBRARY targets
 190     sname = self.sname
 191     if sname.endswith('.objlist'):
 192         sname = sname[0:-8]
 193
 194     modules = []
 195     if sname in subsystems:
 196         modules.append(sname)
 197
 198     m = getattr(self, 'samba_modules', None)
 199     if m is not None:
 200         modules.extend(TO_LIST(m))
 201
 202     m = getattr(self, 'samba_subsystem', None)
 203     if m is not None:
 204         modules.append(m)
 205
 206     sentinal = getattr(self, 'init_function_sentinal', 'NULL')
 207
 208     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
 209     cflags = getattr(self, 'samba_cflags', [])[:]
 210
 211     if modules == []:
 212         cflags.append('-DSTATIC_%s_MODULES=%s' % (sname.replace('-','_'), sentinal))
 213         self.ccflags = cflags
 214         return
 215
 216     for m in modules:
 217         bld.ASSERT(m in subsystems,
 218                    "No init_function defined for module '%s' in target '%s'" % (m, self.sname))
 219         init_fn_list = []
 220         for d in subsystems[m]:
 221             if targets[d['TARGET']] != 'DISABLED':
 222                 init_fn_list.append(d['INIT_FUNCTION'])
 223         if init_fn_list == []:
 224             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, sentinal))
 225         else:
 226             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, ','.join(init_fn_list) + ',' + sentinal))
 227     self.ccflags = cflags
 228
 229
 230
 231 def check_duplicate_sources(bld, tgt_list):
 232     '''see if we are compiling the same source file more than once
 233        without an allow_duplicates attribute'''
 234
 235     debug('deps: checking for duplicate sources')
 236
 237     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 238     ret = True
 239
 240     global tstart
 241
 242     for t in tgt_list:
 243         source_list = TO_LIST(getattr(t, 'source', ''))
 244         tpath = os.path.normpath(os_path_relpath(t.path.abspath(bld.env), t.env.BUILD_DIRECTORY + '/default'))
 245         obj_sources = set()
 246         for s in source_list:
 247             p = os.path.normpath(os.path.join(tpath, s))
 248             if p in obj_sources:
 249                 Logs.error("ERROR: source %s appears twice in target '%s'" % (p, t.sname))
 250                 sys.exit(1)
 251             obj_sources.add(p)
 252         t.samba_source_set = obj_sources
 253
 254     subsystems = {}
 255
 256     # build a list of targets that each source file is part of
 257     for t in tgt_list:
 258         sources = []
 259         if not targets[t.sname] in [ 'LIBRARY', 'BINARY', 'PYTHON' ]:
 260             continue
 261         for obj in t.add_objects:
 262             t2 = t.bld.name_to_obj(obj, bld.env)
 263             source_set = getattr(t2, 'samba_source_set', set())
 264             for s in source_set:
 265                 if not s in subsystems:
 266                     subsystems[s] = {}
 267                 if not t.sname in subsystems[s]:
 268                     subsystems[s][t.sname] = []
 269                 subsystems[s][t.sname].append(t2.sname)
 270
 271     for s in subsystems:
 272         if len(subsystems[s]) > 1 and Options.options.SHOW_DUPLICATES:
 273             Logs.warn("WARNING: source %s is in more than one target: %s" % (s, subsystems[s].keys()))
 274         for tname in subsystems[s]:
 275             if len(subsystems[s][tname]) > 1:
 276                 Logs.error("ERROR: source %s is in more than one subsystem of target '%s': %s" % (s, tname, subsystems[s][tname]))
 277                 sys.exit(1)
 278
 279     return ret
 280
 281
 282 def check_orpaned_targets(bld, tgt_list):
 283     '''check if any build targets are orphaned'''
 284
 285     target_dict = LOCAL_CACHE(bld, 'TARGET_TYPE')
 286
 287     debug('deps: checking for orphaned targets')
 288
 289     for t in tgt_list:
 290         if getattr(t, 'samba_used', False) == True:
 291             continue
 292         type = target_dict[t.sname]
 293         if not type in ['BINARY', 'LIBRARY', 'MODULE', 'ET', 'PYTHON']:
 294             if re.search('^PIDL_', t.sname) is None:
 295                 Logs.warn("Target %s of type %s is unused by any other target" % (t.sname, type))
 296
 297
 298 def check_group_ordering(bld, tgt_list):
 299     '''see if we have any dependencies that violate the group ordering
 300
 301     It is an error for a target to depend on a target from a later
 302     build group
 303     '''
 304
 305     def group_name(g):
 306         tm = bld.task_manager
 307         return [x for x in tm.groups_names if id(tm.groups_names[x]) == id(g)][0]
 308
 309     for g in bld.task_manager.groups:
 310         gname = group_name(g)
 311         for t in g.tasks_gen:
 312             t.samba_group = gname
 313
 314     grp_map = {}
 315     idx = 0
 316     for g in bld.task_manager.groups:
 317         name = group_name(g)
 318         grp_map[name] = idx
 319         idx += 1
 320
 321     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 322
 323     ret = True
 324     for t in tgt_list:
 325         tdeps = getattr(t, 'add_objects', []) + getattr(t, 'uselib_local', [])
 326         for d in tdeps:
 327             t2 = bld.name_to_obj(d, bld.env)
 328             if t2 is None:
 329                 continue
 330             map1 = grp_map[t.samba_group]
 331             map2 = grp_map[t2.samba_group]
 332
 333             if map2 > map1:
 334                 Logs.error("Target %r in build group %r depends on target %r from later build group %r" % (
 335                            t.sname, t.samba_group, t2.sname, t2.samba_group))
 336                 ret = False
 337
 338     return ret
 339
 340
 341 def show_final_deps(bld, tgt_list):
 342     '''show the final dependencies for all targets'''
 343
 344     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 345
 346     for t in tgt_list:
 347         if not targets[t.sname] in ['LIBRARY', 'BINARY', 'PYTHON']:
 348             continue
 349         debug('deps: final dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
 350               t.sname, t.uselib, t.uselib_local, t.add_objects)
 351
 352
 353 def add_samba_attributes(bld, tgt_list):
 354     '''ensure a target has a the required samba attributes'''
 355
 356     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 357
 358     for t in tgt_list:
 359         if t.name != '':
 360             t.sname = t.name
 361         else:
 362             t.sname = t.target
 363         t.samba_type = targets[t.sname]
 364         t.samba_abspath = t.path.abspath(bld.env)
 365         t.samba_deps_extended = t.samba_deps[:]
 366         t.samba_includes_extended = TO_LIST(t.samba_includes)[:]
 367         t.ccflags = getattr(t, 'samba_cflags', '')
 368
 369 def replace_grouping_libraries(bld, tgt_list):
 370     '''replace dependencies based on grouping libraries
 371
 372     If a library is marked as a grouping library, then any target that
 373     depends on a subsystem that is part of that grouping library gets
 374     that dependency replaced with a dependency on the grouping library
 375     '''
 376
 377     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 378
 379     grouping = {}
 380
 381     # find our list of grouping libraries, mapped from the subsystems they depend on
 382     for t in tgt_list:
 383         if not getattr(t, 'grouping_library', False):
 384             continue
 385         for dep in t.samba_deps_extended:
 386             bld.ASSERT(dep in targets, "grouping library target %s not declared in %s" % (dep, t.sname))
 387             if targets[dep] == 'SUBSYSTEM':
 388                 grouping[dep] = t.sname
 389
 390     # now replace any dependencies on elements of grouping libraries
 391     for t in tgt_list:
 392         for i in range(len(t.samba_deps_extended)):
 393             dep = t.samba_deps_extended[i]
 394             if dep in grouping:
 395                 if t.sname != grouping[dep]:
 396                     debug("deps: target %s: replacing dependency %s with grouping library %s" % (t.sname, dep, grouping[dep]))
 397                     t.samba_deps_extended[i] = grouping[dep]
 398
 399
 400
 401 def build_direct_deps(bld, tgt_list):
 402     '''build the direct_objects and direct_libs sets for each target'''
 403
 404     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 405     syslib_deps  = LOCAL_CACHE(bld, 'SYSLIB_DEPS')
 406
 407     global_deps = bld.env.GLOBAL_DEPENDENCIES
 408     global_deps_exclude = set()
 409     for dep in global_deps:
 410         t = bld.name_to_obj(dep, bld.env)
 411         for d in t.samba_deps:
 412             # prevent loops from the global dependencies list
 413             global_deps_exclude.add(d)
 414             global_deps_exclude.add(d + '.objlist')
 415
 416     for t in tgt_list:
 417         t.direct_objects = set()
 418         t.direct_libs = set()
 419         t.direct_syslibs = set()
 420         deps = t.samba_deps_extended[:]
 421         if getattr(t, 'samba_use_global_deps', False) and not t.sname in global_deps_exclude:
 422             deps.extend(global_deps)
 423         for d in deps:
 424             if d == t.sname: continue
 425             if not d in targets:
 426                 Logs.error("Unknown dependency '%s' in '%s'" % (d, t.sname))
 427                 sys.exit(1)
 428             if targets[d] in [ 'EMPTY', 'DISABLED' ]:
 429                 continue
 430             if targets[d] == 'PYTHON' and targets[t.sname] != 'PYTHON' and t.sname.find('.objlist') == -1:
 431                 # this check should be more restrictive, but for now we have pidl-generated python
 432                 # code that directly depends on other python modules
 433                 Logs.error('ERROR: Target %s has dependency on python module %s' % (t.sname, d))
 434                 sys.exit(1)
 435             if targets[d] == 'SYSLIB':
 436                 t.direct_syslibs.add(d)
 437                 if d in syslib_deps:
 438                     for implied in TO_LIST(syslib_deps[d]):
 439                         if BUILTIN_LIBRARY(bld, implied):
 440                             t.direct_objects.add(implied)
 441                         elif targets[implied] == 'SYSLIB':
 442                             t.direct_syslibs.add(implied)
 443                         elif targets[implied] in ['LIBRARY', 'MODULE']:
 444                             t.direct_libs.add(implied)
 445                         else:
 446                             Logs.error('Implied dependency %s in %s is of type %s' % (
 447                                 implied, t.sname, targets[implied]))
 448                             sys.exit(1)
 449                 continue
 450             t2 = bld.name_to_obj(d, bld.env)
 451             if t2 is None:
 452                 Logs.error("no task %s of type %s in %s" % (d, targets[d], t.sname))
 453                 sys.exit(1)
 454             if t2.samba_type in [ 'LIBRARY', 'MODULE' ]:
 455                 t.direct_libs.add(d)
 456             elif t2.samba_type in [ 'SUBSYSTEM', 'ASN1', 'PYTHON' ]:
 457                 t.direct_objects.add(d)
 458     debug('deps: built direct dependencies')
 459
 460
 461 def dependency_loop(loops, t, target):
 462     '''add a dependency loop to the loops dictionary'''
 463     if t.sname == target:
 464         return
 465     if not target in loops:
 466         loops[target] = set()
 467     if not t.sname in loops[target]:
 468         loops[target].add(t.sname)
 469
 470
 471 def indirect_libs(bld, t, chain, loops):
 472     '''recursively calculate the indirect library dependencies for a target
 473
 474     An indirect library is a library that results from a dependency on
 475     a subsystem
 476     '''
 477
 478     ret = getattr(t, 'indirect_libs', None)
 479     if ret is not None:
 480         return ret
 481
 482     ret = set()
 483     for obj in t.direct_objects:
 484         if obj in chain:
 485             dependency_loop(loops, t, obj)
 486             continue
 487         chain.add(obj)
 488         t2 = bld.name_to_obj(obj, bld.env)
 489         r2 = indirect_libs(bld, t2, chain, loops)
 490         chain.remove(obj)
 491         ret = ret.union(t2.direct_libs)
 492         ret = ret.union(r2)
 493
 494     for obj in indirect_objects(bld, t, set(), loops):
 495         if obj in chain:
 496             dependency_loop(loops, t, obj)
 497             continue
 498         chain.add(obj)
 499         t2 = bld.name_to_obj(obj, bld.env)
 500         r2 = indirect_libs(bld, t2, chain, loops)
 501         chain.remove(obj)
 502         ret = ret.union(t2.direct_libs)
 503         ret = ret.union(r2)
 504
 505     t.indirect_libs = ret
 506
 507     return ret
 508
 509
 510 def indirect_objects(bld, t, chain, loops):
 511     '''recursively calculate the indirect object dependencies for a target
 512
 513     indirect objects are the set of objects from expanding the
 514     subsystem dependencies
 515     '''
 516
 517     ret = getattr(t, 'indirect_objects', None)
 518     if ret is not None: return ret
 519
 520     ret = set()
 521     for lib in t.direct_objects:
 522         if lib in chain:
 523             dependency_loop(loops, t, lib)
 524             continue
 525         chain.add(lib)
 526         t2 = bld.name_to_obj(lib, bld.env)
 527         r2 = indirect_objects(bld, t2, chain, loops)
 528         chain.remove(lib)
 529         ret = ret.union(t2.direct_objects)
 530         ret = ret.union(r2)
 531
 532     t.indirect_objects = ret
 533     return ret
 534
 535
 536 def extended_objects(bld, t, chain):
 537     '''recursively calculate the extended object dependencies for a target
 538
 539     extended objects are the union of:
 540        - direct objects
 541        - indirect objects
 542        - direct and indirect objects of all direct and indirect libraries
 543     '''
 544
 545     ret = getattr(t, 'extended_objects', None)
 546     if ret is not None: return ret
 547
 548     ret = set()
 549     ret = ret.union(t.final_objects)
 550
 551     for lib in t.final_libs:
 552         if lib in chain:
 553             continue
 554         t2 = bld.name_to_obj(lib, bld.env)
 555         chain.add(lib)
 556         r2 = extended_objects(bld, t2, chain)
 557         chain.remove(lib)
 558         ret = ret.union(t2.final_objects)
 559         ret = ret.union(r2)
 560
 561     t.extended_objects = ret
 562     return ret
 563
 564
 565 def includes_objects(bld, t, chain, inc_loops):
 566     '''recursively calculate the includes object dependencies for a target
 567
 568     includes dependencies come from either library or object dependencies
 569     '''
 570     ret = getattr(t, 'includes_objects', None)
 571     if ret is not None:
 572         return ret
 573
 574     ret = t.direct_objects.copy()
 575     ret = ret.union(t.direct_libs)
 576
 577     for obj in t.direct_objects:
 578         if obj in chain:
 579             dependency_loop(inc_loops, t, obj)
 580             continue
 581         chain.add(obj)
 582         t2 = bld.name_to_obj(obj, bld.env)
 583         r2 = includes_objects(bld, t2, chain, inc_loops)
 584         chain.remove(obj)
 585         ret = ret.union(t2.direct_objects)
 586         ret = ret.union(r2)
 587
 588     for lib in t.direct_libs:
 589         if lib in chain:
 590             dependency_loop(inc_loops, t, lib)
 591             continue
 592         chain.add(lib)
 593         t2 = bld.name_to_obj(lib, bld.env)
 594         if t2 is None:
 595             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 596             Logs.error('Target %s of type %s not found in direct_libs for %s' % (
 597                 lib, targets[lib], t.sname))
 598             sys.exit(1)
 599         r2 = includes_objects(bld, t2, chain, inc_loops)
 600         chain.remove(lib)
 601         ret = ret.union(t2.direct_objects)
 602         ret = ret.union(r2)
 603
 604     t.includes_objects = ret
 605     return ret
 606
 607
 608 def break_dependency_loops(bld, tgt_list):
 609     '''find and break dependency loops'''
 610     loops = {}
 611     inc_loops = {}
 612
 613     # build up the list of loops
 614     for t in tgt_list:
 615         indirect_objects(bld, t, set(), loops)
 616         indirect_libs(bld, t, set(), loops)
 617         includes_objects(bld, t, set(), inc_loops)
 618
 619     # break the loops
 620     for t in tgt_list:
 621         if t.sname in loops:
 622             for attr in ['direct_objects', 'indirect_objects', 'direct_libs', 'indirect_libs']:
 623                 objs = getattr(t, attr, set())
 624                 setattr(t, attr, objs.difference(loops[t.sname]))
 625
 626     for loop in loops:
 627         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 628
 629     for loop in inc_loops:
 630         debug('deps: Found include loops for target %s : %s', loop, inc_loops[loop])
 631
 632     # expand the loops mapping by one level
 633     for loop in loops.copy():
 634         for tgt in loops[loop]:
 635             if tgt in loops:
 636                 loops[loop] = loops[loop].union(loops[tgt])
 637
 638     for loop in inc_loops.copy():
 639         for tgt in inc_loops[loop]:
 640             if tgt in inc_loops:
 641                 inc_loops[loop] = inc_loops[loop].union(inc_loops[tgt])
 642
 643
 644     # expand indirect subsystem and library loops
 645     for loop in loops.copy():
 646         t = bld.name_to_obj(loop, bld.env)
 647         if t.samba_type in ['SUBSYSTEM']:
 648             loops[loop] = loops[loop].union(t.indirect_objects)
 649             loops[loop] = loops[loop].union(t.direct_objects)
 650         if t.samba_type in ['LIBRARY','PYTHON']:
 651             loops[loop] = loops[loop].union(t.indirect_libs)
 652             loops[loop] = loops[loop].union(t.direct_libs)
 653         if loop in loops[loop]:
 654             loops[loop].remove(loop)
 655
 656     # expand indirect includes loops
 657     for loop in inc_loops.copy():
 658         t = bld.name_to_obj(loop, bld.env)
 659         inc_loops[loop] = inc_loops[loop].union(t.includes_objects)
 660         if loop in inc_loops[loop]:
 661             inc_loops[loop].remove(loop)
 662
 663     # add in the replacement dependencies
 664     for t in tgt_list:
 665         for loop in loops:
 666             for attr in ['indirect_objects', 'indirect_libs']:
 667                 objs = getattr(t, attr, set())
 668                 if loop in objs:
 669                     diff = loops[loop].difference(objs)
 670                     if t.sname in diff:
 671                         diff.remove(t.sname)
 672                     if diff:
 673                         debug('deps: Expanded target %s of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 674                         objs = objs.union(diff)
 675                 setattr(t, attr, objs)
 676
 677         for loop in inc_loops:
 678             objs = getattr(t, 'includes_objects', set())
 679             if loop in objs:
 680                 diff = inc_loops[loop].difference(objs)
 681                 if t.sname in diff:
 682                     diff.remove(t.sname)
 683                 if diff:
 684                     debug('deps: Expanded target %s includes of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 685                     objs = objs.union(diff)
 686             setattr(t, 'includes_objects', objs)
 687
 688
 689 def reduce_objects(bld, tgt_list):
 690     '''reduce objects by looking for indirect object dependencies'''
 691     rely_on = {}
 692
 693     for t in tgt_list:
 694         t.extended_objects = None
 695
 696     changed = False
 697
 698     for type in ['BINARY', 'PYTHON', 'LIBRARY']:
 699         for t in tgt_list:
 700             if t.samba_type != type: continue
 701             # if we will indirectly link to a target then we don't need it
 702             new = t.final_objects.copy()
 703             for l in t.final_libs:
 704                 t2 = bld.name_to_obj(l, bld.env)
 705                 t2_obj = extended_objects(bld, t2, set())
 706                 dup = new.intersection(t2_obj)
 707                 if t.sname in rely_on:
 708                     dup = dup.difference(rely_on[t.sname])
 709                 if dup:
 710                     debug('deps: removing dups from %s of type %s: %s also in %s %s',
 711                           t.sname, t.samba_type, dup, t2.samba_type, l)
 712                     new = new.difference(dup)
 713                     changed = True
 714                     if not l in rely_on:
 715                         rely_on[l] = set()
 716                     rely_on[l] = rely_on[l].union(dup)
 717             t.final_objects = new
 718
 719     if not changed:
 720         return False
 721
 722     # add back in any objects that were relied upon by the reduction rules
 723     for r in rely_on:
 724         t = bld.name_to_obj(r, bld.env)
 725         t.final_objects = t.final_objects.union(rely_on[r])
 726
 727     return True
 728
 729
 730 def show_library_loop(bld, lib1, lib2, path, seen):
 731     '''show the detailed path of a library loop between lib1 and lib2'''
 732
 733     t = bld.name_to_obj(lib1, bld.env)
 734     if not lib2 in getattr(t, 'final_libs', set()):
 735         return
 736
 737     for d in t.samba_deps_extended:
 738         if d in seen:
 739             continue
 740         seen.add(d)
 741         path2 = path + '=>' + d
 742         if d == lib2:
 743             Logs.warn('library loop path: ' + path2)
 744             return
 745         show_library_loop(bld, d, lib2, path2, seen)
 746         seen.remove(d)
 747
 748
 749 def calculate_final_deps(bld, tgt_list, loops):
 750     '''calculate the final library and object dependencies'''
 751     for t in tgt_list:
 752         # start with the maximum possible list
 753         t.final_libs    = t.direct_libs.union(indirect_libs(bld, t, set(), loops))
 754         t.final_objects = t.direct_objects.union(indirect_objects(bld, t, set(), loops))
 755
 756     for t in tgt_list:
 757         # don't depend on ourselves
 758         if t.sname in t.final_libs:
 759             t.final_libs.remove(t.sname)
 760         if t.sname in t.final_objects:
 761             t.final_objects.remove(t.sname)
 762
 763     # handle any non-shared binaries
 764     for t in tgt_list:
 765         if t.samba_type == 'BINARY' and bld.NONSHARED_BINARY(t.sname):
 766             subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 767             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 768
 769             # replace lib deps with objlist deps
 770             for l in t.final_libs:
 771                 objname = l + '.objlist'
 772                 t2 = bld.name_to_obj(objname, bld.env)
 773                 if t2 is None:
 774                     Logs.error('ERROR: subsystem %s not found' % objname)
 775                     sys.exit(1)
 776                 t.final_objects.add(objname)
 777                 t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 778                 if l in subsystem_list:
 779                     # its a subsystem - we also need the contents of any modules
 780                     for d in subsystem_list[l]:
 781                         module_name = d['TARGET']
 782                         if targets[module_name] == 'LIBRARY':
 783                             objname = module_name + '.objlist'
 784                         elif targets[module_name] == 'SUBSYSTEM':
 785                             objname = module_name
 786                         else:
 787                             continue
 788                         t2 = bld.name_to_obj(objname, bld.env)
 789                         if t2 is None:
 790                             Logs.error('ERROR: subsystem %s not found' % objname)
 791                             sys.exit(1)
 792                         t.final_objects.add(objname)
 793                         t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 794             t.final_libs = set()
 795
 796     # find any library loops
 797     for t in tgt_list:
 798         if t.samba_type in ['LIBRARY', 'PYTHON']:
 799             for l in t.final_libs.copy():
 800                 t2 = bld.name_to_obj(l, bld.env)
 801                 if t.sname in t2.final_libs:
 802                     if getattr(bld.env, "ALLOW_CIRCULAR_LIB_DEPENDENCIES", False):
 803                         # we could break this in either direction. If one of the libraries
 804                         # has a version number, and will this be distributed publicly, then
 805                         # we should make it the lower level library in the DAG
 806                         Logs.warn('deps: removing library loop %s from %s' % (t.sname, t2.sname))
 807                         dependency_loop(loops, t, t2.sname)
 808                         t2.final_libs.remove(t.sname)
 809                     else:
 810                         Logs.error('ERROR: circular library dependency between %s and %s'
 811                             % (t.sname, t2.sname))
 812                         show_library_loop(bld, t.sname, t2.sname, t.sname, set())
 813                         show_library_loop(bld, t2.sname, t.sname, t2.sname, set())
 814                         sys.exit(1)
 815
 816     for loop in loops:
 817         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 818
 819     # we now need to make corrections for any library loops we broke up
 820     # any target that depended on the target of the loop and doesn't
 821     # depend on the source of the loop needs to get the loop source added
 822     for type in ['BINARY','PYTHON','LIBRARY','BINARY']:
 823         for t in tgt_list:
 824             if t.samba_type != type: continue
 825             for loop in loops:
 826                 if loop in t.final_libs:
 827                     diff = loops[loop].difference(t.final_libs)
 828                     if t.sname in diff:
 829                         diff.remove(t.sname)
 830                     if t.sname in diff:
 831                         diff.remove(t.sname)
 832                     # make sure we don't recreate the loop again!
 833                     for d in diff.copy():
 834                         t2 = bld.name_to_obj(d, bld.env)
 835                         if t2.samba_type == 'LIBRARY':
 836                             if t.sname in t2.final_libs:
 837                                 debug('deps: removing expansion %s from %s', d, t.sname)
 838                                 diff.remove(d)
 839                     if diff:
 840                         debug('deps: Expanded target %s by loop %s libraries (loop %s) %s', t.sname, loop,
 841                               loops[loop], diff)
 842                         t.final_libs = t.final_libs.union(diff)
 843
 844     # remove objects that are also available in linked libs
 845     count = 0
 846     while reduce_objects(bld, tgt_list):
 847         count += 1
 848         if count > 100:
 849             Logs.warn("WARNING: Unable to remove all inter-target object duplicates")
 850             break
 851     debug('deps: Object reduction took %u iterations', count)
 852
 853     # add in any syslib dependencies
 854     for t in tgt_list:
 855         if not t.samba_type in ['BINARY','PYTHON','LIBRARY']:
 856             continue
 857         syslibs = set()
 858         for d in t.final_objects:
 859             t2 = bld.name_to_obj(d, bld.env)
 860             syslibs = syslibs.union(t2.direct_syslibs)
 861         # this adds the indirect syslibs as well, which may not be needed
 862         # depending on the linker flags
 863         for d in t.final_libs:
 864             t2 = bld.name_to_obj(d, bld.env)
 865             syslibs = syslibs.union(t2.direct_syslibs)
 866         t.final_syslibs = syslibs
 867
 868
 869     # find any unresolved library loops
 870     lib_loop_error = False
 871     for t in tgt_list:
 872         if t.samba_type in ['LIBRARY', 'PYTHON']:
 873             for l in t.final_libs.copy():
 874                 t2 = bld.name_to_obj(l, bld.env)
 875                 if t.sname in t2.final_libs:
 876                     Logs.error('ERROR: Unresolved library loop %s from %s' % (t.sname, t2.sname))
 877                     lib_loop_error = True
 878     if lib_loop_error:
 879         sys.exit(1)
 880
 881     debug('deps: removed duplicate dependencies')
 882
 883
 884 def show_dependencies(bld, target, seen):
 885     '''recursively show the dependencies of target'''
 886
 887     if target in seen:
 888         return
 889
 890     t = bld.name_to_obj(target, bld.env)
 891     if t is None:
 892         Logs.error("ERROR: Unable to find target '%s'" % target)
 893         sys.exit(1)
 894
 895     Logs.info('%s(OBJECTS): %s' % (target, t.direct_objects))
 896     Logs.info('%s(LIBS): %s' % (target, t.direct_libs))
 897     Logs.info('%s(SYSLIBS): %s' % (target, t.direct_syslibs))
 898
 899     seen.add(target)
 900
 901     for t2 in t.direct_objects:
 902         show_dependencies(bld, t2, seen)
 903
 904
 905 def show_object_duplicates(bld, tgt_list):
 906     '''show a list of object files that are included in more than
 907     one library or binary'''
 908
 909     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 910
 911     used_by = {}
 912
 913     Logs.info("showing duplicate objects")
 914
 915     for t in tgt_list:
 916         if not targets[t.sname] in [ 'LIBRARY', 'PYTHON' ]:
 917             continue
 918         for n in getattr(t, 'final_objects', set()):
 919             t2 = bld.name_to_obj(n, bld.env)
 920             if not n in used_by:
 921                 used_by[n] = set()
 922             used_by[n].add(t.sname)
 923
 924     for n in used_by:
 925         if len(used_by[n]) > 1:
 926             Logs.info("target '%s' is used by %s" % (n, used_by[n]))
 927
 928     Logs.info("showing indirect dependency counts (sorted by count)")
 929
 930     def indirect_count(t1, t2):
 931         return len(t2.indirect_objects) - len(t1.indirect_objects)
 932
 933     sorted_list = sorted(tgt_list, cmp=indirect_count)
 934     for t in sorted_list:
 935         if len(t.indirect_objects) > 1:
 936             Logs.info("%s depends on %u indirect objects" % (t.sname, len(t.indirect_objects)))
 937
 938
 939 ######################################################################
 940 # this provides a way to save our dependency calculations between runs
 941 savedeps_version = 3
 942 savedeps_inputs  = ['samba_deps', 'samba_includes', 'local_include', 'local_include_first', 'samba_cflags', 'source', 'grouping_library']
 943 savedeps_outputs = ['uselib', 'uselib_local', 'add_objects', 'includes', 'ccflags', 'ldflags', 'samba_deps_extended']
 944 savedeps_outenv  = ['INC_PATHS']
 945 savedeps_envvars = ['NONSHARED_BINARIES', 'GLOBAL_DEPENDENCIES']
 946 savedeps_caches  = ['GLOBAL_DEPENDENCIES', 'TARGET_TYPE', 'INIT_FUNCTIONS', 'SYSLIB_DEPS']
 947 savedeps_files   = ['buildtools/wafsamba/samba_deps.py']
 948
 949 def save_samba_deps(bld, tgt_list):
 950     '''save the dependency calculations between builds, to make
 951        further builds faster'''
 952     denv = Environment.Environment()
 953
 954     denv.version = savedeps_version
 955     denv.savedeps_inputs = savedeps_inputs
 956     denv.savedeps_outputs = savedeps_outputs
 957     denv.input = {}
 958     denv.output = {}
 959     denv.outenv = {}
 960     denv.caches = {}
 961     denv.envvar = {}
 962     denv.files  = {}
 963
 964     for f in savedeps_files:
 965         denv.files[f] = os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime
 966
 967     for c in savedeps_caches:
 968         denv.caches[c] = LOCAL_CACHE(bld, c)
 969
 970     for e in savedeps_envvars:
 971         denv.envvar[e] = bld.env[e]
 972
 973     for t in tgt_list:
 974         # save all the input attributes for each target
 975         tdeps = {}
 976         for attr in savedeps_inputs:
 977             v = getattr(t, attr, None)
 978             if v is not None:
 979                 tdeps[attr] = v
 980         if tdeps != {}:
 981             denv.input[t.sname] = tdeps
 982
 983         # save all the output attributes for each target
 984         tdeps = {}
 985         for attr in savedeps_outputs:
 986             v = getattr(t, attr, None)
 987             if v is not None:
 988                 tdeps[attr] = v
 989         if tdeps != {}:
 990             denv.output[t.sname] = tdeps
 991
 992         tdeps = {}
 993         for attr in savedeps_outenv:
 994             if attr in t.env:
 995                 tdeps[attr] = t.env[attr]
 996         if tdeps != {}:
 997             denv.outenv[t.sname] = tdeps
 998
 999     depsfile = os.path.join(bld.bdir, "sambadeps")
1000     denv.store(depsfile)
1001
1002
1003
1004 def load_samba_deps(bld, tgt_list):
1005     '''load a previous set of build dependencies if possible'''
1006     depsfile = os.path.join(bld.bdir, "sambadeps")
1007     denv = Environment.Environment()
1008     try:
1009         debug('deps: checking saved dependencies')
1010         denv.load(depsfile)
1011         if (denv.version != savedeps_version or
1012             denv.savedeps_inputs != savedeps_inputs or
1013             denv.savedeps_outputs != savedeps_outputs):
1014             return False
1015     except:
1016         return False
1017
1018     # check if critical files have changed
1019     for f in savedeps_files:
1020         if f not in denv.files:
1021             return False
1022         if denv.files[f] != os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime:
1023             return False
1024
1025     # check if caches are the same
1026     for c in savedeps_caches:
1027         if c not in denv.caches or denv.caches[c] != LOCAL_CACHE(bld, c):
1028             return False
1029
1030     # check if caches are the same
1031     for e in savedeps_envvars:
1032         if e not in denv.envvar or denv.envvar[e] != bld.env[e]:
1033             return False
1034
1035     # check inputs are the same
1036     for t in tgt_list:
1037         tdeps = {}
1038         for attr in savedeps_inputs:
1039             v = getattr(t, attr, None)
1040             if v is not None:
1041                 tdeps[attr] = v
1042         if t.sname in denv.input:
1043             olddeps = denv.input[t.sname]
1044         else:
1045             olddeps = {}
1046         if tdeps != olddeps:
1047             #print '%s: \ntdeps=%s \nodeps=%s' % (t.sname, tdeps, olddeps)
1048             return False
1049
1050     # put outputs in place
1051     for t in tgt_list:
1052         if not t.sname in denv.output: continue
1053         tdeps = denv.output[t.sname]
1054         for a in tdeps:
1055             setattr(t, a, tdeps[a])
1056
1057     # put output env vars in place
1058     for t in tgt_list:
1059         if not t.sname in denv.outenv: continue
1060         tdeps = denv.outenv[t.sname]
1061         for a in tdeps:
1062             t.env[a] = tdeps[a]
1063
1064     debug('deps: loaded saved dependencies')
1065     return True
1066
1067
1068
1069 def check_project_rules(bld):
1070     '''check the project rules - ensuring the targets are sane'''
1071
1072     loops = {}
1073     inc_loops = {}
1074
1075     tgt_list = get_tgt_list(bld)
1076
1077     add_samba_attributes(bld, tgt_list)
1078
1079     force_project_rules = (Options.options.SHOWDEPS or
1080                            Options.options.SHOW_DUPLICATES)
1081
1082     if not force_project_rules and load_samba_deps(bld, tgt_list):
1083         return
1084
1085     global tstart
1086     tstart = time.clock()
1087
1088     bld.new_rules = True
1089     Logs.info("Checking project rules ...")
1090
1091     debug('deps: project rules checking started')
1092
1093     expand_subsystem_deps(bld)
1094
1095     debug("deps: expand_subsystem_deps: %f" % (time.clock() - tstart))
1096
1097     replace_grouping_libraries(bld, tgt_list)
1098
1099     debug("deps: replace_grouping_libraries: %f" % (time.clock() - tstart))
1100
1101     build_direct_deps(bld, tgt_list)
1102
1103     debug("deps: build_direct_deps: %f" % (time.clock() - tstart))
1104
1105     break_dependency_loops(bld, tgt_list)
1106
1107     debug("deps: break_dependency_loops: %f" % (time.clock() - tstart))
1108
1109     if Options.options.SHOWDEPS:
1110             show_dependencies(bld, Options.options.SHOWDEPS, set())
1111
1112     calculate_final_deps(bld, tgt_list, loops)
1113
1114     debug("deps: calculate_final_deps: %f" % (time.clock() - tstart))
1115
1116     if Options.options.SHOW_DUPLICATES:
1117             show_object_duplicates(bld, tgt_list)
1118
1119     # run the various attribute generators
1120     for f in [ build_dependencies, build_includes, add_init_functions ]:
1121         debug('deps: project rules checking %s', f)
1122         for t in tgt_list: f(t)
1123         debug("deps: %s: %f" % (f, time.clock() - tstart))
1124
1125     debug('deps: project rules stage1 completed')
1126
1127     #check_orpaned_targets(bld, tgt_list)
1128
1129     if not check_duplicate_sources(bld, tgt_list):
1130         Logs.error("Duplicate sources present - aborting")
1131         sys.exit(1)
1132
1133     debug("deps: check_duplicate_sources: %f" % (time.clock() - tstart))
1134
1135     if not check_group_ordering(bld, tgt_list):
1136         Logs.error("Bad group ordering - aborting")
1137         sys.exit(1)
1138
1139     debug("deps: check_group_ordering: %f" % (time.clock() - tstart))
1140
1141     show_final_deps(bld, tgt_list)
1142
1143     debug("deps: show_final_deps: %f" % (time.clock() - tstart))
1144
1145     debug('deps: project rules checking completed - %u targets checked',
1146           len(tgt_list))
1147
1148     if not bld.is_install:
1149         save_samba_deps(bld, tgt_list)
1150
1151     debug("deps: save_samba_deps: %f" % (time.clock() - tstart))
1152
1153     Logs.info("Project rules pass")
1154
1155
1156 def CHECK_PROJECT_RULES(bld):
1157     '''enable checking of project targets for sanity'''
1158     if bld.env.added_project_rules:
1159         return
1160     bld.env.added_project_rules = True
1161     bld.add_pre_fun(check_project_rules)
1162 Build.BuildContext.CHECK_PROJECT_RULES = CHECK_PROJECT_RULES
1163
1164