buildtools/wafsamba/samba_deps.py

   1 # Samba automatic dependency handling and project rules
   2
   3 import Build, os, sys, re, Environment, Logs, time
   4 from samba_utils import *
   5 from samba_autoconf import *
   6 from samba_bundled import BUILTIN_LIBRARY
   7
   8 @conf
   9 def ADD_GLOBAL_DEPENDENCY(ctx, dep):
  10     '''add a dependency for all binaries and libraries'''
  11     if not 'GLOBAL_DEPENDENCIES' in ctx.env:
  12         ctx.env.GLOBAL_DEPENDENCIES = []
  13     ctx.env.GLOBAL_DEPENDENCIES.append(dep)
  14
  15
  16 @conf
  17 def BREAK_CIRCULAR_LIBRARY_DEPENDENCIES(ctx):
  18     '''indicate that circular dependencies between libraries should be broken.'''
  19     ctx.env.ALLOW_CIRCULAR_LIB_DEPENDENCIES = True
  20
  21
  22 @conf
  23 def SET_SYSLIB_DEPS(conf, target, deps):
  24     '''setup some implied dependencies for a SYSLIB'''
  25     cache = LOCAL_CACHE(conf, 'SYSLIB_DEPS')
  26     cache[target] = deps
  27
  28
  29 def expand_subsystem_deps(bld):
  30     '''expand the reverse dependencies resulting from subsystem
  31        attributes of modules. This is walking over the complete list
  32        of declared subsystems, and expands the samba_deps_extended list for any
  33        module<->subsystem dependencies'''
  34
  35     subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
  36     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
  37
  38     for subsystem_name in subsystem_list:
  39         bld.ASSERT(subsystem_name in targets, "Subsystem target %s not declared" % subsystem_name)
  40         type = targets[subsystem_name]
  41         if type == 'DISABLED' or type == 'EMPTY':
  42             continue
  43
  44         # for example,
  45         #    subsystem_name = dcerpc_server (a subsystem)
  46         #    subsystem      = dcerpc_server (a subsystem object)
  47         #    module_name    = rpc_epmapper (a module within the dcerpc_server subsystem)
  48         #    module         = rpc_epmapper (a module object within the dcerpc_server subsystem)
  49
  50         subsystem = bld.name_to_obj(subsystem_name, bld.env)
  51         bld.ASSERT(subsystem is not None, "Unable to find subsystem %s" % subsystem_name)
  52         for d in subsystem_list[subsystem_name]:
  53             module_name = d['TARGET']
  54             module_type = targets[module_name]
  55             if module_type in ['DISABLED', 'EMPTY']:
  56                 continue
  57             bld.ASSERT(subsystem is not None,
  58                        "Subsystem target %s for %s (%s) not found" % (subsystem_name, module_name, module_type))
  59             if module_type in ['SUBSYSTEM']:
  60                 # if a module is a plain object type (not a library) then the
  61                 # subsystem it is part of needs to have it as a dependency, so targets
  62                 # that depend on this subsystem get the modules of that subsystem
  63                 subsystem.samba_deps_extended.append(module_name)
  64         subsystem.samba_deps_extended = unique_list(subsystem.samba_deps_extended)
  65
  66
  67
  68 def build_dependencies(self):
  69     '''This builds the dependency list for a target. It runs after all the targets are declared
  70
  71     The reason this is not just done in the SAMBA_*() rules is that we have no way of knowing
  72     the full dependency list for a target until we have all of the targets declared.
  73     '''
  74
  75     if self.samba_type in ['LIBRARY', 'BINARY', 'PYTHON']:
  76         self.uselib        = list(self.final_syslibs)
  77         self.uselib_local  = list(self.final_libs)
  78         self.add_objects   = list(self.final_objects)
  79
  80         # extra link flags from pkg_config
  81         libs = self.final_syslibs.copy()
  82
  83         (ccflags, ldflags) = library_flags(self, list(libs))
  84         new_ldflags        = getattr(self, 'samba_ldflags', [])[:]
  85         new_ldflags.extend(ldflags)
  86         self.ldflags       = new_ldflags
  87
  88         if getattr(self, 'allow_undefined_symbols', False) and self.env.undefined_ldflags:
  89             for f in self.env.undefined_ldflags:
  90                 self.ldflags.remove(f)
  91
  92         debug('deps: computed dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
  93               self.sname, self.uselib, self.uselib_local, self.add_objects)
  94
  95     if self.samba_type in ['SUBSYSTEM']:
  96         # this is needed for the ccflags of libs that come from pkg_config
  97         self.uselib = list(self.final_syslibs)
  98         self.uselib.extend(list(self.direct_syslibs))
  99         for lib in self.final_libs:
 100             t = self.bld.name_to_obj(lib, self.bld.env)
 101             self.uselib.extend(list(t.final_syslibs))
 102         self.uselib = unique_list(self.uselib)
 103
 104     if getattr(self, 'uselib', None):
 105         up_list = []
 106         for l in self.uselib:
 107            up_list.append(l.upper())
 108         self.uselib = up_list
 109
 110
 111 def build_includes(self):
 112     '''This builds the right set of includes for a target.
 113
 114     One tricky part of this is that the includes= attribute for a
 115     target needs to use paths which are relative to that targets
 116     declaration directory (which we can get at via t.path).
 117
 118     The way this works is the includes list gets added as
 119     samba_includes in the main build task declaration. Then this
 120     function runs after all of the tasks are declared, and it
 121     processes the samba_includes attribute to produce a includes=
 122     attribute
 123     '''
 124
 125     if getattr(self, 'samba_includes', None) is None:
 126         return
 127
 128     bld = self.bld
 129
 130     inc_deps = includes_objects(bld, self, set(), {})
 131
 132     includes = []
 133
 134     # maybe add local includes
 135     if getattr(self, 'local_include', True) == True and getattr(self, 'local_include_first', True):
 136         includes.append('.')
 137
 138     includes.extend(self.samba_includes_extended)
 139
 140     if 'EXTRA_INCLUDES' in bld.env and getattr(self, 'global_include', True):
 141         includes.extend(bld.env['EXTRA_INCLUDES'])
 142
 143     includes.append('#')
 144
 145     inc_set = set()
 146     inc_abs = []
 147
 148     for d in inc_deps:
 149         t = bld.name_to_obj(d, bld.env)
 150         bld.ASSERT(t is not None, "Unable to find dependency %s for %s" % (d, self.sname))
 151         inclist = getattr(t, 'samba_includes_extended', [])[:]
 152         if getattr(t, 'local_include', True) == True:
 153             inclist.append('.')
 154         if inclist == []:
 155             continue
 156         tpath = t.samba_abspath
 157         for inc in inclist:
 158             npath = tpath + '/' + inc
 159             if not npath in inc_set:
 160                 inc_abs.append(npath)
 161                 inc_set.add(npath)
 162
 163     mypath = self.path.abspath(bld.env)
 164     for inc in inc_abs:
 165         relpath = os_path_relpath(inc, mypath)
 166         includes.append(relpath)
 167
 168     if getattr(self, 'local_include', True) == True and not getattr(self, 'local_include_first', True):
 169         includes.append('.')
 170
 171     # now transform the includes list to be relative to the top directory
 172     # which is represented by '#' in waf. This allows waf to cache the
 173     # includes lists more efficiently
 174     includes_top = []
 175     for i in includes:
 176         if i[0] == '#':
 177             # some are already top based
 178             includes_top.append(i)
 179             continue
 180         absinc = os.path.join(self.path.abspath(), i)
 181         relinc = os_path_relpath(absinc, self.bld.srcnode.abspath())
 182         includes_top.append('#' + relinc)
 183
 184     self.includes = unique_list(includes_top)
 185     debug('deps: includes for target %s: includes=%s',
 186           self.sname, self.includes)
 187
 188
 189
 190
 191 def add_init_functions(self):
 192     '''This builds the right set of init functions'''
 193
 194     bld = self.bld
 195
 196     subsystems = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 197
 198     # cope with the separated object lists from BINARY and LIBRARY targets
 199     sname = self.sname
 200     if sname.endswith('.objlist'):
 201         sname = sname[0:-8]
 202
 203     modules = []
 204     if sname in subsystems:
 205         modules.append(sname)
 206
 207     m = getattr(self, 'samba_modules', None)
 208     if m is not None:
 209         modules.extend(TO_LIST(m))
 210
 211     m = getattr(self, 'samba_subsystem', None)
 212     if m is not None:
 213         modules.append(m)
 214
 215     sentinal = getattr(self, 'init_function_sentinal', 'NULL')
 216
 217     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
 218     cflags = getattr(self, 'samba_cflags', [])[:]
 219
 220     if modules == []:
 221         sname = sname.replace('-','_')
 222         sname = sname.replace('/','_')
 223         cflags.append('-DSTATIC_%s_MODULES=%s' % (sname, sentinal))
 224         if sentinal == 'NULL':
 225             cflags.append('-DSTATIC_%s_MODULES_PROTO' % sname)
 226         self.ccflags = cflags
 227         return
 228
 229     for m in modules:
 230         bld.ASSERT(m in subsystems,
 231                    "No init_function defined for module '%s' in target '%s'" % (m, self.sname))
 232         init_fn_list = []
 233         for d in subsystems[m]:
 234             if targets[d['TARGET']] != 'DISABLED':
 235                 init_fn_list.append(d['INIT_FUNCTION'])
 236         if init_fn_list == []:
 237             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, sentinal))
 238             if sentinal == 'NULL':
 239                 cflags.append('-DSTATIC_%s_MODULES_PROTO' % m)
 240         else:
 241             cflags.append('-DSTATIC_%s_MODULES=%s' % (m, ','.join(init_fn_list) + ',' + sentinal))
 242             proto=''
 243             for f in init_fn_list:
 244                 proto = proto + '_MODULE_PROTO(%s)' % f
 245             cflags.append('-DSTATIC_%s_MODULES_PROTO=%s' % (m, proto))
 246     self.ccflags = cflags
 247
 248
 249
 250 def check_duplicate_sources(bld, tgt_list):
 251     '''see if we are compiling the same source file more than once
 252        without an allow_duplicates attribute'''
 253
 254     debug('deps: checking for duplicate sources')
 255
 256     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 257     ret = True
 258
 259     global tstart
 260
 261     for t in tgt_list:
 262         source_list = TO_LIST(getattr(t, 'source', ''))
 263         tpath = os.path.normpath(os_path_relpath(t.path.abspath(bld.env), t.env.BUILD_DIRECTORY + '/default'))
 264         obj_sources = set()
 265         for s in source_list:
 266             p = os.path.normpath(os.path.join(tpath, s))
 267             if p in obj_sources:
 268                 Logs.error("ERROR: source %s appears twice in target '%s'" % (p, t.sname))
 269                 sys.exit(1)
 270             obj_sources.add(p)
 271         t.samba_source_set = obj_sources
 272
 273     subsystems = {}
 274
 275     # build a list of targets that each source file is part of
 276     for t in tgt_list:
 277         sources = []
 278         if not targets[t.sname] in [ 'LIBRARY', 'BINARY', 'PYTHON' ]:
 279             continue
 280         for obj in t.add_objects:
 281             t2 = t.bld.name_to_obj(obj, bld.env)
 282             source_set = getattr(t2, 'samba_source_set', set())
 283             for s in source_set:
 284                 if not s in subsystems:
 285                     subsystems[s] = {}
 286                 if not t.sname in subsystems[s]:
 287                     subsystems[s][t.sname] = []
 288                 subsystems[s][t.sname].append(t2.sname)
 289
 290     for s in subsystems:
 291         if len(subsystems[s]) > 1 and Options.options.SHOW_DUPLICATES:
 292             Logs.warn("WARNING: source %s is in more than one target: %s" % (s, subsystems[s].keys()))
 293         for tname in subsystems[s]:
 294             if len(subsystems[s][tname]) > 1:
 295                 raise Utils.WafError("ERROR: source %s is in more than one subsystem of target '%s': %s" % (s, tname, subsystems[s][tname]))
 296
 297     return ret
 298
 299
 300 def check_orpaned_targets(bld, tgt_list):
 301     '''check if any build targets are orphaned'''
 302
 303     target_dict = LOCAL_CACHE(bld, 'TARGET_TYPE')
 304
 305     debug('deps: checking for orphaned targets')
 306
 307     for t in tgt_list:
 308         if getattr(t, 'samba_used', False) == True:
 309             continue
 310         type = target_dict[t.sname]
 311         if not type in ['BINARY', 'LIBRARY', 'MODULE', 'ET', 'PYTHON']:
 312             if re.search('^PIDL_', t.sname) is None:
 313                 Logs.warn("Target %s of type %s is unused by any other target" % (t.sname, type))
 314
 315
 316 def check_group_ordering(bld, tgt_list):
 317     '''see if we have any dependencies that violate the group ordering
 318
 319     It is an error for a target to depend on a target from a later
 320     build group
 321     '''
 322
 323     def group_name(g):
 324         tm = bld.task_manager
 325         return [x for x in tm.groups_names if id(tm.groups_names[x]) == id(g)][0]
 326
 327     for g in bld.task_manager.groups:
 328         gname = group_name(g)
 329         for t in g.tasks_gen:
 330             t.samba_group = gname
 331
 332     grp_map = {}
 333     idx = 0
 334     for g in bld.task_manager.groups:
 335         name = group_name(g)
 336         grp_map[name] = idx
 337         idx += 1
 338
 339     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 340
 341     ret = True
 342     for t in tgt_list:
 343         tdeps = getattr(t, 'add_objects', []) + getattr(t, 'uselib_local', [])
 344         for d in tdeps:
 345             t2 = bld.name_to_obj(d, bld.env)
 346             if t2 is None:
 347                 continue
 348             map1 = grp_map[t.samba_group]
 349             map2 = grp_map[t2.samba_group]
 350
 351             if map2 > map1:
 352                 Logs.error("Target %r in build group %r depends on target %r from later build group %r" % (
 353                            t.sname, t.samba_group, t2.sname, t2.samba_group))
 354                 ret = False
 355
 356     return ret
 357
 358
 359 def show_final_deps(bld, tgt_list):
 360     '''show the final dependencies for all targets'''
 361
 362     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 363
 364     for t in tgt_list:
 365         if not targets[t.sname] in ['LIBRARY', 'BINARY', 'PYTHON', 'SUBSYSTEM']:
 366             continue
 367         debug('deps: final dependencies for target %s: uselib=%s uselib_local=%s add_objects=%s',
 368               t.sname, t.uselib, getattr(t, 'uselib_local', []), getattr(t, 'add_objects', []))
 369
 370
 371 def add_samba_attributes(bld, tgt_list):
 372     '''ensure a target has a the required samba attributes'''
 373
 374     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 375
 376     for t in tgt_list:
 377         if t.name != '':
 378             t.sname = t.name
 379         else:
 380             t.sname = t.target
 381         t.samba_type = targets[t.sname]
 382         t.samba_abspath = t.path.abspath(bld.env)
 383         t.samba_deps_extended = t.samba_deps[:]
 384         t.samba_includes_extended = TO_LIST(t.samba_includes)[:]
 385         t.ccflags = getattr(t, 'samba_cflags', '')
 386
 387 def replace_grouping_libraries(bld, tgt_list):
 388     '''replace dependencies based on grouping libraries
 389
 390     If a library is marked as a grouping library, then any target that
 391     depends on a subsystem that is part of that grouping library gets
 392     that dependency replaced with a dependency on the grouping library
 393     '''
 394
 395     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 396
 397     grouping = {}
 398
 399     # find our list of grouping libraries, mapped from the subsystems they depend on
 400     for t in tgt_list:
 401         if not getattr(t, 'grouping_library', False):
 402             continue
 403         for dep in t.samba_deps_extended:
 404             bld.ASSERT(dep in targets, "grouping library target %s not declared in %s" % (dep, t.sname))
 405             if targets[dep] == 'SUBSYSTEM':
 406                 grouping[dep] = t.sname
 407
 408     # now replace any dependencies on elements of grouping libraries
 409     for t in tgt_list:
 410         for i in range(len(t.samba_deps_extended)):
 411             dep = t.samba_deps_extended[i]
 412             if dep in grouping:
 413                 if t.sname != grouping[dep]:
 414                     debug("deps: target %s: replacing dependency %s with grouping library %s" % (t.sname, dep, grouping[dep]))
 415                     t.samba_deps_extended[i] = grouping[dep]
 416
 417
 418
 419 def build_direct_deps(bld, tgt_list):
 420     '''build the direct_objects and direct_libs sets for each target'''
 421
 422     targets  = LOCAL_CACHE(bld, 'TARGET_TYPE')
 423     syslib_deps  = LOCAL_CACHE(bld, 'SYSLIB_DEPS')
 424
 425     global_deps = bld.env.GLOBAL_DEPENDENCIES
 426     global_deps_exclude = set()
 427     for dep in global_deps:
 428         t = bld.name_to_obj(dep, bld.env)
 429         for d in t.samba_deps:
 430             # prevent loops from the global dependencies list
 431             global_deps_exclude.add(d)
 432             global_deps_exclude.add(d + '.objlist')
 433
 434     for t in tgt_list:
 435         t.direct_objects = set()
 436         t.direct_libs = set()
 437         t.direct_syslibs = set()
 438         deps = t.samba_deps_extended[:]
 439         if getattr(t, 'samba_use_global_deps', False) and not t.sname in global_deps_exclude:
 440             deps.extend(global_deps)
 441         for d in deps:
 442             if d == t.sname: continue
 443             if not d in targets:
 444                 Logs.error("Unknown dependency '%s' in '%s'" % (d, t.sname))
 445                 sys.exit(1)
 446             if targets[d] in [ 'EMPTY', 'DISABLED' ]:
 447                 continue
 448             if targets[d] == 'PYTHON' and targets[t.sname] != 'PYTHON' and t.sname.find('.objlist') == -1:
 449                 # this check should be more restrictive, but for now we have pidl-generated python
 450                 # code that directly depends on other python modules
 451                 Logs.error('ERROR: Target %s has dependency on python module %s' % (t.sname, d))
 452                 sys.exit(1)
 453             if targets[d] == 'SYSLIB':
 454                 t.direct_syslibs.add(d)
 455                 if d in syslib_deps:
 456                     for implied in TO_LIST(syslib_deps[d]):
 457                         if BUILTIN_LIBRARY(bld, implied):
 458                             t.direct_objects.add(implied)
 459                         elif targets[implied] == 'SYSLIB':
 460                             t.direct_syslibs.add(implied)
 461                         elif targets[implied] in ['LIBRARY', 'MODULE']:
 462                             t.direct_libs.add(implied)
 463                         else:
 464                             Logs.error('Implied dependency %s in %s is of type %s' % (
 465                                 implied, t.sname, targets[implied]))
 466                             sys.exit(1)
 467                 continue
 468             t2 = bld.name_to_obj(d, bld.env)
 469             if t2 is None:
 470                 Logs.error("no task %s of type %s in %s" % (d, targets[d], t.sname))
 471                 sys.exit(1)
 472             if t2.samba_type in [ 'LIBRARY', 'MODULE' ]:
 473                 t.direct_libs.add(d)
 474             elif t2.samba_type in [ 'SUBSYSTEM', 'ASN1', 'PYTHON' ]:
 475                 t.direct_objects.add(d)
 476     debug('deps: built direct dependencies')
 477
 478
 479 def dependency_loop(loops, t, target):
 480     '''add a dependency loop to the loops dictionary'''
 481     if t.sname == target:
 482         return
 483     if not target in loops:
 484         loops[target] = set()
 485     if not t.sname in loops[target]:
 486         loops[target].add(t.sname)
 487
 488
 489 def indirect_libs(bld, t, chain, loops):
 490     '''recursively calculate the indirect library dependencies for a target
 491
 492     An indirect library is a library that results from a dependency on
 493     a subsystem
 494     '''
 495
 496     ret = getattr(t, 'indirect_libs', None)
 497     if ret is not None:
 498         return ret
 499
 500     ret = set()
 501     for obj in t.direct_objects:
 502         if obj in chain:
 503             dependency_loop(loops, t, obj)
 504             continue
 505         chain.add(obj)
 506         t2 = bld.name_to_obj(obj, bld.env)
 507         r2 = indirect_libs(bld, t2, chain, loops)
 508         chain.remove(obj)
 509         ret = ret.union(t2.direct_libs)
 510         ret = ret.union(r2)
 511
 512     for obj in indirect_objects(bld, t, set(), loops):
 513         if obj in chain:
 514             dependency_loop(loops, t, obj)
 515             continue
 516         chain.add(obj)
 517         t2 = bld.name_to_obj(obj, bld.env)
 518         r2 = indirect_libs(bld, t2, chain, loops)
 519         chain.remove(obj)
 520         ret = ret.union(t2.direct_libs)
 521         ret = ret.union(r2)
 522
 523     t.indirect_libs = ret
 524
 525     return ret
 526
 527
 528 def indirect_objects(bld, t, chain, loops):
 529     '''recursively calculate the indirect object dependencies for a target
 530
 531     indirect objects are the set of objects from expanding the
 532     subsystem dependencies
 533     '''
 534
 535     ret = getattr(t, 'indirect_objects', None)
 536     if ret is not None: return ret
 537
 538     ret = set()
 539     for lib in t.direct_objects:
 540         if lib in chain:
 541             dependency_loop(loops, t, lib)
 542             continue
 543         chain.add(lib)
 544         t2 = bld.name_to_obj(lib, bld.env)
 545         r2 = indirect_objects(bld, t2, chain, loops)
 546         chain.remove(lib)
 547         ret = ret.union(t2.direct_objects)
 548         ret = ret.union(r2)
 549
 550     t.indirect_objects = ret
 551     return ret
 552
 553
 554 def extended_objects(bld, t, chain):
 555     '''recursively calculate the extended object dependencies for a target
 556
 557     extended objects are the union of:
 558        - direct objects
 559        - indirect objects
 560        - direct and indirect objects of all direct and indirect libraries
 561     '''
 562
 563     ret = getattr(t, 'extended_objects', None)
 564     if ret is not None: return ret
 565
 566     ret = set()
 567     ret = ret.union(t.final_objects)
 568
 569     for lib in t.final_libs:
 570         if lib in chain:
 571             continue
 572         t2 = bld.name_to_obj(lib, bld.env)
 573         chain.add(lib)
 574         r2 = extended_objects(bld, t2, chain)
 575         chain.remove(lib)
 576         ret = ret.union(t2.final_objects)
 577         ret = ret.union(r2)
 578
 579     t.extended_objects = ret
 580     return ret
 581
 582
 583 def includes_objects(bld, t, chain, inc_loops):
 584     '''recursively calculate the includes object dependencies for a target
 585
 586     includes dependencies come from either library or object dependencies
 587     '''
 588     ret = getattr(t, 'includes_objects', None)
 589     if ret is not None:
 590         return ret
 591
 592     ret = t.direct_objects.copy()
 593     ret = ret.union(t.direct_libs)
 594
 595     for obj in t.direct_objects:
 596         if obj in chain:
 597             dependency_loop(inc_loops, t, obj)
 598             continue
 599         chain.add(obj)
 600         t2 = bld.name_to_obj(obj, bld.env)
 601         r2 = includes_objects(bld, t2, chain, inc_loops)
 602         chain.remove(obj)
 603         ret = ret.union(t2.direct_objects)
 604         ret = ret.union(r2)
 605
 606     for lib in t.direct_libs:
 607         if lib in chain:
 608             dependency_loop(inc_loops, t, lib)
 609             continue
 610         chain.add(lib)
 611         t2 = bld.name_to_obj(lib, bld.env)
 612         if t2 is None:
 613             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 614             Logs.error('Target %s of type %s not found in direct_libs for %s' % (
 615                 lib, targets[lib], t.sname))
 616             sys.exit(1)
 617         r2 = includes_objects(bld, t2, chain, inc_loops)
 618         chain.remove(lib)
 619         ret = ret.union(t2.direct_objects)
 620         ret = ret.union(r2)
 621
 622     t.includes_objects = ret
 623     return ret
 624
 625
 626 def break_dependency_loops(bld, tgt_list):
 627     '''find and break dependency loops'''
 628     loops = {}
 629     inc_loops = {}
 630
 631     # build up the list of loops
 632     for t in tgt_list:
 633         indirect_objects(bld, t, set(), loops)
 634         indirect_libs(bld, t, set(), loops)
 635         includes_objects(bld, t, set(), inc_loops)
 636
 637     # break the loops
 638     for t in tgt_list:
 639         if t.sname in loops:
 640             for attr in ['direct_objects', 'indirect_objects', 'direct_libs', 'indirect_libs']:
 641                 objs = getattr(t, attr, set())
 642                 setattr(t, attr, objs.difference(loops[t.sname]))
 643
 644     for loop in loops:
 645         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 646
 647     for loop in inc_loops:
 648         debug('deps: Found include loops for target %s : %s', loop, inc_loops[loop])
 649
 650     # expand the loops mapping by one level
 651     for loop in loops.copy():
 652         for tgt in loops[loop]:
 653             if tgt in loops:
 654                 loops[loop] = loops[loop].union(loops[tgt])
 655
 656     for loop in inc_loops.copy():
 657         for tgt in inc_loops[loop]:
 658             if tgt in inc_loops:
 659                 inc_loops[loop] = inc_loops[loop].union(inc_loops[tgt])
 660
 661
 662     # expand indirect subsystem and library loops
 663     for loop in loops.copy():
 664         t = bld.name_to_obj(loop, bld.env)
 665         if t.samba_type in ['SUBSYSTEM']:
 666             loops[loop] = loops[loop].union(t.indirect_objects)
 667             loops[loop] = loops[loop].union(t.direct_objects)
 668         if t.samba_type in ['LIBRARY','PYTHON']:
 669             loops[loop] = loops[loop].union(t.indirect_libs)
 670             loops[loop] = loops[loop].union(t.direct_libs)
 671         if loop in loops[loop]:
 672             loops[loop].remove(loop)
 673
 674     # expand indirect includes loops
 675     for loop in inc_loops.copy():
 676         t = bld.name_to_obj(loop, bld.env)
 677         inc_loops[loop] = inc_loops[loop].union(t.includes_objects)
 678         if loop in inc_loops[loop]:
 679             inc_loops[loop].remove(loop)
 680
 681     # add in the replacement dependencies
 682     for t in tgt_list:
 683         for loop in loops:
 684             for attr in ['indirect_objects', 'indirect_libs']:
 685                 objs = getattr(t, attr, set())
 686                 if loop in objs:
 687                     diff = loops[loop].difference(objs)
 688                     if t.sname in diff:
 689                         diff.remove(t.sname)
 690                     if diff:
 691                         debug('deps: Expanded target %s of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 692                         objs = objs.union(diff)
 693                 setattr(t, attr, objs)
 694
 695         for loop in inc_loops:
 696             objs = getattr(t, 'includes_objects', set())
 697             if loop in objs:
 698                 diff = inc_loops[loop].difference(objs)
 699                 if t.sname in diff:
 700                     diff.remove(t.sname)
 701                 if diff:
 702                     debug('deps: Expanded target %s includes of type %s from loop %s by %s', t.sname, t.samba_type, loop, diff)
 703                     objs = objs.union(diff)
 704             setattr(t, 'includes_objects', objs)
 705
 706
 707 def reduce_objects(bld, tgt_list):
 708     '''reduce objects by looking for indirect object dependencies'''
 709     rely_on = {}
 710
 711     for t in tgt_list:
 712         t.extended_objects = None
 713
 714     changed = False
 715
 716     for type in ['BINARY', 'PYTHON', 'LIBRARY']:
 717         for t in tgt_list:
 718             if t.samba_type != type: continue
 719             # if we will indirectly link to a target then we don't need it
 720             new = t.final_objects.copy()
 721             for l in t.final_libs:
 722                 t2 = bld.name_to_obj(l, bld.env)
 723                 t2_obj = extended_objects(bld, t2, set())
 724                 dup = new.intersection(t2_obj)
 725                 if t.sname in rely_on:
 726                     dup = dup.difference(rely_on[t.sname])
 727                 if dup:
 728                     debug('deps: removing dups from %s of type %s: %s also in %s %s',
 729                           t.sname, t.samba_type, dup, t2.samba_type, l)
 730                     new = new.difference(dup)
 731                     changed = True
 732                     if not l in rely_on:
 733                         rely_on[l] = set()
 734                     rely_on[l] = rely_on[l].union(dup)
 735             t.final_objects = new
 736
 737     if not changed:
 738         return False
 739
 740     # add back in any objects that were relied upon by the reduction rules
 741     for r in rely_on:
 742         t = bld.name_to_obj(r, bld.env)
 743         t.final_objects = t.final_objects.union(rely_on[r])
 744
 745     return True
 746
 747
 748 def show_library_loop(bld, lib1, lib2, path, seen):
 749     '''show the detailed path of a library loop between lib1 and lib2'''
 750
 751     t = bld.name_to_obj(lib1, bld.env)
 752     if not lib2 in getattr(t, 'final_libs', set()):
 753         return
 754
 755     for d in t.samba_deps_extended:
 756         if d in seen:
 757             continue
 758         seen.add(d)
 759         path2 = path + '=>' + d
 760         if d == lib2:
 761             Logs.warn('library loop path: ' + path2)
 762             return
 763         show_library_loop(bld, d, lib2, path2, seen)
 764         seen.remove(d)
 765
 766
 767 def calculate_final_deps(bld, tgt_list, loops):
 768     '''calculate the final library and object dependencies'''
 769     for t in tgt_list:
 770         # start with the maximum possible list
 771         t.final_libs    = t.direct_libs.union(indirect_libs(bld, t, set(), loops))
 772         t.final_objects = t.direct_objects.union(indirect_objects(bld, t, set(), loops))
 773
 774     for t in tgt_list:
 775         # don't depend on ourselves
 776         if t.sname in t.final_libs:
 777             t.final_libs.remove(t.sname)
 778         if t.sname in t.final_objects:
 779             t.final_objects.remove(t.sname)
 780
 781     # handle any non-shared binaries
 782     for t in tgt_list:
 783         if t.samba_type == 'BINARY' and bld.NONSHARED_BINARY(t.sname):
 784             subsystem_list = LOCAL_CACHE(bld, 'INIT_FUNCTIONS')
 785             targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 786
 787             # replace lib deps with objlist deps
 788             for l in t.final_libs:
 789                 objname = l + '.objlist'
 790                 t2 = bld.name_to_obj(objname, bld.env)
 791                 if t2 is None:
 792                     Logs.error('ERROR: subsystem %s not found' % objname)
 793                     sys.exit(1)
 794                 t.final_objects.add(objname)
 795                 t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 796                 if l in subsystem_list:
 797                     # its a subsystem - we also need the contents of any modules
 798                     for d in subsystem_list[l]:
 799                         module_name = d['TARGET']
 800                         if targets[module_name] == 'LIBRARY':
 801                             objname = module_name + '.objlist'
 802                         elif targets[module_name] == 'SUBSYSTEM':
 803                             objname = module_name
 804                         else:
 805                             continue
 806                         t2 = bld.name_to_obj(objname, bld.env)
 807                         if t2 is None:
 808                             Logs.error('ERROR: subsystem %s not found' % objname)
 809                             sys.exit(1)
 810                         t.final_objects.add(objname)
 811                         t.final_objects = t.final_objects.union(extended_objects(bld, t2, set()))
 812             t.final_libs = set()
 813
 814     # find any library loops
 815     for t in tgt_list:
 816         if t.samba_type in ['LIBRARY', 'PYTHON']:
 817             for l in t.final_libs.copy():
 818                 t2 = bld.name_to_obj(l, bld.env)
 819                 if t.sname in t2.final_libs:
 820                     if getattr(bld.env, "ALLOW_CIRCULAR_LIB_DEPENDENCIES", False):
 821                         # we could break this in either direction. If one of the libraries
 822                         # has a version number, and will this be distributed publicly, then
 823                         # we should make it the lower level library in the DAG
 824                         Logs.warn('deps: removing library loop %s from %s' % (t.sname, t2.sname))
 825                         dependency_loop(loops, t, t2.sname)
 826                         t2.final_libs.remove(t.sname)
 827                     else:
 828                         Logs.error('ERROR: circular library dependency between %s and %s'
 829                             % (t.sname, t2.sname))
 830                         show_library_loop(bld, t.sname, t2.sname, t.sname, set())
 831                         show_library_loop(bld, t2.sname, t.sname, t2.sname, set())
 832                         sys.exit(1)
 833
 834     for loop in loops:
 835         debug('deps: Found dependency loops for target %s : %s', loop, loops[loop])
 836
 837     # we now need to make corrections for any library loops we broke up
 838     # any target that depended on the target of the loop and doesn't
 839     # depend on the source of the loop needs to get the loop source added
 840     for type in ['BINARY','PYTHON','LIBRARY','BINARY']:
 841         for t in tgt_list:
 842             if t.samba_type != type: continue
 843             for loop in loops:
 844                 if loop in t.final_libs:
 845                     diff = loops[loop].difference(t.final_libs)
 846                     if t.sname in diff:
 847                         diff.remove(t.sname)
 848                     if t.sname in diff:
 849                         diff.remove(t.sname)
 850                     # make sure we don't recreate the loop again!
 851                     for d in diff.copy():
 852                         t2 = bld.name_to_obj(d, bld.env)
 853                         if t2.samba_type == 'LIBRARY':
 854                             if t.sname in t2.final_libs:
 855                                 debug('deps: removing expansion %s from %s', d, t.sname)
 856                                 diff.remove(d)
 857                     if diff:
 858                         debug('deps: Expanded target %s by loop %s libraries (loop %s) %s', t.sname, loop,
 859                               loops[loop], diff)
 860                         t.final_libs = t.final_libs.union(diff)
 861
 862     # remove objects that are also available in linked libs
 863     count = 0
 864     while reduce_objects(bld, tgt_list):
 865         count += 1
 866         if count > 100:
 867             Logs.warn("WARNING: Unable to remove all inter-target object duplicates")
 868             break
 869     debug('deps: Object reduction took %u iterations', count)
 870
 871     # add in any syslib dependencies
 872     for t in tgt_list:
 873         if not t.samba_type in ['BINARY','PYTHON','LIBRARY','SUBSYSTEM']:
 874             continue
 875         syslibs = set()
 876         for d in t.final_objects:
 877             t2 = bld.name_to_obj(d, bld.env)
 878             syslibs = syslibs.union(t2.direct_syslibs)
 879         # this adds the indirect syslibs as well, which may not be needed
 880         # depending on the linker flags
 881         for d in t.final_libs:
 882             t2 = bld.name_to_obj(d, bld.env)
 883             syslibs = syslibs.union(t2.direct_syslibs)
 884         t.final_syslibs = syslibs
 885
 886
 887     # find any unresolved library loops
 888     lib_loop_error = False
 889     for t in tgt_list:
 890         if t.samba_type in ['LIBRARY', 'PYTHON']:
 891             for l in t.final_libs.copy():
 892                 t2 = bld.name_to_obj(l, bld.env)
 893                 if t.sname in t2.final_libs:
 894                     Logs.error('ERROR: Unresolved library loop %s from %s' % (t.sname, t2.sname))
 895                     lib_loop_error = True
 896     if lib_loop_error:
 897         sys.exit(1)
 898
 899     debug('deps: removed duplicate dependencies')
 900
 901
 902 def show_dependencies(bld, target, seen):
 903     '''recursively show the dependencies of target'''
 904
 905     if target in seen:
 906         return
 907
 908     t = bld.name_to_obj(target, bld.env)
 909     if t is None:
 910         Logs.error("ERROR: Unable to find target '%s'" % target)
 911         sys.exit(1)
 912
 913     Logs.info('%s(OBJECTS): %s' % (target, t.direct_objects))
 914     Logs.info('%s(LIBS): %s' % (target, t.direct_libs))
 915     Logs.info('%s(SYSLIBS): %s' % (target, t.direct_syslibs))
 916
 917     seen.add(target)
 918
 919     for t2 in t.direct_objects:
 920         show_dependencies(bld, t2, seen)
 921
 922
 923 def show_object_duplicates(bld, tgt_list):
 924     '''show a list of object files that are included in more than
 925     one library or binary'''
 926
 927     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 928
 929     used_by = {}
 930
 931     Logs.info("showing duplicate objects")
 932
 933     for t in tgt_list:
 934         if not targets[t.sname] in [ 'LIBRARY', 'PYTHON' ]:
 935             continue
 936         for n in getattr(t, 'final_objects', set()):
 937             t2 = bld.name_to_obj(n, bld.env)
 938             if not n in used_by:
 939                 used_by[n] = set()
 940             used_by[n].add(t.sname)
 941
 942     for n in used_by:
 943         if len(used_by[n]) > 1:
 944             Logs.info("target '%s' is used by %s" % (n, used_by[n]))
 945
 946     Logs.info("showing indirect dependency counts (sorted by count)")
 947
 948     def indirect_count(t1, t2):
 949         return len(t2.indirect_objects) - len(t1.indirect_objects)
 950
 951     sorted_list = sorted(tgt_list, cmp=indirect_count)
 952     for t in sorted_list:
 953         if len(t.indirect_objects) > 1:
 954             Logs.info("%s depends on %u indirect objects" % (t.sname, len(t.indirect_objects)))
 955
 956
 957 ######################################################################
 958 # this provides a way to save our dependency calculations between runs
 959 savedeps_version = 3
 960 savedeps_inputs  = ['samba_deps', 'samba_includes', 'local_include', 'local_include_first', 'samba_cflags',
 961                     'source', 'grouping_library', 'samba_ldflags', 'allow_undefined_symbols',
 962                     'use_global_deps', 'global_include' ]
 963 savedeps_outputs = ['uselib', 'uselib_local', 'add_objects', 'includes', 'ccflags', 'ldflags', 'samba_deps_extended']
 964 savedeps_outenv  = ['INC_PATHS']
 965 savedeps_envvars = ['NONSHARED_BINARIES', 'GLOBAL_DEPENDENCIES', 'EXTRA_CFLAGS', 'EXTRA_LDFLAGS', 'EXTRA_INCLUDES' ]
 966 savedeps_caches  = ['GLOBAL_DEPENDENCIES', 'TARGET_TYPE', 'INIT_FUNCTIONS', 'SYSLIB_DEPS']
 967 savedeps_files   = ['buildtools/wafsamba/samba_deps.py']
 968
 969 def save_samba_deps(bld, tgt_list):
 970     '''save the dependency calculations between builds, to make
 971        further builds faster'''
 972     denv = Environment.Environment()
 973
 974     denv.version = savedeps_version
 975     denv.savedeps_inputs = savedeps_inputs
 976     denv.savedeps_outputs = savedeps_outputs
 977     denv.input = {}
 978     denv.output = {}
 979     denv.outenv = {}
 980     denv.caches = {}
 981     denv.envvar = {}
 982     denv.files  = {}
 983
 984     for f in savedeps_files:
 985         denv.files[f] = os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime
 986
 987     for c in savedeps_caches:
 988         denv.caches[c] = LOCAL_CACHE(bld, c)
 989
 990     for e in savedeps_envvars:
 991         denv.envvar[e] = bld.env[e]
 992
 993     for t in tgt_list:
 994         # save all the input attributes for each target
 995         tdeps = {}
 996         for attr in savedeps_inputs:
 997             v = getattr(t, attr, None)
 998             if v is not None:
 999                 tdeps[attr] = v
1000         if tdeps != {}:
1001             denv.input[t.sname] = tdeps
1002
1003         # save all the output attributes for each target
1004         tdeps = {}
1005         for attr in savedeps_outputs:
1006             v = getattr(t, attr, None)
1007             if v is not None:
1008                 tdeps[attr] = v
1009         if tdeps != {}:
1010             denv.output[t.sname] = tdeps
1011
1012         tdeps = {}
1013         for attr in savedeps_outenv:
1014             if attr in t.env:
1015                 tdeps[attr] = t.env[attr]
1016         if tdeps != {}:
1017             denv.outenv[t.sname] = tdeps
1018
1019     depsfile = os.path.join(bld.bdir, "sambadeps")
1020     denv.store(depsfile)
1021
1022
1023
1024 def load_samba_deps(bld, tgt_list):
1025     '''load a previous set of build dependencies if possible'''
1026     depsfile = os.path.join(bld.bdir, "sambadeps")
1027     denv = Environment.Environment()
1028     try:
1029         debug('deps: checking saved dependencies')
1030         denv.load(depsfile)
1031         if (denv.version != savedeps_version or
1032             denv.savedeps_inputs != savedeps_inputs or
1033             denv.savedeps_outputs != savedeps_outputs):
1034             return False
1035     except:
1036         return False
1037
1038     # check if critical files have changed
1039     for f in savedeps_files:
1040         if f not in denv.files:
1041             return False
1042         if denv.files[f] != os.stat(os.path.join(bld.srcnode.abspath(), f)).st_mtime:
1043             return False
1044
1045     # check if caches are the same
1046     for c in savedeps_caches:
1047         if c not in denv.caches or denv.caches[c] != LOCAL_CACHE(bld, c):
1048             return False
1049
1050     # check if caches are the same
1051     for e in savedeps_envvars:
1052         if e not in denv.envvar or denv.envvar[e] != bld.env[e]:
1053             return False
1054
1055     # check inputs are the same
1056     for t in tgt_list:
1057         tdeps = {}
1058         for attr in savedeps_inputs:
1059             v = getattr(t, attr, None)
1060             if v is not None:
1061                 tdeps[attr] = v
1062         if t.sname in denv.input:
1063             olddeps = denv.input[t.sname]
1064         else:
1065             olddeps = {}
1066         if tdeps != olddeps:
1067             #print '%s: \ntdeps=%s \nodeps=%s' % (t.sname, tdeps, olddeps)
1068             return False
1069
1070     # put outputs in place
1071     for t in tgt_list:
1072         if not t.sname in denv.output: continue
1073         tdeps = denv.output[t.sname]
1074         for a in tdeps:
1075             setattr(t, a, tdeps[a])
1076
1077     # put output env vars in place
1078     for t in tgt_list:
1079         if not t.sname in denv.outenv: continue
1080         tdeps = denv.outenv[t.sname]
1081         for a in tdeps:
1082             t.env[a] = tdeps[a]
1083
1084     debug('deps: loaded saved dependencies')
1085     return True
1086
1087
1088
1089 def check_project_rules(bld):
1090     '''check the project rules - ensuring the targets are sane'''
1091
1092     loops = {}
1093     inc_loops = {}
1094
1095     tgt_list = get_tgt_list(bld)
1096
1097     add_samba_attributes(bld, tgt_list)
1098
1099     force_project_rules = (Options.options.SHOWDEPS or
1100                            Options.options.SHOW_DUPLICATES)
1101
1102     if not force_project_rules and load_samba_deps(bld, tgt_list):
1103         return
1104
1105     global tstart
1106     tstart = time.clock()
1107
1108     bld.new_rules = True
1109     Logs.info("Checking project rules ...")
1110
1111     debug('deps: project rules checking started')
1112
1113     expand_subsystem_deps(bld)
1114
1115     debug("deps: expand_subsystem_deps: %f" % (time.clock() - tstart))
1116
1117     replace_grouping_libraries(bld, tgt_list)
1118
1119     debug("deps: replace_grouping_libraries: %f" % (time.clock() - tstart))
1120
1121     build_direct_deps(bld, tgt_list)
1122
1123     debug("deps: build_direct_deps: %f" % (time.clock() - tstart))
1124
1125     break_dependency_loops(bld, tgt_list)
1126
1127     debug("deps: break_dependency_loops: %f" % (time.clock() - tstart))
1128
1129     if Options.options.SHOWDEPS:
1130             show_dependencies(bld, Options.options.SHOWDEPS, set())
1131
1132     calculate_final_deps(bld, tgt_list, loops)
1133
1134     debug("deps: calculate_final_deps: %f" % (time.clock() - tstart))
1135
1136     if Options.options.SHOW_DUPLICATES:
1137             show_object_duplicates(bld, tgt_list)
1138
1139     # run the various attribute generators
1140     for f in [ build_dependencies, build_includes, add_init_functions ]:
1141         debug('deps: project rules checking %s', f)
1142         for t in tgt_list: f(t)
1143         debug("deps: %s: %f" % (f, time.clock() - tstart))
1144
1145     debug('deps: project rules stage1 completed')
1146
1147     #check_orpaned_targets(bld, tgt_list)
1148
1149     if not check_duplicate_sources(bld, tgt_list):
1150         Logs.error("Duplicate sources present - aborting")
1151         sys.exit(1)
1152
1153     debug("deps: check_duplicate_sources: %f" % (time.clock() - tstart))
1154
1155     if not check_group_ordering(bld, tgt_list):
1156         Logs.error("Bad group ordering - aborting")
1157         sys.exit(1)
1158
1159     debug("deps: check_group_ordering: %f" % (time.clock() - tstart))
1160
1161     show_final_deps(bld, tgt_list)
1162
1163     debug("deps: show_final_deps: %f" % (time.clock() - tstart))
1164
1165     debug('deps: project rules checking completed - %u targets checked',
1166           len(tgt_list))
1167
1168     if not bld.is_install:
1169         save_samba_deps(bld, tgt_list)
1170
1171     debug("deps: save_samba_deps: %f" % (time.clock() - tstart))
1172
1173     Logs.info("Project rules pass")
1174
1175
1176 def CHECK_PROJECT_RULES(bld):
1177     '''enable checking of project targets for sanity'''
1178     if bld.env.added_project_rules:
1179         return
1180     bld.env.added_project_rules = True
1181     bld.add_pre_fun(check_project_rules)
1182 Build.BuildContext.CHECK_PROJECT_RULES = CHECK_PROJECT_RULES
1183
1184