buildtools/wafsamba/symbols.py

   1 # a waf tool to extract symbols from object files or libraries
   2 # using nm, producing a set of exposed defined/undefined symbols
   3
   4 import Utils, Build, subprocess, Logs
   5 from samba_wildcard import fake_build_environment
   6 from samba_utils import *
   7
   8 # these are the data structures used in symbols.py:
   9 #
  10 # bld.env.symbol_map : dictionary mapping public symbol names to list of
  11 #                      subsystem names where that symbol exists
  12 #
  13 # t.in_library       : list of libraries that t is in
  14 #
  15 # bld.env.public_symbols: set of public symbols for each subsystem
  16 # bld.env.used_symbols  : set of used symbols for each subsystem
  17 #
  18 # bld.env.syslib_symbols: dictionary mapping system library name to set of symbols
  19 #                         for that library
  20 #
  21 # LOCAL_CACHE(bld, 'TARGET_TYPE') : dictionary mapping subsystem name to target type
  22
  23 def symbols_extract(objfiles, dynamic=False):
  24     '''extract symbols from objfile, returning a dictionary containing
  25        the set of undefined and public symbols for each file'''
  26
  27     ret = {}
  28
  29     cmd = ["nm"]
  30     if dynamic:
  31         # needed for some .so files
  32         cmd.append("-D")
  33     cmd.extend(objfiles)
  34
  35     nmpipe = subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout
  36     if len(objfiles) == 1:
  37         filename = objfiles[0]
  38         ret[filename] = { "PUBLIC": set(), "UNDEFINED" : set()}
  39
  40     for line in nmpipe:
  41         line = line.strip()
  42         if line.endswith(':'):
  43             filename = line[:-1]
  44             ret[filename] = { "PUBLIC": set(), "UNDEFINED" : set() }
  45             continue
  46         cols = line.split(" ")
  47         if cols == ['']:
  48             continue
  49         # see if the line starts with an address
  50         if len(cols) == 3:
  51             symbol_type = cols[1]
  52             symbol = cols[2]
  53         else:
  54             symbol_type = cols[0]
  55             symbol = cols[1]
  56         if symbol_type in "BDGTRVWSi":
  57             # its a public symbol
  58             ret[filename]["PUBLIC"].add(symbol)
  59         elif symbol_type in "U":
  60             ret[filename]["UNDEFINED"].add(symbol)
  61
  62     return ret
  63
  64
  65 def real_name(name):
  66     if name.find(".objlist") != -1:
  67         name = name[:-8]
  68     return name
  69
  70
  71 def find_syslib_path(bld, libname, deps):
  72     '''find the path to the syslib we will link against'''
  73     # the strategy is to use the targets that depend on the library, and run ldd
  74     # on it to find the real location of the library that is used
  75
  76     linkpath = deps[0].link_task.outputs[0].abspath(bld.env)
  77
  78     if libname == "python":
  79         libname += bld.env.PYTHON_VERSION
  80
  81     ret = None
  82
  83     lddpipe = subprocess.Popen(['ldd', linkpath], stdout=subprocess.PIPE).stdout
  84     for line in lddpipe:
  85         line = line.strip()
  86         cols = line.split(" ")
  87         if len(cols) < 3 or cols[1] != "=>":
  88             continue
  89         if cols[0].startswith("lib%s." % libname.lower()):
  90             ret = cols[2]
  91         if cols[0].startswith("libc."):
  92             # save this one too
  93             bld.env.libc_path = cols[2]
  94     return ret
  95
  96
  97 def build_symbol_sets(bld, tgt_list):
  98     '''build the public_symbols and undefined_symbols attributes for each target'''
  99
 100     if bld.env.public_symbols:
 101         return
 102
 103     objlist = []  # list of object file
 104     objmap = {}   # map from object filename to target (subsystem) name
 105
 106     for t in tgt_list:
 107         t.public_symbols = set()
 108         t.undefined_symbols = set()
 109         t.used_symbols = set()
 110         for tsk in getattr(t, 'compiled_tasks', []):
 111             for output in tsk.outputs:
 112                 objpath = output.abspath(bld.env)
 113                 objlist.append(objpath)
 114                 objmap[objpath] = t
 115
 116     symbols = symbols_extract(objlist)
 117     for obj in objlist:
 118         t = objmap[obj]
 119         t.public_symbols = t.public_symbols.union(symbols[obj]["PUBLIC"])
 120         t.undefined_symbols = t.undefined_symbols.union(symbols[obj]["UNDEFINED"])
 121         t.used_symbols = t.used_symbols.union(symbols[obj]["UNDEFINED"])
 122
 123     t.undefined_symbols = t.undefined_symbols.difference(t.public_symbols)
 124
 125     # and the reverse map of public symbols to subsystem name
 126     bld.env.symbol_map = {}
 127
 128     for t in tgt_list:
 129         for s in t.public_symbols:
 130             if not s in bld.env.symbol_map:
 131                 bld.env.symbol_map[s] = []
 132             bld.env.symbol_map[s].append(real_name(t.sname))
 133
 134     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 135
 136     bld.env.public_symbols = {}
 137     for t in tgt_list:
 138         name = real_name(t.sname)
 139         if name in bld.env.public_symbols:
 140             bld.env.public_symbols[name] = bld.env.public_symbols[name].union(t.public_symbols)
 141         else:
 142             bld.env.public_symbols[name] = t.public_symbols
 143         if t.samba_type == 'LIBRARY':
 144             for dep in t.add_objects:
 145                 t2 = bld.name_to_obj(dep, bld.env)
 146                 bld.ASSERT(t2 is not None, "Library '%s' has unknown dependency '%s'" % (name, dep))
 147                 bld.env.public_symbols[name] = bld.env.public_symbols[name].union(t2.public_symbols)
 148
 149     bld.env.used_symbols = {}
 150     for t in tgt_list:
 151         name = real_name(t.sname)
 152         if name in bld.env.used_symbols:
 153             bld.env.used_symbols[name] = bld.env.used_symbols[name].union(t.used_symbols)
 154         else:
 155             bld.env.used_symbols[name] = t.used_symbols
 156         if t.samba_type == 'LIBRARY':
 157             for dep in t.add_objects:
 158                 t2 = bld.name_to_obj(dep, bld.env)
 159                 bld.ASSERT(t2 is not None, "Library '%s' has unknown dependency '%s'" % (name, dep))
 160                 bld.env.used_symbols[name] = bld.env.used_symbols[name].union(t2.used_symbols)
 161
 162
 163 def build_syslib_sets(bld, tgt_list):
 164     '''build the public_symbols for all syslibs'''
 165
 166     if bld.env.syslib_symbols:
 167         return
 168
 169     # work out what syslibs we depend on, and what targets those are used in
 170     syslibs = {}
 171     objmap = {}
 172     for t in tgt_list:
 173         if getattr(t, 'uselib', []) and t.samba_type in [ 'LIBRARY', 'BINARY', 'PYTHON' ]:
 174             for lib in t.uselib:
 175                 if lib in ['PYEMBED', 'PYEXT']:
 176                     lib = "python"
 177                 if not lib in syslibs:
 178                     syslibs[lib] = []
 179                 syslibs[lib].append(t)
 180
 181     # work out the paths to each syslib
 182     syslib_paths = []
 183     for lib in syslibs:
 184         path = find_syslib_path(bld, lib, syslibs[lib])
 185         if path is None:
 186             Logs.warn("Unable to find syslib path for %s" % lib)
 187         if path is not None:
 188             syslib_paths.append(path)
 189             objmap[path] = lib.lower()
 190
 191     # add in libc
 192     syslib_paths.append(bld.env.libc_path)
 193     objmap[bld.env.libc_path] = 'c'
 194
 195     symbols = symbols_extract(syslib_paths, dynamic=True)
 196
 197     # keep a map of syslib names to public symbols
 198     bld.env.syslib_symbols = {}
 199     for lib in symbols:
 200         bld.env.syslib_symbols[lib] = symbols[lib]["PUBLIC"]
 201
 202     # add to the map of symbols to dependencies
 203     for lib in symbols:
 204         for sym in symbols[lib]["PUBLIC"]:
 205             if not sym in bld.env.symbol_map:
 206                 bld.env.symbol_map[sym] = []
 207             bld.env.symbol_map[sym].append(objmap[lib])
 208
 209     # keep the libc symbols as well, as these are useful for some of the
 210     # sanity checks
 211     bld.env.libc_symbols = symbols[bld.env.libc_path]["PUBLIC"]
 212
 213     # add to the combined map of dependency name to public_symbols
 214     for lib in bld.env.syslib_symbols:
 215         bld.env.public_symbols[objmap[lib]] = bld.env.syslib_symbols[lib]
 216
 217
 218 def build_autodeps(bld, t):
 219     '''build the set of dependencies for a target'''
 220     deps = set()
 221     name = real_name(t.sname)
 222
 223     targets    = LOCAL_CACHE(bld, 'TARGET_TYPE')
 224
 225     for sym in t.undefined_symbols:
 226         if sym in t.public_symbols:
 227             continue
 228         if sym in bld.env.symbol_map:
 229             depname = bld.env.symbol_map[sym]
 230             if depname == [ name ]:
 231                 # self dependencies aren't interesting
 232                 continue
 233             if t.in_library == depname:
 234                 # no need to depend on the library we are part of
 235                 continue
 236             if depname[0] in ['c', 'python']:
 237                 # these don't go into autodeps
 238                 continue
 239             if targets[depname[0]] in [ 'SYSLIB' ]:
 240                 deps.add(depname[0])
 241                 continue
 242             t2 = bld.name_to_obj(depname[0], bld.env)
 243             if len(t2.in_library) != 1:
 244                 deps.add(depname[0])
 245                 continue
 246             if t2.in_library == t.in_library:
 247                 # if we're part of the same library, we don't need to autodep
 248                 continue
 249             deps.add(t2.in_library[0])
 250     t.autodeps = deps
 251
 252
 253 def build_library_names(bld, tgt_list):
 254     '''add a in_library attribute to all targets that are part of a library'''
 255
 256     if bld.env.done_build_library_names:
 257         return
 258
 259     for t in tgt_list:
 260         t.in_library = []
 261
 262     for t in tgt_list:
 263         if t.samba_type in [ 'LIBRARY' ]:
 264             for obj in t.samba_deps_extended:
 265                 t2 = bld.name_to_obj(obj, bld.env)
 266                 if t2 and t2.samba_type in [ 'SUBSYSTEM', 'ASN1' ]:
 267                     if not t.sname in t2.in_library:
 268                         t2.in_library.append(t.sname)
 269     bld.env.done_build_library_names = True
 270
 271
 272 def check_library_deps(bld, t):
 273     '''check that all the autodeps that have mutual dependency of this
 274     target are in the same library as the target'''
 275
 276     name = real_name(t.sname)
 277
 278     if len(t.in_library) > 1:
 279         Logs.warn("WARNING: Target '%s' in multiple libraries: %s" % (t.sname, t.in_library))
 280
 281     for dep in t.autodeps:
 282         t2 = bld.name_to_obj(dep, bld.env)
 283         if t2 is None:
 284             continue
 285         for dep2 in t2.autodeps:
 286             if dep2 == name and t.in_library != t2.in_library:
 287                 Logs.warn("WARNING: mutual dependency %s <=> %s" % (name, real_name(t2.sname)))
 288                 Logs.warn("Libraries should match. %s != %s" % (t.in_library, t2.in_library))
 289                 # raise Utils.WafError("illegal mutual dependency")
 290
 291
 292 def check_syslib_collisions(bld, tgt_list):
 293     '''check if a target has any symbol collisions with a syslib
 294
 295     We do not want any code in Samba to use a symbol name from a
 296     system library. The chance of that causing problems is just too
 297     high. Note that libreplace uses a rep_XX approach of renaming
 298     symbols via macros
 299     '''
 300
 301     has_error = False
 302     for t in tgt_list:
 303         for lib in bld.env.syslib_symbols:
 304             common = t.public_symbols.intersection(bld.env.syslib_symbols[lib])
 305             if common:
 306                 Logs.error("ERROR: Target '%s' has symbols '%s' which is also in syslib '%s'" % (t.sname, common, lib))
 307                 has_error = True
 308     if has_error:
 309         raise Utils.WafError("symbols in common with system libraries")
 310
 311
 312 def check_dependencies(bld, t):
 313     '''check for depenencies that should be changed'''
 314
 315     if bld.name_to_obj(t.sname + ".objlist", bld.env):
 316         return
 317
 318     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 319
 320     remaining = t.undefined_symbols.copy()
 321     remaining = remaining.difference(t.public_symbols)
 322
 323     sname = real_name(t.sname)
 324
 325     deps = set(t.samba_deps)
 326     for d in t.samba_deps:
 327         if targets[d] in [ 'EMPTY', 'DISABLED', 'SYSLIB' ]:
 328             continue
 329         bld.ASSERT(d in bld.env.public_symbols, "Failed to find symbol list for dependency '%s'" % d)
 330         diff = remaining.intersection(bld.env.public_symbols[d])
 331         if not diff and targets[sname] != 'LIBRARY':
 332             Logs.info("Target '%s' has no dependency on %s" % (sname, d))
 333         else:
 334             remaining = remaining.difference(diff)
 335
 336     t.unsatisfied_symbols = set()
 337     needed = {}
 338     for sym in remaining:
 339         if sym in bld.env.symbol_map:
 340             dep = bld.env.symbol_map[sym]
 341             if not dep[0] in needed:
 342                 needed[dep[0]] = set()
 343             needed[dep[0]].add(sym)
 344         else:
 345             t.unsatisfied_symbols.add(sym)
 346
 347     for dep in needed:
 348         Logs.info("Target '%s' should add dep '%s' for symbols %s" % (sname, dep, " ".join(needed[dep])))
 349
 350
 351
 352 def check_syslib_dependencies(bld, t):
 353     '''check for syslib depenencies'''
 354
 355     if bld.name_to_obj(t.sname + ".objlist", bld.env):
 356         return
 357
 358     sname = real_name(t.sname)
 359
 360     remaining = set()
 361
 362     features = TO_LIST(t.features)
 363     if 'pyembed' in features or 'pyext' in features:
 364         t.unsatisfied_symbols = t.unsatisfied_symbols.difference(bld.env.public_symbols['python'])
 365
 366     needed = {}
 367     for sym in t.unsatisfied_symbols:
 368         if sym in bld.env.symbol_map:
 369             dep = bld.env.symbol_map[sym][0]
 370             if dep == 'c':
 371                 continue
 372             if not dep in needed:
 373                 needed[dep] = set()
 374             needed[dep].add(sym)
 375         else:
 376             remaining.add(sym)
 377
 378     for dep in needed:
 379         Logs.info("Target '%s' should add syslib dep '%s' for symbols %s" % (sname, dep, " ".join(needed[dep])))
 380
 381     if remaining:
 382         debug("deps: Target '%s' has unsatisfied symbols: %s" % (sname, " ".join(remaining)))
 383
 384
 385
 386 def symbols_symbolcheck(task):
 387     '''check the internal dependency lists'''
 388     bld = task.env.bld
 389     tgt_list = get_tgt_list(bld)
 390
 391     build_symbol_sets(bld, tgt_list)
 392     build_library_names(bld, tgt_list)
 393
 394     for t in tgt_list:
 395         t.autodeps = set()
 396         if getattr(t, 'source', ''):
 397             build_autodeps(bld, t)
 398
 399     for t in tgt_list:
 400         check_dependencies(bld, t)
 401
 402     for t in tgt_list:
 403         check_library_deps(bld, t)
 404
 405 def symbols_syslibcheck(task):
 406     '''check the syslib dependencies'''
 407     bld = task.env.bld
 408     tgt_list = get_tgt_list(bld)
 409
 410     build_syslib_sets(bld, tgt_list)
 411     check_syslib_collisions(bld, tgt_list)
 412
 413     for t in tgt_list:
 414         check_syslib_dependencies(bld, t)
 415
 416
 417 def symbols_whyneeded(task):
 418     """check why 'target' needs to link to 'subsystem'"""
 419     bld = task.env.bld
 420     tgt_list = get_tgt_list(bld)
 421
 422     why = Options.options.WHYNEEDED.split(":")
 423     if len(why) != 2:
 424         raise Utils.WafError("usage: WHYNEEDED=TARGET:DEPENDENCY")
 425     target = why[0]
 426     subsystem = why[1]
 427
 428     build_symbol_sets(bld, tgt_list)
 429     build_library_names(bld, tgt_list)
 430     build_syslib_sets(bld, tgt_list)
 431
 432     Logs.info("Checking why %s needs to link to %s" % (target, subsystem))
 433     if not target in bld.env.used_symbols:
 434         Logs.warn("unable to find target '%s' in used_symbols dict" % target)
 435         return
 436     if not subsystem in bld.env.public_symbols:
 437         Logs.warn("unable to find subsystem '%s' in public_symbols dict" % subsystem)
 438         return
 439     overlap = bld.env.used_symbols[target].intersection(bld.env.public_symbols[subsystem])
 440     if not overlap:
 441         Logs.info("target '%s' doesn't use any public symbols from '%s'" % (target, subsystem))
 442     else:
 443         Logs.info("target '%s' uses symbols %s from '%s'" % (target, overlap, subsystem))
 444
 445
 446
 447 def symbols_dupcheck(task):
 448     '''check for symbols defined in two different subsystems'''
 449     bld = task.env.bld
 450     tgt_list = get_tgt_list(bld)
 451
 452     targets = LOCAL_CACHE(bld, 'TARGET_TYPE')
 453
 454     Logs.info("Checking for duplicate symbols")
 455     for sym in bld.env.symbol_map:
 456         subsystems = set(bld.env.symbol_map[sym])
 457         if len(subsystems) == 1:
 458             continue
 459
 460         if sym in ['main', '_init', '_fini', 'init_samba_module', 'samba_init_module', 'ldb_init_module' ]:
 461             # these are expected to be in many subsystems
 462             continue
 463
 464         # if all of them are in system libraries, we can ignore them. This copes
 465         # with the duplication between libc, libpthread and libattr
 466         all_syslib = True
 467         for s in subsystems:
 468             if s != 'c' and (not s in targets or targets[s] != 'SYSLIB'):
 469                 all_syslib = False
 470         if all_syslib:
 471             continue
 472         Logs.info("symbol %s appears in %s" % (sym, subsystems))
 473
 474
 475 def SYMBOL_CHECK(bld):
 476     '''check our dependency lists'''
 477     if Options.options.SYMBOLCHECK:
 478         bld.SET_BUILD_GROUP('symbolcheck')
 479         task = bld(rule=symbols_symbolcheck, always=True, name='symbol checking')
 480         task.env.bld = bld
 481
 482         bld.SET_BUILD_GROUP('syslibcheck')
 483         task = bld(rule=symbols_syslibcheck, always=True, name='syslib checking')
 484         task.env.bld = bld
 485
 486         bld.SET_BUILD_GROUP('syslibcheck')
 487         task = bld(rule=symbols_dupcheck, always=True, name='symbol duplicate checking')
 488         task.env.bld = bld
 489
 490     if Options.options.WHYNEEDED:
 491         bld.SET_BUILD_GROUP('syslibcheck')
 492         task = bld(rule=symbols_whyneeded, always=True, name='check why a dependency is needed')
 493         task.env.bld = bld
 494
 495
 496 Build.BuildContext.SYMBOL_CHECK = SYMBOL_CHECK