config/expandlibs_exec.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
   6 a given command line, and executes that command line with the expanded
   7 arguments.
   8
   9 With the --extract argument (useful for e.g. $(AR)), it extracts object files
  10 from static libraries (or use those listed in library descriptors directly).
  11
  12 With the --uselist argument (useful for e.g. $(CC)), it replaces all object
  13 files with a list file. This can be used to avoid limitations in the length
  14 of a command line. The kind of list file format used depends on the
  15 EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
  16 or 'linkerscript' for GNU ld linker scripts.
  17 See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
  18
  19 With the --symbol-order argument, followed by a file name, it will add the
  20 relevant linker options to change the order in which the linker puts the
  21 symbols appear in the resulting binary. Only works for ELF targets.
  22 '''
  23 from __future__ import with_statement
  24 import sys
  25 import os
  26 from expandlibs import (
  27     ExpandArgs,
  28     relativize,
  29     isDynamicLib,
  30     isObject,
  31     ensureParentDir,
  32     ExpandLibsDeps,
  33 )
  34 import expandlibs_config as conf
  35 from optparse import OptionParser
  36 import subprocess
  37 import tempfile
  38 import shutil
  39 import subprocess
  40 import re
  41 from mozbuild.makeutil import Makefile
  42
  43 # The are the insert points for a GNU ld linker script, assuming a more
  44 # or less "standard" default linker script. This is not a dict because
  45 # order is important.
  46 SECTION_INSERT_BEFORE = [
  47   ('.text', '.fini'),
  48   ('.rodata', '.rodata1'),
  49   ('.data.rel.ro', '.dynamic'),
  50   ('.data', '.data1'),
  51 ]
  52
  53 class ExpandArgsMore(ExpandArgs):
  54     ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
  55     def __enter__(self):
  56         self.tmp = []
  57         return self
  58
  59     def __exit__(self, type, value, tb):
  60         '''Automatically remove temporary files'''
  61         for tmp in self.tmp:
  62             if os.path.isdir(tmp):
  63                 shutil.rmtree(tmp, True)
  64             else:
  65                 os.remove(tmp)
  66
  67     def extract(self):
  68         self[0:] = self._extract(self)
  69
  70     def _extract(self, args):
  71         '''When a static library name is found, either extract its contents
  72         in a temporary directory or use the information found in the
  73         corresponding lib descriptor.
  74         '''
  75         ar_extract = conf.AR_EXTRACT.split()
  76         newlist = []
  77         for arg in args:
  78             if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
  79                 if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
  80                     newlist += self._extract(self._expand_desc(arg))
  81                     continue
  82                 elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'):
  83                     tmp = tempfile.mkdtemp(dir=os.curdir)
  84                     self.tmp.append(tmp)
  85                     if conf.AR == 'lib':
  86                         out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg])
  87                         files = out.splitlines()
  88                         # If lib -list returns a list full of dlls, it's an
  89                         # import lib.
  90                         if all(isDynamicLib(f) for f in files):
  91                             newlist += [arg]
  92                             continue
  93                         for f in files:
  94                             subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp)
  95                     else:
  96                         subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)
  97                     objs = []
  98                     for root, dirs, files in os.walk(tmp):
  99                         objs += [relativize(os.path.join(root, f)) for f in files if isObject(f)]
 100                     newlist += sorted(objs)
 101                     continue
 102             newlist += [arg]
 103         return newlist
 104
 105     def makelist(self):
 106         '''Replaces object file names with a temporary list file, using a
 107         list format depending on the EXPAND_LIBS_LIST_STYLE variable
 108         '''
 109         objs = [o for o in self if isObject(o)]
 110         if not len(objs): return
 111         fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)
 112         if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":
 113             content = ['INPUT("%s")\n' % obj for obj in objs]
 114             ref = tmp
 115         elif conf.EXPAND_LIBS_LIST_STYLE == "filelist":
 116             content = ["%s\n" % obj for obj in objs]
 117             ref = "-Wl,-filelist," + tmp
 118         elif conf.EXPAND_LIBS_LIST_STYLE == "list":
 119             content = ["%s\n" % obj for obj in objs]
 120             ref = "@" + tmp
 121         else:
 122             os.close(fd)
 123             os.remove(tmp)
 124             return
 125         self.tmp.append(tmp)
 126         f = os.fdopen(fd, "w")
 127         f.writelines(content)
 128         f.close()
 129         idx = self.index(objs[0])
 130         newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]
 131         self[0:] = newlist
 132
 133     def _getFoldedSections(self):
 134         '''Returns a dict about folded sections.
 135         When section A and B are folded into section C, the dict contains:
 136         { 'A': 'C',
 137           'B': 'C',
 138           'C': ['A', 'B'] }'''
 139         if not conf.LD_PRINT_ICF_SECTIONS:
 140             return {}
 141
 142         proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
 143         (stdout, stderr) = proc.communicate()
 144         result = {}
 145         # gold's --print-icf-sections output looks like the following:
 146         # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
 147         # In terms of words, chances are this will change in the future,
 148         # especially considering "into" is misplaced. Splitting on quotes
 149         # seems safer.
 150         for l in stderr.split('\n'):
 151             quoted = l.split("'")
 152             if len(quoted) > 5 and quoted[1] != quoted[5]:
 153                 result[quoted[1]] = [quoted[5]]
 154                 if quoted[5] in result:
 155                     result[quoted[5]].append(quoted[1])
 156                 else:
 157                     result[quoted[5]] = [quoted[1]]
 158         return result
 159
 160     def _getOrderedSections(self, ordered_symbols):
 161         '''Given an ordered list of symbols, returns the corresponding list
 162         of sections following the order.'''
 163         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
 164             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 165         finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
 166         folded = self._getFoldedSections()
 167         sections = set()
 168         ordered_sections = []
 169         for symbol in ordered_symbols:
 170             symbol_sections = finder.getSections(symbol)
 171             all_symbol_sections = []
 172             for section in symbol_sections:
 173                 if section in folded:
 174                     if isinstance(folded[section], str):
 175                         section = folded[section]
 176                     all_symbol_sections.append(section)
 177                     all_symbol_sections.extend(folded[section])
 178                 else:
 179                     all_symbol_sections.append(section)
 180             for section in all_symbol_sections:
 181                 if not section in sections:
 182                     ordered_sections.append(section)
 183                     sections.add(section)
 184         return ordered_sections
 185
 186     def orderSymbols(self, order):
 187         '''Given a file containing a list of symbols, adds the appropriate
 188         argument to make the linker put the symbols in that order.'''
 189         with open(order) as file:
 190             sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
 191         split_sections = {}
 192         linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
 193         for s in sections:
 194             for linked_section in linked_sections:
 195                 if s.startswith(linked_section):
 196                     if linked_section in split_sections:
 197                         split_sections[linked_section].append(s)
 198                     else:
 199                         split_sections[linked_section] = [s]
 200                     break
 201         content = []
 202         # Order is important
 203         linked_sections = [s for s in linked_sections if s in split_sections]
 204
 205         if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
 206             option = '-Wl,--section-ordering-file,%s'
 207             content = sections
 208             for linked_section in linked_sections:
 209                 content.extend(split_sections[linked_section])
 210                 content.append('%s.*' % linked_section)
 211                 content.append(linked_section)
 212
 213         elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
 214             option = '-Wl,-T,%s'
 215             section_insert_before = dict(SECTION_INSERT_BEFORE)
 216             for linked_section in linked_sections:
 217                 content.append('SECTIONS {')
 218                 content.append('  %s : {' % linked_section)
 219                 content.extend('    *(%s)' % s for s in split_sections[linked_section])
 220                 content.append('  }')
 221                 content.append('}')
 222                 content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
 223         else:
 224             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 225
 226         fd, tmp = tempfile.mkstemp(dir=os.curdir)
 227         f = os.fdopen(fd, "w")
 228         f.write('\n'.join(content)+'\n')
 229         f.close()
 230         self.tmp.append(tmp)
 231         self.append(option % tmp)
 232
 233 class SectionFinder(object):
 234     '''Instances of this class allow to map symbol names to sections in
 235     object files.'''
 236
 237     def __init__(self, objs):
 238         '''Creates an instance, given a list of object files.'''
 239         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
 240             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 241         self.mapping = {}
 242         for obj in objs:
 243             if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
 244                 raise Exception('%s is not an object nor a static library' % obj)
 245             for symbol, section in SectionFinder._getSymbols(obj):
 246                 sym = SectionFinder._normalize(symbol)
 247                 if sym in self.mapping:
 248                     if not section in self.mapping[sym]:
 249                         self.mapping[sym].append(section)
 250                 else:
 251                     self.mapping[sym] = [section]
 252
 253     def getSections(self, symbol):
 254         '''Given a symbol, returns a list of sections containing it or the
 255         corresponding thunks. When the given symbol is a thunk, returns the
 256         list of sections containing its corresponding normal symbol and the
 257         other thunks for that symbol.'''
 258         sym = SectionFinder._normalize(symbol)
 259         if sym in self.mapping:
 260             return self.mapping[sym]
 261         return []
 262
 263     @staticmethod
 264     def _normalize(symbol):
 265         '''For normal symbols, return the given symbol. For thunks, return
 266         the corresponding normal symbol.'''
 267         if re.match('^_ZThn[0-9]+_', symbol):
 268             return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
 269         return symbol
 270
 271     @staticmethod
 272     def _getSymbols(obj):
 273         '''Returns a list of (symbol, section) contained in the given object
 274         file.'''
 275         proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
 276         (stdout, stderr) = proc.communicate()
 277         syms = []
 278         for line in stdout.splitlines():
 279             # Each line has the following format:
 280             # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
 281             tmp = line.split(' ',1)
 282             # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
 283             # We only need to consider cases where "<section>\t<length> <symbol>" is present,
 284             # and where the [FfO] flag is either F (function) or O (object).
 285             if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
 286                 tmp = tmp[1][8:].split()
 287                 # That gives us ["<section>","<length>", "<symbol>"]
 288                 syms.append((tmp[-1], tmp[0]))
 289         return syms
 290
 291 def print_command(out, args):
 292     print >>out, "Executing: " + " ".join(args)
 293     for tmp in [f for f in args.tmp if os.path.isfile(f)]:
 294         print >>out, tmp + ":"
 295         with open(tmp) as file:
 296             print >>out, "".join(["    " + l for l in file.readlines()])
 297     out.flush()
 298
 299 def main():
 300     parser = OptionParser()
 301     parser.add_option("--depend", dest="depend", metavar="FILE",
 302         help="generate dependencies for the given execution and store it in the given file")
 303     parser.add_option("--target", dest="target", metavar="FILE",
 304         help="designate the target for dependencies")
 305     parser.add_option("--extract", action="store_true", dest="extract",
 306         help="when a library has no descriptor file, extract it first, when possible")
 307     parser.add_option("--uselist", action="store_true", dest="uselist",
 308         help="use a list file for objects when executing a command")
 309     parser.add_option("--verbose", action="store_true", dest="verbose",
 310         help="display executed command and temporary files content")
 311     parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
 312         help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
 313
 314     (options, args) = parser.parse_args()
 315
 316     if not options.target:
 317         options.depend = False
 318     if options.depend:
 319         deps = ExpandLibsDeps(args)
 320         # Filter out common command wrappers
 321         while os.path.basename(deps[0]) in ['ccache', 'distcc']:
 322             deps.pop(0)
 323         # Remove command
 324         deps.pop(0)
 325     with ExpandArgsMore(args) as args:
 326         if options.extract:
 327             args.extract()
 328         if options.symbol_order:
 329             args.orderSymbols(options.symbol_order)
 330         if options.uselist:
 331             args.makelist()
 332
 333         if options.verbose:
 334             print_command(sys.stderr, args)
 335         try:
 336             proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
 337         except Exception, e:
 338             print >>sys.stderr, 'error: Launching', args, ':', e
 339             raise e
 340         (stdout, stderr) = proc.communicate()
 341         if proc.returncode and not options.verbose:
 342             print_command(sys.stderr, args)
 343         sys.stderr.write(stdout)
 344         sys.stderr.flush()
 345         if proc.returncode:
 346             exit(proc.returncode)
 347     if not options.depend:
 348         return
 349     ensureParentDir(options.depend)
 350     mk = Makefile()
 351     deps = [dep for dep in deps if os.path.isfile(dep) and dep != options.target]
 352     no_dynamic_lib = [dep for dep in deps if not isDynamicLib(dep)]
 353     mk.create_rule([options.target]).add_dependencies(no_dynamic_lib)
 354     if len(deps) != len(no_dynamic_lib):
 355         mk.create_rule(['%s_order_only' % options.target]).add_dependencies(dep for dep in deps if isDynamicLib(dep))
 356
 357     with open(options.depend, 'w') as depfile:
 358         mk.dump(depfile, removal_guard=True)
 359
 360 if __name__ == '__main__':
 361     main()