config/expandlibs_exec.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
   6 a given command line, and executes that command line with the expanded
   7 arguments.
   8
   9 With the --extract argument (useful for e.g. $(AR)), it extracts object files
  10 from static libraries (or use those listed in library descriptors directly).
  11
  12 With the --uselist argument (useful for e.g. $(CC)), it replaces all object
  13 files with a list file. This can be used to avoid limitations in the length
  14 of a command line. The kind of list file format used depends on the
  15 EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
  16 or 'linkerscript' for GNU ld linker scripts.
  17 See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
  18
  19 With the --symbol-order argument, followed by a file name, it will add the
  20 relevant linker options to change the order in which the linker puts the
  21 symbols appear in the resulting binary. Only works for ELF targets.
  22 '''
  23 from __future__ import with_statement
  24 import sys
  25 import os
  26 from expandlibs import (
  27     ExpandArgs,
  28     relativize,
  29     isDynamicLib,
  30     isObject,
  31 )
  32 import expandlibs_config as conf
  33 from optparse import OptionParser
  34 import subprocess
  35 import tempfile
  36 import shutil
  37 import subprocess
  38 import re
  39 from mozbuild.makeutil import Makefile
  40
  41 # The are the insert points for a GNU ld linker script, assuming a more
  42 # or less "standard" default linker script. This is not a dict because
  43 # order is important.
  44 SECTION_INSERT_BEFORE = [
  45   ('.text', '.fini'),
  46   ('.rodata', '.rodata1'),
  47   ('.data.rel.ro', '.dynamic'),
  48   ('.data', '.data1'),
  49 ]
  50
  51 class ExpandArgsMore(ExpandArgs):
  52     ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
  53     def __enter__(self):
  54         self.tmp = []
  55         return self
  56
  57     def __exit__(self, type, value, tb):
  58         '''Automatically remove temporary files'''
  59         for tmp in self.tmp:
  60             if os.path.isdir(tmp):
  61                 shutil.rmtree(tmp, True)
  62             else:
  63                 os.remove(tmp)
  64
  65     def extract(self):
  66         self[0:] = self._extract(self)
  67
  68     def _extract(self, args):
  69         '''When a static library name is found, either extract its contents
  70         in a temporary directory or use the information found in the
  71         corresponding lib descriptor.
  72         '''
  73         ar_extract = conf.AR_EXTRACT.split()
  74         newlist = []
  75
  76         def lookup(base, f):
  77             for root, dirs, files in os.walk(base):
  78                 if f in files:
  79                     return os.path.join(root, f)
  80
  81         for arg in args:
  82             if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
  83                 if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
  84                     newlist += self._extract(self._expand_desc(arg))
  85                     continue
  86                 elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'):
  87                     tmp = tempfile.mkdtemp(dir=os.curdir)
  88                     self.tmp.append(tmp)
  89                     if conf.AR == 'lib':
  90                         out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg])
  91                         files = out.splitlines()
  92                         # If lib -list returns a list full of dlls, it's an
  93                         # import lib.
  94                         if all(isDynamicLib(f) for f in files):
  95                             newlist += [arg]
  96                             continue
  97                         for f in files:
  98                             subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp)
  99                     else:
 100                         subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)
 101                     objs = []
 102                     basedir = os.path.dirname(arg)
 103                     for root, dirs, files in os.walk(tmp):
 104                         for f in files:
 105                             if isObject(f):
 106                                 # If the file extracted from the library also
 107                                 # exists in the directory containing the
 108                                 # library, or one of its subdirectories, use
 109                                 # that instead.
 110                                 maybe_obj = lookup(os.path.join(basedir, os.path.relpath(root, tmp)), f)
 111                                 if maybe_obj:
 112                                     objs.append(relativize(maybe_obj))
 113                                 else:
 114                                     objs.append(relativize(os.path.join(root, f)))
 115                     newlist += sorted(objs)
 116                     continue
 117             newlist += [arg]
 118         return newlist
 119
 120     def makelist(self):
 121         '''Replaces object file names with a temporary list file, using a
 122         list format depending on the EXPAND_LIBS_LIST_STYLE variable
 123         '''
 124         objs = [o for o in self if isObject(o)]
 125         if not len(objs): return
 126         fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)
 127         if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":
 128             content = ['INPUT("%s")\n' % obj for obj in objs]
 129             ref = tmp
 130         elif conf.EXPAND_LIBS_LIST_STYLE == "filelist":
 131             content = ["%s\n" % obj for obj in objs]
 132             ref = "-Wl,-filelist," + tmp
 133         elif conf.EXPAND_LIBS_LIST_STYLE == "list":
 134             content = ["%s\n" % obj for obj in objs]
 135             ref = "@" + tmp
 136         else:
 137             os.close(fd)
 138             os.remove(tmp)
 139             return
 140         self.tmp.append(tmp)
 141         f = os.fdopen(fd, "w")
 142         f.writelines(content)
 143         f.close()
 144         idx = self.index(objs[0])
 145         newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]
 146         self[0:] = newlist
 147
 148     def _getFoldedSections(self):
 149         '''Returns a dict about folded sections.
 150         When section A and B are folded into section C, the dict contains:
 151         { 'A': 'C',
 152           'B': 'C',
 153           'C': ['A', 'B'] }'''
 154         if not conf.LD_PRINT_ICF_SECTIONS:
 155             return {}
 156
 157         proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
 158         (stdout, stderr) = proc.communicate()
 159         result = {}
 160         # gold's --print-icf-sections output looks like the following:
 161         # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
 162         # In terms of words, chances are this will change in the future,
 163         # especially considering "into" is misplaced. Splitting on quotes
 164         # seems safer.
 165         for l in stderr.split('\n'):
 166             quoted = l.split("'")
 167             if len(quoted) > 5 and quoted[1] != quoted[5]:
 168                 result[quoted[1]] = [quoted[5]]
 169                 if quoted[5] in result:
 170                     result[quoted[5]].append(quoted[1])
 171                 else:
 172                     result[quoted[5]] = [quoted[1]]
 173         return result
 174
 175     def _getOrderedSections(self, ordered_symbols):
 176         '''Given an ordered list of symbols, returns the corresponding list
 177         of sections following the order.'''
 178         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
 179             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 180         finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
 181         folded = self._getFoldedSections()
 182         sections = set()
 183         ordered_sections = []
 184         for symbol in ordered_symbols:
 185             symbol_sections = finder.getSections(symbol)
 186             all_symbol_sections = []
 187             for section in symbol_sections:
 188                 if section in folded:
 189                     if isinstance(folded[section], str):
 190                         section = folded[section]
 191                     all_symbol_sections.append(section)
 192                     all_symbol_sections.extend(folded[section])
 193                 else:
 194                     all_symbol_sections.append(section)
 195             for section in all_symbol_sections:
 196                 if not section in sections:
 197                     ordered_sections.append(section)
 198                     sections.add(section)
 199         return ordered_sections
 200
 201     def orderSymbols(self, order):
 202         '''Given a file containing a list of symbols, adds the appropriate
 203         argument to make the linker put the symbols in that order.'''
 204         with open(order) as file:
 205             sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
 206         split_sections = {}
 207         linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
 208         for s in sections:
 209             for linked_section in linked_sections:
 210                 if s.startswith(linked_section):
 211                     if linked_section in split_sections:
 212                         split_sections[linked_section].append(s)
 213                     else:
 214                         split_sections[linked_section] = [s]
 215                     break
 216         content = []
 217         # Order is important
 218         linked_sections = [s for s in linked_sections if s in split_sections]
 219
 220         if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
 221             option = '-Wl,--section-ordering-file,%s'
 222             content = sections
 223             for linked_section in linked_sections:
 224                 content.extend(split_sections[linked_section])
 225                 content.append('%s.*' % linked_section)
 226                 content.append(linked_section)
 227
 228         elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
 229             option = '-Wl,-T,%s'
 230             section_insert_before = dict(SECTION_INSERT_BEFORE)
 231             for linked_section in linked_sections:
 232                 content.append('SECTIONS {')
 233                 content.append('  %s : {' % linked_section)
 234                 content.extend('    *(%s)' % s for s in split_sections[linked_section])
 235                 content.append('  }')
 236                 content.append('}')
 237                 content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
 238         else:
 239             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 240
 241         fd, tmp = tempfile.mkstemp(dir=os.curdir)
 242         f = os.fdopen(fd, "w")
 243         f.write('\n'.join(content)+'\n')
 244         f.close()
 245         self.tmp.append(tmp)
 246         self.append(option % tmp)
 247
 248 class SectionFinder(object):
 249     '''Instances of this class allow to map symbol names to sections in
 250     object files.'''
 251
 252     def __init__(self, objs):
 253         '''Creates an instance, given a list of object files.'''
 254         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
 255             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 256         self.mapping = {}
 257         for obj in objs:
 258             if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
 259                 raise Exception('%s is not an object nor a static library' % obj)
 260             for symbol, section in SectionFinder._getSymbols(obj):
 261                 sym = SectionFinder._normalize(symbol)
 262                 if sym in self.mapping:
 263                     if not section in self.mapping[sym]:
 264                         self.mapping[sym].append(section)
 265                 else:
 266                     self.mapping[sym] = [section]
 267
 268     def getSections(self, symbol):
 269         '''Given a symbol, returns a list of sections containing it or the
 270         corresponding thunks. When the given symbol is a thunk, returns the
 271         list of sections containing its corresponding normal symbol and the
 272         other thunks for that symbol.'''
 273         sym = SectionFinder._normalize(symbol)
 274         if sym in self.mapping:
 275             return self.mapping[sym]
 276         return []
 277
 278     @staticmethod
 279     def _normalize(symbol):
 280         '''For normal symbols, return the given symbol. For thunks, return
 281         the corresponding normal symbol.'''
 282         if re.match('^_ZThn[0-9]+_', symbol):
 283             return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
 284         return symbol
 285
 286     @staticmethod
 287     def _getSymbols(obj):
 288         '''Returns a list of (symbol, section) contained in the given object
 289         file.'''
 290         proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
 291         (stdout, stderr) = proc.communicate()
 292         syms = []
 293         for line in stdout.splitlines():
 294             # Each line has the following format:
 295             # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
 296             tmp = line.split(' ',1)
 297             # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
 298             # We only need to consider cases where "<section>\t<length> <symbol>" is present,
 299             # and where the [FfO] flag is either F (function) or O (object).
 300             if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
 301                 tmp = tmp[1][8:].split()
 302                 # That gives us ["<section>","<length>", "<symbol>"]
 303                 syms.append((tmp[-1], tmp[0]))
 304         return syms
 305
 306 def print_command(out, args):
 307     print >>out, "Executing: " + " ".join(args)
 308     for tmp in [f for f in args.tmp if os.path.isfile(f)]:
 309         print >>out, tmp + ":"
 310         with open(tmp) as file:
 311             print >>out, "".join(["    " + l for l in file.readlines()])
 312     out.flush()
 313
 314 def main(args, proc_callback=None):
 315     parser = OptionParser()
 316     parser.add_option("--extract", action="store_true", dest="extract",
 317         help="when a library has no descriptor file, extract it first, when possible")
 318     parser.add_option("--uselist", action="store_true", dest="uselist",
 319         help="use a list file for objects when executing a command")
 320     parser.add_option("--verbose", action="store_true", dest="verbose",
 321         help="display executed command and temporary files content")
 322     parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
 323         help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
 324
 325     (options, args) = parser.parse_args(args)
 326
 327     with ExpandArgsMore(args) as args:
 328         if options.extract:
 329             args.extract()
 330         if options.symbol_order:
 331             args.orderSymbols(options.symbol_order)
 332         if options.uselist:
 333             args.makelist()
 334
 335         if options.verbose:
 336             print_command(sys.stderr, args)
 337         try:
 338             proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
 339             if proc_callback:
 340                 proc_callback(proc)
 341         except Exception, e:
 342             print >>sys.stderr, 'error: Launching', args, ':', e
 343             raise e
 344         (stdout, stderr) = proc.communicate()
 345         if proc.returncode and not options.verbose:
 346             print_command(sys.stderr, args)
 347         sys.stderr.write(stdout)
 348         sys.stderr.flush()
 349         if proc.returncode:
 350             return proc.returncode
 351         return 0
 352
 353 if __name__ == '__main__':
 354     exit(main(sys.argv[1:]))