config/expandlibs_exec.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
   6 a given command line, and executes that command line with the expanded
   7 arguments.
   8
   9 With the --extract argument (useful for e.g. $(AR)), it extracts object files
  10 from static libraries (or use those listed in library descriptors directly).
  11
  12 With the --uselist argument (useful for e.g. $(CC)), it replaces all object
  13 files with a list file. This can be used to avoid limitations in the length
  14 of a command line. The kind of list file format used depends on the
  15 EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
  16 or 'linkerscript' for GNU ld linker scripts.
  17 See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
  18
  19 With the --symbol-order argument, followed by a file name, it will add the
  20 relevant linker options to change the order in which the linker puts the
  21 symbols appear in the resulting binary. Only works for ELF targets.
  22 '''
  23 from __future__ import with_statement
  24 import sys
  25 import os
  26 from expandlibs import ExpandArgs, relativize, isObject, ensureParentDir, ExpandLibsDeps
  27 import expandlibs_config as conf
  28 from optparse import OptionParser
  29 import subprocess
  30 import tempfile
  31 import shutil
  32 import subprocess
  33 import re
  34
  35 # The are the insert points for a GNU ld linker script, assuming a more
  36 # or less "standard" default linker script. This is not a dict because
  37 # order is important.
  38 SECTION_INSERT_BEFORE = [
  39   ('.text', '.fini'),
  40   ('.rodata', '.rodata1'),
  41   ('.data.rel.ro', '.dynamic'),
  42   ('.data', '.data1'),
  43 ]
  44
  45 class ExpandArgsMore(ExpandArgs):
  46     ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
  47     def __enter__(self):
  48         self.tmp = []
  49         return self
  50
  51     def __exit__(self, type, value, tb):
  52         '''Automatically remove temporary files'''
  53         for tmp in self.tmp:
  54             if os.path.isdir(tmp):
  55                 shutil.rmtree(tmp, True)
  56             else:
  57                 os.remove(tmp)
  58
  59     def extract(self):
  60         self[0:] = self._extract(self)
  61
  62     def _extract(self, args):
  63         '''When a static library name is found, either extract its contents
  64         in a temporary directory or use the information found in the
  65         corresponding lib descriptor.
  66         '''
  67         ar_extract = conf.AR_EXTRACT.split()
  68         newlist = []
  69         for arg in args:
  70             if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
  71                 if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
  72                     newlist += self._extract(self._expand_desc(arg))
  73                     continue
  74                 elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'):
  75                     tmp = tempfile.mkdtemp(dir=os.curdir)
  76                     self.tmp.append(tmp)
  77                     if conf.AR == 'lib':
  78                         out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg])
  79                         for l in out.splitlines():
  80                             subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % l, os.path.abspath(arg)], cwd=tmp)
  81                     else:
  82                         subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)
  83                     objs = []
  84                     for root, dirs, files in os.walk(tmp):
  85                         objs += [relativize(os.path.join(root, f)) for f in files if isObject(f)]
  86                     newlist += sorted(objs)
  87                     continue
  88             newlist += [arg]
  89         return newlist
  90
  91     def makelist(self):
  92         '''Replaces object file names with a temporary list file, using a
  93         list format depending on the EXPAND_LIBS_LIST_STYLE variable
  94         '''
  95         objs = [o for o in self if isObject(o)]
  96         if not len(objs): return
  97         fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)
  98         if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":
  99             content = ['INPUT("%s")\n' % obj for obj in objs]
 100             ref = tmp
 101         elif conf.EXPAND_LIBS_LIST_STYLE == "filelist":
 102             content = ["%s\n" % obj for obj in objs]
 103             ref = "-Wl,-filelist," + tmp
 104         elif conf.EXPAND_LIBS_LIST_STYLE == "list":
 105             content = ["%s\n" % obj for obj in objs]
 106             ref = "@" + tmp
 107         else:
 108             os.close(fd)
 109             os.remove(tmp)
 110             return
 111         self.tmp.append(tmp)
 112         f = os.fdopen(fd, "w")
 113         f.writelines(content)
 114         f.close()
 115         idx = self.index(objs[0])
 116         newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]
 117         self[0:] = newlist
 118
 119     def _getFoldedSections(self):
 120         '''Returns a dict about folded sections.
 121         When section A and B are folded into section C, the dict contains:
 122         { 'A': 'C',
 123           'B': 'C',
 124           'C': ['A', 'B'] }'''
 125         if not conf.LD_PRINT_ICF_SECTIONS:
 126             return {}
 127
 128         proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
 129         (stdout, stderr) = proc.communicate()
 130         result = {}
 131         # gold's --print-icf-sections output looks like the following:
 132         # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
 133         # In terms of words, chances are this will change in the future,
 134         # especially considering "into" is misplaced. Splitting on quotes
 135         # seems safer.
 136         for l in stderr.split('\n'):
 137             quoted = l.split("'")
 138             if len(quoted) > 5 and quoted[1] != quoted[5]:
 139                 result[quoted[1]] = [quoted[5]]
 140                 if quoted[5] in result:
 141                     result[quoted[5]].append(quoted[1])
 142                 else:
 143                     result[quoted[5]] = [quoted[1]]
 144         return result
 145
 146     def _getOrderedSections(self, ordered_symbols):
 147         '''Given an ordered list of symbols, returns the corresponding list
 148         of sections following the order.'''
 149         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
 150             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 151         finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
 152         folded = self._getFoldedSections()
 153         sections = set()
 154         ordered_sections = []
 155         for symbol in ordered_symbols:
 156             symbol_sections = finder.getSections(symbol)
 157             all_symbol_sections = []
 158             for section in symbol_sections:
 159                 if section in folded:
 160                     if isinstance(folded[section], str):
 161                         section = folded[section]
 162                     all_symbol_sections.append(section)
 163                     all_symbol_sections.extend(folded[section])
 164                 else:
 165                     all_symbol_sections.append(section)
 166             for section in all_symbol_sections:
 167                 if not section in sections:
 168                     ordered_sections.append(section)
 169                     sections.add(section)
 170         return ordered_sections
 171
 172     def orderSymbols(self, order):
 173         '''Given a file containing a list of symbols, adds the appropriate
 174         argument to make the linker put the symbols in that order.'''
 175         with open(order) as file:
 176             sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
 177         split_sections = {}
 178         linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
 179         for s in sections:
 180             for linked_section in linked_sections:
 181                 if s.startswith(linked_section):
 182                     if linked_section in split_sections:
 183                         split_sections[linked_section].append(s)
 184                     else:
 185                         split_sections[linked_section] = [s]
 186                     break
 187         content = []
 188         # Order is important
 189         linked_sections = [s for s in linked_sections if s in split_sections]
 190
 191         if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
 192             option = '-Wl,--section-ordering-file,%s'
 193             content = sections
 194             for linked_section in linked_sections:
 195                 content.extend(split_sections[linked_section])
 196                 content.append('%s.*' % linked_section)
 197                 content.append(linked_section)
 198
 199         elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
 200             option = '-Wl,-T,%s'
 201             section_insert_before = dict(SECTION_INSERT_BEFORE)
 202             for linked_section in linked_sections:
 203                 content.append('SECTIONS {')
 204                 content.append('  %s : {' % linked_section)
 205                 content.extend('    *(%s)' % s for s in split_sections[linked_section])
 206                 content.append('  }')
 207                 content.append('}')
 208                 content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
 209         else:
 210             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 211
 212         fd, tmp = tempfile.mkstemp(dir=os.curdir)
 213         f = os.fdopen(fd, "w")
 214         f.write('\n'.join(content)+'\n')
 215         f.close()
 216         self.tmp.append(tmp)
 217         self.append(option % tmp)
 218
 219 class SectionFinder(object):
 220     '''Instances of this class allow to map symbol names to sections in
 221     object files.'''
 222
 223     def __init__(self, objs):
 224         '''Creates an instance, given a list of object files.'''
 225         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
 226             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 227         self.mapping = {}
 228         for obj in objs:
 229             if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
 230                 raise Exception('%s is not an object nor a static library' % obj)
 231             for symbol, section in SectionFinder._getSymbols(obj):
 232                 sym = SectionFinder._normalize(symbol)
 233                 if sym in self.mapping:
 234                     if not section in self.mapping[sym]:
 235                         self.mapping[sym].append(section)
 236                 else:
 237                     self.mapping[sym] = [section]
 238
 239     def getSections(self, symbol):
 240         '''Given a symbol, returns a list of sections containing it or the
 241         corresponding thunks. When the given symbol is a thunk, returns the
 242         list of sections containing its corresponding normal symbol and the
 243         other thunks for that symbol.'''
 244         sym = SectionFinder._normalize(symbol)
 245         if sym in self.mapping:
 246             return self.mapping[sym]
 247         return []
 248
 249     @staticmethod
 250     def _normalize(symbol):
 251         '''For normal symbols, return the given symbol. For thunks, return
 252         the corresponding normal symbol.'''
 253         if re.match('^_ZThn[0-9]+_', symbol):
 254             return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
 255         return symbol
 256
 257     @staticmethod
 258     def _getSymbols(obj):
 259         '''Returns a list of (symbol, section) contained in the given object
 260         file.'''
 261         proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
 262         (stdout, stderr) = proc.communicate()
 263         syms = []
 264         for line in stdout.splitlines():
 265             # Each line has the following format:
 266             # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
 267             tmp = line.split(' ',1)
 268             # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
 269             # We only need to consider cases where "<section>\t<length> <symbol>" is present,
 270             # and where the [FfO] flag is either F (function) or O (object).
 271             if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
 272                 tmp = tmp[1][8:].split()
 273                 # That gives us ["<section>","<length>", "<symbol>"]
 274                 syms.append((tmp[-1], tmp[0]))
 275         return syms
 276
 277 def print_command(out, args):
 278     print >>out, "Executing: " + " ".join(args)
 279     for tmp in [f for f in args.tmp if os.path.isfile(f)]:
 280         print >>out, tmp + ":"
 281         with open(tmp) as file:
 282             print >>out, "".join(["    " + l for l in file.readlines()])
 283     out.flush()
 284
 285 def main():
 286     parser = OptionParser()
 287     parser.add_option("--depend", dest="depend", metavar="FILE",
 288         help="generate dependencies for the given execution and store it in the given file")
 289     parser.add_option("--target", dest="target", metavar="FILE",
 290         help="designate the target for dependencies")
 291     parser.add_option("--extract", action="store_true", dest="extract",
 292         help="when a library has no descriptor file, extract it first, when possible")
 293     parser.add_option("--uselist", action="store_true", dest="uselist",
 294         help="use a list file for objects when executing a command")
 295     parser.add_option("--verbose", action="store_true", dest="verbose",
 296         help="display executed command and temporary files content")
 297     parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
 298         help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
 299
 300     (options, args) = parser.parse_args()
 301
 302     if not options.target:
 303         options.depend = False
 304     if options.depend:
 305         deps = ExpandLibsDeps(args)
 306         # Filter out common command wrappers
 307         while os.path.basename(deps[0]) in ['ccache', 'distcc']:
 308             deps.pop(0)
 309         # Remove command
 310         deps.pop(0)
 311     with ExpandArgsMore(args) as args:
 312         if options.extract:
 313             args.extract()
 314         if options.symbol_order:
 315             args.orderSymbols(options.symbol_order)
 316         if options.uselist:
 317             args.makelist()
 318
 319         if options.verbose:
 320             print_command(sys.stderr, args)
 321         proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
 322         (stdout, stderr) = proc.communicate()
 323         if proc.returncode and not options.verbose:
 324             print_command(sys.stderr, args)
 325         sys.stderr.write(stdout)
 326         sys.stderr.flush()
 327         if proc.returncode:
 328             exit(proc.returncode)
 329     if not options.depend:
 330         return
 331     ensureParentDir(options.depend)
 332     with open(options.depend, 'w') as depfile:
 333         depfile.write("%s : %s\n" % (options.target, ' '.join(dep for dep in deps if os.path.isfile(dep) and dep != options.target)))
 334
 335         for dep in deps:
 336             if os.path.isfile(dep) and dep != options.target:
 337                 depfile.write("%s :\n" % dep)
 338
 339 if __name__ == '__main__':
 340     main()