config/expandlibs_exec.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
   6 a given command line, and executes that command line with the expanded
   7 arguments.
   8
   9 With the --extract argument (useful for e.g. $(AR)), it extracts object files
  10 from static libraries (or use those listed in library descriptors directly).
  11
  12 With the --uselist argument (useful for e.g. $(CC)), it replaces all object
  13 files with a list file. This can be used to avoid limitations in the length
  14 of a command line. The kind of list file format used depends on the
  15 EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
  16 or 'linkerscript' for GNU ld linker scripts.
  17 See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
  18
  19 With the --symbol-order argument, followed by a file name, it will add the
  20 relevant linker options to change the order in which the linker puts the
  21 symbols appear in the resulting binary. Only works for ELF targets.
  22 '''
  23 from __future__ import with_statement
  24 import sys
  25 import os
  26 from expandlibs import ExpandArgs, relativize, isObject, ensureParentDir, ExpandLibsDeps
  27 import expandlibs_config as conf
  28 from optparse import OptionParser
  29 import subprocess
  30 import tempfile
  31 import shutil
  32 import subprocess
  33 import re
  34
  35 # The are the insert points for a GNU ld linker script, assuming a more
  36 # or less "standard" default linker script. This is not a dict because
  37 # order is important.
  38 SECTION_INSERT_BEFORE = [
  39   ('.text', '.fini'),
  40   ('.rodata', '.rodata1'),
  41   ('.data.rel.ro', '.dynamic'),
  42   ('.data', '.data1'),
  43 ]
  44
  45 class ExpandArgsMore(ExpandArgs):
  46     ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
  47     def __enter__(self):
  48         self.tmp = []
  49         return self
  50
  51     def __exit__(self, type, value, tb):
  52         '''Automatically remove temporary files'''
  53         for tmp in self.tmp:
  54             if os.path.isdir(tmp):
  55                 shutil.rmtree(tmp, True)
  56             else:
  57                 os.remove(tmp)
  58
  59     def extract(self):
  60         self[0:] = self._extract(self)
  61
  62     def _extract(self, args):
  63         '''When a static library name is found, either extract its contents
  64         in a temporary directory or use the information found in the
  65         corresponding lib descriptor.
  66         '''
  67         ar_extract = conf.AR_EXTRACT.split()
  68         newlist = []
  69         for arg in args:
  70             if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
  71                 if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
  72                     newlist += self._extract(self._expand_desc(arg))
  73                 elif os.path.exists(arg) and len(ar_extract):
  74                     tmp = tempfile.mkdtemp(dir=os.curdir)
  75                     self.tmp.append(tmp)
  76                     subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)
  77                     objs = []
  78                     for root, dirs, files in os.walk(tmp):
  79                         objs += [relativize(os.path.join(root, f)) for f in files if isObject(f)]
  80                     newlist += objs
  81                 else:
  82                     newlist += [arg]
  83             else:
  84                 newlist += [arg]
  85         return newlist
  86
  87     def makelist(self):
  88         '''Replaces object file names with a temporary list file, using a
  89         list format depending on the EXPAND_LIBS_LIST_STYLE variable
  90         '''
  91         objs = [o for o in self if isObject(o)]
  92         if not len(objs): return
  93         fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)
  94         if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":
  95             content = ['INPUT("%s")\n' % obj for obj in objs]
  96             ref = tmp
  97         elif conf.EXPAND_LIBS_LIST_STYLE == "list":
  98             content = ["%s\n" % obj for obj in objs]
  99             ref = "@" + tmp
 100         else:
 101             os.close(fd)
 102             os.remove(tmp)
 103             return
 104         self.tmp.append(tmp)
 105         f = os.fdopen(fd, "w")
 106         f.writelines(content)
 107         f.close()
 108         idx = self.index(objs[0])
 109         newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]
 110         self[0:] = newlist
 111
 112     def _getFoldedSections(self):
 113         '''Returns a dict about folded sections.
 114         When section A and B are folded into section C, the dict contains:
 115         { 'A': 'C',
 116           'B': 'C',
 117           'C': ['A', 'B'] }'''
 118         if not conf.LD_PRINT_ICF_SECTIONS:
 119             return {}
 120
 121         proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
 122         (stdout, stderr) = proc.communicate()
 123         result = {}
 124         # gold's --print-icf-sections output looks like the following:
 125         # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
 126         # In terms of words, chances are this will change in the future,
 127         # especially considering "into" is misplaced. Splitting on quotes
 128         # seems safer.
 129         for l in stderr.split('\n'):
 130             quoted = l.split("'")
 131             if len(quoted) > 5 and quoted[1] != quoted[5]:
 132                 result[quoted[1]] = quoted[5]
 133                 if quoted[5] in result:
 134                     result[quoted[5]].append(quoted[1])
 135                 else:
 136                     result[quoted[5]] = [quoted[1]]
 137         return result
 138
 139     def _getOrderedSections(self, ordered_symbols):
 140         '''Given an ordered list of symbols, returns the corresponding list
 141         of sections following the order.'''
 142         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
 143             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 144         finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
 145         folded = self._getFoldedSections()
 146         sections = set()
 147         ordered_sections = []
 148         for symbol in ordered_symbols:
 149             symbol_sections = finder.getSections(symbol)
 150             all_symbol_sections = []
 151             for section in symbol_sections:
 152                 if section in folded:
 153                     if isinstance(folded[section], str):
 154                         section = folded[section]
 155                     all_symbol_sections.append(section)
 156                     all_symbol_sections.extend(folded[section])
 157                 else:
 158                     all_symbol_sections.append(section)
 159             for section in all_symbol_sections:
 160                 if not section in sections:
 161                     ordered_sections.append(section)
 162                     sections.add(section)
 163         return ordered_sections
 164
 165     def orderSymbols(self, order):
 166         '''Given a file containing a list of symbols, adds the appropriate
 167         argument to make the linker put the symbols in that order.'''
 168         with open(order) as file:
 169             sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
 170         split_sections = {}
 171         linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
 172         for s in sections:
 173             for linked_section in linked_sections:
 174                 if s.startswith(linked_section):
 175                     if linked_section in split_sections:
 176                         split_sections[linked_section].append(s)
 177                     else:
 178                         split_sections[linked_section] = [s]
 179                     break
 180         content = []
 181         # Order is important
 182         linked_sections = [s for s in linked_sections if s in split_sections]
 183
 184         if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
 185             option = '-Wl,--section-ordering-file,%s'
 186             content = sections
 187             for linked_section in linked_sections:
 188                 content.extend(split_sections[linked_section])
 189                 content.append('%s.*' % linked_section)
 190                 content.append(linked_section)
 191
 192         elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
 193             option = '-Wl,-T,%s'
 194             section_insert_before = dict(SECTION_INSERT_BEFORE)
 195             for linked_section in linked_sections:
 196                 content.append('SECTIONS {')
 197                 content.append('  %s : {' % linked_section)
 198                 content.extend('    *(%s)' % s for s in split_sections[linked_section])
 199                 content.append('  }')
 200                 content.append('}')
 201                 content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
 202         else:
 203             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 204
 205         fd, tmp = tempfile.mkstemp(dir=os.curdir)
 206         f = os.fdopen(fd, "w")
 207         f.write('\n'.join(content)+'\n')
 208         f.close()
 209         self.tmp.append(tmp)
 210         self.append(option % tmp)
 211
 212 class SectionFinder(object):
 213     '''Instances of this class allow to map symbol names to sections in
 214     object files.'''
 215
 216     def __init__(self, objs):
 217         '''Creates an instance, given a list of object files.'''
 218         if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
 219             raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
 220         self.mapping = {}
 221         for obj in objs:
 222             if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
 223                 raise Exception('%s is not an object nor a static library' % obj)
 224             for symbol, section in SectionFinder._getSymbols(obj):
 225                 sym = SectionFinder._normalize(symbol)
 226                 if sym in self.mapping:
 227                     if not section in self.mapping[sym]:
 228                         self.mapping[sym].append(section)
 229                 else:
 230                     self.mapping[sym] = [section]
 231
 232     def getSections(self, symbol):
 233         '''Given a symbol, returns a list of sections containing it or the
 234         corresponding thunks. When the given symbol is a thunk, returns the
 235         list of sections containing its corresponding normal symbol and the
 236         other thunks for that symbol.'''
 237         sym = SectionFinder._normalize(symbol)
 238         if sym in self.mapping:
 239             return self.mapping[sym]
 240         return []
 241
 242     @staticmethod
 243     def _normalize(symbol):
 244         '''For normal symbols, return the given symbol. For thunks, return
 245         the corresponding normal symbol.'''
 246         if re.match('^_ZThn[0-9]+_', symbol):
 247             return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
 248         return symbol
 249
 250     @staticmethod
 251     def _getSymbols(obj):
 252         '''Returns a list of (symbol, section) contained in the given object
 253         file.'''
 254         proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
 255         (stdout, stderr) = proc.communicate()
 256         syms = []
 257         for line in stdout.splitlines():
 258             # Each line has the following format:
 259             # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
 260             tmp = line.split(' ',1)
 261             # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
 262             # We only need to consider cases where "<section>\t<length> <symbol>" is present,
 263             # and where the [FfO] flag is either F (function) or O (object).
 264             if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
 265                 tmp = tmp[1][8:].split()
 266                 # That gives us ["<section>","<length>", "<symbol>"]
 267                 syms.append((tmp[-1], tmp[0]))
 268         return syms
 269
 270 def main():
 271     parser = OptionParser()
 272     parser.add_option("--depend", dest="depend", metavar="FILE",
 273         help="generate dependencies for the given execution and store it in the given file")
 274     parser.add_option("--target", dest="target", metavar="FILE",
 275         help="designate the target for dependencies")
 276     parser.add_option("--extract", action="store_true", dest="extract",
 277         help="when a library has no descriptor file, extract it first, when possible")
 278     parser.add_option("--uselist", action="store_true", dest="uselist",
 279         help="use a list file for objects when executing a command")
 280     parser.add_option("--verbose", action="store_true", dest="verbose",
 281         help="display executed command and temporary files content")
 282     parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
 283         help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
 284
 285     (options, args) = parser.parse_args()
 286
 287     if not options.target:
 288         options.depend = False
 289     if options.depend:
 290         deps = ExpandLibsDeps(args)
 291         # Filter out common command wrappers
 292         while os.path.basename(deps[0]) in ['ccache', 'distcc']:
 293             deps.pop(0)
 294         # Remove command
 295         deps.pop(0)
 296     with ExpandArgsMore(args) as args:
 297         if options.extract:
 298             args.extract()
 299         if options.symbol_order:
 300             args.orderSymbols(options.symbol_order)
 301         if options.uselist:
 302             args.makelist()
 303
 304         if options.verbose:
 305             print >>sys.stderr, "Executing: " + " ".join(args)
 306             for tmp in [f for f in args.tmp if os.path.isfile(f)]:
 307                 print >>sys.stderr, tmp + ":"
 308                 with open(tmp) as file:
 309                     print >>sys.stderr, "".join(["    " + l for l in file.readlines()])
 310             sys.stderr.flush()
 311         ret = subprocess.call(args)
 312         if ret:
 313             exit(ret)
 314     if not options.depend:
 315         return
 316     ensureParentDir(options.depend)
 317     with open(options.depend, 'w') as depfile:
 318         depfile.write("%s : %s\n" % (options.target, ' '.join(dep for dep in deps if os.path.isfile(dep) and dep != options.target)))
 319
 320
 321 if __name__ == '__main__':
 322     main()