Bumping manifests a=b2g-bump
[gecko.git] / config / expandlibs_exec.py
blobc05343022959ad48bebb2eac7d566f0a64ea71d4
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
6 a given command line, and executes that command line with the expanded
7 arguments.
9 With the --extract argument (useful for e.g. $(AR)), it extracts object files
10 from static libraries (or use those listed in library descriptors directly).
12 With the --uselist argument (useful for e.g. $(CC)), it replaces all object
13 files with a list file. This can be used to avoid limitations in the length
14 of a command line. The kind of list file format used depends on the
15 EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
16 or 'linkerscript' for GNU ld linker scripts.
17 See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
19 With the --symbol-order argument, followed by a file name, it will add the
20 relevant linker options to change the order in which the linker puts the
21 symbols appear in the resulting binary. Only works for ELF targets.
22 '''
23 from __future__ import with_statement
24 import sys
25 import os
26 from expandlibs import (
27 ExpandArgs,
28 relativize,
29 isDynamicLib,
30 isObject,
32 import expandlibs_config as conf
33 from optparse import OptionParser
34 import subprocess
35 import tempfile
36 import shutil
37 import subprocess
38 import re
39 from mozbuild.makeutil import Makefile
41 # The are the insert points for a GNU ld linker script, assuming a more
42 # or less "standard" default linker script. This is not a dict because
43 # order is important.
44 SECTION_INSERT_BEFORE = [
45 ('.text', '.fini'),
46 ('.rodata', '.rodata1'),
47 ('.data.rel.ro', '.dynamic'),
48 ('.data', '.data1'),
51 class ExpandArgsMore(ExpandArgs):
52 ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
53 def __enter__(self):
54 self.tmp = []
55 return self
57 def __exit__(self, type, value, tb):
58 '''Automatically remove temporary files'''
59 for tmp in self.tmp:
60 if os.path.isdir(tmp):
61 shutil.rmtree(tmp, True)
62 else:
63 os.remove(tmp)
65 def extract(self):
66 self[0:] = self._extract(self)
68 def _extract(self, args):
69 '''When a static library name is found, either extract its contents
70 in a temporary directory or use the information found in the
71 corresponding lib descriptor.
72 '''
73 ar_extract = conf.AR_EXTRACT.split()
74 newlist = []
76 def lookup(base, f):
77 for root, dirs, files in os.walk(base):
78 if f in files:
79 return os.path.join(root, f)
81 for arg in args:
82 if os.path.splitext(arg)[1] == conf.LIB_SUFFIX:
83 if os.path.exists(arg + conf.LIBS_DESC_SUFFIX):
84 newlist += self._extract(self._expand_desc(arg))
85 continue
86 elif os.path.exists(arg) and (len(ar_extract) or conf.AR == 'lib'):
87 tmp = tempfile.mkdtemp(dir=os.curdir)
88 self.tmp.append(tmp)
89 if conf.AR == 'lib':
90 out = subprocess.check_output([conf.AR, '-NOLOGO', '-LIST', arg])
91 files = out.splitlines()
92 # If lib -list returns a list full of dlls, it's an
93 # import lib.
94 if all(isDynamicLib(f) for f in files):
95 newlist += [arg]
96 continue
97 for f in files:
98 subprocess.call([conf.AR, '-NOLOGO', '-EXTRACT:%s' % f, os.path.abspath(arg)], cwd=tmp)
99 else:
100 subprocess.call(ar_extract + [os.path.abspath(arg)], cwd=tmp)
101 objs = []
102 basedir = os.path.dirname(arg)
103 for root, dirs, files in os.walk(tmp):
104 for f in files:
105 if isObject(f):
106 # If the file extracted from the library also
107 # exists in the directory containing the
108 # library, or one of its subdirectories, use
109 # that instead.
110 maybe_obj = lookup(os.path.join(basedir, os.path.relpath(root, tmp)), f)
111 if maybe_obj:
112 objs.append(relativize(maybe_obj))
113 else:
114 objs.append(relativize(os.path.join(root, f)))
115 newlist += sorted(objs)
116 continue
117 newlist += [arg]
118 return newlist
120 def makelist(self):
121 '''Replaces object file names with a temporary list file, using a
122 list format depending on the EXPAND_LIBS_LIST_STYLE variable
124 objs = [o for o in self if isObject(o)]
125 if not len(objs): return
126 fd, tmp = tempfile.mkstemp(suffix=".list",dir=os.curdir)
127 if conf.EXPAND_LIBS_LIST_STYLE == "linkerscript":
128 content = ['INPUT("%s")\n' % obj for obj in objs]
129 ref = tmp
130 elif conf.EXPAND_LIBS_LIST_STYLE == "filelist":
131 content = ["%s\n" % obj for obj in objs]
132 ref = "-Wl,-filelist," + tmp
133 elif conf.EXPAND_LIBS_LIST_STYLE == "list":
134 content = ["%s\n" % obj for obj in objs]
135 ref = "@" + tmp
136 else:
137 os.close(fd)
138 os.remove(tmp)
139 return
140 self.tmp.append(tmp)
141 f = os.fdopen(fd, "w")
142 f.writelines(content)
143 f.close()
144 idx = self.index(objs[0])
145 newlist = self[0:idx] + [ref] + [item for item in self[idx:] if item not in objs]
146 self[0:] = newlist
148 def _getFoldedSections(self):
149 '''Returns a dict about folded sections.
150 When section A and B are folded into section C, the dict contains:
151 { 'A': 'C',
152 'B': 'C',
153 'C': ['A', 'B'] }'''
154 if not conf.LD_PRINT_ICF_SECTIONS:
155 return {}
157 proc = subprocess.Popen(self + [conf.LD_PRINT_ICF_SECTIONS], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
158 (stdout, stderr) = proc.communicate()
159 result = {}
160 # gold's --print-icf-sections output looks like the following:
161 # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
162 # In terms of words, chances are this will change in the future,
163 # especially considering "into" is misplaced. Splitting on quotes
164 # seems safer.
165 for l in stderr.split('\n'):
166 quoted = l.split("'")
167 if len(quoted) > 5 and quoted[1] != quoted[5]:
168 result[quoted[1]] = [quoted[5]]
169 if quoted[5] in result:
170 result[quoted[5]].append(quoted[1])
171 else:
172 result[quoted[5]] = [quoted[1]]
173 return result
175 def _getOrderedSections(self, ordered_symbols):
176 '''Given an ordered list of symbols, returns the corresponding list
177 of sections following the order.'''
178 if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
179 raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
180 finder = SectionFinder([arg for arg in self if isObject(arg) or os.path.splitext(arg)[1] == conf.LIB_SUFFIX])
181 folded = self._getFoldedSections()
182 sections = set()
183 ordered_sections = []
184 for symbol in ordered_symbols:
185 symbol_sections = finder.getSections(symbol)
186 all_symbol_sections = []
187 for section in symbol_sections:
188 if section in folded:
189 if isinstance(folded[section], str):
190 section = folded[section]
191 all_symbol_sections.append(section)
192 all_symbol_sections.extend(folded[section])
193 else:
194 all_symbol_sections.append(section)
195 for section in all_symbol_sections:
196 if not section in sections:
197 ordered_sections.append(section)
198 sections.add(section)
199 return ordered_sections
201 def orderSymbols(self, order):
202 '''Given a file containing a list of symbols, adds the appropriate
203 argument to make the linker put the symbols in that order.'''
204 with open(order) as file:
205 sections = self._getOrderedSections([l.strip() for l in file.readlines() if l.strip()])
206 split_sections = {}
207 linked_sections = [s[0] for s in SECTION_INSERT_BEFORE]
208 for s in sections:
209 for linked_section in linked_sections:
210 if s.startswith(linked_section):
211 if linked_section in split_sections:
212 split_sections[linked_section].append(s)
213 else:
214 split_sections[linked_section] = [s]
215 break
216 content = []
217 # Order is important
218 linked_sections = [s for s in linked_sections if s in split_sections]
220 if conf.EXPAND_LIBS_ORDER_STYLE == 'section-ordering-file':
221 option = '-Wl,--section-ordering-file,%s'
222 content = sections
223 for linked_section in linked_sections:
224 content.extend(split_sections[linked_section])
225 content.append('%s.*' % linked_section)
226 content.append(linked_section)
228 elif conf.EXPAND_LIBS_ORDER_STYLE == 'linkerscript':
229 option = '-Wl,-T,%s'
230 section_insert_before = dict(SECTION_INSERT_BEFORE)
231 for linked_section in linked_sections:
232 content.append('SECTIONS {')
233 content.append(' %s : {' % linked_section)
234 content.extend(' *(%s)' % s for s in split_sections[linked_section])
235 content.append(' }')
236 content.append('}')
237 content.append('INSERT BEFORE %s' % section_insert_before[linked_section])
238 else:
239 raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
241 fd, tmp = tempfile.mkstemp(dir=os.curdir)
242 f = os.fdopen(fd, "w")
243 f.write('\n'.join(content)+'\n')
244 f.close()
245 self.tmp.append(tmp)
246 self.append(option % tmp)
248 class SectionFinder(object):
249 '''Instances of this class allow to map symbol names to sections in
250 object files.'''
252 def __init__(self, objs):
253 '''Creates an instance, given a list of object files.'''
254 if not conf.EXPAND_LIBS_ORDER_STYLE in ['linkerscript', 'section-ordering-file']:
255 raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf.EXPAND_LIBS_ORDER_STYLE)
256 self.mapping = {}
257 for obj in objs:
258 if not isObject(obj) and os.path.splitext(obj)[1] != conf.LIB_SUFFIX:
259 raise Exception('%s is not an object nor a static library' % obj)
260 for symbol, section in SectionFinder._getSymbols(obj):
261 sym = SectionFinder._normalize(symbol)
262 if sym in self.mapping:
263 if not section in self.mapping[sym]:
264 self.mapping[sym].append(section)
265 else:
266 self.mapping[sym] = [section]
268 def getSections(self, symbol):
269 '''Given a symbol, returns a list of sections containing it or the
270 corresponding thunks. When the given symbol is a thunk, returns the
271 list of sections containing its corresponding normal symbol and the
272 other thunks for that symbol.'''
273 sym = SectionFinder._normalize(symbol)
274 if sym in self.mapping:
275 return self.mapping[sym]
276 return []
278 @staticmethod
279 def _normalize(symbol):
280 '''For normal symbols, return the given symbol. For thunks, return
281 the corresponding normal symbol.'''
282 if re.match('^_ZThn[0-9]+_', symbol):
283 return re.sub('^_ZThn[0-9]+_', '_Z', symbol)
284 return symbol
286 @staticmethod
287 def _getSymbols(obj):
288 '''Returns a list of (symbol, section) contained in the given object
289 file.'''
290 proc = subprocess.Popen(['objdump', '-t', obj], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
291 (stdout, stderr) = proc.communicate()
292 syms = []
293 for line in stdout.splitlines():
294 # Each line has the following format:
295 # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
296 tmp = line.split(' ',1)
297 # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
298 # We only need to consider cases where "<section>\t<length> <symbol>" is present,
299 # and where the [FfO] flag is either F (function) or O (object).
300 if len(tmp) > 1 and len(tmp[1]) > 6 and tmp[1][6] in ['O', 'F']:
301 tmp = tmp[1][8:].split()
302 # That gives us ["<section>","<length>", "<symbol>"]
303 syms.append((tmp[-1], tmp[0]))
304 return syms
306 def print_command(out, args):
307 print >>out, "Executing: " + " ".join(args)
308 for tmp in [f for f in args.tmp if os.path.isfile(f)]:
309 print >>out, tmp + ":"
310 with open(tmp) as file:
311 print >>out, "".join([" " + l for l in file.readlines()])
312 out.flush()
314 def main(args, proc_callback=None):
315 parser = OptionParser()
316 parser.add_option("--extract", action="store_true", dest="extract",
317 help="when a library has no descriptor file, extract it first, when possible")
318 parser.add_option("--uselist", action="store_true", dest="uselist",
319 help="use a list file for objects when executing a command")
320 parser.add_option("--verbose", action="store_true", dest="verbose",
321 help="display executed command and temporary files content")
322 parser.add_option("--symbol-order", dest="symbol_order", metavar="FILE",
323 help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
325 (options, args) = parser.parse_args(args)
327 with ExpandArgsMore(args) as args:
328 if options.extract:
329 args.extract()
330 if options.symbol_order:
331 args.orderSymbols(options.symbol_order)
332 if options.uselist:
333 args.makelist()
335 if options.verbose:
336 print_command(sys.stderr, args)
337 try:
338 proc = subprocess.Popen(args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
339 if proc_callback:
340 proc_callback(proc)
341 except Exception, e:
342 print >>sys.stderr, 'error: Launching', args, ':', e
343 raise e
344 (stdout, stderr) = proc.communicate()
345 if proc.returncode and not options.verbose:
346 print_command(sys.stderr, args)
347 sys.stderr.write(stdout)
348 sys.stderr.flush()
349 if proc.returncode:
350 return proc.returncode
351 return 0
353 if __name__ == '__main__':
354 exit(main(sys.argv[1:]))