1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 '''expandlibs-exec.py applies expandlibs rules, and some more (see below) to
6 a given command line, and executes that command line with the expanded
9 With the --extract argument (useful for e.g. $(AR)), it extracts object files
10 from static libraries (or use those listed in library descriptors directly).
12 With the --uselist argument (useful for e.g. $(CC)), it replaces all object
13 files with a list file. This can be used to avoid limitations in the length
14 of a command line. The kind of list file format used depends on the
15 EXPAND_LIBS_LIST_STYLE variable: 'list' for MSVC style lists (@file.list)
16 or 'linkerscript' for GNU ld linker scripts.
17 See https://bugzilla.mozilla.org/show_bug.cgi?id=584474#c59 for more details.
19 With the --symbol-order argument, followed by a file name, it will add the
20 relevant linker options to change the order in which the linker puts the
21 symbols appear in the resulting binary. Only works for ELF targets.
23 from __future__
import with_statement
26 from expandlibs
import (
34 import expandlibs_config
as conf
35 from optparse
import OptionParser
41 from mozbuild
.makeutil
import Makefile
43 # The are the insert points for a GNU ld linker script, assuming a more
44 # or less "standard" default linker script. This is not a dict because
46 SECTION_INSERT_BEFORE
= [
48 ('.rodata', '.rodata1'),
49 ('.data.rel.ro', '.dynamic'),
53 class ExpandArgsMore(ExpandArgs
):
54 ''' Meant to be used as 'with ExpandArgsMore(args) as ...: '''
59 def __exit__(self
, type, value
, tb
):
60 '''Automatically remove temporary files'''
62 if os
.path
.isdir(tmp
):
63 shutil
.rmtree(tmp
, True)
68 self
[0:] = self
._extract
(self
)
70 def _extract(self
, args
):
71 '''When a static library name is found, either extract its contents
72 in a temporary directory or use the information found in the
73 corresponding lib descriptor.
75 ar_extract
= conf
.AR_EXTRACT
.split()
78 if os
.path
.splitext(arg
)[1] == conf
.LIB_SUFFIX
:
79 if os
.path
.exists(arg
+ conf
.LIBS_DESC_SUFFIX
):
80 newlist
+= self
._extract
(self
._expand
_desc
(arg
))
82 elif os
.path
.exists(arg
) and (len(ar_extract
) or conf
.AR
== 'lib'):
83 tmp
= tempfile
.mkdtemp(dir=os
.curdir
)
86 out
= subprocess
.check_output([conf
.AR
, '-NOLOGO', '-LIST', arg
])
87 files
= out
.splitlines()
88 # If lib -list returns a list full of dlls, it's an
90 if all(isDynamicLib(f
) for f
in files
):
94 subprocess
.call([conf
.AR
, '-NOLOGO', '-EXTRACT:%s' % f
, os
.path
.abspath(arg
)], cwd
=tmp
)
96 subprocess
.call(ar_extract
+ [os
.path
.abspath(arg
)], cwd
=tmp
)
98 for root
, dirs
, files
in os
.walk(tmp
):
99 objs
+= [relativize(os
.path
.join(root
, f
)) for f
in files
if isObject(f
)]
100 newlist
+= sorted(objs
)
106 '''Replaces object file names with a temporary list file, using a
107 list format depending on the EXPAND_LIBS_LIST_STYLE variable
109 objs
= [o
for o
in self
if isObject(o
)]
110 if not len(objs
): return
111 fd
, tmp
= tempfile
.mkstemp(suffix
=".list",dir=os
.curdir
)
112 if conf
.EXPAND_LIBS_LIST_STYLE
== "linkerscript":
113 content
= ['INPUT("%s")\n' % obj
for obj
in objs
]
115 elif conf
.EXPAND_LIBS_LIST_STYLE
== "filelist":
116 content
= ["%s\n" % obj
for obj
in objs
]
117 ref
= "-Wl,-filelist," + tmp
118 elif conf
.EXPAND_LIBS_LIST_STYLE
== "list":
119 content
= ["%s\n" % obj
for obj
in objs
]
126 f
= os
.fdopen(fd
, "w")
127 f
.writelines(content
)
129 idx
= self
.index(objs
[0])
130 newlist
= self
[0:idx
] + [ref
] + [item
for item
in self
[idx
:] if item
not in objs
]
133 def _getFoldedSections(self
):
134 '''Returns a dict about folded sections.
135 When section A and B are folded into section C, the dict contains:
139 if not conf
.LD_PRINT_ICF_SECTIONS
:
142 proc
= subprocess
.Popen(self
+ [conf
.LD_PRINT_ICF_SECTIONS
], stdout
= subprocess
.PIPE
, stderr
= subprocess
.PIPE
)
143 (stdout
, stderr
) = proc
.communicate()
145 # gold's --print-icf-sections output looks like the following:
146 # ld: ICF folding section '.section' in file 'file.o'into '.section' in file 'file.o'
147 # In terms of words, chances are this will change in the future,
148 # especially considering "into" is misplaced. Splitting on quotes
150 for l
in stderr
.split('\n'):
151 quoted
= l
.split("'")
152 if len(quoted
) > 5 and quoted
[1] != quoted
[5]:
153 result
[quoted
[1]] = [quoted
[5]]
154 if quoted
[5] in result
:
155 result
[quoted
[5]].append(quoted
[1])
157 result
[quoted
[5]] = [quoted
[1]]
160 def _getOrderedSections(self
, ordered_symbols
):
161 '''Given an ordered list of symbols, returns the corresponding list
162 of sections following the order.'''
163 if not conf
.EXPAND_LIBS_ORDER_STYLE
in ['linkerscript', 'section-ordering-file']:
164 raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf
.EXPAND_LIBS_ORDER_STYLE
)
165 finder
= SectionFinder([arg
for arg
in self
if isObject(arg
) or os
.path
.splitext(arg
)[1] == conf
.LIB_SUFFIX
])
166 folded
= self
._getFoldedSections
()
168 ordered_sections
= []
169 for symbol
in ordered_symbols
:
170 symbol_sections
= finder
.getSections(symbol
)
171 all_symbol_sections
= []
172 for section
in symbol_sections
:
173 if section
in folded
:
174 if isinstance(folded
[section
], str):
175 section
= folded
[section
]
176 all_symbol_sections
.append(section
)
177 all_symbol_sections
.extend(folded
[section
])
179 all_symbol_sections
.append(section
)
180 for section
in all_symbol_sections
:
181 if not section
in sections
:
182 ordered_sections
.append(section
)
183 sections
.add(section
)
184 return ordered_sections
186 def orderSymbols(self
, order
):
187 '''Given a file containing a list of symbols, adds the appropriate
188 argument to make the linker put the symbols in that order.'''
189 with
open(order
) as file:
190 sections
= self
._getOrderedSections
([l
.strip() for l
in file.readlines() if l
.strip()])
192 linked_sections
= [s
[0] for s
in SECTION_INSERT_BEFORE
]
194 for linked_section
in linked_sections
:
195 if s
.startswith(linked_section
):
196 if linked_section
in split_sections
:
197 split_sections
[linked_section
].append(s
)
199 split_sections
[linked_section
] = [s
]
203 linked_sections
= [s
for s
in linked_sections
if s
in split_sections
]
205 if conf
.EXPAND_LIBS_ORDER_STYLE
== 'section-ordering-file':
206 option
= '-Wl,--section-ordering-file,%s'
208 for linked_section
in linked_sections
:
209 content
.extend(split_sections
[linked_section
])
210 content
.append('%s.*' % linked_section
)
211 content
.append(linked_section
)
213 elif conf
.EXPAND_LIBS_ORDER_STYLE
== 'linkerscript':
215 section_insert_before
= dict(SECTION_INSERT_BEFORE
)
216 for linked_section
in linked_sections
:
217 content
.append('SECTIONS {')
218 content
.append(' %s : {' % linked_section
)
219 content
.extend(' *(%s)' % s
for s
in split_sections
[linked_section
])
222 content
.append('INSERT BEFORE %s' % section_insert_before
[linked_section
])
224 raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf
.EXPAND_LIBS_ORDER_STYLE
)
226 fd
, tmp
= tempfile
.mkstemp(dir=os
.curdir
)
227 f
= os
.fdopen(fd
, "w")
228 f
.write('\n'.join(content
)+'\n')
231 self
.append(option
% tmp
)
233 class SectionFinder(object):
234 '''Instances of this class allow to map symbol names to sections in
237 def __init__(self
, objs
):
238 '''Creates an instance, given a list of object files.'''
239 if not conf
.EXPAND_LIBS_ORDER_STYLE
in ['linkerscript', 'section-ordering-file']:
240 raise Exception('EXPAND_LIBS_ORDER_STYLE "%s" is not supported' % conf
.EXPAND_LIBS_ORDER_STYLE
)
243 if not isObject(obj
) and os
.path
.splitext(obj
)[1] != conf
.LIB_SUFFIX
:
244 raise Exception('%s is not an object nor a static library' % obj
)
245 for symbol
, section
in SectionFinder
._getSymbols
(obj
):
246 sym
= SectionFinder
._normalize
(symbol
)
247 if sym
in self
.mapping
:
248 if not section
in self
.mapping
[sym
]:
249 self
.mapping
[sym
].append(section
)
251 self
.mapping
[sym
] = [section
]
253 def getSections(self
, symbol
):
254 '''Given a symbol, returns a list of sections containing it or the
255 corresponding thunks. When the given symbol is a thunk, returns the
256 list of sections containing its corresponding normal symbol and the
257 other thunks for that symbol.'''
258 sym
= SectionFinder
._normalize
(symbol
)
259 if sym
in self
.mapping
:
260 return self
.mapping
[sym
]
264 def _normalize(symbol
):
265 '''For normal symbols, return the given symbol. For thunks, return
266 the corresponding normal symbol.'''
267 if re
.match('^_ZThn[0-9]+_', symbol
):
268 return re
.sub('^_ZThn[0-9]+_', '_Z', symbol
)
272 def _getSymbols(obj
):
273 '''Returns a list of (symbol, section) contained in the given object
275 proc
= subprocess
.Popen(['objdump', '-t', obj
], stdout
= subprocess
.PIPE
, stderr
= subprocess
.PIPE
)
276 (stdout
, stderr
) = proc
.communicate()
278 for line
in stdout
.splitlines():
279 # Each line has the following format:
280 # <addr> [lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>
281 tmp
= line
.split(' ',1)
282 # This gives us ["<addr>", "[lgu!][w ][C ][W ][Ii ][dD ][FfO ] <section>\t<length> <symbol>"]
283 # We only need to consider cases where "<section>\t<length> <symbol>" is present,
284 # and where the [FfO] flag is either F (function) or O (object).
285 if len(tmp
) > 1 and len(tmp
[1]) > 6 and tmp
[1][6] in ['O', 'F']:
286 tmp
= tmp
[1][8:].split()
287 # That gives us ["<section>","<length>", "<symbol>"]
288 syms
.append((tmp
[-1], tmp
[0]))
291 def print_command(out
, args
):
292 print >>out
, "Executing: " + " ".join(args
)
293 for tmp
in [f
for f
in args
.tmp
if os
.path
.isfile(f
)]:
294 print >>out
, tmp
+ ":"
295 with
open(tmp
) as file:
296 print >>out
, "".join([" " + l
for l
in file.readlines()])
300 parser
= OptionParser()
301 parser
.add_option("--depend", dest
="depend", metavar
="FILE",
302 help="generate dependencies for the given execution and store it in the given file")
303 parser
.add_option("--target", dest
="target", metavar
="FILE",
304 help="designate the target for dependencies")
305 parser
.add_option("--extract", action
="store_true", dest
="extract",
306 help="when a library has no descriptor file, extract it first, when possible")
307 parser
.add_option("--uselist", action
="store_true", dest
="uselist",
308 help="use a list file for objects when executing a command")
309 parser
.add_option("--verbose", action
="store_true", dest
="verbose",
310 help="display executed command and temporary files content")
311 parser
.add_option("--symbol-order", dest
="symbol_order", metavar
="FILE",
312 help="use the given list of symbols to order symbols in the resulting binary when using with a linker")
314 (options
, args
) = parser
.parse_args()
316 if not options
.target
:
317 options
.depend
= False
319 deps
= ExpandLibsDeps(args
)
320 # Filter out common command wrappers
321 while os
.path
.basename(deps
[0]) in ['ccache', 'distcc']:
325 with
ExpandArgsMore(args
) as args
:
328 if options
.symbol_order
:
329 args
.orderSymbols(options
.symbol_order
)
334 print_command(sys
.stderr
, args
)
336 proc
= subprocess
.Popen(args
, stdout
= subprocess
.PIPE
, stderr
= subprocess
.STDOUT
)
338 print >>sys
.stderr
, 'error: Launching', args
, ':', e
340 (stdout
, stderr
) = proc
.communicate()
341 if proc
.returncode
and not options
.verbose
:
342 print_command(sys
.stderr
, args
)
343 sys
.stderr
.write(stdout
)
346 exit(proc
.returncode
)
347 if not options
.depend
:
349 ensureParentDir(options
.depend
)
351 deps
= [dep
for dep
in deps
if os
.path
.isfile(dep
) and dep
!= options
.target
352 and os
.path
.abspath(dep
) != os
.path
.abspath(options
.depend
)]
353 no_dynamic_lib
= [dep
for dep
in deps
if not isDynamicLib(dep
)]
354 mk
.create_rule([options
.target
]).add_dependencies(no_dynamic_lib
)
355 if len(deps
) != len(no_dynamic_lib
):
356 mk
.create_rule(['%s_order_only' % options
.target
]).add_dependencies(dep
for dep
in deps
if isDynamicLib(dep
))
358 with
open(options
.depend
, 'w') as depfile
:
359 mk
.dump(depfile
, removal_guard
=True)
361 if __name__
== '__main__':