1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 # This script is used to capture the content of config.status-generated
6 # files and subsequently restore their timestamp if they haven't changed.
17 import mozpack
.path
as mozpath
21 def __new__(cls
, size
):
23 import multiprocessing
24 size
= min(size
, multiprocessing
.cpu_count())
25 return multiprocessing
.Pool(size
)
27 return super(Pool
, cls
).__new
__(cls
)
29 def imap_unordered(self
, fn
, iterable
):
30 return itertools
.imap(fn
, iterable
)
40 def __init__(self
, path
):
42 self
._content
= open(path
, 'rb').read()
44 self
._times
= (stat
.st_atime
, stat
.st_mtime
)
56 '''Returns whether the file was modified since the instance was
57 created. Result is memoized.'''
58 if hasattr(self
, '_modified'):
62 if os
.path
.exists(self
._path
):
63 if open(self
._path
, 'rb').read() == self
._content
:
65 self
._modified
= modified
68 def update_time(self
):
69 '''If the file hasn't changed since the instance was created,
70 restore its old modification time.'''
72 os
.utime(self
._path
, self
._times
)
75 # As defined in the various sub-configures in the tree
94 CONFIGURE_DATA
= 'configure.pkl'
97 # Autoconf, in some of the sub-configures used in the tree, likes to error
98 # out when "precious" variables change in value. The solution it gives to
99 # straighten things is to either run make distclean or remove config.cache.
100 # There's no reason not to do the latter automatically instead of failing,
101 # doing the cleanup (which, on buildbots means a full clobber), and
102 # restarting from scratch.
103 def maybe_clear_cache(data
):
104 env
= dict(data
['env'])
105 for kind
in ('target', 'host', 'build'):
108 env
['%s_alias' % kind
] = arg
109 # configure can take variables assignments in its arguments, and that
110 # overrides whatever is in the environment.
111 for arg
in data
['args']:
112 if arg
[:1] != '-' and '=' in arg
:
113 key
, value
= arg
.split('=', 1)
116 comment
= re
.compile(r
'^\s+#')
118 with
open(data
['cache-file']) as f
:
120 if not comment
.match(line
) and '=' in line
:
121 key
, value
= line
.rstrip(os
.linesep
).split('=', 1)
122 # If the value is quoted, unquote it
124 value
= value
[1:-1].replace("'\\''", "'")
126 for precious
in PRECIOUS_VARS
:
127 # If there is no entry at all for that precious variable, then
128 # its value is not precious for that particular configure.
129 if 'ac_cv_env_%s_set' % precious
not in cache
:
131 is_set
= cache
.get('ac_cv_env_%s_set' % precious
) == 'set'
132 value
= cache
.get('ac_cv_env_%s_value' % precious
) if is_set
else None
133 if value
!= env
.get(precious
):
134 print 'Removing %s because of %s value change from:' \
135 % (data
['cache-file'], precious
)
136 print ' %s' % (value
if value
is not None else 'undefined')
138 print ' %s' % env
.get(precious
, 'undefined')
139 os
.remove(data
['cache-file'])
144 def split_template(s
):
145 """Given a "file:template" string, returns "file", "template". If the string
146 is of the form "file" (without a template), returns "file", "file.in"."""
148 return s
.split(':', 1)
149 return s
, '%s.in' % s
152 def get_config_files(data
):
153 config_status
= mozpath
.join(data
['objdir'], 'config.status')
154 if not os
.path
.exists(config_status
):
157 configure
= mozpath
.join(data
['srcdir'], 'configure')
161 # Scan the config.status output for information about configuration files
163 config_status_output
= subprocess
.check_output(
164 [data
['shell'], '-c', '%s --help' % config_status
],
165 stderr
=subprocess
.STDOUT
).splitlines()
167 for line
in config_status_output
:
168 if line
.startswith('Configuration') and line
.endswith(':'):
169 if line
.endswith('commands:'):
173 elif not line
.strip():
176 for f
, t
in (split_template(couple
) for couple
in line
.split()):
177 f
= mozpath
.join(data
['objdir'], f
)
178 t
= mozpath
.join(data
['srcdir'], t
)
179 if state
== 'commands':
180 command_files
.append(f
)
182 config_files
.append((f
, t
))
184 return config_files
, command_files
187 def prepare(srcdir
, objdir
, shell
, args
):
188 parser
= argparse
.ArgumentParser()
189 parser
.add_argument('--target', type=str)
190 parser
.add_argument('--host', type=str)
191 parser
.add_argument('--build', type=str)
192 parser
.add_argument('--cache-file', type=str)
193 # The --srcdir argument is simply ignored. It's a useless autoconf feature
194 # that we don't support well anyways. This makes it stripped from `others`
195 # and allows to skip setting it when calling the subconfigure (configure
196 # will take it from the configure path anyways).
197 parser
.add_argument('--srcdir', type=str)
199 data_file
= os
.path
.join(objdir
, CONFIGURE_DATA
)
201 if os
.path
.exists(data_file
):
202 with
open(data_file
, 'rb') as f
:
203 data
= pickle
.load(f
)
204 previous_args
= data
['args']
206 # Msys likes to break environment variables and command line arguments,
207 # so read those from stdin, as they are passed from the configure script
208 # when necessary (on windows).
209 # However, for some reason, $PATH is not handled like other environment
210 # variables, and msys remangles it even when giving it is already a msys
211 # $PATH. Fortunately, the mangling/demangling is just find for $PATH, so
212 # we can just take the value from the environment. Msys will convert it
213 # back properly when calling subconfigure.
214 input = sys
.stdin
.read()
216 data
= {a
: b
for [a
, b
] in eval(input)}
217 environ
= {a
: b
for a
, b
in data
['env']}
218 environ
['PATH'] = os
.environ
['PATH']
223 args
, others
= parser
.parse_known_args(args
)
226 'target': args
.target
,
236 data
['cache-file'] = mozpath
.normpath(mozpath
.join(os
.getcwd(),
239 data
['cache-file'] = mozpath
.join(objdir
, 'config.cache')
241 if previous_args
is not None:
242 data
['previous-args'] = previous_args
247 if e
.errno
!= errno
.EEXIST
:
250 with
open(data_file
, 'wb') as f
:
254 def prefix_lines(text
, prefix
):
255 return ''.join('%s> %s' % (prefix
, line
) for line
in text
.splitlines(True))
262 with
open(os
.path
.join(objdir
, CONFIGURE_DATA
), 'rb') as f
:
263 data
= pickle
.load(f
)
265 data
['objdir'] = objdir
267 cache_file
= data
['cache-file']
269 if os
.path
.exists(cache_file
):
270 cleared_cache
= maybe_clear_cache(data
)
272 config_files
, command_files
= get_config_files(data
)
274 for f
, t
in config_files
:
275 contents
.append(File(f
))
277 # AC_CONFIG_COMMANDS actually only registers tags, not file names
278 # but most commands are tagged with the file name they create.
279 # However, a few don't, or are tagged with a directory name (and their
280 # command is just to create that directory)
281 for f
in command_files
:
282 if os
.path
.isfile(f
):
283 contents
.append(File(f
))
285 # Only run configure if one of the following is true:
286 # - config.status doesn't exist
287 # - config.status is older than configure
288 # - the configure arguments changed
289 # - the environment changed in a way that requires a cache clear.
290 configure
= mozpath
.join(data
['srcdir'], 'configure')
291 config_status_path
= mozpath
.join(objdir
, 'config.status')
292 skip_configure
= True
293 if not os
.path
.exists(config_status_path
):
294 skip_configure
= False
297 config_status
= File(config_status_path
)
298 if config_status
.mtime
< os
.path
.getmtime(configure
) or \
299 data
.get('previous-args', data
['args']) != data
['args'] or \
301 skip_configure
= False
303 relobjdir
= os
.path
.relpath(objdir
, os
.getcwd())
305 if not skip_configure
:
306 command
= [data
['shell'], configure
]
307 for kind
in ('target', 'build', 'host'):
308 if data
.get(kind
) is not None:
309 command
+= ['--%s=%s' % (kind
, data
[kind
])]
310 command
+= data
['args']
311 command
+= ['--cache-file=%s' % cache_file
]
313 # Pass --no-create to configure so that it doesn't run config.status.
314 # We're going to run it ourselves.
315 command
+= ['--no-create']
317 print prefix_lines('configuring', relobjdir
)
318 print prefix_lines('running %s' % ' '.join(command
[:-1]), relobjdir
)
321 output
+= subprocess
.check_output(command
,
322 stderr
=subprocess
.STDOUT
, cwd
=objdir
, env
=data
['env'])
323 except subprocess
.CalledProcessError
as e
:
324 return relobjdir
, e
.returncode
, e
.output
326 # Leave config.status with a new timestamp if configure is newer than
327 # its original mtime.
328 if config_status
and os
.path
.getmtime(configure
) <= config_status
.mtime
:
329 config_status
.update_time()
331 # Only run config.status if one of the following is true:
332 # - config.status changed or did not exist
333 # - one of the templates for config files is newer than the corresponding
335 skip_config_status
= True
336 if not config_status
or config_status
.modified
:
337 # If config.status doesn't exist after configure (because it's not
338 # an autoconf configure), skip it.
339 if os
.path
.exists(config_status_path
):
340 skip_config_status
= False
342 # config.status changed or was created, so we need to update the
343 # list of config and command files.
344 config_files
, command_files
= get_config_files(data
)
345 for f
, t
in config_files
:
346 if not os
.path
.exists(t
) or \
347 os
.path
.getmtime(f
) < os
.path
.getmtime(t
):
348 skip_config_status
= False
350 if not skip_config_status
:
352 print prefix_lines('running config.status', relobjdir
)
355 output
+= subprocess
.check_output([data
['shell'], '-c',
356 './config.status'], stderr
=subprocess
.STDOUT
, cwd
=objdir
,
358 except subprocess
.CalledProcessError
as e
:
365 return relobjdir
, ret
, output
368 def subconfigure(args
):
369 parser
= argparse
.ArgumentParser()
370 parser
.add_argument('--list', type=str,
371 help='File containing a list of subconfigures to run')
372 parser
.add_argument('--skip', type=str,
373 help='File containing a list of Subconfigures to skip')
374 parser
.add_argument('subconfigures', type=str, nargs
='*',
375 help='Subconfigures to run if no list file is given')
376 args
, others
= parser
.parse_known_args(args
)
377 subconfigures
= args
.subconfigures
379 subconfigures
.extend(open(args
.list, 'rb').read().splitlines())
381 skips
= set(open(args
.skip
, 'rb').read().splitlines())
382 subconfigures
= [s
for s
in subconfigures
if s
not in skips
]
384 if not subconfigures
:
388 # One would think using a ThreadPool would be faster, considering
389 # everything happens in subprocesses anyways, but no, it's actually
390 # slower on Windows. (20s difference overall!)
391 pool
= Pool(len(subconfigures
))
392 for relobjdir
, returncode
, output
in \
393 pool
.imap_unordered(run
, subconfigures
):
394 print prefix_lines(output
, relobjdir
)
396 ret
= max(returncode
, ret
)
405 if args
[0] != '--prepare':
406 return subconfigure(args
)
408 topsrcdir
= os
.path
.abspath(args
[1])
410 # subdir can be of the form srcdir:objdir
412 srcdir
, subdir
= subdir
.split(':', 1)
415 srcdir
= os
.path
.join(topsrcdir
, srcdir
)
416 objdir
= os
.path
.abspath(subdir
)
418 return prepare(srcdir
, objdir
, args
[3], args
[4:])
421 if __name__
== '__main__':
422 sys
.exit(main(sys
.argv
[1:]))