build/subconfigure.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # This script is used to capture the content of config.status-generated
   6 # files and subsequently restore their timestamp if they haven't changed.
   7
   8 import argparse
   9 import errno
  10 import os
  11 import re
  12 import subprocess
  13 import sys
  14 import pickle
  15
  16 import mozpack.path as mozpath
  17
  18 class File(object):
  19     def __init__(self, path):
  20         self._path = path
  21         self._content = open(path, 'rb').read()
  22         stat = os.stat(path)
  23         self._times = (stat.st_atime, stat.st_mtime)
  24
  25     @property
  26     def path(self):
  27         return self._path
  28
  29     @property
  30     def mtime(self):
  31         return self._times[1]
  32
  33     @property
  34     def modified(self):
  35         '''Returns whether the file was modified since the instance was
  36         created. Result is memoized.'''
  37         if hasattr(self, '_modified'):
  38             return self._modified
  39
  40         modified = True
  41         if os.path.exists(self._path):
  42             if open(self._path, 'rb').read() == self._content:
  43                 modified = False
  44         self._modified = modified
  45         return modified
  46
  47     def update_time(self):
  48         '''If the file hasn't changed since the instance was created,
  49            restore its old modification time.'''
  50         if not self.modified:
  51             os.utime(self._path, self._times)
  52
  53
  54 # As defined in the various sub-configures in the tree
  55 PRECIOUS_VARS = set([
  56     'build_alias',
  57     'host_alias',
  58     'target_alias',
  59     'CC',
  60     'CFLAGS',
  61     'LDFLAGS',
  62     'LIBS',
  63     'CPPFLAGS',
  64     'CPP',
  65     'CCC',
  66     'CXXFLAGS',
  67     'CXX',
  68     'CCASFLAGS',
  69     'CCAS',
  70 ])
  71
  72
  73 # Autoconf, in some of the sub-configures used in the tree, likes to error
  74 # out when "precious" variables change in value. The solution it gives to
  75 # straighten things is to either run make distclean or remove config.cache.
  76 # There's no reason not to do the latter automatically instead of failing,
  77 # doing the cleanup (which, on buildbots means a full clobber), and
  78 # restarting from scratch.
  79 def maybe_clear_cache(data):
  80     env = dict(data['env'])
  81     for kind in ('target', 'host', 'build'):
  82         arg = data[kind]
  83         if arg is not None:
  84             env['%s_alias' % kind] = arg
  85     # configure can take variables assignments in its arguments, and that
  86     # overrides whatever is in the environment.
  87     for arg in data['args']:
  88         if arg[:1] != '-' and '=' in arg:
  89             key, value = arg.split('=', 1)
  90             env[key] = value
  91
  92     comment = re.compile(r'^\s+#')
  93     cache = {}
  94     with open(data['cache-file']) as f:
  95         for line in f:
  96             if not comment.match(line) and '=' in line:
  97                 key, value = line.rstrip(os.linesep).split('=', 1)
  98                 # If the value is quoted, unquote it
  99                 if value[:1] == "'":
 100                     value = value[1:-1].replace("'\\''", "'")
 101                 cache[key] = value
 102     for precious in PRECIOUS_VARS:
 103         # If there is no entry at all for that precious variable, then
 104         # its value is not precious for that particular configure.
 105         if 'ac_cv_env_%s_set' % precious not in cache:
 106             continue
 107         is_set = cache.get('ac_cv_env_%s_set' % precious) == 'set'
 108         value = cache.get('ac_cv_env_%s_value' % precious) if is_set else None
 109         if value != env.get(precious):
 110             print 'Removing %s because of %s value change from:' \
 111                 % (data['cache-file'], precious)
 112             print '  %s' % (value if value is not None else 'undefined')
 113             print 'to:'
 114             print '  %s' % env.get(precious, 'undefined')
 115             os.remove(data['cache-file'])
 116             return True
 117     return False
 118
 119
 120 def split_template(s):
 121     """Given a "file:template" string, returns "file", "template". If the string
 122     is of the form "file" (without a template), returns "file", "file.in"."""
 123     if ':' in s:
 124         return s.split(':', 1)
 125     return s, '%s.in' % s
 126
 127
 128 def get_config_files(data):
 129     config_status = mozpath.join(data['objdir'], 'config.status')
 130     if not os.path.exists(config_status):
 131         return [], []
 132
 133     configure = mozpath.join(data['srcdir'], 'configure')
 134     config_files = []
 135     command_files = []
 136
 137     # Scan the config.status output for information about configuration files
 138     # it generates.
 139     config_status_output = subprocess.check_output(
 140         [data['shell'], '-c', '%s --help' % config_status],
 141         stderr=subprocess.STDOUT).splitlines()
 142     state = None
 143     for line in config_status_output:
 144         if line.startswith('Configuration') and line.endswith(':'):
 145             if line.endswith('commands:'):
 146                 state = 'commands'
 147             else:
 148                 state = 'config'
 149         elif not line.strip():
 150             state = None
 151         elif state:
 152             for f, t in (split_template(couple) for couple in line.split()):
 153                 f = mozpath.join(data['objdir'], f)
 154                 t = mozpath.join(data['srcdir'], t)
 155                 if state == 'commands':
 156                     command_files.append(f)
 157                 else:
 158                     config_files.append((f, t))
 159
 160     return config_files, command_files
 161
 162
 163 def prepare(data_file, srcdir, objdir, shell, args):
 164     parser = argparse.ArgumentParser()
 165     parser.add_argument('--target', type=str)
 166     parser.add_argument('--host', type=str)
 167     parser.add_argument('--build', type=str)
 168     parser.add_argument('--cache-file', type=str)
 169     # The --srcdir argument is simply ignored. It's a useless autoconf feature
 170     # that we don't support well anyways. This makes it stripped from `others`
 171     # and allows to skip setting it when calling the subconfigure (configure
 172     # will take it from the configure path anyways).
 173     parser.add_argument('--srcdir', type=str)
 174
 175     data_file = os.path.join(objdir, data_file)
 176     previous_args = None
 177     if os.path.exists(data_file):
 178         with open(data_file, 'rb') as f:
 179             data = pickle.load(f)
 180             previous_args = data['args']
 181
 182     # Msys likes to break environment variables and command line arguments,
 183     # so read those from stdin, as they are passed from the configure script
 184     # when necessary (on windows).
 185     # However, for some reason, $PATH is not handled like other environment
 186     # variables, and msys remangles it even when giving it is already a msys
 187     # $PATH. Fortunately, the mangling/demangling is just find for $PATH, so
 188     # we can just take the value from the environment. Msys will convert it
 189     # back properly when calling subconfigure.
 190     input = sys.stdin.read()
 191     if input:
 192         data = {a: b for [a, b] in eval(input)}
 193         environ = {a: b for a, b in data['env']}
 194         environ['PATH'] = os.environ['PATH']
 195         args = data['args']
 196     else:
 197         environ = os.environ
 198
 199     args, others = parser.parse_known_args(args)
 200
 201     data = {
 202         'target': args.target,
 203         'host': args.host,
 204         'build': args.build,
 205         'args': others,
 206         'shell': shell,
 207         'srcdir': srcdir,
 208         'env': environ,
 209     }
 210
 211     if args.cache_file:
 212         data['cache-file'] = mozpath.normpath(mozpath.join(os.getcwd(),
 213             args.cache_file))
 214     else:
 215         data['cache-file'] = mozpath.join(objdir, 'config.cache')
 216
 217     if previous_args is not None:
 218         data['previous-args'] = previous_args
 219
 220     try:
 221         os.makedirs(objdir)
 222     except OSError as e:
 223         if e.errno != errno.EEXIST:
 224             raise
 225
 226     with open(data_file, 'wb') as f:
 227         pickle.dump(data, f)
 228
 229
 230 def run(data_file, objdir):
 231     ret = 0
 232
 233     with open(os.path.join(objdir, data_file), 'rb') as f:
 234         data = pickle.load(f)
 235
 236     data['objdir'] = objdir
 237
 238     cache_file = data['cache-file']
 239     cleared_cache = True
 240     if os.path.exists(cache_file):
 241         cleared_cache = maybe_clear_cache(data)
 242
 243     config_files, command_files = get_config_files(data)
 244     contents = []
 245     for f, t in config_files:
 246         contents.append(File(f))
 247
 248     # AC_CONFIG_COMMANDS actually only registers tags, not file names
 249     # but most commands are tagged with the file name they create.
 250     # However, a few don't, or are tagged with a directory name (and their
 251     # command is just to create that directory)
 252     for f in command_files:
 253         if os.path.isfile(f):
 254             contents.append(File(f))
 255
 256     # Only run configure if one of the following is true:
 257     # - config.status doesn't exist
 258     # - config.status is older than configure
 259     # - the configure arguments changed
 260     # - the environment changed in a way that requires a cache clear.
 261     configure = mozpath.join(data['srcdir'], 'configure')
 262     config_status_path = mozpath.join(objdir, 'config.status')
 263     skip_configure = True
 264     if not os.path.exists(config_status_path):
 265         skip_configure = False
 266         config_status = None
 267     else:
 268         config_status = File(config_status_path)
 269         if config_status.mtime < os.path.getmtime(configure) or \
 270                 data.get('previous-args', data['args']) != data['args'] or \
 271                 cleared_cache:
 272             skip_configure = False
 273
 274     if not skip_configure:
 275         command = [data['shell'], configure]
 276         for kind in ('target', 'build', 'host'):
 277             if data.get(kind) is not None:
 278                 command += ['--%s=%s' % (kind, data[kind])]
 279         command += data['args']
 280         command += ['--cache-file=%s' % cache_file]
 281
 282         # Pass --no-create to configure so that it doesn't run config.status.
 283         # We're going to run it ourselves.
 284         command += ['--no-create']
 285
 286         print 'configuring in %s' % os.path.relpath(objdir, os.getcwd())
 287         print 'running %s' % ' '.join(command[:-1])
 288         sys.stdout.flush()
 289         ret = subprocess.call(command, cwd=objdir, env=data['env'])
 290
 291         if ret:
 292             return ret
 293
 294         # Leave config.status with a new timestamp if configure is newer than
 295         # its original mtime.
 296         if config_status and os.path.getmtime(configure) <= config_status.mtime:
 297             config_status.update_time()
 298
 299     # Only run config.status if one of the following is true:
 300     # - config.status changed or did not exist
 301     # - one of the templates for config files is newer than the corresponding
 302     #   config file.
 303     skip_config_status = True
 304     if not config_status or config_status.modified:
 305         # If config.status doesn't exist after configure (because it's not
 306         # an autoconf configure), skip it.
 307         if os.path.exists(config_status_path):
 308             skip_config_status = False
 309     else:
 310         # config.status changed or was created, so we need to update the
 311         # list of config and command files.
 312         config_files, command_files = get_config_files(data)
 313         for f, t in config_files:
 314             if not os.path.exists(t) or \
 315                     os.path.getmtime(f) < os.path.getmtime(t):
 316                 skip_config_status = False
 317
 318     if not skip_config_status:
 319         if skip_configure:
 320             print 'running config.status in %s' % os.path.relpath(objdir,
 321                 os.getcwd())
 322             sys.stdout.flush()
 323         ret = subprocess.call([data['shell'], '-c', './config.status'],
 324             cwd=objdir, env=data['env'])
 325
 326         for f in contents:
 327             f.update_time()
 328
 329     return ret
 330
 331
 332 CONFIGURE_DATA = 'configure.pkl'
 333
 334 def main(args):
 335     if args[0] != '--prepare':
 336         if len(args) != 1:
 337             raise Exception('Usage: %s relativeobjdir' % __file__)
 338         return run(CONFIGURE_DATA, args[0])
 339
 340     topsrcdir = os.path.abspath(args[1])
 341     subdir = args[2]
 342     # subdir can be of the form srcdir:objdir
 343     if ':' in subdir:
 344         srcdir, subdir = subdir.split(':', 1)
 345     else:
 346         srcdir = subdir
 347     srcdir = os.path.join(topsrcdir, srcdir)
 348     objdir = os.path.abspath(subdir)
 349
 350     return prepare(CONFIGURE_DATA, srcdir, objdir, args[3], args[4:])
 351
 352
 353 if __name__ == '__main__':
 354     sys.exit(main(sys.argv[1:]))