build/subconfigure.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # This script is used to capture the content of config.status-generated
   6 # files and subsequently restore their timestamp if they haven't changed.
   7
   8 import argparse
   9 import errno
  10 import itertools
  11 import os
  12 import re
  13 import subprocess
  14 import sys
  15 import pickle
  16
  17 import mozpack.path as mozpath
  18
  19
  20 class Pool(object):
  21     def __new__(cls, size):
  22         try:
  23             import multiprocessing
  24             size = min(size, multiprocessing.cpu_count())
  25             return multiprocessing.Pool(size)
  26         except:
  27             return super(Pool, cls).__new__(cls)
  28
  29     def imap_unordered(self, fn, iterable):
  30         return itertools.imap(fn, iterable)
  31
  32     def close(self):
  33         pass
  34
  35     def join(self):
  36         pass
  37
  38
  39 class File(object):
  40     def __init__(self, path):
  41         self._path = path
  42         self._content = open(path, 'rb').read()
  43         stat = os.stat(path)
  44         self._times = (stat.st_atime, stat.st_mtime)
  45
  46     @property
  47     def path(self):
  48         return self._path
  49
  50     @property
  51     def mtime(self):
  52         return self._times[1]
  53
  54     @property
  55     def modified(self):
  56         '''Returns whether the file was modified since the instance was
  57         created. Result is memoized.'''
  58         if hasattr(self, '_modified'):
  59             return self._modified
  60
  61         modified = True
  62         if os.path.exists(self._path):
  63             if open(self._path, 'rb').read() == self._content:
  64                 modified = False
  65         self._modified = modified
  66         return modified
  67
  68     def update_time(self):
  69         '''If the file hasn't changed since the instance was created,
  70            restore its old modification time.'''
  71         if not self.modified:
  72             os.utime(self._path, self._times)
  73
  74
  75 # As defined in the various sub-configures in the tree
  76 PRECIOUS_VARS = set([
  77     'build_alias',
  78     'host_alias',
  79     'target_alias',
  80     'CC',
  81     'CFLAGS',
  82     'LDFLAGS',
  83     'LIBS',
  84     'CPPFLAGS',
  85     'CPP',
  86     'CCC',
  87     'CXXFLAGS',
  88     'CXX',
  89     'CCASFLAGS',
  90     'CCAS',
  91 ])
  92
  93
  94 CONFIGURE_DATA = 'configure.pkl'
  95
  96
  97 # Autoconf, in some of the sub-configures used in the tree, likes to error
  98 # out when "precious" variables change in value. The solution it gives to
  99 # straighten things is to either run make distclean or remove config.cache.
 100 # There's no reason not to do the latter automatically instead of failing,
 101 # doing the cleanup (which, on buildbots means a full clobber), and
 102 # restarting from scratch.
 103 def maybe_clear_cache(data):
 104     env = dict(data['env'])
 105     for kind in ('target', 'host', 'build'):
 106         arg = data[kind]
 107         if arg is not None:
 108             env['%s_alias' % kind] = arg
 109     # configure can take variables assignments in its arguments, and that
 110     # overrides whatever is in the environment.
 111     for arg in data['args']:
 112         if arg[:1] != '-' and '=' in arg:
 113             key, value = arg.split('=', 1)
 114             env[key] = value
 115
 116     comment = re.compile(r'^\s+#')
 117     cache = {}
 118     with open(data['cache-file']) as f:
 119         for line in f:
 120             if not comment.match(line) and '=' in line:
 121                 key, value = line.rstrip(os.linesep).split('=', 1)
 122                 # If the value is quoted, unquote it
 123                 if value[:1] == "'":
 124                     value = value[1:-1].replace("'\\''", "'")
 125                 cache[key] = value
 126     for precious in PRECIOUS_VARS:
 127         # If there is no entry at all for that precious variable, then
 128         # its value is not precious for that particular configure.
 129         if 'ac_cv_env_%s_set' % precious not in cache:
 130             continue
 131         is_set = cache.get('ac_cv_env_%s_set' % precious) == 'set'
 132         value = cache.get('ac_cv_env_%s_value' % precious) if is_set else None
 133         if value != env.get(precious):
 134             print 'Removing %s because of %s value change from:' \
 135                 % (data['cache-file'], precious)
 136             print '  %s' % (value if value is not None else 'undefined')
 137             print 'to:'
 138             print '  %s' % env.get(precious, 'undefined')
 139             os.remove(data['cache-file'])
 140             return True
 141     return False
 142
 143
 144 def split_template(s):
 145     """Given a "file:template" string, returns "file", "template". If the string
 146     is of the form "file" (without a template), returns "file", "file.in"."""
 147     if ':' in s:
 148         return s.split(':', 1)
 149     return s, '%s.in' % s
 150
 151
 152 def get_config_files(data):
 153     config_status = mozpath.join(data['objdir'], 'config.status')
 154     if not os.path.exists(config_status):
 155         return [], []
 156
 157     configure = mozpath.join(data['srcdir'], 'configure')
 158     config_files = []
 159     command_files = []
 160
 161     # Scan the config.status output for information about configuration files
 162     # it generates.
 163     config_status_output = subprocess.check_output(
 164         [data['shell'], '-c', '%s --help' % config_status],
 165         stderr=subprocess.STDOUT).splitlines()
 166     state = None
 167     for line in config_status_output:
 168         if line.startswith('Configuration') and line.endswith(':'):
 169             if line.endswith('commands:'):
 170                 state = 'commands'
 171             else:
 172                 state = 'config'
 173         elif not line.strip():
 174             state = None
 175         elif state:
 176             for f, t in (split_template(couple) for couple in line.split()):
 177                 f = mozpath.join(data['objdir'], f)
 178                 t = mozpath.join(data['srcdir'], t)
 179                 if state == 'commands':
 180                     command_files.append(f)
 181                 else:
 182                     config_files.append((f, t))
 183
 184     return config_files, command_files
 185
 186
 187 def prepare(srcdir, objdir, shell, args):
 188     parser = argparse.ArgumentParser()
 189     parser.add_argument('--target', type=str)
 190     parser.add_argument('--host', type=str)
 191     parser.add_argument('--build', type=str)
 192     parser.add_argument('--cache-file', type=str)
 193     # The --srcdir argument is simply ignored. It's a useless autoconf feature
 194     # that we don't support well anyways. This makes it stripped from `others`
 195     # and allows to skip setting it when calling the subconfigure (configure
 196     # will take it from the configure path anyways).
 197     parser.add_argument('--srcdir', type=str)
 198
 199     data_file = os.path.join(objdir, CONFIGURE_DATA)
 200     previous_args = None
 201     if os.path.exists(data_file):
 202         with open(data_file, 'rb') as f:
 203             data = pickle.load(f)
 204             previous_args = data['args']
 205
 206     # Msys likes to break environment variables and command line arguments,
 207     # so read those from stdin, as they are passed from the configure script
 208     # when necessary (on windows).
 209     # However, for some reason, $PATH is not handled like other environment
 210     # variables, and msys remangles it even when giving it is already a msys
 211     # $PATH. Fortunately, the mangling/demangling is just find for $PATH, so
 212     # we can just take the value from the environment. Msys will convert it
 213     # back properly when calling subconfigure.
 214     input = sys.stdin.read()
 215     if input:
 216         data = {a: b for [a, b] in eval(input)}
 217         environ = {a: b for a, b in data['env']}
 218         environ['PATH'] = os.environ['PATH']
 219         args = data['args']
 220     else:
 221         environ = os.environ
 222
 223     args, others = parser.parse_known_args(args)
 224
 225     data = {
 226         'target': args.target,
 227         'host': args.host,
 228         'build': args.build,
 229         'args': others,
 230         'shell': shell,
 231         'srcdir': srcdir,
 232         'env': environ,
 233     }
 234
 235     if args.cache_file:
 236         data['cache-file'] = mozpath.normpath(mozpath.join(os.getcwd(),
 237             args.cache_file))
 238     else:
 239         data['cache-file'] = mozpath.join(objdir, 'config.cache')
 240
 241     if previous_args is not None:
 242         data['previous-args'] = previous_args
 243
 244     try:
 245         os.makedirs(objdir)
 246     except OSError as e:
 247         if e.errno != errno.EEXIST:
 248             raise
 249
 250     with open(data_file, 'wb') as f:
 251         pickle.dump(data, f)
 252
 253
 254 def prefix_lines(text, prefix):
 255     return ''.join('%s> %s' % (prefix, line) for line in text.splitlines(True))
 256
 257
 258 def run(objdir):
 259     ret = 0
 260     output = ''
 261
 262     with open(os.path.join(objdir, CONFIGURE_DATA), 'rb') as f:
 263         data = pickle.load(f)
 264
 265     data['objdir'] = objdir
 266
 267     cache_file = data['cache-file']
 268     cleared_cache = True
 269     if os.path.exists(cache_file):
 270         cleared_cache = maybe_clear_cache(data)
 271
 272     config_files, command_files = get_config_files(data)
 273     contents = []
 274     for f, t in config_files:
 275         contents.append(File(f))
 276
 277     # AC_CONFIG_COMMANDS actually only registers tags, not file names
 278     # but most commands are tagged with the file name they create.
 279     # However, a few don't, or are tagged with a directory name (and their
 280     # command is just to create that directory)
 281     for f in command_files:
 282         if os.path.isfile(f):
 283             contents.append(File(f))
 284
 285     # Only run configure if one of the following is true:
 286     # - config.status doesn't exist
 287     # - config.status is older than configure
 288     # - the configure arguments changed
 289     # - the environment changed in a way that requires a cache clear.
 290     configure = mozpath.join(data['srcdir'], 'configure')
 291     config_status_path = mozpath.join(objdir, 'config.status')
 292     skip_configure = True
 293     if not os.path.exists(config_status_path):
 294         skip_configure = False
 295         config_status = None
 296     else:
 297         config_status = File(config_status_path)
 298         if config_status.mtime < os.path.getmtime(configure) or \
 299                 data.get('previous-args', data['args']) != data['args'] or \
 300                 cleared_cache:
 301             skip_configure = False
 302
 303     relobjdir = os.path.relpath(objdir, os.getcwd())
 304
 305     if not skip_configure:
 306         command = [data['shell'], configure]
 307         for kind in ('target', 'build', 'host'):
 308             if data.get(kind) is not None:
 309                 command += ['--%s=%s' % (kind, data[kind])]
 310         command += data['args']
 311         command += ['--cache-file=%s' % cache_file]
 312
 313         # Pass --no-create to configure so that it doesn't run config.status.
 314         # We're going to run it ourselves.
 315         command += ['--no-create']
 316
 317         print prefix_lines('configuring', relobjdir)
 318         print prefix_lines('running %s' % ' '.join(command[:-1]), relobjdir)
 319         sys.stdout.flush()
 320         try:
 321             output += subprocess.check_output(command,
 322                 stderr=subprocess.STDOUT, cwd=objdir, env=data['env'])
 323         except subprocess.CalledProcessError as e:
 324             return relobjdir, e.returncode, e.output
 325
 326         # Leave config.status with a new timestamp if configure is newer than
 327         # its original mtime.
 328         if config_status and os.path.getmtime(configure) <= config_status.mtime:
 329             config_status.update_time()
 330
 331     # Only run config.status if one of the following is true:
 332     # - config.status changed or did not exist
 333     # - one of the templates for config files is newer than the corresponding
 334     #   config file.
 335     skip_config_status = True
 336     if not config_status or config_status.modified:
 337         # If config.status doesn't exist after configure (because it's not
 338         # an autoconf configure), skip it.
 339         if os.path.exists(config_status_path):
 340             skip_config_status = False
 341     else:
 342         # config.status changed or was created, so we need to update the
 343         # list of config and command files.
 344         config_files, command_files = get_config_files(data)
 345         for f, t in config_files:
 346             if not os.path.exists(t) or \
 347                     os.path.getmtime(f) < os.path.getmtime(t):
 348                 skip_config_status = False
 349
 350     if not skip_config_status:
 351         if skip_configure:
 352             print prefix_lines('running config.status', relobjdir)
 353             sys.stdout.flush()
 354         try:
 355             output += subprocess.check_output([data['shell'], '-c',
 356                 './config.status'], stderr=subprocess.STDOUT, cwd=objdir,
 357                 env=data['env'])
 358         except subprocess.CalledProcessError as e:
 359             ret = e.returncode
 360             output += e.output
 361
 362         for f in contents:
 363             f.update_time()
 364
 365     return relobjdir, ret, output
 366
 367
 368 def subconfigure(args):
 369     parser = argparse.ArgumentParser()
 370     parser.add_argument('--list', type=str,
 371         help='File containing a list of subconfigures to run')
 372     parser.add_argument('--skip', type=str,
 373         help='File containing a list of Subconfigures to skip')
 374     parser.add_argument('subconfigures', type=str, nargs='*',
 375         help='Subconfigures to run if no list file is given')
 376     args, others = parser.parse_known_args(args)
 377     subconfigures = args.subconfigures
 378     if args.list:
 379         subconfigures.extend(open(args.list, 'rb').read().splitlines())
 380     if args.skip:
 381         skips = set(open(args.skip, 'rb').read().splitlines())
 382         subconfigures = [s for s in subconfigures if s not in skips]
 383
 384     if not subconfigures:
 385         return 0
 386
 387     ret = 0
 388     # One would think using a ThreadPool would be faster, considering
 389     # everything happens in subprocesses anyways, but no, it's actually
 390     # slower on Windows. (20s difference overall!)
 391     pool = Pool(len(subconfigures))
 392     for relobjdir, returncode, output in \
 393             pool.imap_unordered(run, subconfigures):
 394         print prefix_lines(output, relobjdir)
 395         sys.stdout.flush()
 396         ret = max(returncode, ret)
 397         if ret:
 398             break
 399     pool.close()
 400     pool.join()
 401     return ret
 402
 403
 404 def main(args):
 405     if args[0] != '--prepare':
 406         return subconfigure(args)
 407
 408     topsrcdir = os.path.abspath(args[1])
 409     subdir = args[2]
 410     # subdir can be of the form srcdir:objdir
 411     if ':' in subdir:
 412         srcdir, subdir = subdir.split(':', 1)
 413     else:
 414         srcdir = subdir
 415     srcdir = os.path.join(topsrcdir, srcdir)
 416     objdir = os.path.abspath(subdir)
 417
 418     return prepare(srcdir, objdir, args[3], args[4:])
 419
 420
 421 if __name__ == '__main__':
 422     sys.exit(main(sys.argv[1:]))