scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2018 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52 try:
  53     os.cpu_count
  54 except:
  55     import multiprocessing
  56     os.cpu_count = lambda: multiprocessing.cpu_count()
  57
  58 try:
  59     re.fullmatch
  60 except:
  61     re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)
  62
  63 try:
  64     subprocess.run
  65 except:
  66     class _CompletedProcess:
  67         def __init__(self, args, returncode, stdout=None, stderr=None):
  68             self.args = args
  69             self.returncode = returncode
  70             self.stdout = stdout
  71             self.stderr = stderr
  72
  73     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  74         assert(timeout is None)
  75         with subprocess.Popen(*popenargs, **kwargs) as process:
  76             try:
  77                 stdout, stderr = process.communicate(input)
  78             except:
  79                 process.kill()
  80                 process.wait()
  81                 raise
  82             returncode = process.poll()
  83             if check and returncode:
  84                 raise subprocess.CalledProcessError(returncode, popenargs)
  85         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  86
  87     subprocess.run = _run
  88
  89
  90 class Context(object):
  91     """The global state associated with builds in a given directory."""
  92
  93     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  94                  action):
  95         """Initialize the context."""
  96         self.topdir = topdir
  97         self.parallelism = parallelism
  98         self.keep = keep
  99         self.replace_sources = replace_sources
 100         self.strip = strip
 101         self.srcdir = os.path.join(topdir, 'src')
 102         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 103         self.build_state_json = os.path.join(topdir, 'build-state.json')
 104         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 105         self.installdir = os.path.join(topdir, 'install')
 106         self.host_libraries_installdir = os.path.join(self.installdir,
 107                                                       'host-libraries')
 108         self.builddir = os.path.join(topdir, 'build')
 109         self.logsdir = os.path.join(topdir, 'logs')
 110         self.logsdir_old = os.path.join(topdir, 'logs-old')
 111         self.makefile = os.path.join(self.builddir, 'Makefile')
 112         self.wrapper = os.path.join(self.builddir, 'wrapper')
 113         self.save_logs = os.path.join(self.builddir, 'save-logs')
 114         self.script_text = self.get_script_text()
 115         if action != 'checkout':
 116             self.build_triplet = self.get_build_triplet()
 117             self.glibc_version = self.get_glibc_version()
 118         self.configs = {}
 119         self.glibc_configs = {}
 120         self.makefile_pieces = ['.PHONY: all\n']
 121         self.add_all_configs()
 122         self.load_versions_json()
 123         self.load_build_state_json()
 124         self.status_log_list = []
 125         self.email_warning = False
 126
 127     def get_script_text(self):
 128         """Return the text of this script."""
 129         with open(sys.argv[0], 'r') as f:
 130             return f.read()
 131
 132     def exec_self(self):
 133         """Re-execute this script with the same arguments."""
 134         sys.stdout.flush()
 135         os.execv(sys.executable, [sys.executable] + sys.argv)
 136
 137     def get_build_triplet(self):
 138         """Determine the build triplet with config.guess."""
 139         config_guess = os.path.join(self.component_srcdir('gcc'),
 140                                     'config.guess')
 141         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 142                                 check=True, universal_newlines=True).stdout
 143         return cg_out.rstrip()
 144
 145     def get_glibc_version(self):
 146         """Determine the glibc version number (major.minor)."""
 147         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 148         with open(version_h, 'r') as f:
 149             lines = f.readlines()
 150         starttext = '#define VERSION "'
 151         for l in lines:
 152             if l.startswith(starttext):
 153                 l = l[len(starttext):]
 154                 l = l.rstrip('"\n')
 155                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 156                 return '%s.%s' % m.group(1, 2)
 157         print('error: could not determine glibc version')
 158         exit(1)
 159
 160     def add_all_configs(self):
 161         """Add all known glibc build configurations."""
 162         self.add_config(arch='aarch64',
 163                         os_name='linux-gnu',
 164                         extra_glibcs=[{'variant': 'disable-multi-arch',
 165                                        'cfg': ['--disable-multi-arch']}])
 166         self.add_config(arch='aarch64_be',
 167                         os_name='linux-gnu')
 168         self.add_config(arch='alpha',
 169                         os_name='linux-gnu')
 170         self.add_config(arch='arm',
 171                         os_name='linux-gnueabi')
 172         self.add_config(arch='armeb',
 173                         os_name='linux-gnueabi')
 174         self.add_config(arch='armeb',
 175                         os_name='linux-gnueabi',
 176                         variant='be8',
 177                         gcc_cfg=['--with-arch=armv7-a'])
 178         self.add_config(arch='arm',
 179                         os_name='linux-gnueabihf',
 180                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 181                         extra_glibcs=[{'variant': 'v7a',
 182                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 183                                       {'variant': 'v7a-disable-multi-arch',
 184                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 185                                        'cfg': ['--disable-multi-arch']}])
 186         self.add_config(arch='armeb',
 187                         os_name='linux-gnueabihf',
 188                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 189         self.add_config(arch='armeb',
 190                         os_name='linux-gnueabihf',
 191                         variant='be8',
 192                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 193                                  '--with-fpu=vfpv3'])
 194         self.add_config(arch='hppa',
 195                         os_name='linux-gnu')
 196         self.add_config(arch='i686',
 197                         os_name='gnu')
 198         self.add_config(arch='ia64',
 199                         os_name='linux-gnu',
 200                         first_gcc_cfg=['--with-system-libunwind'])
 201         self.add_config(arch='m68k',
 202                         os_name='linux-gnu',
 203                         gcc_cfg=['--disable-multilib'])
 204         self.add_config(arch='m68k',
 205                         os_name='linux-gnu',
 206                         variant='coldfire',
 207                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 208         self.add_config(arch='m68k',
 209                         os_name='linux-gnu',
 210                         variant='coldfire-soft',
 211                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 212                                  '--disable-multilib'])
 213         self.add_config(arch='microblaze',
 214                         os_name='linux-gnu',
 215                         gcc_cfg=['--disable-multilib'])
 216         self.add_config(arch='microblazeel',
 217                         os_name='linux-gnu',
 218                         gcc_cfg=['--disable-multilib'])
 219         self.add_config(arch='mips64',
 220                         os_name='linux-gnu',
 221                         gcc_cfg=['--with-mips-plt'],
 222                         glibcs=[{'variant': 'n32'},
 223                                 {'arch': 'mips',
 224                                  'ccopts': '-mabi=32'},
 225                                 {'variant': 'n64',
 226                                  'ccopts': '-mabi=64'}])
 227         self.add_config(arch='mips64',
 228                         os_name='linux-gnu',
 229                         variant='soft',
 230                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 231                         glibcs=[{'variant': 'n32-soft'},
 232                                 {'variant': 'soft',
 233                                  'arch': 'mips',
 234                                  'ccopts': '-mabi=32'},
 235                                 {'variant': 'n64-soft',
 236                                  'ccopts': '-mabi=64'}])
 237         self.add_config(arch='mips64',
 238                         os_name='linux-gnu',
 239                         variant='nan2008',
 240                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 241                                  '--with-arch-64=mips64r2',
 242                                  '--with-arch-32=mips32r2'],
 243                         glibcs=[{'variant': 'n32-nan2008'},
 244                                 {'variant': 'nan2008',
 245                                  'arch': 'mips',
 246                                  'ccopts': '-mabi=32'},
 247                                 {'variant': 'n64-nan2008',
 248                                  'ccopts': '-mabi=64'}])
 249         self.add_config(arch='mips64',
 250                         os_name='linux-gnu',
 251                         variant='nan2008-soft',
 252                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 253                                  '--with-arch-64=mips64r2',
 254                                  '--with-arch-32=mips32r2',
 255                                  '--with-float=soft'],
 256                         glibcs=[{'variant': 'n32-nan2008-soft'},
 257                                 {'variant': 'nan2008-soft',
 258                                  'arch': 'mips',
 259                                  'ccopts': '-mabi=32'},
 260                                 {'variant': 'n64-nan2008-soft',
 261                                  'ccopts': '-mabi=64'}])
 262         self.add_config(arch='mips64el',
 263                         os_name='linux-gnu',
 264                         gcc_cfg=['--with-mips-plt'],
 265                         glibcs=[{'variant': 'n32'},
 266                                 {'arch': 'mipsel',
 267                                  'ccopts': '-mabi=32'},
 268                                 {'variant': 'n64',
 269                                  'ccopts': '-mabi=64'}])
 270         self.add_config(arch='mips64el',
 271                         os_name='linux-gnu',
 272                         variant='soft',
 273                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 274                         glibcs=[{'variant': 'n32-soft'},
 275                                 {'variant': 'soft',
 276                                  'arch': 'mipsel',
 277                                  'ccopts': '-mabi=32'},
 278                                 {'variant': 'n64-soft',
 279                                  'ccopts': '-mabi=64'}])
 280         self.add_config(arch='mips64el',
 281                         os_name='linux-gnu',
 282                         variant='nan2008',
 283                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 284                                  '--with-arch-64=mips64r2',
 285                                  '--with-arch-32=mips32r2'],
 286                         glibcs=[{'variant': 'n32-nan2008'},
 287                                 {'variant': 'nan2008',
 288                                  'arch': 'mipsel',
 289                                  'ccopts': '-mabi=32'},
 290                                 {'variant': 'n64-nan2008',
 291                                  'ccopts': '-mabi=64'}])
 292         self.add_config(arch='mips64el',
 293                         os_name='linux-gnu',
 294                         variant='nan2008-soft',
 295                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 296                                  '--with-arch-64=mips64r2',
 297                                  '--with-arch-32=mips32r2',
 298                                  '--with-float=soft'],
 299                         glibcs=[{'variant': 'n32-nan2008-soft'},
 300                                 {'variant': 'nan2008-soft',
 301                                  'arch': 'mipsel',
 302                                  'ccopts': '-mabi=32'},
 303                                 {'variant': 'n64-nan2008-soft',
 304                                  'ccopts': '-mabi=64'}])
 305         self.add_config(arch='nios2',
 306                         os_name='linux-gnu')
 307         self.add_config(arch='powerpc',
 308                         os_name='linux-gnu',
 309                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 310                         extra_glibcs=[{'variant': 'power4',
 311                                        'ccopts': '-mcpu=power4',
 312                                        'cfg': ['--with-cpu=power4']}])
 313         self.add_config(arch='powerpc',
 314                         os_name='linux-gnu',
 315                         variant='soft',
 316                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 317                                  '--enable-secureplt'])
 318         self.add_config(arch='powerpc64',
 319                         os_name='linux-gnu',
 320                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 321         self.add_config(arch='powerpc64le',
 322                         os_name='linux-gnu',
 323                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 324         self.add_config(arch='powerpc',
 325                         os_name='linux-gnuspe',
 326                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 327                                  '--enable-e500-double'])
 328         self.add_config(arch='powerpc',
 329                         os_name='linux-gnuspe',
 330                         variant='e500v1',
 331                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 332         self.add_config(arch='riscv64',
 333                         os_name='linux-gnu',
 334                         variant='rv64imac-lp64',
 335                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 336                                  '--disable-multilib'])
 337         self.add_config(arch='riscv64',
 338                         os_name='linux-gnu',
 339                         variant='rv64imafdc-lp64',
 340                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 341                                  '--disable-multilib'])
 342         self.add_config(arch='riscv64',
 343                         os_name='linux-gnu',
 344                         variant='rv64imafdc-lp64d',
 345                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 346                                  '--disable-multilib'])
 347         self.add_config(arch='s390x',
 348                         os_name='linux-gnu',
 349                         glibcs=[{},
 350                                 {'arch': 's390', 'ccopts': '-m31'}])
 351         self.add_config(arch='sh3',
 352                         os_name='linux-gnu')
 353         self.add_config(arch='sh3eb',
 354                         os_name='linux-gnu')
 355         self.add_config(arch='sh4',
 356                         os_name='linux-gnu')
 357         self.add_config(arch='sh4eb',
 358                         os_name='linux-gnu')
 359         self.add_config(arch='sh4',
 360                         os_name='linux-gnu',
 361                         variant='soft',
 362                         gcc_cfg=['--without-fp'])
 363         self.add_config(arch='sh4eb',
 364                         os_name='linux-gnu',
 365                         variant='soft',
 366                         gcc_cfg=['--without-fp'])
 367         self.add_config(arch='sparc64',
 368                         os_name='linux-gnu',
 369                         glibcs=[{},
 370                                 {'arch': 'sparcv9',
 371                                  'ccopts': '-m32 -mlong-double-128'}],
 372                         extra_glibcs=[{'variant': 'disable-multi-arch',
 373                                        'cfg': ['--disable-multi-arch']},
 374                                       {'variant': 'disable-multi-arch',
 375                                        'arch': 'sparcv9',
 376                                        'ccopts': '-m32 -mlong-double-128',
 377                                        'cfg': ['--disable-multi-arch']}])
 378         self.add_config(arch='tilegx',
 379                         os_name='linux-gnu',
 380                         glibcs=[{},
 381                                 {'variant': '32', 'ccopts': '-m32'}])
 382         self.add_config(arch='tilegxbe',
 383                         os_name='linux-gnu',
 384                         glibcs=[{},
 385                                 {'variant': '32', 'ccopts': '-m32'}])
 386         self.add_config(arch='x86_64',
 387                         os_name='linux-gnu',
 388                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 389                         glibcs=[{},
 390                                 {'variant': 'x32', 'ccopts': '-mx32'},
 391                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 392                         extra_glibcs=[{'variant': 'disable-multi-arch',
 393                                        'cfg': ['--disable-multi-arch']},
 394                                       {'variant': 'static-pie',
 395                                        'cfg': ['--enable-static-pie']},
 396                                       {'variant': 'x32-static-pie',
 397                                        'ccopts': '-mx32',
 398                                        'cfg': ['--enable-static-pie']},
 399                                       {'variant': 'static-pie',
 400                                        'arch': 'i686',
 401                                        'ccopts': '-m32 -march=i686',
 402                                        'cfg': ['--enable-static-pie']},
 403                                       {'variant': 'disable-multi-arch',
 404                                        'arch': 'i686',
 405                                        'ccopts': '-m32 -march=i686',
 406                                        'cfg': ['--disable-multi-arch']},
 407                                       {'arch': 'i486',
 408                                        'ccopts': '-m32 -march=i486'},
 409                                       {'arch': 'i586',
 410                                        'ccopts': '-m32 -march=i586'}])
 411
 412     def add_config(self, **args):
 413         """Add an individual build configuration."""
 414         cfg = Config(self, **args)
 415         if cfg.name in self.configs:
 416             print('error: duplicate config %s' % cfg.name)
 417             exit(1)
 418         self.configs[cfg.name] = cfg
 419         for c in cfg.all_glibcs:
 420             if c.name in self.glibc_configs:
 421                 print('error: duplicate glibc config %s' % c.name)
 422                 exit(1)
 423             self.glibc_configs[c.name] = c
 424
 425     def component_srcdir(self, component):
 426         """Return the source directory for a given component, e.g. gcc."""
 427         return os.path.join(self.srcdir, component)
 428
 429     def component_builddir(self, action, config, component, subconfig=None):
 430         """Return the directory to use for a build."""
 431         if config is None:
 432             # Host libraries.
 433             assert subconfig is None
 434             return os.path.join(self.builddir, action, component)
 435         if subconfig is None:
 436             return os.path.join(self.builddir, action, config, component)
 437         else:
 438             # glibc build as part of compiler build.
 439             return os.path.join(self.builddir, action, config, component,
 440                                 subconfig)
 441
 442     def compiler_installdir(self, config):
 443         """Return the directory in which to install a compiler."""
 444         return os.path.join(self.installdir, 'compilers', config)
 445
 446     def compiler_bindir(self, config):
 447         """Return the directory in which to find compiler binaries."""
 448         return os.path.join(self.compiler_installdir(config), 'bin')
 449
 450     def compiler_sysroot(self, config):
 451         """Return the sysroot directory for a compiler."""
 452         return os.path.join(self.compiler_installdir(config), 'sysroot')
 453
 454     def glibc_installdir(self, config):
 455         """Return the directory in which to install glibc."""
 456         return os.path.join(self.installdir, 'glibcs', config)
 457
 458     def run_builds(self, action, configs):
 459         """Run the requested builds."""
 460         if action == 'checkout':
 461             self.checkout(configs)
 462             return
 463         if action == 'bot-cycle':
 464             if configs:
 465                 print('error: configurations specified for bot-cycle')
 466                 exit(1)
 467             self.bot_cycle()
 468             return
 469         if action == 'bot':
 470             if configs:
 471                 print('error: configurations specified for bot')
 472                 exit(1)
 473             self.bot()
 474             return
 475         if action == 'host-libraries' and configs:
 476             print('error: configurations specified for host-libraries')
 477             exit(1)
 478         self.clear_last_build_state(action)
 479         build_time = datetime.datetime.utcnow()
 480         if action == 'host-libraries':
 481             build_components = ('gmp', 'mpfr', 'mpc')
 482             old_components = ()
 483             old_versions = {}
 484             self.build_host_libraries()
 485         elif action == 'compilers':
 486             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 487                                 'gnumach', 'hurd')
 488             old_components = ('gmp', 'mpfr', 'mpc')
 489             old_versions = self.build_state['host-libraries']['build-versions']
 490             self.build_compilers(configs)
 491         else:
 492             build_components = ('glibc',)
 493             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 494                               'mig', 'gnumach', 'hurd')
 495             old_versions = self.build_state['compilers']['build-versions']
 496             self.build_glibcs(configs)
 497         self.write_files()
 498         self.do_build()
 499         if configs:
 500             # Partial build, do not update stored state.
 501             return
 502         build_versions = {}
 503         for k in build_components:
 504             if k in self.versions:
 505                 build_versions[k] = {'version': self.versions[k]['version'],
 506                                      'revision': self.versions[k]['revision']}
 507         for k in old_components:
 508             if k in old_versions:
 509                 build_versions[k] = {'version': old_versions[k]['version'],
 510                                      'revision': old_versions[k]['revision']}
 511         self.update_build_state(action, build_time, build_versions)
 512
 513     @staticmethod
 514     def remove_dirs(*args):
 515         """Remove directories and their contents if they exist."""
 516         for dir in args:
 517             shutil.rmtree(dir, ignore_errors=True)
 518
 519     @staticmethod
 520     def remove_recreate_dirs(*args):
 521         """Remove directories if they exist, and create them as empty."""
 522         Context.remove_dirs(*args)
 523         for dir in args:
 524             os.makedirs(dir, exist_ok=True)
 525
 526     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 527         """Add makefile text for a list of commands."""
 528         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 529         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 530                                     (target, target, target, commands))
 531         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 532
 533     def write_files(self):
 534         """Write out the Makefile and wrapper script."""
 535         mftext = ''.join(self.makefile_pieces)
 536         with open(self.makefile, 'w') as f:
 537             f.write(mftext)
 538         wrapper_text = (
 539             '#!/bin/sh\n'
 540             'prev_base=$1\n'
 541             'this_base=$2\n'
 542             'desc=$3\n'
 543             'dir=$4\n'
 544             'path=$5\n'
 545             'shift 5\n'
 546             'prev_status=$prev_base-status.txt\n'
 547             'this_status=$this_base-status.txt\n'
 548             'this_log=$this_base-log.txt\n'
 549             'date > "$this_log"\n'
 550             'echo >> "$this_log"\n'
 551             'echo "Description: $desc" >> "$this_log"\n'
 552             'printf "%s" "Command:" >> "$this_log"\n'
 553             'for word in "$@"; do\n'
 554             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 555             '    printf " %s" "$word"\n'
 556             '  else\n'
 557             '    printf " \'"\n'
 558             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 559             '    printf "\'"\n'
 560             '  fi\n'
 561             'done >> "$this_log"\n'
 562             'echo >> "$this_log"\n'
 563             'echo "Directory: $dir" >> "$this_log"\n'
 564             'echo "Path addition: $path" >> "$this_log"\n'
 565             'echo >> "$this_log"\n'
 566             'record_status ()\n'
 567             '{\n'
 568             '  echo >> "$this_log"\n'
 569             '  echo "$1: $desc" > "$this_status"\n'
 570             '  echo "$1: $desc" >> "$this_log"\n'
 571             '  echo >> "$this_log"\n'
 572             '  date >> "$this_log"\n'
 573             '  echo "$1: $desc"\n'
 574             '  exit 0\n'
 575             '}\n'
 576             'check_error ()\n'
 577             '{\n'
 578             '  if [ "$1" != "0" ]; then\n'
 579             '    record_status FAIL\n'
 580             '  fi\n'
 581             '}\n'
 582             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 583             '    record_status UNRESOLVED\n'
 584             'fi\n'
 585             'if [ "$dir" ]; then\n'
 586             '  cd "$dir"\n'
 587             '  check_error "$?"\n'
 588             'fi\n'
 589             'if [ "$path" ]; then\n'
 590             '  PATH=$path:$PATH\n'
 591             'fi\n'
 592             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 593             'check_error "$?"\n'
 594             'record_status PASS\n')
 595         with open(self.wrapper, 'w') as f:
 596             f.write(wrapper_text)
 597         # Mode 0o755.
 598         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 599                      stat.S_IROTH|stat.S_IXOTH)
 600         os.chmod(self.wrapper, mode_exec)
 601         save_logs_text = (
 602             '#!/bin/sh\n'
 603             'if ! [ -f tests.sum ]; then\n'
 604             '  echo "No test summary available."\n'
 605             '  exit 0\n'
 606             'fi\n'
 607             'save_file ()\n'
 608             '{\n'
 609             '  echo "Contents of $1:"\n'
 610             '  echo\n'
 611             '  cat "$1"\n'
 612             '  echo\n'
 613             '  echo "End of contents of $1."\n'
 614             '  echo\n'
 615             '}\n'
 616             'save_file tests.sum\n'
 617             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 618             'for t in $non_pass_tests; do\n'
 619             '  if [ -f "$t.out" ]; then\n'
 620             '    save_file "$t.out"\n'
 621             '  fi\n'
 622             'done\n')
 623         with open(self.save_logs, 'w') as f:
 624             f.write(save_logs_text)
 625         os.chmod(self.save_logs, mode_exec)
 626
 627     def do_build(self):
 628         """Do the actual build."""
 629         cmd = ['make', '-j%d' % self.parallelism]
 630         subprocess.run(cmd, cwd=self.builddir, check=True)
 631
 632     def build_host_libraries(self):
 633         """Build the host libraries."""
 634         installdir = self.host_libraries_installdir
 635         builddir = os.path.join(self.builddir, 'host-libraries')
 636         logsdir = os.path.join(self.logsdir, 'host-libraries')
 637         self.remove_recreate_dirs(installdir, builddir, logsdir)
 638         cmdlist = CommandList('host-libraries', self.keep)
 639         self.build_host_library(cmdlist, 'gmp')
 640         self.build_host_library(cmdlist, 'mpfr',
 641                                 ['--with-gmp=%s' % installdir])
 642         self.build_host_library(cmdlist, 'mpc',
 643                                 ['--with-gmp=%s' % installdir,
 644                                 '--with-mpfr=%s' % installdir])
 645         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 646         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 647
 648     def build_host_library(self, cmdlist, lib, extra_opts=None):
 649         """Build one host library."""
 650         srcdir = self.component_srcdir(lib)
 651         builddir = self.component_builddir('host-libraries', None, lib)
 652         installdir = self.host_libraries_installdir
 653         cmdlist.push_subdesc(lib)
 654         cmdlist.create_use_dir(builddir)
 655         cfg_cmd = [os.path.join(srcdir, 'configure'),
 656                    '--prefix=%s' % installdir,
 657                    '--disable-shared']
 658         if extra_opts:
 659             cfg_cmd.extend (extra_opts)
 660         cmdlist.add_command('configure', cfg_cmd)
 661         cmdlist.add_command('build', ['make'])
 662         cmdlist.add_command('check', ['make', 'check'])
 663         cmdlist.add_command('install', ['make', 'install'])
 664         cmdlist.cleanup_dir()
 665         cmdlist.pop_subdesc()
 666
 667     def build_compilers(self, configs):
 668         """Build the compilers."""
 669         if not configs:
 670             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 671             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 672             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 673             configs = sorted(self.configs.keys())
 674         for c in configs:
 675             self.configs[c].build()
 676
 677     def build_glibcs(self, configs):
 678         """Build the glibcs."""
 679         if not configs:
 680             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 681             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 682             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 683             configs = sorted(self.glibc_configs.keys())
 684         for c in configs:
 685             self.glibc_configs[c].build()
 686
 687     def load_versions_json(self):
 688         """Load information about source directory versions."""
 689         if not os.access(self.versions_json, os.F_OK):
 690             self.versions = {}
 691             return
 692         with open(self.versions_json, 'r') as f:
 693             self.versions = json.load(f)
 694
 695     def store_json(self, data, filename):
 696         """Store information in a JSON file."""
 697         filename_tmp = filename + '.tmp'
 698         with open(filename_tmp, 'w') as f:
 699             json.dump(data, f, indent=2, sort_keys=True)
 700         os.rename(filename_tmp, filename)
 701
 702     def store_versions_json(self):
 703         """Store information about source directory versions."""
 704         self.store_json(self.versions, self.versions_json)
 705
 706     def set_component_version(self, component, version, explicit, revision):
 707         """Set the version information for a component."""
 708         self.versions[component] = {'version': version,
 709                                     'explicit': explicit,
 710                                     'revision': revision}
 711         self.store_versions_json()
 712
 713     def checkout(self, versions):
 714         """Check out the desired component versions."""
 715         default_versions = {'binutils': 'vcs-2.30',
 716                             'gcc': 'vcs-7',
 717                             'glibc': 'vcs-mainline',
 718                             'gmp': '6.1.2',
 719                             'linux': '4.15',
 720                             'mpc': '1.1.0',
 721                             'mpfr': '4.0.1',
 722                             'mig': 'vcs-mainline',
 723                             'gnumach': 'vcs-mainline',
 724                             'hurd': 'vcs-mainline'}
 725         use_versions = {}
 726         explicit_versions = {}
 727         for v in versions:
 728             found_v = False
 729             for k in default_versions.keys():
 730                 kx = k + '-'
 731                 if v.startswith(kx):
 732                     vx = v[len(kx):]
 733                     if k in use_versions:
 734                         print('error: multiple versions for %s' % k)
 735                         exit(1)
 736                     use_versions[k] = vx
 737                     explicit_versions[k] = True
 738                     found_v = True
 739                     break
 740             if not found_v:
 741                 print('error: unknown component in %s' % v)
 742                 exit(1)
 743         for k in default_versions.keys():
 744             if k not in use_versions:
 745                 if k in self.versions and self.versions[k]['explicit']:
 746                     use_versions[k] = self.versions[k]['version']
 747                     explicit_versions[k] = True
 748                 else:
 749                     use_versions[k] = default_versions[k]
 750                     explicit_versions[k] = False
 751         os.makedirs(self.srcdir, exist_ok=True)
 752         for k in sorted(default_versions.keys()):
 753             update = os.access(self.component_srcdir(k), os.F_OK)
 754             v = use_versions[k]
 755             if (update and
 756                 k in self.versions and
 757                 v != self.versions[k]['version']):
 758                 if not self.replace_sources:
 759                     print('error: version of %s has changed from %s to %s, '
 760                           'use --replace-sources to check out again' %
 761                           (k, self.versions[k]['version'], v))
 762                     exit(1)
 763                 shutil.rmtree(self.component_srcdir(k))
 764                 update = False
 765             if v.startswith('vcs-'):
 766                 revision = self.checkout_vcs(k, v[4:], update)
 767             else:
 768                 self.checkout_tar(k, v, update)
 769                 revision = v
 770             self.set_component_version(k, v, explicit_versions[k], revision)
 771         if self.get_script_text() != self.script_text:
 772             # Rerun the checkout process in case the updated script
 773             # uses different default versions or new components.
 774             self.exec_self()
 775
 776     def checkout_vcs(self, component, version, update):
 777         """Check out the given version of the given component from version
 778         control.  Return a revision identifier."""
 779         if component == 'binutils':
 780             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 781             if version == 'mainline':
 782                 git_branch = 'master'
 783             else:
 784                 trans = str.maketrans({'.': '_'})
 785                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 786             return self.git_checkout(component, git_url, git_branch, update)
 787         elif component == 'gcc':
 788             if version == 'mainline':
 789                 branch = 'trunk'
 790             else:
 791                 trans = str.maketrans({'.': '_'})
 792                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 793             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 794             return self.gcc_checkout(svn_url, update)
 795         elif component == 'glibc':
 796             git_url = 'git://sourceware.org/git/glibc.git'
 797             if version == 'mainline':
 798                 git_branch = 'master'
 799             else:
 800                 git_branch = 'release/%s/master' % version
 801             r = self.git_checkout(component, git_url, git_branch, update)
 802             self.fix_glibc_timestamps()
 803             return r
 804         elif component == 'gnumach':
 805             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 806             git_branch = 'master'
 807             r = self.git_checkout(component, git_url, git_branch, update)
 808             subprocess.run(['autoreconf', '-i'],
 809                            cwd=self.component_srcdir(component), check=True)
 810             return r
 811         elif component == 'mig':
 812             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 813             git_branch = 'master'
 814             r = self.git_checkout(component, git_url, git_branch, update)
 815             subprocess.run(['autoreconf', '-i'],
 816                            cwd=self.component_srcdir(component), check=True)
 817             return r
 818         elif component == 'hurd':
 819             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 820             git_branch = 'master'
 821             r = self.git_checkout(component, git_url, git_branch, update)
 822             subprocess.run(['autoconf'],
 823                            cwd=self.component_srcdir(component), check=True)
 824             return r
 825         else:
 826             print('error: component %s coming from VCS' % component)
 827             exit(1)
 828
 829     def git_checkout(self, component, git_url, git_branch, update):
 830         """Check out a component from git.  Return a commit identifier."""
 831         if update:
 832             subprocess.run(['git', 'remote', 'prune', 'origin'],
 833                            cwd=self.component_srcdir(component), check=True)
 834             if self.replace_sources:
 835                 subprocess.run(['git', 'clean', '-dxfq'],
 836                                cwd=self.component_srcdir(component), check=True)
 837             subprocess.run(['git', 'pull', '-q'],
 838                            cwd=self.component_srcdir(component), check=True)
 839         else:
 840             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 841                             self.component_srcdir(component)], check=True)
 842         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 843                            cwd=self.component_srcdir(component),
 844                            stdout=subprocess.PIPE,
 845                            check=True, universal_newlines=True).stdout
 846         return r.rstrip()
 847
 848     def fix_glibc_timestamps(self):
 849         """Fix timestamps in a glibc checkout."""
 850         # Ensure that builds do not try to regenerate generated files
 851         # in the source tree.
 852         srcdir = self.component_srcdir('glibc')
 853         for dirpath, dirnames, filenames in os.walk(srcdir):
 854             for f in filenames:
 855                 if (f == 'configure' or
 856                     f == 'preconfigure' or
 857                     f.endswith('-kw.h')):
 858                     to_touch = os.path.join(dirpath, f)
 859                     subprocess.run(['touch', to_touch], check=True)
 860
 861     def gcc_checkout(self, svn_url, update):
 862         """Check out GCC from SVN.  Return the revision number."""
 863         if not update:
 864             subprocess.run(['svn', 'co', '-q', svn_url,
 865                             self.component_srcdir('gcc')], check=True)
 866         subprocess.run(['contrib/gcc_update', '--silent'],
 867                        cwd=self.component_srcdir('gcc'), check=True)
 868         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 869                            stdout=subprocess.PIPE,
 870                            check=True, universal_newlines=True).stdout
 871         return r.rstrip()
 872
 873     def checkout_tar(self, component, version, update):
 874         """Check out the given version of the given component from a
 875         tarball."""
 876         if update:
 877             return
 878         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 879                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 880                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 881                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 882                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 883                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 884                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 885                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 886                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 887         if component not in url_map:
 888             print('error: component %s coming from tarball' % component)
 889             exit(1)
 890         url = url_map[component] % {'version': version}
 891         filename = os.path.join(self.srcdir, url.split('/')[-1])
 892         response = urllib.request.urlopen(url)
 893         data = response.read()
 894         with open(filename, 'wb') as f:
 895             f.write(data)
 896         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 897                        check=True)
 898         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 899                   self.component_srcdir(component))
 900         os.remove(filename)
 901
 902     def load_build_state_json(self):
 903         """Load information about the state of previous builds."""
 904         if os.access(self.build_state_json, os.F_OK):
 905             with open(self.build_state_json, 'r') as f:
 906                 self.build_state = json.load(f)
 907         else:
 908             self.build_state = {}
 909         for k in ('host-libraries', 'compilers', 'glibcs'):
 910             if k not in self.build_state:
 911                 self.build_state[k] = {}
 912             if 'build-time' not in self.build_state[k]:
 913                 self.build_state[k]['build-time'] = ''
 914             if 'build-versions' not in self.build_state[k]:
 915                 self.build_state[k]['build-versions'] = {}
 916             if 'build-results' not in self.build_state[k]:
 917                 self.build_state[k]['build-results'] = {}
 918             if 'result-changes' not in self.build_state[k]:
 919                 self.build_state[k]['result-changes'] = {}
 920             if 'ever-passed' not in self.build_state[k]:
 921                 self.build_state[k]['ever-passed'] = []
 922
 923     def store_build_state_json(self):
 924         """Store information about the state of previous builds."""
 925         self.store_json(self.build_state, self.build_state_json)
 926
 927     def clear_last_build_state(self, action):
 928         """Clear information about the state of part of the build."""
 929         # We clear the last build time and versions when starting a
 930         # new build.  The results of the last build are kept around,
 931         # as comparison is still meaningful if this build is aborted
 932         # and a new one started.
 933         self.build_state[action]['build-time'] = ''
 934         self.build_state[action]['build-versions'] = {}
 935         self.store_build_state_json()
 936
 937     def update_build_state(self, action, build_time, build_versions):
 938         """Update the build state after a build."""
 939         build_time = build_time.replace(microsecond=0)
 940         self.build_state[action]['build-time'] = str(build_time)
 941         self.build_state[action]['build-versions'] = build_versions
 942         build_results = {}
 943         for log in self.status_log_list:
 944             with open(log, 'r') as f:
 945                 log_text = f.read()
 946             log_text = log_text.rstrip()
 947             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 948             result = m.group(1)
 949             test_name = m.group(2)
 950             assert test_name not in build_results
 951             build_results[test_name] = result
 952         old_build_results = self.build_state[action]['build-results']
 953         self.build_state[action]['build-results'] = build_results
 954         result_changes = {}
 955         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 956         for t in all_tests:
 957             if t in old_build_results:
 958                 old_res = old_build_results[t]
 959             else:
 960                 old_res = '(New test)'
 961             if t in build_results:
 962                 new_res = build_results[t]
 963             else:
 964                 new_res = '(Test removed)'
 965             if old_res != new_res:
 966                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 967         self.build_state[action]['result-changes'] = result_changes
 968         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 969                            if t in build_results}
 970         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 971         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 972                                                          new_passes)
 973         self.store_build_state_json()
 974
 975     def load_bot_config_json(self):
 976         """Load bot configuration."""
 977         with open(self.bot_config_json, 'r') as f:
 978             self.bot_config = json.load(f)
 979
 980     def part_build_old(self, action, delay):
 981         """Return whether the last build for a given action was at least a
 982         given number of seconds ago, or does not have a time recorded."""
 983         old_time_str = self.build_state[action]['build-time']
 984         if not old_time_str:
 985             return True
 986         old_time = datetime.datetime.strptime(old_time_str,
 987                                               '%Y-%m-%d %H:%M:%S')
 988         new_time = datetime.datetime.utcnow()
 989         delta = new_time - old_time
 990         return delta.total_seconds() >= delay
 991
 992     def bot_cycle(self):
 993         """Run a single round of checkout and builds."""
 994         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 995         self.load_bot_config_json()
 996         actions = ('host-libraries', 'compilers', 'glibcs')
 997         self.bot_run_self(['--replace-sources'], 'checkout')
 998         self.load_versions_json()
 999         if self.get_script_text() != self.script_text:
1000             print('Script changed, re-execing.')
1001             # On script change, all parts of the build should be rerun.
1002             for a in actions:
1003                 self.clear_last_build_state(a)
1004             self.exec_self()
1005         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1006                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1007                                           'mig', 'gnumach', 'hurd'),
1008                             'glibcs': ('glibc',)}
1009         must_build = {}
1010         for a in actions:
1011             build_vers = self.build_state[a]['build-versions']
1012             must_build[a] = False
1013             if not self.build_state[a]['build-time']:
1014                 must_build[a] = True
1015             old_vers = {}
1016             new_vers = {}
1017             for c in check_components[a]:
1018                 if c in build_vers:
1019                     old_vers[c] = build_vers[c]
1020                 new_vers[c] = {'version': self.versions[c]['version'],
1021                                'revision': self.versions[c]['revision']}
1022             if new_vers == old_vers:
1023                 print('Versions for %s unchanged.' % a)
1024             else:
1025                 print('Versions changed or rebuild forced for %s.' % a)
1026                 if a == 'compilers' and not self.part_build_old(
1027                         a, self.bot_config['compilers-rebuild-delay']):
1028                     print('Not requiring rebuild of compilers this soon.')
1029                 else:
1030                     must_build[a] = True
1031         if must_build['host-libraries']:
1032             must_build['compilers'] = True
1033         if must_build['compilers']:
1034             must_build['glibcs'] = True
1035         for a in actions:
1036             if must_build[a]:
1037                 print('Must rebuild %s.' % a)
1038                 self.clear_last_build_state(a)
1039             else:
1040                 print('No need to rebuild %s.' % a)
1041         if os.access(self.logsdir, os.F_OK):
1042             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1043             shutil.copytree(self.logsdir, self.logsdir_old)
1044         for a in actions:
1045             if must_build[a]:
1046                 build_time = datetime.datetime.utcnow()
1047                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1048                 self.bot_run_self([], a)
1049                 self.load_build_state_json()
1050                 self.bot_build_mail(a, build_time)
1051         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1052
1053     def bot_build_mail(self, action, build_time):
1054         """Send email with the results of a build."""
1055         if not ('email-from' in self.bot_config and
1056                 'email-server' in self.bot_config and
1057                 'email-subject' in self.bot_config and
1058                 'email-to' in self.bot_config):
1059             if not self.email_warning:
1060                 print("Email not configured, not sending.")
1061                 self.email_warning = True
1062             return
1063
1064         build_time = build_time.replace(microsecond=0)
1065         subject = (self.bot_config['email-subject'] %
1066                    {'action': action,
1067                     'build-time': str(build_time)})
1068         results = self.build_state[action]['build-results']
1069         changes = self.build_state[action]['result-changes']
1070         ever_passed = set(self.build_state[action]['ever-passed'])
1071         versions = self.build_state[action]['build-versions']
1072         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1073         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1074         all_fails = {k for k in results if results[k] == 'FAIL'}
1075         if new_regressions:
1076             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1077             new_reg_text = ('New regressions:\n\n%s\n\n' %
1078                             '\n'.join(new_reg_list))
1079         else:
1080             new_reg_text = ''
1081         if all_regressions:
1082             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1083             all_reg_text = ('All regressions:\n\n%s\n\n' %
1084                             '\n'.join(all_reg_list))
1085         else:
1086             all_reg_text = ''
1087         if all_fails:
1088             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1089             all_fail_text = ('All failures:\n\n%s\n\n' %
1090                              '\n'.join(all_fail_list))
1091         else:
1092             all_fail_text = ''
1093         if changes:
1094             changes_list = sorted(changes.keys())
1095             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1096             changes_text = ('All changed results:\n\n%s\n\n' %
1097                             '\n'.join(changes_list))
1098         else:
1099             changes_text = ''
1100         results_text = (new_reg_text + all_reg_text + all_fail_text +
1101                         changes_text)
1102         if not results_text:
1103             results_text = 'Clean build with unchanged results.\n\n'
1104         versions_list = sorted(versions.keys())
1105         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1106                                           versions[k]['revision'])
1107                          for k in versions_list]
1108         versions_text = ('Component versions for this build:\n\n%s\n' %
1109                          '\n'.join(versions_list))
1110         body_text = results_text + versions_text
1111         msg = email.mime.text.MIMEText(body_text)
1112         msg['Subject'] = subject
1113         msg['From'] = self.bot_config['email-from']
1114         msg['To'] = self.bot_config['email-to']
1115         msg['Message-ID'] = email.utils.make_msgid()
1116         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1117         with smtplib.SMTP(self.bot_config['email-server']) as s:
1118             s.send_message(msg)
1119
1120     def bot_run_self(self, opts, action, check=True):
1121         """Run a copy of this script with given options."""
1122         cmd = [sys.executable, sys.argv[0], '--keep=none',
1123                '-j%d' % self.parallelism]
1124         cmd.extend(opts)
1125         cmd.extend([self.topdir, action])
1126         sys.stdout.flush()
1127         subprocess.run(cmd, check=check)
1128
1129     def bot(self):
1130         """Run repeated rounds of checkout and builds."""
1131         while True:
1132             self.load_bot_config_json()
1133             if not self.bot_config['run']:
1134                 print('Bot exiting by request.')
1135                 exit(0)
1136             self.bot_run_self([], 'bot-cycle', check=False)
1137             self.load_bot_config_json()
1138             if not self.bot_config['run']:
1139                 print('Bot exiting by request.')
1140                 exit(0)
1141             time.sleep(self.bot_config['delay'])
1142             if self.get_script_text() != self.script_text:
1143                 print('Script changed, bot re-execing.')
1144                 self.exec_self()
1145
1146
1147 class Config(object):
1148     """A configuration for building a compiler and associated libraries."""
1149
1150     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1151                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1152         """Initialize a Config object."""
1153         self.ctx = ctx
1154         self.arch = arch
1155         self.os = os_name
1156         self.variant = variant
1157         if variant is None:
1158             self.name = '%s-%s' % (arch, os_name)
1159         else:
1160             self.name = '%s-%s-%s' % (arch, os_name, variant)
1161         self.triplet = '%s-glibc-%s' % (arch, os_name)
1162         if gcc_cfg is None:
1163             self.gcc_cfg = []
1164         else:
1165             self.gcc_cfg = gcc_cfg
1166         if first_gcc_cfg is None:
1167             self.first_gcc_cfg = []
1168         else:
1169             self.first_gcc_cfg = first_gcc_cfg
1170         if glibcs is None:
1171             glibcs = [{'variant': variant}]
1172         if extra_glibcs is None:
1173             extra_glibcs = []
1174         glibcs = [Glibc(self, **g) for g in glibcs]
1175         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1176         self.all_glibcs = glibcs + extra_glibcs
1177         self.compiler_glibcs = glibcs
1178         self.installdir = ctx.compiler_installdir(self.name)
1179         self.bindir = ctx.compiler_bindir(self.name)
1180         self.sysroot = ctx.compiler_sysroot(self.name)
1181         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1182         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1183
1184     def component_builddir(self, component):
1185         """Return the directory to use for a (non-glibc) build."""
1186         return self.ctx.component_builddir('compilers', self.name, component)
1187
1188     def build(self):
1189         """Generate commands to build this compiler."""
1190         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1191                                       self.logsdir)
1192         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1193         cmdlist.add_command('check-host-libraries',
1194                             ['test', '-f',
1195                              os.path.join(self.ctx.host_libraries_installdir,
1196                                           'ok')])
1197         cmdlist.use_path(self.bindir)
1198         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1199                               ['--disable-gdb',
1200                                '--disable-libdecnumber',
1201                                '--disable-readline',
1202                                '--disable-sim'])
1203         if self.os.startswith('linux'):
1204             self.install_linux_headers(cmdlist)
1205         self.build_gcc(cmdlist, True)
1206         if self.os == 'gnu':
1207             self.install_gnumach_headers(cmdlist)
1208             self.build_cross_tool(cmdlist, 'mig', 'mig')
1209             self.install_hurd_headers(cmdlist)
1210         for g in self.compiler_glibcs:
1211             cmdlist.push_subdesc('glibc')
1212             cmdlist.push_subdesc(g.name)
1213             g.build_glibc(cmdlist, True)
1214             cmdlist.pop_subdesc()
1215             cmdlist.pop_subdesc()
1216         self.build_gcc(cmdlist, False)
1217         cmdlist.add_command('done', ['touch',
1218                                      os.path.join(self.installdir, 'ok')])
1219         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1220                                       self.logsdir)
1221
1222     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1223         """Build one cross tool."""
1224         srcdir = self.ctx.component_srcdir(tool_src)
1225         builddir = self.component_builddir(tool_build)
1226         cmdlist.push_subdesc(tool_build)
1227         cmdlist.create_use_dir(builddir)
1228         cfg_cmd = [os.path.join(srcdir, 'configure'),
1229                    '--prefix=%s' % self.installdir,
1230                    '--build=%s' % self.ctx.build_triplet,
1231                    '--host=%s' % self.ctx.build_triplet,
1232                    '--target=%s' % self.triplet,
1233                    '--with-sysroot=%s' % self.sysroot]
1234         if extra_opts:
1235             cfg_cmd.extend(extra_opts)
1236         cmdlist.add_command('configure', cfg_cmd)
1237         cmdlist.add_command('build', ['make'])
1238         # Parallel "make install" for GCC has race conditions that can
1239         # cause it to fail; see
1240         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1241         # problems are not known for binutils, but doing the
1242         # installation in parallel within a particular toolchain build
1243         # (as opposed to installation of one toolchain from
1244         # build-many-glibcs.py running in parallel to the installation
1245         # of other toolchains being built) is not known to be
1246         # significantly beneficial, so it is simplest just to disable
1247         # parallel install for cross tools here.
1248         cmdlist.add_command('install', ['make', '-j1', 'install'])
1249         cmdlist.cleanup_dir()
1250         cmdlist.pop_subdesc()
1251
1252     def install_linux_headers(self, cmdlist):
1253         """Install Linux kernel headers."""
1254         arch_map = {'aarch64': 'arm64',
1255                     'alpha': 'alpha',
1256                     'arm': 'arm',
1257                     'hppa': 'parisc',
1258                     'i486': 'x86',
1259                     'i586': 'x86',
1260                     'i686': 'x86',
1261                     'i786': 'x86',
1262                     'ia64': 'ia64',
1263                     'm68k': 'm68k',
1264                     'microblaze': 'microblaze',
1265                     'mips': 'mips',
1266                     'nios2': 'nios2',
1267                     'powerpc': 'powerpc',
1268                     's390': 's390',
1269                     'riscv32': 'riscv',
1270                     'riscv64': 'riscv',
1271                     'sh': 'sh',
1272                     'sparc': 'sparc',
1273                     'tile': 'tile',
1274                     'x86_64': 'x86'}
1275         linux_arch = None
1276         for k in arch_map:
1277             if self.arch.startswith(k):
1278                 linux_arch = arch_map[k]
1279                 break
1280         assert linux_arch is not None
1281         srcdir = self.ctx.component_srcdir('linux')
1282         builddir = self.component_builddir('linux')
1283         headers_dir = os.path.join(self.sysroot, 'usr')
1284         cmdlist.push_subdesc('linux')
1285         cmdlist.create_use_dir(builddir)
1286         cmdlist.add_command('install-headers',
1287                             ['make', '-C', srcdir, 'O=%s' % builddir,
1288                              'ARCH=%s' % linux_arch,
1289                              'INSTALL_HDR_PATH=%s' % headers_dir,
1290                              'headers_install'])
1291         cmdlist.cleanup_dir()
1292         cmdlist.pop_subdesc()
1293
1294     def install_gnumach_headers(self, cmdlist):
1295         """Install GNU Mach headers."""
1296         srcdir = self.ctx.component_srcdir('gnumach')
1297         builddir = self.component_builddir('gnumach')
1298         cmdlist.push_subdesc('gnumach')
1299         cmdlist.create_use_dir(builddir)
1300         cmdlist.add_command('configure',
1301                             [os.path.join(srcdir, 'configure'),
1302                              '--build=%s' % self.ctx.build_triplet,
1303                              '--host=%s' % self.triplet,
1304                              '--prefix=',
1305                              'CC=%s-gcc -nostdlib' % self.triplet])
1306         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1307                                         'install-data'])
1308         cmdlist.cleanup_dir()
1309         cmdlist.pop_subdesc()
1310
1311     def install_hurd_headers(self, cmdlist):
1312         """Install Hurd headers."""
1313         srcdir = self.ctx.component_srcdir('hurd')
1314         builddir = self.component_builddir('hurd')
1315         cmdlist.push_subdesc('hurd')
1316         cmdlist.create_use_dir(builddir)
1317         cmdlist.add_command('configure',
1318                             [os.path.join(srcdir, 'configure'),
1319                              '--build=%s' % self.ctx.build_triplet,
1320                              '--host=%s' % self.triplet,
1321                              '--prefix=',
1322                              '--disable-profile', '--without-parted',
1323                              'CC=%s-gcc -nostdlib' % self.triplet])
1324         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1325                                         'no_deps=t', 'install-headers'])
1326         cmdlist.cleanup_dir()
1327         cmdlist.pop_subdesc()
1328
1329     def build_gcc(self, cmdlist, bootstrap):
1330         """Build GCC."""
1331         # libsanitizer commonly breaks because of glibc header
1332         # changes, or on unusual targets.  libssp is of little
1333         # relevance with glibc's own stack checking support.
1334         # libcilkrts does not support GNU/Hurd (and has been removed
1335         # in GCC 8, so --disable-libcilkrts can be removed once glibc
1336         # no longer supports building with older GCC versions).
1337         cfg_opts = list(self.gcc_cfg)
1338         cfg_opts += ['--disable-libsanitizer', '--disable-libssp',
1339                      '--disable-libcilkrts']
1340         host_libs = self.ctx.host_libraries_installdir
1341         cfg_opts += ['--with-gmp=%s' % host_libs,
1342                      '--with-mpfr=%s' % host_libs,
1343                      '--with-mpc=%s' % host_libs]
1344         if bootstrap:
1345             tool_build = 'gcc-first'
1346             # Building a static-only, C-only compiler that is
1347             # sufficient to build glibc.  Various libraries and
1348             # features that may require libc headers must be disabled.
1349             # When configuring with a sysroot, --with-newlib is
1350             # required to define inhibit_libc (to stop some parts of
1351             # libgcc including libc headers); --without-headers is not
1352             # sufficient.
1353             cfg_opts += ['--enable-languages=c', '--disable-shared',
1354                          '--disable-threads',
1355                          '--disable-libatomic',
1356                          '--disable-decimal-float',
1357                          '--disable-libffi',
1358                          '--disable-libgomp',
1359                          '--disable-libitm',
1360                          '--disable-libmpx',
1361                          '--disable-libquadmath',
1362                          '--without-headers', '--with-newlib',
1363                          '--with-glibc-version=%s' % self.ctx.glibc_version
1364                          ]
1365             cfg_opts += self.first_gcc_cfg
1366         else:
1367             tool_build = 'gcc'
1368             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1369                          '--enable-threads']
1370         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1371
1372
1373 class Glibc(object):
1374     """A configuration for building glibc."""
1375
1376     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1377                  cfg=None, ccopts=None):
1378         """Initialize a Glibc object."""
1379         self.ctx = compiler.ctx
1380         self.compiler = compiler
1381         if arch is None:
1382             self.arch = compiler.arch
1383         else:
1384             self.arch = arch
1385         if os_name is None:
1386             self.os = compiler.os
1387         else:
1388             self.os = os_name
1389         self.variant = variant
1390         if variant is None:
1391             self.name = '%s-%s' % (self.arch, self.os)
1392         else:
1393             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1394         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1395         if cfg is None:
1396             self.cfg = []
1397         else:
1398             self.cfg = cfg
1399         self.ccopts = ccopts
1400
1401     def tool_name(self, tool):
1402         """Return the name of a cross-compilation tool."""
1403         ctool = '%s-%s' % (self.compiler.triplet, tool)
1404         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1405             ctool = '%s %s' % (ctool, self.ccopts)
1406         return ctool
1407
1408     def build(self):
1409         """Generate commands to build this glibc."""
1410         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1411         installdir = self.ctx.glibc_installdir(self.name)
1412         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1413         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1414         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1415         cmdlist.add_command('check-compilers',
1416                             ['test', '-f',
1417                              os.path.join(self.compiler.installdir, 'ok')])
1418         cmdlist.use_path(self.compiler.bindir)
1419         self.build_glibc(cmdlist, False)
1420         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1421                                       logsdir)
1422
1423     def build_glibc(self, cmdlist, for_compiler):
1424         """Generate commands to build this glibc, either as part of a compiler
1425         build or with the bootstrapped compiler (and in the latter case, run
1426         tests as well)."""
1427         srcdir = self.ctx.component_srcdir('glibc')
1428         if for_compiler:
1429             builddir = self.ctx.component_builddir('compilers',
1430                                                    self.compiler.name, 'glibc',
1431                                                    self.name)
1432             installdir = self.compiler.sysroot
1433             srcdir_copy = self.ctx.component_builddir('compilers',
1434                                                       self.compiler.name,
1435                                                       'glibc-src',
1436                                                       self.name)
1437         else:
1438             builddir = self.ctx.component_builddir('glibcs', self.name,
1439                                                    'glibc')
1440             installdir = self.ctx.glibc_installdir(self.name)
1441             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1442                                                       'glibc-src')
1443         cmdlist.create_use_dir(builddir)
1444         # glibc builds write into the source directory, and even if
1445         # not intentionally there is a risk of bugs that involve
1446         # writing into the working directory.  To avoid possible
1447         # concurrency issues, copy the source directory.
1448         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1449         use_usr = self.os != 'gnu'
1450         prefix = '/usr' if use_usr else ''
1451         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1452                    '--prefix=%s' % prefix,
1453                    '--enable-profile',
1454                    '--build=%s' % self.ctx.build_triplet,
1455                    '--host=%s' % self.triplet,
1456                    'CC=%s' % self.tool_name('gcc'),
1457                    'CXX=%s' % self.tool_name('g++'),
1458                    'AR=%s' % self.tool_name('ar'),
1459                    'AS=%s' % self.tool_name('as'),
1460                    'LD=%s' % self.tool_name('ld'),
1461                    'NM=%s' % self.tool_name('nm'),
1462                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1463                    'OBJDUMP=%s' % self.tool_name('objdump'),
1464                    'RANLIB=%s' % self.tool_name('ranlib'),
1465                    'READELF=%s' % self.tool_name('readelf'),
1466                    'STRIP=%s' % self.tool_name('strip')]
1467         if self.os == 'gnu':
1468             cfg_cmd += ['MIG=%s' % self.tool_name('mig')]
1469         cfg_cmd += self.cfg
1470         cmdlist.add_command('configure', cfg_cmd)
1471         cmdlist.add_command('build', ['make'])
1472         cmdlist.add_command('install', ['make', 'install',
1473                                         'install_root=%s' % installdir])
1474         # GCC uses paths such as lib/../lib64, so make sure lib
1475         # directories always exist.
1476         mkdir_cmd = ['mkdir', '-p',
1477                      os.path.join(installdir, 'lib')]
1478         if use_usr:
1479             mkdir_cmd += [os.path.join(installdir, 'usr', 'lib')]
1480         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1481         if not for_compiler:
1482             if self.ctx.strip:
1483                 cmdlist.add_command('strip',
1484                                     ['sh', '-c',
1485                                      ('%s $(find %s/lib* -name "*.so")' %
1486                                       (self.tool_name('strip'), installdir))])
1487             cmdlist.add_command('check', ['make', 'check'])
1488             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1489                                 always_run=True)
1490         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1491         cmdlist.cleanup_dir()
1492
1493
1494 class Command(object):
1495     """A command run in the build process."""
1496
1497     def __init__(self, desc, num, dir, path, command, always_run=False):
1498         """Initialize a Command object."""
1499         self.dir = dir
1500         self.path = path
1501         self.desc = desc
1502         trans = str.maketrans({' ': '-'})
1503         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1504         self.command = command
1505         self.always_run = always_run
1506
1507     @staticmethod
1508     def shell_make_quote_string(s):
1509         """Given a string not containing a newline, quote it for use by the
1510         shell and make."""
1511         assert '\n' not in s
1512         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1513             return s
1514         strans = str.maketrans({"'": "'\\''"})
1515         s = "'%s'" % s.translate(strans)
1516         mtrans = str.maketrans({'$': '$$'})
1517         return s.translate(mtrans)
1518
1519     @staticmethod
1520     def shell_make_quote_list(l, translate_make):
1521         """Given a list of strings not containing newlines, quote them for use
1522         by the shell and make, returning a single string.  If translate_make
1523         is true and the first string is 'make', change it to $(MAKE)."""
1524         l = [Command.shell_make_quote_string(s) for s in l]
1525         if translate_make and l[0] == 'make':
1526             l[0] = '$(MAKE)'
1527         return ' '.join(l)
1528
1529     def shell_make_quote(self):
1530         """Return this command quoted for the shell and make."""
1531         return self.shell_make_quote_list(self.command, True)
1532
1533
1534 class CommandList(object):
1535     """A list of commands run in the build process."""
1536
1537     def __init__(self, desc, keep):
1538         """Initialize a CommandList object."""
1539         self.cmdlist = []
1540         self.dir = None
1541         self.path = None
1542         self.desc = [desc]
1543         self.keep = keep
1544
1545     def desc_txt(self, desc):
1546         """Return the description to use for a command."""
1547         return '%s %s' % (' '.join(self.desc), desc)
1548
1549     def use_dir(self, dir):
1550         """Set the default directory for subsequent commands."""
1551         self.dir = dir
1552
1553     def use_path(self, path):
1554         """Set a directory to be prepended to the PATH for subsequent
1555         commands."""
1556         self.path = path
1557
1558     def push_subdesc(self, subdesc):
1559         """Set the default subdescription for subsequent commands (e.g., the
1560         name of a component being built, within the series of commands
1561         building it)."""
1562         self.desc.append(subdesc)
1563
1564     def pop_subdesc(self):
1565         """Pop a subdescription from the list of descriptions."""
1566         self.desc.pop()
1567
1568     def create_use_dir(self, dir):
1569         """Remove and recreate a directory and use it for subsequent
1570         commands."""
1571         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1572         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1573         self.use_dir(dir)
1574
1575     def create_copy_dir(self, src, dest):
1576         """Remove a directory and recreate it as a copy from the given
1577         source."""
1578         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1579         parent = os.path.dirname(dest)
1580         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1581         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1582
1583     def add_command_dir(self, desc, dir, command, always_run=False):
1584         """Add a command to run in a given directory."""
1585         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1586                       command, always_run)
1587         self.cmdlist.append(cmd)
1588
1589     def add_command(self, desc, command, always_run=False):
1590         """Add a command to run in the default directory."""
1591         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1592                       self.path, command, always_run)
1593         self.cmdlist.append(cmd)
1594
1595     def cleanup_dir(self, desc='cleanup', dir=None):
1596         """Clean up a build directory.  If no directory is specified, the
1597         default directory is cleaned up and ceases to be the default
1598         directory."""
1599         if dir is None:
1600             dir = self.dir
1601             self.use_dir(None)
1602         if self.keep != 'all':
1603             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1604                                  always_run=(self.keep == 'none'))
1605
1606     def makefile_commands(self, wrapper, logsdir):
1607         """Return the sequence of commands in the form of text for a Makefile.
1608         The given wrapper script takes arguments: base of logs for
1609         previous command, or empty; base of logs for this command;
1610         description; directory; PATH addition; the command itself."""
1611         # prev_base is the base of the name for logs of the previous
1612         # command that is not always-run (that is, a build command,
1613         # whose failure should stop subsequent build commands from
1614         # being run, as opposed to a cleanup command, which is run
1615         # even if previous commands failed).
1616         prev_base = ''
1617         cmds = []
1618         for c in self.cmdlist:
1619             ctxt = c.shell_make_quote()
1620             if prev_base and not c.always_run:
1621                 prev_log = os.path.join(logsdir, prev_base)
1622             else:
1623                 prev_log = ''
1624             this_log = os.path.join(logsdir, c.logbase)
1625             if not c.always_run:
1626                 prev_base = c.logbase
1627             if c.dir is None:
1628                 dir = ''
1629             else:
1630                 dir = c.dir
1631             if c.path is None:
1632                 path = ''
1633             else:
1634                 path = c.path
1635             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1636             prelim_txt = Command.shell_make_quote_list(prelims, False)
1637             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1638         return '\n'.join(cmds)
1639
1640     def status_logs(self, logsdir):
1641         """Return the list of log files with command status."""
1642         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1643                 for c in self.cmdlist]
1644
1645
1646 def get_parser():
1647     """Return an argument parser for this module."""
1648     parser = argparse.ArgumentParser(description=__doc__)
1649     parser.add_argument('-j', dest='parallelism',
1650                         help='Run this number of jobs in parallel',
1651                         type=int, default=os.cpu_count())
1652     parser.add_argument('--keep', dest='keep',
1653                         help='Whether to keep all build directories, '
1654                         'none or only those from failed builds',
1655                         default='none', choices=('none', 'all', 'failed'))
1656     parser.add_argument('--replace-sources', action='store_true',
1657                         help='Remove and replace source directories '
1658                         'with the wrong version of a component')
1659     parser.add_argument('--strip', action='store_true',
1660                         help='Strip installed glibc libraries')
1661     parser.add_argument('topdir',
1662                         help='Toplevel working directory')
1663     parser.add_argument('action',
1664                         help='What to do',
1665                         choices=('checkout', 'bot-cycle', 'bot',
1666                                  'host-libraries', 'compilers', 'glibcs'))
1667     parser.add_argument('configs',
1668                         help='Versions to check out or configurations to build',
1669                         nargs='*')
1670     return parser
1671
1672
1673 def main(argv):
1674     """The main entry point."""
1675     parser = get_parser()
1676     opts = parser.parse_args(argv)
1677     topdir = os.path.abspath(opts.topdir)
1678     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1679                   opts.strip, opts.action)
1680     ctx.run_builds(opts.action, opts.configs)
1681
1682
1683 if __name__ == '__main__':
1684     main(sys.argv[1:])