scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2018 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52 try:
  53     os.cpu_count
  54 except:
  55     import multiprocessing
  56     os.cpu_count = lambda: multiprocessing.cpu_count()
  57
  58 try:
  59     re.fullmatch
  60 except:
  61     re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)
  62
  63 try:
  64     subprocess.run
  65 except:
  66     class _CompletedProcess:
  67         def __init__(self, args, returncode, stdout=None, stderr=None):
  68             self.args = args
  69             self.returncode = returncode
  70             self.stdout = stdout
  71             self.stderr = stderr
  72
  73     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  74         assert(timeout is None)
  75         with subprocess.Popen(*popenargs, **kwargs) as process:
  76             try:
  77                 stdout, stderr = process.communicate(input)
  78             except:
  79                 process.kill()
  80                 process.wait()
  81                 raise
  82             returncode = process.poll()
  83             if check and returncode:
  84                 raise subprocess.CalledProcessError(returncode, popenargs)
  85         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  86
  87     subprocess.run = _run
  88
  89
  90 class Context(object):
  91     """The global state associated with builds in a given directory."""
  92
  93     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  94                  action):
  95         """Initialize the context."""
  96         self.topdir = topdir
  97         self.parallelism = parallelism
  98         self.keep = keep
  99         self.replace_sources = replace_sources
 100         self.strip = strip
 101         self.srcdir = os.path.join(topdir, 'src')
 102         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 103         self.build_state_json = os.path.join(topdir, 'build-state.json')
 104         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 105         self.installdir = os.path.join(topdir, 'install')
 106         self.host_libraries_installdir = os.path.join(self.installdir,
 107                                                       'host-libraries')
 108         self.builddir = os.path.join(topdir, 'build')
 109         self.logsdir = os.path.join(topdir, 'logs')
 110         self.logsdir_old = os.path.join(topdir, 'logs-old')
 111         self.makefile = os.path.join(self.builddir, 'Makefile')
 112         self.wrapper = os.path.join(self.builddir, 'wrapper')
 113         self.save_logs = os.path.join(self.builddir, 'save-logs')
 114         self.script_text = self.get_script_text()
 115         if action != 'checkout':
 116             self.build_triplet = self.get_build_triplet()
 117             self.glibc_version = self.get_glibc_version()
 118         self.configs = {}
 119         self.glibc_configs = {}
 120         self.makefile_pieces = ['.PHONY: all\n']
 121         self.add_all_configs()
 122         self.load_versions_json()
 123         self.load_build_state_json()
 124         self.status_log_list = []
 125         self.email_warning = False
 126
 127     def get_script_text(self):
 128         """Return the text of this script."""
 129         with open(sys.argv[0], 'r') as f:
 130             return f.read()
 131
 132     def exec_self(self):
 133         """Re-execute this script with the same arguments."""
 134         sys.stdout.flush()
 135         os.execv(sys.executable, [sys.executable] + sys.argv)
 136
 137     def get_build_triplet(self):
 138         """Determine the build triplet with config.guess."""
 139         config_guess = os.path.join(self.component_srcdir('gcc'),
 140                                     'config.guess')
 141         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 142                                 check=True, universal_newlines=True).stdout
 143         return cg_out.rstrip()
 144
 145     def get_glibc_version(self):
 146         """Determine the glibc version number (major.minor)."""
 147         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 148         with open(version_h, 'r') as f:
 149             lines = f.readlines()
 150         starttext = '#define VERSION "'
 151         for l in lines:
 152             if l.startswith(starttext):
 153                 l = l[len(starttext):]
 154                 l = l.rstrip('"\n')
 155                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 156                 return '%s.%s' % m.group(1, 2)
 157         print('error: could not determine glibc version')
 158         exit(1)
 159
 160     def add_all_configs(self):
 161         """Add all known glibc build configurations."""
 162         self.add_config(arch='aarch64',
 163                         os_name='linux-gnu',
 164                         extra_glibcs=[{'variant': 'disable-multi-arch',
 165                                        'cfg': ['--disable-multi-arch']}])
 166         self.add_config(arch='aarch64_be',
 167                         os_name='linux-gnu')
 168         self.add_config(arch='alpha',
 169                         os_name='linux-gnu')
 170         self.add_config(arch='arm',
 171                         os_name='linux-gnueabi')
 172         self.add_config(arch='armeb',
 173                         os_name='linux-gnueabi')
 174         self.add_config(arch='armeb',
 175                         os_name='linux-gnueabi',
 176                         variant='be8',
 177                         gcc_cfg=['--with-arch=armv7-a'])
 178         self.add_config(arch='arm',
 179                         os_name='linux-gnueabihf',
 180                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 181                         extra_glibcs=[{'variant': 'v7a',
 182                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 183                                       {'variant': 'v7a-disable-multi-arch',
 184                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 185                                        'cfg': ['--disable-multi-arch']}])
 186         self.add_config(arch='armeb',
 187                         os_name='linux-gnueabihf',
 188                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 189         self.add_config(arch='armeb',
 190                         os_name='linux-gnueabihf',
 191                         variant='be8',
 192                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 193                                  '--with-fpu=vfpv3'])
 194         self.add_config(arch='hppa',
 195                         os_name='linux-gnu')
 196         self.add_config(arch='i686',
 197                         os_name='gnu')
 198         self.add_config(arch='ia64',
 199                         os_name='linux-gnu',
 200                         first_gcc_cfg=['--with-system-libunwind'])
 201         self.add_config(arch='m68k',
 202                         os_name='linux-gnu',
 203                         gcc_cfg=['--disable-multilib'])
 204         self.add_config(arch='m68k',
 205                         os_name='linux-gnu',
 206                         variant='coldfire',
 207                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 208         self.add_config(arch='m68k',
 209                         os_name='linux-gnu',
 210                         variant='coldfire-soft',
 211                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 212                                  '--disable-multilib'])
 213         self.add_config(arch='microblaze',
 214                         os_name='linux-gnu',
 215                         gcc_cfg=['--disable-multilib'])
 216         self.add_config(arch='microblazeel',
 217                         os_name='linux-gnu',
 218                         gcc_cfg=['--disable-multilib'])
 219         self.add_config(arch='mips64',
 220                         os_name='linux-gnu',
 221                         gcc_cfg=['--with-mips-plt'],
 222                         glibcs=[{'variant': 'n32'},
 223                                 {'arch': 'mips',
 224                                  'ccopts': '-mabi=32'},
 225                                 {'variant': 'n64',
 226                                  'ccopts': '-mabi=64'}])
 227         self.add_config(arch='mips64',
 228                         os_name='linux-gnu',
 229                         variant='soft',
 230                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 231                         glibcs=[{'variant': 'n32-soft'},
 232                                 {'variant': 'soft',
 233                                  'arch': 'mips',
 234                                  'ccopts': '-mabi=32'},
 235                                 {'variant': 'n64-soft',
 236                                  'ccopts': '-mabi=64'}])
 237         self.add_config(arch='mips64',
 238                         os_name='linux-gnu',
 239                         variant='nan2008',
 240                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 241                                  '--with-arch-64=mips64r2',
 242                                  '--with-arch-32=mips32r2'],
 243                         glibcs=[{'variant': 'n32-nan2008'},
 244                                 {'variant': 'nan2008',
 245                                  'arch': 'mips',
 246                                  'ccopts': '-mabi=32'},
 247                                 {'variant': 'n64-nan2008',
 248                                  'ccopts': '-mabi=64'}])
 249         self.add_config(arch='mips64',
 250                         os_name='linux-gnu',
 251                         variant='nan2008-soft',
 252                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 253                                  '--with-arch-64=mips64r2',
 254                                  '--with-arch-32=mips32r2',
 255                                  '--with-float=soft'],
 256                         glibcs=[{'variant': 'n32-nan2008-soft'},
 257                                 {'variant': 'nan2008-soft',
 258                                  'arch': 'mips',
 259                                  'ccopts': '-mabi=32'},
 260                                 {'variant': 'n64-nan2008-soft',
 261                                  'ccopts': '-mabi=64'}])
 262         self.add_config(arch='mips64el',
 263                         os_name='linux-gnu',
 264                         gcc_cfg=['--with-mips-plt'],
 265                         glibcs=[{'variant': 'n32'},
 266                                 {'arch': 'mipsel',
 267                                  'ccopts': '-mabi=32'},
 268                                 {'variant': 'n64',
 269                                  'ccopts': '-mabi=64'}])
 270         self.add_config(arch='mips64el',
 271                         os_name='linux-gnu',
 272                         variant='soft',
 273                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 274                         glibcs=[{'variant': 'n32-soft'},
 275                                 {'variant': 'soft',
 276                                  'arch': 'mipsel',
 277                                  'ccopts': '-mabi=32'},
 278                                 {'variant': 'n64-soft',
 279                                  'ccopts': '-mabi=64'}])
 280         self.add_config(arch='mips64el',
 281                         os_name='linux-gnu',
 282                         variant='nan2008',
 283                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 284                                  '--with-arch-64=mips64r2',
 285                                  '--with-arch-32=mips32r2'],
 286                         glibcs=[{'variant': 'n32-nan2008'},
 287                                 {'variant': 'nan2008',
 288                                  'arch': 'mipsel',
 289                                  'ccopts': '-mabi=32'},
 290                                 {'variant': 'n64-nan2008',
 291                                  'ccopts': '-mabi=64'}])
 292         self.add_config(arch='mips64el',
 293                         os_name='linux-gnu',
 294                         variant='nan2008-soft',
 295                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 296                                  '--with-arch-64=mips64r2',
 297                                  '--with-arch-32=mips32r2',
 298                                  '--with-float=soft'],
 299                         glibcs=[{'variant': 'n32-nan2008-soft'},
 300                                 {'variant': 'nan2008-soft',
 301                                  'arch': 'mipsel',
 302                                  'ccopts': '-mabi=32'},
 303                                 {'variant': 'n64-nan2008-soft',
 304                                  'ccopts': '-mabi=64'}])
 305         self.add_config(arch='nios2',
 306                         os_name='linux-gnu')
 307         self.add_config(arch='powerpc',
 308                         os_name='linux-gnu',
 309                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 310                         extra_glibcs=[{'variant': 'power4',
 311                                        'ccopts': '-mcpu=power4',
 312                                        'cfg': ['--with-cpu=power4']}])
 313         self.add_config(arch='powerpc',
 314                         os_name='linux-gnu',
 315                         variant='soft',
 316                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 317                                  '--enable-secureplt'])
 318         self.add_config(arch='powerpc64',
 319                         os_name='linux-gnu',
 320                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 321         self.add_config(arch='powerpc64le',
 322                         os_name='linux-gnu',
 323                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 324         self.add_config(arch='powerpc',
 325                         os_name='linux-gnuspe',
 326                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 327                                  '--enable-e500-double', '--enable-obsolete'])
 328         self.add_config(arch='powerpc',
 329                         os_name='linux-gnuspe',
 330                         variant='e500v1',
 331                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 332                                  '--enable-obsolete'])
 333         self.add_config(arch='riscv64',
 334                         os_name='linux-gnu',
 335                         variant='rv64imac-lp64',
 336                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 337                                  '--disable-multilib'])
 338         self.add_config(arch='riscv64',
 339                         os_name='linux-gnu',
 340                         variant='rv64imafdc-lp64',
 341                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 342                                  '--disable-multilib'])
 343         self.add_config(arch='riscv64',
 344                         os_name='linux-gnu',
 345                         variant='rv64imafdc-lp64d',
 346                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 347                                  '--disable-multilib'])
 348         self.add_config(arch='s390x',
 349                         os_name='linux-gnu',
 350                         glibcs=[{},
 351                                 {'arch': 's390', 'ccopts': '-m31'}])
 352         self.add_config(arch='sh3',
 353                         os_name='linux-gnu')
 354         self.add_config(arch='sh3eb',
 355                         os_name='linux-gnu')
 356         self.add_config(arch='sh4',
 357                         os_name='linux-gnu')
 358         self.add_config(arch='sh4eb',
 359                         os_name='linux-gnu')
 360         self.add_config(arch='sh4',
 361                         os_name='linux-gnu',
 362                         variant='soft',
 363                         gcc_cfg=['--without-fp'])
 364         self.add_config(arch='sh4eb',
 365                         os_name='linux-gnu',
 366                         variant='soft',
 367                         gcc_cfg=['--without-fp'])
 368         self.add_config(arch='sparc64',
 369                         os_name='linux-gnu',
 370                         glibcs=[{},
 371                                 {'arch': 'sparcv9',
 372                                  'ccopts': '-m32 -mlong-double-128'}],
 373                         extra_glibcs=[{'variant': 'disable-multi-arch',
 374                                        'cfg': ['--disable-multi-arch']},
 375                                       {'variant': 'disable-multi-arch',
 376                                        'arch': 'sparcv9',
 377                                        'ccopts': '-m32 -mlong-double-128',
 378                                        'cfg': ['--disable-multi-arch']}])
 379         self.add_config(arch='x86_64',
 380                         os_name='linux-gnu',
 381                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 382                         glibcs=[{},
 383                                 {'variant': 'x32', 'ccopts': '-mx32'},
 384                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 385                         extra_glibcs=[{'variant': 'disable-multi-arch',
 386                                        'cfg': ['--disable-multi-arch']},
 387                                       {'variant': 'static-pie',
 388                                        'cfg': ['--enable-static-pie']},
 389                                       {'variant': 'x32-static-pie',
 390                                        'ccopts': '-mx32',
 391                                        'cfg': ['--enable-static-pie']},
 392                                       {'variant': 'static-pie',
 393                                        'arch': 'i686',
 394                                        'ccopts': '-m32 -march=i686',
 395                                        'cfg': ['--enable-static-pie']},
 396                                       {'variant': 'disable-multi-arch',
 397                                        'arch': 'i686',
 398                                        'ccopts': '-m32 -march=i686',
 399                                        'cfg': ['--disable-multi-arch']},
 400                                       {'arch': 'i486',
 401                                        'ccopts': '-m32 -march=i486'},
 402                                       {'arch': 'i586',
 403                                        'ccopts': '-m32 -march=i586'}])
 404
 405     def add_config(self, **args):
 406         """Add an individual build configuration."""
 407         cfg = Config(self, **args)
 408         if cfg.name in self.configs:
 409             print('error: duplicate config %s' % cfg.name)
 410             exit(1)
 411         self.configs[cfg.name] = cfg
 412         for c in cfg.all_glibcs:
 413             if c.name in self.glibc_configs:
 414                 print('error: duplicate glibc config %s' % c.name)
 415                 exit(1)
 416             self.glibc_configs[c.name] = c
 417
 418     def component_srcdir(self, component):
 419         """Return the source directory for a given component, e.g. gcc."""
 420         return os.path.join(self.srcdir, component)
 421
 422     def component_builddir(self, action, config, component, subconfig=None):
 423         """Return the directory to use for a build."""
 424         if config is None:
 425             # Host libraries.
 426             assert subconfig is None
 427             return os.path.join(self.builddir, action, component)
 428         if subconfig is None:
 429             return os.path.join(self.builddir, action, config, component)
 430         else:
 431             # glibc build as part of compiler build.
 432             return os.path.join(self.builddir, action, config, component,
 433                                 subconfig)
 434
 435     def compiler_installdir(self, config):
 436         """Return the directory in which to install a compiler."""
 437         return os.path.join(self.installdir, 'compilers', config)
 438
 439     def compiler_bindir(self, config):
 440         """Return the directory in which to find compiler binaries."""
 441         return os.path.join(self.compiler_installdir(config), 'bin')
 442
 443     def compiler_sysroot(self, config):
 444         """Return the sysroot directory for a compiler."""
 445         return os.path.join(self.compiler_installdir(config), 'sysroot')
 446
 447     def glibc_installdir(self, config):
 448         """Return the directory in which to install glibc."""
 449         return os.path.join(self.installdir, 'glibcs', config)
 450
 451     def run_builds(self, action, configs):
 452         """Run the requested builds."""
 453         if action == 'checkout':
 454             self.checkout(configs)
 455             return
 456         if action == 'bot-cycle':
 457             if configs:
 458                 print('error: configurations specified for bot-cycle')
 459                 exit(1)
 460             self.bot_cycle()
 461             return
 462         if action == 'bot':
 463             if configs:
 464                 print('error: configurations specified for bot')
 465                 exit(1)
 466             self.bot()
 467             return
 468         if action == 'host-libraries' and configs:
 469             print('error: configurations specified for host-libraries')
 470             exit(1)
 471         self.clear_last_build_state(action)
 472         build_time = datetime.datetime.utcnow()
 473         if action == 'host-libraries':
 474             build_components = ('gmp', 'mpfr', 'mpc')
 475             old_components = ()
 476             old_versions = {}
 477             self.build_host_libraries()
 478         elif action == 'compilers':
 479             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 480                                 'gnumach', 'hurd')
 481             old_components = ('gmp', 'mpfr', 'mpc')
 482             old_versions = self.build_state['host-libraries']['build-versions']
 483             self.build_compilers(configs)
 484         else:
 485             build_components = ('glibc',)
 486             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 487                               'mig', 'gnumach', 'hurd')
 488             old_versions = self.build_state['compilers']['build-versions']
 489             self.build_glibcs(configs)
 490         self.write_files()
 491         self.do_build()
 492         if configs:
 493             # Partial build, do not update stored state.
 494             return
 495         build_versions = {}
 496         for k in build_components:
 497             if k in self.versions:
 498                 build_versions[k] = {'version': self.versions[k]['version'],
 499                                      'revision': self.versions[k]['revision']}
 500         for k in old_components:
 501             if k in old_versions:
 502                 build_versions[k] = {'version': old_versions[k]['version'],
 503                                      'revision': old_versions[k]['revision']}
 504         self.update_build_state(action, build_time, build_versions)
 505
 506     @staticmethod
 507     def remove_dirs(*args):
 508         """Remove directories and their contents if they exist."""
 509         for dir in args:
 510             shutil.rmtree(dir, ignore_errors=True)
 511
 512     @staticmethod
 513     def remove_recreate_dirs(*args):
 514         """Remove directories if they exist, and create them as empty."""
 515         Context.remove_dirs(*args)
 516         for dir in args:
 517             os.makedirs(dir, exist_ok=True)
 518
 519     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 520         """Add makefile text for a list of commands."""
 521         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 522         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 523                                     (target, target, target, commands))
 524         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 525
 526     def write_files(self):
 527         """Write out the Makefile and wrapper script."""
 528         mftext = ''.join(self.makefile_pieces)
 529         with open(self.makefile, 'w') as f:
 530             f.write(mftext)
 531         wrapper_text = (
 532             '#!/bin/sh\n'
 533             'prev_base=$1\n'
 534             'this_base=$2\n'
 535             'desc=$3\n'
 536             'dir=$4\n'
 537             'path=$5\n'
 538             'shift 5\n'
 539             'prev_status=$prev_base-status.txt\n'
 540             'this_status=$this_base-status.txt\n'
 541             'this_log=$this_base-log.txt\n'
 542             'date > "$this_log"\n'
 543             'echo >> "$this_log"\n'
 544             'echo "Description: $desc" >> "$this_log"\n'
 545             'printf "%s" "Command:" >> "$this_log"\n'
 546             'for word in "$@"; do\n'
 547             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 548             '    printf " %s" "$word"\n'
 549             '  else\n'
 550             '    printf " \'"\n'
 551             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 552             '    printf "\'"\n'
 553             '  fi\n'
 554             'done >> "$this_log"\n'
 555             'echo >> "$this_log"\n'
 556             'echo "Directory: $dir" >> "$this_log"\n'
 557             'echo "Path addition: $path" >> "$this_log"\n'
 558             'echo >> "$this_log"\n'
 559             'record_status ()\n'
 560             '{\n'
 561             '  echo >> "$this_log"\n'
 562             '  echo "$1: $desc" > "$this_status"\n'
 563             '  echo "$1: $desc" >> "$this_log"\n'
 564             '  echo >> "$this_log"\n'
 565             '  date >> "$this_log"\n'
 566             '  echo "$1: $desc"\n'
 567             '  exit 0\n'
 568             '}\n'
 569             'check_error ()\n'
 570             '{\n'
 571             '  if [ "$1" != "0" ]; then\n'
 572             '    record_status FAIL\n'
 573             '  fi\n'
 574             '}\n'
 575             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 576             '    record_status UNRESOLVED\n'
 577             'fi\n'
 578             'if [ "$dir" ]; then\n'
 579             '  cd "$dir"\n'
 580             '  check_error "$?"\n'
 581             'fi\n'
 582             'if [ "$path" ]; then\n'
 583             '  PATH=$path:$PATH\n'
 584             'fi\n'
 585             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 586             'check_error "$?"\n'
 587             'record_status PASS\n')
 588         with open(self.wrapper, 'w') as f:
 589             f.write(wrapper_text)
 590         # Mode 0o755.
 591         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 592                      stat.S_IROTH|stat.S_IXOTH)
 593         os.chmod(self.wrapper, mode_exec)
 594         save_logs_text = (
 595             '#!/bin/sh\n'
 596             'if ! [ -f tests.sum ]; then\n'
 597             '  echo "No test summary available."\n'
 598             '  exit 0\n'
 599             'fi\n'
 600             'save_file ()\n'
 601             '{\n'
 602             '  echo "Contents of $1:"\n'
 603             '  echo\n'
 604             '  cat "$1"\n'
 605             '  echo\n'
 606             '  echo "End of contents of $1."\n'
 607             '  echo\n'
 608             '}\n'
 609             'save_file tests.sum\n'
 610             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 611             'for t in $non_pass_tests; do\n'
 612             '  if [ -f "$t.out" ]; then\n'
 613             '    save_file "$t.out"\n'
 614             '  fi\n'
 615             'done\n')
 616         with open(self.save_logs, 'w') as f:
 617             f.write(save_logs_text)
 618         os.chmod(self.save_logs, mode_exec)
 619
 620     def do_build(self):
 621         """Do the actual build."""
 622         cmd = ['make', '-j%d' % self.parallelism]
 623         subprocess.run(cmd, cwd=self.builddir, check=True)
 624
 625     def build_host_libraries(self):
 626         """Build the host libraries."""
 627         installdir = self.host_libraries_installdir
 628         builddir = os.path.join(self.builddir, 'host-libraries')
 629         logsdir = os.path.join(self.logsdir, 'host-libraries')
 630         self.remove_recreate_dirs(installdir, builddir, logsdir)
 631         cmdlist = CommandList('host-libraries', self.keep)
 632         self.build_host_library(cmdlist, 'gmp')
 633         self.build_host_library(cmdlist, 'mpfr',
 634                                 ['--with-gmp=%s' % installdir])
 635         self.build_host_library(cmdlist, 'mpc',
 636                                 ['--with-gmp=%s' % installdir,
 637                                 '--with-mpfr=%s' % installdir])
 638         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 639         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 640
 641     def build_host_library(self, cmdlist, lib, extra_opts=None):
 642         """Build one host library."""
 643         srcdir = self.component_srcdir(lib)
 644         builddir = self.component_builddir('host-libraries', None, lib)
 645         installdir = self.host_libraries_installdir
 646         cmdlist.push_subdesc(lib)
 647         cmdlist.create_use_dir(builddir)
 648         cfg_cmd = [os.path.join(srcdir, 'configure'),
 649                    '--prefix=%s' % installdir,
 650                    '--disable-shared']
 651         if extra_opts:
 652             cfg_cmd.extend (extra_opts)
 653         cmdlist.add_command('configure', cfg_cmd)
 654         cmdlist.add_command('build', ['make'])
 655         cmdlist.add_command('check', ['make', 'check'])
 656         cmdlist.add_command('install', ['make', 'install'])
 657         cmdlist.cleanup_dir()
 658         cmdlist.pop_subdesc()
 659
 660     def build_compilers(self, configs):
 661         """Build the compilers."""
 662         if not configs:
 663             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 664             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 665             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 666             configs = sorted(self.configs.keys())
 667         for c in configs:
 668             self.configs[c].build()
 669
 670     def build_glibcs(self, configs):
 671         """Build the glibcs."""
 672         if not configs:
 673             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 674             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 675             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 676             configs = sorted(self.glibc_configs.keys())
 677         for c in configs:
 678             self.glibc_configs[c].build()
 679
 680     def load_versions_json(self):
 681         """Load information about source directory versions."""
 682         if not os.access(self.versions_json, os.F_OK):
 683             self.versions = {}
 684             return
 685         with open(self.versions_json, 'r') as f:
 686             self.versions = json.load(f)
 687
 688     def store_json(self, data, filename):
 689         """Store information in a JSON file."""
 690         filename_tmp = filename + '.tmp'
 691         with open(filename_tmp, 'w') as f:
 692             json.dump(data, f, indent=2, sort_keys=True)
 693         os.rename(filename_tmp, filename)
 694
 695     def store_versions_json(self):
 696         """Store information about source directory versions."""
 697         self.store_json(self.versions, self.versions_json)
 698
 699     def set_component_version(self, component, version, explicit, revision):
 700         """Set the version information for a component."""
 701         self.versions[component] = {'version': version,
 702                                     'explicit': explicit,
 703                                     'revision': revision}
 704         self.store_versions_json()
 705
 706     def checkout(self, versions):
 707         """Check out the desired component versions."""
 708         default_versions = {'binutils': 'vcs-2.30',
 709                             'gcc': 'vcs-8',
 710                             'glibc': 'vcs-mainline',
 711                             'gmp': '6.1.2',
 712                             'linux': '4.16',
 713                             'mpc': '1.1.0',
 714                             'mpfr': '4.0.1',
 715                             'mig': 'vcs-mainline',
 716                             'gnumach': 'vcs-mainline',
 717                             'hurd': 'vcs-mainline'}
 718         use_versions = {}
 719         explicit_versions = {}
 720         for v in versions:
 721             found_v = False
 722             for k in default_versions.keys():
 723                 kx = k + '-'
 724                 if v.startswith(kx):
 725                     vx = v[len(kx):]
 726                     if k in use_versions:
 727                         print('error: multiple versions for %s' % k)
 728                         exit(1)
 729                     use_versions[k] = vx
 730                     explicit_versions[k] = True
 731                     found_v = True
 732                     break
 733             if not found_v:
 734                 print('error: unknown component in %s' % v)
 735                 exit(1)
 736         for k in default_versions.keys():
 737             if k not in use_versions:
 738                 if k in self.versions and self.versions[k]['explicit']:
 739                     use_versions[k] = self.versions[k]['version']
 740                     explicit_versions[k] = True
 741                 else:
 742                     use_versions[k] = default_versions[k]
 743                     explicit_versions[k] = False
 744         os.makedirs(self.srcdir, exist_ok=True)
 745         for k in sorted(default_versions.keys()):
 746             update = os.access(self.component_srcdir(k), os.F_OK)
 747             v = use_versions[k]
 748             if (update and
 749                 k in self.versions and
 750                 v != self.versions[k]['version']):
 751                 if not self.replace_sources:
 752                     print('error: version of %s has changed from %s to %s, '
 753                           'use --replace-sources to check out again' %
 754                           (k, self.versions[k]['version'], v))
 755                     exit(1)
 756                 shutil.rmtree(self.component_srcdir(k))
 757                 update = False
 758             if v.startswith('vcs-'):
 759                 revision = self.checkout_vcs(k, v[4:], update)
 760             else:
 761                 self.checkout_tar(k, v, update)
 762                 revision = v
 763             self.set_component_version(k, v, explicit_versions[k], revision)
 764         if self.get_script_text() != self.script_text:
 765             # Rerun the checkout process in case the updated script
 766             # uses different default versions or new components.
 767             self.exec_self()
 768
 769     def checkout_vcs(self, component, version, update):
 770         """Check out the given version of the given component from version
 771         control.  Return a revision identifier."""
 772         if component == 'binutils':
 773             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 774             if version == 'mainline':
 775                 git_branch = 'master'
 776             else:
 777                 trans = str.maketrans({'.': '_'})
 778                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 779             return self.git_checkout(component, git_url, git_branch, update)
 780         elif component == 'gcc':
 781             if version == 'mainline':
 782                 branch = 'trunk'
 783             else:
 784                 trans = str.maketrans({'.': '_'})
 785                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 786             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 787             return self.gcc_checkout(svn_url, update)
 788         elif component == 'glibc':
 789             git_url = 'git://sourceware.org/git/glibc.git'
 790             if version == 'mainline':
 791                 git_branch = 'master'
 792             else:
 793                 git_branch = 'release/%s/master' % version
 794             r = self.git_checkout(component, git_url, git_branch, update)
 795             self.fix_glibc_timestamps()
 796             return r
 797         elif component == 'gnumach':
 798             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 799             git_branch = 'master'
 800             r = self.git_checkout(component, git_url, git_branch, update)
 801             subprocess.run(['autoreconf', '-i'],
 802                            cwd=self.component_srcdir(component), check=True)
 803             return r
 804         elif component == 'mig':
 805             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 806             git_branch = 'master'
 807             r = self.git_checkout(component, git_url, git_branch, update)
 808             subprocess.run(['autoreconf', '-i'],
 809                            cwd=self.component_srcdir(component), check=True)
 810             return r
 811         elif component == 'hurd':
 812             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 813             git_branch = 'master'
 814             r = self.git_checkout(component, git_url, git_branch, update)
 815             subprocess.run(['autoconf'],
 816                            cwd=self.component_srcdir(component), check=True)
 817             return r
 818         else:
 819             print('error: component %s coming from VCS' % component)
 820             exit(1)
 821
 822     def git_checkout(self, component, git_url, git_branch, update):
 823         """Check out a component from git.  Return a commit identifier."""
 824         if update:
 825             subprocess.run(['git', 'remote', 'prune', 'origin'],
 826                            cwd=self.component_srcdir(component), check=True)
 827             if self.replace_sources:
 828                 subprocess.run(['git', 'clean', '-dxfq'],
 829                                cwd=self.component_srcdir(component), check=True)
 830             subprocess.run(['git', 'pull', '-q'],
 831                            cwd=self.component_srcdir(component), check=True)
 832         else:
 833             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 834                             self.component_srcdir(component)], check=True)
 835         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 836                            cwd=self.component_srcdir(component),
 837                            stdout=subprocess.PIPE,
 838                            check=True, universal_newlines=True).stdout
 839         return r.rstrip()
 840
 841     def fix_glibc_timestamps(self):
 842         """Fix timestamps in a glibc checkout."""
 843         # Ensure that builds do not try to regenerate generated files
 844         # in the source tree.
 845         srcdir = self.component_srcdir('glibc')
 846         for dirpath, dirnames, filenames in os.walk(srcdir):
 847             for f in filenames:
 848                 if (f == 'configure' or
 849                     f == 'preconfigure' or
 850                     f.endswith('-kw.h')):
 851                     to_touch = os.path.join(dirpath, f)
 852                     subprocess.run(['touch', to_touch], check=True)
 853
 854     def gcc_checkout(self, svn_url, update):
 855         """Check out GCC from SVN.  Return the revision number."""
 856         if not update:
 857             subprocess.run(['svn', 'co', '-q', svn_url,
 858                             self.component_srcdir('gcc')], check=True)
 859         subprocess.run(['contrib/gcc_update', '--silent'],
 860                        cwd=self.component_srcdir('gcc'), check=True)
 861         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 862                            stdout=subprocess.PIPE,
 863                            check=True, universal_newlines=True).stdout
 864         return r.rstrip()
 865
 866     def checkout_tar(self, component, version, update):
 867         """Check out the given version of the given component from a
 868         tarball."""
 869         if update:
 870             return
 871         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 872                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 873                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 874                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 875                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 876                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 877                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 878                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 879                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 880         if component not in url_map:
 881             print('error: component %s coming from tarball' % component)
 882             exit(1)
 883         url = url_map[component] % {'version': version}
 884         filename = os.path.join(self.srcdir, url.split('/')[-1])
 885         response = urllib.request.urlopen(url)
 886         data = response.read()
 887         with open(filename, 'wb') as f:
 888             f.write(data)
 889         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 890                        check=True)
 891         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 892                   self.component_srcdir(component))
 893         os.remove(filename)
 894
 895     def load_build_state_json(self):
 896         """Load information about the state of previous builds."""
 897         if os.access(self.build_state_json, os.F_OK):
 898             with open(self.build_state_json, 'r') as f:
 899                 self.build_state = json.load(f)
 900         else:
 901             self.build_state = {}
 902         for k in ('host-libraries', 'compilers', 'glibcs'):
 903             if k not in self.build_state:
 904                 self.build_state[k] = {}
 905             if 'build-time' not in self.build_state[k]:
 906                 self.build_state[k]['build-time'] = ''
 907             if 'build-versions' not in self.build_state[k]:
 908                 self.build_state[k]['build-versions'] = {}
 909             if 'build-results' not in self.build_state[k]:
 910                 self.build_state[k]['build-results'] = {}
 911             if 'result-changes' not in self.build_state[k]:
 912                 self.build_state[k]['result-changes'] = {}
 913             if 'ever-passed' not in self.build_state[k]:
 914                 self.build_state[k]['ever-passed'] = []
 915
 916     def store_build_state_json(self):
 917         """Store information about the state of previous builds."""
 918         self.store_json(self.build_state, self.build_state_json)
 919
 920     def clear_last_build_state(self, action):
 921         """Clear information about the state of part of the build."""
 922         # We clear the last build time and versions when starting a
 923         # new build.  The results of the last build are kept around,
 924         # as comparison is still meaningful if this build is aborted
 925         # and a new one started.
 926         self.build_state[action]['build-time'] = ''
 927         self.build_state[action]['build-versions'] = {}
 928         self.store_build_state_json()
 929
 930     def update_build_state(self, action, build_time, build_versions):
 931         """Update the build state after a build."""
 932         build_time = build_time.replace(microsecond=0)
 933         self.build_state[action]['build-time'] = str(build_time)
 934         self.build_state[action]['build-versions'] = build_versions
 935         build_results = {}
 936         for log in self.status_log_list:
 937             with open(log, 'r') as f:
 938                 log_text = f.read()
 939             log_text = log_text.rstrip()
 940             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 941             result = m.group(1)
 942             test_name = m.group(2)
 943             assert test_name not in build_results
 944             build_results[test_name] = result
 945         old_build_results = self.build_state[action]['build-results']
 946         self.build_state[action]['build-results'] = build_results
 947         result_changes = {}
 948         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 949         for t in all_tests:
 950             if t in old_build_results:
 951                 old_res = old_build_results[t]
 952             else:
 953                 old_res = '(New test)'
 954             if t in build_results:
 955                 new_res = build_results[t]
 956             else:
 957                 new_res = '(Test removed)'
 958             if old_res != new_res:
 959                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 960         self.build_state[action]['result-changes'] = result_changes
 961         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 962                            if t in build_results}
 963         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 964         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 965                                                          new_passes)
 966         self.store_build_state_json()
 967
 968     def load_bot_config_json(self):
 969         """Load bot configuration."""
 970         with open(self.bot_config_json, 'r') as f:
 971             self.bot_config = json.load(f)
 972
 973     def part_build_old(self, action, delay):
 974         """Return whether the last build for a given action was at least a
 975         given number of seconds ago, or does not have a time recorded."""
 976         old_time_str = self.build_state[action]['build-time']
 977         if not old_time_str:
 978             return True
 979         old_time = datetime.datetime.strptime(old_time_str,
 980                                               '%Y-%m-%d %H:%M:%S')
 981         new_time = datetime.datetime.utcnow()
 982         delta = new_time - old_time
 983         return delta.total_seconds() >= delay
 984
 985     def bot_cycle(self):
 986         """Run a single round of checkout and builds."""
 987         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 988         self.load_bot_config_json()
 989         actions = ('host-libraries', 'compilers', 'glibcs')
 990         self.bot_run_self(['--replace-sources'], 'checkout')
 991         self.load_versions_json()
 992         if self.get_script_text() != self.script_text:
 993             print('Script changed, re-execing.')
 994             # On script change, all parts of the build should be rerun.
 995             for a in actions:
 996                 self.clear_last_build_state(a)
 997             self.exec_self()
 998         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 999                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1000                                           'mig', 'gnumach', 'hurd'),
1001                             'glibcs': ('glibc',)}
1002         must_build = {}
1003         for a in actions:
1004             build_vers = self.build_state[a]['build-versions']
1005             must_build[a] = False
1006             if not self.build_state[a]['build-time']:
1007                 must_build[a] = True
1008             old_vers = {}
1009             new_vers = {}
1010             for c in check_components[a]:
1011                 if c in build_vers:
1012                     old_vers[c] = build_vers[c]
1013                 new_vers[c] = {'version': self.versions[c]['version'],
1014                                'revision': self.versions[c]['revision']}
1015             if new_vers == old_vers:
1016                 print('Versions for %s unchanged.' % a)
1017             else:
1018                 print('Versions changed or rebuild forced for %s.' % a)
1019                 if a == 'compilers' and not self.part_build_old(
1020                         a, self.bot_config['compilers-rebuild-delay']):
1021                     print('Not requiring rebuild of compilers this soon.')
1022                 else:
1023                     must_build[a] = True
1024         if must_build['host-libraries']:
1025             must_build['compilers'] = True
1026         if must_build['compilers']:
1027             must_build['glibcs'] = True
1028         for a in actions:
1029             if must_build[a]:
1030                 print('Must rebuild %s.' % a)
1031                 self.clear_last_build_state(a)
1032             else:
1033                 print('No need to rebuild %s.' % a)
1034         if os.access(self.logsdir, os.F_OK):
1035             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1036             shutil.copytree(self.logsdir, self.logsdir_old)
1037         for a in actions:
1038             if must_build[a]:
1039                 build_time = datetime.datetime.utcnow()
1040                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1041                 self.bot_run_self([], a)
1042                 self.load_build_state_json()
1043                 self.bot_build_mail(a, build_time)
1044         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1045
1046     def bot_build_mail(self, action, build_time):
1047         """Send email with the results of a build."""
1048         if not ('email-from' in self.bot_config and
1049                 'email-server' in self.bot_config and
1050                 'email-subject' in self.bot_config and
1051                 'email-to' in self.bot_config):
1052             if not self.email_warning:
1053                 print("Email not configured, not sending.")
1054                 self.email_warning = True
1055             return
1056
1057         build_time = build_time.replace(microsecond=0)
1058         subject = (self.bot_config['email-subject'] %
1059                    {'action': action,
1060                     'build-time': str(build_time)})
1061         results = self.build_state[action]['build-results']
1062         changes = self.build_state[action]['result-changes']
1063         ever_passed = set(self.build_state[action]['ever-passed'])
1064         versions = self.build_state[action]['build-versions']
1065         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1066         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1067         all_fails = {k for k in results if results[k] == 'FAIL'}
1068         if new_regressions:
1069             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1070             new_reg_text = ('New regressions:\n\n%s\n\n' %
1071                             '\n'.join(new_reg_list))
1072         else:
1073             new_reg_text = ''
1074         if all_regressions:
1075             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1076             all_reg_text = ('All regressions:\n\n%s\n\n' %
1077                             '\n'.join(all_reg_list))
1078         else:
1079             all_reg_text = ''
1080         if all_fails:
1081             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1082             all_fail_text = ('All failures:\n\n%s\n\n' %
1083                              '\n'.join(all_fail_list))
1084         else:
1085             all_fail_text = ''
1086         if changes:
1087             changes_list = sorted(changes.keys())
1088             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1089             changes_text = ('All changed results:\n\n%s\n\n' %
1090                             '\n'.join(changes_list))
1091         else:
1092             changes_text = ''
1093         results_text = (new_reg_text + all_reg_text + all_fail_text +
1094                         changes_text)
1095         if not results_text:
1096             results_text = 'Clean build with unchanged results.\n\n'
1097         versions_list = sorted(versions.keys())
1098         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1099                                           versions[k]['revision'])
1100                          for k in versions_list]
1101         versions_text = ('Component versions for this build:\n\n%s\n' %
1102                          '\n'.join(versions_list))
1103         body_text = results_text + versions_text
1104         msg = email.mime.text.MIMEText(body_text)
1105         msg['Subject'] = subject
1106         msg['From'] = self.bot_config['email-from']
1107         msg['To'] = self.bot_config['email-to']
1108         msg['Message-ID'] = email.utils.make_msgid()
1109         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1110         with smtplib.SMTP(self.bot_config['email-server']) as s:
1111             s.send_message(msg)
1112
1113     def bot_run_self(self, opts, action, check=True):
1114         """Run a copy of this script with given options."""
1115         cmd = [sys.executable, sys.argv[0], '--keep=none',
1116                '-j%d' % self.parallelism]
1117         cmd.extend(opts)
1118         cmd.extend([self.topdir, action])
1119         sys.stdout.flush()
1120         subprocess.run(cmd, check=check)
1121
1122     def bot(self):
1123         """Run repeated rounds of checkout and builds."""
1124         while True:
1125             self.load_bot_config_json()
1126             if not self.bot_config['run']:
1127                 print('Bot exiting by request.')
1128                 exit(0)
1129             self.bot_run_self([], 'bot-cycle', check=False)
1130             self.load_bot_config_json()
1131             if not self.bot_config['run']:
1132                 print('Bot exiting by request.')
1133                 exit(0)
1134             time.sleep(self.bot_config['delay'])
1135             if self.get_script_text() != self.script_text:
1136                 print('Script changed, bot re-execing.')
1137                 self.exec_self()
1138
1139
1140 class Config(object):
1141     """A configuration for building a compiler and associated libraries."""
1142
1143     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1144                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1145         """Initialize a Config object."""
1146         self.ctx = ctx
1147         self.arch = arch
1148         self.os = os_name
1149         self.variant = variant
1150         if variant is None:
1151             self.name = '%s-%s' % (arch, os_name)
1152         else:
1153             self.name = '%s-%s-%s' % (arch, os_name, variant)
1154         self.triplet = '%s-glibc-%s' % (arch, os_name)
1155         if gcc_cfg is None:
1156             self.gcc_cfg = []
1157         else:
1158             self.gcc_cfg = gcc_cfg
1159         if first_gcc_cfg is None:
1160             self.first_gcc_cfg = []
1161         else:
1162             self.first_gcc_cfg = first_gcc_cfg
1163         if glibcs is None:
1164             glibcs = [{'variant': variant}]
1165         if extra_glibcs is None:
1166             extra_glibcs = []
1167         glibcs = [Glibc(self, **g) for g in glibcs]
1168         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1169         self.all_glibcs = glibcs + extra_glibcs
1170         self.compiler_glibcs = glibcs
1171         self.installdir = ctx.compiler_installdir(self.name)
1172         self.bindir = ctx.compiler_bindir(self.name)
1173         self.sysroot = ctx.compiler_sysroot(self.name)
1174         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1175         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1176
1177     def component_builddir(self, component):
1178         """Return the directory to use for a (non-glibc) build."""
1179         return self.ctx.component_builddir('compilers', self.name, component)
1180
1181     def build(self):
1182         """Generate commands to build this compiler."""
1183         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1184                                       self.logsdir)
1185         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1186         cmdlist.add_command('check-host-libraries',
1187                             ['test', '-f',
1188                              os.path.join(self.ctx.host_libraries_installdir,
1189                                           'ok')])
1190         cmdlist.use_path(self.bindir)
1191         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1192                               ['--disable-gdb',
1193                                '--disable-libdecnumber',
1194                                '--disable-readline',
1195                                '--disable-sim'])
1196         if self.os.startswith('linux'):
1197             self.install_linux_headers(cmdlist)
1198         self.build_gcc(cmdlist, True)
1199         if self.os == 'gnu':
1200             self.install_gnumach_headers(cmdlist)
1201             self.build_cross_tool(cmdlist, 'mig', 'mig')
1202             self.install_hurd_headers(cmdlist)
1203         for g in self.compiler_glibcs:
1204             cmdlist.push_subdesc('glibc')
1205             cmdlist.push_subdesc(g.name)
1206             g.build_glibc(cmdlist, True)
1207             cmdlist.pop_subdesc()
1208             cmdlist.pop_subdesc()
1209         self.build_gcc(cmdlist, False)
1210         cmdlist.add_command('done', ['touch',
1211                                      os.path.join(self.installdir, 'ok')])
1212         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1213                                       self.logsdir)
1214
1215     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1216         """Build one cross tool."""
1217         srcdir = self.ctx.component_srcdir(tool_src)
1218         builddir = self.component_builddir(tool_build)
1219         cmdlist.push_subdesc(tool_build)
1220         cmdlist.create_use_dir(builddir)
1221         cfg_cmd = [os.path.join(srcdir, 'configure'),
1222                    '--prefix=%s' % self.installdir,
1223                    '--build=%s' % self.ctx.build_triplet,
1224                    '--host=%s' % self.ctx.build_triplet,
1225                    '--target=%s' % self.triplet,
1226                    '--with-sysroot=%s' % self.sysroot]
1227         if extra_opts:
1228             cfg_cmd.extend(extra_opts)
1229         cmdlist.add_command('configure', cfg_cmd)
1230         cmdlist.add_command('build', ['make'])
1231         # Parallel "make install" for GCC has race conditions that can
1232         # cause it to fail; see
1233         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1234         # problems are not known for binutils, but doing the
1235         # installation in parallel within a particular toolchain build
1236         # (as opposed to installation of one toolchain from
1237         # build-many-glibcs.py running in parallel to the installation
1238         # of other toolchains being built) is not known to be
1239         # significantly beneficial, so it is simplest just to disable
1240         # parallel install for cross tools here.
1241         cmdlist.add_command('install', ['make', '-j1', 'install'])
1242         cmdlist.cleanup_dir()
1243         cmdlist.pop_subdesc()
1244
1245     def install_linux_headers(self, cmdlist):
1246         """Install Linux kernel headers."""
1247         arch_map = {'aarch64': 'arm64',
1248                     'alpha': 'alpha',
1249                     'arm': 'arm',
1250                     'hppa': 'parisc',
1251                     'i486': 'x86',
1252                     'i586': 'x86',
1253                     'i686': 'x86',
1254                     'i786': 'x86',
1255                     'ia64': 'ia64',
1256                     'm68k': 'm68k',
1257                     'microblaze': 'microblaze',
1258                     'mips': 'mips',
1259                     'nios2': 'nios2',
1260                     'powerpc': 'powerpc',
1261                     's390': 's390',
1262                     'riscv32': 'riscv',
1263                     'riscv64': 'riscv',
1264                     'sh': 'sh',
1265                     'sparc': 'sparc',
1266                     'x86_64': 'x86'}
1267         linux_arch = None
1268         for k in arch_map:
1269             if self.arch.startswith(k):
1270                 linux_arch = arch_map[k]
1271                 break
1272         assert linux_arch is not None
1273         srcdir = self.ctx.component_srcdir('linux')
1274         builddir = self.component_builddir('linux')
1275         headers_dir = os.path.join(self.sysroot, 'usr')
1276         cmdlist.push_subdesc('linux')
1277         cmdlist.create_use_dir(builddir)
1278         cmdlist.add_command('install-headers',
1279                             ['make', '-C', srcdir, 'O=%s' % builddir,
1280                              'ARCH=%s' % linux_arch,
1281                              'INSTALL_HDR_PATH=%s' % headers_dir,
1282                              'headers_install'])
1283         cmdlist.cleanup_dir()
1284         cmdlist.pop_subdesc()
1285
1286     def install_gnumach_headers(self, cmdlist):
1287         """Install GNU Mach headers."""
1288         srcdir = self.ctx.component_srcdir('gnumach')
1289         builddir = self.component_builddir('gnumach')
1290         cmdlist.push_subdesc('gnumach')
1291         cmdlist.create_use_dir(builddir)
1292         cmdlist.add_command('configure',
1293                             [os.path.join(srcdir, 'configure'),
1294                              '--build=%s' % self.ctx.build_triplet,
1295                              '--host=%s' % self.triplet,
1296                              '--prefix=',
1297                              'CC=%s-gcc -nostdlib' % self.triplet])
1298         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1299                                         'install-data'])
1300         cmdlist.cleanup_dir()
1301         cmdlist.pop_subdesc()
1302
1303     def install_hurd_headers(self, cmdlist):
1304         """Install Hurd headers."""
1305         srcdir = self.ctx.component_srcdir('hurd')
1306         builddir = self.component_builddir('hurd')
1307         cmdlist.push_subdesc('hurd')
1308         cmdlist.create_use_dir(builddir)
1309         cmdlist.add_command('configure',
1310                             [os.path.join(srcdir, 'configure'),
1311                              '--build=%s' % self.ctx.build_triplet,
1312                              '--host=%s' % self.triplet,
1313                              '--prefix=',
1314                              '--disable-profile', '--without-parted',
1315                              'CC=%s-gcc -nostdlib' % self.triplet])
1316         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1317                                         'no_deps=t', 'install-headers'])
1318         cmdlist.cleanup_dir()
1319         cmdlist.pop_subdesc()
1320
1321     def build_gcc(self, cmdlist, bootstrap):
1322         """Build GCC."""
1323         # libsanitizer commonly breaks because of glibc header
1324         # changes, or on unusual targets.  libssp is of little
1325         # relevance with glibc's own stack checking support.
1326         # libcilkrts does not support GNU/Hurd (and has been removed
1327         # in GCC 8, so --disable-libcilkrts can be removed once glibc
1328         # no longer supports building with older GCC versions).
1329         cfg_opts = list(self.gcc_cfg)
1330         cfg_opts += ['--disable-libsanitizer', '--disable-libssp',
1331                      '--disable-libcilkrts']
1332         host_libs = self.ctx.host_libraries_installdir
1333         cfg_opts += ['--with-gmp=%s' % host_libs,
1334                      '--with-mpfr=%s' % host_libs,
1335                      '--with-mpc=%s' % host_libs]
1336         if bootstrap:
1337             tool_build = 'gcc-first'
1338             # Building a static-only, C-only compiler that is
1339             # sufficient to build glibc.  Various libraries and
1340             # features that may require libc headers must be disabled.
1341             # When configuring with a sysroot, --with-newlib is
1342             # required to define inhibit_libc (to stop some parts of
1343             # libgcc including libc headers); --without-headers is not
1344             # sufficient.
1345             cfg_opts += ['--enable-languages=c', '--disable-shared',
1346                          '--disable-threads',
1347                          '--disable-libatomic',
1348                          '--disable-decimal-float',
1349                          '--disable-libffi',
1350                          '--disable-libgomp',
1351                          '--disable-libitm',
1352                          '--disable-libmpx',
1353                          '--disable-libquadmath',
1354                          '--without-headers', '--with-newlib',
1355                          '--with-glibc-version=%s' % self.ctx.glibc_version
1356                          ]
1357             cfg_opts += self.first_gcc_cfg
1358         else:
1359             tool_build = 'gcc'
1360             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1361                          '--enable-threads']
1362         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1363
1364
1365 class Glibc(object):
1366     """A configuration for building glibc."""
1367
1368     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1369                  cfg=None, ccopts=None):
1370         """Initialize a Glibc object."""
1371         self.ctx = compiler.ctx
1372         self.compiler = compiler
1373         if arch is None:
1374             self.arch = compiler.arch
1375         else:
1376             self.arch = arch
1377         if os_name is None:
1378             self.os = compiler.os
1379         else:
1380             self.os = os_name
1381         self.variant = variant
1382         if variant is None:
1383             self.name = '%s-%s' % (self.arch, self.os)
1384         else:
1385             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1386         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1387         if cfg is None:
1388             self.cfg = []
1389         else:
1390             self.cfg = cfg
1391         self.ccopts = ccopts
1392
1393     def tool_name(self, tool):
1394         """Return the name of a cross-compilation tool."""
1395         ctool = '%s-%s' % (self.compiler.triplet, tool)
1396         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1397             ctool = '%s %s' % (ctool, self.ccopts)
1398         return ctool
1399
1400     def build(self):
1401         """Generate commands to build this glibc."""
1402         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1403         installdir = self.ctx.glibc_installdir(self.name)
1404         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1405         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1406         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1407         cmdlist.add_command('check-compilers',
1408                             ['test', '-f',
1409                              os.path.join(self.compiler.installdir, 'ok')])
1410         cmdlist.use_path(self.compiler.bindir)
1411         self.build_glibc(cmdlist, False)
1412         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1413                                       logsdir)
1414
1415     def build_glibc(self, cmdlist, for_compiler):
1416         """Generate commands to build this glibc, either as part of a compiler
1417         build or with the bootstrapped compiler (and in the latter case, run
1418         tests as well)."""
1419         srcdir = self.ctx.component_srcdir('glibc')
1420         if for_compiler:
1421             builddir = self.ctx.component_builddir('compilers',
1422                                                    self.compiler.name, 'glibc',
1423                                                    self.name)
1424             installdir = self.compiler.sysroot
1425             srcdir_copy = self.ctx.component_builddir('compilers',
1426                                                       self.compiler.name,
1427                                                       'glibc-src',
1428                                                       self.name)
1429         else:
1430             builddir = self.ctx.component_builddir('glibcs', self.name,
1431                                                    'glibc')
1432             installdir = self.ctx.glibc_installdir(self.name)
1433             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1434                                                       'glibc-src')
1435         cmdlist.create_use_dir(builddir)
1436         # glibc builds write into the source directory, and even if
1437         # not intentionally there is a risk of bugs that involve
1438         # writing into the working directory.  To avoid possible
1439         # concurrency issues, copy the source directory.
1440         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1441         use_usr = self.os != 'gnu'
1442         prefix = '/usr' if use_usr else ''
1443         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1444                    '--prefix=%s' % prefix,
1445                    '--enable-profile',
1446                    '--build=%s' % self.ctx.build_triplet,
1447                    '--host=%s' % self.triplet,
1448                    'CC=%s' % self.tool_name('gcc'),
1449                    'CXX=%s' % self.tool_name('g++'),
1450                    'AR=%s' % self.tool_name('ar'),
1451                    'AS=%s' % self.tool_name('as'),
1452                    'LD=%s' % self.tool_name('ld'),
1453                    'NM=%s' % self.tool_name('nm'),
1454                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1455                    'OBJDUMP=%s' % self.tool_name('objdump'),
1456                    'RANLIB=%s' % self.tool_name('ranlib'),
1457                    'READELF=%s' % self.tool_name('readelf'),
1458                    'STRIP=%s' % self.tool_name('strip')]
1459         if self.os == 'gnu':
1460             cfg_cmd += ['MIG=%s' % self.tool_name('mig')]
1461         cfg_cmd += self.cfg
1462         cmdlist.add_command('configure', cfg_cmd)
1463         cmdlist.add_command('build', ['make'])
1464         cmdlist.add_command('install', ['make', 'install',
1465                                         'install_root=%s' % installdir])
1466         # GCC uses paths such as lib/../lib64, so make sure lib
1467         # directories always exist.
1468         mkdir_cmd = ['mkdir', '-p',
1469                      os.path.join(installdir, 'lib')]
1470         if use_usr:
1471             mkdir_cmd += [os.path.join(installdir, 'usr', 'lib')]
1472         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1473         if not for_compiler:
1474             if self.ctx.strip:
1475                 cmdlist.add_command('strip',
1476                                     ['sh', '-c',
1477                                      ('%s $(find %s/lib* -name "*.so")' %
1478                                       (self.tool_name('strip'), installdir))])
1479             cmdlist.add_command('check', ['make', 'check'])
1480             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1481                                 always_run=True)
1482         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1483         cmdlist.cleanup_dir()
1484
1485
1486 class Command(object):
1487     """A command run in the build process."""
1488
1489     def __init__(self, desc, num, dir, path, command, always_run=False):
1490         """Initialize a Command object."""
1491         self.dir = dir
1492         self.path = path
1493         self.desc = desc
1494         trans = str.maketrans({' ': '-'})
1495         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1496         self.command = command
1497         self.always_run = always_run
1498
1499     @staticmethod
1500     def shell_make_quote_string(s):
1501         """Given a string not containing a newline, quote it for use by the
1502         shell and make."""
1503         assert '\n' not in s
1504         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1505             return s
1506         strans = str.maketrans({"'": "'\\''"})
1507         s = "'%s'" % s.translate(strans)
1508         mtrans = str.maketrans({'$': '$$'})
1509         return s.translate(mtrans)
1510
1511     @staticmethod
1512     def shell_make_quote_list(l, translate_make):
1513         """Given a list of strings not containing newlines, quote them for use
1514         by the shell and make, returning a single string.  If translate_make
1515         is true and the first string is 'make', change it to $(MAKE)."""
1516         l = [Command.shell_make_quote_string(s) for s in l]
1517         if translate_make and l[0] == 'make':
1518             l[0] = '$(MAKE)'
1519         return ' '.join(l)
1520
1521     def shell_make_quote(self):
1522         """Return this command quoted for the shell and make."""
1523         return self.shell_make_quote_list(self.command, True)
1524
1525
1526 class CommandList(object):
1527     """A list of commands run in the build process."""
1528
1529     def __init__(self, desc, keep):
1530         """Initialize a CommandList object."""
1531         self.cmdlist = []
1532         self.dir = None
1533         self.path = None
1534         self.desc = [desc]
1535         self.keep = keep
1536
1537     def desc_txt(self, desc):
1538         """Return the description to use for a command."""
1539         return '%s %s' % (' '.join(self.desc), desc)
1540
1541     def use_dir(self, dir):
1542         """Set the default directory for subsequent commands."""
1543         self.dir = dir
1544
1545     def use_path(self, path):
1546         """Set a directory to be prepended to the PATH for subsequent
1547         commands."""
1548         self.path = path
1549
1550     def push_subdesc(self, subdesc):
1551         """Set the default subdescription for subsequent commands (e.g., the
1552         name of a component being built, within the series of commands
1553         building it)."""
1554         self.desc.append(subdesc)
1555
1556     def pop_subdesc(self):
1557         """Pop a subdescription from the list of descriptions."""
1558         self.desc.pop()
1559
1560     def create_use_dir(self, dir):
1561         """Remove and recreate a directory and use it for subsequent
1562         commands."""
1563         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1564         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1565         self.use_dir(dir)
1566
1567     def create_copy_dir(self, src, dest):
1568         """Remove a directory and recreate it as a copy from the given
1569         source."""
1570         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1571         parent = os.path.dirname(dest)
1572         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1573         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1574
1575     def add_command_dir(self, desc, dir, command, always_run=False):
1576         """Add a command to run in a given directory."""
1577         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1578                       command, always_run)
1579         self.cmdlist.append(cmd)
1580
1581     def add_command(self, desc, command, always_run=False):
1582         """Add a command to run in the default directory."""
1583         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1584                       self.path, command, always_run)
1585         self.cmdlist.append(cmd)
1586
1587     def cleanup_dir(self, desc='cleanup', dir=None):
1588         """Clean up a build directory.  If no directory is specified, the
1589         default directory is cleaned up and ceases to be the default
1590         directory."""
1591         if dir is None:
1592             dir = self.dir
1593             self.use_dir(None)
1594         if self.keep != 'all':
1595             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1596                                  always_run=(self.keep == 'none'))
1597
1598     def makefile_commands(self, wrapper, logsdir):
1599         """Return the sequence of commands in the form of text for a Makefile.
1600         The given wrapper script takes arguments: base of logs for
1601         previous command, or empty; base of logs for this command;
1602         description; directory; PATH addition; the command itself."""
1603         # prev_base is the base of the name for logs of the previous
1604         # command that is not always-run (that is, a build command,
1605         # whose failure should stop subsequent build commands from
1606         # being run, as opposed to a cleanup command, which is run
1607         # even if previous commands failed).
1608         prev_base = ''
1609         cmds = []
1610         for c in self.cmdlist:
1611             ctxt = c.shell_make_quote()
1612             if prev_base and not c.always_run:
1613                 prev_log = os.path.join(logsdir, prev_base)
1614             else:
1615                 prev_log = ''
1616             this_log = os.path.join(logsdir, c.logbase)
1617             if not c.always_run:
1618                 prev_base = c.logbase
1619             if c.dir is None:
1620                 dir = ''
1621             else:
1622                 dir = c.dir
1623             if c.path is None:
1624                 path = ''
1625             else:
1626                 path = c.path
1627             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1628             prelim_txt = Command.shell_make_quote_list(prelims, False)
1629             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1630         return '\n'.join(cmds)
1631
1632     def status_logs(self, logsdir):
1633         """Return the list of log files with command status."""
1634         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1635                 for c in self.cmdlist]
1636
1637
1638 def get_parser():
1639     """Return an argument parser for this module."""
1640     parser = argparse.ArgumentParser(description=__doc__)
1641     parser.add_argument('-j', dest='parallelism',
1642                         help='Run this number of jobs in parallel',
1643                         type=int, default=os.cpu_count())
1644     parser.add_argument('--keep', dest='keep',
1645                         help='Whether to keep all build directories, '
1646                         'none or only those from failed builds',
1647                         default='none', choices=('none', 'all', 'failed'))
1648     parser.add_argument('--replace-sources', action='store_true',
1649                         help='Remove and replace source directories '
1650                         'with the wrong version of a component')
1651     parser.add_argument('--strip', action='store_true',
1652                         help='Strip installed glibc libraries')
1653     parser.add_argument('topdir',
1654                         help='Toplevel working directory')
1655     parser.add_argument('action',
1656                         help='What to do',
1657                         choices=('checkout', 'bot-cycle', 'bot',
1658                                  'host-libraries', 'compilers', 'glibcs'))
1659     parser.add_argument('configs',
1660                         help='Versions to check out or configurations to build',
1661                         nargs='*')
1662     return parser
1663
1664
1665 def main(argv):
1666     """The main entry point."""
1667     parser = get_parser()
1668     opts = parser.parse_args(argv)
1669     topdir = os.path.abspath(opts.topdir)
1670     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1671                   opts.strip, opts.action)
1672     ctx.run_builds(opts.action, opts.configs)
1673
1674
1675 if __name__ == '__main__':
1676     main(sys.argv[1:])