scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2017 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52 try:
  53     os.cpu_count
  54 except:
  55     import multiprocessing
  56     os.cpu_count = lambda: multiprocessing.cpu_count()
  57
  58 try:
  59     re.fullmatch
  60 except:
  61     re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)
  62
  63 try:
  64     subprocess.run
  65 except:
  66     class _CompletedProcess:
  67         def __init__(self, args, returncode, stdout=None, stderr=None):
  68             self.args = args
  69             self.returncode = returncode
  70             self.stdout = stdout
  71             self.stderr = stderr
  72
  73     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  74         assert(timeout is None)
  75         with subprocess.Popen(*popenargs, **kwargs) as process:
  76             try:
  77                 stdout, stderr = process.communicate(input)
  78             except:
  79                 process.kill()
  80                 process.wait()
  81                 raise
  82             returncode = process.poll()
  83             if check and returncode:
  84                 raise subprocess.CalledProcessError(returncode, popenargs)
  85         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  86
  87     subprocess.run = _run
  88
  89
  90 class Context(object):
  91     """The global state associated with builds in a given directory."""
  92
  93     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  94                  action):
  95         """Initialize the context."""
  96         self.topdir = topdir
  97         self.parallelism = parallelism
  98         self.keep = keep
  99         self.replace_sources = replace_sources
 100         self.strip = strip
 101         self.srcdir = os.path.join(topdir, 'src')
 102         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 103         self.build_state_json = os.path.join(topdir, 'build-state.json')
 104         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 105         self.installdir = os.path.join(topdir, 'install')
 106         self.host_libraries_installdir = os.path.join(self.installdir,
 107                                                       'host-libraries')
 108         self.builddir = os.path.join(topdir, 'build')
 109         self.logsdir = os.path.join(topdir, 'logs')
 110         self.logsdir_old = os.path.join(topdir, 'logs-old')
 111         self.makefile = os.path.join(self.builddir, 'Makefile')
 112         self.wrapper = os.path.join(self.builddir, 'wrapper')
 113         self.save_logs = os.path.join(self.builddir, 'save-logs')
 114         self.script_text = self.get_script_text()
 115         if action != 'checkout':
 116             self.build_triplet = self.get_build_triplet()
 117             self.glibc_version = self.get_glibc_version()
 118         self.configs = {}
 119         self.glibc_configs = {}
 120         self.makefile_pieces = ['.PHONY: all\n']
 121         self.add_all_configs()
 122         self.load_versions_json()
 123         self.load_build_state_json()
 124         self.status_log_list = []
 125         self.email_warning = False
 126
 127     def get_script_text(self):
 128         """Return the text of this script."""
 129         with open(sys.argv[0], 'r') as f:
 130             return f.read()
 131
 132     def exec_self(self):
 133         """Re-execute this script with the same arguments."""
 134         sys.stdout.flush()
 135         os.execv(sys.executable, [sys.executable] + sys.argv)
 136
 137     def get_build_triplet(self):
 138         """Determine the build triplet with config.guess."""
 139         config_guess = os.path.join(self.component_srcdir('gcc'),
 140                                     'config.guess')
 141         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 142                                 check=True, universal_newlines=True).stdout
 143         return cg_out.rstrip()
 144
 145     def get_glibc_version(self):
 146         """Determine the glibc version number (major.minor)."""
 147         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 148         with open(version_h, 'r') as f:
 149             lines = f.readlines()
 150         starttext = '#define VERSION "'
 151         for l in lines:
 152             if l.startswith(starttext):
 153                 l = l[len(starttext):]
 154                 l = l.rstrip('"\n')
 155                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 156                 return '%s.%s' % m.group(1, 2)
 157         print('error: could not determine glibc version')
 158         exit(1)
 159
 160     def add_all_configs(self):
 161         """Add all known glibc build configurations."""
 162         self.add_config(arch='aarch64',
 163                         os_name='linux-gnu')
 164         self.add_config(arch='aarch64_be',
 165                         os_name='linux-gnu')
 166         self.add_config(arch='alpha',
 167                         os_name='linux-gnu')
 168         self.add_config(arch='arm',
 169                         os_name='linux-gnueabi')
 170         self.add_config(arch='armeb',
 171                         os_name='linux-gnueabi')
 172         self.add_config(arch='armeb',
 173                         os_name='linux-gnueabi',
 174                         variant='be8',
 175                         gcc_cfg=['--with-arch=armv7-a'])
 176         self.add_config(arch='arm',
 177                         os_name='linux-gnueabihf',
 178                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 179                         extra_glibcs=[{'variant': 'v7a',
 180                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 181                                       {'variant': 'v7a-disable-multi-arch',
 182                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 183                                        'cfg': ['--disable-multi-arch']}])
 184         self.add_config(arch='armeb',
 185                         os_name='linux-gnueabihf',
 186                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 187         self.add_config(arch='armeb',
 188                         os_name='linux-gnueabihf',
 189                         variant='be8',
 190                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 191                                  '--with-fpu=vfpv3'])
 192         self.add_config(arch='hppa',
 193                         os_name='linux-gnu')
 194         self.add_config(arch='ia64',
 195                         os_name='linux-gnu',
 196                         first_gcc_cfg=['--with-system-libunwind'])
 197         self.add_config(arch='m68k',
 198                         os_name='linux-gnu',
 199                         gcc_cfg=['--disable-multilib'])
 200         self.add_config(arch='m68k',
 201                         os_name='linux-gnu',
 202                         variant='coldfire',
 203                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 204         self.add_config(arch='microblaze',
 205                         os_name='linux-gnu',
 206                         gcc_cfg=['--disable-multilib'])
 207         self.add_config(arch='microblazeel',
 208                         os_name='linux-gnu',
 209                         gcc_cfg=['--disable-multilib'])
 210         self.add_config(arch='mips64',
 211                         os_name='linux-gnu',
 212                         gcc_cfg=['--with-mips-plt'],
 213                         glibcs=[{'variant': 'n32'},
 214                                 {'arch': 'mips',
 215                                  'ccopts': '-mabi=32'},
 216                                 {'variant': 'n64',
 217                                  'ccopts': '-mabi=64'}])
 218         self.add_config(arch='mips64',
 219                         os_name='linux-gnu',
 220                         variant='soft',
 221                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 222                         glibcs=[{'variant': 'n32-soft'},
 223                                 {'variant': 'soft',
 224                                  'arch': 'mips',
 225                                  'ccopts': '-mabi=32'},
 226                                 {'variant': 'n64-soft',
 227                                  'ccopts': '-mabi=64'}])
 228         self.add_config(arch='mips64',
 229                         os_name='linux-gnu',
 230                         variant='nan2008',
 231                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 232                                  '--with-arch-64=mips64r2',
 233                                  '--with-arch-32=mips32r2'],
 234                         glibcs=[{'variant': 'n32-nan2008'},
 235                                 {'variant': 'nan2008',
 236                                  'arch': 'mips',
 237                                  'ccopts': '-mabi=32'},
 238                                 {'variant': 'n64-nan2008',
 239                                  'ccopts': '-mabi=64'}])
 240         self.add_config(arch='mips64',
 241                         os_name='linux-gnu',
 242                         variant='nan2008-soft',
 243                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 244                                  '--with-arch-64=mips64r2',
 245                                  '--with-arch-32=mips32r2',
 246                                  '--with-float=soft'],
 247                         glibcs=[{'variant': 'n32-nan2008-soft'},
 248                                 {'variant': 'nan2008-soft',
 249                                  'arch': 'mips',
 250                                  'ccopts': '-mabi=32'},
 251                                 {'variant': 'n64-nan2008-soft',
 252                                  'ccopts': '-mabi=64'}])
 253         self.add_config(arch='mips64el',
 254                         os_name='linux-gnu',
 255                         gcc_cfg=['--with-mips-plt'],
 256                         glibcs=[{'variant': 'n32'},
 257                                 {'arch': 'mipsel',
 258                                  'ccopts': '-mabi=32'},
 259                                 {'variant': 'n64',
 260                                  'ccopts': '-mabi=64'}])
 261         self.add_config(arch='mips64el',
 262                         os_name='linux-gnu',
 263                         variant='soft',
 264                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 265                         glibcs=[{'variant': 'n32-soft'},
 266                                 {'variant': 'soft',
 267                                  'arch': 'mipsel',
 268                                  'ccopts': '-mabi=32'},
 269                                 {'variant': 'n64-soft',
 270                                  'ccopts': '-mabi=64'}])
 271         self.add_config(arch='mips64el',
 272                         os_name='linux-gnu',
 273                         variant='nan2008',
 274                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 275                                  '--with-arch-64=mips64r2',
 276                                  '--with-arch-32=mips32r2'],
 277                         glibcs=[{'variant': 'n32-nan2008'},
 278                                 {'variant': 'nan2008',
 279                                  'arch': 'mipsel',
 280                                  'ccopts': '-mabi=32'},
 281                                 {'variant': 'n64-nan2008',
 282                                  'ccopts': '-mabi=64'}])
 283         self.add_config(arch='mips64el',
 284                         os_name='linux-gnu',
 285                         variant='nan2008-soft',
 286                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 287                                  '--with-arch-64=mips64r2',
 288                                  '--with-arch-32=mips32r2',
 289                                  '--with-float=soft'],
 290                         glibcs=[{'variant': 'n32-nan2008-soft'},
 291                                 {'variant': 'nan2008-soft',
 292                                  'arch': 'mipsel',
 293                                  'ccopts': '-mabi=32'},
 294                                 {'variant': 'n64-nan2008-soft',
 295                                  'ccopts': '-mabi=64'}])
 296         self.add_config(arch='nios2',
 297                         os_name='linux-gnu')
 298         self.add_config(arch='powerpc',
 299                         os_name='linux-gnu',
 300                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 301                         extra_glibcs=[{'variant': 'power4',
 302                                        'ccopts': '-mcpu=power4',
 303                                        'cfg': ['--with-cpu=power4']}])
 304         self.add_config(arch='powerpc',
 305                         os_name='linux-gnu',
 306                         variant='soft',
 307                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 308                                  '--enable-secureplt'])
 309         self.add_config(arch='powerpc64',
 310                         os_name='linux-gnu',
 311                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 312         self.add_config(arch='powerpc64le',
 313                         os_name='linux-gnu',
 314                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 315         self.add_config(arch='powerpc',
 316                         os_name='linux-gnuspe',
 317                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 318                                  '--enable-e500-double'])
 319         self.add_config(arch='powerpc',
 320                         os_name='linux-gnuspe',
 321                         variant='e500v1',
 322                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 323         self.add_config(arch='s390x',
 324                         os_name='linux-gnu',
 325                         glibcs=[{},
 326                                 {'arch': 's390', 'ccopts': '-m31'}])
 327         self.add_config(arch='sh3',
 328                         os_name='linux-gnu')
 329         self.add_config(arch='sh3eb',
 330                         os_name='linux-gnu')
 331         self.add_config(arch='sh4',
 332                         os_name='linux-gnu')
 333         self.add_config(arch='sh4eb',
 334                         os_name='linux-gnu')
 335         self.add_config(arch='sh4',
 336                         os_name='linux-gnu',
 337                         variant='soft',
 338                         gcc_cfg=['--without-fp'])
 339         self.add_config(arch='sh4eb',
 340                         os_name='linux-gnu',
 341                         variant='soft',
 342                         gcc_cfg=['--without-fp'])
 343         self.add_config(arch='sparc64',
 344                         os_name='linux-gnu',
 345                         glibcs=[{},
 346                                 {'arch': 'sparcv9',
 347                                  'ccopts': '-m32 -mlong-double-128'}],
 348                         extra_glibcs=[{'variant': 'disable-multi-arch',
 349                                        'cfg': ['--disable-multi-arch']},
 350                                       {'variant': 'disable-multi-arch',
 351                                        'arch': 'sparcv9',
 352                                        'ccopts': '-m32 -mlong-double-128',
 353                                        'cfg': ['--disable-multi-arch']}])
 354         self.add_config(arch='tilegx',
 355                         os_name='linux-gnu',
 356                         glibcs=[{},
 357                                 {'variant': '32', 'ccopts': '-m32'}])
 358         self.add_config(arch='tilegxbe',
 359                         os_name='linux-gnu',
 360                         glibcs=[{},
 361                                 {'variant': '32', 'ccopts': '-m32'}])
 362         self.add_config(arch='tilepro',
 363                         os_name='linux-gnu')
 364         self.add_config(arch='x86_64',
 365                         os_name='linux-gnu',
 366                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 367                         glibcs=[{},
 368                                 {'variant': 'x32', 'ccopts': '-mx32'},
 369                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 370                         extra_glibcs=[{'variant': 'disable-multi-arch',
 371                                        'cfg': ['--disable-multi-arch']},
 372                                       {'variant': 'disable-multi-arch',
 373                                        'arch': 'i686',
 374                                        'ccopts': '-m32 -march=i686',
 375                                        'cfg': ['--disable-multi-arch']},
 376                                       {'arch': 'i486',
 377                                        'ccopts': '-m32 -march=i486'},
 378                                       {'arch': 'i586',
 379                                        'ccopts': '-m32 -march=i586'}])
 380
 381     def add_config(self, **args):
 382         """Add an individual build configuration."""
 383         cfg = Config(self, **args)
 384         if cfg.name in self.configs:
 385             print('error: duplicate config %s' % cfg.name)
 386             exit(1)
 387         self.configs[cfg.name] = cfg
 388         for c in cfg.all_glibcs:
 389             if c.name in self.glibc_configs:
 390                 print('error: duplicate glibc config %s' % c.name)
 391                 exit(1)
 392             self.glibc_configs[c.name] = c
 393
 394     def component_srcdir(self, component):
 395         """Return the source directory for a given component, e.g. gcc."""
 396         return os.path.join(self.srcdir, component)
 397
 398     def component_builddir(self, action, config, component, subconfig=None):
 399         """Return the directory to use for a build."""
 400         if config is None:
 401             # Host libraries.
 402             assert subconfig is None
 403             return os.path.join(self.builddir, action, component)
 404         if subconfig is None:
 405             return os.path.join(self.builddir, action, config, component)
 406         else:
 407             # glibc build as part of compiler build.
 408             return os.path.join(self.builddir, action, config, component,
 409                                 subconfig)
 410
 411     def compiler_installdir(self, config):
 412         """Return the directory in which to install a compiler."""
 413         return os.path.join(self.installdir, 'compilers', config)
 414
 415     def compiler_bindir(self, config):
 416         """Return the directory in which to find compiler binaries."""
 417         return os.path.join(self.compiler_installdir(config), 'bin')
 418
 419     def compiler_sysroot(self, config):
 420         """Return the sysroot directory for a compiler."""
 421         return os.path.join(self.compiler_installdir(config), 'sysroot')
 422
 423     def glibc_installdir(self, config):
 424         """Return the directory in which to install glibc."""
 425         return os.path.join(self.installdir, 'glibcs', config)
 426
 427     def run_builds(self, action, configs):
 428         """Run the requested builds."""
 429         if action == 'checkout':
 430             self.checkout(configs)
 431             return
 432         if action == 'bot-cycle':
 433             if configs:
 434                 print('error: configurations specified for bot-cycle')
 435                 exit(1)
 436             self.bot_cycle()
 437             return
 438         if action == 'bot':
 439             if configs:
 440                 print('error: configurations specified for bot')
 441                 exit(1)
 442             self.bot()
 443             return
 444         if action == 'host-libraries' and configs:
 445             print('error: configurations specified for host-libraries')
 446             exit(1)
 447         self.clear_last_build_state(action)
 448         build_time = datetime.datetime.utcnow()
 449         if action == 'host-libraries':
 450             build_components = ('gmp', 'mpfr', 'mpc')
 451             old_components = ()
 452             old_versions = {}
 453             self.build_host_libraries()
 454         elif action == 'compilers':
 455             build_components = ('binutils', 'gcc', 'glibc', 'linux')
 456             old_components = ('gmp', 'mpfr', 'mpc')
 457             old_versions = self.build_state['host-libraries']['build-versions']
 458             self.build_compilers(configs)
 459         else:
 460             build_components = ('glibc',)
 461             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
 462             old_versions = self.build_state['compilers']['build-versions']
 463             self.build_glibcs(configs)
 464         self.write_files()
 465         self.do_build()
 466         if configs:
 467             # Partial build, do not update stored state.
 468             return
 469         build_versions = {}
 470         for k in build_components:
 471             if k in self.versions:
 472                 build_versions[k] = {'version': self.versions[k]['version'],
 473                                      'revision': self.versions[k]['revision']}
 474         for k in old_components:
 475             if k in old_versions:
 476                 build_versions[k] = {'version': old_versions[k]['version'],
 477                                      'revision': old_versions[k]['revision']}
 478         self.update_build_state(action, build_time, build_versions)
 479
 480     @staticmethod
 481     def remove_dirs(*args):
 482         """Remove directories and their contents if they exist."""
 483         for dir in args:
 484             shutil.rmtree(dir, ignore_errors=True)
 485
 486     @staticmethod
 487     def remove_recreate_dirs(*args):
 488         """Remove directories if they exist, and create them as empty."""
 489         Context.remove_dirs(*args)
 490         for dir in args:
 491             os.makedirs(dir, exist_ok=True)
 492
 493     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 494         """Add makefile text for a list of commands."""
 495         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 496         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 497                                     (target, target, target, commands))
 498         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 499
 500     def write_files(self):
 501         """Write out the Makefile and wrapper script."""
 502         mftext = ''.join(self.makefile_pieces)
 503         with open(self.makefile, 'w') as f:
 504             f.write(mftext)
 505         wrapper_text = (
 506             '#!/bin/sh\n'
 507             'prev_base=$1\n'
 508             'this_base=$2\n'
 509             'desc=$3\n'
 510             'dir=$4\n'
 511             'path=$5\n'
 512             'shift 5\n'
 513             'prev_status=$prev_base-status.txt\n'
 514             'this_status=$this_base-status.txt\n'
 515             'this_log=$this_base-log.txt\n'
 516             'date > "$this_log"\n'
 517             'echo >> "$this_log"\n'
 518             'echo "Description: $desc" >> "$this_log"\n'
 519             'printf "%s" "Command:" >> "$this_log"\n'
 520             'for word in "$@"; do\n'
 521             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 522             '    printf " %s" "$word"\n'
 523             '  else\n'
 524             '    printf " \'"\n'
 525             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 526             '    printf "\'"\n'
 527             '  fi\n'
 528             'done >> "$this_log"\n'
 529             'echo >> "$this_log"\n'
 530             'echo "Directory: $dir" >> "$this_log"\n'
 531             'echo "Path addition: $path" >> "$this_log"\n'
 532             'echo >> "$this_log"\n'
 533             'record_status ()\n'
 534             '{\n'
 535             '  echo >> "$this_log"\n'
 536             '  echo "$1: $desc" > "$this_status"\n'
 537             '  echo "$1: $desc" >> "$this_log"\n'
 538             '  echo >> "$this_log"\n'
 539             '  date >> "$this_log"\n'
 540             '  echo "$1: $desc"\n'
 541             '  exit 0\n'
 542             '}\n'
 543             'check_error ()\n'
 544             '{\n'
 545             '  if [ "$1" != "0" ]; then\n'
 546             '    record_status FAIL\n'
 547             '  fi\n'
 548             '}\n'
 549             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 550             '    record_status UNRESOLVED\n'
 551             'fi\n'
 552             'if [ "$dir" ]; then\n'
 553             '  cd "$dir"\n'
 554             '  check_error "$?"\n'
 555             'fi\n'
 556             'if [ "$path" ]; then\n'
 557             '  PATH=$path:$PATH\n'
 558             'fi\n'
 559             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 560             'check_error "$?"\n'
 561             'record_status PASS\n')
 562         with open(self.wrapper, 'w') as f:
 563             f.write(wrapper_text)
 564         # Mode 0o755.
 565         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 566                      stat.S_IROTH|stat.S_IXOTH)
 567         os.chmod(self.wrapper, mode_exec)
 568         save_logs_text = (
 569             '#!/bin/sh\n'
 570             'if ! [ -f tests.sum ]; then\n'
 571             '  echo "No test summary available."\n'
 572             '  exit 0\n'
 573             'fi\n'
 574             'save_file ()\n'
 575             '{\n'
 576             '  echo "Contents of $1:"\n'
 577             '  echo\n'
 578             '  cat "$1"\n'
 579             '  echo\n'
 580             '  echo "End of contents of $1."\n'
 581             '  echo\n'
 582             '}\n'
 583             'save_file tests.sum\n'
 584             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 585             'for t in $non_pass_tests; do\n'
 586             '  if [ -f "$t.out" ]; then\n'
 587             '    save_file "$t.out"\n'
 588             '  fi\n'
 589             'done\n')
 590         with open(self.save_logs, 'w') as f:
 591             f.write(save_logs_text)
 592         os.chmod(self.save_logs, mode_exec)
 593
 594     def do_build(self):
 595         """Do the actual build."""
 596         cmd = ['make', '-j%d' % self.parallelism]
 597         subprocess.run(cmd, cwd=self.builddir, check=True)
 598
 599     def build_host_libraries(self):
 600         """Build the host libraries."""
 601         installdir = self.host_libraries_installdir
 602         builddir = os.path.join(self.builddir, 'host-libraries')
 603         logsdir = os.path.join(self.logsdir, 'host-libraries')
 604         self.remove_recreate_dirs(installdir, builddir, logsdir)
 605         cmdlist = CommandList('host-libraries', self.keep)
 606         self.build_host_library(cmdlist, 'gmp')
 607         self.build_host_library(cmdlist, 'mpfr',
 608                                 ['--with-gmp=%s' % installdir])
 609         self.build_host_library(cmdlist, 'mpc',
 610                                 ['--with-gmp=%s' % installdir,
 611                                 '--with-mpfr=%s' % installdir])
 612         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 613         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 614
 615     def build_host_library(self, cmdlist, lib, extra_opts=None):
 616         """Build one host library."""
 617         srcdir = self.component_srcdir(lib)
 618         builddir = self.component_builddir('host-libraries', None, lib)
 619         installdir = self.host_libraries_installdir
 620         cmdlist.push_subdesc(lib)
 621         cmdlist.create_use_dir(builddir)
 622         cfg_cmd = [os.path.join(srcdir, 'configure'),
 623                    '--prefix=%s' % installdir,
 624                    '--disable-shared']
 625         if extra_opts:
 626             cfg_cmd.extend (extra_opts)
 627         cmdlist.add_command('configure', cfg_cmd)
 628         cmdlist.add_command('build', ['make'])
 629         cmdlist.add_command('check', ['make', 'check'])
 630         cmdlist.add_command('install', ['make', 'install'])
 631         cmdlist.cleanup_dir()
 632         cmdlist.pop_subdesc()
 633
 634     def build_compilers(self, configs):
 635         """Build the compilers."""
 636         if not configs:
 637             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 638             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 639             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 640             configs = sorted(self.configs.keys())
 641         for c in configs:
 642             self.configs[c].build()
 643
 644     def build_glibcs(self, configs):
 645         """Build the glibcs."""
 646         if not configs:
 647             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 648             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 649             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 650             configs = sorted(self.glibc_configs.keys())
 651         for c in configs:
 652             self.glibc_configs[c].build()
 653
 654     def load_versions_json(self):
 655         """Load information about source directory versions."""
 656         if not os.access(self.versions_json, os.F_OK):
 657             self.versions = {}
 658             return
 659         with open(self.versions_json, 'r') as f:
 660             self.versions = json.load(f)
 661
 662     def store_json(self, data, filename):
 663         """Store information in a JSON file."""
 664         filename_tmp = filename + '.tmp'
 665         with open(filename_tmp, 'w') as f:
 666             json.dump(data, f, indent=2, sort_keys=True)
 667         os.rename(filename_tmp, filename)
 668
 669     def store_versions_json(self):
 670         """Store information about source directory versions."""
 671         self.store_json(self.versions, self.versions_json)
 672
 673     def set_component_version(self, component, version, explicit, revision):
 674         """Set the version information for a component."""
 675         self.versions[component] = {'version': version,
 676                                     'explicit': explicit,
 677                                     'revision': revision}
 678         self.store_versions_json()
 679
 680     def checkout(self, versions):
 681         """Check out the desired component versions."""
 682         default_versions = {'binutils': 'vcs-2.29',
 683                             'gcc': 'vcs-7',
 684                             'glibc': 'vcs-mainline',
 685                             'gmp': '6.1.2',
 686                             'linux': '4.14',
 687                             'mpc': '1.0.3',
 688                             'mpfr': '3.1.6'}
 689         use_versions = {}
 690         explicit_versions = {}
 691         for v in versions:
 692             found_v = False
 693             for k in default_versions.keys():
 694                 kx = k + '-'
 695                 if v.startswith(kx):
 696                     vx = v[len(kx):]
 697                     if k in use_versions:
 698                         print('error: multiple versions for %s' % k)
 699                         exit(1)
 700                     use_versions[k] = vx
 701                     explicit_versions[k] = True
 702                     found_v = True
 703                     break
 704             if not found_v:
 705                 print('error: unknown component in %s' % v)
 706                 exit(1)
 707         for k in default_versions.keys():
 708             if k not in use_versions:
 709                 if k in self.versions and self.versions[k]['explicit']:
 710                     use_versions[k] = self.versions[k]['version']
 711                     explicit_versions[k] = True
 712                 else:
 713                     use_versions[k] = default_versions[k]
 714                     explicit_versions[k] = False
 715         os.makedirs(self.srcdir, exist_ok=True)
 716         for k in sorted(default_versions.keys()):
 717             update = os.access(self.component_srcdir(k), os.F_OK)
 718             v = use_versions[k]
 719             if (update and
 720                 k in self.versions and
 721                 v != self.versions[k]['version']):
 722                 if not self.replace_sources:
 723                     print('error: version of %s has changed from %s to %s, '
 724                           'use --replace-sources to check out again' %
 725                           (k, self.versions[k]['version'], v))
 726                     exit(1)
 727                 shutil.rmtree(self.component_srcdir(k))
 728                 update = False
 729             if v.startswith('vcs-'):
 730                 revision = self.checkout_vcs(k, v[4:], update)
 731             else:
 732                 self.checkout_tar(k, v, update)
 733                 revision = v
 734             self.set_component_version(k, v, explicit_versions[k], revision)
 735         if self.get_script_text() != self.script_text:
 736             # Rerun the checkout process in case the updated script
 737             # uses different default versions or new components.
 738             self.exec_self()
 739
 740     def checkout_vcs(self, component, version, update):
 741         """Check out the given version of the given component from version
 742         control.  Return a revision identifier."""
 743         if component == 'binutils':
 744             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 745             if version == 'mainline':
 746                 git_branch = 'master'
 747             else:
 748                 trans = str.maketrans({'.': '_'})
 749                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 750             return self.git_checkout(component, git_url, git_branch, update)
 751         elif component == 'gcc':
 752             if version == 'mainline':
 753                 branch = 'trunk'
 754             else:
 755                 trans = str.maketrans({'.': '_'})
 756                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 757             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 758             return self.gcc_checkout(svn_url, update)
 759         elif component == 'glibc':
 760             git_url = 'git://sourceware.org/git/glibc.git'
 761             if version == 'mainline':
 762                 git_branch = 'master'
 763             else:
 764                 git_branch = 'release/%s/master' % version
 765             r = self.git_checkout(component, git_url, git_branch, update)
 766             self.fix_glibc_timestamps()
 767             return r
 768         else:
 769             print('error: component %s coming from VCS' % component)
 770             exit(1)
 771
 772     def git_checkout(self, component, git_url, git_branch, update):
 773         """Check out a component from git.  Return a commit identifier."""
 774         if update:
 775             subprocess.run(['git', 'remote', 'prune', 'origin'],
 776                            cwd=self.component_srcdir(component), check=True)
 777             subprocess.run(['git', 'pull', '-q'],
 778                            cwd=self.component_srcdir(component), check=True)
 779         else:
 780             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 781                             self.component_srcdir(component)], check=True)
 782         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 783                            cwd=self.component_srcdir(component),
 784                            stdout=subprocess.PIPE,
 785                            check=True, universal_newlines=True).stdout
 786         return r.rstrip()
 787
 788     def fix_glibc_timestamps(self):
 789         """Fix timestamps in a glibc checkout."""
 790         # Ensure that builds do not try to regenerate generated files
 791         # in the source tree.
 792         srcdir = self.component_srcdir('glibc')
 793         for dirpath, dirnames, filenames in os.walk(srcdir):
 794             for f in filenames:
 795                 if (f == 'configure' or
 796                     f == 'preconfigure' or
 797                     f.endswith('-kw.h')):
 798                     to_touch = os.path.join(dirpath, f)
 799                     subprocess.run(['touch', to_touch], check=True)
 800
 801     def gcc_checkout(self, svn_url, update):
 802         """Check out GCC from SVN.  Return the revision number."""
 803         if not update:
 804             subprocess.run(['svn', 'co', '-q', svn_url,
 805                             self.component_srcdir('gcc')], check=True)
 806         subprocess.run(['contrib/gcc_update', '--silent'],
 807                        cwd=self.component_srcdir('gcc'), check=True)
 808         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 809                            stdout=subprocess.PIPE,
 810                            check=True, universal_newlines=True).stdout
 811         return r.rstrip()
 812
 813     def checkout_tar(self, component, version, update):
 814         """Check out the given version of the given component from a
 815         tarball."""
 816         if update:
 817             return
 818         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 819                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 820                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 821                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 822                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 823                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
 824         if component not in url_map:
 825             print('error: component %s coming from tarball' % component)
 826             exit(1)
 827         url = url_map[component] % {'version': version}
 828         filename = os.path.join(self.srcdir, url.split('/')[-1])
 829         response = urllib.request.urlopen(url)
 830         data = response.read()
 831         with open(filename, 'wb') as f:
 832             f.write(data)
 833         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 834                        check=True)
 835         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 836                   self.component_srcdir(component))
 837         os.remove(filename)
 838
 839     def load_build_state_json(self):
 840         """Load information about the state of previous builds."""
 841         if os.access(self.build_state_json, os.F_OK):
 842             with open(self.build_state_json, 'r') as f:
 843                 self.build_state = json.load(f)
 844         else:
 845             self.build_state = {}
 846         for k in ('host-libraries', 'compilers', 'glibcs'):
 847             if k not in self.build_state:
 848                 self.build_state[k] = {}
 849             if 'build-time' not in self.build_state[k]:
 850                 self.build_state[k]['build-time'] = ''
 851             if 'build-versions' not in self.build_state[k]:
 852                 self.build_state[k]['build-versions'] = {}
 853             if 'build-results' not in self.build_state[k]:
 854                 self.build_state[k]['build-results'] = {}
 855             if 'result-changes' not in self.build_state[k]:
 856                 self.build_state[k]['result-changes'] = {}
 857             if 'ever-passed' not in self.build_state[k]:
 858                 self.build_state[k]['ever-passed'] = []
 859
 860     def store_build_state_json(self):
 861         """Store information about the state of previous builds."""
 862         self.store_json(self.build_state, self.build_state_json)
 863
 864     def clear_last_build_state(self, action):
 865         """Clear information about the state of part of the build."""
 866         # We clear the last build time and versions when starting a
 867         # new build.  The results of the last build are kept around,
 868         # as comparison is still meaningful if this build is aborted
 869         # and a new one started.
 870         self.build_state[action]['build-time'] = ''
 871         self.build_state[action]['build-versions'] = {}
 872         self.store_build_state_json()
 873
 874     def update_build_state(self, action, build_time, build_versions):
 875         """Update the build state after a build."""
 876         build_time = build_time.replace(microsecond=0)
 877         self.build_state[action]['build-time'] = str(build_time)
 878         self.build_state[action]['build-versions'] = build_versions
 879         build_results = {}
 880         for log in self.status_log_list:
 881             with open(log, 'r') as f:
 882                 log_text = f.read()
 883             log_text = log_text.rstrip()
 884             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 885             result = m.group(1)
 886             test_name = m.group(2)
 887             assert test_name not in build_results
 888             build_results[test_name] = result
 889         old_build_results = self.build_state[action]['build-results']
 890         self.build_state[action]['build-results'] = build_results
 891         result_changes = {}
 892         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 893         for t in all_tests:
 894             if t in old_build_results:
 895                 old_res = old_build_results[t]
 896             else:
 897                 old_res = '(New test)'
 898             if t in build_results:
 899                 new_res = build_results[t]
 900             else:
 901                 new_res = '(Test removed)'
 902             if old_res != new_res:
 903                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 904         self.build_state[action]['result-changes'] = result_changes
 905         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 906                            if t in build_results}
 907         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 908         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 909                                                          new_passes)
 910         self.store_build_state_json()
 911
 912     def load_bot_config_json(self):
 913         """Load bot configuration."""
 914         with open(self.bot_config_json, 'r') as f:
 915             self.bot_config = json.load(f)
 916
 917     def part_build_old(self, action, delay):
 918         """Return whether the last build for a given action was at least a
 919         given number of seconds ago, or does not have a time recorded."""
 920         old_time_str = self.build_state[action]['build-time']
 921         if not old_time_str:
 922             return True
 923         old_time = datetime.datetime.strptime(old_time_str,
 924                                               '%Y-%m-%d %H:%M:%S')
 925         new_time = datetime.datetime.utcnow()
 926         delta = new_time - old_time
 927         return delta.total_seconds() >= delay
 928
 929     def bot_cycle(self):
 930         """Run a single round of checkout and builds."""
 931         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 932         self.load_bot_config_json()
 933         actions = ('host-libraries', 'compilers', 'glibcs')
 934         self.bot_run_self(['--replace-sources'], 'checkout')
 935         self.load_versions_json()
 936         if self.get_script_text() != self.script_text:
 937             print('Script changed, re-execing.')
 938             # On script change, all parts of the build should be rerun.
 939             for a in actions:
 940                 self.clear_last_build_state(a)
 941             self.exec_self()
 942         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 943                             'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
 944                             'glibcs': ('glibc',)}
 945         must_build = {}
 946         for a in actions:
 947             build_vers = self.build_state[a]['build-versions']
 948             must_build[a] = False
 949             if not self.build_state[a]['build-time']:
 950                 must_build[a] = True
 951             old_vers = {}
 952             new_vers = {}
 953             for c in check_components[a]:
 954                 if c in build_vers:
 955                     old_vers[c] = build_vers[c]
 956                 new_vers[c] = {'version': self.versions[c]['version'],
 957                                'revision': self.versions[c]['revision']}
 958             if new_vers == old_vers:
 959                 print('Versions for %s unchanged.' % a)
 960             else:
 961                 print('Versions changed or rebuild forced for %s.' % a)
 962                 if a == 'compilers' and not self.part_build_old(
 963                         a, self.bot_config['compilers-rebuild-delay']):
 964                     print('Not requiring rebuild of compilers this soon.')
 965                 else:
 966                     must_build[a] = True
 967         if must_build['host-libraries']:
 968             must_build['compilers'] = True
 969         if must_build['compilers']:
 970             must_build['glibcs'] = True
 971         for a in actions:
 972             if must_build[a]:
 973                 print('Must rebuild %s.' % a)
 974                 self.clear_last_build_state(a)
 975             else:
 976                 print('No need to rebuild %s.' % a)
 977         if os.access(self.logsdir, os.F_OK):
 978             shutil.rmtree(self.logsdir_old, ignore_errors=True)
 979             shutil.copytree(self.logsdir, self.logsdir_old)
 980         for a in actions:
 981             if must_build[a]:
 982                 build_time = datetime.datetime.utcnow()
 983                 print('Rebuilding %s at %s.' % (a, str(build_time)))
 984                 self.bot_run_self([], a)
 985                 self.load_build_state_json()
 986                 self.bot_build_mail(a, build_time)
 987         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
 988
 989     def bot_build_mail(self, action, build_time):
 990         """Send email with the results of a build."""
 991         if not ('email-from' in self.bot_config and
 992                 'email-server' in self.bot_config and
 993                 'email-subject' in self.bot_config and
 994                 'email-to' in self.bot_config):
 995             if not self.email_warning:
 996                 print("Email not configured, not sending.")
 997                 self.email_warning = True
 998             return
 999
1000         build_time = build_time.replace(microsecond=0)
1001         subject = (self.bot_config['email-subject'] %
1002                    {'action': action,
1003                     'build-time': str(build_time)})
1004         results = self.build_state[action]['build-results']
1005         changes = self.build_state[action]['result-changes']
1006         ever_passed = set(self.build_state[action]['ever-passed'])
1007         versions = self.build_state[action]['build-versions']
1008         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1009         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1010         all_fails = {k for k in results if results[k] == 'FAIL'}
1011         if new_regressions:
1012             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1013             new_reg_text = ('New regressions:\n\n%s\n\n' %
1014                             '\n'.join(new_reg_list))
1015         else:
1016             new_reg_text = ''
1017         if all_regressions:
1018             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1019             all_reg_text = ('All regressions:\n\n%s\n\n' %
1020                             '\n'.join(all_reg_list))
1021         else:
1022             all_reg_text = ''
1023         if all_fails:
1024             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1025             all_fail_text = ('All failures:\n\n%s\n\n' %
1026                              '\n'.join(all_fail_list))
1027         else:
1028             all_fail_text = ''
1029         if changes:
1030             changes_list = sorted(changes.keys())
1031             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1032             changes_text = ('All changed results:\n\n%s\n\n' %
1033                             '\n'.join(changes_list))
1034         else:
1035             changes_text = ''
1036         results_text = (new_reg_text + all_reg_text + all_fail_text +
1037                         changes_text)
1038         if not results_text:
1039             results_text = 'Clean build with unchanged results.\n\n'
1040         versions_list = sorted(versions.keys())
1041         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1042                                           versions[k]['revision'])
1043                          for k in versions_list]
1044         versions_text = ('Component versions for this build:\n\n%s\n' %
1045                          '\n'.join(versions_list))
1046         body_text = results_text + versions_text
1047         msg = email.mime.text.MIMEText(body_text)
1048         msg['Subject'] = subject
1049         msg['From'] = self.bot_config['email-from']
1050         msg['To'] = self.bot_config['email-to']
1051         msg['Message-ID'] = email.utils.make_msgid()
1052         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1053         with smtplib.SMTP(self.bot_config['email-server']) as s:
1054             s.send_message(msg)
1055
1056     def bot_run_self(self, opts, action, check=True):
1057         """Run a copy of this script with given options."""
1058         cmd = [sys.executable, sys.argv[0], '--keep=none',
1059                '-j%d' % self.parallelism]
1060         cmd.extend(opts)
1061         cmd.extend([self.topdir, action])
1062         sys.stdout.flush()
1063         subprocess.run(cmd, check=check)
1064
1065     def bot(self):
1066         """Run repeated rounds of checkout and builds."""
1067         while True:
1068             self.load_bot_config_json()
1069             if not self.bot_config['run']:
1070                 print('Bot exiting by request.')
1071                 exit(0)
1072             self.bot_run_self([], 'bot-cycle', check=False)
1073             self.load_bot_config_json()
1074             if not self.bot_config['run']:
1075                 print('Bot exiting by request.')
1076                 exit(0)
1077             time.sleep(self.bot_config['delay'])
1078             if self.get_script_text() != self.script_text:
1079                 print('Script changed, bot re-execing.')
1080                 self.exec_self()
1081
1082
1083 class Config(object):
1084     """A configuration for building a compiler and associated libraries."""
1085
1086     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1087                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1088         """Initialize a Config object."""
1089         self.ctx = ctx
1090         self.arch = arch
1091         self.os = os_name
1092         self.variant = variant
1093         if variant is None:
1094             self.name = '%s-%s' % (arch, os_name)
1095         else:
1096             self.name = '%s-%s-%s' % (arch, os_name, variant)
1097         self.triplet = '%s-glibc-%s' % (arch, os_name)
1098         if gcc_cfg is None:
1099             self.gcc_cfg = []
1100         else:
1101             self.gcc_cfg = gcc_cfg
1102         if first_gcc_cfg is None:
1103             self.first_gcc_cfg = []
1104         else:
1105             self.first_gcc_cfg = first_gcc_cfg
1106         if glibcs is None:
1107             glibcs = [{'variant': variant}]
1108         if extra_glibcs is None:
1109             extra_glibcs = []
1110         glibcs = [Glibc(self, **g) for g in glibcs]
1111         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1112         self.all_glibcs = glibcs + extra_glibcs
1113         self.compiler_glibcs = glibcs
1114         self.installdir = ctx.compiler_installdir(self.name)
1115         self.bindir = ctx.compiler_bindir(self.name)
1116         self.sysroot = ctx.compiler_sysroot(self.name)
1117         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1118         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1119
1120     def component_builddir(self, component):
1121         """Return the directory to use for a (non-glibc) build."""
1122         return self.ctx.component_builddir('compilers', self.name, component)
1123
1124     def build(self):
1125         """Generate commands to build this compiler."""
1126         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1127                                       self.logsdir)
1128         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1129         cmdlist.add_command('check-host-libraries',
1130                             ['test', '-f',
1131                              os.path.join(self.ctx.host_libraries_installdir,
1132                                           'ok')])
1133         cmdlist.use_path(self.bindir)
1134         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1135                               ['--disable-gdb',
1136                                '--disable-libdecnumber',
1137                                '--disable-readline',
1138                                '--disable-sim'])
1139         if self.os.startswith('linux'):
1140             self.install_linux_headers(cmdlist)
1141         self.build_gcc(cmdlist, True)
1142         for g in self.compiler_glibcs:
1143             cmdlist.push_subdesc('glibc')
1144             cmdlist.push_subdesc(g.name)
1145             g.build_glibc(cmdlist, True)
1146             cmdlist.pop_subdesc()
1147             cmdlist.pop_subdesc()
1148         self.build_gcc(cmdlist, False)
1149         cmdlist.add_command('done', ['touch',
1150                                      os.path.join(self.installdir, 'ok')])
1151         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1152                                       self.logsdir)
1153
1154     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1155         """Build one cross tool."""
1156         srcdir = self.ctx.component_srcdir(tool_src)
1157         builddir = self.component_builddir(tool_build)
1158         cmdlist.push_subdesc(tool_build)
1159         cmdlist.create_use_dir(builddir)
1160         cfg_cmd = [os.path.join(srcdir, 'configure'),
1161                    '--prefix=%s' % self.installdir,
1162                    '--build=%s' % self.ctx.build_triplet,
1163                    '--host=%s' % self.ctx.build_triplet,
1164                    '--target=%s' % self.triplet,
1165                    '--with-sysroot=%s' % self.sysroot]
1166         if extra_opts:
1167             cfg_cmd.extend(extra_opts)
1168         cmdlist.add_command('configure', cfg_cmd)
1169         cmdlist.add_command('build', ['make'])
1170         # Parallel "make install" for GCC has race conditions that can
1171         # cause it to fail; see
1172         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1173         # problems are not known for binutils, but doing the
1174         # installation in parallel within a particular toolchain build
1175         # (as opposed to installation of one toolchain from
1176         # build-many-glibcs.py running in parallel to the installation
1177         # of other toolchains being built) is not known to be
1178         # significantly beneficial, so it is simplest just to disable
1179         # parallel install for cross tools here.
1180         cmdlist.add_command('install', ['make', '-j1', 'install'])
1181         cmdlist.cleanup_dir()
1182         cmdlist.pop_subdesc()
1183
1184     def install_linux_headers(self, cmdlist):
1185         """Install Linux kernel headers."""
1186         arch_map = {'aarch64': 'arm64',
1187                     'alpha': 'alpha',
1188                     'arm': 'arm',
1189                     'hppa': 'parisc',
1190                     'i486': 'x86',
1191                     'i586': 'x86',
1192                     'i686': 'x86',
1193                     'i786': 'x86',
1194                     'ia64': 'ia64',
1195                     'm68k': 'm68k',
1196                     'microblaze': 'microblaze',
1197                     'mips': 'mips',
1198                     'nios2': 'nios2',
1199                     'powerpc': 'powerpc',
1200                     's390': 's390',
1201                     'sh': 'sh',
1202                     'sparc': 'sparc',
1203                     'tile': 'tile',
1204                     'x86_64': 'x86'}
1205         linux_arch = None
1206         for k in arch_map:
1207             if self.arch.startswith(k):
1208                 linux_arch = arch_map[k]
1209                 break
1210         assert linux_arch is not None
1211         srcdir = self.ctx.component_srcdir('linux')
1212         builddir = self.component_builddir('linux')
1213         headers_dir = os.path.join(self.sysroot, 'usr')
1214         cmdlist.push_subdesc('linux')
1215         cmdlist.create_use_dir(builddir)
1216         cmdlist.add_command('install-headers',
1217                             ['make', '-C', srcdir, 'O=%s' % builddir,
1218                              'ARCH=%s' % linux_arch,
1219                              'INSTALL_HDR_PATH=%s' % headers_dir,
1220                              'headers_install'])
1221         cmdlist.cleanup_dir()
1222         cmdlist.pop_subdesc()
1223
1224     def build_gcc(self, cmdlist, bootstrap):
1225         """Build GCC."""
1226         # libsanitizer commonly breaks because of glibc header
1227         # changes, or on unusual targets.  libssp is of little
1228         # relevance with glibc's own stack checking support.
1229         cfg_opts = list(self.gcc_cfg)
1230         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1231         host_libs = self.ctx.host_libraries_installdir
1232         cfg_opts += ['--with-gmp=%s' % host_libs,
1233                      '--with-mpfr=%s' % host_libs,
1234                      '--with-mpc=%s' % host_libs]
1235         if bootstrap:
1236             tool_build = 'gcc-first'
1237             # Building a static-only, C-only compiler that is
1238             # sufficient to build glibc.  Various libraries and
1239             # features that may require libc headers must be disabled.
1240             # When configuring with a sysroot, --with-newlib is
1241             # required to define inhibit_libc (to stop some parts of
1242             # libgcc including libc headers); --without-headers is not
1243             # sufficient.
1244             cfg_opts += ['--enable-languages=c', '--disable-shared',
1245                          '--disable-threads',
1246                          '--disable-libatomic',
1247                          '--disable-decimal-float',
1248                          '--disable-libffi',
1249                          '--disable-libgomp',
1250                          '--disable-libitm',
1251                          '--disable-libmpx',
1252                          '--disable-libquadmath',
1253                          '--without-headers', '--with-newlib',
1254                          '--with-glibc-version=%s' % self.ctx.glibc_version
1255                          ]
1256             cfg_opts += self.first_gcc_cfg
1257         else:
1258             tool_build = 'gcc'
1259             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1260                          '--enable-threads']
1261         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1262
1263
1264 class Glibc(object):
1265     """A configuration for building glibc."""
1266
1267     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1268                  cfg=None, ccopts=None):
1269         """Initialize a Glibc object."""
1270         self.ctx = compiler.ctx
1271         self.compiler = compiler
1272         if arch is None:
1273             self.arch = compiler.arch
1274         else:
1275             self.arch = arch
1276         if os_name is None:
1277             self.os = compiler.os
1278         else:
1279             self.os = os_name
1280         self.variant = variant
1281         if variant is None:
1282             self.name = '%s-%s' % (self.arch, self.os)
1283         else:
1284             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1285         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1286         if cfg is None:
1287             self.cfg = []
1288         else:
1289             self.cfg = cfg
1290         self.ccopts = ccopts
1291
1292     def tool_name(self, tool):
1293         """Return the name of a cross-compilation tool."""
1294         ctool = '%s-%s' % (self.compiler.triplet, tool)
1295         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1296             ctool = '%s %s' % (ctool, self.ccopts)
1297         return ctool
1298
1299     def build(self):
1300         """Generate commands to build this glibc."""
1301         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1302         installdir = self.ctx.glibc_installdir(self.name)
1303         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1304         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1305         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1306         cmdlist.add_command('check-compilers',
1307                             ['test', '-f',
1308                              os.path.join(self.compiler.installdir, 'ok')])
1309         cmdlist.use_path(self.compiler.bindir)
1310         self.build_glibc(cmdlist, False)
1311         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1312                                       logsdir)
1313
1314     def build_glibc(self, cmdlist, for_compiler):
1315         """Generate commands to build this glibc, either as part of a compiler
1316         build or with the bootstrapped compiler (and in the latter case, run
1317         tests as well)."""
1318         srcdir = self.ctx.component_srcdir('glibc')
1319         if for_compiler:
1320             builddir = self.ctx.component_builddir('compilers',
1321                                                    self.compiler.name, 'glibc',
1322                                                    self.name)
1323             installdir = self.compiler.sysroot
1324             srcdir_copy = self.ctx.component_builddir('compilers',
1325                                                       self.compiler.name,
1326                                                       'glibc-src',
1327                                                       self.name)
1328         else:
1329             builddir = self.ctx.component_builddir('glibcs', self.name,
1330                                                    'glibc')
1331             installdir = self.ctx.glibc_installdir(self.name)
1332             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1333                                                       'glibc-src')
1334         cmdlist.create_use_dir(builddir)
1335         # glibc builds write into the source directory, and even if
1336         # not intentionally there is a risk of bugs that involve
1337         # writing into the working directory.  To avoid possible
1338         # concurrency issues, copy the source directory.
1339         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1340         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1341                    '--prefix=/usr',
1342                    '--enable-profile',
1343                    '--build=%s' % self.ctx.build_triplet,
1344                    '--host=%s' % self.triplet,
1345                    'CC=%s' % self.tool_name('gcc'),
1346                    'CXX=%s' % self.tool_name('g++'),
1347                    'AR=%s' % self.tool_name('ar'),
1348                    'AS=%s' % self.tool_name('as'),
1349                    'LD=%s' % self.tool_name('ld'),
1350                    'NM=%s' % self.tool_name('nm'),
1351                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1352                    'OBJDUMP=%s' % self.tool_name('objdump'),
1353                    'RANLIB=%s' % self.tool_name('ranlib'),
1354                    'READELF=%s' % self.tool_name('readelf'),
1355                    'STRIP=%s' % self.tool_name('strip')]
1356         cfg_cmd += self.cfg
1357         cmdlist.add_command('configure', cfg_cmd)
1358         cmdlist.add_command('build', ['make'])
1359         cmdlist.add_command('install', ['make', 'install',
1360                                         'install_root=%s' % installdir])
1361         # GCC uses paths such as lib/../lib64, so make sure lib
1362         # directories always exist.
1363         cmdlist.add_command('mkdir-lib', ['mkdir', '-p',
1364                                           os.path.join(installdir, 'lib'),
1365                                           os.path.join(installdir,
1366                                                        'usr', 'lib')])
1367         if not for_compiler:
1368             if self.ctx.strip:
1369                 cmdlist.add_command('strip',
1370                                     ['sh', '-c',
1371                                      ('%s %s/lib*/*.so' %
1372                                       (self.tool_name('strip'), installdir))])
1373             cmdlist.add_command('check', ['make', 'check'])
1374             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1375                                 always_run=True)
1376         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1377         cmdlist.cleanup_dir()
1378
1379
1380 class Command(object):
1381     """A command run in the build process."""
1382
1383     def __init__(self, desc, num, dir, path, command, always_run=False):
1384         """Initialize a Command object."""
1385         self.dir = dir
1386         self.path = path
1387         self.desc = desc
1388         trans = str.maketrans({' ': '-'})
1389         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1390         self.command = command
1391         self.always_run = always_run
1392
1393     @staticmethod
1394     def shell_make_quote_string(s):
1395         """Given a string not containing a newline, quote it for use by the
1396         shell and make."""
1397         assert '\n' not in s
1398         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1399             return s
1400         strans = str.maketrans({"'": "'\\''"})
1401         s = "'%s'" % s.translate(strans)
1402         mtrans = str.maketrans({'$': '$$'})
1403         return s.translate(mtrans)
1404
1405     @staticmethod
1406     def shell_make_quote_list(l, translate_make):
1407         """Given a list of strings not containing newlines, quote them for use
1408         by the shell and make, returning a single string.  If translate_make
1409         is true and the first string is 'make', change it to $(MAKE)."""
1410         l = [Command.shell_make_quote_string(s) for s in l]
1411         if translate_make and l[0] == 'make':
1412             l[0] = '$(MAKE)'
1413         return ' '.join(l)
1414
1415     def shell_make_quote(self):
1416         """Return this command quoted for the shell and make."""
1417         return self.shell_make_quote_list(self.command, True)
1418
1419
1420 class CommandList(object):
1421     """A list of commands run in the build process."""
1422
1423     def __init__(self, desc, keep):
1424         """Initialize a CommandList object."""
1425         self.cmdlist = []
1426         self.dir = None
1427         self.path = None
1428         self.desc = [desc]
1429         self.keep = keep
1430
1431     def desc_txt(self, desc):
1432         """Return the description to use for a command."""
1433         return '%s %s' % (' '.join(self.desc), desc)
1434
1435     def use_dir(self, dir):
1436         """Set the default directory for subsequent commands."""
1437         self.dir = dir
1438
1439     def use_path(self, path):
1440         """Set a directory to be prepended to the PATH for subsequent
1441         commands."""
1442         self.path = path
1443
1444     def push_subdesc(self, subdesc):
1445         """Set the default subdescription for subsequent commands (e.g., the
1446         name of a component being built, within the series of commands
1447         building it)."""
1448         self.desc.append(subdesc)
1449
1450     def pop_subdesc(self):
1451         """Pop a subdescription from the list of descriptions."""
1452         self.desc.pop()
1453
1454     def create_use_dir(self, dir):
1455         """Remove and recreate a directory and use it for subsequent
1456         commands."""
1457         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1458         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1459         self.use_dir(dir)
1460
1461     def create_copy_dir(self, src, dest):
1462         """Remove a directory and recreate it as a copy from the given
1463         source."""
1464         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1465         parent = os.path.dirname(dest)
1466         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1467         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1468
1469     def add_command_dir(self, desc, dir, command, always_run=False):
1470         """Add a command to run in a given directory."""
1471         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1472                       command, always_run)
1473         self.cmdlist.append(cmd)
1474
1475     def add_command(self, desc, command, always_run=False):
1476         """Add a command to run in the default directory."""
1477         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1478                       self.path, command, always_run)
1479         self.cmdlist.append(cmd)
1480
1481     def cleanup_dir(self, desc='cleanup', dir=None):
1482         """Clean up a build directory.  If no directory is specified, the
1483         default directory is cleaned up and ceases to be the default
1484         directory."""
1485         if dir is None:
1486             dir = self.dir
1487             self.use_dir(None)
1488         if self.keep != 'all':
1489             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1490                                  always_run=(self.keep == 'none'))
1491
1492     def makefile_commands(self, wrapper, logsdir):
1493         """Return the sequence of commands in the form of text for a Makefile.
1494         The given wrapper script takes arguments: base of logs for
1495         previous command, or empty; base of logs for this command;
1496         description; directory; PATH addition; the command itself."""
1497         # prev_base is the base of the name for logs of the previous
1498         # command that is not always-run (that is, a build command,
1499         # whose failure should stop subsequent build commands from
1500         # being run, as opposed to a cleanup command, which is run
1501         # even if previous commands failed).
1502         prev_base = ''
1503         cmds = []
1504         for c in self.cmdlist:
1505             ctxt = c.shell_make_quote()
1506             if prev_base and not c.always_run:
1507                 prev_log = os.path.join(logsdir, prev_base)
1508             else:
1509                 prev_log = ''
1510             this_log = os.path.join(logsdir, c.logbase)
1511             if not c.always_run:
1512                 prev_base = c.logbase
1513             if c.dir is None:
1514                 dir = ''
1515             else:
1516                 dir = c.dir
1517             if c.path is None:
1518                 path = ''
1519             else:
1520                 path = c.path
1521             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1522             prelim_txt = Command.shell_make_quote_list(prelims, False)
1523             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1524         return '\n'.join(cmds)
1525
1526     def status_logs(self, logsdir):
1527         """Return the list of log files with command status."""
1528         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1529                 for c in self.cmdlist]
1530
1531
1532 def get_parser():
1533     """Return an argument parser for this module."""
1534     parser = argparse.ArgumentParser(description=__doc__)
1535     parser.add_argument('-j', dest='parallelism',
1536                         help='Run this number of jobs in parallel',
1537                         type=int, default=os.cpu_count())
1538     parser.add_argument('--keep', dest='keep',
1539                         help='Whether to keep all build directories, '
1540                         'none or only those from failed builds',
1541                         default='none', choices=('none', 'all', 'failed'))
1542     parser.add_argument('--replace-sources', action='store_true',
1543                         help='Remove and replace source directories '
1544                         'with the wrong version of a component')
1545     parser.add_argument('--strip', action='store_true',
1546                         help='Strip installed glibc libraries')
1547     parser.add_argument('topdir',
1548                         help='Toplevel working directory')
1549     parser.add_argument('action',
1550                         help='What to do',
1551                         choices=('checkout', 'bot-cycle', 'bot',
1552                                  'host-libraries', 'compilers', 'glibcs'))
1553     parser.add_argument('configs',
1554                         help='Versions to check out or configurations to build',
1555                         nargs='*')
1556     return parser
1557
1558
1559 def main(argv):
1560     """The main entry point."""
1561     parser = get_parser()
1562     opts = parser.parse_args(argv)
1563     topdir = os.path.abspath(opts.topdir)
1564     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1565                   opts.strip, opts.action)
1566     ctx.run_builds(opts.action, opts.configs)
1567
1568
1569 if __name__ == '__main__':
1570     main(sys.argv[1:])