scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2017 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52 try:
  53     os.cpu_count
  54 except:
  55     import multiprocessing
  56     os.cpu_count = lambda: multiprocessing.cpu_count()
  57
  58 try:
  59     re.fullmatch
  60 except:
  61     re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)
  62
  63 try:
  64     subprocess.run
  65 except:
  66     class _CompletedProcess:
  67         def __init__(self, args, returncode, stdout=None, stderr=None):
  68             self.args = args
  69             self.returncode = returncode
  70             self.stdout = stdout
  71             self.stderr = stderr
  72
  73     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  74         assert(timeout is None)
  75         with subprocess.Popen(*popenargs, **kwargs) as process:
  76             try:
  77                 stdout, stderr = process.communicate(input)
  78             except:
  79                 process.kill()
  80                 process.wait()
  81                 raise
  82             returncode = process.poll()
  83             if check and returncode:
  84                 raise subprocess.CalledProcessError(returncode, popenargs)
  85         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  86
  87     subprocess.run = _run
  88
  89
  90 class Context(object):
  91     """The global state associated with builds in a given directory."""
  92
  93     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  94                  action):
  95         """Initialize the context."""
  96         self.topdir = topdir
  97         self.parallelism = parallelism
  98         self.keep = keep
  99         self.replace_sources = replace_sources
 100         self.strip = strip
 101         self.srcdir = os.path.join(topdir, 'src')
 102         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 103         self.build_state_json = os.path.join(topdir, 'build-state.json')
 104         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 105         self.installdir = os.path.join(topdir, 'install')
 106         self.host_libraries_installdir = os.path.join(self.installdir,
 107                                                       'host-libraries')
 108         self.builddir = os.path.join(topdir, 'build')
 109         self.logsdir = os.path.join(topdir, 'logs')
 110         self.logsdir_old = os.path.join(topdir, 'logs-old')
 111         self.makefile = os.path.join(self.builddir, 'Makefile')
 112         self.wrapper = os.path.join(self.builddir, 'wrapper')
 113         self.save_logs = os.path.join(self.builddir, 'save-logs')
 114         self.script_text = self.get_script_text()
 115         if action != 'checkout':
 116             self.build_triplet = self.get_build_triplet()
 117             self.glibc_version = self.get_glibc_version()
 118         self.configs = {}
 119         self.glibc_configs = {}
 120         self.makefile_pieces = ['.PHONY: all\n']
 121         self.add_all_configs()
 122         self.load_versions_json()
 123         self.load_build_state_json()
 124         self.status_log_list = []
 125         self.email_warning = False
 126
 127     def get_script_text(self):
 128         """Return the text of this script."""
 129         with open(sys.argv[0], 'r') as f:
 130             return f.read()
 131
 132     def exec_self(self):
 133         """Re-execute this script with the same arguments."""
 134         sys.stdout.flush()
 135         os.execv(sys.executable, [sys.executable] + sys.argv)
 136
 137     def get_build_triplet(self):
 138         """Determine the build triplet with config.guess."""
 139         config_guess = os.path.join(self.component_srcdir('gcc'),
 140                                     'config.guess')
 141         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 142                                 check=True, universal_newlines=True).stdout
 143         return cg_out.rstrip()
 144
 145     def get_glibc_version(self):
 146         """Determine the glibc version number (major.minor)."""
 147         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 148         with open(version_h, 'r') as f:
 149             lines = f.readlines()
 150         starttext = '#define VERSION "'
 151         for l in lines:
 152             if l.startswith(starttext):
 153                 l = l[len(starttext):]
 154                 l = l.rstrip('"\n')
 155                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 156                 return '%s.%s' % m.group(1, 2)
 157         print('error: could not determine glibc version')
 158         exit(1)
 159
 160     def add_all_configs(self):
 161         """Add all known glibc build configurations."""
 162         self.add_config(arch='aarch64',
 163                         os_name='linux-gnu')
 164         self.add_config(arch='aarch64_be',
 165                         os_name='linux-gnu')
 166         self.add_config(arch='alpha',
 167                         os_name='linux-gnu')
 168         self.add_config(arch='arm',
 169                         os_name='linux-gnueabi')
 170         self.add_config(arch='armeb',
 171                         os_name='linux-gnueabi')
 172         self.add_config(arch='armeb',
 173                         os_name='linux-gnueabi',
 174                         variant='be8',
 175                         gcc_cfg=['--with-arch=armv7-a'])
 176         self.add_config(arch='arm',
 177                         os_name='linux-gnueabihf')
 178         self.add_config(arch='armeb',
 179                         os_name='linux-gnueabihf')
 180         self.add_config(arch='armeb',
 181                         os_name='linux-gnueabihf',
 182                         variant='be8',
 183                         gcc_cfg=['--with-arch=armv7-a'])
 184         self.add_config(arch='hppa',
 185                         os_name='linux-gnu')
 186         self.add_config(arch='ia64',
 187                         os_name='linux-gnu',
 188                         first_gcc_cfg=['--with-system-libunwind'])
 189         self.add_config(arch='m68k',
 190                         os_name='linux-gnu',
 191                         gcc_cfg=['--disable-multilib'])
 192         self.add_config(arch='m68k',
 193                         os_name='linux-gnu',
 194                         variant='coldfire',
 195                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 196         self.add_config(arch='microblaze',
 197                         os_name='linux-gnu',
 198                         gcc_cfg=['--disable-multilib'])
 199         self.add_config(arch='microblazeel',
 200                         os_name='linux-gnu',
 201                         gcc_cfg=['--disable-multilib'])
 202         self.add_config(arch='mips64',
 203                         os_name='linux-gnu',
 204                         gcc_cfg=['--with-mips-plt'],
 205                         glibcs=[{'variant': 'n32'},
 206                                 {'arch': 'mips',
 207                                  'ccopts': '-mabi=32'},
 208                                 {'variant': 'n64',
 209                                  'ccopts': '-mabi=64'}])
 210         self.add_config(arch='mips64',
 211                         os_name='linux-gnu',
 212                         variant='soft',
 213                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 214                         glibcs=[{'variant': 'n32-soft',
 215                                  'cfg': ['--without-fp']},
 216                                 {'variant': 'soft',
 217                                  'arch': 'mips',
 218                                  'ccopts': '-mabi=32',
 219                                  'cfg': ['--without-fp']},
 220                                 {'variant': 'n64-soft',
 221                                  'ccopts': '-mabi=64',
 222                                  'cfg': ['--without-fp']}])
 223         self.add_config(arch='mips64',
 224                         os_name='linux-gnu',
 225                         variant='nan2008',
 226                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 227                                  '--with-arch-64=mips64r2',
 228                                  '--with-arch-32=mips32r2'],
 229                         glibcs=[{'variant': 'n32-nan2008'},
 230                                 {'variant': 'nan2008',
 231                                  'arch': 'mips',
 232                                  'ccopts': '-mabi=32'},
 233                                 {'variant': 'n64-nan2008',
 234                                  'ccopts': '-mabi=64'}])
 235         self.add_config(arch='mips64',
 236                         os_name='linux-gnu',
 237                         variant='nan2008-soft',
 238                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 239                                  '--with-arch-64=mips64r2',
 240                                  '--with-arch-32=mips32r2',
 241                                  '--with-float=soft'],
 242                         glibcs=[{'variant': 'n32-nan2008-soft',
 243                                  'cfg': ['--without-fp']},
 244                                 {'variant': 'nan2008-soft',
 245                                  'arch': 'mips',
 246                                  'ccopts': '-mabi=32',
 247                                  'cfg': ['--without-fp']},
 248                                 {'variant': 'n64-nan2008-soft',
 249                                  'ccopts': '-mabi=64',
 250                                  'cfg': ['--without-fp']}])
 251         self.add_config(arch='mips64el',
 252                         os_name='linux-gnu',
 253                         gcc_cfg=['--with-mips-plt'],
 254                         glibcs=[{'variant': 'n32'},
 255                                 {'arch': 'mipsel',
 256                                  'ccopts': '-mabi=32'},
 257                                 {'variant': 'n64',
 258                                  'ccopts': '-mabi=64'}])
 259         self.add_config(arch='mips64el',
 260                         os_name='linux-gnu',
 261                         variant='soft',
 262                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 263                         glibcs=[{'variant': 'n32-soft',
 264                                  'cfg': ['--without-fp']},
 265                                 {'variant': 'soft',
 266                                  'arch': 'mipsel',
 267                                  'ccopts': '-mabi=32',
 268                                  'cfg': ['--without-fp']},
 269                                 {'variant': 'n64-soft',
 270                                  'ccopts': '-mabi=64',
 271                                  'cfg': ['--without-fp']}])
 272         self.add_config(arch='mips64el',
 273                         os_name='linux-gnu',
 274                         variant='nan2008',
 275                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 276                                  '--with-arch-64=mips64r2',
 277                                  '--with-arch-32=mips32r2'],
 278                         glibcs=[{'variant': 'n32-nan2008'},
 279                                 {'variant': 'nan2008',
 280                                  'arch': 'mipsel',
 281                                  'ccopts': '-mabi=32'},
 282                                 {'variant': 'n64-nan2008',
 283                                  'ccopts': '-mabi=64'}])
 284         self.add_config(arch='mips64el',
 285                         os_name='linux-gnu',
 286                         variant='nan2008-soft',
 287                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 288                                  '--with-arch-64=mips64r2',
 289                                  '--with-arch-32=mips32r2',
 290                                  '--with-float=soft'],
 291                         glibcs=[{'variant': 'n32-nan2008-soft',
 292                                  'cfg': ['--without-fp']},
 293                                 {'variant': 'nan2008-soft',
 294                                  'arch': 'mipsel',
 295                                  'ccopts': '-mabi=32',
 296                                  'cfg': ['--without-fp']},
 297                                 {'variant': 'n64-nan2008-soft',
 298                                  'ccopts': '-mabi=64',
 299                                  'cfg': ['--without-fp']}])
 300         self.add_config(arch='nios2',
 301                         os_name='linux-gnu')
 302         self.add_config(arch='powerpc',
 303                         os_name='linux-gnu',
 304                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 305                         extra_glibcs=[{'variant': 'power4',
 306                                        'ccopts': '-mcpu=power4',
 307                                        'cfg': ['--with-cpu=power4']}])
 308         self.add_config(arch='powerpc',
 309                         os_name='linux-gnu',
 310                         variant='soft',
 311                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 312                                  '--enable-secureplt'],
 313                         glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
 314         self.add_config(arch='powerpc64',
 315                         os_name='linux-gnu',
 316                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 317         self.add_config(arch='powerpc64le',
 318                         os_name='linux-gnu',
 319                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 320         self.add_config(arch='powerpc',
 321                         os_name='linux-gnuspe',
 322                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 323                                  '--enable-e500-double'],
 324                         glibcs=[{'cfg': ['--without-fp']}])
 325         self.add_config(arch='powerpc',
 326                         os_name='linux-gnuspe',
 327                         variant='e500v1',
 328                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 329                         glibcs=[{'variant': 'e500v1', 'cfg': ['--without-fp']}])
 330         self.add_config(arch='s390x',
 331                         os_name='linux-gnu',
 332                         glibcs=[{},
 333                                 {'arch': 's390', 'ccopts': '-m31'}])
 334         self.add_config(arch='sh3',
 335                         os_name='linux-gnu')
 336         self.add_config(arch='sh3eb',
 337                         os_name='linux-gnu')
 338         self.add_config(arch='sh4',
 339                         os_name='linux-gnu')
 340         self.add_config(arch='sh4eb',
 341                         os_name='linux-gnu')
 342         self.add_config(arch='sh4',
 343                         os_name='linux-gnu',
 344                         variant='soft',
 345                         gcc_cfg=['--without-fp'],
 346                         glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
 347         self.add_config(arch='sh4eb',
 348                         os_name='linux-gnu',
 349                         variant='soft',
 350                         gcc_cfg=['--without-fp'],
 351                         glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
 352         self.add_config(arch='sparc64',
 353                         os_name='linux-gnu',
 354                         glibcs=[{},
 355                                 {'arch': 'sparcv9',
 356                                  'ccopts': '-m32 -mlong-double-128'}])
 357         self.add_config(arch='tilegx',
 358                         os_name='linux-gnu',
 359                         glibcs=[{},
 360                                 {'variant': '32', 'ccopts': '-m32'}])
 361         self.add_config(arch='tilegxbe',
 362                         os_name='linux-gnu',
 363                         glibcs=[{},
 364                                 {'variant': '32', 'ccopts': '-m32'}])
 365         self.add_config(arch='tilepro',
 366                         os_name='linux-gnu')
 367         self.add_config(arch='x86_64',
 368                         os_name='linux-gnu',
 369                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 370                         glibcs=[{},
 371                                 {'variant': 'x32', 'ccopts': '-mx32'},
 372                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 373                         extra_glibcs=[{'variant': 'disable-multi-arch',
 374                                        'cfg': ['--disable-multi-arch']},
 375                                       {'variant': 'disable-multi-arch',
 376                                        'arch': 'i686',
 377                                        'ccopts': '-m32 -march=i686',
 378                                        'cfg': ['--disable-multi-arch']},
 379                                       {'arch': 'i486',
 380                                        'ccopts': '-m32 -march=i486'},
 381                                       {'arch': 'i586',
 382                                        'ccopts': '-m32 -march=i586'}])
 383
 384     def add_config(self, **args):
 385         """Add an individual build configuration."""
 386         cfg = Config(self, **args)
 387         if cfg.name in self.configs:
 388             print('error: duplicate config %s' % cfg.name)
 389             exit(1)
 390         self.configs[cfg.name] = cfg
 391         for c in cfg.all_glibcs:
 392             if c.name in self.glibc_configs:
 393                 print('error: duplicate glibc config %s' % c.name)
 394                 exit(1)
 395             self.glibc_configs[c.name] = c
 396
 397     def component_srcdir(self, component):
 398         """Return the source directory for a given component, e.g. gcc."""
 399         return os.path.join(self.srcdir, component)
 400
 401     def component_builddir(self, action, config, component, subconfig=None):
 402         """Return the directory to use for a build."""
 403         if config is None:
 404             # Host libraries.
 405             assert subconfig is None
 406             return os.path.join(self.builddir, action, component)
 407         if subconfig is None:
 408             return os.path.join(self.builddir, action, config, component)
 409         else:
 410             # glibc build as part of compiler build.
 411             return os.path.join(self.builddir, action, config, component,
 412                                 subconfig)
 413
 414     def compiler_installdir(self, config):
 415         """Return the directory in which to install a compiler."""
 416         return os.path.join(self.installdir, 'compilers', config)
 417
 418     def compiler_bindir(self, config):
 419         """Return the directory in which to find compiler binaries."""
 420         return os.path.join(self.compiler_installdir(config), 'bin')
 421
 422     def compiler_sysroot(self, config):
 423         """Return the sysroot directory for a compiler."""
 424         return os.path.join(self.compiler_installdir(config), 'sysroot')
 425
 426     def glibc_installdir(self, config):
 427         """Return the directory in which to install glibc."""
 428         return os.path.join(self.installdir, 'glibcs', config)
 429
 430     def run_builds(self, action, configs):
 431         """Run the requested builds."""
 432         if action == 'checkout':
 433             self.checkout(configs)
 434             return
 435         if action == 'bot-cycle':
 436             if configs:
 437                 print('error: configurations specified for bot-cycle')
 438                 exit(1)
 439             self.bot_cycle()
 440             return
 441         if action == 'bot':
 442             if configs:
 443                 print('error: configurations specified for bot')
 444                 exit(1)
 445             self.bot()
 446             return
 447         if action == 'host-libraries' and configs:
 448             print('error: configurations specified for host-libraries')
 449             exit(1)
 450         self.clear_last_build_state(action)
 451         build_time = datetime.datetime.utcnow()
 452         if action == 'host-libraries':
 453             build_components = ('gmp', 'mpfr', 'mpc')
 454             old_components = ()
 455             old_versions = {}
 456             self.build_host_libraries()
 457         elif action == 'compilers':
 458             build_components = ('binutils', 'gcc', 'glibc', 'linux')
 459             old_components = ('gmp', 'mpfr', 'mpc')
 460             old_versions = self.build_state['host-libraries']['build-versions']
 461             self.build_compilers(configs)
 462         else:
 463             build_components = ('glibc',)
 464             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
 465             old_versions = self.build_state['compilers']['build-versions']
 466             self.build_glibcs(configs)
 467         self.write_files()
 468         self.do_build()
 469         if configs:
 470             # Partial build, do not update stored state.
 471             return
 472         build_versions = {}
 473         for k in build_components:
 474             if k in self.versions:
 475                 build_versions[k] = {'version': self.versions[k]['version'],
 476                                      'revision': self.versions[k]['revision']}
 477         for k in old_components:
 478             if k in old_versions:
 479                 build_versions[k] = {'version': old_versions[k]['version'],
 480                                      'revision': old_versions[k]['revision']}
 481         self.update_build_state(action, build_time, build_versions)
 482
 483     @staticmethod
 484     def remove_dirs(*args):
 485         """Remove directories and their contents if they exist."""
 486         for dir in args:
 487             shutil.rmtree(dir, ignore_errors=True)
 488
 489     @staticmethod
 490     def remove_recreate_dirs(*args):
 491         """Remove directories if they exist, and create them as empty."""
 492         Context.remove_dirs(*args)
 493         for dir in args:
 494             os.makedirs(dir, exist_ok=True)
 495
 496     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 497         """Add makefile text for a list of commands."""
 498         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 499         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 500                                     (target, target, target, commands))
 501         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 502
 503     def write_files(self):
 504         """Write out the Makefile and wrapper script."""
 505         mftext = ''.join(self.makefile_pieces)
 506         with open(self.makefile, 'w') as f:
 507             f.write(mftext)
 508         wrapper_text = (
 509             '#!/bin/sh\n'
 510             'prev_base=$1\n'
 511             'this_base=$2\n'
 512             'desc=$3\n'
 513             'dir=$4\n'
 514             'path=$5\n'
 515             'shift 5\n'
 516             'prev_status=$prev_base-status.txt\n'
 517             'this_status=$this_base-status.txt\n'
 518             'this_log=$this_base-log.txt\n'
 519             'date > "$this_log"\n'
 520             'echo >> "$this_log"\n'
 521             'echo "Description: $desc" >> "$this_log"\n'
 522             'printf "%s" "Command:" >> "$this_log"\n'
 523             'for word in "$@"; do\n'
 524             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 525             '    printf " %s" "$word"\n'
 526             '  else\n'
 527             '    printf " \'"\n'
 528             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 529             '    printf "\'"\n'
 530             '  fi\n'
 531             'done >> "$this_log"\n'
 532             'echo >> "$this_log"\n'
 533             'echo "Directory: $dir" >> "$this_log"\n'
 534             'echo "Path addition: $path" >> "$this_log"\n'
 535             'echo >> "$this_log"\n'
 536             'record_status ()\n'
 537             '{\n'
 538             '  echo >> "$this_log"\n'
 539             '  echo "$1: $desc" > "$this_status"\n'
 540             '  echo "$1: $desc" >> "$this_log"\n'
 541             '  echo >> "$this_log"\n'
 542             '  date >> "$this_log"\n'
 543             '  echo "$1: $desc"\n'
 544             '  exit 0\n'
 545             '}\n'
 546             'check_error ()\n'
 547             '{\n'
 548             '  if [ "$1" != "0" ]; then\n'
 549             '    record_status FAIL\n'
 550             '  fi\n'
 551             '}\n'
 552             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 553             '    record_status UNRESOLVED\n'
 554             'fi\n'
 555             'if [ "$dir" ]; then\n'
 556             '  cd "$dir"\n'
 557             '  check_error "$?"\n'
 558             'fi\n'
 559             'if [ "$path" ]; then\n'
 560             '  PATH=$path:$PATH\n'
 561             'fi\n'
 562             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 563             'check_error "$?"\n'
 564             'record_status PASS\n')
 565         with open(self.wrapper, 'w') as f:
 566             f.write(wrapper_text)
 567         # Mode 0o755.
 568         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 569                      stat.S_IROTH|stat.S_IXOTH)
 570         os.chmod(self.wrapper, mode_exec)
 571         save_logs_text = (
 572             '#!/bin/sh\n'
 573             'if ! [ -f tests.sum ]; then\n'
 574             '  echo "No test summary available."\n'
 575             '  exit 0\n'
 576             'fi\n'
 577             'save_file ()\n'
 578             '{\n'
 579             '  echo "Contents of $1:"\n'
 580             '  echo\n'
 581             '  cat "$1"\n'
 582             '  echo\n'
 583             '  echo "End of contents of $1."\n'
 584             '  echo\n'
 585             '}\n'
 586             'save_file tests.sum\n'
 587             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 588             'for t in $non_pass_tests; do\n'
 589             '  if [ -f "$t.out" ]; then\n'
 590             '    save_file "$t.out"\n'
 591             '  fi\n'
 592             'done\n')
 593         with open(self.save_logs, 'w') as f:
 594             f.write(save_logs_text)
 595         os.chmod(self.save_logs, mode_exec)
 596
 597     def do_build(self):
 598         """Do the actual build."""
 599         cmd = ['make', '-j%d' % self.parallelism]
 600         subprocess.run(cmd, cwd=self.builddir, check=True)
 601
 602     def build_host_libraries(self):
 603         """Build the host libraries."""
 604         installdir = self.host_libraries_installdir
 605         builddir = os.path.join(self.builddir, 'host-libraries')
 606         logsdir = os.path.join(self.logsdir, 'host-libraries')
 607         self.remove_recreate_dirs(installdir, builddir, logsdir)
 608         cmdlist = CommandList('host-libraries', self.keep)
 609         self.build_host_library(cmdlist, 'gmp')
 610         self.build_host_library(cmdlist, 'mpfr',
 611                                 ['--with-gmp=%s' % installdir])
 612         self.build_host_library(cmdlist, 'mpc',
 613                                 ['--with-gmp=%s' % installdir,
 614                                 '--with-mpfr=%s' % installdir])
 615         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 616         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 617
 618     def build_host_library(self, cmdlist, lib, extra_opts=None):
 619         """Build one host library."""
 620         srcdir = self.component_srcdir(lib)
 621         builddir = self.component_builddir('host-libraries', None, lib)
 622         installdir = self.host_libraries_installdir
 623         cmdlist.push_subdesc(lib)
 624         cmdlist.create_use_dir(builddir)
 625         cfg_cmd = [os.path.join(srcdir, 'configure'),
 626                    '--prefix=%s' % installdir,
 627                    '--disable-shared']
 628         if extra_opts:
 629             cfg_cmd.extend (extra_opts)
 630         cmdlist.add_command('configure', cfg_cmd)
 631         cmdlist.add_command('build', ['make'])
 632         cmdlist.add_command('check', ['make', 'check'])
 633         cmdlist.add_command('install', ['make', 'install'])
 634         cmdlist.cleanup_dir()
 635         cmdlist.pop_subdesc()
 636
 637     def build_compilers(self, configs):
 638         """Build the compilers."""
 639         if not configs:
 640             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 641             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 642             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 643             configs = sorted(self.configs.keys())
 644         for c in configs:
 645             self.configs[c].build()
 646
 647     def build_glibcs(self, configs):
 648         """Build the glibcs."""
 649         if not configs:
 650             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 651             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 652             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 653             configs = sorted(self.glibc_configs.keys())
 654         for c in configs:
 655             self.glibc_configs[c].build()
 656
 657     def load_versions_json(self):
 658         """Load information about source directory versions."""
 659         if not os.access(self.versions_json, os.F_OK):
 660             self.versions = {}
 661             return
 662         with open(self.versions_json, 'r') as f:
 663             self.versions = json.load(f)
 664
 665     def store_json(self, data, filename):
 666         """Store information in a JSON file."""
 667         filename_tmp = filename + '.tmp'
 668         with open(filename_tmp, 'w') as f:
 669             json.dump(data, f, indent=2, sort_keys=True)
 670         os.rename(filename_tmp, filename)
 671
 672     def store_versions_json(self):
 673         """Store information about source directory versions."""
 674         self.store_json(self.versions, self.versions_json)
 675
 676     def set_component_version(self, component, version, explicit, revision):
 677         """Set the version information for a component."""
 678         self.versions[component] = {'version': version,
 679                                     'explicit': explicit,
 680                                     'revision': revision}
 681         self.store_versions_json()
 682
 683     def checkout(self, versions):
 684         """Check out the desired component versions."""
 685         default_versions = {'binutils': 'vcs-2.29',
 686                             'gcc': 'vcs-7',
 687                             'glibc': 'vcs-mainline',
 688                             'gmp': '6.1.2',
 689                             'linux': '4.13',
 690                             'mpc': '1.0.3',
 691                             'mpfr': '3.1.6'}
 692         use_versions = {}
 693         explicit_versions = {}
 694         for v in versions:
 695             found_v = False
 696             for k in default_versions.keys():
 697                 kx = k + '-'
 698                 if v.startswith(kx):
 699                     vx = v[len(kx):]
 700                     if k in use_versions:
 701                         print('error: multiple versions for %s' % k)
 702                         exit(1)
 703                     use_versions[k] = vx
 704                     explicit_versions[k] = True
 705                     found_v = True
 706                     break
 707             if not found_v:
 708                 print('error: unknown component in %s' % v)
 709                 exit(1)
 710         for k in default_versions.keys():
 711             if k not in use_versions:
 712                 if k in self.versions and self.versions[k]['explicit']:
 713                     use_versions[k] = self.versions[k]['version']
 714                     explicit_versions[k] = True
 715                 else:
 716                     use_versions[k] = default_versions[k]
 717                     explicit_versions[k] = False
 718         os.makedirs(self.srcdir, exist_ok=True)
 719         for k in sorted(default_versions.keys()):
 720             update = os.access(self.component_srcdir(k), os.F_OK)
 721             v = use_versions[k]
 722             if (update and
 723                 k in self.versions and
 724                 v != self.versions[k]['version']):
 725                 if not self.replace_sources:
 726                     print('error: version of %s has changed from %s to %s, '
 727                           'use --replace-sources to check out again' %
 728                           (k, self.versions[k]['version'], v))
 729                     exit(1)
 730                 shutil.rmtree(self.component_srcdir(k))
 731                 update = False
 732             if v.startswith('vcs-'):
 733                 revision = self.checkout_vcs(k, v[4:], update)
 734             else:
 735                 self.checkout_tar(k, v, update)
 736                 revision = v
 737             self.set_component_version(k, v, explicit_versions[k], revision)
 738         if self.get_script_text() != self.script_text:
 739             # Rerun the checkout process in case the updated script
 740             # uses different default versions or new components.
 741             self.exec_self()
 742
 743     def checkout_vcs(self, component, version, update):
 744         """Check out the given version of the given component from version
 745         control.  Return a revision identifier."""
 746         if component == 'binutils':
 747             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 748             if version == 'mainline':
 749                 git_branch = 'master'
 750             else:
 751                 trans = str.maketrans({'.': '_'})
 752                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 753             return self.git_checkout(component, git_url, git_branch, update)
 754         elif component == 'gcc':
 755             if version == 'mainline':
 756                 branch = 'trunk'
 757             else:
 758                 trans = str.maketrans({'.': '_'})
 759                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 760             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 761             return self.gcc_checkout(svn_url, update)
 762         elif component == 'glibc':
 763             git_url = 'git://sourceware.org/git/glibc.git'
 764             if version == 'mainline':
 765                 git_branch = 'master'
 766             else:
 767                 git_branch = 'release/%s/master' % version
 768             r = self.git_checkout(component, git_url, git_branch, update)
 769             self.fix_glibc_timestamps()
 770             return r
 771         else:
 772             print('error: component %s coming from VCS' % component)
 773             exit(1)
 774
 775     def git_checkout(self, component, git_url, git_branch, update):
 776         """Check out a component from git.  Return a commit identifier."""
 777         if update:
 778             subprocess.run(['git', 'remote', 'prune', 'origin'],
 779                            cwd=self.component_srcdir(component), check=True)
 780             subprocess.run(['git', 'pull', '-q'],
 781                            cwd=self.component_srcdir(component), check=True)
 782         else:
 783             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 784                             self.component_srcdir(component)], check=True)
 785         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 786                            cwd=self.component_srcdir(component),
 787                            stdout=subprocess.PIPE,
 788                            check=True, universal_newlines=True).stdout
 789         return r.rstrip()
 790
 791     def fix_glibc_timestamps(self):
 792         """Fix timestamps in a glibc checkout."""
 793         # Ensure that builds do not try to regenerate generated files
 794         # in the source tree.
 795         srcdir = self.component_srcdir('glibc')
 796         for dirpath, dirnames, filenames in os.walk(srcdir):
 797             for f in filenames:
 798                 if (f == 'configure' or
 799                     f == 'preconfigure' or
 800                     f.endswith('-kw.h')):
 801                     to_touch = os.path.join(dirpath, f)
 802                     subprocess.run(['touch', to_touch], check=True)
 803
 804     def gcc_checkout(self, svn_url, update):
 805         """Check out GCC from SVN.  Return the revision number."""
 806         if not update:
 807             subprocess.run(['svn', 'co', '-q', svn_url,
 808                             self.component_srcdir('gcc')], check=True)
 809         subprocess.run(['contrib/gcc_update', '--silent'],
 810                        cwd=self.component_srcdir('gcc'), check=True)
 811         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 812                            stdout=subprocess.PIPE,
 813                            check=True, universal_newlines=True).stdout
 814         return r.rstrip()
 815
 816     def checkout_tar(self, component, version, update):
 817         """Check out the given version of the given component from a
 818         tarball."""
 819         if update:
 820             return
 821         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 822                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 823                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 824                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 825                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 826                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
 827         if component not in url_map:
 828             print('error: component %s coming from tarball' % component)
 829             exit(1)
 830         url = url_map[component] % {'version': version}
 831         filename = os.path.join(self.srcdir, url.split('/')[-1])
 832         response = urllib.request.urlopen(url)
 833         data = response.read()
 834         with open(filename, 'wb') as f:
 835             f.write(data)
 836         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 837                        check=True)
 838         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 839                   self.component_srcdir(component))
 840         os.remove(filename)
 841
 842     def load_build_state_json(self):
 843         """Load information about the state of previous builds."""
 844         if os.access(self.build_state_json, os.F_OK):
 845             with open(self.build_state_json, 'r') as f:
 846                 self.build_state = json.load(f)
 847         else:
 848             self.build_state = {}
 849         for k in ('host-libraries', 'compilers', 'glibcs'):
 850             if k not in self.build_state:
 851                 self.build_state[k] = {}
 852             if 'build-time' not in self.build_state[k]:
 853                 self.build_state[k]['build-time'] = ''
 854             if 'build-versions' not in self.build_state[k]:
 855                 self.build_state[k]['build-versions'] = {}
 856             if 'build-results' not in self.build_state[k]:
 857                 self.build_state[k]['build-results'] = {}
 858             if 'result-changes' not in self.build_state[k]:
 859                 self.build_state[k]['result-changes'] = {}
 860             if 'ever-passed' not in self.build_state[k]:
 861                 self.build_state[k]['ever-passed'] = []
 862
 863     def store_build_state_json(self):
 864         """Store information about the state of previous builds."""
 865         self.store_json(self.build_state, self.build_state_json)
 866
 867     def clear_last_build_state(self, action):
 868         """Clear information about the state of part of the build."""
 869         # We clear the last build time and versions when starting a
 870         # new build.  The results of the last build are kept around,
 871         # as comparison is still meaningful if this build is aborted
 872         # and a new one started.
 873         self.build_state[action]['build-time'] = ''
 874         self.build_state[action]['build-versions'] = {}
 875         self.store_build_state_json()
 876
 877     def update_build_state(self, action, build_time, build_versions):
 878         """Update the build state after a build."""
 879         build_time = build_time.replace(microsecond=0)
 880         self.build_state[action]['build-time'] = str(build_time)
 881         self.build_state[action]['build-versions'] = build_versions
 882         build_results = {}
 883         for log in self.status_log_list:
 884             with open(log, 'r') as f:
 885                 log_text = f.read()
 886             log_text = log_text.rstrip()
 887             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 888             result = m.group(1)
 889             test_name = m.group(2)
 890             assert test_name not in build_results
 891             build_results[test_name] = result
 892         old_build_results = self.build_state[action]['build-results']
 893         self.build_state[action]['build-results'] = build_results
 894         result_changes = {}
 895         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 896         for t in all_tests:
 897             if t in old_build_results:
 898                 old_res = old_build_results[t]
 899             else:
 900                 old_res = '(New test)'
 901             if t in build_results:
 902                 new_res = build_results[t]
 903             else:
 904                 new_res = '(Test removed)'
 905             if old_res != new_res:
 906                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 907         self.build_state[action]['result-changes'] = result_changes
 908         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 909                            if t in build_results}
 910         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 911         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 912                                                          new_passes)
 913         self.store_build_state_json()
 914
 915     def load_bot_config_json(self):
 916         """Load bot configuration."""
 917         with open(self.bot_config_json, 'r') as f:
 918             self.bot_config = json.load(f)
 919
 920     def part_build_old(self, action, delay):
 921         """Return whether the last build for a given action was at least a
 922         given number of seconds ago, or does not have a time recorded."""
 923         old_time_str = self.build_state[action]['build-time']
 924         if not old_time_str:
 925             return True
 926         old_time = datetime.datetime.strptime(old_time_str,
 927                                               '%Y-%m-%d %H:%M:%S')
 928         new_time = datetime.datetime.utcnow()
 929         delta = new_time - old_time
 930         return delta.total_seconds() >= delay
 931
 932     def bot_cycle(self):
 933         """Run a single round of checkout and builds."""
 934         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 935         self.load_bot_config_json()
 936         actions = ('host-libraries', 'compilers', 'glibcs')
 937         self.bot_run_self(['--replace-sources'], 'checkout')
 938         self.load_versions_json()
 939         if self.get_script_text() != self.script_text:
 940             print('Script changed, re-execing.')
 941             # On script change, all parts of the build should be rerun.
 942             for a in actions:
 943                 self.clear_last_build_state(a)
 944             self.exec_self()
 945         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 946                             'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
 947                             'glibcs': ('glibc',)}
 948         must_build = {}
 949         for a in actions:
 950             build_vers = self.build_state[a]['build-versions']
 951             must_build[a] = False
 952             if not self.build_state[a]['build-time']:
 953                 must_build[a] = True
 954             old_vers = {}
 955             new_vers = {}
 956             for c in check_components[a]:
 957                 if c in build_vers:
 958                     old_vers[c] = build_vers[c]
 959                 new_vers[c] = {'version': self.versions[c]['version'],
 960                                'revision': self.versions[c]['revision']}
 961             if new_vers == old_vers:
 962                 print('Versions for %s unchanged.' % a)
 963             else:
 964                 print('Versions changed or rebuild forced for %s.' % a)
 965                 if a == 'compilers' and not self.part_build_old(
 966                         a, self.bot_config['compilers-rebuild-delay']):
 967                     print('Not requiring rebuild of compilers this soon.')
 968                 else:
 969                     must_build[a] = True
 970         if must_build['host-libraries']:
 971             must_build['compilers'] = True
 972         if must_build['compilers']:
 973             must_build['glibcs'] = True
 974         for a in actions:
 975             if must_build[a]:
 976                 print('Must rebuild %s.' % a)
 977                 self.clear_last_build_state(a)
 978             else:
 979                 print('No need to rebuild %s.' % a)
 980         if os.access(self.logsdir, os.F_OK):
 981             shutil.rmtree(self.logsdir_old, ignore_errors=True)
 982             shutil.copytree(self.logsdir, self.logsdir_old)
 983         for a in actions:
 984             if must_build[a]:
 985                 build_time = datetime.datetime.utcnow()
 986                 print('Rebuilding %s at %s.' % (a, str(build_time)))
 987                 self.bot_run_self([], a)
 988                 self.load_build_state_json()
 989                 self.bot_build_mail(a, build_time)
 990         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
 991
 992     def bot_build_mail(self, action, build_time):
 993         """Send email with the results of a build."""
 994         if not ('email-from' in self.bot_config and
 995                 'email-server' in self.bot_config and
 996                 'email-subject' in self.bot_config and
 997                 'email-to' in self.bot_config):
 998             if not self.email_warning:
 999                 print("Email not configured, not sending.")
1000                 self.email_warning = True
1001             return
1002
1003         build_time = build_time.replace(microsecond=0)
1004         subject = (self.bot_config['email-subject'] %
1005                    {'action': action,
1006                     'build-time': str(build_time)})
1007         results = self.build_state[action]['build-results']
1008         changes = self.build_state[action]['result-changes']
1009         ever_passed = set(self.build_state[action]['ever-passed'])
1010         versions = self.build_state[action]['build-versions']
1011         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1012         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1013         all_fails = {k for k in results if results[k] == 'FAIL'}
1014         if new_regressions:
1015             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1016             new_reg_text = ('New regressions:\n\n%s\n\n' %
1017                             '\n'.join(new_reg_list))
1018         else:
1019             new_reg_text = ''
1020         if all_regressions:
1021             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1022             all_reg_text = ('All regressions:\n\n%s\n\n' %
1023                             '\n'.join(all_reg_list))
1024         else:
1025             all_reg_text = ''
1026         if all_fails:
1027             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1028             all_fail_text = ('All failures:\n\n%s\n\n' %
1029                              '\n'.join(all_fail_list))
1030         else:
1031             all_fail_text = ''
1032         if changes:
1033             changes_list = sorted(changes.keys())
1034             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1035             changes_text = ('All changed results:\n\n%s\n\n' %
1036                             '\n'.join(changes_list))
1037         else:
1038             changes_text = ''
1039         results_text = (new_reg_text + all_reg_text + all_fail_text +
1040                         changes_text)
1041         if not results_text:
1042             results_text = 'Clean build with unchanged results.\n\n'
1043         versions_list = sorted(versions.keys())
1044         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1045                                           versions[k]['revision'])
1046                          for k in versions_list]
1047         versions_text = ('Component versions for this build:\n\n%s\n' %
1048                          '\n'.join(versions_list))
1049         body_text = results_text + versions_text
1050         msg = email.mime.text.MIMEText(body_text)
1051         msg['Subject'] = subject
1052         msg['From'] = self.bot_config['email-from']
1053         msg['To'] = self.bot_config['email-to']
1054         msg['Message-ID'] = email.utils.make_msgid()
1055         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1056         with smtplib.SMTP(self.bot_config['email-server']) as s:
1057             s.send_message(msg)
1058
1059     def bot_run_self(self, opts, action, check=True):
1060         """Run a copy of this script with given options."""
1061         cmd = [sys.executable, sys.argv[0], '--keep=none',
1062                '-j%d' % self.parallelism]
1063         cmd.extend(opts)
1064         cmd.extend([self.topdir, action])
1065         sys.stdout.flush()
1066         subprocess.run(cmd, check=check)
1067
1068     def bot(self):
1069         """Run repeated rounds of checkout and builds."""
1070         while True:
1071             self.load_bot_config_json()
1072             if not self.bot_config['run']:
1073                 print('Bot exiting by request.')
1074                 exit(0)
1075             self.bot_run_self([], 'bot-cycle', check=False)
1076             self.load_bot_config_json()
1077             if not self.bot_config['run']:
1078                 print('Bot exiting by request.')
1079                 exit(0)
1080             time.sleep(self.bot_config['delay'])
1081             if self.get_script_text() != self.script_text:
1082                 print('Script changed, bot re-execing.')
1083                 self.exec_self()
1084
1085
1086 class Config(object):
1087     """A configuration for building a compiler and associated libraries."""
1088
1089     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1090                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1091         """Initialize a Config object."""
1092         self.ctx = ctx
1093         self.arch = arch
1094         self.os = os_name
1095         self.variant = variant
1096         if variant is None:
1097             self.name = '%s-%s' % (arch, os_name)
1098         else:
1099             self.name = '%s-%s-%s' % (arch, os_name, variant)
1100         self.triplet = '%s-glibc-%s' % (arch, os_name)
1101         if gcc_cfg is None:
1102             self.gcc_cfg = []
1103         else:
1104             self.gcc_cfg = gcc_cfg
1105         if first_gcc_cfg is None:
1106             self.first_gcc_cfg = []
1107         else:
1108             self.first_gcc_cfg = first_gcc_cfg
1109         if glibcs is None:
1110             glibcs = [{'variant': variant}]
1111         if extra_glibcs is None:
1112             extra_glibcs = []
1113         glibcs = [Glibc(self, **g) for g in glibcs]
1114         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1115         self.all_glibcs = glibcs + extra_glibcs
1116         self.compiler_glibcs = glibcs
1117         self.installdir = ctx.compiler_installdir(self.name)
1118         self.bindir = ctx.compiler_bindir(self.name)
1119         self.sysroot = ctx.compiler_sysroot(self.name)
1120         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1121         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1122
1123     def component_builddir(self, component):
1124         """Return the directory to use for a (non-glibc) build."""
1125         return self.ctx.component_builddir('compilers', self.name, component)
1126
1127     def build(self):
1128         """Generate commands to build this compiler."""
1129         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1130                                       self.logsdir)
1131         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1132         cmdlist.add_command('check-host-libraries',
1133                             ['test', '-f',
1134                              os.path.join(self.ctx.host_libraries_installdir,
1135                                           'ok')])
1136         cmdlist.use_path(self.bindir)
1137         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1138                               ['--disable-gdb',
1139                                '--disable-libdecnumber',
1140                                '--disable-readline',
1141                                '--disable-sim'])
1142         if self.os.startswith('linux'):
1143             self.install_linux_headers(cmdlist)
1144         self.build_gcc(cmdlist, True)
1145         for g in self.compiler_glibcs:
1146             cmdlist.push_subdesc('glibc')
1147             cmdlist.push_subdesc(g.name)
1148             g.build_glibc(cmdlist, True)
1149             cmdlist.pop_subdesc()
1150             cmdlist.pop_subdesc()
1151         self.build_gcc(cmdlist, False)
1152         cmdlist.add_command('done', ['touch',
1153                                      os.path.join(self.installdir, 'ok')])
1154         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1155                                       self.logsdir)
1156
1157     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1158         """Build one cross tool."""
1159         srcdir = self.ctx.component_srcdir(tool_src)
1160         builddir = self.component_builddir(tool_build)
1161         cmdlist.push_subdesc(tool_build)
1162         cmdlist.create_use_dir(builddir)
1163         cfg_cmd = [os.path.join(srcdir, 'configure'),
1164                    '--prefix=%s' % self.installdir,
1165                    '--build=%s' % self.ctx.build_triplet,
1166                    '--host=%s' % self.ctx.build_triplet,
1167                    '--target=%s' % self.triplet,
1168                    '--with-sysroot=%s' % self.sysroot]
1169         if extra_opts:
1170             cfg_cmd.extend(extra_opts)
1171         cmdlist.add_command('configure', cfg_cmd)
1172         cmdlist.add_command('build', ['make'])
1173         # Parallel "make install" for GCC has race conditions that can
1174         # cause it to fail; see
1175         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1176         # problems are not known for binutils, but doing the
1177         # installation in parallel within a particular toolchain build
1178         # (as opposed to installation of one toolchain from
1179         # build-many-glibcs.py running in parallel to the installation
1180         # of other toolchains being built) is not known to be
1181         # significantly beneficial, so it is simplest just to disable
1182         # parallel install for cross tools here.
1183         cmdlist.add_command('install', ['make', '-j1', 'install'])
1184         cmdlist.cleanup_dir()
1185         cmdlist.pop_subdesc()
1186
1187     def install_linux_headers(self, cmdlist):
1188         """Install Linux kernel headers."""
1189         arch_map = {'aarch64': 'arm64',
1190                     'alpha': 'alpha',
1191                     'arm': 'arm',
1192                     'hppa': 'parisc',
1193                     'i486': 'x86',
1194                     'i586': 'x86',
1195                     'i686': 'x86',
1196                     'i786': 'x86',
1197                     'ia64': 'ia64',
1198                     'm68k': 'm68k',
1199                     'microblaze': 'microblaze',
1200                     'mips': 'mips',
1201                     'nios2': 'nios2',
1202                     'powerpc': 'powerpc',
1203                     's390': 's390',
1204                     'sh': 'sh',
1205                     'sparc': 'sparc',
1206                     'tile': 'tile',
1207                     'x86_64': 'x86'}
1208         linux_arch = None
1209         for k in arch_map:
1210             if self.arch.startswith(k):
1211                 linux_arch = arch_map[k]
1212                 break
1213         assert linux_arch is not None
1214         srcdir = self.ctx.component_srcdir('linux')
1215         builddir = self.component_builddir('linux')
1216         headers_dir = os.path.join(self.sysroot, 'usr')
1217         cmdlist.push_subdesc('linux')
1218         cmdlist.create_use_dir(builddir)
1219         cmdlist.add_command('install-headers',
1220                             ['make', '-C', srcdir, 'O=%s' % builddir,
1221                              'ARCH=%s' % linux_arch,
1222                              'INSTALL_HDR_PATH=%s' % headers_dir,
1223                              'headers_install'])
1224         cmdlist.cleanup_dir()
1225         cmdlist.pop_subdesc()
1226
1227     def build_gcc(self, cmdlist, bootstrap):
1228         """Build GCC."""
1229         # libsanitizer commonly breaks because of glibc header
1230         # changes, or on unusual targets.  libssp is of little
1231         # relevance with glibc's own stack checking support.
1232         cfg_opts = list(self.gcc_cfg)
1233         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1234         host_libs = self.ctx.host_libraries_installdir
1235         cfg_opts += ['--with-gmp=%s' % host_libs,
1236                      '--with-mpfr=%s' % host_libs,
1237                      '--with-mpc=%s' % host_libs]
1238         if bootstrap:
1239             tool_build = 'gcc-first'
1240             # Building a static-only, C-only compiler that is
1241             # sufficient to build glibc.  Various libraries and
1242             # features that may require libc headers must be disabled.
1243             # When configuring with a sysroot, --with-newlib is
1244             # required to define inhibit_libc (to stop some parts of
1245             # libgcc including libc headers); --without-headers is not
1246             # sufficient.
1247             cfg_opts += ['--enable-languages=c', '--disable-shared',
1248                          '--disable-threads',
1249                          '--disable-libatomic',
1250                          '--disable-decimal-float',
1251                          '--disable-libffi',
1252                          '--disable-libgomp',
1253                          '--disable-libitm',
1254                          '--disable-libmpx',
1255                          '--disable-libquadmath',
1256                          '--without-headers', '--with-newlib',
1257                          '--with-glibc-version=%s' % self.ctx.glibc_version
1258                          ]
1259             cfg_opts += self.first_gcc_cfg
1260         else:
1261             tool_build = 'gcc'
1262             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1263                          '--enable-threads']
1264         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1265
1266
1267 class Glibc(object):
1268     """A configuration for building glibc."""
1269
1270     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1271                  cfg=None, ccopts=None):
1272         """Initialize a Glibc object."""
1273         self.ctx = compiler.ctx
1274         self.compiler = compiler
1275         if arch is None:
1276             self.arch = compiler.arch
1277         else:
1278             self.arch = arch
1279         if os_name is None:
1280             self.os = compiler.os
1281         else:
1282             self.os = os_name
1283         self.variant = variant
1284         if variant is None:
1285             self.name = '%s-%s' % (self.arch, self.os)
1286         else:
1287             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1288         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1289         if cfg is None:
1290             self.cfg = []
1291         else:
1292             self.cfg = cfg
1293         self.ccopts = ccopts
1294
1295     def tool_name(self, tool):
1296         """Return the name of a cross-compilation tool."""
1297         ctool = '%s-%s' % (self.compiler.triplet, tool)
1298         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1299             ctool = '%s %s' % (ctool, self.ccopts)
1300         return ctool
1301
1302     def build(self):
1303         """Generate commands to build this glibc."""
1304         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1305         installdir = self.ctx.glibc_installdir(self.name)
1306         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1307         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1308         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1309         cmdlist.add_command('check-compilers',
1310                             ['test', '-f',
1311                              os.path.join(self.compiler.installdir, 'ok')])
1312         cmdlist.use_path(self.compiler.bindir)
1313         self.build_glibc(cmdlist, False)
1314         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1315                                       logsdir)
1316
1317     def build_glibc(self, cmdlist, for_compiler):
1318         """Generate commands to build this glibc, either as part of a compiler
1319         build or with the bootstrapped compiler (and in the latter case, run
1320         tests as well)."""
1321         srcdir = self.ctx.component_srcdir('glibc')
1322         if for_compiler:
1323             builddir = self.ctx.component_builddir('compilers',
1324                                                    self.compiler.name, 'glibc',
1325                                                    self.name)
1326             installdir = self.compiler.sysroot
1327             srcdir_copy = self.ctx.component_builddir('compilers',
1328                                                       self.compiler.name,
1329                                                       'glibc-src',
1330                                                       self.name)
1331         else:
1332             builddir = self.ctx.component_builddir('glibcs', self.name,
1333                                                    'glibc')
1334             installdir = self.ctx.glibc_installdir(self.name)
1335             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1336                                                       'glibc-src')
1337         cmdlist.create_use_dir(builddir)
1338         # glibc builds write into the source directory, and even if
1339         # not intentionally there is a risk of bugs that involve
1340         # writing into the working directory.  To avoid possible
1341         # concurrency issues, copy the source directory.
1342         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1343         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1344                    '--prefix=/usr',
1345                    '--enable-add-ons',
1346                    '--enable-profile',
1347                    '--build=%s' % self.ctx.build_triplet,
1348                    '--host=%s' % self.triplet,
1349                    'CC=%s' % self.tool_name('gcc'),
1350                    'CXX=%s' % self.tool_name('g++'),
1351                    'AR=%s' % self.tool_name('ar'),
1352                    'AS=%s' % self.tool_name('as'),
1353                    'LD=%s' % self.tool_name('ld'),
1354                    'NM=%s' % self.tool_name('nm'),
1355                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1356                    'OBJDUMP=%s' % self.tool_name('objdump'),
1357                    'RANLIB=%s' % self.tool_name('ranlib'),
1358                    'READELF=%s' % self.tool_name('readelf'),
1359                    'STRIP=%s' % self.tool_name('strip')]
1360         cfg_cmd += self.cfg
1361         cmdlist.add_command('configure', cfg_cmd)
1362         cmdlist.add_command('build', ['make'])
1363         cmdlist.add_command('install', ['make', 'install',
1364                                         'install_root=%s' % installdir])
1365         # GCC uses paths such as lib/../lib64, so make sure lib
1366         # directories always exist.
1367         cmdlist.add_command('mkdir-lib', ['mkdir', '-p',
1368                                           os.path.join(installdir, 'lib'),
1369                                           os.path.join(installdir,
1370                                                        'usr', 'lib')])
1371         if not for_compiler:
1372             if self.ctx.strip:
1373                 cmdlist.add_command('strip',
1374                                     ['sh', '-c',
1375                                      ('%s %s/lib*/*.so' %
1376                                       (self.tool_name('strip'), installdir))])
1377             cmdlist.add_command('check', ['make', 'check'])
1378             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1379                                 always_run=True)
1380         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1381         cmdlist.cleanup_dir()
1382
1383
1384 class Command(object):
1385     """A command run in the build process."""
1386
1387     def __init__(self, desc, num, dir, path, command, always_run=False):
1388         """Initialize a Command object."""
1389         self.dir = dir
1390         self.path = path
1391         self.desc = desc
1392         trans = str.maketrans({' ': '-'})
1393         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1394         self.command = command
1395         self.always_run = always_run
1396
1397     @staticmethod
1398     def shell_make_quote_string(s):
1399         """Given a string not containing a newline, quote it for use by the
1400         shell and make."""
1401         assert '\n' not in s
1402         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1403             return s
1404         strans = str.maketrans({"'": "'\\''"})
1405         s = "'%s'" % s.translate(strans)
1406         mtrans = str.maketrans({'$': '$$'})
1407         return s.translate(mtrans)
1408
1409     @staticmethod
1410     def shell_make_quote_list(l, translate_make):
1411         """Given a list of strings not containing newlines, quote them for use
1412         by the shell and make, returning a single string.  If translate_make
1413         is true and the first string is 'make', change it to $(MAKE)."""
1414         l = [Command.shell_make_quote_string(s) for s in l]
1415         if translate_make and l[0] == 'make':
1416             l[0] = '$(MAKE)'
1417         return ' '.join(l)
1418
1419     def shell_make_quote(self):
1420         """Return this command quoted for the shell and make."""
1421         return self.shell_make_quote_list(self.command, True)
1422
1423
1424 class CommandList(object):
1425     """A list of commands run in the build process."""
1426
1427     def __init__(self, desc, keep):
1428         """Initialize a CommandList object."""
1429         self.cmdlist = []
1430         self.dir = None
1431         self.path = None
1432         self.desc = [desc]
1433         self.keep = keep
1434
1435     def desc_txt(self, desc):
1436         """Return the description to use for a command."""
1437         return '%s %s' % (' '.join(self.desc), desc)
1438
1439     def use_dir(self, dir):
1440         """Set the default directory for subsequent commands."""
1441         self.dir = dir
1442
1443     def use_path(self, path):
1444         """Set a directory to be prepended to the PATH for subsequent
1445         commands."""
1446         self.path = path
1447
1448     def push_subdesc(self, subdesc):
1449         """Set the default subdescription for subsequent commands (e.g., the
1450         name of a component being built, within the series of commands
1451         building it)."""
1452         self.desc.append(subdesc)
1453
1454     def pop_subdesc(self):
1455         """Pop a subdescription from the list of descriptions."""
1456         self.desc.pop()
1457
1458     def create_use_dir(self, dir):
1459         """Remove and recreate a directory and use it for subsequent
1460         commands."""
1461         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1462         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1463         self.use_dir(dir)
1464
1465     def create_copy_dir(self, src, dest):
1466         """Remove a directory and recreate it as a copy from the given
1467         source."""
1468         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1469         parent = os.path.dirname(dest)
1470         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1471         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1472
1473     def add_command_dir(self, desc, dir, command, always_run=False):
1474         """Add a command to run in a given directory."""
1475         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1476                       command, always_run)
1477         self.cmdlist.append(cmd)
1478
1479     def add_command(self, desc, command, always_run=False):
1480         """Add a command to run in the default directory."""
1481         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1482                       self.path, command, always_run)
1483         self.cmdlist.append(cmd)
1484
1485     def cleanup_dir(self, desc='cleanup', dir=None):
1486         """Clean up a build directory.  If no directory is specified, the
1487         default directory is cleaned up and ceases to be the default
1488         directory."""
1489         if dir is None:
1490             dir = self.dir
1491             self.use_dir(None)
1492         if self.keep != 'all':
1493             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1494                                  always_run=(self.keep == 'none'))
1495
1496     def makefile_commands(self, wrapper, logsdir):
1497         """Return the sequence of commands in the form of text for a Makefile.
1498         The given wrapper script takes arguments: base of logs for
1499         previous command, or empty; base of logs for this command;
1500         description; directory; PATH addition; the command itself."""
1501         # prev_base is the base of the name for logs of the previous
1502         # command that is not always-run (that is, a build command,
1503         # whose failure should stop subsequent build commands from
1504         # being run, as opposed to a cleanup command, which is run
1505         # even if previous commands failed).
1506         prev_base = ''
1507         cmds = []
1508         for c in self.cmdlist:
1509             ctxt = c.shell_make_quote()
1510             if prev_base and not c.always_run:
1511                 prev_log = os.path.join(logsdir, prev_base)
1512             else:
1513                 prev_log = ''
1514             this_log = os.path.join(logsdir, c.logbase)
1515             if not c.always_run:
1516                 prev_base = c.logbase
1517             if c.dir is None:
1518                 dir = ''
1519             else:
1520                 dir = c.dir
1521             if c.path is None:
1522                 path = ''
1523             else:
1524                 path = c.path
1525             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1526             prelim_txt = Command.shell_make_quote_list(prelims, False)
1527             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1528         return '\n'.join(cmds)
1529
1530     def status_logs(self, logsdir):
1531         """Return the list of log files with command status."""
1532         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1533                 for c in self.cmdlist]
1534
1535
1536 def get_parser():
1537     """Return an argument parser for this module."""
1538     parser = argparse.ArgumentParser(description=__doc__)
1539     parser.add_argument('-j', dest='parallelism',
1540                         help='Run this number of jobs in parallel',
1541                         type=int, default=os.cpu_count())
1542     parser.add_argument('--keep', dest='keep',
1543                         help='Whether to keep all build directories, '
1544                         'none or only those from failed builds',
1545                         default='none', choices=('none', 'all', 'failed'))
1546     parser.add_argument('--replace-sources', action='store_true',
1547                         help='Remove and replace source directories '
1548                         'with the wrong version of a component')
1549     parser.add_argument('--strip', action='store_true',
1550                         help='Strip installed glibc libraries')
1551     parser.add_argument('topdir',
1552                         help='Toplevel working directory')
1553     parser.add_argument('action',
1554                         help='What to do',
1555                         choices=('checkout', 'bot-cycle', 'bot',
1556                                  'host-libraries', 'compilers', 'glibcs'))
1557     parser.add_argument('configs',
1558                         help='Versions to check out or configurations to build',
1559                         nargs='*')
1560     return parser
1561
1562
1563 def main(argv):
1564     """The main entry point."""
1565     parser = get_parser()
1566     opts = parser.parse_args(argv)
1567     topdir = os.path.abspath(opts.topdir)
1568     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1569                   opts.strip, opts.action)
1570     ctx.run_builds(opts.action, opts.configs)
1571
1572
1573 if __name__ == '__main__':
1574     main(sys.argv[1:])