scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'host-libraries', to build libraries required by the toolchain,
  27 'compilers', to build cross-compilers for various configurations, or
  28 'glibcs', to build glibc for various configurations and run the
  29 compilation parts of the testsuite.  Subsequent arguments name the
  30 versions of components to check out (<component>-<version), for
  31 'checkout', or, for actions other than 'checkout' and 'bot-cycle',
  32 name configurations for which compilers or glibc are to be built.
  33
  34 """
  35
  36 import argparse
  37 import datetime
  38 import email.mime.text
  39 import email.utils
  40 import json
  41 import os
  42 import re
  43 import shutil
  44 import smtplib
  45 import stat
  46 import subprocess
  47 import sys
  48 import urllib.request
  49
  50
  51 class Context(object):
  52     """The global state associated with builds in a given directory."""
  53
  54     def __init__(self, topdir, parallelism, keep, replace_sources, action):
  55         """Initialize the context."""
  56         self.topdir = topdir
  57         self.parallelism = parallelism
  58         self.keep = keep
  59         self.replace_sources = replace_sources
  60         self.srcdir = os.path.join(topdir, 'src')
  61         self.versions_json = os.path.join(self.srcdir, 'versions.json')
  62         self.build_state_json = os.path.join(topdir, 'build-state.json')
  63         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
  64         self.installdir = os.path.join(topdir, 'install')
  65         self.host_libraries_installdir = os.path.join(self.installdir,
  66                                                       'host-libraries')
  67         self.builddir = os.path.join(topdir, 'build')
  68         self.logsdir = os.path.join(topdir, 'logs')
  69         self.makefile = os.path.join(self.builddir, 'Makefile')
  70         self.wrapper = os.path.join(self.builddir, 'wrapper')
  71         self.save_logs = os.path.join(self.builddir, 'save-logs')
  72         self.script_text = self.get_script_text()
  73         if action != 'checkout':
  74             self.build_triplet = self.get_build_triplet()
  75             self.glibc_version = self.get_glibc_version()
  76         self.configs = {}
  77         self.glibc_configs = {}
  78         self.makefile_pieces = ['.PHONY: all\n']
  79         self.add_all_configs()
  80         self.load_versions_json()
  81         self.load_build_state_json()
  82         self.status_log_list = []
  83
  84     def get_script_text(self):
  85         """Return the text of this script."""
  86         with open(sys.argv[0], 'r') as f:
  87             return f.read()
  88
  89     def exec_self(self):
  90         """Re-execute this script with the same arguments."""
  91         os.execv(sys.executable, [sys.executable] + sys.argv)
  92
  93     def get_build_triplet(self):
  94         """Determine the build triplet with config.guess."""
  95         config_guess = os.path.join(self.component_srcdir('gcc'),
  96                                     'config.guess')
  97         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
  98                                 check=True, universal_newlines=True).stdout
  99         return cg_out.rstrip()
 100
 101     def get_glibc_version(self):
 102         """Determine the glibc version number (major.minor)."""
 103         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 104         with open(version_h, 'r') as f:
 105             lines = f.readlines()
 106         starttext = '#define VERSION "'
 107         for l in lines:
 108             if l.startswith(starttext):
 109                 l = l[len(starttext):]
 110                 l = l.rstrip('"\n')
 111                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 112                 return '%s.%s' % m.group(1, 2)
 113         print('error: could not determine glibc version')
 114         exit(1)
 115
 116     def add_all_configs(self):
 117         """Add all known glibc build configurations."""
 118         # On architectures missing __builtin_trap support, these
 119         # options may be needed as a workaround; see
 120         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70216> for SH.
 121         no_isolate = ('-fno-isolate-erroneous-paths-dereference'
 122                       ' -fno-isolate-erroneous-paths-attribute')
 123         self.add_config(arch='aarch64',
 124                         os_name='linux-gnu')
 125         self.add_config(arch='aarch64_be',
 126                         os_name='linux-gnu')
 127         self.add_config(arch='alpha',
 128                         os_name='linux-gnu')
 129         self.add_config(arch='arm',
 130                         os_name='linux-gnueabi')
 131         self.add_config(arch='armeb',
 132                         os_name='linux-gnueabi')
 133         self.add_config(arch='armeb',
 134                         os_name='linux-gnueabi',
 135                         variant='be8',
 136                         gcc_cfg=['--with-arch=armv7-a'])
 137         self.add_config(arch='arm',
 138                         os_name='linux-gnueabihf')
 139         self.add_config(arch='armeb',
 140                         os_name='linux-gnueabihf')
 141         self.add_config(arch='armeb',
 142                         os_name='linux-gnueabihf',
 143                         variant='be8',
 144                         gcc_cfg=['--with-arch=armv7-a'])
 145         self.add_config(arch='hppa',
 146                         os_name='linux-gnu')
 147         self.add_config(arch='ia64',
 148                         os_name='linux-gnu',
 149                         first_gcc_cfg=['--with-system-libunwind'])
 150         self.add_config(arch='m68k',
 151                         os_name='linux-gnu',
 152                         gcc_cfg=['--disable-multilib'])
 153         self.add_config(arch='m68k',
 154                         os_name='linux-gnu',
 155                         variant='coldfire',
 156                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 157         self.add_config(arch='microblaze',
 158                         os_name='linux-gnu',
 159                         gcc_cfg=['--disable-multilib'])
 160         self.add_config(arch='microblazeel',
 161                         os_name='linux-gnu',
 162                         gcc_cfg=['--disable-multilib'])
 163         self.add_config(arch='mips64',
 164                         os_name='linux-gnu',
 165                         gcc_cfg=['--with-mips-plt'],
 166                         glibcs=[{'variant': 'n32'},
 167                                 {'arch': 'mips',
 168                                  'ccopts': '-mabi=32'},
 169                                 {'variant': 'n64',
 170                                  'ccopts': '-mabi=64'}])
 171         self.add_config(arch='mips64',
 172                         os_name='linux-gnu',
 173                         variant='soft',
 174                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 175                         glibcs=[{'variant': 'n32-soft',
 176                                  'cfg': ['--without-fp']},
 177                                 {'variant': 'soft',
 178                                  'arch': 'mips',
 179                                  'ccopts': '-mabi=32',
 180                                  'cfg': ['--without-fp']},
 181                                 {'variant': 'n64-soft',
 182                                  'ccopts': '-mabi=64',
 183                                  'cfg': ['--without-fp']}])
 184         self.add_config(arch='mips64',
 185                         os_name='linux-gnu',
 186                         variant='nan2008',
 187                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 188                                  '--with-arch-64=mips64r2',
 189                                  '--with-arch-32=mips32r2'],
 190                         glibcs=[{'variant': 'n32-nan2008'},
 191                                 {'variant': 'nan2008',
 192                                  'arch': 'mips',
 193                                  'ccopts': '-mabi=32'},
 194                                 {'variant': 'n64-nan2008',
 195                                  'ccopts': '-mabi=64'}])
 196         self.add_config(arch='mips64',
 197                         os_name='linux-gnu',
 198                         variant='nan2008-soft',
 199                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 200                                  '--with-arch-64=mips64r2',
 201                                  '--with-arch-32=mips32r2',
 202                                  '--with-float=soft'],
 203                         glibcs=[{'variant': 'n32-nan2008-soft',
 204                                  'cfg': ['--without-fp']},
 205                                 {'variant': 'nan2008-soft',
 206                                  'arch': 'mips',
 207                                  'ccopts': '-mabi=32',
 208                                  'cfg': ['--without-fp']},
 209                                 {'variant': 'n64-nan2008-soft',
 210                                  'ccopts': '-mabi=64',
 211                                  'cfg': ['--without-fp']}])
 212         self.add_config(arch='mips64el',
 213                         os_name='linux-gnu',
 214                         gcc_cfg=['--with-mips-plt'],
 215                         glibcs=[{'variant': 'n32'},
 216                                 {'arch': 'mipsel',
 217                                  'ccopts': '-mabi=32'},
 218                                 {'variant': 'n64',
 219                                  'ccopts': '-mabi=64'}])
 220         self.add_config(arch='mips64el',
 221                         os_name='linux-gnu',
 222                         variant='soft',
 223                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 224                         glibcs=[{'variant': 'n32-soft',
 225                                  'cfg': ['--without-fp']},
 226                                 {'variant': 'soft',
 227                                  'arch': 'mipsel',
 228                                  'ccopts': '-mabi=32',
 229                                  'cfg': ['--without-fp']},
 230                                 {'variant': 'n64-soft',
 231                                  'ccopts': '-mabi=64',
 232                                  'cfg': ['--without-fp']}])
 233         self.add_config(arch='mips64el',
 234                         os_name='linux-gnu',
 235                         variant='nan2008',
 236                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 237                                  '--with-arch-64=mips64r2',
 238                                  '--with-arch-32=mips32r2'],
 239                         glibcs=[{'variant': 'n32-nan2008'},
 240                                 {'variant': 'nan2008',
 241                                  'arch': 'mipsel',
 242                                  'ccopts': '-mabi=32'},
 243                                 {'variant': 'n64-nan2008',
 244                                  'ccopts': '-mabi=64'}])
 245         self.add_config(arch='mips64el',
 246                         os_name='linux-gnu',
 247                         variant='nan2008-soft',
 248                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 249                                  '--with-arch-64=mips64r2',
 250                                  '--with-arch-32=mips32r2',
 251                                  '--with-float=soft'],
 252                         glibcs=[{'variant': 'n32-nan2008-soft',
 253                                  'cfg': ['--without-fp']},
 254                                 {'variant': 'nan2008-soft',
 255                                  'arch': 'mipsel',
 256                                  'ccopts': '-mabi=32',
 257                                  'cfg': ['--without-fp']},
 258                                 {'variant': 'n64-nan2008-soft',
 259                                  'ccopts': '-mabi=64',
 260                                  'cfg': ['--without-fp']}])
 261         self.add_config(arch='nios2',
 262                         os_name='linux-gnu')
 263         self.add_config(arch='powerpc',
 264                         os_name='linux-gnu',
 265                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 266         self.add_config(arch='powerpc',
 267                         os_name='linux-gnu',
 268                         variant='soft',
 269                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 270                                  '--enable-secureplt'],
 271                         glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
 272         self.add_config(arch='powerpc64',
 273                         os_name='linux-gnu',
 274                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 275         self.add_config(arch='powerpc64le',
 276                         os_name='linux-gnu',
 277                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 278         self.add_config(arch='powerpc',
 279                         os_name='linux-gnuspe',
 280                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 281                                  '--enable-e500-double'],
 282                         glibcs=[{'cfg': ['--without-fp']}])
 283         self.add_config(arch='powerpc',
 284                         os_name='linux-gnuspe',
 285                         variant='e500v1',
 286                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 287                         glibcs=[{'variant': 'e500v1', 'cfg': ['--without-fp']}])
 288         self.add_config(arch='s390x',
 289                         os_name='linux-gnu',
 290                         glibcs=[{},
 291                                 {'arch': 's390', 'ccopts': '-m31'}])
 292         self.add_config(arch='sh3',
 293                         os_name='linux-gnu',
 294                         glibcs=[{'ccopts': no_isolate}])
 295         self.add_config(arch='sh3eb',
 296                         os_name='linux-gnu',
 297                         glibcs=[{'ccopts': no_isolate}])
 298         self.add_config(arch='sh4',
 299                         os_name='linux-gnu',
 300                         glibcs=[{'ccopts': no_isolate}])
 301         self.add_config(arch='sh4eb',
 302                         os_name='linux-gnu',
 303                         glibcs=[{'ccopts': no_isolate}])
 304         self.add_config(arch='sh4',
 305                         os_name='linux-gnu',
 306                         variant='soft',
 307                         gcc_cfg=['--without-fp'],
 308                         glibcs=[{'variant': 'soft',
 309                                  'cfg': ['--without-fp'],
 310                                  'ccopts': no_isolate}])
 311         self.add_config(arch='sh4eb',
 312                         os_name='linux-gnu',
 313                         variant='soft',
 314                         gcc_cfg=['--without-fp'],
 315                         glibcs=[{'variant': 'soft',
 316                                  'cfg': ['--without-fp'],
 317                                  'ccopts': no_isolate}])
 318         self.add_config(arch='sparc64',
 319                         os_name='linux-gnu',
 320                         glibcs=[{},
 321                                 {'arch': 'sparcv9',
 322                                  'ccopts': '-m32 -mlong-double-128'}])
 323         self.add_config(arch='tilegx',
 324                         os_name='linux-gnu',
 325                         glibcs=[{},
 326                                 {'variant': '32', 'ccopts': '-m32'}])
 327         self.add_config(arch='tilegxbe',
 328                         os_name='linux-gnu',
 329                         glibcs=[{},
 330                                 {'variant': '32', 'ccopts': '-m32'}])
 331         self.add_config(arch='tilepro',
 332                         os_name='linux-gnu')
 333         self.add_config(arch='x86_64',
 334                         os_name='linux-gnu',
 335                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 336                         glibcs=[{},
 337                                 {'variant': 'x32', 'ccopts': '-mx32'},
 338                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 339                         extra_glibcs=[{'variant': 'disable-multi-arch',
 340                                        'cfg': ['--disable-multi-arch']},
 341                                       {'variant': 'disable-multi-arch',
 342                                        'arch': 'i686',
 343                                        'ccopts': '-m32 -march=i686',
 344                                        'cfg': ['--disable-multi-arch']},
 345                                       {'arch': 'i486',
 346                                        'ccopts': '-m32 -march=i486'},
 347                                       {'arch': 'i586',
 348                                        'ccopts': '-m32 -march=i586'}])
 349
 350     def add_config(self, **args):
 351         """Add an individual build configuration."""
 352         cfg = Config(self, **args)
 353         if cfg.name in self.configs:
 354             print('error: duplicate config %s' % cfg.name)
 355             exit(1)
 356         self.configs[cfg.name] = cfg
 357         for c in cfg.all_glibcs:
 358             if c.name in self.glibc_configs:
 359                 print('error: duplicate glibc config %s' % c.name)
 360                 exit(1)
 361             self.glibc_configs[c.name] = c
 362
 363     def component_srcdir(self, component):
 364         """Return the source directory for a given component, e.g. gcc."""
 365         return os.path.join(self.srcdir, component)
 366
 367     def component_builddir(self, action, config, component, subconfig=None):
 368         """Return the directory to use for a build."""
 369         if config is None:
 370             # Host libraries.
 371             assert subconfig is None
 372             return os.path.join(self.builddir, action, component)
 373         if subconfig is None:
 374             return os.path.join(self.builddir, action, config, component)
 375         else:
 376             # glibc build as part of compiler build.
 377             return os.path.join(self.builddir, action, config, component,
 378                                 subconfig)
 379
 380     def compiler_installdir(self, config):
 381         """Return the directory in which to install a compiler."""
 382         return os.path.join(self.installdir, 'compilers', config)
 383
 384     def compiler_bindir(self, config):
 385         """Return the directory in which to find compiler binaries."""
 386         return os.path.join(self.compiler_installdir(config), 'bin')
 387
 388     def compiler_sysroot(self, config):
 389         """Return the sysroot directory for a compiler."""
 390         return os.path.join(self.compiler_installdir(config), 'sysroot')
 391
 392     def glibc_installdir(self, config):
 393         """Return the directory in which to install glibc."""
 394         return os.path.join(self.installdir, 'glibcs', config)
 395
 396     def run_builds(self, action, configs):
 397         """Run the requested builds."""
 398         if action == 'checkout':
 399             self.checkout(configs)
 400             return
 401         if action == 'bot-cycle':
 402             if configs:
 403                 print('error: configurations specified for bot-cycle')
 404                 exit(1)
 405             self.bot_cycle()
 406             return
 407         if action == 'host-libraries' and configs:
 408             print('error: configurations specified for host-libraries')
 409             exit(1)
 410         self.clear_last_build_state(action)
 411         build_time = datetime.datetime.utcnow()
 412         if action == 'host-libraries':
 413             build_components = ('gmp', 'mpfr', 'mpc')
 414             old_components = ()
 415             old_versions = {}
 416             self.build_host_libraries()
 417         elif action == 'compilers':
 418             build_components = ('binutils', 'gcc', 'glibc', 'linux')
 419             old_components = ('gmp', 'mpfr', 'mpc')
 420             old_versions = self.build_state['host-libraries']['build-versions']
 421             self.build_compilers(configs)
 422         else:
 423             build_components = ('glibc',)
 424             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
 425             old_versions = self.build_state['compilers']['build-versions']
 426             self.build_glibcs(configs)
 427         self.write_files()
 428         self.do_build()
 429         if configs:
 430             # Partial build, do not update stored state.
 431             return
 432         build_versions = {}
 433         for k in build_components:
 434             if k in self.versions:
 435                 build_versions[k] = {'version': self.versions[k]['version'],
 436                                      'revision': self.versions[k]['revision']}
 437         for k in old_components:
 438             if k in old_versions:
 439                 build_versions[k] = {'version': old_versions[k]['version'],
 440                                      'revision': old_versions[k]['revision']}
 441         self.update_build_state(action, build_time, build_versions)
 442
 443     @staticmethod
 444     def remove_dirs(*args):
 445         """Remove directories and their contents if they exist."""
 446         for dir in args:
 447             shutil.rmtree(dir, ignore_errors=True)
 448
 449     @staticmethod
 450     def remove_recreate_dirs(*args):
 451         """Remove directories if they exist, and create them as empty."""
 452         Context.remove_dirs(*args)
 453         for dir in args:
 454             os.makedirs(dir, exist_ok=True)
 455
 456     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 457         """Add makefile text for a list of commands."""
 458         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 459         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 460                                     (target, target, target, commands))
 461         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 462
 463     def write_files(self):
 464         """Write out the Makefile and wrapper script."""
 465         mftext = ''.join(self.makefile_pieces)
 466         with open(self.makefile, 'w') as f:
 467             f.write(mftext)
 468         wrapper_text = (
 469             '#!/bin/sh\n'
 470             'prev_base=$1\n'
 471             'this_base=$2\n'
 472             'desc=$3\n'
 473             'dir=$4\n'
 474             'path=$5\n'
 475             'shift 5\n'
 476             'prev_status=$prev_base-status.txt\n'
 477             'this_status=$this_base-status.txt\n'
 478             'this_log=$this_base-log.txt\n'
 479             'date > "$this_log"\n'
 480             'echo >> "$this_log"\n'
 481             'echo "Description: $desc" >> "$this_log"\n'
 482             'printf "%s" "Command:" >> "$this_log"\n'
 483             'for word in "$@"; do\n'
 484             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 485             '    printf " %s" "$word"\n'
 486             '  else\n'
 487             '    printf " \'"\n'
 488             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 489             '    printf "\'"\n'
 490             '  fi\n'
 491             'done >> "$this_log"\n'
 492             'echo >> "$this_log"\n'
 493             'echo "Directory: $dir" >> "$this_log"\n'
 494             'echo "Path addition: $path" >> "$this_log"\n'
 495             'echo >> "$this_log"\n'
 496             'record_status ()\n'
 497             '{\n'
 498             '  echo >> "$this_log"\n'
 499             '  echo "$1: $desc" > "$this_status"\n'
 500             '  echo "$1: $desc" >> "$this_log"\n'
 501             '  echo >> "$this_log"\n'
 502             '  date >> "$this_log"\n'
 503             '  echo "$1: $desc"\n'
 504             '  exit 0\n'
 505             '}\n'
 506             'check_error ()\n'
 507             '{\n'
 508             '  if [ "$1" != "0" ]; then\n'
 509             '    record_status FAIL\n'
 510             '  fi\n'
 511             '}\n'
 512             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 513             '    record_status UNRESOLVED\n'
 514             'fi\n'
 515             'if [ "$dir" ]; then\n'
 516             '  cd "$dir"\n'
 517             '  check_error "$?"\n'
 518             'fi\n'
 519             'if [ "$path" ]; then\n'
 520             '  PATH=$path:$PATH\n'
 521             'fi\n'
 522             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 523             'check_error "$?"\n'
 524             'record_status PASS\n')
 525         with open(self.wrapper, 'w') as f:
 526             f.write(wrapper_text)
 527         # Mode 0o755.
 528         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 529                      stat.S_IROTH|stat.S_IXOTH)
 530         os.chmod(self.wrapper, mode_exec)
 531         save_logs_text = (
 532             '#!/bin/sh\n'
 533             'if ! [ -f tests.sum ]; then\n'
 534             '  echo "No test summary available."\n'
 535             '  exit 0\n'
 536             'fi\n'
 537             'save_file ()\n'
 538             '{\n'
 539             '  echo "Contents of $1:"\n'
 540             '  echo\n'
 541             '  cat "$1"\n'
 542             '  echo\n'
 543             '  echo "End of contents of $1."\n'
 544             '  echo\n'
 545             '}\n'
 546             'save_file tests.sum\n'
 547             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 548             'for t in $non_pass_tests; do\n'
 549             '  if [ -f "$t.out" ]; then\n'
 550             '    save_file "$t.out"\n'
 551             '  fi\n'
 552             'done\n')
 553         with open(self.save_logs, 'w') as f:
 554             f.write(save_logs_text)
 555         os.chmod(self.save_logs, mode_exec)
 556
 557     def do_build(self):
 558         """Do the actual build."""
 559         cmd = ['make', '-j%d' % self.parallelism]
 560         subprocess.run(cmd, cwd=self.builddir, check=True)
 561
 562     def build_host_libraries(self):
 563         """Build the host libraries."""
 564         installdir = self.host_libraries_installdir
 565         builddir = os.path.join(self.builddir, 'host-libraries')
 566         logsdir = os.path.join(self.logsdir, 'host-libraries')
 567         self.remove_recreate_dirs(installdir, builddir, logsdir)
 568         cmdlist = CommandList('host-libraries', self.keep)
 569         self.build_host_library(cmdlist, 'gmp')
 570         self.build_host_library(cmdlist, 'mpfr',
 571                                 ['--with-gmp=%s' % installdir])
 572         self.build_host_library(cmdlist, 'mpc',
 573                                 ['--with-gmp=%s' % installdir,
 574                                 '--with-mpfr=%s' % installdir])
 575         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 576         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 577
 578     def build_host_library(self, cmdlist, lib, extra_opts=None):
 579         """Build one host library."""
 580         srcdir = self.component_srcdir(lib)
 581         builddir = self.component_builddir('host-libraries', None, lib)
 582         installdir = self.host_libraries_installdir
 583         cmdlist.push_subdesc(lib)
 584         cmdlist.create_use_dir(builddir)
 585         cfg_cmd = [os.path.join(srcdir, 'configure'),
 586                    '--prefix=%s' % installdir,
 587                    '--disable-shared']
 588         if extra_opts:
 589             cfg_cmd.extend (extra_opts)
 590         cmdlist.add_command('configure', cfg_cmd)
 591         cmdlist.add_command('build', ['make'])
 592         cmdlist.add_command('check', ['make', 'check'])
 593         cmdlist.add_command('install', ['make', 'install'])
 594         cmdlist.cleanup_dir()
 595         cmdlist.pop_subdesc()
 596
 597     def build_compilers(self, configs):
 598         """Build the compilers."""
 599         if not configs:
 600             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 601             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 602             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 603             configs = sorted(self.configs.keys())
 604         for c in configs:
 605             self.configs[c].build()
 606
 607     def build_glibcs(self, configs):
 608         """Build the glibcs."""
 609         if not configs:
 610             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 611             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 612             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 613             configs = sorted(self.glibc_configs.keys())
 614         for c in configs:
 615             self.glibc_configs[c].build()
 616
 617     def load_versions_json(self):
 618         """Load information about source directory versions."""
 619         if not os.access(self.versions_json, os.F_OK):
 620             self.versions = {}
 621             return
 622         with open(self.versions_json, 'r') as f:
 623             self.versions = json.load(f)
 624
 625     def store_json(self, data, filename):
 626         """Store information in a JSON file."""
 627         filename_tmp = filename + '.tmp'
 628         with open(filename_tmp, 'w') as f:
 629             json.dump(data, f, indent=2, sort_keys=True)
 630         os.rename(filename_tmp, filename)
 631
 632     def store_versions_json(self):
 633         """Store information about source directory versions."""
 634         self.store_json(self.versions, self.versions_json)
 635
 636     def set_component_version(self, component, version, explicit, revision):
 637         """Set the version information for a component."""
 638         self.versions[component] = {'version': version,
 639                                     'explicit': explicit,
 640                                     'revision': revision}
 641         self.store_versions_json()
 642
 643     def checkout(self, versions):
 644         """Check out the desired component versions."""
 645         default_versions = {'binutils': 'vcs-2.27',
 646                             'gcc': 'vcs-6',
 647                             'glibc': 'vcs-mainline',
 648                             'gmp': '6.1.1',
 649                             'linux': '4.8.6',
 650                             'mpc': '1.0.3',
 651                             'mpfr': '3.1.5'}
 652         use_versions = {}
 653         explicit_versions = {}
 654         for v in versions:
 655             found_v = False
 656             for k in default_versions.keys():
 657                 kx = k + '-'
 658                 if v.startswith(kx):
 659                     vx = v[len(kx):]
 660                     if k in use_versions:
 661                         print('error: multiple versions for %s' % k)
 662                         exit(1)
 663                     use_versions[k] = vx
 664                     explicit_versions[k] = True
 665                     found_v = True
 666                     break
 667             if not found_v:
 668                 print('error: unknown component in %s' % v)
 669                 exit(1)
 670         for k in default_versions.keys():
 671             if k not in use_versions:
 672                 if k in self.versions and self.versions[k]['explicit']:
 673                     use_versions[k] = self.versions[k]['version']
 674                     explicit_versions[k] = True
 675                 else:
 676                     use_versions[k] = default_versions[k]
 677                     explicit_versions[k] = False
 678         os.makedirs(self.srcdir, exist_ok=True)
 679         for k in sorted(default_versions.keys()):
 680             update = os.access(self.component_srcdir(k), os.F_OK)
 681             v = use_versions[k]
 682             if (update and
 683                 k in self.versions and
 684                 v != self.versions[k]['version']):
 685                 if not self.replace_sources:
 686                     print('error: version of %s has changed from %s to %s, '
 687                           'use --replace-sources to check out again' %
 688                           (k, self.versions[k]['version'], v))
 689                     exit(1)
 690                 shutil.rmtree(self.component_srcdir(k))
 691                 update = False
 692             if v.startswith('vcs-'):
 693                 revision = self.checkout_vcs(k, v[4:], update)
 694             else:
 695                 self.checkout_tar(k, v, update)
 696                 revision = v
 697             self.set_component_version(k, v, explicit_versions[k], revision)
 698         if self.get_script_text() != self.script_text:
 699             # Rerun the checkout process in case the updated script
 700             # uses different default versions or new components.
 701             self.exec_self()
 702
 703     def checkout_vcs(self, component, version, update):
 704         """Check out the given version of the given component from version
 705         control.  Return a revision identifier."""
 706         if component == 'binutils':
 707             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 708             if version == 'mainline':
 709                 git_branch = 'master'
 710             else:
 711                 trans = str.maketrans({'.': '_'})
 712                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 713             return self.git_checkout(component, git_url, git_branch, update)
 714         elif component == 'gcc':
 715             if version == 'mainline':
 716                 branch = 'trunk'
 717             else:
 718                 trans = str.maketrans({'.': '_'})
 719                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 720             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 721             return self.gcc_checkout(svn_url, update)
 722         elif component == 'glibc':
 723             git_url = 'git://sourceware.org/git/glibc.git'
 724             if version == 'mainline':
 725                 git_branch = 'master'
 726             else:
 727                 git_branch = 'release/%s/master' % version
 728             r = self.git_checkout(component, git_url, git_branch, update)
 729             self.fix_glibc_timestamps()
 730             return r
 731         else:
 732             print('error: component %s coming from VCS' % component)
 733             exit(1)
 734
 735     def git_checkout(self, component, git_url, git_branch, update):
 736         """Check out a component from git.  Return a commit identifier."""
 737         if update:
 738             subprocess.run(['git', 'remote', 'prune', 'origin'],
 739                            cwd=self.component_srcdir(component), check=True)
 740             subprocess.run(['git', 'pull', '-q'],
 741                            cwd=self.component_srcdir(component), check=True)
 742         else:
 743             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 744                             self.component_srcdir(component)], check=True)
 745         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 746                            cwd=self.component_srcdir(component),
 747                            stdout=subprocess.PIPE,
 748                            check=True, universal_newlines=True).stdout
 749         return r.rstrip()
 750
 751     def fix_glibc_timestamps(self):
 752         """Fix timestamps in a glibc checkout."""
 753         # Ensure that builds do not try to regenerate generated files
 754         # in the source tree.
 755         srcdir = self.component_srcdir('glibc')
 756         for dirpath, dirnames, filenames in os.walk(srcdir):
 757             for f in filenames:
 758                 if (f == 'configure' or
 759                     f == 'preconfigure' or
 760                     f.endswith('-kw.h')):
 761                     to_touch = os.path.join(dirpath, f)
 762                     subprocess.run(['touch', to_touch], check=True)
 763
 764     def gcc_checkout(self, svn_url, update):
 765         """Check out GCC from SVN.  Return the revision number."""
 766         if not update:
 767             subprocess.run(['svn', 'co', '-q', svn_url,
 768                             self.component_srcdir('gcc')], check=True)
 769         subprocess.run(['contrib/gcc_update', '--silent'],
 770                        cwd=self.component_srcdir('gcc'), check=True)
 771         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 772                            stdout=subprocess.PIPE,
 773                            check=True, universal_newlines=True).stdout
 774         return r.rstrip()
 775
 776     def checkout_tar(self, component, version, update):
 777         """Check out the given version of the given component from a
 778         tarball."""
 779         if update:
 780             return
 781         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 782                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 783                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 784                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 785                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 786                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
 787         if component not in url_map:
 788             print('error: component %s coming from tarball' % component)
 789             exit(1)
 790         url = url_map[component] % {'version': version}
 791         filename = os.path.join(self.srcdir, url.split('/')[-1])
 792         response = urllib.request.urlopen(url)
 793         data = response.read()
 794         with open(filename, 'wb') as f:
 795             f.write(data)
 796         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 797                        check=True)
 798         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 799                   self.component_srcdir(component))
 800         os.remove(filename)
 801
 802     def load_build_state_json(self):
 803         """Load information about the state of previous builds."""
 804         if os.access(self.build_state_json, os.F_OK):
 805             with open(self.build_state_json, 'r') as f:
 806                 self.build_state = json.load(f)
 807         else:
 808             self.build_state = {}
 809         for k in ('host-libraries', 'compilers', 'glibcs'):
 810             if k not in self.build_state:
 811                 self.build_state[k] = {}
 812             if 'build-time' not in self.build_state[k]:
 813                 self.build_state[k]['build-time'] = ''
 814             if 'build-versions' not in self.build_state[k]:
 815                 self.build_state[k]['build-versions'] = {}
 816             if 'build-results' not in self.build_state[k]:
 817                 self.build_state[k]['build-results'] = {}
 818             if 'result-changes' not in self.build_state[k]:
 819                 self.build_state[k]['result-changes'] = {}
 820             if 'ever-passed' not in self.build_state[k]:
 821                 self.build_state[k]['ever-passed'] = []
 822
 823     def store_build_state_json(self):
 824         """Store information about the state of previous builds."""
 825         self.store_json(self.build_state, self.build_state_json)
 826
 827     def clear_last_build_state(self, action):
 828         """Clear information about the state of part of the build."""
 829         # We clear the last build time and versions when starting a
 830         # new build.  The results of the last build are kept around,
 831         # as comparison is still meaningful if this build is aborted
 832         # and a new one started.
 833         self.build_state[action]['build-time'] = ''
 834         self.build_state[action]['build-versions'] = {}
 835         self.store_build_state_json()
 836
 837     def update_build_state(self, action, build_time, build_versions):
 838         """Update the build state after a build."""
 839         build_time = build_time.replace(microsecond=0)
 840         self.build_state[action]['build-time'] = str(build_time)
 841         self.build_state[action]['build-versions'] = build_versions
 842         build_results = {}
 843         for log in self.status_log_list:
 844             with open(log, 'r') as f:
 845                 log_text = f.read()
 846             log_text = log_text.rstrip()
 847             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 848             result = m.group(1)
 849             test_name = m.group(2)
 850             assert test_name not in build_results
 851             build_results[test_name] = result
 852         old_build_results = self.build_state[action]['build-results']
 853         self.build_state[action]['build-results'] = build_results
 854         result_changes = {}
 855         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 856         for t in all_tests:
 857             if t in old_build_results:
 858                 old_res = old_build_results[t]
 859             else:
 860                 old_res = '(New test)'
 861             if t in build_results:
 862                 new_res = build_results[t]
 863             else:
 864                 new_res = '(Test removed)'
 865             if old_res != new_res:
 866                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 867         self.build_state[action]['result-changes'] = result_changes
 868         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 869                            if t in build_results}
 870         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 871         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 872                                                          new_passes)
 873         self.store_build_state_json()
 874
 875     def load_bot_config_json(self):
 876         """Load bot configuration."""
 877         with open(self.bot_config_json, 'r') as f:
 878             self.bot_config = json.load(f)
 879
 880     def part_build_old(self, action, delay):
 881         """Return whether the last build for a given action was at least a
 882         given number of seconds ago, or does not have a time recorded."""
 883         old_time_str = self.build_state[action]['build-time']
 884         if not old_time_str:
 885             return True
 886         old_time = datetime.datetime.strptime(old_time_str,
 887                                               '%Y-%m-%d %H:%M:%S')
 888         new_time = datetime.datetime.utcnow()
 889         delta = new_time - old_time
 890         return delta.total_seconds() >= delay
 891
 892     def bot_cycle(self):
 893         """Run a single round of checkout and builds."""
 894         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 895         self.load_bot_config_json()
 896         actions = ('host-libraries', 'compilers', 'glibcs')
 897         self.bot_run_self(['--replace-sources'], 'checkout')
 898         self.load_versions_json()
 899         if self.get_script_text() != self.script_text:
 900             print('Script changed, re-execing.')
 901             # On script change, all parts of the build should be rerun.
 902             for a in actions:
 903                 self.clear_last_build_state(a)
 904             self.exec_self()
 905         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 906                             'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
 907                             'glibcs': ('glibc',)}
 908         must_build = {}
 909         for a in actions:
 910             build_vers = self.build_state[a]['build-versions']
 911             must_build[a] = False
 912             if not self.build_state[a]['build-time']:
 913                 must_build[a] = True
 914             old_vers = {}
 915             new_vers = {}
 916             for c in check_components[a]:
 917                 if c in build_vers:
 918                     old_vers[c] = build_vers[c]
 919                 new_vers[c] = {'version': self.versions[c]['version'],
 920                                'revision': self.versions[c]['revision']}
 921             if new_vers == old_vers:
 922                 print('Versions for %s unchanged.' % a)
 923             else:
 924                 print('Versions changed or rebuild forced for %s.' % a)
 925                 if a == 'compilers' and not self.part_build_old(
 926                         a, self.bot_config['compilers-rebuild-delay']):
 927                     print('Not requiring rebuild of compilers this soon.')
 928                 else:
 929                     must_build[a] = True
 930         if must_build['host-libraries']:
 931             must_build['compilers'] = True
 932         if must_build['compilers']:
 933             must_build['glibcs'] = True
 934         for a in actions:
 935             if must_build[a]:
 936                 print('Must rebuild %s.' % a)
 937                 self.clear_last_build_state(a)
 938             else:
 939                 print('No need to rebuild %s.' % a)
 940         for a in actions:
 941             if must_build[a]:
 942                 build_time = datetime.datetime.utcnow()
 943                 print('Rebuilding %s at %s.' % (a, str(build_time)))
 944                 self.bot_run_self([], a)
 945                 self.load_build_state_json()
 946                 self.bot_build_mail(a, build_time)
 947         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
 948
 949     def bot_build_mail(self, action, build_time):
 950         """Send email with the results of a build."""
 951         build_time = build_time.replace(microsecond=0)
 952         subject = (self.bot_config['email-subject'] %
 953                    {'action': action,
 954                     'build-time': str(build_time)})
 955         results = self.build_state[action]['build-results']
 956         changes = self.build_state[action]['result-changes']
 957         ever_passed = set(self.build_state[action]['ever-passed'])
 958         versions = self.build_state[action]['build-versions']
 959         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
 960         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
 961         all_fails = {k for k in results if results[k] == 'FAIL'}
 962         if new_regressions:
 963             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
 964             new_reg_text = ('New regressions:\n\n%s\n\n' %
 965                             '\n'.join(new_reg_list))
 966         else:
 967             new_reg_text = ''
 968         if all_regressions:
 969             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
 970             all_reg_text = ('All regressions:\n\n%s\n\n' %
 971                             '\n'.join(all_reg_list))
 972         else:
 973             all_reg_text = ''
 974         if all_fails:
 975             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
 976             all_fail_text = ('All failures:\n\n%s\n\n' %
 977                              '\n'.join(all_fail_list))
 978         else:
 979             all_fail_text = ''
 980         if changes:
 981             changes_list = sorted(changes.keys())
 982             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
 983             changes_text = ('All changed results:\n\n%s\n\n' %
 984                             '\n'.join(changes_list))
 985         else:
 986             changes_text = ''
 987         results_text = (new_reg_text + all_reg_text + all_fail_text +
 988                         changes_text)
 989         if not results_text:
 990             results_text = 'Clean build with unchanged results.\n\n'
 991         versions_list = sorted(versions.keys())
 992         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
 993                                           versions[k]['revision'])
 994                          for k in versions_list]
 995         versions_text = ('Component versions for this build:\n\n%s\n' %
 996                          '\n'.join(versions_list))
 997         body_text = results_text + versions_text
 998         msg = email.mime.text.MIMEText(body_text)
 999         msg['Subject'] = subject
1000         msg['From'] = self.bot_config['email-from']
1001         msg['To'] = self.bot_config['email-to']
1002         msg['Message-ID'] = email.utils.make_msgid()
1003         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1004         with smtplib.SMTP(self.bot_config['email-server']) as s:
1005             s.send_message(msg)
1006
1007     def bot_run_self(self, opts, action):
1008         """Run a copy of this script with given options."""
1009         cmd = [sys.executable, sys.argv[0], '--keep=none',
1010                '-j%d' % self.parallelism]
1011         cmd.extend(opts)
1012         cmd.extend([self.topdir, action])
1013         subprocess.run(cmd, check=True)
1014
1015
1016 class Config(object):
1017     """A configuration for building a compiler and associated libraries."""
1018
1019     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1020                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1021         """Initialize a Config object."""
1022         self.ctx = ctx
1023         self.arch = arch
1024         self.os = os_name
1025         self.variant = variant
1026         if variant is None:
1027             self.name = '%s-%s' % (arch, os_name)
1028         else:
1029             self.name = '%s-%s-%s' % (arch, os_name, variant)
1030         self.triplet = '%s-glibc-%s' % (arch, os_name)
1031         if gcc_cfg is None:
1032             self.gcc_cfg = []
1033         else:
1034             self.gcc_cfg = gcc_cfg
1035         if first_gcc_cfg is None:
1036             self.first_gcc_cfg = []
1037         else:
1038             self.first_gcc_cfg = first_gcc_cfg
1039         if glibcs is None:
1040             glibcs = [{'variant': variant}]
1041         if extra_glibcs is None:
1042             extra_glibcs = []
1043         glibcs = [Glibc(self, **g) for g in glibcs]
1044         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1045         self.all_glibcs = glibcs + extra_glibcs
1046         self.compiler_glibcs = glibcs
1047         self.installdir = ctx.compiler_installdir(self.name)
1048         self.bindir = ctx.compiler_bindir(self.name)
1049         self.sysroot = ctx.compiler_sysroot(self.name)
1050         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1051         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1052
1053     def component_builddir(self, component):
1054         """Return the directory to use for a (non-glibc) build."""
1055         return self.ctx.component_builddir('compilers', self.name, component)
1056
1057     def build(self):
1058         """Generate commands to build this compiler."""
1059         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1060                                       self.logsdir)
1061         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1062         cmdlist.add_command('check-host-libraries',
1063                             ['test', '-f',
1064                              os.path.join(self.ctx.host_libraries_installdir,
1065                                           'ok')])
1066         cmdlist.use_path(self.bindir)
1067         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1068                               ['--disable-gdb',
1069                                '--disable-libdecnumber',
1070                                '--disable-readline',
1071                                '--disable-sim'])
1072         if self.os.startswith('linux'):
1073             self.install_linux_headers(cmdlist)
1074         self.build_gcc(cmdlist, True)
1075         for g in self.compiler_glibcs:
1076             cmdlist.push_subdesc('glibc')
1077             cmdlist.push_subdesc(g.name)
1078             g.build_glibc(cmdlist, True)
1079             cmdlist.pop_subdesc()
1080             cmdlist.pop_subdesc()
1081         self.build_gcc(cmdlist, False)
1082         cmdlist.add_command('done', ['touch',
1083                                      os.path.join(self.installdir, 'ok')])
1084         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1085                                       self.logsdir)
1086
1087     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1088         """Build one cross tool."""
1089         srcdir = self.ctx.component_srcdir(tool_src)
1090         builddir = self.component_builddir(tool_build)
1091         cmdlist.push_subdesc(tool_build)
1092         cmdlist.create_use_dir(builddir)
1093         cfg_cmd = [os.path.join(srcdir, 'configure'),
1094                    '--prefix=%s' % self.installdir,
1095                    '--build=%s' % self.ctx.build_triplet,
1096                    '--host=%s' % self.ctx.build_triplet,
1097                    '--target=%s' % self.triplet,
1098                    '--with-sysroot=%s' % self.sysroot]
1099         if extra_opts:
1100             cfg_cmd.extend(extra_opts)
1101         cmdlist.add_command('configure', cfg_cmd)
1102         cmdlist.add_command('build', ['make'])
1103         cmdlist.add_command('install', ['make', 'install'])
1104         cmdlist.cleanup_dir()
1105         cmdlist.pop_subdesc()
1106
1107     def install_linux_headers(self, cmdlist):
1108         """Install Linux kernel headers."""
1109         arch_map = {'aarch64': 'arm64',
1110                     'alpha': 'alpha',
1111                     'arm': 'arm',
1112                     'hppa': 'parisc',
1113                     'i486': 'x86',
1114                     'i586': 'x86',
1115                     'i686': 'x86',
1116                     'i786': 'x86',
1117                     'ia64': 'ia64',
1118                     'm68k': 'm68k',
1119                     'microblaze': 'microblaze',
1120                     'mips': 'mips',
1121                     'nios2': 'nios2',
1122                     'powerpc': 'powerpc',
1123                     's390': 's390',
1124                     'sh': 'sh',
1125                     'sparc': 'sparc',
1126                     'tile': 'tile',
1127                     'x86_64': 'x86'}
1128         linux_arch = None
1129         for k in arch_map:
1130             if self.arch.startswith(k):
1131                 linux_arch = arch_map[k]
1132                 break
1133         assert linux_arch is not None
1134         srcdir = self.ctx.component_srcdir('linux')
1135         builddir = self.component_builddir('linux')
1136         headers_dir = os.path.join(self.sysroot, 'usr')
1137         cmdlist.push_subdesc('linux')
1138         cmdlist.create_use_dir(builddir)
1139         cmdlist.add_command('install-headers',
1140                             ['make', '-C', srcdir, 'O=%s' % builddir,
1141                              'ARCH=%s' % linux_arch,
1142                              'INSTALL_HDR_PATH=%s' % headers_dir,
1143                              'headers_install'])
1144         cmdlist.cleanup_dir()
1145         cmdlist.pop_subdesc()
1146
1147     def build_gcc(self, cmdlist, bootstrap):
1148         """Build GCC."""
1149         # libsanitizer commonly breaks because of glibc header
1150         # changes, or on unusual targets.  libssp is of little
1151         # relevance with glibc's own stack checking support.
1152         cfg_opts = list(self.gcc_cfg)
1153         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1154         host_libs = self.ctx.host_libraries_installdir
1155         cfg_opts += ['--with-gmp=%s' % host_libs,
1156                      '--with-mpfr=%s' % host_libs,
1157                      '--with-mpc=%s' % host_libs]
1158         if bootstrap:
1159             tool_build = 'gcc-first'
1160             # Building a static-only, C-only compiler that is
1161             # sufficient to build glibc.  Various libraries and
1162             # features that may require libc headers must be disabled.
1163             # When configuring with a sysroot, --with-newlib is
1164             # required to define inhibit_libc (to stop some parts of
1165             # libgcc including libc headers); --without-headers is not
1166             # sufficient.
1167             cfg_opts += ['--enable-languages=c', '--disable-shared',
1168                          '--disable-threads',
1169                          '--disable-libatomic',
1170                          '--disable-decimal-float',
1171                          '--disable-libffi',
1172                          '--disable-libgomp',
1173                          '--disable-libitm',
1174                          '--disable-libmpx',
1175                          '--disable-libquadmath',
1176                          '--without-headers', '--with-newlib',
1177                          '--with-glibc-version=%s' % self.ctx.glibc_version
1178                          ]
1179             cfg_opts += self.first_gcc_cfg
1180         else:
1181             tool_build = 'gcc'
1182             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1183                          '--enable-threads']
1184         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1185
1186
1187 class Glibc(object):
1188     """A configuration for building glibc."""
1189
1190     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1191                  cfg=None, ccopts=None):
1192         """Initialize a Glibc object."""
1193         self.ctx = compiler.ctx
1194         self.compiler = compiler
1195         if arch is None:
1196             self.arch = compiler.arch
1197         else:
1198             self.arch = arch
1199         if os_name is None:
1200             self.os = compiler.os
1201         else:
1202             self.os = os_name
1203         self.variant = variant
1204         if variant is None:
1205             self.name = '%s-%s' % (self.arch, self.os)
1206         else:
1207             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1208         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1209         if cfg is None:
1210             self.cfg = []
1211         else:
1212             self.cfg = cfg
1213         self.ccopts = ccopts
1214
1215     def tool_name(self, tool):
1216         """Return the name of a cross-compilation tool."""
1217         ctool = '%s-%s' % (self.compiler.triplet, tool)
1218         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1219             ctool = '%s %s' % (ctool, self.ccopts)
1220         return ctool
1221
1222     def build(self):
1223         """Generate commands to build this glibc."""
1224         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1225         installdir = self.ctx.glibc_installdir(self.name)
1226         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1227         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1228         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1229         cmdlist.add_command('check-compilers',
1230                             ['test', '-f',
1231                              os.path.join(self.compiler.installdir, 'ok')])
1232         cmdlist.use_path(self.compiler.bindir)
1233         self.build_glibc(cmdlist, False)
1234         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1235                                       logsdir)
1236
1237     def build_glibc(self, cmdlist, for_compiler):
1238         """Generate commands to build this glibc, either as part of a compiler
1239         build or with the bootstrapped compiler (and in the latter case, run
1240         tests as well)."""
1241         srcdir = self.ctx.component_srcdir('glibc')
1242         if for_compiler:
1243             builddir = self.ctx.component_builddir('compilers',
1244                                                    self.compiler.name, 'glibc',
1245                                                    self.name)
1246             installdir = self.compiler.sysroot
1247             srcdir_copy = self.ctx.component_builddir('compilers',
1248                                                       self.compiler.name,
1249                                                       'glibc-src',
1250                                                       self.name)
1251         else:
1252             builddir = self.ctx.component_builddir('glibcs', self.name,
1253                                                    'glibc')
1254             installdir = self.ctx.glibc_installdir(self.name)
1255             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1256                                                       'glibc-src')
1257         cmdlist.create_use_dir(builddir)
1258         # glibc builds write into the source directory, and even if
1259         # not intentionally there is a risk of bugs that involve
1260         # writing into the working directory.  To avoid possible
1261         # concurrency issues, copy the source directory.
1262         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1263         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1264                    '--prefix=/usr',
1265                    '--enable-add-ons',
1266                    '--build=%s' % self.ctx.build_triplet,
1267                    '--host=%s' % self.triplet,
1268                    'CC=%s' % self.tool_name('gcc'),
1269                    'CXX=%s' % self.tool_name('g++'),
1270                    'AR=%s' % self.tool_name('ar'),
1271                    'AS=%s' % self.tool_name('as'),
1272                    'LD=%s' % self.tool_name('ld'),
1273                    'NM=%s' % self.tool_name('nm'),
1274                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1275                    'OBJDUMP=%s' % self.tool_name('objdump'),
1276                    'RANLIB=%s' % self.tool_name('ranlib'),
1277                    'READELF=%s' % self.tool_name('readelf'),
1278                    'STRIP=%s' % self.tool_name('strip')]
1279         cfg_cmd += self.cfg
1280         cmdlist.add_command('configure', cfg_cmd)
1281         cmdlist.add_command('build', ['make'])
1282         cmdlist.add_command('install', ['make', 'install',
1283                                         'install_root=%s' % installdir])
1284         # GCC uses paths such as lib/../lib64, so make sure lib
1285         # directories always exist.
1286         cmdlist.add_command('mkdir-lib', ['mkdir', '-p',
1287                                           os.path.join(installdir, 'lib'),
1288                                           os.path.join(installdir,
1289                                                        'usr', 'lib')])
1290         if not for_compiler:
1291             cmdlist.add_command('check', ['make', 'check'])
1292             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1293                                 always_run=True)
1294         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1295         cmdlist.cleanup_dir()
1296
1297
1298 class Command(object):
1299     """A command run in the build process."""
1300
1301     def __init__(self, desc, num, dir, path, command, always_run=False):
1302         """Initialize a Command object."""
1303         self.dir = dir
1304         self.path = path
1305         self.desc = desc
1306         trans = str.maketrans({' ': '-'})
1307         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1308         self.command = command
1309         self.always_run = always_run
1310
1311     @staticmethod
1312     def shell_make_quote_string(s):
1313         """Given a string not containing a newline, quote it for use by the
1314         shell and make."""
1315         assert '\n' not in s
1316         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1317             return s
1318         strans = str.maketrans({"'": "'\\''"})
1319         s = "'%s'" % s.translate(strans)
1320         mtrans = str.maketrans({'$': '$$'})
1321         return s.translate(mtrans)
1322
1323     @staticmethod
1324     def shell_make_quote_list(l, translate_make):
1325         """Given a list of strings not containing newlines, quote them for use
1326         by the shell and make, returning a single string.  If translate_make
1327         is true and the first string is 'make', change it to $(MAKE)."""
1328         l = [Command.shell_make_quote_string(s) for s in l]
1329         if translate_make and l[0] == 'make':
1330             l[0] = '$(MAKE)'
1331         return ' '.join(l)
1332
1333     def shell_make_quote(self):
1334         """Return this command quoted for the shell and make."""
1335         return self.shell_make_quote_list(self.command, True)
1336
1337
1338 class CommandList(object):
1339     """A list of commands run in the build process."""
1340
1341     def __init__(self, desc, keep):
1342         """Initialize a CommandList object."""
1343         self.cmdlist = []
1344         self.dir = None
1345         self.path = None
1346         self.desc = [desc]
1347         self.keep = keep
1348
1349     def desc_txt(self, desc):
1350         """Return the description to use for a command."""
1351         return '%s %s' % (' '.join(self.desc), desc)
1352
1353     def use_dir(self, dir):
1354         """Set the default directory for subsequent commands."""
1355         self.dir = dir
1356
1357     def use_path(self, path):
1358         """Set a directory to be prepended to the PATH for subsequent
1359         commands."""
1360         self.path = path
1361
1362     def push_subdesc(self, subdesc):
1363         """Set the default subdescription for subsequent commands (e.g., the
1364         name of a component being built, within the series of commands
1365         building it)."""
1366         self.desc.append(subdesc)
1367
1368     def pop_subdesc(self):
1369         """Pop a subdescription from the list of descriptions."""
1370         self.desc.pop()
1371
1372     def create_use_dir(self, dir):
1373         """Remove and recreate a directory and use it for subsequent
1374         commands."""
1375         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1376         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1377         self.use_dir(dir)
1378
1379     def create_copy_dir(self, src, dest):
1380         """Remove a directory and recreate it as a copy from the given
1381         source."""
1382         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1383         parent = os.path.dirname(dest)
1384         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1385         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1386
1387     def add_command_dir(self, desc, dir, command, always_run=False):
1388         """Add a command to run in a given directory."""
1389         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1390                       command, always_run)
1391         self.cmdlist.append(cmd)
1392
1393     def add_command(self, desc, command, always_run=False):
1394         """Add a command to run in the default directory."""
1395         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1396                       self.path, command, always_run)
1397         self.cmdlist.append(cmd)
1398
1399     def cleanup_dir(self, desc='cleanup', dir=None):
1400         """Clean up a build directory.  If no directory is specified, the
1401         default directory is cleaned up and ceases to be the default
1402         directory."""
1403         if dir is None:
1404             dir = self.dir
1405             self.use_dir(None)
1406         if self.keep != 'all':
1407             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1408                                  always_run=(self.keep == 'none'))
1409
1410     def makefile_commands(self, wrapper, logsdir):
1411         """Return the sequence of commands in the form of text for a Makefile.
1412         The given wrapper script takes arguments: base of logs for
1413         previous command, or empty; base of logs for this command;
1414         description; directory; PATH addition; the command itself."""
1415         # prev_base is the base of the name for logs of the previous
1416         # command that is not always-run (that is, a build command,
1417         # whose failure should stop subsequent build commands from
1418         # being run, as opposed to a cleanup command, which is run
1419         # even if previous commands failed).
1420         prev_base = ''
1421         cmds = []
1422         for c in self.cmdlist:
1423             ctxt = c.shell_make_quote()
1424             if prev_base and not c.always_run:
1425                 prev_log = os.path.join(logsdir, prev_base)
1426             else:
1427                 prev_log = ''
1428             this_log = os.path.join(logsdir, c.logbase)
1429             if not c.always_run:
1430                 prev_base = c.logbase
1431             if c.dir is None:
1432                 dir = ''
1433             else:
1434                 dir = c.dir
1435             if c.path is None:
1436                 path = ''
1437             else:
1438                 path = c.path
1439             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1440             prelim_txt = Command.shell_make_quote_list(prelims, False)
1441             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1442         return '\n'.join(cmds)
1443
1444     def status_logs(self, logsdir):
1445         """Return the list of log files with command status."""
1446         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1447                 for c in self.cmdlist]
1448
1449
1450 def get_parser():
1451     """Return an argument parser for this module."""
1452     parser = argparse.ArgumentParser(description=__doc__)
1453     parser.add_argument('-j', dest='parallelism',
1454                         help='Run this number of jobs in parallel',
1455                         type=int, default=os.cpu_count())
1456     parser.add_argument('--keep', dest='keep',
1457                         help='Whether to keep all build directories, '
1458                         'none or only those from failed builds',
1459                         default='none', choices=('none', 'all', 'failed'))
1460     parser.add_argument('--replace-sources', action='store_true',
1461                         help='Remove and replace source directories '
1462                         'with the wrong version of a component')
1463     parser.add_argument('topdir',
1464                         help='Toplevel working directory')
1465     parser.add_argument('action',
1466                         help='What to do',
1467                         choices=('checkout', 'bot-cycle', 'host-libraries',
1468                                  'compilers', 'glibcs'))
1469     parser.add_argument('configs',
1470                         help='Versions to check out or configurations to build',
1471                         nargs='*')
1472     return parser
1473
1474
1475 def main(argv):
1476     """The main entry point."""
1477     parser = get_parser()
1478     opts = parser.parse_args(argv)
1479     topdir = os.path.abspath(opts.topdir)
1480     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1481                   opts.action)
1482     ctx.run_builds(opts.action, opts.configs)
1483
1484
1485 if __name__ == '__main__':
1486     main(sys.argv[1:])