scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52
  53 class Context(object):
  54     """The global state associated with builds in a given directory."""
  55
  56     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  57                  action):
  58         """Initialize the context."""
  59         self.topdir = topdir
  60         self.parallelism = parallelism
  61         self.keep = keep
  62         self.replace_sources = replace_sources
  63         self.strip = strip
  64         self.srcdir = os.path.join(topdir, 'src')
  65         self.versions_json = os.path.join(self.srcdir, 'versions.json')
  66         self.build_state_json = os.path.join(topdir, 'build-state.json')
  67         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
  68         self.installdir = os.path.join(topdir, 'install')
  69         self.host_libraries_installdir = os.path.join(self.installdir,
  70                                                       'host-libraries')
  71         self.builddir = os.path.join(topdir, 'build')
  72         self.logsdir = os.path.join(topdir, 'logs')
  73         self.logsdir_old = os.path.join(topdir, 'logs-old')
  74         self.makefile = os.path.join(self.builddir, 'Makefile')
  75         self.wrapper = os.path.join(self.builddir, 'wrapper')
  76         self.save_logs = os.path.join(self.builddir, 'save-logs')
  77         self.script_text = self.get_script_text()
  78         if action != 'checkout':
  79             self.build_triplet = self.get_build_triplet()
  80             self.glibc_version = self.get_glibc_version()
  81         self.configs = {}
  82         self.glibc_configs = {}
  83         self.makefile_pieces = ['.PHONY: all\n']
  84         self.add_all_configs()
  85         self.load_versions_json()
  86         self.load_build_state_json()
  87         self.status_log_list = []
  88
  89     def get_script_text(self):
  90         """Return the text of this script."""
  91         with open(sys.argv[0], 'r') as f:
  92             return f.read()
  93
  94     def exec_self(self):
  95         """Re-execute this script with the same arguments."""
  96         os.execv(sys.executable, [sys.executable] + sys.argv)
  97
  98     def get_build_triplet(self):
  99         """Determine the build triplet with config.guess."""
 100         config_guess = os.path.join(self.component_srcdir('gcc'),
 101                                     'config.guess')
 102         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 103                                 check=True, universal_newlines=True).stdout
 104         return cg_out.rstrip()
 105
 106     def get_glibc_version(self):
 107         """Determine the glibc version number (major.minor)."""
 108         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 109         with open(version_h, 'r') as f:
 110             lines = f.readlines()
 111         starttext = '#define VERSION "'
 112         for l in lines:
 113             if l.startswith(starttext):
 114                 l = l[len(starttext):]
 115                 l = l.rstrip('"\n')
 116                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 117                 return '%s.%s' % m.group(1, 2)
 118         print('error: could not determine glibc version')
 119         exit(1)
 120
 121     def add_all_configs(self):
 122         """Add all known glibc build configurations."""
 123         # On architectures missing __builtin_trap support, these
 124         # options may be needed as a workaround; see
 125         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70216> for SH.
 126         no_isolate = ('-fno-isolate-erroneous-paths-dereference'
 127                       ' -fno-isolate-erroneous-paths-attribute')
 128         self.add_config(arch='aarch64',
 129                         os_name='linux-gnu')
 130         self.add_config(arch='aarch64_be',
 131                         os_name='linux-gnu')
 132         self.add_config(arch='alpha',
 133                         os_name='linux-gnu')
 134         self.add_config(arch='arm',
 135                         os_name='linux-gnueabi')
 136         self.add_config(arch='armeb',
 137                         os_name='linux-gnueabi')
 138         self.add_config(arch='armeb',
 139                         os_name='linux-gnueabi',
 140                         variant='be8',
 141                         gcc_cfg=['--with-arch=armv7-a'])
 142         self.add_config(arch='arm',
 143                         os_name='linux-gnueabihf')
 144         self.add_config(arch='armeb',
 145                         os_name='linux-gnueabihf')
 146         self.add_config(arch='armeb',
 147                         os_name='linux-gnueabihf',
 148                         variant='be8',
 149                         gcc_cfg=['--with-arch=armv7-a'])
 150         self.add_config(arch='hppa',
 151                         os_name='linux-gnu')
 152         self.add_config(arch='ia64',
 153                         os_name='linux-gnu',
 154                         first_gcc_cfg=['--with-system-libunwind'])
 155         self.add_config(arch='m68k',
 156                         os_name='linux-gnu',
 157                         gcc_cfg=['--disable-multilib'])
 158         self.add_config(arch='m68k',
 159                         os_name='linux-gnu',
 160                         variant='coldfire',
 161                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 162         self.add_config(arch='microblaze',
 163                         os_name='linux-gnu',
 164                         gcc_cfg=['--disable-multilib'])
 165         self.add_config(arch='microblazeel',
 166                         os_name='linux-gnu',
 167                         gcc_cfg=['--disable-multilib'])
 168         self.add_config(arch='mips64',
 169                         os_name='linux-gnu',
 170                         gcc_cfg=['--with-mips-plt'],
 171                         glibcs=[{'variant': 'n32'},
 172                                 {'arch': 'mips',
 173                                  'ccopts': '-mabi=32'},
 174                                 {'variant': 'n64',
 175                                  'ccopts': '-mabi=64'}])
 176         self.add_config(arch='mips64',
 177                         os_name='linux-gnu',
 178                         variant='soft',
 179                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 180                         glibcs=[{'variant': 'n32-soft',
 181                                  'cfg': ['--without-fp']},
 182                                 {'variant': 'soft',
 183                                  'arch': 'mips',
 184                                  'ccopts': '-mabi=32',
 185                                  'cfg': ['--without-fp']},
 186                                 {'variant': 'n64-soft',
 187                                  'ccopts': '-mabi=64',
 188                                  'cfg': ['--without-fp']}])
 189         self.add_config(arch='mips64',
 190                         os_name='linux-gnu',
 191                         variant='nan2008',
 192                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 193                                  '--with-arch-64=mips64r2',
 194                                  '--with-arch-32=mips32r2'],
 195                         glibcs=[{'variant': 'n32-nan2008'},
 196                                 {'variant': 'nan2008',
 197                                  'arch': 'mips',
 198                                  'ccopts': '-mabi=32'},
 199                                 {'variant': 'n64-nan2008',
 200                                  'ccopts': '-mabi=64'}])
 201         self.add_config(arch='mips64',
 202                         os_name='linux-gnu',
 203                         variant='nan2008-soft',
 204                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 205                                  '--with-arch-64=mips64r2',
 206                                  '--with-arch-32=mips32r2',
 207                                  '--with-float=soft'],
 208                         glibcs=[{'variant': 'n32-nan2008-soft',
 209                                  'cfg': ['--without-fp']},
 210                                 {'variant': 'nan2008-soft',
 211                                  'arch': 'mips',
 212                                  'ccopts': '-mabi=32',
 213                                  'cfg': ['--without-fp']},
 214                                 {'variant': 'n64-nan2008-soft',
 215                                  'ccopts': '-mabi=64',
 216                                  'cfg': ['--without-fp']}])
 217         self.add_config(arch='mips64el',
 218                         os_name='linux-gnu',
 219                         gcc_cfg=['--with-mips-plt'],
 220                         glibcs=[{'variant': 'n32'},
 221                                 {'arch': 'mipsel',
 222                                  'ccopts': '-mabi=32'},
 223                                 {'variant': 'n64',
 224                                  'ccopts': '-mabi=64'}])
 225         self.add_config(arch='mips64el',
 226                         os_name='linux-gnu',
 227                         variant='soft',
 228                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 229                         glibcs=[{'variant': 'n32-soft',
 230                                  'cfg': ['--without-fp']},
 231                                 {'variant': 'soft',
 232                                  'arch': 'mipsel',
 233                                  'ccopts': '-mabi=32',
 234                                  'cfg': ['--without-fp']},
 235                                 {'variant': 'n64-soft',
 236                                  'ccopts': '-mabi=64',
 237                                  'cfg': ['--without-fp']}])
 238         self.add_config(arch='mips64el',
 239                         os_name='linux-gnu',
 240                         variant='nan2008',
 241                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 242                                  '--with-arch-64=mips64r2',
 243                                  '--with-arch-32=mips32r2'],
 244                         glibcs=[{'variant': 'n32-nan2008'},
 245                                 {'variant': 'nan2008',
 246                                  'arch': 'mipsel',
 247                                  'ccopts': '-mabi=32'},
 248                                 {'variant': 'n64-nan2008',
 249                                  'ccopts': '-mabi=64'}])
 250         self.add_config(arch='mips64el',
 251                         os_name='linux-gnu',
 252                         variant='nan2008-soft',
 253                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 254                                  '--with-arch-64=mips64r2',
 255                                  '--with-arch-32=mips32r2',
 256                                  '--with-float=soft'],
 257                         glibcs=[{'variant': 'n32-nan2008-soft',
 258                                  'cfg': ['--without-fp']},
 259                                 {'variant': 'nan2008-soft',
 260                                  'arch': 'mipsel',
 261                                  'ccopts': '-mabi=32',
 262                                  'cfg': ['--without-fp']},
 263                                 {'variant': 'n64-nan2008-soft',
 264                                  'ccopts': '-mabi=64',
 265                                  'cfg': ['--without-fp']}])
 266         self.add_config(arch='nios2',
 267                         os_name='linux-gnu')
 268         self.add_config(arch='powerpc',
 269                         os_name='linux-gnu',
 270                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 271         self.add_config(arch='powerpc',
 272                         os_name='linux-gnu',
 273                         variant='soft',
 274                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 275                                  '--enable-secureplt'],
 276                         glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
 277         self.add_config(arch='powerpc64',
 278                         os_name='linux-gnu',
 279                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 280         self.add_config(arch='powerpc64le',
 281                         os_name='linux-gnu',
 282                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 283         self.add_config(arch='powerpc',
 284                         os_name='linux-gnuspe',
 285                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 286                                  '--enable-e500-double'],
 287                         glibcs=[{'cfg': ['--without-fp']}])
 288         self.add_config(arch='powerpc',
 289                         os_name='linux-gnuspe',
 290                         variant='e500v1',
 291                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 292                         glibcs=[{'variant': 'e500v1', 'cfg': ['--without-fp']}])
 293         self.add_config(arch='s390x',
 294                         os_name='linux-gnu',
 295                         glibcs=[{},
 296                                 {'arch': 's390', 'ccopts': '-m31'}])
 297         self.add_config(arch='sh3',
 298                         os_name='linux-gnu',
 299                         glibcs=[{'ccopts': no_isolate}])
 300         self.add_config(arch='sh3eb',
 301                         os_name='linux-gnu',
 302                         glibcs=[{'ccopts': no_isolate}])
 303         self.add_config(arch='sh4',
 304                         os_name='linux-gnu',
 305                         glibcs=[{'ccopts': no_isolate}])
 306         self.add_config(arch='sh4eb',
 307                         os_name='linux-gnu',
 308                         glibcs=[{'ccopts': no_isolate}])
 309         self.add_config(arch='sh4',
 310                         os_name='linux-gnu',
 311                         variant='soft',
 312                         gcc_cfg=['--without-fp'],
 313                         glibcs=[{'variant': 'soft',
 314                                  'cfg': ['--without-fp'],
 315                                  'ccopts': no_isolate}])
 316         self.add_config(arch='sh4eb',
 317                         os_name='linux-gnu',
 318                         variant='soft',
 319                         gcc_cfg=['--without-fp'],
 320                         glibcs=[{'variant': 'soft',
 321                                  'cfg': ['--without-fp'],
 322                                  'ccopts': no_isolate}])
 323         self.add_config(arch='sparc64',
 324                         os_name='linux-gnu',
 325                         glibcs=[{},
 326                                 {'arch': 'sparcv9',
 327                                  'ccopts': '-m32 -mlong-double-128'}])
 328         self.add_config(arch='tilegx',
 329                         os_name='linux-gnu',
 330                         glibcs=[{},
 331                                 {'variant': '32', 'ccopts': '-m32'}])
 332         self.add_config(arch='tilegxbe',
 333                         os_name='linux-gnu',
 334                         glibcs=[{},
 335                                 {'variant': '32', 'ccopts': '-m32'}])
 336         self.add_config(arch='tilepro',
 337                         os_name='linux-gnu')
 338         self.add_config(arch='x86_64',
 339                         os_name='linux-gnu',
 340                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 341                         glibcs=[{},
 342                                 {'variant': 'x32', 'ccopts': '-mx32'},
 343                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 344                         extra_glibcs=[{'variant': 'disable-multi-arch',
 345                                        'cfg': ['--disable-multi-arch']},
 346                                       {'variant': 'disable-multi-arch',
 347                                        'arch': 'i686',
 348                                        'ccopts': '-m32 -march=i686',
 349                                        'cfg': ['--disable-multi-arch']},
 350                                       {'arch': 'i486',
 351                                        'ccopts': '-m32 -march=i486'},
 352                                       {'arch': 'i586',
 353                                        'ccopts': '-m32 -march=i586'}])
 354
 355     def add_config(self, **args):
 356         """Add an individual build configuration."""
 357         cfg = Config(self, **args)
 358         if cfg.name in self.configs:
 359             print('error: duplicate config %s' % cfg.name)
 360             exit(1)
 361         self.configs[cfg.name] = cfg
 362         for c in cfg.all_glibcs:
 363             if c.name in self.glibc_configs:
 364                 print('error: duplicate glibc config %s' % c.name)
 365                 exit(1)
 366             self.glibc_configs[c.name] = c
 367
 368     def component_srcdir(self, component):
 369         """Return the source directory for a given component, e.g. gcc."""
 370         return os.path.join(self.srcdir, component)
 371
 372     def component_builddir(self, action, config, component, subconfig=None):
 373         """Return the directory to use for a build."""
 374         if config is None:
 375             # Host libraries.
 376             assert subconfig is None
 377             return os.path.join(self.builddir, action, component)
 378         if subconfig is None:
 379             return os.path.join(self.builddir, action, config, component)
 380         else:
 381             # glibc build as part of compiler build.
 382             return os.path.join(self.builddir, action, config, component,
 383                                 subconfig)
 384
 385     def compiler_installdir(self, config):
 386         """Return the directory in which to install a compiler."""
 387         return os.path.join(self.installdir, 'compilers', config)
 388
 389     def compiler_bindir(self, config):
 390         """Return the directory in which to find compiler binaries."""
 391         return os.path.join(self.compiler_installdir(config), 'bin')
 392
 393     def compiler_sysroot(self, config):
 394         """Return the sysroot directory for a compiler."""
 395         return os.path.join(self.compiler_installdir(config), 'sysroot')
 396
 397     def glibc_installdir(self, config):
 398         """Return the directory in which to install glibc."""
 399         return os.path.join(self.installdir, 'glibcs', config)
 400
 401     def run_builds(self, action, configs):
 402         """Run the requested builds."""
 403         if action == 'checkout':
 404             self.checkout(configs)
 405             return
 406         if action == 'bot-cycle':
 407             if configs:
 408                 print('error: configurations specified for bot-cycle')
 409                 exit(1)
 410             self.bot_cycle()
 411             return
 412         if action == 'bot':
 413             if configs:
 414                 print('error: configurations specified for bot')
 415                 exit(1)
 416             self.bot()
 417             return
 418         if action == 'host-libraries' and configs:
 419             print('error: configurations specified for host-libraries')
 420             exit(1)
 421         self.clear_last_build_state(action)
 422         build_time = datetime.datetime.utcnow()
 423         if action == 'host-libraries':
 424             build_components = ('gmp', 'mpfr', 'mpc')
 425             old_components = ()
 426             old_versions = {}
 427             self.build_host_libraries()
 428         elif action == 'compilers':
 429             build_components = ('binutils', 'gcc', 'glibc', 'linux')
 430             old_components = ('gmp', 'mpfr', 'mpc')
 431             old_versions = self.build_state['host-libraries']['build-versions']
 432             self.build_compilers(configs)
 433         else:
 434             build_components = ('glibc',)
 435             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
 436             old_versions = self.build_state['compilers']['build-versions']
 437             self.build_glibcs(configs)
 438         self.write_files()
 439         self.do_build()
 440         if configs:
 441             # Partial build, do not update stored state.
 442             return
 443         build_versions = {}
 444         for k in build_components:
 445             if k in self.versions:
 446                 build_versions[k] = {'version': self.versions[k]['version'],
 447                                      'revision': self.versions[k]['revision']}
 448         for k in old_components:
 449             if k in old_versions:
 450                 build_versions[k] = {'version': old_versions[k]['version'],
 451                                      'revision': old_versions[k]['revision']}
 452         self.update_build_state(action, build_time, build_versions)
 453
 454     @staticmethod
 455     def remove_dirs(*args):
 456         """Remove directories and their contents if they exist."""
 457         for dir in args:
 458             shutil.rmtree(dir, ignore_errors=True)
 459
 460     @staticmethod
 461     def remove_recreate_dirs(*args):
 462         """Remove directories if they exist, and create them as empty."""
 463         Context.remove_dirs(*args)
 464         for dir in args:
 465             os.makedirs(dir, exist_ok=True)
 466
 467     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 468         """Add makefile text for a list of commands."""
 469         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 470         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 471                                     (target, target, target, commands))
 472         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 473
 474     def write_files(self):
 475         """Write out the Makefile and wrapper script."""
 476         mftext = ''.join(self.makefile_pieces)
 477         with open(self.makefile, 'w') as f:
 478             f.write(mftext)
 479         wrapper_text = (
 480             '#!/bin/sh\n'
 481             'prev_base=$1\n'
 482             'this_base=$2\n'
 483             'desc=$3\n'
 484             'dir=$4\n'
 485             'path=$5\n'
 486             'shift 5\n'
 487             'prev_status=$prev_base-status.txt\n'
 488             'this_status=$this_base-status.txt\n'
 489             'this_log=$this_base-log.txt\n'
 490             'date > "$this_log"\n'
 491             'echo >> "$this_log"\n'
 492             'echo "Description: $desc" >> "$this_log"\n'
 493             'printf "%s" "Command:" >> "$this_log"\n'
 494             'for word in "$@"; do\n'
 495             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 496             '    printf " %s" "$word"\n'
 497             '  else\n'
 498             '    printf " \'"\n'
 499             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 500             '    printf "\'"\n'
 501             '  fi\n'
 502             'done >> "$this_log"\n'
 503             'echo >> "$this_log"\n'
 504             'echo "Directory: $dir" >> "$this_log"\n'
 505             'echo "Path addition: $path" >> "$this_log"\n'
 506             'echo >> "$this_log"\n'
 507             'record_status ()\n'
 508             '{\n'
 509             '  echo >> "$this_log"\n'
 510             '  echo "$1: $desc" > "$this_status"\n'
 511             '  echo "$1: $desc" >> "$this_log"\n'
 512             '  echo >> "$this_log"\n'
 513             '  date >> "$this_log"\n'
 514             '  echo "$1: $desc"\n'
 515             '  exit 0\n'
 516             '}\n'
 517             'check_error ()\n'
 518             '{\n'
 519             '  if [ "$1" != "0" ]; then\n'
 520             '    record_status FAIL\n'
 521             '  fi\n'
 522             '}\n'
 523             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 524             '    record_status UNRESOLVED\n'
 525             'fi\n'
 526             'if [ "$dir" ]; then\n'
 527             '  cd "$dir"\n'
 528             '  check_error "$?"\n'
 529             'fi\n'
 530             'if [ "$path" ]; then\n'
 531             '  PATH=$path:$PATH\n'
 532             'fi\n'
 533             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 534             'check_error "$?"\n'
 535             'record_status PASS\n')
 536         with open(self.wrapper, 'w') as f:
 537             f.write(wrapper_text)
 538         # Mode 0o755.
 539         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 540                      stat.S_IROTH|stat.S_IXOTH)
 541         os.chmod(self.wrapper, mode_exec)
 542         save_logs_text = (
 543             '#!/bin/sh\n'
 544             'if ! [ -f tests.sum ]; then\n'
 545             '  echo "No test summary available."\n'
 546             '  exit 0\n'
 547             'fi\n'
 548             'save_file ()\n'
 549             '{\n'
 550             '  echo "Contents of $1:"\n'
 551             '  echo\n'
 552             '  cat "$1"\n'
 553             '  echo\n'
 554             '  echo "End of contents of $1."\n'
 555             '  echo\n'
 556             '}\n'
 557             'save_file tests.sum\n'
 558             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 559             'for t in $non_pass_tests; do\n'
 560             '  if [ -f "$t.out" ]; then\n'
 561             '    save_file "$t.out"\n'
 562             '  fi\n'
 563             'done\n')
 564         with open(self.save_logs, 'w') as f:
 565             f.write(save_logs_text)
 566         os.chmod(self.save_logs, mode_exec)
 567
 568     def do_build(self):
 569         """Do the actual build."""
 570         cmd = ['make', '-j%d' % self.parallelism]
 571         subprocess.run(cmd, cwd=self.builddir, check=True)
 572
 573     def build_host_libraries(self):
 574         """Build the host libraries."""
 575         installdir = self.host_libraries_installdir
 576         builddir = os.path.join(self.builddir, 'host-libraries')
 577         logsdir = os.path.join(self.logsdir, 'host-libraries')
 578         self.remove_recreate_dirs(installdir, builddir, logsdir)
 579         cmdlist = CommandList('host-libraries', self.keep)
 580         self.build_host_library(cmdlist, 'gmp')
 581         self.build_host_library(cmdlist, 'mpfr',
 582                                 ['--with-gmp=%s' % installdir])
 583         self.build_host_library(cmdlist, 'mpc',
 584                                 ['--with-gmp=%s' % installdir,
 585                                 '--with-mpfr=%s' % installdir])
 586         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 587         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 588
 589     def build_host_library(self, cmdlist, lib, extra_opts=None):
 590         """Build one host library."""
 591         srcdir = self.component_srcdir(lib)
 592         builddir = self.component_builddir('host-libraries', None, lib)
 593         installdir = self.host_libraries_installdir
 594         cmdlist.push_subdesc(lib)
 595         cmdlist.create_use_dir(builddir)
 596         cfg_cmd = [os.path.join(srcdir, 'configure'),
 597                    '--prefix=%s' % installdir,
 598                    '--disable-shared']
 599         if extra_opts:
 600             cfg_cmd.extend (extra_opts)
 601         cmdlist.add_command('configure', cfg_cmd)
 602         cmdlist.add_command('build', ['make'])
 603         cmdlist.add_command('check', ['make', 'check'])
 604         cmdlist.add_command('install', ['make', 'install'])
 605         cmdlist.cleanup_dir()
 606         cmdlist.pop_subdesc()
 607
 608     def build_compilers(self, configs):
 609         """Build the compilers."""
 610         if not configs:
 611             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 612             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 613             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 614             configs = sorted(self.configs.keys())
 615         for c in configs:
 616             self.configs[c].build()
 617
 618     def build_glibcs(self, configs):
 619         """Build the glibcs."""
 620         if not configs:
 621             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 622             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 623             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 624             configs = sorted(self.glibc_configs.keys())
 625         for c in configs:
 626             self.glibc_configs[c].build()
 627
 628     def load_versions_json(self):
 629         """Load information about source directory versions."""
 630         if not os.access(self.versions_json, os.F_OK):
 631             self.versions = {}
 632             return
 633         with open(self.versions_json, 'r') as f:
 634             self.versions = json.load(f)
 635
 636     def store_json(self, data, filename):
 637         """Store information in a JSON file."""
 638         filename_tmp = filename + '.tmp'
 639         with open(filename_tmp, 'w') as f:
 640             json.dump(data, f, indent=2, sort_keys=True)
 641         os.rename(filename_tmp, filename)
 642
 643     def store_versions_json(self):
 644         """Store information about source directory versions."""
 645         self.store_json(self.versions, self.versions_json)
 646
 647     def set_component_version(self, component, version, explicit, revision):
 648         """Set the version information for a component."""
 649         self.versions[component] = {'version': version,
 650                                     'explicit': explicit,
 651                                     'revision': revision}
 652         self.store_versions_json()
 653
 654     def checkout(self, versions):
 655         """Check out the desired component versions."""
 656         default_versions = {'binutils': 'vcs-2.27',
 657                             'gcc': 'vcs-6',
 658                             'glibc': 'vcs-mainline',
 659                             'gmp': '6.1.1',
 660                             'linux': '4.9',
 661                             'mpc': '1.0.3',
 662                             'mpfr': '3.1.5'}
 663         use_versions = {}
 664         explicit_versions = {}
 665         for v in versions:
 666             found_v = False
 667             for k in default_versions.keys():
 668                 kx = k + '-'
 669                 if v.startswith(kx):
 670                     vx = v[len(kx):]
 671                     if k in use_versions:
 672                         print('error: multiple versions for %s' % k)
 673                         exit(1)
 674                     use_versions[k] = vx
 675                     explicit_versions[k] = True
 676                     found_v = True
 677                     break
 678             if not found_v:
 679                 print('error: unknown component in %s' % v)
 680                 exit(1)
 681         for k in default_versions.keys():
 682             if k not in use_versions:
 683                 if k in self.versions and self.versions[k]['explicit']:
 684                     use_versions[k] = self.versions[k]['version']
 685                     explicit_versions[k] = True
 686                 else:
 687                     use_versions[k] = default_versions[k]
 688                     explicit_versions[k] = False
 689         os.makedirs(self.srcdir, exist_ok=True)
 690         for k in sorted(default_versions.keys()):
 691             update = os.access(self.component_srcdir(k), os.F_OK)
 692             v = use_versions[k]
 693             if (update and
 694                 k in self.versions and
 695                 v != self.versions[k]['version']):
 696                 if not self.replace_sources:
 697                     print('error: version of %s has changed from %s to %s, '
 698                           'use --replace-sources to check out again' %
 699                           (k, self.versions[k]['version'], v))
 700                     exit(1)
 701                 shutil.rmtree(self.component_srcdir(k))
 702                 update = False
 703             if v.startswith('vcs-'):
 704                 revision = self.checkout_vcs(k, v[4:], update)
 705             else:
 706                 self.checkout_tar(k, v, update)
 707                 revision = v
 708             self.set_component_version(k, v, explicit_versions[k], revision)
 709         if self.get_script_text() != self.script_text:
 710             # Rerun the checkout process in case the updated script
 711             # uses different default versions or new components.
 712             self.exec_self()
 713
 714     def checkout_vcs(self, component, version, update):
 715         """Check out the given version of the given component from version
 716         control.  Return a revision identifier."""
 717         if component == 'binutils':
 718             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 719             if version == 'mainline':
 720                 git_branch = 'master'
 721             else:
 722                 trans = str.maketrans({'.': '_'})
 723                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 724             return self.git_checkout(component, git_url, git_branch, update)
 725         elif component == 'gcc':
 726             if version == 'mainline':
 727                 branch = 'trunk'
 728             else:
 729                 trans = str.maketrans({'.': '_'})
 730                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 731             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 732             return self.gcc_checkout(svn_url, update)
 733         elif component == 'glibc':
 734             git_url = 'git://sourceware.org/git/glibc.git'
 735             if version == 'mainline':
 736                 git_branch = 'master'
 737             else:
 738                 git_branch = 'release/%s/master' % version
 739             r = self.git_checkout(component, git_url, git_branch, update)
 740             self.fix_glibc_timestamps()
 741             return r
 742         else:
 743             print('error: component %s coming from VCS' % component)
 744             exit(1)
 745
 746     def git_checkout(self, component, git_url, git_branch, update):
 747         """Check out a component from git.  Return a commit identifier."""
 748         if update:
 749             subprocess.run(['git', 'remote', 'prune', 'origin'],
 750                            cwd=self.component_srcdir(component), check=True)
 751             subprocess.run(['git', 'pull', '-q'],
 752                            cwd=self.component_srcdir(component), check=True)
 753         else:
 754             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 755                             self.component_srcdir(component)], check=True)
 756         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 757                            cwd=self.component_srcdir(component),
 758                            stdout=subprocess.PIPE,
 759                            check=True, universal_newlines=True).stdout
 760         return r.rstrip()
 761
 762     def fix_glibc_timestamps(self):
 763         """Fix timestamps in a glibc checkout."""
 764         # Ensure that builds do not try to regenerate generated files
 765         # in the source tree.
 766         srcdir = self.component_srcdir('glibc')
 767         for dirpath, dirnames, filenames in os.walk(srcdir):
 768             for f in filenames:
 769                 if (f == 'configure' or
 770                     f == 'preconfigure' or
 771                     f.endswith('-kw.h')):
 772                     to_touch = os.path.join(dirpath, f)
 773                     subprocess.run(['touch', to_touch], check=True)
 774
 775     def gcc_checkout(self, svn_url, update):
 776         """Check out GCC from SVN.  Return the revision number."""
 777         if not update:
 778             subprocess.run(['svn', 'co', '-q', svn_url,
 779                             self.component_srcdir('gcc')], check=True)
 780         subprocess.run(['contrib/gcc_update', '--silent'],
 781                        cwd=self.component_srcdir('gcc'), check=True)
 782         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 783                            stdout=subprocess.PIPE,
 784                            check=True, universal_newlines=True).stdout
 785         return r.rstrip()
 786
 787     def checkout_tar(self, component, version, update):
 788         """Check out the given version of the given component from a
 789         tarball."""
 790         if update:
 791             return
 792         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 793                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 794                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 795                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 796                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 797                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
 798         if component not in url_map:
 799             print('error: component %s coming from tarball' % component)
 800             exit(1)
 801         url = url_map[component] % {'version': version}
 802         filename = os.path.join(self.srcdir, url.split('/')[-1])
 803         response = urllib.request.urlopen(url)
 804         data = response.read()
 805         with open(filename, 'wb') as f:
 806             f.write(data)
 807         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 808                        check=True)
 809         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 810                   self.component_srcdir(component))
 811         os.remove(filename)
 812
 813     def load_build_state_json(self):
 814         """Load information about the state of previous builds."""
 815         if os.access(self.build_state_json, os.F_OK):
 816             with open(self.build_state_json, 'r') as f:
 817                 self.build_state = json.load(f)
 818         else:
 819             self.build_state = {}
 820         for k in ('host-libraries', 'compilers', 'glibcs'):
 821             if k not in self.build_state:
 822                 self.build_state[k] = {}
 823             if 'build-time' not in self.build_state[k]:
 824                 self.build_state[k]['build-time'] = ''
 825             if 'build-versions' not in self.build_state[k]:
 826                 self.build_state[k]['build-versions'] = {}
 827             if 'build-results' not in self.build_state[k]:
 828                 self.build_state[k]['build-results'] = {}
 829             if 'result-changes' not in self.build_state[k]:
 830                 self.build_state[k]['result-changes'] = {}
 831             if 'ever-passed' not in self.build_state[k]:
 832                 self.build_state[k]['ever-passed'] = []
 833
 834     def store_build_state_json(self):
 835         """Store information about the state of previous builds."""
 836         self.store_json(self.build_state, self.build_state_json)
 837
 838     def clear_last_build_state(self, action):
 839         """Clear information about the state of part of the build."""
 840         # We clear the last build time and versions when starting a
 841         # new build.  The results of the last build are kept around,
 842         # as comparison is still meaningful if this build is aborted
 843         # and a new one started.
 844         self.build_state[action]['build-time'] = ''
 845         self.build_state[action]['build-versions'] = {}
 846         self.store_build_state_json()
 847
 848     def update_build_state(self, action, build_time, build_versions):
 849         """Update the build state after a build."""
 850         build_time = build_time.replace(microsecond=0)
 851         self.build_state[action]['build-time'] = str(build_time)
 852         self.build_state[action]['build-versions'] = build_versions
 853         build_results = {}
 854         for log in self.status_log_list:
 855             with open(log, 'r') as f:
 856                 log_text = f.read()
 857             log_text = log_text.rstrip()
 858             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 859             result = m.group(1)
 860             test_name = m.group(2)
 861             assert test_name not in build_results
 862             build_results[test_name] = result
 863         old_build_results = self.build_state[action]['build-results']
 864         self.build_state[action]['build-results'] = build_results
 865         result_changes = {}
 866         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 867         for t in all_tests:
 868             if t in old_build_results:
 869                 old_res = old_build_results[t]
 870             else:
 871                 old_res = '(New test)'
 872             if t in build_results:
 873                 new_res = build_results[t]
 874             else:
 875                 new_res = '(Test removed)'
 876             if old_res != new_res:
 877                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 878         self.build_state[action]['result-changes'] = result_changes
 879         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 880                            if t in build_results}
 881         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 882         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 883                                                          new_passes)
 884         self.store_build_state_json()
 885
 886     def load_bot_config_json(self):
 887         """Load bot configuration."""
 888         with open(self.bot_config_json, 'r') as f:
 889             self.bot_config = json.load(f)
 890
 891     def part_build_old(self, action, delay):
 892         """Return whether the last build for a given action was at least a
 893         given number of seconds ago, or does not have a time recorded."""
 894         old_time_str = self.build_state[action]['build-time']
 895         if not old_time_str:
 896             return True
 897         old_time = datetime.datetime.strptime(old_time_str,
 898                                               '%Y-%m-%d %H:%M:%S')
 899         new_time = datetime.datetime.utcnow()
 900         delta = new_time - old_time
 901         return delta.total_seconds() >= delay
 902
 903     def bot_cycle(self):
 904         """Run a single round of checkout and builds."""
 905         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 906         self.load_bot_config_json()
 907         actions = ('host-libraries', 'compilers', 'glibcs')
 908         self.bot_run_self(['--replace-sources'], 'checkout')
 909         self.load_versions_json()
 910         if self.get_script_text() != self.script_text:
 911             print('Script changed, re-execing.')
 912             # On script change, all parts of the build should be rerun.
 913             for a in actions:
 914                 self.clear_last_build_state(a)
 915             self.exec_self()
 916         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 917                             'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
 918                             'glibcs': ('glibc',)}
 919         must_build = {}
 920         for a in actions:
 921             build_vers = self.build_state[a]['build-versions']
 922             must_build[a] = False
 923             if not self.build_state[a]['build-time']:
 924                 must_build[a] = True
 925             old_vers = {}
 926             new_vers = {}
 927             for c in check_components[a]:
 928                 if c in build_vers:
 929                     old_vers[c] = build_vers[c]
 930                 new_vers[c] = {'version': self.versions[c]['version'],
 931                                'revision': self.versions[c]['revision']}
 932             if new_vers == old_vers:
 933                 print('Versions for %s unchanged.' % a)
 934             else:
 935                 print('Versions changed or rebuild forced for %s.' % a)
 936                 if a == 'compilers' and not self.part_build_old(
 937                         a, self.bot_config['compilers-rebuild-delay']):
 938                     print('Not requiring rebuild of compilers this soon.')
 939                 else:
 940                     must_build[a] = True
 941         if must_build['host-libraries']:
 942             must_build['compilers'] = True
 943         if must_build['compilers']:
 944             must_build['glibcs'] = True
 945         for a in actions:
 946             if must_build[a]:
 947                 print('Must rebuild %s.' % a)
 948                 self.clear_last_build_state(a)
 949             else:
 950                 print('No need to rebuild %s.' % a)
 951         if os.access(self.logsdir, os.F_OK):
 952             shutil.rmtree(self.logsdir_old, ignore_errors=True)
 953             shutil.copytree(self.logsdir, self.logsdir_old)
 954         for a in actions:
 955             if must_build[a]:
 956                 build_time = datetime.datetime.utcnow()
 957                 print('Rebuilding %s at %s.' % (a, str(build_time)))
 958                 self.bot_run_self([], a)
 959                 self.load_build_state_json()
 960                 self.bot_build_mail(a, build_time)
 961         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
 962
 963     def bot_build_mail(self, action, build_time):
 964         """Send email with the results of a build."""
 965         build_time = build_time.replace(microsecond=0)
 966         subject = (self.bot_config['email-subject'] %
 967                    {'action': action,
 968                     'build-time': str(build_time)})
 969         results = self.build_state[action]['build-results']
 970         changes = self.build_state[action]['result-changes']
 971         ever_passed = set(self.build_state[action]['ever-passed'])
 972         versions = self.build_state[action]['build-versions']
 973         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
 974         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
 975         all_fails = {k for k in results if results[k] == 'FAIL'}
 976         if new_regressions:
 977             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
 978             new_reg_text = ('New regressions:\n\n%s\n\n' %
 979                             '\n'.join(new_reg_list))
 980         else:
 981             new_reg_text = ''
 982         if all_regressions:
 983             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
 984             all_reg_text = ('All regressions:\n\n%s\n\n' %
 985                             '\n'.join(all_reg_list))
 986         else:
 987             all_reg_text = ''
 988         if all_fails:
 989             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
 990             all_fail_text = ('All failures:\n\n%s\n\n' %
 991                              '\n'.join(all_fail_list))
 992         else:
 993             all_fail_text = ''
 994         if changes:
 995             changes_list = sorted(changes.keys())
 996             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
 997             changes_text = ('All changed results:\n\n%s\n\n' %
 998                             '\n'.join(changes_list))
 999         else:
1000             changes_text = ''
1001         results_text = (new_reg_text + all_reg_text + all_fail_text +
1002                         changes_text)
1003         if not results_text:
1004             results_text = 'Clean build with unchanged results.\n\n'
1005         versions_list = sorted(versions.keys())
1006         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1007                                           versions[k]['revision'])
1008                          for k in versions_list]
1009         versions_text = ('Component versions for this build:\n\n%s\n' %
1010                          '\n'.join(versions_list))
1011         body_text = results_text + versions_text
1012         msg = email.mime.text.MIMEText(body_text)
1013         msg['Subject'] = subject
1014         msg['From'] = self.bot_config['email-from']
1015         msg['To'] = self.bot_config['email-to']
1016         msg['Message-ID'] = email.utils.make_msgid()
1017         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1018         with smtplib.SMTP(self.bot_config['email-server']) as s:
1019             s.send_message(msg)
1020
1021     def bot_run_self(self, opts, action, check=True):
1022         """Run a copy of this script with given options."""
1023         cmd = [sys.executable, sys.argv[0], '--keep=none',
1024                '-j%d' % self.parallelism]
1025         cmd.extend(opts)
1026         cmd.extend([self.topdir, action])
1027         sys.stdout.flush()
1028         subprocess.run(cmd, check=check)
1029
1030     def bot(self):
1031         """Run repeated rounds of checkout and builds."""
1032         while True:
1033             self.load_bot_config_json()
1034             if not self.bot_config['run']:
1035                 print('Bot exiting by request.')
1036                 exit(0)
1037             self.bot_run_self([], 'bot-cycle', check=False)
1038             self.load_bot_config_json()
1039             if not self.bot_config['run']:
1040                 print('Bot exiting by request.')
1041                 exit(0)
1042             time.sleep(self.bot_config['delay'])
1043             if self.get_script_text() != self.script_text:
1044                 print('Script changed, bot re-execing.')
1045                 self.exec_self()
1046
1047
1048 class Config(object):
1049     """A configuration for building a compiler and associated libraries."""
1050
1051     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1052                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1053         """Initialize a Config object."""
1054         self.ctx = ctx
1055         self.arch = arch
1056         self.os = os_name
1057         self.variant = variant
1058         if variant is None:
1059             self.name = '%s-%s' % (arch, os_name)
1060         else:
1061             self.name = '%s-%s-%s' % (arch, os_name, variant)
1062         self.triplet = '%s-glibc-%s' % (arch, os_name)
1063         if gcc_cfg is None:
1064             self.gcc_cfg = []
1065         else:
1066             self.gcc_cfg = gcc_cfg
1067         if first_gcc_cfg is None:
1068             self.first_gcc_cfg = []
1069         else:
1070             self.first_gcc_cfg = first_gcc_cfg
1071         if glibcs is None:
1072             glibcs = [{'variant': variant}]
1073         if extra_glibcs is None:
1074             extra_glibcs = []
1075         glibcs = [Glibc(self, **g) for g in glibcs]
1076         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1077         self.all_glibcs = glibcs + extra_glibcs
1078         self.compiler_glibcs = glibcs
1079         self.installdir = ctx.compiler_installdir(self.name)
1080         self.bindir = ctx.compiler_bindir(self.name)
1081         self.sysroot = ctx.compiler_sysroot(self.name)
1082         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1083         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1084
1085     def component_builddir(self, component):
1086         """Return the directory to use for a (non-glibc) build."""
1087         return self.ctx.component_builddir('compilers', self.name, component)
1088
1089     def build(self):
1090         """Generate commands to build this compiler."""
1091         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1092                                       self.logsdir)
1093         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1094         cmdlist.add_command('check-host-libraries',
1095                             ['test', '-f',
1096                              os.path.join(self.ctx.host_libraries_installdir,
1097                                           'ok')])
1098         cmdlist.use_path(self.bindir)
1099         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1100                               ['--disable-gdb',
1101                                '--disable-libdecnumber',
1102                                '--disable-readline',
1103                                '--disable-sim'])
1104         if self.os.startswith('linux'):
1105             self.install_linux_headers(cmdlist)
1106         self.build_gcc(cmdlist, True)
1107         for g in self.compiler_glibcs:
1108             cmdlist.push_subdesc('glibc')
1109             cmdlist.push_subdesc(g.name)
1110             g.build_glibc(cmdlist, True)
1111             cmdlist.pop_subdesc()
1112             cmdlist.pop_subdesc()
1113         self.build_gcc(cmdlist, False)
1114         cmdlist.add_command('done', ['touch',
1115                                      os.path.join(self.installdir, 'ok')])
1116         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1117                                       self.logsdir)
1118
1119     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1120         """Build one cross tool."""
1121         srcdir = self.ctx.component_srcdir(tool_src)
1122         builddir = self.component_builddir(tool_build)
1123         cmdlist.push_subdesc(tool_build)
1124         cmdlist.create_use_dir(builddir)
1125         cfg_cmd = [os.path.join(srcdir, 'configure'),
1126                    '--prefix=%s' % self.installdir,
1127                    '--build=%s' % self.ctx.build_triplet,
1128                    '--host=%s' % self.ctx.build_triplet,
1129                    '--target=%s' % self.triplet,
1130                    '--with-sysroot=%s' % self.sysroot]
1131         if extra_opts:
1132             cfg_cmd.extend(extra_opts)
1133         cmdlist.add_command('configure', cfg_cmd)
1134         cmdlist.add_command('build', ['make'])
1135         cmdlist.add_command('install', ['make', 'install'])
1136         cmdlist.cleanup_dir()
1137         cmdlist.pop_subdesc()
1138
1139     def install_linux_headers(self, cmdlist):
1140         """Install Linux kernel headers."""
1141         arch_map = {'aarch64': 'arm64',
1142                     'alpha': 'alpha',
1143                     'arm': 'arm',
1144                     'hppa': 'parisc',
1145                     'i486': 'x86',
1146                     'i586': 'x86',
1147                     'i686': 'x86',
1148                     'i786': 'x86',
1149                     'ia64': 'ia64',
1150                     'm68k': 'm68k',
1151                     'microblaze': 'microblaze',
1152                     'mips': 'mips',
1153                     'nios2': 'nios2',
1154                     'powerpc': 'powerpc',
1155                     's390': 's390',
1156                     'sh': 'sh',
1157                     'sparc': 'sparc',
1158                     'tile': 'tile',
1159                     'x86_64': 'x86'}
1160         linux_arch = None
1161         for k in arch_map:
1162             if self.arch.startswith(k):
1163                 linux_arch = arch_map[k]
1164                 break
1165         assert linux_arch is not None
1166         srcdir = self.ctx.component_srcdir('linux')
1167         builddir = self.component_builddir('linux')
1168         headers_dir = os.path.join(self.sysroot, 'usr')
1169         cmdlist.push_subdesc('linux')
1170         cmdlist.create_use_dir(builddir)
1171         cmdlist.add_command('install-headers',
1172                             ['make', '-C', srcdir, 'O=%s' % builddir,
1173                              'ARCH=%s' % linux_arch,
1174                              'INSTALL_HDR_PATH=%s' % headers_dir,
1175                              'headers_install'])
1176         cmdlist.cleanup_dir()
1177         cmdlist.pop_subdesc()
1178
1179     def build_gcc(self, cmdlist, bootstrap):
1180         """Build GCC."""
1181         # libsanitizer commonly breaks because of glibc header
1182         # changes, or on unusual targets.  libssp is of little
1183         # relevance with glibc's own stack checking support.
1184         cfg_opts = list(self.gcc_cfg)
1185         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1186         host_libs = self.ctx.host_libraries_installdir
1187         cfg_opts += ['--with-gmp=%s' % host_libs,
1188                      '--with-mpfr=%s' % host_libs,
1189                      '--with-mpc=%s' % host_libs]
1190         if bootstrap:
1191             tool_build = 'gcc-first'
1192             # Building a static-only, C-only compiler that is
1193             # sufficient to build glibc.  Various libraries and
1194             # features that may require libc headers must be disabled.
1195             # When configuring with a sysroot, --with-newlib is
1196             # required to define inhibit_libc (to stop some parts of
1197             # libgcc including libc headers); --without-headers is not
1198             # sufficient.
1199             cfg_opts += ['--enable-languages=c', '--disable-shared',
1200                          '--disable-threads',
1201                          '--disable-libatomic',
1202                          '--disable-decimal-float',
1203                          '--disable-libffi',
1204                          '--disable-libgomp',
1205                          '--disable-libitm',
1206                          '--disable-libmpx',
1207                          '--disable-libquadmath',
1208                          '--without-headers', '--with-newlib',
1209                          '--with-glibc-version=%s' % self.ctx.glibc_version
1210                          ]
1211             cfg_opts += self.first_gcc_cfg
1212         else:
1213             tool_build = 'gcc'
1214             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1215                          '--enable-threads']
1216         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1217
1218
1219 class Glibc(object):
1220     """A configuration for building glibc."""
1221
1222     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1223                  cfg=None, ccopts=None):
1224         """Initialize a Glibc object."""
1225         self.ctx = compiler.ctx
1226         self.compiler = compiler
1227         if arch is None:
1228             self.arch = compiler.arch
1229         else:
1230             self.arch = arch
1231         if os_name is None:
1232             self.os = compiler.os
1233         else:
1234             self.os = os_name
1235         self.variant = variant
1236         if variant is None:
1237             self.name = '%s-%s' % (self.arch, self.os)
1238         else:
1239             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1240         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1241         if cfg is None:
1242             self.cfg = []
1243         else:
1244             self.cfg = cfg
1245         self.ccopts = ccopts
1246
1247     def tool_name(self, tool):
1248         """Return the name of a cross-compilation tool."""
1249         ctool = '%s-%s' % (self.compiler.triplet, tool)
1250         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1251             ctool = '%s %s' % (ctool, self.ccopts)
1252         return ctool
1253
1254     def build(self):
1255         """Generate commands to build this glibc."""
1256         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1257         installdir = self.ctx.glibc_installdir(self.name)
1258         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1259         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1260         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1261         cmdlist.add_command('check-compilers',
1262                             ['test', '-f',
1263                              os.path.join(self.compiler.installdir, 'ok')])
1264         cmdlist.use_path(self.compiler.bindir)
1265         self.build_glibc(cmdlist, False)
1266         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1267                                       logsdir)
1268
1269     def build_glibc(self, cmdlist, for_compiler):
1270         """Generate commands to build this glibc, either as part of a compiler
1271         build or with the bootstrapped compiler (and in the latter case, run
1272         tests as well)."""
1273         srcdir = self.ctx.component_srcdir('glibc')
1274         if for_compiler:
1275             builddir = self.ctx.component_builddir('compilers',
1276                                                    self.compiler.name, 'glibc',
1277                                                    self.name)
1278             installdir = self.compiler.sysroot
1279             srcdir_copy = self.ctx.component_builddir('compilers',
1280                                                       self.compiler.name,
1281                                                       'glibc-src',
1282                                                       self.name)
1283         else:
1284             builddir = self.ctx.component_builddir('glibcs', self.name,
1285                                                    'glibc')
1286             installdir = self.ctx.glibc_installdir(self.name)
1287             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1288                                                       'glibc-src')
1289         cmdlist.create_use_dir(builddir)
1290         # glibc builds write into the source directory, and even if
1291         # not intentionally there is a risk of bugs that involve
1292         # writing into the working directory.  To avoid possible
1293         # concurrency issues, copy the source directory.
1294         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1295         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1296                    '--prefix=/usr',
1297                    '--enable-add-ons',
1298                    '--build=%s' % self.ctx.build_triplet,
1299                    '--host=%s' % self.triplet,
1300                    'CC=%s' % self.tool_name('gcc'),
1301                    'CXX=%s' % self.tool_name('g++'),
1302                    'AR=%s' % self.tool_name('ar'),
1303                    'AS=%s' % self.tool_name('as'),
1304                    'LD=%s' % self.tool_name('ld'),
1305                    'NM=%s' % self.tool_name('nm'),
1306                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1307                    'OBJDUMP=%s' % self.tool_name('objdump'),
1308                    'RANLIB=%s' % self.tool_name('ranlib'),
1309                    'READELF=%s' % self.tool_name('readelf'),
1310                    'STRIP=%s' % self.tool_name('strip')]
1311         cfg_cmd += self.cfg
1312         cmdlist.add_command('configure', cfg_cmd)
1313         cmdlist.add_command('build', ['make'])
1314         cmdlist.add_command('install', ['make', 'install',
1315                                         'install_root=%s' % installdir])
1316         # GCC uses paths such as lib/../lib64, so make sure lib
1317         # directories always exist.
1318         cmdlist.add_command('mkdir-lib', ['mkdir', '-p',
1319                                           os.path.join(installdir, 'lib'),
1320                                           os.path.join(installdir,
1321                                                        'usr', 'lib')])
1322         if not for_compiler:
1323             if self.ctx.strip:
1324                 cmdlist.add_command('strip',
1325                                     ['sh', '-c',
1326                                      ('%s %s/lib*/*.so' %
1327                                       (self.tool_name('strip'), installdir))])
1328             cmdlist.add_command('check', ['make', 'check'])
1329             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1330                                 always_run=True)
1331         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1332         cmdlist.cleanup_dir()
1333
1334
1335 class Command(object):
1336     """A command run in the build process."""
1337
1338     def __init__(self, desc, num, dir, path, command, always_run=False):
1339         """Initialize a Command object."""
1340         self.dir = dir
1341         self.path = path
1342         self.desc = desc
1343         trans = str.maketrans({' ': '-'})
1344         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1345         self.command = command
1346         self.always_run = always_run
1347
1348     @staticmethod
1349     def shell_make_quote_string(s):
1350         """Given a string not containing a newline, quote it for use by the
1351         shell and make."""
1352         assert '\n' not in s
1353         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1354             return s
1355         strans = str.maketrans({"'": "'\\''"})
1356         s = "'%s'" % s.translate(strans)
1357         mtrans = str.maketrans({'$': '$$'})
1358         return s.translate(mtrans)
1359
1360     @staticmethod
1361     def shell_make_quote_list(l, translate_make):
1362         """Given a list of strings not containing newlines, quote them for use
1363         by the shell and make, returning a single string.  If translate_make
1364         is true and the first string is 'make', change it to $(MAKE)."""
1365         l = [Command.shell_make_quote_string(s) for s in l]
1366         if translate_make and l[0] == 'make':
1367             l[0] = '$(MAKE)'
1368         return ' '.join(l)
1369
1370     def shell_make_quote(self):
1371         """Return this command quoted for the shell and make."""
1372         return self.shell_make_quote_list(self.command, True)
1373
1374
1375 class CommandList(object):
1376     """A list of commands run in the build process."""
1377
1378     def __init__(self, desc, keep):
1379         """Initialize a CommandList object."""
1380         self.cmdlist = []
1381         self.dir = None
1382         self.path = None
1383         self.desc = [desc]
1384         self.keep = keep
1385
1386     def desc_txt(self, desc):
1387         """Return the description to use for a command."""
1388         return '%s %s' % (' '.join(self.desc), desc)
1389
1390     def use_dir(self, dir):
1391         """Set the default directory for subsequent commands."""
1392         self.dir = dir
1393
1394     def use_path(self, path):
1395         """Set a directory to be prepended to the PATH for subsequent
1396         commands."""
1397         self.path = path
1398
1399     def push_subdesc(self, subdesc):
1400         """Set the default subdescription for subsequent commands (e.g., the
1401         name of a component being built, within the series of commands
1402         building it)."""
1403         self.desc.append(subdesc)
1404
1405     def pop_subdesc(self):
1406         """Pop a subdescription from the list of descriptions."""
1407         self.desc.pop()
1408
1409     def create_use_dir(self, dir):
1410         """Remove and recreate a directory and use it for subsequent
1411         commands."""
1412         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1413         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1414         self.use_dir(dir)
1415
1416     def create_copy_dir(self, src, dest):
1417         """Remove a directory and recreate it as a copy from the given
1418         source."""
1419         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1420         parent = os.path.dirname(dest)
1421         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1422         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1423
1424     def add_command_dir(self, desc, dir, command, always_run=False):
1425         """Add a command to run in a given directory."""
1426         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1427                       command, always_run)
1428         self.cmdlist.append(cmd)
1429
1430     def add_command(self, desc, command, always_run=False):
1431         """Add a command to run in the default directory."""
1432         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1433                       self.path, command, always_run)
1434         self.cmdlist.append(cmd)
1435
1436     def cleanup_dir(self, desc='cleanup', dir=None):
1437         """Clean up a build directory.  If no directory is specified, the
1438         default directory is cleaned up and ceases to be the default
1439         directory."""
1440         if dir is None:
1441             dir = self.dir
1442             self.use_dir(None)
1443         if self.keep != 'all':
1444             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1445                                  always_run=(self.keep == 'none'))
1446
1447     def makefile_commands(self, wrapper, logsdir):
1448         """Return the sequence of commands in the form of text for a Makefile.
1449         The given wrapper script takes arguments: base of logs for
1450         previous command, or empty; base of logs for this command;
1451         description; directory; PATH addition; the command itself."""
1452         # prev_base is the base of the name for logs of the previous
1453         # command that is not always-run (that is, a build command,
1454         # whose failure should stop subsequent build commands from
1455         # being run, as opposed to a cleanup command, which is run
1456         # even if previous commands failed).
1457         prev_base = ''
1458         cmds = []
1459         for c in self.cmdlist:
1460             ctxt = c.shell_make_quote()
1461             if prev_base and not c.always_run:
1462                 prev_log = os.path.join(logsdir, prev_base)
1463             else:
1464                 prev_log = ''
1465             this_log = os.path.join(logsdir, c.logbase)
1466             if not c.always_run:
1467                 prev_base = c.logbase
1468             if c.dir is None:
1469                 dir = ''
1470             else:
1471                 dir = c.dir
1472             if c.path is None:
1473                 path = ''
1474             else:
1475                 path = c.path
1476             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1477             prelim_txt = Command.shell_make_quote_list(prelims, False)
1478             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1479         return '\n'.join(cmds)
1480
1481     def status_logs(self, logsdir):
1482         """Return the list of log files with command status."""
1483         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1484                 for c in self.cmdlist]
1485
1486
1487 def get_parser():
1488     """Return an argument parser for this module."""
1489     parser = argparse.ArgumentParser(description=__doc__)
1490     parser.add_argument('-j', dest='parallelism',
1491                         help='Run this number of jobs in parallel',
1492                         type=int, default=os.cpu_count())
1493     parser.add_argument('--keep', dest='keep',
1494                         help='Whether to keep all build directories, '
1495                         'none or only those from failed builds',
1496                         default='none', choices=('none', 'all', 'failed'))
1497     parser.add_argument('--replace-sources', action='store_true',
1498                         help='Remove and replace source directories '
1499                         'with the wrong version of a component')
1500     parser.add_argument('--strip', action='store_true',
1501                         help='Strip installed glibc libraries')
1502     parser.add_argument('topdir',
1503                         help='Toplevel working directory')
1504     parser.add_argument('action',
1505                         help='What to do',
1506                         choices=('checkout', 'bot-cycle', 'bot',
1507                                  'host-libraries', 'compilers', 'glibcs'))
1508     parser.add_argument('configs',
1509                         help='Versions to check out or configurations to build',
1510                         nargs='*')
1511     return parser
1512
1513
1514 def main(argv):
1515     """The main entry point."""
1516     parser = get_parser()
1517     opts = parser.parse_args(argv)
1518     topdir = os.path.abspath(opts.topdir)
1519     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1520                   opts.strip, opts.action)
1521     ctx.run_builds(opts.action, opts.configs)
1522
1523
1524 if __name__ == '__main__':
1525     main(sys.argv[1:])