scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2018 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52 try:
  53     os.cpu_count
  54 except:
  55     import multiprocessing
  56     os.cpu_count = lambda: multiprocessing.cpu_count()
  57
  58 try:
  59     re.fullmatch
  60 except:
  61     re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)
  62
  63 try:
  64     subprocess.run
  65 except:
  66     class _CompletedProcess:
  67         def __init__(self, args, returncode, stdout=None, stderr=None):
  68             self.args = args
  69             self.returncode = returncode
  70             self.stdout = stdout
  71             self.stderr = stderr
  72
  73     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  74         assert(timeout is None)
  75         with subprocess.Popen(*popenargs, **kwargs) as process:
  76             try:
  77                 stdout, stderr = process.communicate(input)
  78             except:
  79                 process.kill()
  80                 process.wait()
  81                 raise
  82             returncode = process.poll()
  83             if check and returncode:
  84                 raise subprocess.CalledProcessError(returncode, popenargs)
  85         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  86
  87     subprocess.run = _run
  88
  89
  90 class Context(object):
  91     """The global state associated with builds in a given directory."""
  92
  93     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  94                  action):
  95         """Initialize the context."""
  96         self.topdir = topdir
  97         self.parallelism = parallelism
  98         self.keep = keep
  99         self.replace_sources = replace_sources
 100         self.strip = strip
 101         self.srcdir = os.path.join(topdir, 'src')
 102         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 103         self.build_state_json = os.path.join(topdir, 'build-state.json')
 104         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 105         self.installdir = os.path.join(topdir, 'install')
 106         self.host_libraries_installdir = os.path.join(self.installdir,
 107                                                       'host-libraries')
 108         self.builddir = os.path.join(topdir, 'build')
 109         self.logsdir = os.path.join(topdir, 'logs')
 110         self.logsdir_old = os.path.join(topdir, 'logs-old')
 111         self.makefile = os.path.join(self.builddir, 'Makefile')
 112         self.wrapper = os.path.join(self.builddir, 'wrapper')
 113         self.save_logs = os.path.join(self.builddir, 'save-logs')
 114         self.script_text = self.get_script_text()
 115         if action != 'checkout':
 116             self.build_triplet = self.get_build_triplet()
 117             self.glibc_version = self.get_glibc_version()
 118         self.configs = {}
 119         self.glibc_configs = {}
 120         self.makefile_pieces = ['.PHONY: all\n']
 121         self.add_all_configs()
 122         self.load_versions_json()
 123         self.load_build_state_json()
 124         self.status_log_list = []
 125         self.email_warning = False
 126
 127     def get_script_text(self):
 128         """Return the text of this script."""
 129         with open(sys.argv[0], 'r') as f:
 130             return f.read()
 131
 132     def exec_self(self):
 133         """Re-execute this script with the same arguments."""
 134         sys.stdout.flush()
 135         os.execv(sys.executable, [sys.executable] + sys.argv)
 136
 137     def get_build_triplet(self):
 138         """Determine the build triplet with config.guess."""
 139         config_guess = os.path.join(self.component_srcdir('gcc'),
 140                                     'config.guess')
 141         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 142                                 check=True, universal_newlines=True).stdout
 143         return cg_out.rstrip()
 144
 145     def get_glibc_version(self):
 146         """Determine the glibc version number (major.minor)."""
 147         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 148         with open(version_h, 'r') as f:
 149             lines = f.readlines()
 150         starttext = '#define VERSION "'
 151         for l in lines:
 152             if l.startswith(starttext):
 153                 l = l[len(starttext):]
 154                 l = l.rstrip('"\n')
 155                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 156                 return '%s.%s' % m.group(1, 2)
 157         print('error: could not determine glibc version')
 158         exit(1)
 159
 160     def add_all_configs(self):
 161         """Add all known glibc build configurations."""
 162         self.add_config(arch='aarch64',
 163                         os_name='linux-gnu',
 164                         extra_glibcs=[{'variant': 'disable-multi-arch',
 165                                        'cfg': ['--disable-multi-arch']}])
 166         self.add_config(arch='aarch64_be',
 167                         os_name='linux-gnu')
 168         self.add_config(arch='alpha',
 169                         os_name='linux-gnu')
 170         self.add_config(arch='arm',
 171                         os_name='linux-gnueabi')
 172         self.add_config(arch='armeb',
 173                         os_name='linux-gnueabi')
 174         self.add_config(arch='armeb',
 175                         os_name='linux-gnueabi',
 176                         variant='be8',
 177                         gcc_cfg=['--with-arch=armv7-a'])
 178         self.add_config(arch='arm',
 179                         os_name='linux-gnueabihf',
 180                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 181                         extra_glibcs=[{'variant': 'v7a',
 182                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 183                                       {'variant': 'v7a-disable-multi-arch',
 184                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 185                                        'cfg': ['--disable-multi-arch']}])
 186         self.add_config(arch='armeb',
 187                         os_name='linux-gnueabihf',
 188                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 189         self.add_config(arch='armeb',
 190                         os_name='linux-gnueabihf',
 191                         variant='be8',
 192                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 193                                  '--with-fpu=vfpv3'])
 194         self.add_config(arch='hppa',
 195                         os_name='linux-gnu')
 196         self.add_config(arch='ia64',
 197                         os_name='linux-gnu',
 198                         first_gcc_cfg=['--with-system-libunwind'])
 199         self.add_config(arch='m68k',
 200                         os_name='linux-gnu',
 201                         gcc_cfg=['--disable-multilib'])
 202         self.add_config(arch='m68k',
 203                         os_name='linux-gnu',
 204                         variant='coldfire',
 205                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 206         self.add_config(arch='microblaze',
 207                         os_name='linux-gnu',
 208                         gcc_cfg=['--disable-multilib'])
 209         self.add_config(arch='microblazeel',
 210                         os_name='linux-gnu',
 211                         gcc_cfg=['--disable-multilib'])
 212         self.add_config(arch='mips64',
 213                         os_name='linux-gnu',
 214                         gcc_cfg=['--with-mips-plt'],
 215                         glibcs=[{'variant': 'n32'},
 216                                 {'arch': 'mips',
 217                                  'ccopts': '-mabi=32'},
 218                                 {'variant': 'n64',
 219                                  'ccopts': '-mabi=64'}])
 220         self.add_config(arch='mips64',
 221                         os_name='linux-gnu',
 222                         variant='soft',
 223                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 224                         glibcs=[{'variant': 'n32-soft'},
 225                                 {'variant': 'soft',
 226                                  'arch': 'mips',
 227                                  'ccopts': '-mabi=32'},
 228                                 {'variant': 'n64-soft',
 229                                  'ccopts': '-mabi=64'}])
 230         self.add_config(arch='mips64',
 231                         os_name='linux-gnu',
 232                         variant='nan2008',
 233                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 234                                  '--with-arch-64=mips64r2',
 235                                  '--with-arch-32=mips32r2'],
 236                         glibcs=[{'variant': 'n32-nan2008'},
 237                                 {'variant': 'nan2008',
 238                                  'arch': 'mips',
 239                                  'ccopts': '-mabi=32'},
 240                                 {'variant': 'n64-nan2008',
 241                                  'ccopts': '-mabi=64'}])
 242         self.add_config(arch='mips64',
 243                         os_name='linux-gnu',
 244                         variant='nan2008-soft',
 245                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 246                                  '--with-arch-64=mips64r2',
 247                                  '--with-arch-32=mips32r2',
 248                                  '--with-float=soft'],
 249                         glibcs=[{'variant': 'n32-nan2008-soft'},
 250                                 {'variant': 'nan2008-soft',
 251                                  'arch': 'mips',
 252                                  'ccopts': '-mabi=32'},
 253                                 {'variant': 'n64-nan2008-soft',
 254                                  'ccopts': '-mabi=64'}])
 255         self.add_config(arch='mips64el',
 256                         os_name='linux-gnu',
 257                         gcc_cfg=['--with-mips-plt'],
 258                         glibcs=[{'variant': 'n32'},
 259                                 {'arch': 'mipsel',
 260                                  'ccopts': '-mabi=32'},
 261                                 {'variant': 'n64',
 262                                  'ccopts': '-mabi=64'}])
 263         self.add_config(arch='mips64el',
 264                         os_name='linux-gnu',
 265                         variant='soft',
 266                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 267                         glibcs=[{'variant': 'n32-soft'},
 268                                 {'variant': 'soft',
 269                                  'arch': 'mipsel',
 270                                  'ccopts': '-mabi=32'},
 271                                 {'variant': 'n64-soft',
 272                                  'ccopts': '-mabi=64'}])
 273         self.add_config(arch='mips64el',
 274                         os_name='linux-gnu',
 275                         variant='nan2008',
 276                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 277                                  '--with-arch-64=mips64r2',
 278                                  '--with-arch-32=mips32r2'],
 279                         glibcs=[{'variant': 'n32-nan2008'},
 280                                 {'variant': 'nan2008',
 281                                  'arch': 'mipsel',
 282                                  'ccopts': '-mabi=32'},
 283                                 {'variant': 'n64-nan2008',
 284                                  'ccopts': '-mabi=64'}])
 285         self.add_config(arch='mips64el',
 286                         os_name='linux-gnu',
 287                         variant='nan2008-soft',
 288                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 289                                  '--with-arch-64=mips64r2',
 290                                  '--with-arch-32=mips32r2',
 291                                  '--with-float=soft'],
 292                         glibcs=[{'variant': 'n32-nan2008-soft'},
 293                                 {'variant': 'nan2008-soft',
 294                                  'arch': 'mipsel',
 295                                  'ccopts': '-mabi=32'},
 296                                 {'variant': 'n64-nan2008-soft',
 297                                  'ccopts': '-mabi=64'}])
 298         self.add_config(arch='nios2',
 299                         os_name='linux-gnu')
 300         self.add_config(arch='powerpc',
 301                         os_name='linux-gnu',
 302                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 303                         extra_glibcs=[{'variant': 'power4',
 304                                        'ccopts': '-mcpu=power4',
 305                                        'cfg': ['--with-cpu=power4']}])
 306         self.add_config(arch='powerpc',
 307                         os_name='linux-gnu',
 308                         variant='soft',
 309                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 310                                  '--enable-secureplt'])
 311         self.add_config(arch='powerpc64',
 312                         os_name='linux-gnu',
 313                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 314         self.add_config(arch='powerpc64le',
 315                         os_name='linux-gnu',
 316                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 317         self.add_config(arch='powerpc',
 318                         os_name='linux-gnuspe',
 319                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 320                                  '--enable-e500-double'])
 321         self.add_config(arch='powerpc',
 322                         os_name='linux-gnuspe',
 323                         variant='e500v1',
 324                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 325         self.add_config(arch='s390x',
 326                         os_name='linux-gnu',
 327                         glibcs=[{},
 328                                 {'arch': 's390', 'ccopts': '-m31'}])
 329         self.add_config(arch='sh3',
 330                         os_name='linux-gnu')
 331         self.add_config(arch='sh3eb',
 332                         os_name='linux-gnu')
 333         self.add_config(arch='sh4',
 334                         os_name='linux-gnu')
 335         self.add_config(arch='sh4eb',
 336                         os_name='linux-gnu')
 337         self.add_config(arch='sh4',
 338                         os_name='linux-gnu',
 339                         variant='soft',
 340                         gcc_cfg=['--without-fp'])
 341         self.add_config(arch='sh4eb',
 342                         os_name='linux-gnu',
 343                         variant='soft',
 344                         gcc_cfg=['--without-fp'])
 345         self.add_config(arch='sparc64',
 346                         os_name='linux-gnu',
 347                         glibcs=[{},
 348                                 {'arch': 'sparcv9',
 349                                  'ccopts': '-m32 -mlong-double-128'}],
 350                         extra_glibcs=[{'variant': 'disable-multi-arch',
 351                                        'cfg': ['--disable-multi-arch']},
 352                                       {'variant': 'disable-multi-arch',
 353                                        'arch': 'sparcv9',
 354                                        'ccopts': '-m32 -mlong-double-128',
 355                                        'cfg': ['--disable-multi-arch']}])
 356         self.add_config(arch='tilegx',
 357                         os_name='linux-gnu',
 358                         glibcs=[{},
 359                                 {'variant': '32', 'ccopts': '-m32'}])
 360         self.add_config(arch='tilegxbe',
 361                         os_name='linux-gnu',
 362                         glibcs=[{},
 363                                 {'variant': '32', 'ccopts': '-m32'}])
 364         self.add_config(arch='x86_64',
 365                         os_name='linux-gnu',
 366                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 367                         glibcs=[{},
 368                                 {'variant': 'x32', 'ccopts': '-mx32'},
 369                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 370                         extra_glibcs=[{'variant': 'disable-multi-arch',
 371                                        'cfg': ['--disable-multi-arch']},
 372                                       {'variant': 'static-pie',
 373                                        'cfg': ['--enable-static-pie']},
 374                                       {'variant': 'x32-static-pie',
 375                                        'ccopts': '-mx32',
 376                                        'cfg': ['--enable-static-pie']},
 377                                       {'variant': 'static-pie',
 378                                        'arch': 'i686',
 379                                        'ccopts': '-m32 -march=i686',
 380                                        'cfg': ['--enable-static-pie']},
 381                                       {'variant': 'disable-multi-arch',
 382                                        'arch': 'i686',
 383                                        'ccopts': '-m32 -march=i686',
 384                                        'cfg': ['--disable-multi-arch']},
 385                                       {'arch': 'i486',
 386                                        'ccopts': '-m32 -march=i486'},
 387                                       {'arch': 'i586',
 388                                        'ccopts': '-m32 -march=i586'}])
 389
 390     def add_config(self, **args):
 391         """Add an individual build configuration."""
 392         cfg = Config(self, **args)
 393         if cfg.name in self.configs:
 394             print('error: duplicate config %s' % cfg.name)
 395             exit(1)
 396         self.configs[cfg.name] = cfg
 397         for c in cfg.all_glibcs:
 398             if c.name in self.glibc_configs:
 399                 print('error: duplicate glibc config %s' % c.name)
 400                 exit(1)
 401             self.glibc_configs[c.name] = c
 402
 403     def component_srcdir(self, component):
 404         """Return the source directory for a given component, e.g. gcc."""
 405         return os.path.join(self.srcdir, component)
 406
 407     def component_builddir(self, action, config, component, subconfig=None):
 408         """Return the directory to use for a build."""
 409         if config is None:
 410             # Host libraries.
 411             assert subconfig is None
 412             return os.path.join(self.builddir, action, component)
 413         if subconfig is None:
 414             return os.path.join(self.builddir, action, config, component)
 415         else:
 416             # glibc build as part of compiler build.
 417             return os.path.join(self.builddir, action, config, component,
 418                                 subconfig)
 419
 420     def compiler_installdir(self, config):
 421         """Return the directory in which to install a compiler."""
 422         return os.path.join(self.installdir, 'compilers', config)
 423
 424     def compiler_bindir(self, config):
 425         """Return the directory in which to find compiler binaries."""
 426         return os.path.join(self.compiler_installdir(config), 'bin')
 427
 428     def compiler_sysroot(self, config):
 429         """Return the sysroot directory for a compiler."""
 430         return os.path.join(self.compiler_installdir(config), 'sysroot')
 431
 432     def glibc_installdir(self, config):
 433         """Return the directory in which to install glibc."""
 434         return os.path.join(self.installdir, 'glibcs', config)
 435
 436     def run_builds(self, action, configs):
 437         """Run the requested builds."""
 438         if action == 'checkout':
 439             self.checkout(configs)
 440             return
 441         if action == 'bot-cycle':
 442             if configs:
 443                 print('error: configurations specified for bot-cycle')
 444                 exit(1)
 445             self.bot_cycle()
 446             return
 447         if action == 'bot':
 448             if configs:
 449                 print('error: configurations specified for bot')
 450                 exit(1)
 451             self.bot()
 452             return
 453         if action == 'host-libraries' and configs:
 454             print('error: configurations specified for host-libraries')
 455             exit(1)
 456         self.clear_last_build_state(action)
 457         build_time = datetime.datetime.utcnow()
 458         if action == 'host-libraries':
 459             build_components = ('gmp', 'mpfr', 'mpc')
 460             old_components = ()
 461             old_versions = {}
 462             self.build_host_libraries()
 463         elif action == 'compilers':
 464             build_components = ('binutils', 'gcc', 'glibc', 'linux')
 465             old_components = ('gmp', 'mpfr', 'mpc')
 466             old_versions = self.build_state['host-libraries']['build-versions']
 467             self.build_compilers(configs)
 468         else:
 469             build_components = ('glibc',)
 470             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
 471             old_versions = self.build_state['compilers']['build-versions']
 472             self.build_glibcs(configs)
 473         self.write_files()
 474         self.do_build()
 475         if configs:
 476             # Partial build, do not update stored state.
 477             return
 478         build_versions = {}
 479         for k in build_components:
 480             if k in self.versions:
 481                 build_versions[k] = {'version': self.versions[k]['version'],
 482                                      'revision': self.versions[k]['revision']}
 483         for k in old_components:
 484             if k in old_versions:
 485                 build_versions[k] = {'version': old_versions[k]['version'],
 486                                      'revision': old_versions[k]['revision']}
 487         self.update_build_state(action, build_time, build_versions)
 488
 489     @staticmethod
 490     def remove_dirs(*args):
 491         """Remove directories and their contents if they exist."""
 492         for dir in args:
 493             shutil.rmtree(dir, ignore_errors=True)
 494
 495     @staticmethod
 496     def remove_recreate_dirs(*args):
 497         """Remove directories if they exist, and create them as empty."""
 498         Context.remove_dirs(*args)
 499         for dir in args:
 500             os.makedirs(dir, exist_ok=True)
 501
 502     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 503         """Add makefile text for a list of commands."""
 504         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 505         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 506                                     (target, target, target, commands))
 507         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 508
 509     def write_files(self):
 510         """Write out the Makefile and wrapper script."""
 511         mftext = ''.join(self.makefile_pieces)
 512         with open(self.makefile, 'w') as f:
 513             f.write(mftext)
 514         wrapper_text = (
 515             '#!/bin/sh\n'
 516             'prev_base=$1\n'
 517             'this_base=$2\n'
 518             'desc=$3\n'
 519             'dir=$4\n'
 520             'path=$5\n'
 521             'shift 5\n'
 522             'prev_status=$prev_base-status.txt\n'
 523             'this_status=$this_base-status.txt\n'
 524             'this_log=$this_base-log.txt\n'
 525             'date > "$this_log"\n'
 526             'echo >> "$this_log"\n'
 527             'echo "Description: $desc" >> "$this_log"\n'
 528             'printf "%s" "Command:" >> "$this_log"\n'
 529             'for word in "$@"; do\n'
 530             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 531             '    printf " %s" "$word"\n'
 532             '  else\n'
 533             '    printf " \'"\n'
 534             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 535             '    printf "\'"\n'
 536             '  fi\n'
 537             'done >> "$this_log"\n'
 538             'echo >> "$this_log"\n'
 539             'echo "Directory: $dir" >> "$this_log"\n'
 540             'echo "Path addition: $path" >> "$this_log"\n'
 541             'echo >> "$this_log"\n'
 542             'record_status ()\n'
 543             '{\n'
 544             '  echo >> "$this_log"\n'
 545             '  echo "$1: $desc" > "$this_status"\n'
 546             '  echo "$1: $desc" >> "$this_log"\n'
 547             '  echo >> "$this_log"\n'
 548             '  date >> "$this_log"\n'
 549             '  echo "$1: $desc"\n'
 550             '  exit 0\n'
 551             '}\n'
 552             'check_error ()\n'
 553             '{\n'
 554             '  if [ "$1" != "0" ]; then\n'
 555             '    record_status FAIL\n'
 556             '  fi\n'
 557             '}\n'
 558             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 559             '    record_status UNRESOLVED\n'
 560             'fi\n'
 561             'if [ "$dir" ]; then\n'
 562             '  cd "$dir"\n'
 563             '  check_error "$?"\n'
 564             'fi\n'
 565             'if [ "$path" ]; then\n'
 566             '  PATH=$path:$PATH\n'
 567             'fi\n'
 568             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 569             'check_error "$?"\n'
 570             'record_status PASS\n')
 571         with open(self.wrapper, 'w') as f:
 572             f.write(wrapper_text)
 573         # Mode 0o755.
 574         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 575                      stat.S_IROTH|stat.S_IXOTH)
 576         os.chmod(self.wrapper, mode_exec)
 577         save_logs_text = (
 578             '#!/bin/sh\n'
 579             'if ! [ -f tests.sum ]; then\n'
 580             '  echo "No test summary available."\n'
 581             '  exit 0\n'
 582             'fi\n'
 583             'save_file ()\n'
 584             '{\n'
 585             '  echo "Contents of $1:"\n'
 586             '  echo\n'
 587             '  cat "$1"\n'
 588             '  echo\n'
 589             '  echo "End of contents of $1."\n'
 590             '  echo\n'
 591             '}\n'
 592             'save_file tests.sum\n'
 593             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 594             'for t in $non_pass_tests; do\n'
 595             '  if [ -f "$t.out" ]; then\n'
 596             '    save_file "$t.out"\n'
 597             '  fi\n'
 598             'done\n')
 599         with open(self.save_logs, 'w') as f:
 600             f.write(save_logs_text)
 601         os.chmod(self.save_logs, mode_exec)
 602
 603     def do_build(self):
 604         """Do the actual build."""
 605         cmd = ['make', '-j%d' % self.parallelism]
 606         subprocess.run(cmd, cwd=self.builddir, check=True)
 607
 608     def build_host_libraries(self):
 609         """Build the host libraries."""
 610         installdir = self.host_libraries_installdir
 611         builddir = os.path.join(self.builddir, 'host-libraries')
 612         logsdir = os.path.join(self.logsdir, 'host-libraries')
 613         self.remove_recreate_dirs(installdir, builddir, logsdir)
 614         cmdlist = CommandList('host-libraries', self.keep)
 615         self.build_host_library(cmdlist, 'gmp')
 616         self.build_host_library(cmdlist, 'mpfr',
 617                                 ['--with-gmp=%s' % installdir])
 618         self.build_host_library(cmdlist, 'mpc',
 619                                 ['--with-gmp=%s' % installdir,
 620                                 '--with-mpfr=%s' % installdir])
 621         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 622         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 623
 624     def build_host_library(self, cmdlist, lib, extra_opts=None):
 625         """Build one host library."""
 626         srcdir = self.component_srcdir(lib)
 627         builddir = self.component_builddir('host-libraries', None, lib)
 628         installdir = self.host_libraries_installdir
 629         cmdlist.push_subdesc(lib)
 630         cmdlist.create_use_dir(builddir)
 631         cfg_cmd = [os.path.join(srcdir, 'configure'),
 632                    '--prefix=%s' % installdir,
 633                    '--disable-shared']
 634         if extra_opts:
 635             cfg_cmd.extend (extra_opts)
 636         cmdlist.add_command('configure', cfg_cmd)
 637         cmdlist.add_command('build', ['make'])
 638         cmdlist.add_command('check', ['make', 'check'])
 639         cmdlist.add_command('install', ['make', 'install'])
 640         cmdlist.cleanup_dir()
 641         cmdlist.pop_subdesc()
 642
 643     def build_compilers(self, configs):
 644         """Build the compilers."""
 645         if not configs:
 646             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 647             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 648             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 649             configs = sorted(self.configs.keys())
 650         for c in configs:
 651             self.configs[c].build()
 652
 653     def build_glibcs(self, configs):
 654         """Build the glibcs."""
 655         if not configs:
 656             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 657             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 658             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 659             configs = sorted(self.glibc_configs.keys())
 660         for c in configs:
 661             self.glibc_configs[c].build()
 662
 663     def load_versions_json(self):
 664         """Load information about source directory versions."""
 665         if not os.access(self.versions_json, os.F_OK):
 666             self.versions = {}
 667             return
 668         with open(self.versions_json, 'r') as f:
 669             self.versions = json.load(f)
 670
 671     def store_json(self, data, filename):
 672         """Store information in a JSON file."""
 673         filename_tmp = filename + '.tmp'
 674         with open(filename_tmp, 'w') as f:
 675             json.dump(data, f, indent=2, sort_keys=True)
 676         os.rename(filename_tmp, filename)
 677
 678     def store_versions_json(self):
 679         """Store information about source directory versions."""
 680         self.store_json(self.versions, self.versions_json)
 681
 682     def set_component_version(self, component, version, explicit, revision):
 683         """Set the version information for a component."""
 684         self.versions[component] = {'version': version,
 685                                     'explicit': explicit,
 686                                     'revision': revision}
 687         self.store_versions_json()
 688
 689     def checkout(self, versions):
 690         """Check out the desired component versions."""
 691         default_versions = {'binutils': 'vcs-2.29',
 692                             'gcc': 'vcs-7',
 693                             'glibc': 'vcs-mainline',
 694                             'gmp': '6.1.2',
 695                             'linux': '4.14',
 696                             'mpc': '1.0.3',
 697                             'mpfr': '3.1.6'}
 698         use_versions = {}
 699         explicit_versions = {}
 700         for v in versions:
 701             found_v = False
 702             for k in default_versions.keys():
 703                 kx = k + '-'
 704                 if v.startswith(kx):
 705                     vx = v[len(kx):]
 706                     if k in use_versions:
 707                         print('error: multiple versions for %s' % k)
 708                         exit(1)
 709                     use_versions[k] = vx
 710                     explicit_versions[k] = True
 711                     found_v = True
 712                     break
 713             if not found_v:
 714                 print('error: unknown component in %s' % v)
 715                 exit(1)
 716         for k in default_versions.keys():
 717             if k not in use_versions:
 718                 if k in self.versions and self.versions[k]['explicit']:
 719                     use_versions[k] = self.versions[k]['version']
 720                     explicit_versions[k] = True
 721                 else:
 722                     use_versions[k] = default_versions[k]
 723                     explicit_versions[k] = False
 724         os.makedirs(self.srcdir, exist_ok=True)
 725         for k in sorted(default_versions.keys()):
 726             update = os.access(self.component_srcdir(k), os.F_OK)
 727             v = use_versions[k]
 728             if (update and
 729                 k in self.versions and
 730                 v != self.versions[k]['version']):
 731                 if not self.replace_sources:
 732                     print('error: version of %s has changed from %s to %s, '
 733                           'use --replace-sources to check out again' %
 734                           (k, self.versions[k]['version'], v))
 735                     exit(1)
 736                 shutil.rmtree(self.component_srcdir(k))
 737                 update = False
 738             if v.startswith('vcs-'):
 739                 revision = self.checkout_vcs(k, v[4:], update)
 740             else:
 741                 self.checkout_tar(k, v, update)
 742                 revision = v
 743             self.set_component_version(k, v, explicit_versions[k], revision)
 744         if self.get_script_text() != self.script_text:
 745             # Rerun the checkout process in case the updated script
 746             # uses different default versions or new components.
 747             self.exec_self()
 748
 749     def checkout_vcs(self, component, version, update):
 750         """Check out the given version of the given component from version
 751         control.  Return a revision identifier."""
 752         if component == 'binutils':
 753             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 754             if version == 'mainline':
 755                 git_branch = 'master'
 756             else:
 757                 trans = str.maketrans({'.': '_'})
 758                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 759             return self.git_checkout(component, git_url, git_branch, update)
 760         elif component == 'gcc':
 761             if version == 'mainline':
 762                 branch = 'trunk'
 763             else:
 764                 trans = str.maketrans({'.': '_'})
 765                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 766             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 767             return self.gcc_checkout(svn_url, update)
 768         elif component == 'glibc':
 769             git_url = 'git://sourceware.org/git/glibc.git'
 770             if version == 'mainline':
 771                 git_branch = 'master'
 772             else:
 773                 git_branch = 'release/%s/master' % version
 774             r = self.git_checkout(component, git_url, git_branch, update)
 775             self.fix_glibc_timestamps()
 776             return r
 777         else:
 778             print('error: component %s coming from VCS' % component)
 779             exit(1)
 780
 781     def git_checkout(self, component, git_url, git_branch, update):
 782         """Check out a component from git.  Return a commit identifier."""
 783         if update:
 784             subprocess.run(['git', 'remote', 'prune', 'origin'],
 785                            cwd=self.component_srcdir(component), check=True)
 786             subprocess.run(['git', 'pull', '-q'],
 787                            cwd=self.component_srcdir(component), check=True)
 788         else:
 789             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 790                             self.component_srcdir(component)], check=True)
 791         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 792                            cwd=self.component_srcdir(component),
 793                            stdout=subprocess.PIPE,
 794                            check=True, universal_newlines=True).stdout
 795         return r.rstrip()
 796
 797     def fix_glibc_timestamps(self):
 798         """Fix timestamps in a glibc checkout."""
 799         # Ensure that builds do not try to regenerate generated files
 800         # in the source tree.
 801         srcdir = self.component_srcdir('glibc')
 802         for dirpath, dirnames, filenames in os.walk(srcdir):
 803             for f in filenames:
 804                 if (f == 'configure' or
 805                     f == 'preconfigure' or
 806                     f.endswith('-kw.h')):
 807                     to_touch = os.path.join(dirpath, f)
 808                     subprocess.run(['touch', to_touch], check=True)
 809
 810     def gcc_checkout(self, svn_url, update):
 811         """Check out GCC from SVN.  Return the revision number."""
 812         if not update:
 813             subprocess.run(['svn', 'co', '-q', svn_url,
 814                             self.component_srcdir('gcc')], check=True)
 815         subprocess.run(['contrib/gcc_update', '--silent'],
 816                        cwd=self.component_srcdir('gcc'), check=True)
 817         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 818                            stdout=subprocess.PIPE,
 819                            check=True, universal_newlines=True).stdout
 820         return r.rstrip()
 821
 822     def checkout_tar(self, component, version, update):
 823         """Check out the given version of the given component from a
 824         tarball."""
 825         if update:
 826             return
 827         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 828                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 829                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 830                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 831                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 832                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
 833         if component not in url_map:
 834             print('error: component %s coming from tarball' % component)
 835             exit(1)
 836         url = url_map[component] % {'version': version}
 837         filename = os.path.join(self.srcdir, url.split('/')[-1])
 838         response = urllib.request.urlopen(url)
 839         data = response.read()
 840         with open(filename, 'wb') as f:
 841             f.write(data)
 842         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 843                        check=True)
 844         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 845                   self.component_srcdir(component))
 846         os.remove(filename)
 847
 848     def load_build_state_json(self):
 849         """Load information about the state of previous builds."""
 850         if os.access(self.build_state_json, os.F_OK):
 851             with open(self.build_state_json, 'r') as f:
 852                 self.build_state = json.load(f)
 853         else:
 854             self.build_state = {}
 855         for k in ('host-libraries', 'compilers', 'glibcs'):
 856             if k not in self.build_state:
 857                 self.build_state[k] = {}
 858             if 'build-time' not in self.build_state[k]:
 859                 self.build_state[k]['build-time'] = ''
 860             if 'build-versions' not in self.build_state[k]:
 861                 self.build_state[k]['build-versions'] = {}
 862             if 'build-results' not in self.build_state[k]:
 863                 self.build_state[k]['build-results'] = {}
 864             if 'result-changes' not in self.build_state[k]:
 865                 self.build_state[k]['result-changes'] = {}
 866             if 'ever-passed' not in self.build_state[k]:
 867                 self.build_state[k]['ever-passed'] = []
 868
 869     def store_build_state_json(self):
 870         """Store information about the state of previous builds."""
 871         self.store_json(self.build_state, self.build_state_json)
 872
 873     def clear_last_build_state(self, action):
 874         """Clear information about the state of part of the build."""
 875         # We clear the last build time and versions when starting a
 876         # new build.  The results of the last build are kept around,
 877         # as comparison is still meaningful if this build is aborted
 878         # and a new one started.
 879         self.build_state[action]['build-time'] = ''
 880         self.build_state[action]['build-versions'] = {}
 881         self.store_build_state_json()
 882
 883     def update_build_state(self, action, build_time, build_versions):
 884         """Update the build state after a build."""
 885         build_time = build_time.replace(microsecond=0)
 886         self.build_state[action]['build-time'] = str(build_time)
 887         self.build_state[action]['build-versions'] = build_versions
 888         build_results = {}
 889         for log in self.status_log_list:
 890             with open(log, 'r') as f:
 891                 log_text = f.read()
 892             log_text = log_text.rstrip()
 893             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 894             result = m.group(1)
 895             test_name = m.group(2)
 896             assert test_name not in build_results
 897             build_results[test_name] = result
 898         old_build_results = self.build_state[action]['build-results']
 899         self.build_state[action]['build-results'] = build_results
 900         result_changes = {}
 901         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 902         for t in all_tests:
 903             if t in old_build_results:
 904                 old_res = old_build_results[t]
 905             else:
 906                 old_res = '(New test)'
 907             if t in build_results:
 908                 new_res = build_results[t]
 909             else:
 910                 new_res = '(Test removed)'
 911             if old_res != new_res:
 912                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 913         self.build_state[action]['result-changes'] = result_changes
 914         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 915                            if t in build_results}
 916         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 917         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 918                                                          new_passes)
 919         self.store_build_state_json()
 920
 921     def load_bot_config_json(self):
 922         """Load bot configuration."""
 923         with open(self.bot_config_json, 'r') as f:
 924             self.bot_config = json.load(f)
 925
 926     def part_build_old(self, action, delay):
 927         """Return whether the last build for a given action was at least a
 928         given number of seconds ago, or does not have a time recorded."""
 929         old_time_str = self.build_state[action]['build-time']
 930         if not old_time_str:
 931             return True
 932         old_time = datetime.datetime.strptime(old_time_str,
 933                                               '%Y-%m-%d %H:%M:%S')
 934         new_time = datetime.datetime.utcnow()
 935         delta = new_time - old_time
 936         return delta.total_seconds() >= delay
 937
 938     def bot_cycle(self):
 939         """Run a single round of checkout and builds."""
 940         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 941         self.load_bot_config_json()
 942         actions = ('host-libraries', 'compilers', 'glibcs')
 943         self.bot_run_self(['--replace-sources'], 'checkout')
 944         self.load_versions_json()
 945         if self.get_script_text() != self.script_text:
 946             print('Script changed, re-execing.')
 947             # On script change, all parts of the build should be rerun.
 948             for a in actions:
 949                 self.clear_last_build_state(a)
 950             self.exec_self()
 951         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 952                             'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
 953                             'glibcs': ('glibc',)}
 954         must_build = {}
 955         for a in actions:
 956             build_vers = self.build_state[a]['build-versions']
 957             must_build[a] = False
 958             if not self.build_state[a]['build-time']:
 959                 must_build[a] = True
 960             old_vers = {}
 961             new_vers = {}
 962             for c in check_components[a]:
 963                 if c in build_vers:
 964                     old_vers[c] = build_vers[c]
 965                 new_vers[c] = {'version': self.versions[c]['version'],
 966                                'revision': self.versions[c]['revision']}
 967             if new_vers == old_vers:
 968                 print('Versions for %s unchanged.' % a)
 969             else:
 970                 print('Versions changed or rebuild forced for %s.' % a)
 971                 if a == 'compilers' and not self.part_build_old(
 972                         a, self.bot_config['compilers-rebuild-delay']):
 973                     print('Not requiring rebuild of compilers this soon.')
 974                 else:
 975                     must_build[a] = True
 976         if must_build['host-libraries']:
 977             must_build['compilers'] = True
 978         if must_build['compilers']:
 979             must_build['glibcs'] = True
 980         for a in actions:
 981             if must_build[a]:
 982                 print('Must rebuild %s.' % a)
 983                 self.clear_last_build_state(a)
 984             else:
 985                 print('No need to rebuild %s.' % a)
 986         if os.access(self.logsdir, os.F_OK):
 987             shutil.rmtree(self.logsdir_old, ignore_errors=True)
 988             shutil.copytree(self.logsdir, self.logsdir_old)
 989         for a in actions:
 990             if must_build[a]:
 991                 build_time = datetime.datetime.utcnow()
 992                 print('Rebuilding %s at %s.' % (a, str(build_time)))
 993                 self.bot_run_self([], a)
 994                 self.load_build_state_json()
 995                 self.bot_build_mail(a, build_time)
 996         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
 997
 998     def bot_build_mail(self, action, build_time):
 999         """Send email with the results of a build."""
1000         if not ('email-from' in self.bot_config and
1001                 'email-server' in self.bot_config and
1002                 'email-subject' in self.bot_config and
1003                 'email-to' in self.bot_config):
1004             if not self.email_warning:
1005                 print("Email not configured, not sending.")
1006                 self.email_warning = True
1007             return
1008
1009         build_time = build_time.replace(microsecond=0)
1010         subject = (self.bot_config['email-subject'] %
1011                    {'action': action,
1012                     'build-time': str(build_time)})
1013         results = self.build_state[action]['build-results']
1014         changes = self.build_state[action]['result-changes']
1015         ever_passed = set(self.build_state[action]['ever-passed'])
1016         versions = self.build_state[action]['build-versions']
1017         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1018         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1019         all_fails = {k for k in results if results[k] == 'FAIL'}
1020         if new_regressions:
1021             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1022             new_reg_text = ('New regressions:\n\n%s\n\n' %
1023                             '\n'.join(new_reg_list))
1024         else:
1025             new_reg_text = ''
1026         if all_regressions:
1027             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1028             all_reg_text = ('All regressions:\n\n%s\n\n' %
1029                             '\n'.join(all_reg_list))
1030         else:
1031             all_reg_text = ''
1032         if all_fails:
1033             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1034             all_fail_text = ('All failures:\n\n%s\n\n' %
1035                              '\n'.join(all_fail_list))
1036         else:
1037             all_fail_text = ''
1038         if changes:
1039             changes_list = sorted(changes.keys())
1040             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1041             changes_text = ('All changed results:\n\n%s\n\n' %
1042                             '\n'.join(changes_list))
1043         else:
1044             changes_text = ''
1045         results_text = (new_reg_text + all_reg_text + all_fail_text +
1046                         changes_text)
1047         if not results_text:
1048             results_text = 'Clean build with unchanged results.\n\n'
1049         versions_list = sorted(versions.keys())
1050         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1051                                           versions[k]['revision'])
1052                          for k in versions_list]
1053         versions_text = ('Component versions for this build:\n\n%s\n' %
1054                          '\n'.join(versions_list))
1055         body_text = results_text + versions_text
1056         msg = email.mime.text.MIMEText(body_text)
1057         msg['Subject'] = subject
1058         msg['From'] = self.bot_config['email-from']
1059         msg['To'] = self.bot_config['email-to']
1060         msg['Message-ID'] = email.utils.make_msgid()
1061         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1062         with smtplib.SMTP(self.bot_config['email-server']) as s:
1063             s.send_message(msg)
1064
1065     def bot_run_self(self, opts, action, check=True):
1066         """Run a copy of this script with given options."""
1067         cmd = [sys.executable, sys.argv[0], '--keep=none',
1068                '-j%d' % self.parallelism]
1069         cmd.extend(opts)
1070         cmd.extend([self.topdir, action])
1071         sys.stdout.flush()
1072         subprocess.run(cmd, check=check)
1073
1074     def bot(self):
1075         """Run repeated rounds of checkout and builds."""
1076         while True:
1077             self.load_bot_config_json()
1078             if not self.bot_config['run']:
1079                 print('Bot exiting by request.')
1080                 exit(0)
1081             self.bot_run_self([], 'bot-cycle', check=False)
1082             self.load_bot_config_json()
1083             if not self.bot_config['run']:
1084                 print('Bot exiting by request.')
1085                 exit(0)
1086             time.sleep(self.bot_config['delay'])
1087             if self.get_script_text() != self.script_text:
1088                 print('Script changed, bot re-execing.')
1089                 self.exec_self()
1090
1091
1092 class Config(object):
1093     """A configuration for building a compiler and associated libraries."""
1094
1095     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1096                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1097         """Initialize a Config object."""
1098         self.ctx = ctx
1099         self.arch = arch
1100         self.os = os_name
1101         self.variant = variant
1102         if variant is None:
1103             self.name = '%s-%s' % (arch, os_name)
1104         else:
1105             self.name = '%s-%s-%s' % (arch, os_name, variant)
1106         self.triplet = '%s-glibc-%s' % (arch, os_name)
1107         if gcc_cfg is None:
1108             self.gcc_cfg = []
1109         else:
1110             self.gcc_cfg = gcc_cfg
1111         if first_gcc_cfg is None:
1112             self.first_gcc_cfg = []
1113         else:
1114             self.first_gcc_cfg = first_gcc_cfg
1115         if glibcs is None:
1116             glibcs = [{'variant': variant}]
1117         if extra_glibcs is None:
1118             extra_glibcs = []
1119         glibcs = [Glibc(self, **g) for g in glibcs]
1120         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1121         self.all_glibcs = glibcs + extra_glibcs
1122         self.compiler_glibcs = glibcs
1123         self.installdir = ctx.compiler_installdir(self.name)
1124         self.bindir = ctx.compiler_bindir(self.name)
1125         self.sysroot = ctx.compiler_sysroot(self.name)
1126         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1127         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1128
1129     def component_builddir(self, component):
1130         """Return the directory to use for a (non-glibc) build."""
1131         return self.ctx.component_builddir('compilers', self.name, component)
1132
1133     def build(self):
1134         """Generate commands to build this compiler."""
1135         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1136                                       self.logsdir)
1137         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1138         cmdlist.add_command('check-host-libraries',
1139                             ['test', '-f',
1140                              os.path.join(self.ctx.host_libraries_installdir,
1141                                           'ok')])
1142         cmdlist.use_path(self.bindir)
1143         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1144                               ['--disable-gdb',
1145                                '--disable-libdecnumber',
1146                                '--disable-readline',
1147                                '--disable-sim'])
1148         if self.os.startswith('linux'):
1149             self.install_linux_headers(cmdlist)
1150         self.build_gcc(cmdlist, True)
1151         for g in self.compiler_glibcs:
1152             cmdlist.push_subdesc('glibc')
1153             cmdlist.push_subdesc(g.name)
1154             g.build_glibc(cmdlist, True)
1155             cmdlist.pop_subdesc()
1156             cmdlist.pop_subdesc()
1157         self.build_gcc(cmdlist, False)
1158         cmdlist.add_command('done', ['touch',
1159                                      os.path.join(self.installdir, 'ok')])
1160         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1161                                       self.logsdir)
1162
1163     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1164         """Build one cross tool."""
1165         srcdir = self.ctx.component_srcdir(tool_src)
1166         builddir = self.component_builddir(tool_build)
1167         cmdlist.push_subdesc(tool_build)
1168         cmdlist.create_use_dir(builddir)
1169         cfg_cmd = [os.path.join(srcdir, 'configure'),
1170                    '--prefix=%s' % self.installdir,
1171                    '--build=%s' % self.ctx.build_triplet,
1172                    '--host=%s' % self.ctx.build_triplet,
1173                    '--target=%s' % self.triplet,
1174                    '--with-sysroot=%s' % self.sysroot]
1175         if extra_opts:
1176             cfg_cmd.extend(extra_opts)
1177         cmdlist.add_command('configure', cfg_cmd)
1178         cmdlist.add_command('build', ['make'])
1179         # Parallel "make install" for GCC has race conditions that can
1180         # cause it to fail; see
1181         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1182         # problems are not known for binutils, but doing the
1183         # installation in parallel within a particular toolchain build
1184         # (as opposed to installation of one toolchain from
1185         # build-many-glibcs.py running in parallel to the installation
1186         # of other toolchains being built) is not known to be
1187         # significantly beneficial, so it is simplest just to disable
1188         # parallel install for cross tools here.
1189         cmdlist.add_command('install', ['make', '-j1', 'install'])
1190         cmdlist.cleanup_dir()
1191         cmdlist.pop_subdesc()
1192
1193     def install_linux_headers(self, cmdlist):
1194         """Install Linux kernel headers."""
1195         arch_map = {'aarch64': 'arm64',
1196                     'alpha': 'alpha',
1197                     'arm': 'arm',
1198                     'hppa': 'parisc',
1199                     'i486': 'x86',
1200                     'i586': 'x86',
1201                     'i686': 'x86',
1202                     'i786': 'x86',
1203                     'ia64': 'ia64',
1204                     'm68k': 'm68k',
1205                     'microblaze': 'microblaze',
1206                     'mips': 'mips',
1207                     'nios2': 'nios2',
1208                     'powerpc': 'powerpc',
1209                     's390': 's390',
1210                     'sh': 'sh',
1211                     'sparc': 'sparc',
1212                     'tile': 'tile',
1213                     'x86_64': 'x86'}
1214         linux_arch = None
1215         for k in arch_map:
1216             if self.arch.startswith(k):
1217                 linux_arch = arch_map[k]
1218                 break
1219         assert linux_arch is not None
1220         srcdir = self.ctx.component_srcdir('linux')
1221         builddir = self.component_builddir('linux')
1222         headers_dir = os.path.join(self.sysroot, 'usr')
1223         cmdlist.push_subdesc('linux')
1224         cmdlist.create_use_dir(builddir)
1225         cmdlist.add_command('install-headers',
1226                             ['make', '-C', srcdir, 'O=%s' % builddir,
1227                              'ARCH=%s' % linux_arch,
1228                              'INSTALL_HDR_PATH=%s' % headers_dir,
1229                              'headers_install'])
1230         cmdlist.cleanup_dir()
1231         cmdlist.pop_subdesc()
1232
1233     def build_gcc(self, cmdlist, bootstrap):
1234         """Build GCC."""
1235         # libsanitizer commonly breaks because of glibc header
1236         # changes, or on unusual targets.  libssp is of little
1237         # relevance with glibc's own stack checking support.
1238         cfg_opts = list(self.gcc_cfg)
1239         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1240         host_libs = self.ctx.host_libraries_installdir
1241         cfg_opts += ['--with-gmp=%s' % host_libs,
1242                      '--with-mpfr=%s' % host_libs,
1243                      '--with-mpc=%s' % host_libs]
1244         if bootstrap:
1245             tool_build = 'gcc-first'
1246             # Building a static-only, C-only compiler that is
1247             # sufficient to build glibc.  Various libraries and
1248             # features that may require libc headers must be disabled.
1249             # When configuring with a sysroot, --with-newlib is
1250             # required to define inhibit_libc (to stop some parts of
1251             # libgcc including libc headers); --without-headers is not
1252             # sufficient.
1253             cfg_opts += ['--enable-languages=c', '--disable-shared',
1254                          '--disable-threads',
1255                          '--disable-libatomic',
1256                          '--disable-decimal-float',
1257                          '--disable-libffi',
1258                          '--disable-libgomp',
1259                          '--disable-libitm',
1260                          '--disable-libmpx',
1261                          '--disable-libquadmath',
1262                          '--without-headers', '--with-newlib',
1263                          '--with-glibc-version=%s' % self.ctx.glibc_version
1264                          ]
1265             cfg_opts += self.first_gcc_cfg
1266         else:
1267             tool_build = 'gcc'
1268             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1269                          '--enable-threads']
1270         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1271
1272
1273 class Glibc(object):
1274     """A configuration for building glibc."""
1275
1276     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1277                  cfg=None, ccopts=None):
1278         """Initialize a Glibc object."""
1279         self.ctx = compiler.ctx
1280         self.compiler = compiler
1281         if arch is None:
1282             self.arch = compiler.arch
1283         else:
1284             self.arch = arch
1285         if os_name is None:
1286             self.os = compiler.os
1287         else:
1288             self.os = os_name
1289         self.variant = variant
1290         if variant is None:
1291             self.name = '%s-%s' % (self.arch, self.os)
1292         else:
1293             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1294         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1295         if cfg is None:
1296             self.cfg = []
1297         else:
1298             self.cfg = cfg
1299         self.ccopts = ccopts
1300
1301     def tool_name(self, tool):
1302         """Return the name of a cross-compilation tool."""
1303         ctool = '%s-%s' % (self.compiler.triplet, tool)
1304         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1305             ctool = '%s %s' % (ctool, self.ccopts)
1306         return ctool
1307
1308     def build(self):
1309         """Generate commands to build this glibc."""
1310         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1311         installdir = self.ctx.glibc_installdir(self.name)
1312         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1313         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1314         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1315         cmdlist.add_command('check-compilers',
1316                             ['test', '-f',
1317                              os.path.join(self.compiler.installdir, 'ok')])
1318         cmdlist.use_path(self.compiler.bindir)
1319         self.build_glibc(cmdlist, False)
1320         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1321                                       logsdir)
1322
1323     def build_glibc(self, cmdlist, for_compiler):
1324         """Generate commands to build this glibc, either as part of a compiler
1325         build or with the bootstrapped compiler (and in the latter case, run
1326         tests as well)."""
1327         srcdir = self.ctx.component_srcdir('glibc')
1328         if for_compiler:
1329             builddir = self.ctx.component_builddir('compilers',
1330                                                    self.compiler.name, 'glibc',
1331                                                    self.name)
1332             installdir = self.compiler.sysroot
1333             srcdir_copy = self.ctx.component_builddir('compilers',
1334                                                       self.compiler.name,
1335                                                       'glibc-src',
1336                                                       self.name)
1337         else:
1338             builddir = self.ctx.component_builddir('glibcs', self.name,
1339                                                    'glibc')
1340             installdir = self.ctx.glibc_installdir(self.name)
1341             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1342                                                       'glibc-src')
1343         cmdlist.create_use_dir(builddir)
1344         # glibc builds write into the source directory, and even if
1345         # not intentionally there is a risk of bugs that involve
1346         # writing into the working directory.  To avoid possible
1347         # concurrency issues, copy the source directory.
1348         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1349         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1350                    '--prefix=/usr',
1351                    '--enable-profile',
1352                    '--build=%s' % self.ctx.build_triplet,
1353                    '--host=%s' % self.triplet,
1354                    'CC=%s' % self.tool_name('gcc'),
1355                    'CXX=%s' % self.tool_name('g++'),
1356                    'AR=%s' % self.tool_name('ar'),
1357                    'AS=%s' % self.tool_name('as'),
1358                    'LD=%s' % self.tool_name('ld'),
1359                    'NM=%s' % self.tool_name('nm'),
1360                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1361                    'OBJDUMP=%s' % self.tool_name('objdump'),
1362                    'RANLIB=%s' % self.tool_name('ranlib'),
1363                    'READELF=%s' % self.tool_name('readelf'),
1364                    'STRIP=%s' % self.tool_name('strip')]
1365         cfg_cmd += self.cfg
1366         cmdlist.add_command('configure', cfg_cmd)
1367         cmdlist.add_command('build', ['make'])
1368         cmdlist.add_command('install', ['make', 'install',
1369                                         'install_root=%s' % installdir])
1370         # GCC uses paths such as lib/../lib64, so make sure lib
1371         # directories always exist.
1372         cmdlist.add_command('mkdir-lib', ['mkdir', '-p',
1373                                           os.path.join(installdir, 'lib'),
1374                                           os.path.join(installdir,
1375                                                        'usr', 'lib')])
1376         if not for_compiler:
1377             if self.ctx.strip:
1378                 cmdlist.add_command('strip',
1379                                     ['sh', '-c',
1380                                      ('%s $(find %s/lib* -name "*.so")' %
1381                                       (self.tool_name('strip'), installdir))])
1382             cmdlist.add_command('check', ['make', 'check'])
1383             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1384                                 always_run=True)
1385         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1386         cmdlist.cleanup_dir()
1387
1388
1389 class Command(object):
1390     """A command run in the build process."""
1391
1392     def __init__(self, desc, num, dir, path, command, always_run=False):
1393         """Initialize a Command object."""
1394         self.dir = dir
1395         self.path = path
1396         self.desc = desc
1397         trans = str.maketrans({' ': '-'})
1398         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1399         self.command = command
1400         self.always_run = always_run
1401
1402     @staticmethod
1403     def shell_make_quote_string(s):
1404         """Given a string not containing a newline, quote it for use by the
1405         shell and make."""
1406         assert '\n' not in s
1407         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1408             return s
1409         strans = str.maketrans({"'": "'\\''"})
1410         s = "'%s'" % s.translate(strans)
1411         mtrans = str.maketrans({'$': '$$'})
1412         return s.translate(mtrans)
1413
1414     @staticmethod
1415     def shell_make_quote_list(l, translate_make):
1416         """Given a list of strings not containing newlines, quote them for use
1417         by the shell and make, returning a single string.  If translate_make
1418         is true and the first string is 'make', change it to $(MAKE)."""
1419         l = [Command.shell_make_quote_string(s) for s in l]
1420         if translate_make and l[0] == 'make':
1421             l[0] = '$(MAKE)'
1422         return ' '.join(l)
1423
1424     def shell_make_quote(self):
1425         """Return this command quoted for the shell and make."""
1426         return self.shell_make_quote_list(self.command, True)
1427
1428
1429 class CommandList(object):
1430     """A list of commands run in the build process."""
1431
1432     def __init__(self, desc, keep):
1433         """Initialize a CommandList object."""
1434         self.cmdlist = []
1435         self.dir = None
1436         self.path = None
1437         self.desc = [desc]
1438         self.keep = keep
1439
1440     def desc_txt(self, desc):
1441         """Return the description to use for a command."""
1442         return '%s %s' % (' '.join(self.desc), desc)
1443
1444     def use_dir(self, dir):
1445         """Set the default directory for subsequent commands."""
1446         self.dir = dir
1447
1448     def use_path(self, path):
1449         """Set a directory to be prepended to the PATH for subsequent
1450         commands."""
1451         self.path = path
1452
1453     def push_subdesc(self, subdesc):
1454         """Set the default subdescription for subsequent commands (e.g., the
1455         name of a component being built, within the series of commands
1456         building it)."""
1457         self.desc.append(subdesc)
1458
1459     def pop_subdesc(self):
1460         """Pop a subdescription from the list of descriptions."""
1461         self.desc.pop()
1462
1463     def create_use_dir(self, dir):
1464         """Remove and recreate a directory and use it for subsequent
1465         commands."""
1466         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1467         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1468         self.use_dir(dir)
1469
1470     def create_copy_dir(self, src, dest):
1471         """Remove a directory and recreate it as a copy from the given
1472         source."""
1473         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1474         parent = os.path.dirname(dest)
1475         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1476         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1477
1478     def add_command_dir(self, desc, dir, command, always_run=False):
1479         """Add a command to run in a given directory."""
1480         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1481                       command, always_run)
1482         self.cmdlist.append(cmd)
1483
1484     def add_command(self, desc, command, always_run=False):
1485         """Add a command to run in the default directory."""
1486         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1487                       self.path, command, always_run)
1488         self.cmdlist.append(cmd)
1489
1490     def cleanup_dir(self, desc='cleanup', dir=None):
1491         """Clean up a build directory.  If no directory is specified, the
1492         default directory is cleaned up and ceases to be the default
1493         directory."""
1494         if dir is None:
1495             dir = self.dir
1496             self.use_dir(None)
1497         if self.keep != 'all':
1498             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1499                                  always_run=(self.keep == 'none'))
1500
1501     def makefile_commands(self, wrapper, logsdir):
1502         """Return the sequence of commands in the form of text for a Makefile.
1503         The given wrapper script takes arguments: base of logs for
1504         previous command, or empty; base of logs for this command;
1505         description; directory; PATH addition; the command itself."""
1506         # prev_base is the base of the name for logs of the previous
1507         # command that is not always-run (that is, a build command,
1508         # whose failure should stop subsequent build commands from
1509         # being run, as opposed to a cleanup command, which is run
1510         # even if previous commands failed).
1511         prev_base = ''
1512         cmds = []
1513         for c in self.cmdlist:
1514             ctxt = c.shell_make_quote()
1515             if prev_base and not c.always_run:
1516                 prev_log = os.path.join(logsdir, prev_base)
1517             else:
1518                 prev_log = ''
1519             this_log = os.path.join(logsdir, c.logbase)
1520             if not c.always_run:
1521                 prev_base = c.logbase
1522             if c.dir is None:
1523                 dir = ''
1524             else:
1525                 dir = c.dir
1526             if c.path is None:
1527                 path = ''
1528             else:
1529                 path = c.path
1530             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1531             prelim_txt = Command.shell_make_quote_list(prelims, False)
1532             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1533         return '\n'.join(cmds)
1534
1535     def status_logs(self, logsdir):
1536         """Return the list of log files with command status."""
1537         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1538                 for c in self.cmdlist]
1539
1540
1541 def get_parser():
1542     """Return an argument parser for this module."""
1543     parser = argparse.ArgumentParser(description=__doc__)
1544     parser.add_argument('-j', dest='parallelism',
1545                         help='Run this number of jobs in parallel',
1546                         type=int, default=os.cpu_count())
1547     parser.add_argument('--keep', dest='keep',
1548                         help='Whether to keep all build directories, '
1549                         'none or only those from failed builds',
1550                         default='none', choices=('none', 'all', 'failed'))
1551     parser.add_argument('--replace-sources', action='store_true',
1552                         help='Remove and replace source directories '
1553                         'with the wrong version of a component')
1554     parser.add_argument('--strip', action='store_true',
1555                         help='Strip installed glibc libraries')
1556     parser.add_argument('topdir',
1557                         help='Toplevel working directory')
1558     parser.add_argument('action',
1559                         help='What to do',
1560                         choices=('checkout', 'bot-cycle', 'bot',
1561                                  'host-libraries', 'compilers', 'glibcs'))
1562     parser.add_argument('configs',
1563                         help='Versions to check out or configurations to build',
1564                         nargs='*')
1565     return parser
1566
1567
1568 def main(argv):
1569     """The main entry point."""
1570     parser = get_parser()
1571     opts = parser.parse_args(argv)
1572     topdir = os.path.abspath(opts.topdir)
1573     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1574                   opts.strip, opts.action)
1575     ctx.run_builds(opts.action, opts.configs)
1576
1577
1578 if __name__ == '__main__':
1579     main(sys.argv[1:])