scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2017 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52 try:
  53     os.cpu_count
  54 except:
  55     import multiprocessing
  56     os.cpu_count = lambda: multiprocessing.cpu_count()
  57
  58 try:
  59     re.fullmatch
  60 except:
  61     re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)
  62
  63 try:
  64     subprocess.run
  65 except:
  66     class _CompletedProcess:
  67         def __init__(self, args, returncode, stdout=None, stderr=None):
  68             self.args = args
  69             self.returncode = returncode
  70             self.stdout = stdout
  71             self.stderr = stderr
  72
  73     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  74         assert(timeout is None)
  75         with subprocess.Popen(*popenargs, **kwargs) as process:
  76             try:
  77                 stdout, stderr = process.communicate(input)
  78             except:
  79                 process.kill()
  80                 process.wait()
  81                 raise
  82             returncode = process.poll()
  83             if check and returncode:
  84                 raise subprocess.CalledProcessError(returncode, popenargs)
  85         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  86
  87     subprocess.run = _run
  88
  89
  90 class Context(object):
  91     """The global state associated with builds in a given directory."""
  92
  93     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  94                  action):
  95         """Initialize the context."""
  96         self.topdir = topdir
  97         self.parallelism = parallelism
  98         self.keep = keep
  99         self.replace_sources = replace_sources
 100         self.strip = strip
 101         self.srcdir = os.path.join(topdir, 'src')
 102         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 103         self.build_state_json = os.path.join(topdir, 'build-state.json')
 104         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 105         self.installdir = os.path.join(topdir, 'install')
 106         self.host_libraries_installdir = os.path.join(self.installdir,
 107                                                       'host-libraries')
 108         self.builddir = os.path.join(topdir, 'build')
 109         self.logsdir = os.path.join(topdir, 'logs')
 110         self.logsdir_old = os.path.join(topdir, 'logs-old')
 111         self.makefile = os.path.join(self.builddir, 'Makefile')
 112         self.wrapper = os.path.join(self.builddir, 'wrapper')
 113         self.save_logs = os.path.join(self.builddir, 'save-logs')
 114         self.script_text = self.get_script_text()
 115         if action != 'checkout':
 116             self.build_triplet = self.get_build_triplet()
 117             self.glibc_version = self.get_glibc_version()
 118         self.configs = {}
 119         self.glibc_configs = {}
 120         self.makefile_pieces = ['.PHONY: all\n']
 121         self.add_all_configs()
 122         self.load_versions_json()
 123         self.load_build_state_json()
 124         self.status_log_list = []
 125         self.email_warning = False
 126
 127     def get_script_text(self):
 128         """Return the text of this script."""
 129         with open(sys.argv[0], 'r') as f:
 130             return f.read()
 131
 132     def exec_self(self):
 133         """Re-execute this script with the same arguments."""
 134         sys.stdout.flush()
 135         os.execv(sys.executable, [sys.executable] + sys.argv)
 136
 137     def get_build_triplet(self):
 138         """Determine the build triplet with config.guess."""
 139         config_guess = os.path.join(self.component_srcdir('gcc'),
 140                                     'config.guess')
 141         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 142                                 check=True, universal_newlines=True).stdout
 143         return cg_out.rstrip()
 144
 145     def get_glibc_version(self):
 146         """Determine the glibc version number (major.minor)."""
 147         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 148         with open(version_h, 'r') as f:
 149             lines = f.readlines()
 150         starttext = '#define VERSION "'
 151         for l in lines:
 152             if l.startswith(starttext):
 153                 l = l[len(starttext):]
 154                 l = l.rstrip('"\n')
 155                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 156                 return '%s.%s' % m.group(1, 2)
 157         print('error: could not determine glibc version')
 158         exit(1)
 159
 160     def add_all_configs(self):
 161         """Add all known glibc build configurations."""
 162         self.add_config(arch='aarch64',
 163                         os_name='linux-gnu')
 164         self.add_config(arch='aarch64_be',
 165                         os_name='linux-gnu')
 166         self.add_config(arch='alpha',
 167                         os_name='linux-gnu')
 168         self.add_config(arch='arm',
 169                         os_name='linux-gnueabi')
 170         self.add_config(arch='armeb',
 171                         os_name='linux-gnueabi')
 172         self.add_config(arch='armeb',
 173                         os_name='linux-gnueabi',
 174                         variant='be8',
 175                         gcc_cfg=['--with-arch=armv7-a'])
 176         self.add_config(arch='arm',
 177                         os_name='linux-gnueabihf',
 178                         extra_glibcs=[{'variant': 'v7a',
 179                                        'ccopts': '-march=armv7-a'},
 180                                       {'variant': 'v7a-disable-multi-arch',
 181                                        'ccopts': '-march=armv7-a',
 182                                        'cfg': ['--disable-multi-arch']}])
 183         self.add_config(arch='armeb',
 184                         os_name='linux-gnueabihf')
 185         self.add_config(arch='armeb',
 186                         os_name='linux-gnueabihf',
 187                         variant='be8',
 188                         gcc_cfg=['--with-arch=armv7-a'])
 189         self.add_config(arch='hppa',
 190                         os_name='linux-gnu')
 191         self.add_config(arch='ia64',
 192                         os_name='linux-gnu',
 193                         first_gcc_cfg=['--with-system-libunwind'])
 194         self.add_config(arch='m68k',
 195                         os_name='linux-gnu',
 196                         gcc_cfg=['--disable-multilib'])
 197         self.add_config(arch='m68k',
 198                         os_name='linux-gnu',
 199                         variant='coldfire',
 200                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 201         self.add_config(arch='microblaze',
 202                         os_name='linux-gnu',
 203                         gcc_cfg=['--disable-multilib'])
 204         self.add_config(arch='microblazeel',
 205                         os_name='linux-gnu',
 206                         gcc_cfg=['--disable-multilib'])
 207         self.add_config(arch='mips64',
 208                         os_name='linux-gnu',
 209                         gcc_cfg=['--with-mips-plt'],
 210                         glibcs=[{'variant': 'n32'},
 211                                 {'arch': 'mips',
 212                                  'ccopts': '-mabi=32'},
 213                                 {'variant': 'n64',
 214                                  'ccopts': '-mabi=64'}])
 215         self.add_config(arch='mips64',
 216                         os_name='linux-gnu',
 217                         variant='soft',
 218                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 219                         glibcs=[{'variant': 'n32-soft',
 220                                  'cfg': ['--without-fp']},
 221                                 {'variant': 'soft',
 222                                  'arch': 'mips',
 223                                  'ccopts': '-mabi=32',
 224                                  'cfg': ['--without-fp']},
 225                                 {'variant': 'n64-soft',
 226                                  'ccopts': '-mabi=64',
 227                                  'cfg': ['--without-fp']}])
 228         self.add_config(arch='mips64',
 229                         os_name='linux-gnu',
 230                         variant='nan2008',
 231                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 232                                  '--with-arch-64=mips64r2',
 233                                  '--with-arch-32=mips32r2'],
 234                         glibcs=[{'variant': 'n32-nan2008'},
 235                                 {'variant': 'nan2008',
 236                                  'arch': 'mips',
 237                                  'ccopts': '-mabi=32'},
 238                                 {'variant': 'n64-nan2008',
 239                                  'ccopts': '-mabi=64'}])
 240         self.add_config(arch='mips64',
 241                         os_name='linux-gnu',
 242                         variant='nan2008-soft',
 243                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 244                                  '--with-arch-64=mips64r2',
 245                                  '--with-arch-32=mips32r2',
 246                                  '--with-float=soft'],
 247                         glibcs=[{'variant': 'n32-nan2008-soft',
 248                                  'cfg': ['--without-fp']},
 249                                 {'variant': 'nan2008-soft',
 250                                  'arch': 'mips',
 251                                  'ccopts': '-mabi=32',
 252                                  'cfg': ['--without-fp']},
 253                                 {'variant': 'n64-nan2008-soft',
 254                                  'ccopts': '-mabi=64',
 255                                  'cfg': ['--without-fp']}])
 256         self.add_config(arch='mips64el',
 257                         os_name='linux-gnu',
 258                         gcc_cfg=['--with-mips-plt'],
 259                         glibcs=[{'variant': 'n32'},
 260                                 {'arch': 'mipsel',
 261                                  'ccopts': '-mabi=32'},
 262                                 {'variant': 'n64',
 263                                  'ccopts': '-mabi=64'}])
 264         self.add_config(arch='mips64el',
 265                         os_name='linux-gnu',
 266                         variant='soft',
 267                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 268                         glibcs=[{'variant': 'n32-soft',
 269                                  'cfg': ['--without-fp']},
 270                                 {'variant': 'soft',
 271                                  'arch': 'mipsel',
 272                                  'ccopts': '-mabi=32',
 273                                  'cfg': ['--without-fp']},
 274                                 {'variant': 'n64-soft',
 275                                  'ccopts': '-mabi=64',
 276                                  'cfg': ['--without-fp']}])
 277         self.add_config(arch='mips64el',
 278                         os_name='linux-gnu',
 279                         variant='nan2008',
 280                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 281                                  '--with-arch-64=mips64r2',
 282                                  '--with-arch-32=mips32r2'],
 283                         glibcs=[{'variant': 'n32-nan2008'},
 284                                 {'variant': 'nan2008',
 285                                  'arch': 'mipsel',
 286                                  'ccopts': '-mabi=32'},
 287                                 {'variant': 'n64-nan2008',
 288                                  'ccopts': '-mabi=64'}])
 289         self.add_config(arch='mips64el',
 290                         os_name='linux-gnu',
 291                         variant='nan2008-soft',
 292                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 293                                  '--with-arch-64=mips64r2',
 294                                  '--with-arch-32=mips32r2',
 295                                  '--with-float=soft'],
 296                         glibcs=[{'variant': 'n32-nan2008-soft',
 297                                  'cfg': ['--without-fp']},
 298                                 {'variant': 'nan2008-soft',
 299                                  'arch': 'mipsel',
 300                                  'ccopts': '-mabi=32',
 301                                  'cfg': ['--without-fp']},
 302                                 {'variant': 'n64-nan2008-soft',
 303                                  'ccopts': '-mabi=64',
 304                                  'cfg': ['--without-fp']}])
 305         self.add_config(arch='nios2',
 306                         os_name='linux-gnu')
 307         self.add_config(arch='powerpc',
 308                         os_name='linux-gnu',
 309                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 310                         extra_glibcs=[{'variant': 'power4',
 311                                        'ccopts': '-mcpu=power4',
 312                                        'cfg': ['--with-cpu=power4']}])
 313         self.add_config(arch='powerpc',
 314                         os_name='linux-gnu',
 315                         variant='soft',
 316                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 317                                  '--enable-secureplt'],
 318                         glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
 319         self.add_config(arch='powerpc64',
 320                         os_name='linux-gnu',
 321                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 322         self.add_config(arch='powerpc64le',
 323                         os_name='linux-gnu',
 324                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 325         self.add_config(arch='powerpc',
 326                         os_name='linux-gnuspe',
 327                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 328                                  '--enable-e500-double'],
 329                         glibcs=[{'cfg': ['--without-fp']}])
 330         self.add_config(arch='powerpc',
 331                         os_name='linux-gnuspe',
 332                         variant='e500v1',
 333                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 334                         glibcs=[{'variant': 'e500v1', 'cfg': ['--without-fp']}])
 335         self.add_config(arch='s390x',
 336                         os_name='linux-gnu',
 337                         glibcs=[{},
 338                                 {'arch': 's390', 'ccopts': '-m31'}])
 339         self.add_config(arch='sh3',
 340                         os_name='linux-gnu')
 341         self.add_config(arch='sh3eb',
 342                         os_name='linux-gnu')
 343         self.add_config(arch='sh4',
 344                         os_name='linux-gnu')
 345         self.add_config(arch='sh4eb',
 346                         os_name='linux-gnu')
 347         self.add_config(arch='sh4',
 348                         os_name='linux-gnu',
 349                         variant='soft',
 350                         gcc_cfg=['--without-fp'],
 351                         glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
 352         self.add_config(arch='sh4eb',
 353                         os_name='linux-gnu',
 354                         variant='soft',
 355                         gcc_cfg=['--without-fp'],
 356                         glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
 357         self.add_config(arch='sparc64',
 358                         os_name='linux-gnu',
 359                         glibcs=[{},
 360                                 {'arch': 'sparcv9',
 361                                  'ccopts': '-m32 -mlong-double-128'}])
 362         self.add_config(arch='tilegx',
 363                         os_name='linux-gnu',
 364                         glibcs=[{},
 365                                 {'variant': '32', 'ccopts': '-m32'}])
 366         self.add_config(arch='tilegxbe',
 367                         os_name='linux-gnu',
 368                         glibcs=[{},
 369                                 {'variant': '32', 'ccopts': '-m32'}])
 370         self.add_config(arch='tilepro',
 371                         os_name='linux-gnu')
 372         self.add_config(arch='x86_64',
 373                         os_name='linux-gnu',
 374                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 375                         glibcs=[{},
 376                                 {'variant': 'x32', 'ccopts': '-mx32'},
 377                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 378                         extra_glibcs=[{'variant': 'disable-multi-arch',
 379                                        'cfg': ['--disable-multi-arch']},
 380                                       {'variant': 'disable-multi-arch',
 381                                        'arch': 'i686',
 382                                        'ccopts': '-m32 -march=i686',
 383                                        'cfg': ['--disable-multi-arch']},
 384                                       {'arch': 'i486',
 385                                        'ccopts': '-m32 -march=i486'},
 386                                       {'arch': 'i586',
 387                                        'ccopts': '-m32 -march=i586'}])
 388
 389     def add_config(self, **args):
 390         """Add an individual build configuration."""
 391         cfg = Config(self, **args)
 392         if cfg.name in self.configs:
 393             print('error: duplicate config %s' % cfg.name)
 394             exit(1)
 395         self.configs[cfg.name] = cfg
 396         for c in cfg.all_glibcs:
 397             if c.name in self.glibc_configs:
 398                 print('error: duplicate glibc config %s' % c.name)
 399                 exit(1)
 400             self.glibc_configs[c.name] = c
 401
 402     def component_srcdir(self, component):
 403         """Return the source directory for a given component, e.g. gcc."""
 404         return os.path.join(self.srcdir, component)
 405
 406     def component_builddir(self, action, config, component, subconfig=None):
 407         """Return the directory to use for a build."""
 408         if config is None:
 409             # Host libraries.
 410             assert subconfig is None
 411             return os.path.join(self.builddir, action, component)
 412         if subconfig is None:
 413             return os.path.join(self.builddir, action, config, component)
 414         else:
 415             # glibc build as part of compiler build.
 416             return os.path.join(self.builddir, action, config, component,
 417                                 subconfig)
 418
 419     def compiler_installdir(self, config):
 420         """Return the directory in which to install a compiler."""
 421         return os.path.join(self.installdir, 'compilers', config)
 422
 423     def compiler_bindir(self, config):
 424         """Return the directory in which to find compiler binaries."""
 425         return os.path.join(self.compiler_installdir(config), 'bin')
 426
 427     def compiler_sysroot(self, config):
 428         """Return the sysroot directory for a compiler."""
 429         return os.path.join(self.compiler_installdir(config), 'sysroot')
 430
 431     def glibc_installdir(self, config):
 432         """Return the directory in which to install glibc."""
 433         return os.path.join(self.installdir, 'glibcs', config)
 434
 435     def run_builds(self, action, configs):
 436         """Run the requested builds."""
 437         if action == 'checkout':
 438             self.checkout(configs)
 439             return
 440         if action == 'bot-cycle':
 441             if configs:
 442                 print('error: configurations specified for bot-cycle')
 443                 exit(1)
 444             self.bot_cycle()
 445             return
 446         if action == 'bot':
 447             if configs:
 448                 print('error: configurations specified for bot')
 449                 exit(1)
 450             self.bot()
 451             return
 452         if action == 'host-libraries' and configs:
 453             print('error: configurations specified for host-libraries')
 454             exit(1)
 455         self.clear_last_build_state(action)
 456         build_time = datetime.datetime.utcnow()
 457         if action == 'host-libraries':
 458             build_components = ('gmp', 'mpfr', 'mpc')
 459             old_components = ()
 460             old_versions = {}
 461             self.build_host_libraries()
 462         elif action == 'compilers':
 463             build_components = ('binutils', 'gcc', 'glibc', 'linux')
 464             old_components = ('gmp', 'mpfr', 'mpc')
 465             old_versions = self.build_state['host-libraries']['build-versions']
 466             self.build_compilers(configs)
 467         else:
 468             build_components = ('glibc',)
 469             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
 470             old_versions = self.build_state['compilers']['build-versions']
 471             self.build_glibcs(configs)
 472         self.write_files()
 473         self.do_build()
 474         if configs:
 475             # Partial build, do not update stored state.
 476             return
 477         build_versions = {}
 478         for k in build_components:
 479             if k in self.versions:
 480                 build_versions[k] = {'version': self.versions[k]['version'],
 481                                      'revision': self.versions[k]['revision']}
 482         for k in old_components:
 483             if k in old_versions:
 484                 build_versions[k] = {'version': old_versions[k]['version'],
 485                                      'revision': old_versions[k]['revision']}
 486         self.update_build_state(action, build_time, build_versions)
 487
 488     @staticmethod
 489     def remove_dirs(*args):
 490         """Remove directories and their contents if they exist."""
 491         for dir in args:
 492             shutil.rmtree(dir, ignore_errors=True)
 493
 494     @staticmethod
 495     def remove_recreate_dirs(*args):
 496         """Remove directories if they exist, and create them as empty."""
 497         Context.remove_dirs(*args)
 498         for dir in args:
 499             os.makedirs(dir, exist_ok=True)
 500
 501     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 502         """Add makefile text for a list of commands."""
 503         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 504         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 505                                     (target, target, target, commands))
 506         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 507
 508     def write_files(self):
 509         """Write out the Makefile and wrapper script."""
 510         mftext = ''.join(self.makefile_pieces)
 511         with open(self.makefile, 'w') as f:
 512             f.write(mftext)
 513         wrapper_text = (
 514             '#!/bin/sh\n'
 515             'prev_base=$1\n'
 516             'this_base=$2\n'
 517             'desc=$3\n'
 518             'dir=$4\n'
 519             'path=$5\n'
 520             'shift 5\n'
 521             'prev_status=$prev_base-status.txt\n'
 522             'this_status=$this_base-status.txt\n'
 523             'this_log=$this_base-log.txt\n'
 524             'date > "$this_log"\n'
 525             'echo >> "$this_log"\n'
 526             'echo "Description: $desc" >> "$this_log"\n'
 527             'printf "%s" "Command:" >> "$this_log"\n'
 528             'for word in "$@"; do\n'
 529             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 530             '    printf " %s" "$word"\n'
 531             '  else\n'
 532             '    printf " \'"\n'
 533             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 534             '    printf "\'"\n'
 535             '  fi\n'
 536             'done >> "$this_log"\n'
 537             'echo >> "$this_log"\n'
 538             'echo "Directory: $dir" >> "$this_log"\n'
 539             'echo "Path addition: $path" >> "$this_log"\n'
 540             'echo >> "$this_log"\n'
 541             'record_status ()\n'
 542             '{\n'
 543             '  echo >> "$this_log"\n'
 544             '  echo "$1: $desc" > "$this_status"\n'
 545             '  echo "$1: $desc" >> "$this_log"\n'
 546             '  echo >> "$this_log"\n'
 547             '  date >> "$this_log"\n'
 548             '  echo "$1: $desc"\n'
 549             '  exit 0\n'
 550             '}\n'
 551             'check_error ()\n'
 552             '{\n'
 553             '  if [ "$1" != "0" ]; then\n'
 554             '    record_status FAIL\n'
 555             '  fi\n'
 556             '}\n'
 557             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 558             '    record_status UNRESOLVED\n'
 559             'fi\n'
 560             'if [ "$dir" ]; then\n'
 561             '  cd "$dir"\n'
 562             '  check_error "$?"\n'
 563             'fi\n'
 564             'if [ "$path" ]; then\n'
 565             '  PATH=$path:$PATH\n'
 566             'fi\n'
 567             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 568             'check_error "$?"\n'
 569             'record_status PASS\n')
 570         with open(self.wrapper, 'w') as f:
 571             f.write(wrapper_text)
 572         # Mode 0o755.
 573         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 574                      stat.S_IROTH|stat.S_IXOTH)
 575         os.chmod(self.wrapper, mode_exec)
 576         save_logs_text = (
 577             '#!/bin/sh\n'
 578             'if ! [ -f tests.sum ]; then\n'
 579             '  echo "No test summary available."\n'
 580             '  exit 0\n'
 581             'fi\n'
 582             'save_file ()\n'
 583             '{\n'
 584             '  echo "Contents of $1:"\n'
 585             '  echo\n'
 586             '  cat "$1"\n'
 587             '  echo\n'
 588             '  echo "End of contents of $1."\n'
 589             '  echo\n'
 590             '}\n'
 591             'save_file tests.sum\n'
 592             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 593             'for t in $non_pass_tests; do\n'
 594             '  if [ -f "$t.out" ]; then\n'
 595             '    save_file "$t.out"\n'
 596             '  fi\n'
 597             'done\n')
 598         with open(self.save_logs, 'w') as f:
 599             f.write(save_logs_text)
 600         os.chmod(self.save_logs, mode_exec)
 601
 602     def do_build(self):
 603         """Do the actual build."""
 604         cmd = ['make', '-j%d' % self.parallelism]
 605         subprocess.run(cmd, cwd=self.builddir, check=True)
 606
 607     def build_host_libraries(self):
 608         """Build the host libraries."""
 609         installdir = self.host_libraries_installdir
 610         builddir = os.path.join(self.builddir, 'host-libraries')
 611         logsdir = os.path.join(self.logsdir, 'host-libraries')
 612         self.remove_recreate_dirs(installdir, builddir, logsdir)
 613         cmdlist = CommandList('host-libraries', self.keep)
 614         self.build_host_library(cmdlist, 'gmp')
 615         self.build_host_library(cmdlist, 'mpfr',
 616                                 ['--with-gmp=%s' % installdir])
 617         self.build_host_library(cmdlist, 'mpc',
 618                                 ['--with-gmp=%s' % installdir,
 619                                 '--with-mpfr=%s' % installdir])
 620         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 621         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 622
 623     def build_host_library(self, cmdlist, lib, extra_opts=None):
 624         """Build one host library."""
 625         srcdir = self.component_srcdir(lib)
 626         builddir = self.component_builddir('host-libraries', None, lib)
 627         installdir = self.host_libraries_installdir
 628         cmdlist.push_subdesc(lib)
 629         cmdlist.create_use_dir(builddir)
 630         cfg_cmd = [os.path.join(srcdir, 'configure'),
 631                    '--prefix=%s' % installdir,
 632                    '--disable-shared']
 633         if extra_opts:
 634             cfg_cmd.extend (extra_opts)
 635         cmdlist.add_command('configure', cfg_cmd)
 636         cmdlist.add_command('build', ['make'])
 637         cmdlist.add_command('check', ['make', 'check'])
 638         cmdlist.add_command('install', ['make', 'install'])
 639         cmdlist.cleanup_dir()
 640         cmdlist.pop_subdesc()
 641
 642     def build_compilers(self, configs):
 643         """Build the compilers."""
 644         if not configs:
 645             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 646             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 647             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 648             configs = sorted(self.configs.keys())
 649         for c in configs:
 650             self.configs[c].build()
 651
 652     def build_glibcs(self, configs):
 653         """Build the glibcs."""
 654         if not configs:
 655             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 656             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 657             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 658             configs = sorted(self.glibc_configs.keys())
 659         for c in configs:
 660             self.glibc_configs[c].build()
 661
 662     def load_versions_json(self):
 663         """Load information about source directory versions."""
 664         if not os.access(self.versions_json, os.F_OK):
 665             self.versions = {}
 666             return
 667         with open(self.versions_json, 'r') as f:
 668             self.versions = json.load(f)
 669
 670     def store_json(self, data, filename):
 671         """Store information in a JSON file."""
 672         filename_tmp = filename + '.tmp'
 673         with open(filename_tmp, 'w') as f:
 674             json.dump(data, f, indent=2, sort_keys=True)
 675         os.rename(filename_tmp, filename)
 676
 677     def store_versions_json(self):
 678         """Store information about source directory versions."""
 679         self.store_json(self.versions, self.versions_json)
 680
 681     def set_component_version(self, component, version, explicit, revision):
 682         """Set the version information for a component."""
 683         self.versions[component] = {'version': version,
 684                                     'explicit': explicit,
 685                                     'revision': revision}
 686         self.store_versions_json()
 687
 688     def checkout(self, versions):
 689         """Check out the desired component versions."""
 690         default_versions = {'binutils': 'vcs-2.29',
 691                             'gcc': 'vcs-7',
 692                             'glibc': 'vcs-mainline',
 693                             'gmp': '6.1.2',
 694                             'linux': '4.14',
 695                             'mpc': '1.0.3',
 696                             'mpfr': '3.1.6'}
 697         use_versions = {}
 698         explicit_versions = {}
 699         for v in versions:
 700             found_v = False
 701             for k in default_versions.keys():
 702                 kx = k + '-'
 703                 if v.startswith(kx):
 704                     vx = v[len(kx):]
 705                     if k in use_versions:
 706                         print('error: multiple versions for %s' % k)
 707                         exit(1)
 708                     use_versions[k] = vx
 709                     explicit_versions[k] = True
 710                     found_v = True
 711                     break
 712             if not found_v:
 713                 print('error: unknown component in %s' % v)
 714                 exit(1)
 715         for k in default_versions.keys():
 716             if k not in use_versions:
 717                 if k in self.versions and self.versions[k]['explicit']:
 718                     use_versions[k] = self.versions[k]['version']
 719                     explicit_versions[k] = True
 720                 else:
 721                     use_versions[k] = default_versions[k]
 722                     explicit_versions[k] = False
 723         os.makedirs(self.srcdir, exist_ok=True)
 724         for k in sorted(default_versions.keys()):
 725             update = os.access(self.component_srcdir(k), os.F_OK)
 726             v = use_versions[k]
 727             if (update and
 728                 k in self.versions and
 729                 v != self.versions[k]['version']):
 730                 if not self.replace_sources:
 731                     print('error: version of %s has changed from %s to %s, '
 732                           'use --replace-sources to check out again' %
 733                           (k, self.versions[k]['version'], v))
 734                     exit(1)
 735                 shutil.rmtree(self.component_srcdir(k))
 736                 update = False
 737             if v.startswith('vcs-'):
 738                 revision = self.checkout_vcs(k, v[4:], update)
 739             else:
 740                 self.checkout_tar(k, v, update)
 741                 revision = v
 742             self.set_component_version(k, v, explicit_versions[k], revision)
 743         if self.get_script_text() != self.script_text:
 744             # Rerun the checkout process in case the updated script
 745             # uses different default versions or new components.
 746             self.exec_self()
 747
 748     def checkout_vcs(self, component, version, update):
 749         """Check out the given version of the given component from version
 750         control.  Return a revision identifier."""
 751         if component == 'binutils':
 752             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 753             if version == 'mainline':
 754                 git_branch = 'master'
 755             else:
 756                 trans = str.maketrans({'.': '_'})
 757                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 758             return self.git_checkout(component, git_url, git_branch, update)
 759         elif component == 'gcc':
 760             if version == 'mainline':
 761                 branch = 'trunk'
 762             else:
 763                 trans = str.maketrans({'.': '_'})
 764                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 765             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 766             return self.gcc_checkout(svn_url, update)
 767         elif component == 'glibc':
 768             git_url = 'git://sourceware.org/git/glibc.git'
 769             if version == 'mainline':
 770                 git_branch = 'master'
 771             else:
 772                 git_branch = 'release/%s/master' % version
 773             r = self.git_checkout(component, git_url, git_branch, update)
 774             self.fix_glibc_timestamps()
 775             return r
 776         else:
 777             print('error: component %s coming from VCS' % component)
 778             exit(1)
 779
 780     def git_checkout(self, component, git_url, git_branch, update):
 781         """Check out a component from git.  Return a commit identifier."""
 782         if update:
 783             subprocess.run(['git', 'remote', 'prune', 'origin'],
 784                            cwd=self.component_srcdir(component), check=True)
 785             subprocess.run(['git', 'pull', '-q'],
 786                            cwd=self.component_srcdir(component), check=True)
 787         else:
 788             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 789                             self.component_srcdir(component)], check=True)
 790         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 791                            cwd=self.component_srcdir(component),
 792                            stdout=subprocess.PIPE,
 793                            check=True, universal_newlines=True).stdout
 794         return r.rstrip()
 795
 796     def fix_glibc_timestamps(self):
 797         """Fix timestamps in a glibc checkout."""
 798         # Ensure that builds do not try to regenerate generated files
 799         # in the source tree.
 800         srcdir = self.component_srcdir('glibc')
 801         for dirpath, dirnames, filenames in os.walk(srcdir):
 802             for f in filenames:
 803                 if (f == 'configure' or
 804                     f == 'preconfigure' or
 805                     f.endswith('-kw.h')):
 806                     to_touch = os.path.join(dirpath, f)
 807                     subprocess.run(['touch', to_touch], check=True)
 808
 809     def gcc_checkout(self, svn_url, update):
 810         """Check out GCC from SVN.  Return the revision number."""
 811         if not update:
 812             subprocess.run(['svn', 'co', '-q', svn_url,
 813                             self.component_srcdir('gcc')], check=True)
 814         subprocess.run(['contrib/gcc_update', '--silent'],
 815                        cwd=self.component_srcdir('gcc'), check=True)
 816         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 817                            stdout=subprocess.PIPE,
 818                            check=True, universal_newlines=True).stdout
 819         return r.rstrip()
 820
 821     def checkout_tar(self, component, version, update):
 822         """Check out the given version of the given component from a
 823         tarball."""
 824         if update:
 825             return
 826         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 827                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 828                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 829                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 830                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 831                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
 832         if component not in url_map:
 833             print('error: component %s coming from tarball' % component)
 834             exit(1)
 835         url = url_map[component] % {'version': version}
 836         filename = os.path.join(self.srcdir, url.split('/')[-1])
 837         response = urllib.request.urlopen(url)
 838         data = response.read()
 839         with open(filename, 'wb') as f:
 840             f.write(data)
 841         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 842                        check=True)
 843         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 844                   self.component_srcdir(component))
 845         os.remove(filename)
 846
 847     def load_build_state_json(self):
 848         """Load information about the state of previous builds."""
 849         if os.access(self.build_state_json, os.F_OK):
 850             with open(self.build_state_json, 'r') as f:
 851                 self.build_state = json.load(f)
 852         else:
 853             self.build_state = {}
 854         for k in ('host-libraries', 'compilers', 'glibcs'):
 855             if k not in self.build_state:
 856                 self.build_state[k] = {}
 857             if 'build-time' not in self.build_state[k]:
 858                 self.build_state[k]['build-time'] = ''
 859             if 'build-versions' not in self.build_state[k]:
 860                 self.build_state[k]['build-versions'] = {}
 861             if 'build-results' not in self.build_state[k]:
 862                 self.build_state[k]['build-results'] = {}
 863             if 'result-changes' not in self.build_state[k]:
 864                 self.build_state[k]['result-changes'] = {}
 865             if 'ever-passed' not in self.build_state[k]:
 866                 self.build_state[k]['ever-passed'] = []
 867
 868     def store_build_state_json(self):
 869         """Store information about the state of previous builds."""
 870         self.store_json(self.build_state, self.build_state_json)
 871
 872     def clear_last_build_state(self, action):
 873         """Clear information about the state of part of the build."""
 874         # We clear the last build time and versions when starting a
 875         # new build.  The results of the last build are kept around,
 876         # as comparison is still meaningful if this build is aborted
 877         # and a new one started.
 878         self.build_state[action]['build-time'] = ''
 879         self.build_state[action]['build-versions'] = {}
 880         self.store_build_state_json()
 881
 882     def update_build_state(self, action, build_time, build_versions):
 883         """Update the build state after a build."""
 884         build_time = build_time.replace(microsecond=0)
 885         self.build_state[action]['build-time'] = str(build_time)
 886         self.build_state[action]['build-versions'] = build_versions
 887         build_results = {}
 888         for log in self.status_log_list:
 889             with open(log, 'r') as f:
 890                 log_text = f.read()
 891             log_text = log_text.rstrip()
 892             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 893             result = m.group(1)
 894             test_name = m.group(2)
 895             assert test_name not in build_results
 896             build_results[test_name] = result
 897         old_build_results = self.build_state[action]['build-results']
 898         self.build_state[action]['build-results'] = build_results
 899         result_changes = {}
 900         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 901         for t in all_tests:
 902             if t in old_build_results:
 903                 old_res = old_build_results[t]
 904             else:
 905                 old_res = '(New test)'
 906             if t in build_results:
 907                 new_res = build_results[t]
 908             else:
 909                 new_res = '(Test removed)'
 910             if old_res != new_res:
 911                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 912         self.build_state[action]['result-changes'] = result_changes
 913         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 914                            if t in build_results}
 915         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 916         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 917                                                          new_passes)
 918         self.store_build_state_json()
 919
 920     def load_bot_config_json(self):
 921         """Load bot configuration."""
 922         with open(self.bot_config_json, 'r') as f:
 923             self.bot_config = json.load(f)
 924
 925     def part_build_old(self, action, delay):
 926         """Return whether the last build for a given action was at least a
 927         given number of seconds ago, or does not have a time recorded."""
 928         old_time_str = self.build_state[action]['build-time']
 929         if not old_time_str:
 930             return True
 931         old_time = datetime.datetime.strptime(old_time_str,
 932                                               '%Y-%m-%d %H:%M:%S')
 933         new_time = datetime.datetime.utcnow()
 934         delta = new_time - old_time
 935         return delta.total_seconds() >= delay
 936
 937     def bot_cycle(self):
 938         """Run a single round of checkout and builds."""
 939         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 940         self.load_bot_config_json()
 941         actions = ('host-libraries', 'compilers', 'glibcs')
 942         self.bot_run_self(['--replace-sources'], 'checkout')
 943         self.load_versions_json()
 944         if self.get_script_text() != self.script_text:
 945             print('Script changed, re-execing.')
 946             # On script change, all parts of the build should be rerun.
 947             for a in actions:
 948                 self.clear_last_build_state(a)
 949             self.exec_self()
 950         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 951                             'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
 952                             'glibcs': ('glibc',)}
 953         must_build = {}
 954         for a in actions:
 955             build_vers = self.build_state[a]['build-versions']
 956             must_build[a] = False
 957             if not self.build_state[a]['build-time']:
 958                 must_build[a] = True
 959             old_vers = {}
 960             new_vers = {}
 961             for c in check_components[a]:
 962                 if c in build_vers:
 963                     old_vers[c] = build_vers[c]
 964                 new_vers[c] = {'version': self.versions[c]['version'],
 965                                'revision': self.versions[c]['revision']}
 966             if new_vers == old_vers:
 967                 print('Versions for %s unchanged.' % a)
 968             else:
 969                 print('Versions changed or rebuild forced for %s.' % a)
 970                 if a == 'compilers' and not self.part_build_old(
 971                         a, self.bot_config['compilers-rebuild-delay']):
 972                     print('Not requiring rebuild of compilers this soon.')
 973                 else:
 974                     must_build[a] = True
 975         if must_build['host-libraries']:
 976             must_build['compilers'] = True
 977         if must_build['compilers']:
 978             must_build['glibcs'] = True
 979         for a in actions:
 980             if must_build[a]:
 981                 print('Must rebuild %s.' % a)
 982                 self.clear_last_build_state(a)
 983             else:
 984                 print('No need to rebuild %s.' % a)
 985         if os.access(self.logsdir, os.F_OK):
 986             shutil.rmtree(self.logsdir_old, ignore_errors=True)
 987             shutil.copytree(self.logsdir, self.logsdir_old)
 988         for a in actions:
 989             if must_build[a]:
 990                 build_time = datetime.datetime.utcnow()
 991                 print('Rebuilding %s at %s.' % (a, str(build_time)))
 992                 self.bot_run_self([], a)
 993                 self.load_build_state_json()
 994                 self.bot_build_mail(a, build_time)
 995         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
 996
 997     def bot_build_mail(self, action, build_time):
 998         """Send email with the results of a build."""
 999         if not ('email-from' in self.bot_config and
1000                 'email-server' in self.bot_config and
1001                 'email-subject' in self.bot_config and
1002                 'email-to' in self.bot_config):
1003             if not self.email_warning:
1004                 print("Email not configured, not sending.")
1005                 self.email_warning = True
1006             return
1007
1008         build_time = build_time.replace(microsecond=0)
1009         subject = (self.bot_config['email-subject'] %
1010                    {'action': action,
1011                     'build-time': str(build_time)})
1012         results = self.build_state[action]['build-results']
1013         changes = self.build_state[action]['result-changes']
1014         ever_passed = set(self.build_state[action]['ever-passed'])
1015         versions = self.build_state[action]['build-versions']
1016         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1017         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1018         all_fails = {k for k in results if results[k] == 'FAIL'}
1019         if new_regressions:
1020             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1021             new_reg_text = ('New regressions:\n\n%s\n\n' %
1022                             '\n'.join(new_reg_list))
1023         else:
1024             new_reg_text = ''
1025         if all_regressions:
1026             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1027             all_reg_text = ('All regressions:\n\n%s\n\n' %
1028                             '\n'.join(all_reg_list))
1029         else:
1030             all_reg_text = ''
1031         if all_fails:
1032             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1033             all_fail_text = ('All failures:\n\n%s\n\n' %
1034                              '\n'.join(all_fail_list))
1035         else:
1036             all_fail_text = ''
1037         if changes:
1038             changes_list = sorted(changes.keys())
1039             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1040             changes_text = ('All changed results:\n\n%s\n\n' %
1041                             '\n'.join(changes_list))
1042         else:
1043             changes_text = ''
1044         results_text = (new_reg_text + all_reg_text + all_fail_text +
1045                         changes_text)
1046         if not results_text:
1047             results_text = 'Clean build with unchanged results.\n\n'
1048         versions_list = sorted(versions.keys())
1049         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1050                                           versions[k]['revision'])
1051                          for k in versions_list]
1052         versions_text = ('Component versions for this build:\n\n%s\n' %
1053                          '\n'.join(versions_list))
1054         body_text = results_text + versions_text
1055         msg = email.mime.text.MIMEText(body_text)
1056         msg['Subject'] = subject
1057         msg['From'] = self.bot_config['email-from']
1058         msg['To'] = self.bot_config['email-to']
1059         msg['Message-ID'] = email.utils.make_msgid()
1060         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1061         with smtplib.SMTP(self.bot_config['email-server']) as s:
1062             s.send_message(msg)
1063
1064     def bot_run_self(self, opts, action, check=True):
1065         """Run a copy of this script with given options."""
1066         cmd = [sys.executable, sys.argv[0], '--keep=none',
1067                '-j%d' % self.parallelism]
1068         cmd.extend(opts)
1069         cmd.extend([self.topdir, action])
1070         sys.stdout.flush()
1071         subprocess.run(cmd, check=check)
1072
1073     def bot(self):
1074         """Run repeated rounds of checkout and builds."""
1075         while True:
1076             self.load_bot_config_json()
1077             if not self.bot_config['run']:
1078                 print('Bot exiting by request.')
1079                 exit(0)
1080             self.bot_run_self([], 'bot-cycle', check=False)
1081             self.load_bot_config_json()
1082             if not self.bot_config['run']:
1083                 print('Bot exiting by request.')
1084                 exit(0)
1085             time.sleep(self.bot_config['delay'])
1086             if self.get_script_text() != self.script_text:
1087                 print('Script changed, bot re-execing.')
1088                 self.exec_self()
1089
1090
1091 class Config(object):
1092     """A configuration for building a compiler and associated libraries."""
1093
1094     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1095                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1096         """Initialize a Config object."""
1097         self.ctx = ctx
1098         self.arch = arch
1099         self.os = os_name
1100         self.variant = variant
1101         if variant is None:
1102             self.name = '%s-%s' % (arch, os_name)
1103         else:
1104             self.name = '%s-%s-%s' % (arch, os_name, variant)
1105         self.triplet = '%s-glibc-%s' % (arch, os_name)
1106         if gcc_cfg is None:
1107             self.gcc_cfg = []
1108         else:
1109             self.gcc_cfg = gcc_cfg
1110         if first_gcc_cfg is None:
1111             self.first_gcc_cfg = []
1112         else:
1113             self.first_gcc_cfg = first_gcc_cfg
1114         if glibcs is None:
1115             glibcs = [{'variant': variant}]
1116         if extra_glibcs is None:
1117             extra_glibcs = []
1118         glibcs = [Glibc(self, **g) for g in glibcs]
1119         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1120         self.all_glibcs = glibcs + extra_glibcs
1121         self.compiler_glibcs = glibcs
1122         self.installdir = ctx.compiler_installdir(self.name)
1123         self.bindir = ctx.compiler_bindir(self.name)
1124         self.sysroot = ctx.compiler_sysroot(self.name)
1125         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1126         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1127
1128     def component_builddir(self, component):
1129         """Return the directory to use for a (non-glibc) build."""
1130         return self.ctx.component_builddir('compilers', self.name, component)
1131
1132     def build(self):
1133         """Generate commands to build this compiler."""
1134         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1135                                       self.logsdir)
1136         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1137         cmdlist.add_command('check-host-libraries',
1138                             ['test', '-f',
1139                              os.path.join(self.ctx.host_libraries_installdir,
1140                                           'ok')])
1141         cmdlist.use_path(self.bindir)
1142         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1143                               ['--disable-gdb',
1144                                '--disable-libdecnumber',
1145                                '--disable-readline',
1146                                '--disable-sim'])
1147         if self.os.startswith('linux'):
1148             self.install_linux_headers(cmdlist)
1149         self.build_gcc(cmdlist, True)
1150         for g in self.compiler_glibcs:
1151             cmdlist.push_subdesc('glibc')
1152             cmdlist.push_subdesc(g.name)
1153             g.build_glibc(cmdlist, True)
1154             cmdlist.pop_subdesc()
1155             cmdlist.pop_subdesc()
1156         self.build_gcc(cmdlist, False)
1157         cmdlist.add_command('done', ['touch',
1158                                      os.path.join(self.installdir, 'ok')])
1159         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1160                                       self.logsdir)
1161
1162     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1163         """Build one cross tool."""
1164         srcdir = self.ctx.component_srcdir(tool_src)
1165         builddir = self.component_builddir(tool_build)
1166         cmdlist.push_subdesc(tool_build)
1167         cmdlist.create_use_dir(builddir)
1168         cfg_cmd = [os.path.join(srcdir, 'configure'),
1169                    '--prefix=%s' % self.installdir,
1170                    '--build=%s' % self.ctx.build_triplet,
1171                    '--host=%s' % self.ctx.build_triplet,
1172                    '--target=%s' % self.triplet,
1173                    '--with-sysroot=%s' % self.sysroot]
1174         if extra_opts:
1175             cfg_cmd.extend(extra_opts)
1176         cmdlist.add_command('configure', cfg_cmd)
1177         cmdlist.add_command('build', ['make'])
1178         # Parallel "make install" for GCC has race conditions that can
1179         # cause it to fail; see
1180         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1181         # problems are not known for binutils, but doing the
1182         # installation in parallel within a particular toolchain build
1183         # (as opposed to installation of one toolchain from
1184         # build-many-glibcs.py running in parallel to the installation
1185         # of other toolchains being built) is not known to be
1186         # significantly beneficial, so it is simplest just to disable
1187         # parallel install for cross tools here.
1188         cmdlist.add_command('install', ['make', '-j1', 'install'])
1189         cmdlist.cleanup_dir()
1190         cmdlist.pop_subdesc()
1191
1192     def install_linux_headers(self, cmdlist):
1193         """Install Linux kernel headers."""
1194         arch_map = {'aarch64': 'arm64',
1195                     'alpha': 'alpha',
1196                     'arm': 'arm',
1197                     'hppa': 'parisc',
1198                     'i486': 'x86',
1199                     'i586': 'x86',
1200                     'i686': 'x86',
1201                     'i786': 'x86',
1202                     'ia64': 'ia64',
1203                     'm68k': 'm68k',
1204                     'microblaze': 'microblaze',
1205                     'mips': 'mips',
1206                     'nios2': 'nios2',
1207                     'powerpc': 'powerpc',
1208                     's390': 's390',
1209                     'sh': 'sh',
1210                     'sparc': 'sparc',
1211                     'tile': 'tile',
1212                     'x86_64': 'x86'}
1213         linux_arch = None
1214         for k in arch_map:
1215             if self.arch.startswith(k):
1216                 linux_arch = arch_map[k]
1217                 break
1218         assert linux_arch is not None
1219         srcdir = self.ctx.component_srcdir('linux')
1220         builddir = self.component_builddir('linux')
1221         headers_dir = os.path.join(self.sysroot, 'usr')
1222         cmdlist.push_subdesc('linux')
1223         cmdlist.create_use_dir(builddir)
1224         cmdlist.add_command('install-headers',
1225                             ['make', '-C', srcdir, 'O=%s' % builddir,
1226                              'ARCH=%s' % linux_arch,
1227                              'INSTALL_HDR_PATH=%s' % headers_dir,
1228                              'headers_install'])
1229         cmdlist.cleanup_dir()
1230         cmdlist.pop_subdesc()
1231
1232     def build_gcc(self, cmdlist, bootstrap):
1233         """Build GCC."""
1234         # libsanitizer commonly breaks because of glibc header
1235         # changes, or on unusual targets.  libssp is of little
1236         # relevance with glibc's own stack checking support.
1237         cfg_opts = list(self.gcc_cfg)
1238         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1239         host_libs = self.ctx.host_libraries_installdir
1240         cfg_opts += ['--with-gmp=%s' % host_libs,
1241                      '--with-mpfr=%s' % host_libs,
1242                      '--with-mpc=%s' % host_libs]
1243         if bootstrap:
1244             tool_build = 'gcc-first'
1245             # Building a static-only, C-only compiler that is
1246             # sufficient to build glibc.  Various libraries and
1247             # features that may require libc headers must be disabled.
1248             # When configuring with a sysroot, --with-newlib is
1249             # required to define inhibit_libc (to stop some parts of
1250             # libgcc including libc headers); --without-headers is not
1251             # sufficient.
1252             cfg_opts += ['--enable-languages=c', '--disable-shared',
1253                          '--disable-threads',
1254                          '--disable-libatomic',
1255                          '--disable-decimal-float',
1256                          '--disable-libffi',
1257                          '--disable-libgomp',
1258                          '--disable-libitm',
1259                          '--disable-libmpx',
1260                          '--disable-libquadmath',
1261                          '--without-headers', '--with-newlib',
1262                          '--with-glibc-version=%s' % self.ctx.glibc_version
1263                          ]
1264             cfg_opts += self.first_gcc_cfg
1265         else:
1266             tool_build = 'gcc'
1267             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1268                          '--enable-threads']
1269         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1270
1271
1272 class Glibc(object):
1273     """A configuration for building glibc."""
1274
1275     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1276                  cfg=None, ccopts=None):
1277         """Initialize a Glibc object."""
1278         self.ctx = compiler.ctx
1279         self.compiler = compiler
1280         if arch is None:
1281             self.arch = compiler.arch
1282         else:
1283             self.arch = arch
1284         if os_name is None:
1285             self.os = compiler.os
1286         else:
1287             self.os = os_name
1288         self.variant = variant
1289         if variant is None:
1290             self.name = '%s-%s' % (self.arch, self.os)
1291         else:
1292             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1293         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1294         if cfg is None:
1295             self.cfg = []
1296         else:
1297             self.cfg = cfg
1298         self.ccopts = ccopts
1299
1300     def tool_name(self, tool):
1301         """Return the name of a cross-compilation tool."""
1302         ctool = '%s-%s' % (self.compiler.triplet, tool)
1303         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1304             ctool = '%s %s' % (ctool, self.ccopts)
1305         return ctool
1306
1307     def build(self):
1308         """Generate commands to build this glibc."""
1309         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1310         installdir = self.ctx.glibc_installdir(self.name)
1311         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1312         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1313         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1314         cmdlist.add_command('check-compilers',
1315                             ['test', '-f',
1316                              os.path.join(self.compiler.installdir, 'ok')])
1317         cmdlist.use_path(self.compiler.bindir)
1318         self.build_glibc(cmdlist, False)
1319         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1320                                       logsdir)
1321
1322     def build_glibc(self, cmdlist, for_compiler):
1323         """Generate commands to build this glibc, either as part of a compiler
1324         build or with the bootstrapped compiler (and in the latter case, run
1325         tests as well)."""
1326         srcdir = self.ctx.component_srcdir('glibc')
1327         if for_compiler:
1328             builddir = self.ctx.component_builddir('compilers',
1329                                                    self.compiler.name, 'glibc',
1330                                                    self.name)
1331             installdir = self.compiler.sysroot
1332             srcdir_copy = self.ctx.component_builddir('compilers',
1333                                                       self.compiler.name,
1334                                                       'glibc-src',
1335                                                       self.name)
1336         else:
1337             builddir = self.ctx.component_builddir('glibcs', self.name,
1338                                                    'glibc')
1339             installdir = self.ctx.glibc_installdir(self.name)
1340             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1341                                                       'glibc-src')
1342         cmdlist.create_use_dir(builddir)
1343         # glibc builds write into the source directory, and even if
1344         # not intentionally there is a risk of bugs that involve
1345         # writing into the working directory.  To avoid possible
1346         # concurrency issues, copy the source directory.
1347         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1348         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1349                    '--prefix=/usr',
1350                    '--enable-profile',
1351                    '--build=%s' % self.ctx.build_triplet,
1352                    '--host=%s' % self.triplet,
1353                    'CC=%s' % self.tool_name('gcc'),
1354                    'CXX=%s' % self.tool_name('g++'),
1355                    'AR=%s' % self.tool_name('ar'),
1356                    'AS=%s' % self.tool_name('as'),
1357                    'LD=%s' % self.tool_name('ld'),
1358                    'NM=%s' % self.tool_name('nm'),
1359                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1360                    'OBJDUMP=%s' % self.tool_name('objdump'),
1361                    'RANLIB=%s' % self.tool_name('ranlib'),
1362                    'READELF=%s' % self.tool_name('readelf'),
1363                    'STRIP=%s' % self.tool_name('strip')]
1364         cfg_cmd += self.cfg
1365         cmdlist.add_command('configure', cfg_cmd)
1366         cmdlist.add_command('build', ['make'])
1367         cmdlist.add_command('install', ['make', 'install',
1368                                         'install_root=%s' % installdir])
1369         # GCC uses paths such as lib/../lib64, so make sure lib
1370         # directories always exist.
1371         cmdlist.add_command('mkdir-lib', ['mkdir', '-p',
1372                                           os.path.join(installdir, 'lib'),
1373                                           os.path.join(installdir,
1374                                                        'usr', 'lib')])
1375         if not for_compiler:
1376             if self.ctx.strip:
1377                 cmdlist.add_command('strip',
1378                                     ['sh', '-c',
1379                                      ('%s %s/lib*/*.so' %
1380                                       (self.tool_name('strip'), installdir))])
1381             cmdlist.add_command('check', ['make', 'check'])
1382             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1383                                 always_run=True)
1384         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1385         cmdlist.cleanup_dir()
1386
1387
1388 class Command(object):
1389     """A command run in the build process."""
1390
1391     def __init__(self, desc, num, dir, path, command, always_run=False):
1392         """Initialize a Command object."""
1393         self.dir = dir
1394         self.path = path
1395         self.desc = desc
1396         trans = str.maketrans({' ': '-'})
1397         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1398         self.command = command
1399         self.always_run = always_run
1400
1401     @staticmethod
1402     def shell_make_quote_string(s):
1403         """Given a string not containing a newline, quote it for use by the
1404         shell and make."""
1405         assert '\n' not in s
1406         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1407             return s
1408         strans = str.maketrans({"'": "'\\''"})
1409         s = "'%s'" % s.translate(strans)
1410         mtrans = str.maketrans({'$': '$$'})
1411         return s.translate(mtrans)
1412
1413     @staticmethod
1414     def shell_make_quote_list(l, translate_make):
1415         """Given a list of strings not containing newlines, quote them for use
1416         by the shell and make, returning a single string.  If translate_make
1417         is true and the first string is 'make', change it to $(MAKE)."""
1418         l = [Command.shell_make_quote_string(s) for s in l]
1419         if translate_make and l[0] == 'make':
1420             l[0] = '$(MAKE)'
1421         return ' '.join(l)
1422
1423     def shell_make_quote(self):
1424         """Return this command quoted for the shell and make."""
1425         return self.shell_make_quote_list(self.command, True)
1426
1427
1428 class CommandList(object):
1429     """A list of commands run in the build process."""
1430
1431     def __init__(self, desc, keep):
1432         """Initialize a CommandList object."""
1433         self.cmdlist = []
1434         self.dir = None
1435         self.path = None
1436         self.desc = [desc]
1437         self.keep = keep
1438
1439     def desc_txt(self, desc):
1440         """Return the description to use for a command."""
1441         return '%s %s' % (' '.join(self.desc), desc)
1442
1443     def use_dir(self, dir):
1444         """Set the default directory for subsequent commands."""
1445         self.dir = dir
1446
1447     def use_path(self, path):
1448         """Set a directory to be prepended to the PATH for subsequent
1449         commands."""
1450         self.path = path
1451
1452     def push_subdesc(self, subdesc):
1453         """Set the default subdescription for subsequent commands (e.g., the
1454         name of a component being built, within the series of commands
1455         building it)."""
1456         self.desc.append(subdesc)
1457
1458     def pop_subdesc(self):
1459         """Pop a subdescription from the list of descriptions."""
1460         self.desc.pop()
1461
1462     def create_use_dir(self, dir):
1463         """Remove and recreate a directory and use it for subsequent
1464         commands."""
1465         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1466         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1467         self.use_dir(dir)
1468
1469     def create_copy_dir(self, src, dest):
1470         """Remove a directory and recreate it as a copy from the given
1471         source."""
1472         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1473         parent = os.path.dirname(dest)
1474         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1475         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1476
1477     def add_command_dir(self, desc, dir, command, always_run=False):
1478         """Add a command to run in a given directory."""
1479         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1480                       command, always_run)
1481         self.cmdlist.append(cmd)
1482
1483     def add_command(self, desc, command, always_run=False):
1484         """Add a command to run in the default directory."""
1485         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1486                       self.path, command, always_run)
1487         self.cmdlist.append(cmd)
1488
1489     def cleanup_dir(self, desc='cleanup', dir=None):
1490         """Clean up a build directory.  If no directory is specified, the
1491         default directory is cleaned up and ceases to be the default
1492         directory."""
1493         if dir is None:
1494             dir = self.dir
1495             self.use_dir(None)
1496         if self.keep != 'all':
1497             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1498                                  always_run=(self.keep == 'none'))
1499
1500     def makefile_commands(self, wrapper, logsdir):
1501         """Return the sequence of commands in the form of text for a Makefile.
1502         The given wrapper script takes arguments: base of logs for
1503         previous command, or empty; base of logs for this command;
1504         description; directory; PATH addition; the command itself."""
1505         # prev_base is the base of the name for logs of the previous
1506         # command that is not always-run (that is, a build command,
1507         # whose failure should stop subsequent build commands from
1508         # being run, as opposed to a cleanup command, which is run
1509         # even if previous commands failed).
1510         prev_base = ''
1511         cmds = []
1512         for c in self.cmdlist:
1513             ctxt = c.shell_make_quote()
1514             if prev_base and not c.always_run:
1515                 prev_log = os.path.join(logsdir, prev_base)
1516             else:
1517                 prev_log = ''
1518             this_log = os.path.join(logsdir, c.logbase)
1519             if not c.always_run:
1520                 prev_base = c.logbase
1521             if c.dir is None:
1522                 dir = ''
1523             else:
1524                 dir = c.dir
1525             if c.path is None:
1526                 path = ''
1527             else:
1528                 path = c.path
1529             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1530             prelim_txt = Command.shell_make_quote_list(prelims, False)
1531             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1532         return '\n'.join(cmds)
1533
1534     def status_logs(self, logsdir):
1535         """Return the list of log files with command status."""
1536         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1537                 for c in self.cmdlist]
1538
1539
1540 def get_parser():
1541     """Return an argument parser for this module."""
1542     parser = argparse.ArgumentParser(description=__doc__)
1543     parser.add_argument('-j', dest='parallelism',
1544                         help='Run this number of jobs in parallel',
1545                         type=int, default=os.cpu_count())
1546     parser.add_argument('--keep', dest='keep',
1547                         help='Whether to keep all build directories, '
1548                         'none or only those from failed builds',
1549                         default='none', choices=('none', 'all', 'failed'))
1550     parser.add_argument('--replace-sources', action='store_true',
1551                         help='Remove and replace source directories '
1552                         'with the wrong version of a component')
1553     parser.add_argument('--strip', action='store_true',
1554                         help='Strip installed glibc libraries')
1555     parser.add_argument('topdir',
1556                         help='Toplevel working directory')
1557     parser.add_argument('action',
1558                         help='What to do',
1559                         choices=('checkout', 'bot-cycle', 'bot',
1560                                  'host-libraries', 'compilers', 'glibcs'))
1561     parser.add_argument('configs',
1562                         help='Versions to check out or configurations to build',
1563                         nargs='*')
1564     return parser
1565
1566
1567 def main(argv):
1568     """The main entry point."""
1569     parser = get_parser()
1570     opts = parser.parse_args(argv)
1571     topdir = os.path.abspath(opts.topdir)
1572     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1573                   opts.strip, opts.action)
1574     ctx.run_builds(opts.action, opts.configs)
1575
1576
1577 if __name__ == '__main__':
1578     main(sys.argv[1:])