scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2018 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52 try:
  53     os.cpu_count
  54 except:
  55     import multiprocessing
  56     os.cpu_count = lambda: multiprocessing.cpu_count()
  57
  58 try:
  59     re.fullmatch
  60 except:
  61     re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)
  62
  63 try:
  64     subprocess.run
  65 except:
  66     class _CompletedProcess:
  67         def __init__(self, args, returncode, stdout=None, stderr=None):
  68             self.args = args
  69             self.returncode = returncode
  70             self.stdout = stdout
  71             self.stderr = stderr
  72
  73     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  74         assert(timeout is None)
  75         with subprocess.Popen(*popenargs, **kwargs) as process:
  76             try:
  77                 stdout, stderr = process.communicate(input)
  78             except:
  79                 process.kill()
  80                 process.wait()
  81                 raise
  82             returncode = process.poll()
  83             if check and returncode:
  84                 raise subprocess.CalledProcessError(returncode, popenargs)
  85         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  86
  87     subprocess.run = _run
  88
  89
  90 class Context(object):
  91     """The global state associated with builds in a given directory."""
  92
  93     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  94                  action):
  95         """Initialize the context."""
  96         self.topdir = topdir
  97         self.parallelism = parallelism
  98         self.keep = keep
  99         self.replace_sources = replace_sources
 100         self.strip = strip
 101         self.srcdir = os.path.join(topdir, 'src')
 102         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 103         self.build_state_json = os.path.join(topdir, 'build-state.json')
 104         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 105         self.installdir = os.path.join(topdir, 'install')
 106         self.host_libraries_installdir = os.path.join(self.installdir,
 107                                                       'host-libraries')
 108         self.builddir = os.path.join(topdir, 'build')
 109         self.logsdir = os.path.join(topdir, 'logs')
 110         self.logsdir_old = os.path.join(topdir, 'logs-old')
 111         self.makefile = os.path.join(self.builddir, 'Makefile')
 112         self.wrapper = os.path.join(self.builddir, 'wrapper')
 113         self.save_logs = os.path.join(self.builddir, 'save-logs')
 114         self.script_text = self.get_script_text()
 115         if action != 'checkout':
 116             self.build_triplet = self.get_build_triplet()
 117             self.glibc_version = self.get_glibc_version()
 118         self.configs = {}
 119         self.glibc_configs = {}
 120         self.makefile_pieces = ['.PHONY: all\n']
 121         self.add_all_configs()
 122         self.load_versions_json()
 123         self.load_build_state_json()
 124         self.status_log_list = []
 125         self.email_warning = False
 126
 127     def get_script_text(self):
 128         """Return the text of this script."""
 129         with open(sys.argv[0], 'r') as f:
 130             return f.read()
 131
 132     def exec_self(self):
 133         """Re-execute this script with the same arguments."""
 134         sys.stdout.flush()
 135         os.execv(sys.executable, [sys.executable] + sys.argv)
 136
 137     def get_build_triplet(self):
 138         """Determine the build triplet with config.guess."""
 139         config_guess = os.path.join(self.component_srcdir('gcc'),
 140                                     'config.guess')
 141         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 142                                 check=True, universal_newlines=True).stdout
 143         return cg_out.rstrip()
 144
 145     def get_glibc_version(self):
 146         """Determine the glibc version number (major.minor)."""
 147         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 148         with open(version_h, 'r') as f:
 149             lines = f.readlines()
 150         starttext = '#define VERSION "'
 151         for l in lines:
 152             if l.startswith(starttext):
 153                 l = l[len(starttext):]
 154                 l = l.rstrip('"\n')
 155                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 156                 return '%s.%s' % m.group(1, 2)
 157         print('error: could not determine glibc version')
 158         exit(1)
 159
 160     def add_all_configs(self):
 161         """Add all known glibc build configurations."""
 162         self.add_config(arch='aarch64',
 163                         os_name='linux-gnu',
 164                         extra_glibcs=[{'variant': 'disable-multi-arch',
 165                                        'cfg': ['--disable-multi-arch']}])
 166         self.add_config(arch='aarch64_be',
 167                         os_name='linux-gnu')
 168         self.add_config(arch='alpha',
 169                         os_name='linux-gnu')
 170         self.add_config(arch='arm',
 171                         os_name='linux-gnueabi')
 172         self.add_config(arch='armeb',
 173                         os_name='linux-gnueabi')
 174         self.add_config(arch='armeb',
 175                         os_name='linux-gnueabi',
 176                         variant='be8',
 177                         gcc_cfg=['--with-arch=armv7-a'])
 178         self.add_config(arch='arm',
 179                         os_name='linux-gnueabihf',
 180                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 181                         extra_glibcs=[{'variant': 'v7a',
 182                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 183                                       {'variant': 'v7a-disable-multi-arch',
 184                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 185                                        'cfg': ['--disable-multi-arch']}])
 186         self.add_config(arch='armeb',
 187                         os_name='linux-gnueabihf',
 188                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 189         self.add_config(arch='armeb',
 190                         os_name='linux-gnueabihf',
 191                         variant='be8',
 192                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 193                                  '--with-fpu=vfpv3'])
 194         self.add_config(arch='hppa',
 195                         os_name='linux-gnu')
 196         self.add_config(arch='i686',
 197                         os_name='gnu')
 198         self.add_config(arch='ia64',
 199                         os_name='linux-gnu',
 200                         first_gcc_cfg=['--with-system-libunwind'])
 201         self.add_config(arch='m68k',
 202                         os_name='linux-gnu',
 203                         gcc_cfg=['--disable-multilib'])
 204         self.add_config(arch='m68k',
 205                         os_name='linux-gnu',
 206                         variant='coldfire',
 207                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 208         self.add_config(arch='microblaze',
 209                         os_name='linux-gnu',
 210                         gcc_cfg=['--disable-multilib'])
 211         self.add_config(arch='microblazeel',
 212                         os_name='linux-gnu',
 213                         gcc_cfg=['--disable-multilib'])
 214         self.add_config(arch='mips64',
 215                         os_name='linux-gnu',
 216                         gcc_cfg=['--with-mips-plt'],
 217                         glibcs=[{'variant': 'n32'},
 218                                 {'arch': 'mips',
 219                                  'ccopts': '-mabi=32'},
 220                                 {'variant': 'n64',
 221                                  'ccopts': '-mabi=64'}])
 222         self.add_config(arch='mips64',
 223                         os_name='linux-gnu',
 224                         variant='soft',
 225                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 226                         glibcs=[{'variant': 'n32-soft'},
 227                                 {'variant': 'soft',
 228                                  'arch': 'mips',
 229                                  'ccopts': '-mabi=32'},
 230                                 {'variant': 'n64-soft',
 231                                  'ccopts': '-mabi=64'}])
 232         self.add_config(arch='mips64',
 233                         os_name='linux-gnu',
 234                         variant='nan2008',
 235                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 236                                  '--with-arch-64=mips64r2',
 237                                  '--with-arch-32=mips32r2'],
 238                         glibcs=[{'variant': 'n32-nan2008'},
 239                                 {'variant': 'nan2008',
 240                                  'arch': 'mips',
 241                                  'ccopts': '-mabi=32'},
 242                                 {'variant': 'n64-nan2008',
 243                                  'ccopts': '-mabi=64'}])
 244         self.add_config(arch='mips64',
 245                         os_name='linux-gnu',
 246                         variant='nan2008-soft',
 247                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 248                                  '--with-arch-64=mips64r2',
 249                                  '--with-arch-32=mips32r2',
 250                                  '--with-float=soft'],
 251                         glibcs=[{'variant': 'n32-nan2008-soft'},
 252                                 {'variant': 'nan2008-soft',
 253                                  'arch': 'mips',
 254                                  'ccopts': '-mabi=32'},
 255                                 {'variant': 'n64-nan2008-soft',
 256                                  'ccopts': '-mabi=64'}])
 257         self.add_config(arch='mips64el',
 258                         os_name='linux-gnu',
 259                         gcc_cfg=['--with-mips-plt'],
 260                         glibcs=[{'variant': 'n32'},
 261                                 {'arch': 'mipsel',
 262                                  'ccopts': '-mabi=32'},
 263                                 {'variant': 'n64',
 264                                  'ccopts': '-mabi=64'}])
 265         self.add_config(arch='mips64el',
 266                         os_name='linux-gnu',
 267                         variant='soft',
 268                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 269                         glibcs=[{'variant': 'n32-soft'},
 270                                 {'variant': 'soft',
 271                                  'arch': 'mipsel',
 272                                  'ccopts': '-mabi=32'},
 273                                 {'variant': 'n64-soft',
 274                                  'ccopts': '-mabi=64'}])
 275         self.add_config(arch='mips64el',
 276                         os_name='linux-gnu',
 277                         variant='nan2008',
 278                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 279                                  '--with-arch-64=mips64r2',
 280                                  '--with-arch-32=mips32r2'],
 281                         glibcs=[{'variant': 'n32-nan2008'},
 282                                 {'variant': 'nan2008',
 283                                  'arch': 'mipsel',
 284                                  'ccopts': '-mabi=32'},
 285                                 {'variant': 'n64-nan2008',
 286                                  'ccopts': '-mabi=64'}])
 287         self.add_config(arch='mips64el',
 288                         os_name='linux-gnu',
 289                         variant='nan2008-soft',
 290                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 291                                  '--with-arch-64=mips64r2',
 292                                  '--with-arch-32=mips32r2',
 293                                  '--with-float=soft'],
 294                         glibcs=[{'variant': 'n32-nan2008-soft'},
 295                                 {'variant': 'nan2008-soft',
 296                                  'arch': 'mipsel',
 297                                  'ccopts': '-mabi=32'},
 298                                 {'variant': 'n64-nan2008-soft',
 299                                  'ccopts': '-mabi=64'}])
 300         self.add_config(arch='nios2',
 301                         os_name='linux-gnu')
 302         self.add_config(arch='powerpc',
 303                         os_name='linux-gnu',
 304                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 305                         extra_glibcs=[{'variant': 'power4',
 306                                        'ccopts': '-mcpu=power4',
 307                                        'cfg': ['--with-cpu=power4']}])
 308         self.add_config(arch='powerpc',
 309                         os_name='linux-gnu',
 310                         variant='soft',
 311                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 312                                  '--enable-secureplt'])
 313         self.add_config(arch='powerpc64',
 314                         os_name='linux-gnu',
 315                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 316         self.add_config(arch='powerpc64le',
 317                         os_name='linux-gnu',
 318                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 319         self.add_config(arch='powerpc',
 320                         os_name='linux-gnuspe',
 321                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 322                                  '--enable-e500-double'])
 323         self.add_config(arch='powerpc',
 324                         os_name='linux-gnuspe',
 325                         variant='e500v1',
 326                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 327         self.add_config(arch='s390x',
 328                         os_name='linux-gnu',
 329                         glibcs=[{},
 330                                 {'arch': 's390', 'ccopts': '-m31'}])
 331         self.add_config(arch='sh3',
 332                         os_name='linux-gnu')
 333         self.add_config(arch='sh3eb',
 334                         os_name='linux-gnu')
 335         self.add_config(arch='sh4',
 336                         os_name='linux-gnu')
 337         self.add_config(arch='sh4eb',
 338                         os_name='linux-gnu')
 339         self.add_config(arch='sh4',
 340                         os_name='linux-gnu',
 341                         variant='soft',
 342                         gcc_cfg=['--without-fp'])
 343         self.add_config(arch='sh4eb',
 344                         os_name='linux-gnu',
 345                         variant='soft',
 346                         gcc_cfg=['--without-fp'])
 347         self.add_config(arch='sparc64',
 348                         os_name='linux-gnu',
 349                         glibcs=[{},
 350                                 {'arch': 'sparcv9',
 351                                  'ccopts': '-m32 -mlong-double-128'}],
 352                         extra_glibcs=[{'variant': 'disable-multi-arch',
 353                                        'cfg': ['--disable-multi-arch']},
 354                                       {'variant': 'disable-multi-arch',
 355                                        'arch': 'sparcv9',
 356                                        'ccopts': '-m32 -mlong-double-128',
 357                                        'cfg': ['--disable-multi-arch']}])
 358         self.add_config(arch='tilegx',
 359                         os_name='linux-gnu',
 360                         glibcs=[{},
 361                                 {'variant': '32', 'ccopts': '-m32'}])
 362         self.add_config(arch='tilegxbe',
 363                         os_name='linux-gnu',
 364                         glibcs=[{},
 365                                 {'variant': '32', 'ccopts': '-m32'}])
 366         self.add_config(arch='x86_64',
 367                         os_name='linux-gnu',
 368                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 369                         glibcs=[{},
 370                                 {'variant': 'x32', 'ccopts': '-mx32'},
 371                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 372                         extra_glibcs=[{'variant': 'disable-multi-arch',
 373                                        'cfg': ['--disable-multi-arch']},
 374                                       {'variant': 'static-pie',
 375                                        'cfg': ['--enable-static-pie']},
 376                                       {'variant': 'x32-static-pie',
 377                                        'ccopts': '-mx32',
 378                                        'cfg': ['--enable-static-pie']},
 379                                       {'variant': 'static-pie',
 380                                        'arch': 'i686',
 381                                        'ccopts': '-m32 -march=i686',
 382                                        'cfg': ['--enable-static-pie']},
 383                                       {'variant': 'disable-multi-arch',
 384                                        'arch': 'i686',
 385                                        'ccopts': '-m32 -march=i686',
 386                                        'cfg': ['--disable-multi-arch']},
 387                                       {'arch': 'i486',
 388                                        'ccopts': '-m32 -march=i486'},
 389                                       {'arch': 'i586',
 390                                        'ccopts': '-m32 -march=i586'}])
 391
 392     def add_config(self, **args):
 393         """Add an individual build configuration."""
 394         cfg = Config(self, **args)
 395         if cfg.name in self.configs:
 396             print('error: duplicate config %s' % cfg.name)
 397             exit(1)
 398         self.configs[cfg.name] = cfg
 399         for c in cfg.all_glibcs:
 400             if c.name in self.glibc_configs:
 401                 print('error: duplicate glibc config %s' % c.name)
 402                 exit(1)
 403             self.glibc_configs[c.name] = c
 404
 405     def component_srcdir(self, component):
 406         """Return the source directory for a given component, e.g. gcc."""
 407         return os.path.join(self.srcdir, component)
 408
 409     def component_builddir(self, action, config, component, subconfig=None):
 410         """Return the directory to use for a build."""
 411         if config is None:
 412             # Host libraries.
 413             assert subconfig is None
 414             return os.path.join(self.builddir, action, component)
 415         if subconfig is None:
 416             return os.path.join(self.builddir, action, config, component)
 417         else:
 418             # glibc build as part of compiler build.
 419             return os.path.join(self.builddir, action, config, component,
 420                                 subconfig)
 421
 422     def compiler_installdir(self, config):
 423         """Return the directory in which to install a compiler."""
 424         return os.path.join(self.installdir, 'compilers', config)
 425
 426     def compiler_bindir(self, config):
 427         """Return the directory in which to find compiler binaries."""
 428         return os.path.join(self.compiler_installdir(config), 'bin')
 429
 430     def compiler_sysroot(self, config):
 431         """Return the sysroot directory for a compiler."""
 432         return os.path.join(self.compiler_installdir(config), 'sysroot')
 433
 434     def glibc_installdir(self, config):
 435         """Return the directory in which to install glibc."""
 436         return os.path.join(self.installdir, 'glibcs', config)
 437
 438     def run_builds(self, action, configs):
 439         """Run the requested builds."""
 440         if action == 'checkout':
 441             self.checkout(configs)
 442             return
 443         if action == 'bot-cycle':
 444             if configs:
 445                 print('error: configurations specified for bot-cycle')
 446                 exit(1)
 447             self.bot_cycle()
 448             return
 449         if action == 'bot':
 450             if configs:
 451                 print('error: configurations specified for bot')
 452                 exit(1)
 453             self.bot()
 454             return
 455         if action == 'host-libraries' and configs:
 456             print('error: configurations specified for host-libraries')
 457             exit(1)
 458         self.clear_last_build_state(action)
 459         build_time = datetime.datetime.utcnow()
 460         if action == 'host-libraries':
 461             build_components = ('gmp', 'mpfr', 'mpc')
 462             old_components = ()
 463             old_versions = {}
 464             self.build_host_libraries()
 465         elif action == 'compilers':
 466             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 467                                 'gnumach', 'hurd')
 468             old_components = ('gmp', 'mpfr', 'mpc')
 469             old_versions = self.build_state['host-libraries']['build-versions']
 470             self.build_compilers(configs)
 471         else:
 472             build_components = ('glibc',)
 473             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 474                               'mig', 'gnumach', 'hurd')
 475             old_versions = self.build_state['compilers']['build-versions']
 476             self.build_glibcs(configs)
 477         self.write_files()
 478         self.do_build()
 479         if configs:
 480             # Partial build, do not update stored state.
 481             return
 482         build_versions = {}
 483         for k in build_components:
 484             if k in self.versions:
 485                 build_versions[k] = {'version': self.versions[k]['version'],
 486                                      'revision': self.versions[k]['revision']}
 487         for k in old_components:
 488             if k in old_versions:
 489                 build_versions[k] = {'version': old_versions[k]['version'],
 490                                      'revision': old_versions[k]['revision']}
 491         self.update_build_state(action, build_time, build_versions)
 492
 493     @staticmethod
 494     def remove_dirs(*args):
 495         """Remove directories and their contents if they exist."""
 496         for dir in args:
 497             shutil.rmtree(dir, ignore_errors=True)
 498
 499     @staticmethod
 500     def remove_recreate_dirs(*args):
 501         """Remove directories if they exist, and create them as empty."""
 502         Context.remove_dirs(*args)
 503         for dir in args:
 504             os.makedirs(dir, exist_ok=True)
 505
 506     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 507         """Add makefile text for a list of commands."""
 508         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 509         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 510                                     (target, target, target, commands))
 511         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 512
 513     def write_files(self):
 514         """Write out the Makefile and wrapper script."""
 515         mftext = ''.join(self.makefile_pieces)
 516         with open(self.makefile, 'w') as f:
 517             f.write(mftext)
 518         wrapper_text = (
 519             '#!/bin/sh\n'
 520             'prev_base=$1\n'
 521             'this_base=$2\n'
 522             'desc=$3\n'
 523             'dir=$4\n'
 524             'path=$5\n'
 525             'shift 5\n'
 526             'prev_status=$prev_base-status.txt\n'
 527             'this_status=$this_base-status.txt\n'
 528             'this_log=$this_base-log.txt\n'
 529             'date > "$this_log"\n'
 530             'echo >> "$this_log"\n'
 531             'echo "Description: $desc" >> "$this_log"\n'
 532             'printf "%s" "Command:" >> "$this_log"\n'
 533             'for word in "$@"; do\n'
 534             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 535             '    printf " %s" "$word"\n'
 536             '  else\n'
 537             '    printf " \'"\n'
 538             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 539             '    printf "\'"\n'
 540             '  fi\n'
 541             'done >> "$this_log"\n'
 542             'echo >> "$this_log"\n'
 543             'echo "Directory: $dir" >> "$this_log"\n'
 544             'echo "Path addition: $path" >> "$this_log"\n'
 545             'echo >> "$this_log"\n'
 546             'record_status ()\n'
 547             '{\n'
 548             '  echo >> "$this_log"\n'
 549             '  echo "$1: $desc" > "$this_status"\n'
 550             '  echo "$1: $desc" >> "$this_log"\n'
 551             '  echo >> "$this_log"\n'
 552             '  date >> "$this_log"\n'
 553             '  echo "$1: $desc"\n'
 554             '  exit 0\n'
 555             '}\n'
 556             'check_error ()\n'
 557             '{\n'
 558             '  if [ "$1" != "0" ]; then\n'
 559             '    record_status FAIL\n'
 560             '  fi\n'
 561             '}\n'
 562             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 563             '    record_status UNRESOLVED\n'
 564             'fi\n'
 565             'if [ "$dir" ]; then\n'
 566             '  cd "$dir"\n'
 567             '  check_error "$?"\n'
 568             'fi\n'
 569             'if [ "$path" ]; then\n'
 570             '  PATH=$path:$PATH\n'
 571             'fi\n'
 572             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 573             'check_error "$?"\n'
 574             'record_status PASS\n')
 575         with open(self.wrapper, 'w') as f:
 576             f.write(wrapper_text)
 577         # Mode 0o755.
 578         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 579                      stat.S_IROTH|stat.S_IXOTH)
 580         os.chmod(self.wrapper, mode_exec)
 581         save_logs_text = (
 582             '#!/bin/sh\n'
 583             'if ! [ -f tests.sum ]; then\n'
 584             '  echo "No test summary available."\n'
 585             '  exit 0\n'
 586             'fi\n'
 587             'save_file ()\n'
 588             '{\n'
 589             '  echo "Contents of $1:"\n'
 590             '  echo\n'
 591             '  cat "$1"\n'
 592             '  echo\n'
 593             '  echo "End of contents of $1."\n'
 594             '  echo\n'
 595             '}\n'
 596             'save_file tests.sum\n'
 597             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 598             'for t in $non_pass_tests; do\n'
 599             '  if [ -f "$t.out" ]; then\n'
 600             '    save_file "$t.out"\n'
 601             '  fi\n'
 602             'done\n')
 603         with open(self.save_logs, 'w') as f:
 604             f.write(save_logs_text)
 605         os.chmod(self.save_logs, mode_exec)
 606
 607     def do_build(self):
 608         """Do the actual build."""
 609         cmd = ['make', '-j%d' % self.parallelism]
 610         subprocess.run(cmd, cwd=self.builddir, check=True)
 611
 612     def build_host_libraries(self):
 613         """Build the host libraries."""
 614         installdir = self.host_libraries_installdir
 615         builddir = os.path.join(self.builddir, 'host-libraries')
 616         logsdir = os.path.join(self.logsdir, 'host-libraries')
 617         self.remove_recreate_dirs(installdir, builddir, logsdir)
 618         cmdlist = CommandList('host-libraries', self.keep)
 619         self.build_host_library(cmdlist, 'gmp')
 620         self.build_host_library(cmdlist, 'mpfr',
 621                                 ['--with-gmp=%s' % installdir])
 622         self.build_host_library(cmdlist, 'mpc',
 623                                 ['--with-gmp=%s' % installdir,
 624                                 '--with-mpfr=%s' % installdir])
 625         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 626         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 627
 628     def build_host_library(self, cmdlist, lib, extra_opts=None):
 629         """Build one host library."""
 630         srcdir = self.component_srcdir(lib)
 631         builddir = self.component_builddir('host-libraries', None, lib)
 632         installdir = self.host_libraries_installdir
 633         cmdlist.push_subdesc(lib)
 634         cmdlist.create_use_dir(builddir)
 635         cfg_cmd = [os.path.join(srcdir, 'configure'),
 636                    '--prefix=%s' % installdir,
 637                    '--disable-shared']
 638         if extra_opts:
 639             cfg_cmd.extend (extra_opts)
 640         cmdlist.add_command('configure', cfg_cmd)
 641         cmdlist.add_command('build', ['make'])
 642         cmdlist.add_command('check', ['make', 'check'])
 643         cmdlist.add_command('install', ['make', 'install'])
 644         cmdlist.cleanup_dir()
 645         cmdlist.pop_subdesc()
 646
 647     def build_compilers(self, configs):
 648         """Build the compilers."""
 649         if not configs:
 650             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 651             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 652             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 653             configs = sorted(self.configs.keys())
 654         for c in configs:
 655             self.configs[c].build()
 656
 657     def build_glibcs(self, configs):
 658         """Build the glibcs."""
 659         if not configs:
 660             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 661             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 662             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 663             configs = sorted(self.glibc_configs.keys())
 664         for c in configs:
 665             self.glibc_configs[c].build()
 666
 667     def load_versions_json(self):
 668         """Load information about source directory versions."""
 669         if not os.access(self.versions_json, os.F_OK):
 670             self.versions = {}
 671             return
 672         with open(self.versions_json, 'r') as f:
 673             self.versions = json.load(f)
 674
 675     def store_json(self, data, filename):
 676         """Store information in a JSON file."""
 677         filename_tmp = filename + '.tmp'
 678         with open(filename_tmp, 'w') as f:
 679             json.dump(data, f, indent=2, sort_keys=True)
 680         os.rename(filename_tmp, filename)
 681
 682     def store_versions_json(self):
 683         """Store information about source directory versions."""
 684         self.store_json(self.versions, self.versions_json)
 685
 686     def set_component_version(self, component, version, explicit, revision):
 687         """Set the version information for a component."""
 688         self.versions[component] = {'version': version,
 689                                     'explicit': explicit,
 690                                     'revision': revision}
 691         self.store_versions_json()
 692
 693     def checkout(self, versions):
 694         """Check out the desired component versions."""
 695         default_versions = {'binutils': 'vcs-2.30',
 696                             'gcc': 'vcs-7',
 697                             'glibc': 'vcs-mainline',
 698                             'gmp': '6.1.2',
 699                             'linux': '4.14',
 700                             'mpc': '1.1.0',
 701                             'mpfr': '4.0.0',
 702                             'mig': '1.8',
 703                             'gnumach': '1.8',
 704                             'hurd': '0.9'}
 705         use_versions = {}
 706         explicit_versions = {}
 707         for v in versions:
 708             found_v = False
 709             for k in default_versions.keys():
 710                 kx = k + '-'
 711                 if v.startswith(kx):
 712                     vx = v[len(kx):]
 713                     if k in use_versions:
 714                         print('error: multiple versions for %s' % k)
 715                         exit(1)
 716                     use_versions[k] = vx
 717                     explicit_versions[k] = True
 718                     found_v = True
 719                     break
 720             if not found_v:
 721                 print('error: unknown component in %s' % v)
 722                 exit(1)
 723         for k in default_versions.keys():
 724             if k not in use_versions:
 725                 if k in self.versions and self.versions[k]['explicit']:
 726                     use_versions[k] = self.versions[k]['version']
 727                     explicit_versions[k] = True
 728                 else:
 729                     use_versions[k] = default_versions[k]
 730                     explicit_versions[k] = False
 731         os.makedirs(self.srcdir, exist_ok=True)
 732         for k in sorted(default_versions.keys()):
 733             update = os.access(self.component_srcdir(k), os.F_OK)
 734             v = use_versions[k]
 735             if (update and
 736                 k in self.versions and
 737                 v != self.versions[k]['version']):
 738                 if not self.replace_sources:
 739                     print('error: version of %s has changed from %s to %s, '
 740                           'use --replace-sources to check out again' %
 741                           (k, self.versions[k]['version'], v))
 742                     exit(1)
 743                 shutil.rmtree(self.component_srcdir(k))
 744                 update = False
 745             if v.startswith('vcs-'):
 746                 revision = self.checkout_vcs(k, v[4:], update)
 747             else:
 748                 self.checkout_tar(k, v, update)
 749                 revision = v
 750             self.set_component_version(k, v, explicit_versions[k], revision)
 751         if self.get_script_text() != self.script_text:
 752             # Rerun the checkout process in case the updated script
 753             # uses different default versions or new components.
 754             self.exec_self()
 755
 756     def checkout_vcs(self, component, version, update):
 757         """Check out the given version of the given component from version
 758         control.  Return a revision identifier."""
 759         if component == 'binutils':
 760             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 761             if version == 'mainline':
 762                 git_branch = 'master'
 763             else:
 764                 trans = str.maketrans({'.': '_'})
 765                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 766             return self.git_checkout(component, git_url, git_branch, update)
 767         elif component == 'gcc':
 768             if version == 'mainline':
 769                 branch = 'trunk'
 770             else:
 771                 trans = str.maketrans({'.': '_'})
 772                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 773             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 774             return self.gcc_checkout(svn_url, update)
 775         elif component == 'glibc':
 776             git_url = 'git://sourceware.org/git/glibc.git'
 777             if version == 'mainline':
 778                 git_branch = 'master'
 779             else:
 780                 git_branch = 'release/%s/master' % version
 781             r = self.git_checkout(component, git_url, git_branch, update)
 782             self.fix_glibc_timestamps()
 783             return r
 784         else:
 785             print('error: component %s coming from VCS' % component)
 786             exit(1)
 787
 788     def git_checkout(self, component, git_url, git_branch, update):
 789         """Check out a component from git.  Return a commit identifier."""
 790         if update:
 791             subprocess.run(['git', 'remote', 'prune', 'origin'],
 792                            cwd=self.component_srcdir(component), check=True)
 793             subprocess.run(['git', 'pull', '-q'],
 794                            cwd=self.component_srcdir(component), check=True)
 795         else:
 796             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 797                             self.component_srcdir(component)], check=True)
 798         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 799                            cwd=self.component_srcdir(component),
 800                            stdout=subprocess.PIPE,
 801                            check=True, universal_newlines=True).stdout
 802         return r.rstrip()
 803
 804     def fix_glibc_timestamps(self):
 805         """Fix timestamps in a glibc checkout."""
 806         # Ensure that builds do not try to regenerate generated files
 807         # in the source tree.
 808         srcdir = self.component_srcdir('glibc')
 809         for dirpath, dirnames, filenames in os.walk(srcdir):
 810             for f in filenames:
 811                 if (f == 'configure' or
 812                     f == 'preconfigure' or
 813                     f.endswith('-kw.h')):
 814                     to_touch = os.path.join(dirpath, f)
 815                     subprocess.run(['touch', to_touch], check=True)
 816
 817     def gcc_checkout(self, svn_url, update):
 818         """Check out GCC from SVN.  Return the revision number."""
 819         if not update:
 820             subprocess.run(['svn', 'co', '-q', svn_url,
 821                             self.component_srcdir('gcc')], check=True)
 822         subprocess.run(['contrib/gcc_update', '--silent'],
 823                        cwd=self.component_srcdir('gcc'), check=True)
 824         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 825                            stdout=subprocess.PIPE,
 826                            check=True, universal_newlines=True).stdout
 827         return r.rstrip()
 828
 829     def checkout_tar(self, component, version, update):
 830         """Check out the given version of the given component from a
 831         tarball."""
 832         if update:
 833             return
 834         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 835                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 836                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 837                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 838                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 839                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 840                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 841                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 842                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 843         if component not in url_map:
 844             print('error: component %s coming from tarball' % component)
 845             exit(1)
 846         url = url_map[component] % {'version': version}
 847         filename = os.path.join(self.srcdir, url.split('/')[-1])
 848         response = urllib.request.urlopen(url)
 849         data = response.read()
 850         with open(filename, 'wb') as f:
 851             f.write(data)
 852         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 853                        check=True)
 854         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 855                   self.component_srcdir(component))
 856         os.remove(filename)
 857
 858     def load_build_state_json(self):
 859         """Load information about the state of previous builds."""
 860         if os.access(self.build_state_json, os.F_OK):
 861             with open(self.build_state_json, 'r') as f:
 862                 self.build_state = json.load(f)
 863         else:
 864             self.build_state = {}
 865         for k in ('host-libraries', 'compilers', 'glibcs'):
 866             if k not in self.build_state:
 867                 self.build_state[k] = {}
 868             if 'build-time' not in self.build_state[k]:
 869                 self.build_state[k]['build-time'] = ''
 870             if 'build-versions' not in self.build_state[k]:
 871                 self.build_state[k]['build-versions'] = {}
 872             if 'build-results' not in self.build_state[k]:
 873                 self.build_state[k]['build-results'] = {}
 874             if 'result-changes' not in self.build_state[k]:
 875                 self.build_state[k]['result-changes'] = {}
 876             if 'ever-passed' not in self.build_state[k]:
 877                 self.build_state[k]['ever-passed'] = []
 878
 879     def store_build_state_json(self):
 880         """Store information about the state of previous builds."""
 881         self.store_json(self.build_state, self.build_state_json)
 882
 883     def clear_last_build_state(self, action):
 884         """Clear information about the state of part of the build."""
 885         # We clear the last build time and versions when starting a
 886         # new build.  The results of the last build are kept around,
 887         # as comparison is still meaningful if this build is aborted
 888         # and a new one started.
 889         self.build_state[action]['build-time'] = ''
 890         self.build_state[action]['build-versions'] = {}
 891         self.store_build_state_json()
 892
 893     def update_build_state(self, action, build_time, build_versions):
 894         """Update the build state after a build."""
 895         build_time = build_time.replace(microsecond=0)
 896         self.build_state[action]['build-time'] = str(build_time)
 897         self.build_state[action]['build-versions'] = build_versions
 898         build_results = {}
 899         for log in self.status_log_list:
 900             with open(log, 'r') as f:
 901                 log_text = f.read()
 902             log_text = log_text.rstrip()
 903             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 904             result = m.group(1)
 905             test_name = m.group(2)
 906             assert test_name not in build_results
 907             build_results[test_name] = result
 908         old_build_results = self.build_state[action]['build-results']
 909         self.build_state[action]['build-results'] = build_results
 910         result_changes = {}
 911         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 912         for t in all_tests:
 913             if t in old_build_results:
 914                 old_res = old_build_results[t]
 915             else:
 916                 old_res = '(New test)'
 917             if t in build_results:
 918                 new_res = build_results[t]
 919             else:
 920                 new_res = '(Test removed)'
 921             if old_res != new_res:
 922                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 923         self.build_state[action]['result-changes'] = result_changes
 924         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 925                            if t in build_results}
 926         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 927         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 928                                                          new_passes)
 929         self.store_build_state_json()
 930
 931     def load_bot_config_json(self):
 932         """Load bot configuration."""
 933         with open(self.bot_config_json, 'r') as f:
 934             self.bot_config = json.load(f)
 935
 936     def part_build_old(self, action, delay):
 937         """Return whether the last build for a given action was at least a
 938         given number of seconds ago, or does not have a time recorded."""
 939         old_time_str = self.build_state[action]['build-time']
 940         if not old_time_str:
 941             return True
 942         old_time = datetime.datetime.strptime(old_time_str,
 943                                               '%Y-%m-%d %H:%M:%S')
 944         new_time = datetime.datetime.utcnow()
 945         delta = new_time - old_time
 946         return delta.total_seconds() >= delay
 947
 948     def bot_cycle(self):
 949         """Run a single round of checkout and builds."""
 950         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 951         self.load_bot_config_json()
 952         actions = ('host-libraries', 'compilers', 'glibcs')
 953         self.bot_run_self(['--replace-sources'], 'checkout')
 954         self.load_versions_json()
 955         if self.get_script_text() != self.script_text:
 956             print('Script changed, re-execing.')
 957             # On script change, all parts of the build should be rerun.
 958             for a in actions:
 959                 self.clear_last_build_state(a)
 960             self.exec_self()
 961         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 962                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
 963                                           'mig', 'gnumach', 'hurd'),
 964                             'glibcs': ('glibc',)}
 965         must_build = {}
 966         for a in actions:
 967             build_vers = self.build_state[a]['build-versions']
 968             must_build[a] = False
 969             if not self.build_state[a]['build-time']:
 970                 must_build[a] = True
 971             old_vers = {}
 972             new_vers = {}
 973             for c in check_components[a]:
 974                 if c in build_vers:
 975                     old_vers[c] = build_vers[c]
 976                 new_vers[c] = {'version': self.versions[c]['version'],
 977                                'revision': self.versions[c]['revision']}
 978             if new_vers == old_vers:
 979                 print('Versions for %s unchanged.' % a)
 980             else:
 981                 print('Versions changed or rebuild forced for %s.' % a)
 982                 if a == 'compilers' and not self.part_build_old(
 983                         a, self.bot_config['compilers-rebuild-delay']):
 984                     print('Not requiring rebuild of compilers this soon.')
 985                 else:
 986                     must_build[a] = True
 987         if must_build['host-libraries']:
 988             must_build['compilers'] = True
 989         if must_build['compilers']:
 990             must_build['glibcs'] = True
 991         for a in actions:
 992             if must_build[a]:
 993                 print('Must rebuild %s.' % a)
 994                 self.clear_last_build_state(a)
 995             else:
 996                 print('No need to rebuild %s.' % a)
 997         if os.access(self.logsdir, os.F_OK):
 998             shutil.rmtree(self.logsdir_old, ignore_errors=True)
 999             shutil.copytree(self.logsdir, self.logsdir_old)
1000         for a in actions:
1001             if must_build[a]:
1002                 build_time = datetime.datetime.utcnow()
1003                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1004                 self.bot_run_self([], a)
1005                 self.load_build_state_json()
1006                 self.bot_build_mail(a, build_time)
1007         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1008
1009     def bot_build_mail(self, action, build_time):
1010         """Send email with the results of a build."""
1011         if not ('email-from' in self.bot_config and
1012                 'email-server' in self.bot_config and
1013                 'email-subject' in self.bot_config and
1014                 'email-to' in self.bot_config):
1015             if not self.email_warning:
1016                 print("Email not configured, not sending.")
1017                 self.email_warning = True
1018             return
1019
1020         build_time = build_time.replace(microsecond=0)
1021         subject = (self.bot_config['email-subject'] %
1022                    {'action': action,
1023                     'build-time': str(build_time)})
1024         results = self.build_state[action]['build-results']
1025         changes = self.build_state[action]['result-changes']
1026         ever_passed = set(self.build_state[action]['ever-passed'])
1027         versions = self.build_state[action]['build-versions']
1028         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1029         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1030         all_fails = {k for k in results if results[k] == 'FAIL'}
1031         if new_regressions:
1032             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1033             new_reg_text = ('New regressions:\n\n%s\n\n' %
1034                             '\n'.join(new_reg_list))
1035         else:
1036             new_reg_text = ''
1037         if all_regressions:
1038             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1039             all_reg_text = ('All regressions:\n\n%s\n\n' %
1040                             '\n'.join(all_reg_list))
1041         else:
1042             all_reg_text = ''
1043         if all_fails:
1044             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1045             all_fail_text = ('All failures:\n\n%s\n\n' %
1046                              '\n'.join(all_fail_list))
1047         else:
1048             all_fail_text = ''
1049         if changes:
1050             changes_list = sorted(changes.keys())
1051             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1052             changes_text = ('All changed results:\n\n%s\n\n' %
1053                             '\n'.join(changes_list))
1054         else:
1055             changes_text = ''
1056         results_text = (new_reg_text + all_reg_text + all_fail_text +
1057                         changes_text)
1058         if not results_text:
1059             results_text = 'Clean build with unchanged results.\n\n'
1060         versions_list = sorted(versions.keys())
1061         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1062                                           versions[k]['revision'])
1063                          for k in versions_list]
1064         versions_text = ('Component versions for this build:\n\n%s\n' %
1065                          '\n'.join(versions_list))
1066         body_text = results_text + versions_text
1067         msg = email.mime.text.MIMEText(body_text)
1068         msg['Subject'] = subject
1069         msg['From'] = self.bot_config['email-from']
1070         msg['To'] = self.bot_config['email-to']
1071         msg['Message-ID'] = email.utils.make_msgid()
1072         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1073         with smtplib.SMTP(self.bot_config['email-server']) as s:
1074             s.send_message(msg)
1075
1076     def bot_run_self(self, opts, action, check=True):
1077         """Run a copy of this script with given options."""
1078         cmd = [sys.executable, sys.argv[0], '--keep=none',
1079                '-j%d' % self.parallelism]
1080         cmd.extend(opts)
1081         cmd.extend([self.topdir, action])
1082         sys.stdout.flush()
1083         subprocess.run(cmd, check=check)
1084
1085     def bot(self):
1086         """Run repeated rounds of checkout and builds."""
1087         while True:
1088             self.load_bot_config_json()
1089             if not self.bot_config['run']:
1090                 print('Bot exiting by request.')
1091                 exit(0)
1092             self.bot_run_self([], 'bot-cycle', check=False)
1093             self.load_bot_config_json()
1094             if not self.bot_config['run']:
1095                 print('Bot exiting by request.')
1096                 exit(0)
1097             time.sleep(self.bot_config['delay'])
1098             if self.get_script_text() != self.script_text:
1099                 print('Script changed, bot re-execing.')
1100                 self.exec_self()
1101
1102
1103 class Config(object):
1104     """A configuration for building a compiler and associated libraries."""
1105
1106     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1107                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1108         """Initialize a Config object."""
1109         self.ctx = ctx
1110         self.arch = arch
1111         self.os = os_name
1112         self.variant = variant
1113         if variant is None:
1114             self.name = '%s-%s' % (arch, os_name)
1115         else:
1116             self.name = '%s-%s-%s' % (arch, os_name, variant)
1117         self.triplet = '%s-glibc-%s' % (arch, os_name)
1118         if gcc_cfg is None:
1119             self.gcc_cfg = []
1120         else:
1121             self.gcc_cfg = gcc_cfg
1122         if first_gcc_cfg is None:
1123             self.first_gcc_cfg = []
1124         else:
1125             self.first_gcc_cfg = first_gcc_cfg
1126         if glibcs is None:
1127             glibcs = [{'variant': variant}]
1128         if extra_glibcs is None:
1129             extra_glibcs = []
1130         glibcs = [Glibc(self, **g) for g in glibcs]
1131         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1132         self.all_glibcs = glibcs + extra_glibcs
1133         self.compiler_glibcs = glibcs
1134         self.installdir = ctx.compiler_installdir(self.name)
1135         self.bindir = ctx.compiler_bindir(self.name)
1136         self.sysroot = ctx.compiler_sysroot(self.name)
1137         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1138         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1139
1140     def component_builddir(self, component):
1141         """Return the directory to use for a (non-glibc) build."""
1142         return self.ctx.component_builddir('compilers', self.name, component)
1143
1144     def build(self):
1145         """Generate commands to build this compiler."""
1146         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1147                                       self.logsdir)
1148         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1149         cmdlist.add_command('check-host-libraries',
1150                             ['test', '-f',
1151                              os.path.join(self.ctx.host_libraries_installdir,
1152                                           'ok')])
1153         cmdlist.use_path(self.bindir)
1154         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1155                               ['--disable-gdb',
1156                                '--disable-libdecnumber',
1157                                '--disable-readline',
1158                                '--disable-sim'])
1159         if self.os.startswith('linux'):
1160             self.install_linux_headers(cmdlist)
1161         self.build_gcc(cmdlist, True)
1162         if self.os == 'gnu':
1163             self.install_gnumach_headers(cmdlist)
1164             self.build_cross_tool(cmdlist, 'mig', 'mig')
1165             self.install_hurd_headers(cmdlist)
1166         for g in self.compiler_glibcs:
1167             cmdlist.push_subdesc('glibc')
1168             cmdlist.push_subdesc(g.name)
1169             g.build_glibc(cmdlist, True)
1170             cmdlist.pop_subdesc()
1171             cmdlist.pop_subdesc()
1172         self.build_gcc(cmdlist, False)
1173         cmdlist.add_command('done', ['touch',
1174                                      os.path.join(self.installdir, 'ok')])
1175         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1176                                       self.logsdir)
1177
1178     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1179         """Build one cross tool."""
1180         srcdir = self.ctx.component_srcdir(tool_src)
1181         builddir = self.component_builddir(tool_build)
1182         cmdlist.push_subdesc(tool_build)
1183         cmdlist.create_use_dir(builddir)
1184         cfg_cmd = [os.path.join(srcdir, 'configure'),
1185                    '--prefix=%s' % self.installdir,
1186                    '--build=%s' % self.ctx.build_triplet,
1187                    '--host=%s' % self.ctx.build_triplet,
1188                    '--target=%s' % self.triplet,
1189                    '--with-sysroot=%s' % self.sysroot]
1190         if extra_opts:
1191             cfg_cmd.extend(extra_opts)
1192         cmdlist.add_command('configure', cfg_cmd)
1193         cmdlist.add_command('build', ['make'])
1194         # Parallel "make install" for GCC has race conditions that can
1195         # cause it to fail; see
1196         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1197         # problems are not known for binutils, but doing the
1198         # installation in parallel within a particular toolchain build
1199         # (as opposed to installation of one toolchain from
1200         # build-many-glibcs.py running in parallel to the installation
1201         # of other toolchains being built) is not known to be
1202         # significantly beneficial, so it is simplest just to disable
1203         # parallel install for cross tools here.
1204         cmdlist.add_command('install', ['make', '-j1', 'install'])
1205         cmdlist.cleanup_dir()
1206         cmdlist.pop_subdesc()
1207
1208     def install_linux_headers(self, cmdlist):
1209         """Install Linux kernel headers."""
1210         arch_map = {'aarch64': 'arm64',
1211                     'alpha': 'alpha',
1212                     'arm': 'arm',
1213                     'hppa': 'parisc',
1214                     'i486': 'x86',
1215                     'i586': 'x86',
1216                     'i686': 'x86',
1217                     'i786': 'x86',
1218                     'ia64': 'ia64',
1219                     'm68k': 'm68k',
1220                     'microblaze': 'microblaze',
1221                     'mips': 'mips',
1222                     'nios2': 'nios2',
1223                     'powerpc': 'powerpc',
1224                     's390': 's390',
1225                     'sh': 'sh',
1226                     'sparc': 'sparc',
1227                     'tile': 'tile',
1228                     'x86_64': 'x86'}
1229         linux_arch = None
1230         for k in arch_map:
1231             if self.arch.startswith(k):
1232                 linux_arch = arch_map[k]
1233                 break
1234         assert linux_arch is not None
1235         srcdir = self.ctx.component_srcdir('linux')
1236         builddir = self.component_builddir('linux')
1237         headers_dir = os.path.join(self.sysroot, 'usr')
1238         cmdlist.push_subdesc('linux')
1239         cmdlist.create_use_dir(builddir)
1240         cmdlist.add_command('install-headers',
1241                             ['make', '-C', srcdir, 'O=%s' % builddir,
1242                              'ARCH=%s' % linux_arch,
1243                              'INSTALL_HDR_PATH=%s' % headers_dir,
1244                              'headers_install'])
1245         cmdlist.cleanup_dir()
1246         cmdlist.pop_subdesc()
1247
1248     def install_gnumach_headers(self, cmdlist):
1249         """Install GNU Mach headers."""
1250         srcdir = self.ctx.component_srcdir('gnumach')
1251         builddir = self.component_builddir('gnumach')
1252         cmdlist.push_subdesc('gnumach')
1253         cmdlist.create_use_dir(builddir)
1254         cmdlist.add_command('configure',
1255                             [os.path.join(srcdir, 'configure'),
1256                              '--build=%s' % self.ctx.build_triplet,
1257                              '--host=%s' % self.triplet,
1258                              '--prefix=',
1259                              'CC=%s-gcc -nostdlib' % self.triplet])
1260         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1261                                         'install-data'])
1262         cmdlist.cleanup_dir()
1263         cmdlist.pop_subdesc()
1264
1265     def install_hurd_headers(self, cmdlist):
1266         """Install Hurd headers."""
1267         srcdir = self.ctx.component_srcdir('hurd')
1268         builddir = self.component_builddir('hurd')
1269         cmdlist.push_subdesc('hurd')
1270         cmdlist.create_use_dir(builddir)
1271         cmdlist.add_command('configure',
1272                             [os.path.join(srcdir, 'configure'),
1273                              '--build=%s' % self.ctx.build_triplet,
1274                              '--host=%s' % self.triplet,
1275                              '--prefix=',
1276                              '--disable-profile', '--without-parted',
1277                              'CC=%s-gcc -nostdlib' % self.triplet])
1278         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1279                                         'no_deps=t', 'install-headers'])
1280         cmdlist.cleanup_dir()
1281         cmdlist.pop_subdesc()
1282
1283     def build_gcc(self, cmdlist, bootstrap):
1284         """Build GCC."""
1285         # libsanitizer commonly breaks because of glibc header
1286         # changes, or on unusual targets.  libssp is of little
1287         # relevance with glibc's own stack checking support.
1288         cfg_opts = list(self.gcc_cfg)
1289         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1290         host_libs = self.ctx.host_libraries_installdir
1291         cfg_opts += ['--with-gmp=%s' % host_libs,
1292                      '--with-mpfr=%s' % host_libs,
1293                      '--with-mpc=%s' % host_libs]
1294         if bootstrap:
1295             tool_build = 'gcc-first'
1296             # Building a static-only, C-only compiler that is
1297             # sufficient to build glibc.  Various libraries and
1298             # features that may require libc headers must be disabled.
1299             # When configuring with a sysroot, --with-newlib is
1300             # required to define inhibit_libc (to stop some parts of
1301             # libgcc including libc headers); --without-headers is not
1302             # sufficient.
1303             cfg_opts += ['--enable-languages=c', '--disable-shared',
1304                          '--disable-threads',
1305                          '--disable-libatomic',
1306                          '--disable-decimal-float',
1307                          '--disable-libffi',
1308                          '--disable-libgomp',
1309                          '--disable-libitm',
1310                          '--disable-libmpx',
1311                          '--disable-libquadmath',
1312                          '--without-headers', '--with-newlib',
1313                          '--with-glibc-version=%s' % self.ctx.glibc_version
1314                          ]
1315             cfg_opts += self.first_gcc_cfg
1316         else:
1317             tool_build = 'gcc'
1318             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1319                          '--enable-threads']
1320         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1321
1322
1323 class Glibc(object):
1324     """A configuration for building glibc."""
1325
1326     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1327                  cfg=None, ccopts=None):
1328         """Initialize a Glibc object."""
1329         self.ctx = compiler.ctx
1330         self.compiler = compiler
1331         if arch is None:
1332             self.arch = compiler.arch
1333         else:
1334             self.arch = arch
1335         if os_name is None:
1336             self.os = compiler.os
1337         else:
1338             self.os = os_name
1339         self.variant = variant
1340         if variant is None:
1341             self.name = '%s-%s' % (self.arch, self.os)
1342         else:
1343             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1344         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1345         if cfg is None:
1346             self.cfg = []
1347         else:
1348             self.cfg = cfg
1349         self.ccopts = ccopts
1350
1351     def tool_name(self, tool):
1352         """Return the name of a cross-compilation tool."""
1353         ctool = '%s-%s' % (self.compiler.triplet, tool)
1354         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1355             ctool = '%s %s' % (ctool, self.ccopts)
1356         return ctool
1357
1358     def build(self):
1359         """Generate commands to build this glibc."""
1360         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1361         installdir = self.ctx.glibc_installdir(self.name)
1362         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1363         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1364         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1365         cmdlist.add_command('check-compilers',
1366                             ['test', '-f',
1367                              os.path.join(self.compiler.installdir, 'ok')])
1368         cmdlist.use_path(self.compiler.bindir)
1369         self.build_glibc(cmdlist, False)
1370         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1371                                       logsdir)
1372
1373     def build_glibc(self, cmdlist, for_compiler):
1374         """Generate commands to build this glibc, either as part of a compiler
1375         build or with the bootstrapped compiler (and in the latter case, run
1376         tests as well)."""
1377         srcdir = self.ctx.component_srcdir('glibc')
1378         if for_compiler:
1379             builddir = self.ctx.component_builddir('compilers',
1380                                                    self.compiler.name, 'glibc',
1381                                                    self.name)
1382             installdir = self.compiler.sysroot
1383             srcdir_copy = self.ctx.component_builddir('compilers',
1384                                                       self.compiler.name,
1385                                                       'glibc-src',
1386                                                       self.name)
1387         else:
1388             builddir = self.ctx.component_builddir('glibcs', self.name,
1389                                                    'glibc')
1390             installdir = self.ctx.glibc_installdir(self.name)
1391             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1392                                                       'glibc-src')
1393         cmdlist.create_use_dir(builddir)
1394         # glibc builds write into the source directory, and even if
1395         # not intentionally there is a risk of bugs that involve
1396         # writing into the working directory.  To avoid possible
1397         # concurrency issues, copy the source directory.
1398         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1399         use_usr = self.os != 'gnu'
1400         prefix = '/usr' if use_usr else ''
1401         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1402                    '--prefix=%s' % prefix,
1403                    '--enable-profile',
1404                    '--build=%s' % self.ctx.build_triplet,
1405                    '--host=%s' % self.triplet,
1406                    'CC=%s' % self.tool_name('gcc'),
1407                    'CXX=%s' % self.tool_name('g++'),
1408                    'AR=%s' % self.tool_name('ar'),
1409                    'AS=%s' % self.tool_name('as'),
1410                    'LD=%s' % self.tool_name('ld'),
1411                    'NM=%s' % self.tool_name('nm'),
1412                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1413                    'OBJDUMP=%s' % self.tool_name('objdump'),
1414                    'RANLIB=%s' % self.tool_name('ranlib'),
1415                    'READELF=%s' % self.tool_name('readelf'),
1416                    'STRIP=%s' % self.tool_name('strip')]
1417         if self.os == 'gnu':
1418             cfg_cmd += ['MIG=%s' % self.tool_name('mig')]
1419         cfg_cmd += self.cfg
1420         cmdlist.add_command('configure', cfg_cmd)
1421         cmdlist.add_command('build', ['make'])
1422         cmdlist.add_command('install', ['make', 'install',
1423                                         'install_root=%s' % installdir])
1424         # GCC uses paths such as lib/../lib64, so make sure lib
1425         # directories always exist.
1426         mkdir_cmd = ['mkdir', '-p',
1427                      os.path.join(installdir, 'lib')]
1428         if use_usr:
1429             mkdir_cmd += [os.path.join(installdir, 'usr', 'lib')]
1430         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1431         if not for_compiler:
1432             if self.ctx.strip:
1433                 cmdlist.add_command('strip',
1434                                     ['sh', '-c',
1435                                      ('%s $(find %s/lib* -name "*.so")' %
1436                                       (self.tool_name('strip'), installdir))])
1437             cmdlist.add_command('check', ['make', 'check'])
1438             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1439                                 always_run=True)
1440         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1441         cmdlist.cleanup_dir()
1442
1443
1444 class Command(object):
1445     """A command run in the build process."""
1446
1447     def __init__(self, desc, num, dir, path, command, always_run=False):
1448         """Initialize a Command object."""
1449         self.dir = dir
1450         self.path = path
1451         self.desc = desc
1452         trans = str.maketrans({' ': '-'})
1453         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1454         self.command = command
1455         self.always_run = always_run
1456
1457     @staticmethod
1458     def shell_make_quote_string(s):
1459         """Given a string not containing a newline, quote it for use by the
1460         shell and make."""
1461         assert '\n' not in s
1462         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1463             return s
1464         strans = str.maketrans({"'": "'\\''"})
1465         s = "'%s'" % s.translate(strans)
1466         mtrans = str.maketrans({'$': '$$'})
1467         return s.translate(mtrans)
1468
1469     @staticmethod
1470     def shell_make_quote_list(l, translate_make):
1471         """Given a list of strings not containing newlines, quote them for use
1472         by the shell and make, returning a single string.  If translate_make
1473         is true and the first string is 'make', change it to $(MAKE)."""
1474         l = [Command.shell_make_quote_string(s) for s in l]
1475         if translate_make and l[0] == 'make':
1476             l[0] = '$(MAKE)'
1477         return ' '.join(l)
1478
1479     def shell_make_quote(self):
1480         """Return this command quoted for the shell and make."""
1481         return self.shell_make_quote_list(self.command, True)
1482
1483
1484 class CommandList(object):
1485     """A list of commands run in the build process."""
1486
1487     def __init__(self, desc, keep):
1488         """Initialize a CommandList object."""
1489         self.cmdlist = []
1490         self.dir = None
1491         self.path = None
1492         self.desc = [desc]
1493         self.keep = keep
1494
1495     def desc_txt(self, desc):
1496         """Return the description to use for a command."""
1497         return '%s %s' % (' '.join(self.desc), desc)
1498
1499     def use_dir(self, dir):
1500         """Set the default directory for subsequent commands."""
1501         self.dir = dir
1502
1503     def use_path(self, path):
1504         """Set a directory to be prepended to the PATH for subsequent
1505         commands."""
1506         self.path = path
1507
1508     def push_subdesc(self, subdesc):
1509         """Set the default subdescription for subsequent commands (e.g., the
1510         name of a component being built, within the series of commands
1511         building it)."""
1512         self.desc.append(subdesc)
1513
1514     def pop_subdesc(self):
1515         """Pop a subdescription from the list of descriptions."""
1516         self.desc.pop()
1517
1518     def create_use_dir(self, dir):
1519         """Remove and recreate a directory and use it for subsequent
1520         commands."""
1521         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1522         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1523         self.use_dir(dir)
1524
1525     def create_copy_dir(self, src, dest):
1526         """Remove a directory and recreate it as a copy from the given
1527         source."""
1528         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1529         parent = os.path.dirname(dest)
1530         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1531         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1532
1533     def add_command_dir(self, desc, dir, command, always_run=False):
1534         """Add a command to run in a given directory."""
1535         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1536                       command, always_run)
1537         self.cmdlist.append(cmd)
1538
1539     def add_command(self, desc, command, always_run=False):
1540         """Add a command to run in the default directory."""
1541         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1542                       self.path, command, always_run)
1543         self.cmdlist.append(cmd)
1544
1545     def cleanup_dir(self, desc='cleanup', dir=None):
1546         """Clean up a build directory.  If no directory is specified, the
1547         default directory is cleaned up and ceases to be the default
1548         directory."""
1549         if dir is None:
1550             dir = self.dir
1551             self.use_dir(None)
1552         if self.keep != 'all':
1553             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1554                                  always_run=(self.keep == 'none'))
1555
1556     def makefile_commands(self, wrapper, logsdir):
1557         """Return the sequence of commands in the form of text for a Makefile.
1558         The given wrapper script takes arguments: base of logs for
1559         previous command, or empty; base of logs for this command;
1560         description; directory; PATH addition; the command itself."""
1561         # prev_base is the base of the name for logs of the previous
1562         # command that is not always-run (that is, a build command,
1563         # whose failure should stop subsequent build commands from
1564         # being run, as opposed to a cleanup command, which is run
1565         # even if previous commands failed).
1566         prev_base = ''
1567         cmds = []
1568         for c in self.cmdlist:
1569             ctxt = c.shell_make_quote()
1570             if prev_base and not c.always_run:
1571                 prev_log = os.path.join(logsdir, prev_base)
1572             else:
1573                 prev_log = ''
1574             this_log = os.path.join(logsdir, c.logbase)
1575             if not c.always_run:
1576                 prev_base = c.logbase
1577             if c.dir is None:
1578                 dir = ''
1579             else:
1580                 dir = c.dir
1581             if c.path is None:
1582                 path = ''
1583             else:
1584                 path = c.path
1585             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1586             prelim_txt = Command.shell_make_quote_list(prelims, False)
1587             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1588         return '\n'.join(cmds)
1589
1590     def status_logs(self, logsdir):
1591         """Return the list of log files with command status."""
1592         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1593                 for c in self.cmdlist]
1594
1595
1596 def get_parser():
1597     """Return an argument parser for this module."""
1598     parser = argparse.ArgumentParser(description=__doc__)
1599     parser.add_argument('-j', dest='parallelism',
1600                         help='Run this number of jobs in parallel',
1601                         type=int, default=os.cpu_count())
1602     parser.add_argument('--keep', dest='keep',
1603                         help='Whether to keep all build directories, '
1604                         'none or only those from failed builds',
1605                         default='none', choices=('none', 'all', 'failed'))
1606     parser.add_argument('--replace-sources', action='store_true',
1607                         help='Remove and replace source directories '
1608                         'with the wrong version of a component')
1609     parser.add_argument('--strip', action='store_true',
1610                         help='Strip installed glibc libraries')
1611     parser.add_argument('topdir',
1612                         help='Toplevel working directory')
1613     parser.add_argument('action',
1614                         help='What to do',
1615                         choices=('checkout', 'bot-cycle', 'bot',
1616                                  'host-libraries', 'compilers', 'glibcs'))
1617     parser.add_argument('configs',
1618                         help='Versions to check out or configurations to build',
1619                         nargs='*')
1620     return parser
1621
1622
1623 def main(argv):
1624     """The main entry point."""
1625     parser = get_parser()
1626     opts = parser.parse_args(argv)
1627     topdir = os.path.abspath(opts.topdir)
1628     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1629                   opts.strip, opts.action)
1630     ctx.run_builds(opts.action, opts.configs)
1631
1632
1633 if __name__ == '__main__':
1634     main(sys.argv[1:])