scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2018 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52 try:
  53     os.cpu_count
  54 except:
  55     import multiprocessing
  56     os.cpu_count = lambda: multiprocessing.cpu_count()
  57
  58 try:
  59     re.fullmatch
  60 except:
  61     re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)
  62
  63 try:
  64     subprocess.run
  65 except:
  66     class _CompletedProcess:
  67         def __init__(self, args, returncode, stdout=None, stderr=None):
  68             self.args = args
  69             self.returncode = returncode
  70             self.stdout = stdout
  71             self.stderr = stderr
  72
  73     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  74         assert(timeout is None)
  75         with subprocess.Popen(*popenargs, **kwargs) as process:
  76             try:
  77                 stdout, stderr = process.communicate(input)
  78             except:
  79                 process.kill()
  80                 process.wait()
  81                 raise
  82             returncode = process.poll()
  83             if check and returncode:
  84                 raise subprocess.CalledProcessError(returncode, popenargs)
  85         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  86
  87     subprocess.run = _run
  88
  89
  90 class Context(object):
  91     """The global state associated with builds in a given directory."""
  92
  93     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  94                  action):
  95         """Initialize the context."""
  96         self.topdir = topdir
  97         self.parallelism = parallelism
  98         self.keep = keep
  99         self.replace_sources = replace_sources
 100         self.strip = strip
 101         self.srcdir = os.path.join(topdir, 'src')
 102         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 103         self.build_state_json = os.path.join(topdir, 'build-state.json')
 104         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 105         self.installdir = os.path.join(topdir, 'install')
 106         self.host_libraries_installdir = os.path.join(self.installdir,
 107                                                       'host-libraries')
 108         self.builddir = os.path.join(topdir, 'build')
 109         self.logsdir = os.path.join(topdir, 'logs')
 110         self.logsdir_old = os.path.join(topdir, 'logs-old')
 111         self.makefile = os.path.join(self.builddir, 'Makefile')
 112         self.wrapper = os.path.join(self.builddir, 'wrapper')
 113         self.save_logs = os.path.join(self.builddir, 'save-logs')
 114         self.script_text = self.get_script_text()
 115         if action != 'checkout':
 116             self.build_triplet = self.get_build_triplet()
 117             self.glibc_version = self.get_glibc_version()
 118         self.configs = {}
 119         self.glibc_configs = {}
 120         self.makefile_pieces = ['.PHONY: all\n']
 121         self.add_all_configs()
 122         self.load_versions_json()
 123         self.load_build_state_json()
 124         self.status_log_list = []
 125         self.email_warning = False
 126
 127     def get_script_text(self):
 128         """Return the text of this script."""
 129         with open(sys.argv[0], 'r') as f:
 130             return f.read()
 131
 132     def exec_self(self):
 133         """Re-execute this script with the same arguments."""
 134         sys.stdout.flush()
 135         os.execv(sys.executable, [sys.executable] + sys.argv)
 136
 137     def get_build_triplet(self):
 138         """Determine the build triplet with config.guess."""
 139         config_guess = os.path.join(self.component_srcdir('gcc'),
 140                                     'config.guess')
 141         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 142                                 check=True, universal_newlines=True).stdout
 143         return cg_out.rstrip()
 144
 145     def get_glibc_version(self):
 146         """Determine the glibc version number (major.minor)."""
 147         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 148         with open(version_h, 'r') as f:
 149             lines = f.readlines()
 150         starttext = '#define VERSION "'
 151         for l in lines:
 152             if l.startswith(starttext):
 153                 l = l[len(starttext):]
 154                 l = l.rstrip('"\n')
 155                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 156                 return '%s.%s' % m.group(1, 2)
 157         print('error: could not determine glibc version')
 158         exit(1)
 159
 160     def add_all_configs(self):
 161         """Add all known glibc build configurations."""
 162         self.add_config(arch='aarch64',
 163                         os_name='linux-gnu',
 164                         extra_glibcs=[{'variant': 'disable-multi-arch',
 165                                        'cfg': ['--disable-multi-arch']}])
 166         self.add_config(arch='aarch64_be',
 167                         os_name='linux-gnu')
 168         self.add_config(arch='alpha',
 169                         os_name='linux-gnu')
 170         self.add_config(arch='arm',
 171                         os_name='linux-gnueabi')
 172         self.add_config(arch='armeb',
 173                         os_name='linux-gnueabi')
 174         self.add_config(arch='armeb',
 175                         os_name='linux-gnueabi',
 176                         variant='be8',
 177                         gcc_cfg=['--with-arch=armv7-a'])
 178         self.add_config(arch='arm',
 179                         os_name='linux-gnueabihf',
 180                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 181                         extra_glibcs=[{'variant': 'v7a',
 182                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 183                                       {'variant': 'v7a-disable-multi-arch',
 184                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 185                                        'cfg': ['--disable-multi-arch']}])
 186         self.add_config(arch='armeb',
 187                         os_name='linux-gnueabihf',
 188                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 189         self.add_config(arch='armeb',
 190                         os_name='linux-gnueabihf',
 191                         variant='be8',
 192                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 193                                  '--with-fpu=vfpv3'])
 194         self.add_config(arch='hppa',
 195                         os_name='linux-gnu')
 196         self.add_config(arch='i686',
 197                         os_name='gnu')
 198         self.add_config(arch='ia64',
 199                         os_name='linux-gnu',
 200                         first_gcc_cfg=['--with-system-libunwind'])
 201         self.add_config(arch='m68k',
 202                         os_name='linux-gnu',
 203                         gcc_cfg=['--disable-multilib'])
 204         self.add_config(arch='m68k',
 205                         os_name='linux-gnu',
 206                         variant='coldfire',
 207                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 208         self.add_config(arch='m68k',
 209                         os_name='linux-gnu',
 210                         variant='coldfire-soft',
 211                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 212                                  '--disable-multilib'])
 213         self.add_config(arch='microblaze',
 214                         os_name='linux-gnu',
 215                         gcc_cfg=['--disable-multilib'])
 216         self.add_config(arch='microblazeel',
 217                         os_name='linux-gnu',
 218                         gcc_cfg=['--disable-multilib'])
 219         self.add_config(arch='mips64',
 220                         os_name='linux-gnu',
 221                         gcc_cfg=['--with-mips-plt'],
 222                         glibcs=[{'variant': 'n32'},
 223                                 {'arch': 'mips',
 224                                  'ccopts': '-mabi=32'},
 225                                 {'variant': 'n64',
 226                                  'ccopts': '-mabi=64'}])
 227         self.add_config(arch='mips64',
 228                         os_name='linux-gnu',
 229                         variant='soft',
 230                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 231                         glibcs=[{'variant': 'n32-soft'},
 232                                 {'variant': 'soft',
 233                                  'arch': 'mips',
 234                                  'ccopts': '-mabi=32'},
 235                                 {'variant': 'n64-soft',
 236                                  'ccopts': '-mabi=64'}])
 237         self.add_config(arch='mips64',
 238                         os_name='linux-gnu',
 239                         variant='nan2008',
 240                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 241                                  '--with-arch-64=mips64r2',
 242                                  '--with-arch-32=mips32r2'],
 243                         glibcs=[{'variant': 'n32-nan2008'},
 244                                 {'variant': 'nan2008',
 245                                  'arch': 'mips',
 246                                  'ccopts': '-mabi=32'},
 247                                 {'variant': 'n64-nan2008',
 248                                  'ccopts': '-mabi=64'}])
 249         self.add_config(arch='mips64',
 250                         os_name='linux-gnu',
 251                         variant='nan2008-soft',
 252                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 253                                  '--with-arch-64=mips64r2',
 254                                  '--with-arch-32=mips32r2',
 255                                  '--with-float=soft'],
 256                         glibcs=[{'variant': 'n32-nan2008-soft'},
 257                                 {'variant': 'nan2008-soft',
 258                                  'arch': 'mips',
 259                                  'ccopts': '-mabi=32'},
 260                                 {'variant': 'n64-nan2008-soft',
 261                                  'ccopts': '-mabi=64'}])
 262         self.add_config(arch='mips64el',
 263                         os_name='linux-gnu',
 264                         gcc_cfg=['--with-mips-plt'],
 265                         glibcs=[{'variant': 'n32'},
 266                                 {'arch': 'mipsel',
 267                                  'ccopts': '-mabi=32'},
 268                                 {'variant': 'n64',
 269                                  'ccopts': '-mabi=64'}])
 270         self.add_config(arch='mips64el',
 271                         os_name='linux-gnu',
 272                         variant='soft',
 273                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 274                         glibcs=[{'variant': 'n32-soft'},
 275                                 {'variant': 'soft',
 276                                  'arch': 'mipsel',
 277                                  'ccopts': '-mabi=32'},
 278                                 {'variant': 'n64-soft',
 279                                  'ccopts': '-mabi=64'}])
 280         self.add_config(arch='mips64el',
 281                         os_name='linux-gnu',
 282                         variant='nan2008',
 283                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 284                                  '--with-arch-64=mips64r2',
 285                                  '--with-arch-32=mips32r2'],
 286                         glibcs=[{'variant': 'n32-nan2008'},
 287                                 {'variant': 'nan2008',
 288                                  'arch': 'mipsel',
 289                                  'ccopts': '-mabi=32'},
 290                                 {'variant': 'n64-nan2008',
 291                                  'ccopts': '-mabi=64'}])
 292         self.add_config(arch='mips64el',
 293                         os_name='linux-gnu',
 294                         variant='nan2008-soft',
 295                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 296                                  '--with-arch-64=mips64r2',
 297                                  '--with-arch-32=mips32r2',
 298                                  '--with-float=soft'],
 299                         glibcs=[{'variant': 'n32-nan2008-soft'},
 300                                 {'variant': 'nan2008-soft',
 301                                  'arch': 'mipsel',
 302                                  'ccopts': '-mabi=32'},
 303                                 {'variant': 'n64-nan2008-soft',
 304                                  'ccopts': '-mabi=64'}])
 305         self.add_config(arch='nios2',
 306                         os_name='linux-gnu')
 307         self.add_config(arch='powerpc',
 308                         os_name='linux-gnu',
 309                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 310                         extra_glibcs=[{'variant': 'power4',
 311                                        'ccopts': '-mcpu=power4',
 312                                        'cfg': ['--with-cpu=power4']}])
 313         self.add_config(arch='powerpc',
 314                         os_name='linux-gnu',
 315                         variant='soft',
 316                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 317                                  '--enable-secureplt'])
 318         self.add_config(arch='powerpc64',
 319                         os_name='linux-gnu',
 320                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 321         self.add_config(arch='powerpc64le',
 322                         os_name='linux-gnu',
 323                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 324         self.add_config(arch='powerpc',
 325                         os_name='linux-gnuspe',
 326                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 327                                  '--enable-e500-double'])
 328         self.add_config(arch='powerpc',
 329                         os_name='linux-gnuspe',
 330                         variant='e500v1',
 331                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 332         self.add_config(arch='s390x',
 333                         os_name='linux-gnu',
 334                         glibcs=[{},
 335                                 {'arch': 's390', 'ccopts': '-m31'}])
 336         self.add_config(arch='sh3',
 337                         os_name='linux-gnu')
 338         self.add_config(arch='sh3eb',
 339                         os_name='linux-gnu')
 340         self.add_config(arch='sh4',
 341                         os_name='linux-gnu')
 342         self.add_config(arch='sh4eb',
 343                         os_name='linux-gnu')
 344         self.add_config(arch='sh4',
 345                         os_name='linux-gnu',
 346                         variant='soft',
 347                         gcc_cfg=['--without-fp'])
 348         self.add_config(arch='sh4eb',
 349                         os_name='linux-gnu',
 350                         variant='soft',
 351                         gcc_cfg=['--without-fp'])
 352         self.add_config(arch='sparc64',
 353                         os_name='linux-gnu',
 354                         glibcs=[{},
 355                                 {'arch': 'sparcv9',
 356                                  'ccopts': '-m32 -mlong-double-128'}],
 357                         extra_glibcs=[{'variant': 'disable-multi-arch',
 358                                        'cfg': ['--disable-multi-arch']},
 359                                       {'variant': 'disable-multi-arch',
 360                                        'arch': 'sparcv9',
 361                                        'ccopts': '-m32 -mlong-double-128',
 362                                        'cfg': ['--disable-multi-arch']}])
 363         self.add_config(arch='tilegx',
 364                         os_name='linux-gnu',
 365                         glibcs=[{},
 366                                 {'variant': '32', 'ccopts': '-m32'}])
 367         self.add_config(arch='tilegxbe',
 368                         os_name='linux-gnu',
 369                         glibcs=[{},
 370                                 {'variant': '32', 'ccopts': '-m32'}])
 371         self.add_config(arch='x86_64',
 372                         os_name='linux-gnu',
 373                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 374                         glibcs=[{},
 375                                 {'variant': 'x32', 'ccopts': '-mx32'},
 376                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 377                         extra_glibcs=[{'variant': 'disable-multi-arch',
 378                                        'cfg': ['--disable-multi-arch']},
 379                                       {'variant': 'static-pie',
 380                                        'cfg': ['--enable-static-pie']},
 381                                       {'variant': 'x32-static-pie',
 382                                        'ccopts': '-mx32',
 383                                        'cfg': ['--enable-static-pie']},
 384                                       {'variant': 'static-pie',
 385                                        'arch': 'i686',
 386                                        'ccopts': '-m32 -march=i686',
 387                                        'cfg': ['--enable-static-pie']},
 388                                       {'variant': 'disable-multi-arch',
 389                                        'arch': 'i686',
 390                                        'ccopts': '-m32 -march=i686',
 391                                        'cfg': ['--disable-multi-arch']},
 392                                       {'arch': 'i486',
 393                                        'ccopts': '-m32 -march=i486'},
 394                                       {'arch': 'i586',
 395                                        'ccopts': '-m32 -march=i586'}])
 396
 397     def add_config(self, **args):
 398         """Add an individual build configuration."""
 399         cfg = Config(self, **args)
 400         if cfg.name in self.configs:
 401             print('error: duplicate config %s' % cfg.name)
 402             exit(1)
 403         self.configs[cfg.name] = cfg
 404         for c in cfg.all_glibcs:
 405             if c.name in self.glibc_configs:
 406                 print('error: duplicate glibc config %s' % c.name)
 407                 exit(1)
 408             self.glibc_configs[c.name] = c
 409
 410     def component_srcdir(self, component):
 411         """Return the source directory for a given component, e.g. gcc."""
 412         return os.path.join(self.srcdir, component)
 413
 414     def component_builddir(self, action, config, component, subconfig=None):
 415         """Return the directory to use for a build."""
 416         if config is None:
 417             # Host libraries.
 418             assert subconfig is None
 419             return os.path.join(self.builddir, action, component)
 420         if subconfig is None:
 421             return os.path.join(self.builddir, action, config, component)
 422         else:
 423             # glibc build as part of compiler build.
 424             return os.path.join(self.builddir, action, config, component,
 425                                 subconfig)
 426
 427     def compiler_installdir(self, config):
 428         """Return the directory in which to install a compiler."""
 429         return os.path.join(self.installdir, 'compilers', config)
 430
 431     def compiler_bindir(self, config):
 432         """Return the directory in which to find compiler binaries."""
 433         return os.path.join(self.compiler_installdir(config), 'bin')
 434
 435     def compiler_sysroot(self, config):
 436         """Return the sysroot directory for a compiler."""
 437         return os.path.join(self.compiler_installdir(config), 'sysroot')
 438
 439     def glibc_installdir(self, config):
 440         """Return the directory in which to install glibc."""
 441         return os.path.join(self.installdir, 'glibcs', config)
 442
 443     def run_builds(self, action, configs):
 444         """Run the requested builds."""
 445         if action == 'checkout':
 446             self.checkout(configs)
 447             return
 448         if action == 'bot-cycle':
 449             if configs:
 450                 print('error: configurations specified for bot-cycle')
 451                 exit(1)
 452             self.bot_cycle()
 453             return
 454         if action == 'bot':
 455             if configs:
 456                 print('error: configurations specified for bot')
 457                 exit(1)
 458             self.bot()
 459             return
 460         if action == 'host-libraries' and configs:
 461             print('error: configurations specified for host-libraries')
 462             exit(1)
 463         self.clear_last_build_state(action)
 464         build_time = datetime.datetime.utcnow()
 465         if action == 'host-libraries':
 466             build_components = ('gmp', 'mpfr', 'mpc')
 467             old_components = ()
 468             old_versions = {}
 469             self.build_host_libraries()
 470         elif action == 'compilers':
 471             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 472                                 'gnumach', 'hurd')
 473             old_components = ('gmp', 'mpfr', 'mpc')
 474             old_versions = self.build_state['host-libraries']['build-versions']
 475             self.build_compilers(configs)
 476         else:
 477             build_components = ('glibc',)
 478             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 479                               'mig', 'gnumach', 'hurd')
 480             old_versions = self.build_state['compilers']['build-versions']
 481             self.build_glibcs(configs)
 482         self.write_files()
 483         self.do_build()
 484         if configs:
 485             # Partial build, do not update stored state.
 486             return
 487         build_versions = {}
 488         for k in build_components:
 489             if k in self.versions:
 490                 build_versions[k] = {'version': self.versions[k]['version'],
 491                                      'revision': self.versions[k]['revision']}
 492         for k in old_components:
 493             if k in old_versions:
 494                 build_versions[k] = {'version': old_versions[k]['version'],
 495                                      'revision': old_versions[k]['revision']}
 496         self.update_build_state(action, build_time, build_versions)
 497
 498     @staticmethod
 499     def remove_dirs(*args):
 500         """Remove directories and their contents if they exist."""
 501         for dir in args:
 502             shutil.rmtree(dir, ignore_errors=True)
 503
 504     @staticmethod
 505     def remove_recreate_dirs(*args):
 506         """Remove directories if they exist, and create them as empty."""
 507         Context.remove_dirs(*args)
 508         for dir in args:
 509             os.makedirs(dir, exist_ok=True)
 510
 511     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 512         """Add makefile text for a list of commands."""
 513         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 514         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 515                                     (target, target, target, commands))
 516         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 517
 518     def write_files(self):
 519         """Write out the Makefile and wrapper script."""
 520         mftext = ''.join(self.makefile_pieces)
 521         with open(self.makefile, 'w') as f:
 522             f.write(mftext)
 523         wrapper_text = (
 524             '#!/bin/sh\n'
 525             'prev_base=$1\n'
 526             'this_base=$2\n'
 527             'desc=$3\n'
 528             'dir=$4\n'
 529             'path=$5\n'
 530             'shift 5\n'
 531             'prev_status=$prev_base-status.txt\n'
 532             'this_status=$this_base-status.txt\n'
 533             'this_log=$this_base-log.txt\n'
 534             'date > "$this_log"\n'
 535             'echo >> "$this_log"\n'
 536             'echo "Description: $desc" >> "$this_log"\n'
 537             'printf "%s" "Command:" >> "$this_log"\n'
 538             'for word in "$@"; do\n'
 539             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 540             '    printf " %s" "$word"\n'
 541             '  else\n'
 542             '    printf " \'"\n'
 543             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 544             '    printf "\'"\n'
 545             '  fi\n'
 546             'done >> "$this_log"\n'
 547             'echo >> "$this_log"\n'
 548             'echo "Directory: $dir" >> "$this_log"\n'
 549             'echo "Path addition: $path" >> "$this_log"\n'
 550             'echo >> "$this_log"\n'
 551             'record_status ()\n'
 552             '{\n'
 553             '  echo >> "$this_log"\n'
 554             '  echo "$1: $desc" > "$this_status"\n'
 555             '  echo "$1: $desc" >> "$this_log"\n'
 556             '  echo >> "$this_log"\n'
 557             '  date >> "$this_log"\n'
 558             '  echo "$1: $desc"\n'
 559             '  exit 0\n'
 560             '}\n'
 561             'check_error ()\n'
 562             '{\n'
 563             '  if [ "$1" != "0" ]; then\n'
 564             '    record_status FAIL\n'
 565             '  fi\n'
 566             '}\n'
 567             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 568             '    record_status UNRESOLVED\n'
 569             'fi\n'
 570             'if [ "$dir" ]; then\n'
 571             '  cd "$dir"\n'
 572             '  check_error "$?"\n'
 573             'fi\n'
 574             'if [ "$path" ]; then\n'
 575             '  PATH=$path:$PATH\n'
 576             'fi\n'
 577             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 578             'check_error "$?"\n'
 579             'record_status PASS\n')
 580         with open(self.wrapper, 'w') as f:
 581             f.write(wrapper_text)
 582         # Mode 0o755.
 583         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 584                      stat.S_IROTH|stat.S_IXOTH)
 585         os.chmod(self.wrapper, mode_exec)
 586         save_logs_text = (
 587             '#!/bin/sh\n'
 588             'if ! [ -f tests.sum ]; then\n'
 589             '  echo "No test summary available."\n'
 590             '  exit 0\n'
 591             'fi\n'
 592             'save_file ()\n'
 593             '{\n'
 594             '  echo "Contents of $1:"\n'
 595             '  echo\n'
 596             '  cat "$1"\n'
 597             '  echo\n'
 598             '  echo "End of contents of $1."\n'
 599             '  echo\n'
 600             '}\n'
 601             'save_file tests.sum\n'
 602             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 603             'for t in $non_pass_tests; do\n'
 604             '  if [ -f "$t.out" ]; then\n'
 605             '    save_file "$t.out"\n'
 606             '  fi\n'
 607             'done\n')
 608         with open(self.save_logs, 'w') as f:
 609             f.write(save_logs_text)
 610         os.chmod(self.save_logs, mode_exec)
 611
 612     def do_build(self):
 613         """Do the actual build."""
 614         cmd = ['make', '-j%d' % self.parallelism]
 615         subprocess.run(cmd, cwd=self.builddir, check=True)
 616
 617     def build_host_libraries(self):
 618         """Build the host libraries."""
 619         installdir = self.host_libraries_installdir
 620         builddir = os.path.join(self.builddir, 'host-libraries')
 621         logsdir = os.path.join(self.logsdir, 'host-libraries')
 622         self.remove_recreate_dirs(installdir, builddir, logsdir)
 623         cmdlist = CommandList('host-libraries', self.keep)
 624         self.build_host_library(cmdlist, 'gmp')
 625         self.build_host_library(cmdlist, 'mpfr',
 626                                 ['--with-gmp=%s' % installdir])
 627         self.build_host_library(cmdlist, 'mpc',
 628                                 ['--with-gmp=%s' % installdir,
 629                                 '--with-mpfr=%s' % installdir])
 630         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 631         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 632
 633     def build_host_library(self, cmdlist, lib, extra_opts=None):
 634         """Build one host library."""
 635         srcdir = self.component_srcdir(lib)
 636         builddir = self.component_builddir('host-libraries', None, lib)
 637         installdir = self.host_libraries_installdir
 638         cmdlist.push_subdesc(lib)
 639         cmdlist.create_use_dir(builddir)
 640         cfg_cmd = [os.path.join(srcdir, 'configure'),
 641                    '--prefix=%s' % installdir,
 642                    '--disable-shared']
 643         if extra_opts:
 644             cfg_cmd.extend (extra_opts)
 645         cmdlist.add_command('configure', cfg_cmd)
 646         cmdlist.add_command('build', ['make'])
 647         cmdlist.add_command('check', ['make', 'check'])
 648         cmdlist.add_command('install', ['make', 'install'])
 649         cmdlist.cleanup_dir()
 650         cmdlist.pop_subdesc()
 651
 652     def build_compilers(self, configs):
 653         """Build the compilers."""
 654         if not configs:
 655             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 656             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 657             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 658             configs = sorted(self.configs.keys())
 659         for c in configs:
 660             self.configs[c].build()
 661
 662     def build_glibcs(self, configs):
 663         """Build the glibcs."""
 664         if not configs:
 665             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 666             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 667             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 668             configs = sorted(self.glibc_configs.keys())
 669         for c in configs:
 670             self.glibc_configs[c].build()
 671
 672     def load_versions_json(self):
 673         """Load information about source directory versions."""
 674         if not os.access(self.versions_json, os.F_OK):
 675             self.versions = {}
 676             return
 677         with open(self.versions_json, 'r') as f:
 678             self.versions = json.load(f)
 679
 680     def store_json(self, data, filename):
 681         """Store information in a JSON file."""
 682         filename_tmp = filename + '.tmp'
 683         with open(filename_tmp, 'w') as f:
 684             json.dump(data, f, indent=2, sort_keys=True)
 685         os.rename(filename_tmp, filename)
 686
 687     def store_versions_json(self):
 688         """Store information about source directory versions."""
 689         self.store_json(self.versions, self.versions_json)
 690
 691     def set_component_version(self, component, version, explicit, revision):
 692         """Set the version information for a component."""
 693         self.versions[component] = {'version': version,
 694                                     'explicit': explicit,
 695                                     'revision': revision}
 696         self.store_versions_json()
 697
 698     def checkout(self, versions):
 699         """Check out the desired component versions."""
 700         default_versions = {'binutils': 'vcs-2.30',
 701                             'gcc': 'vcs-7',
 702                             'glibc': 'vcs-mainline',
 703                             'gmp': '6.1.2',
 704                             'linux': '4.14',
 705                             'mpc': '1.1.0',
 706                             'mpfr': '4.0.0',
 707                             'mig': 'vcs-mainline',
 708                             'gnumach': 'vcs-mainline',
 709                             'hurd': 'vcs-mainline'}
 710         use_versions = {}
 711         explicit_versions = {}
 712         for v in versions:
 713             found_v = False
 714             for k in default_versions.keys():
 715                 kx = k + '-'
 716                 if v.startswith(kx):
 717                     vx = v[len(kx):]
 718                     if k in use_versions:
 719                         print('error: multiple versions for %s' % k)
 720                         exit(1)
 721                     use_versions[k] = vx
 722                     explicit_versions[k] = True
 723                     found_v = True
 724                     break
 725             if not found_v:
 726                 print('error: unknown component in %s' % v)
 727                 exit(1)
 728         for k in default_versions.keys():
 729             if k not in use_versions:
 730                 if k in self.versions and self.versions[k]['explicit']:
 731                     use_versions[k] = self.versions[k]['version']
 732                     explicit_versions[k] = True
 733                 else:
 734                     use_versions[k] = default_versions[k]
 735                     explicit_versions[k] = False
 736         os.makedirs(self.srcdir, exist_ok=True)
 737         for k in sorted(default_versions.keys()):
 738             update = os.access(self.component_srcdir(k), os.F_OK)
 739             v = use_versions[k]
 740             if (update and
 741                 k in self.versions and
 742                 v != self.versions[k]['version']):
 743                 if not self.replace_sources:
 744                     print('error: version of %s has changed from %s to %s, '
 745                           'use --replace-sources to check out again' %
 746                           (k, self.versions[k]['version'], v))
 747                     exit(1)
 748                 shutil.rmtree(self.component_srcdir(k))
 749                 update = False
 750             if v.startswith('vcs-'):
 751                 revision = self.checkout_vcs(k, v[4:], update)
 752             else:
 753                 self.checkout_tar(k, v, update)
 754                 revision = v
 755             self.set_component_version(k, v, explicit_versions[k], revision)
 756         if self.get_script_text() != self.script_text:
 757             # Rerun the checkout process in case the updated script
 758             # uses different default versions or new components.
 759             self.exec_self()
 760
 761     def checkout_vcs(self, component, version, update):
 762         """Check out the given version of the given component from version
 763         control.  Return a revision identifier."""
 764         if component == 'binutils':
 765             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 766             if version == 'mainline':
 767                 git_branch = 'master'
 768             else:
 769                 trans = str.maketrans({'.': '_'})
 770                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 771             return self.git_checkout(component, git_url, git_branch, update)
 772         elif component == 'gcc':
 773             if version == 'mainline':
 774                 branch = 'trunk'
 775             else:
 776                 trans = str.maketrans({'.': '_'})
 777                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 778             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 779             return self.gcc_checkout(svn_url, update)
 780         elif component == 'glibc':
 781             git_url = 'git://sourceware.org/git/glibc.git'
 782             if version == 'mainline':
 783                 git_branch = 'master'
 784             else:
 785                 git_branch = 'release/%s/master' % version
 786             r = self.git_checkout(component, git_url, git_branch, update)
 787             self.fix_glibc_timestamps()
 788             return r
 789         elif component == 'gnumach':
 790             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 791             git_branch = 'master'
 792             r = self.git_checkout(component, git_url, git_branch, update)
 793             subprocess.run(['autoreconf', '-i'],
 794                            cwd=self.component_srcdir(component), check=True)
 795             return r
 796         elif component == 'mig':
 797             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 798             git_branch = 'master'
 799             r = self.git_checkout(component, git_url, git_branch, update)
 800             subprocess.run(['autoreconf', '-i'],
 801                            cwd=self.component_srcdir(component), check=True)
 802             return r
 803         elif component == 'hurd':
 804             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 805             git_branch = 'master'
 806             r = self.git_checkout(component, git_url, git_branch, update)
 807             subprocess.run(['autoconf'],
 808                            cwd=self.component_srcdir(component), check=True)
 809             return r
 810         else:
 811             print('error: component %s coming from VCS' % component)
 812             exit(1)
 813
 814     def git_checkout(self, component, git_url, git_branch, update):
 815         """Check out a component from git.  Return a commit identifier."""
 816         if update:
 817             subprocess.run(['git', 'remote', 'prune', 'origin'],
 818                            cwd=self.component_srcdir(component), check=True)
 819             subprocess.run(['git', 'pull', '-q'],
 820                            cwd=self.component_srcdir(component), check=True)
 821         else:
 822             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 823                             self.component_srcdir(component)], check=True)
 824         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 825                            cwd=self.component_srcdir(component),
 826                            stdout=subprocess.PIPE,
 827                            check=True, universal_newlines=True).stdout
 828         return r.rstrip()
 829
 830     def fix_glibc_timestamps(self):
 831         """Fix timestamps in a glibc checkout."""
 832         # Ensure that builds do not try to regenerate generated files
 833         # in the source tree.
 834         srcdir = self.component_srcdir('glibc')
 835         for dirpath, dirnames, filenames in os.walk(srcdir):
 836             for f in filenames:
 837                 if (f == 'configure' or
 838                     f == 'preconfigure' or
 839                     f.endswith('-kw.h')):
 840                     to_touch = os.path.join(dirpath, f)
 841                     subprocess.run(['touch', to_touch], check=True)
 842
 843     def gcc_checkout(self, svn_url, update):
 844         """Check out GCC from SVN.  Return the revision number."""
 845         if not update:
 846             subprocess.run(['svn', 'co', '-q', svn_url,
 847                             self.component_srcdir('gcc')], check=True)
 848         subprocess.run(['contrib/gcc_update', '--silent'],
 849                        cwd=self.component_srcdir('gcc'), check=True)
 850         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 851                            stdout=subprocess.PIPE,
 852                            check=True, universal_newlines=True).stdout
 853         return r.rstrip()
 854
 855     def checkout_tar(self, component, version, update):
 856         """Check out the given version of the given component from a
 857         tarball."""
 858         if update:
 859             return
 860         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 861                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 862                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 863                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 864                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 865                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 866                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 867                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 868                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 869         if component not in url_map:
 870             print('error: component %s coming from tarball' % component)
 871             exit(1)
 872         url = url_map[component] % {'version': version}
 873         filename = os.path.join(self.srcdir, url.split('/')[-1])
 874         response = urllib.request.urlopen(url)
 875         data = response.read()
 876         with open(filename, 'wb') as f:
 877             f.write(data)
 878         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 879                        check=True)
 880         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 881                   self.component_srcdir(component))
 882         os.remove(filename)
 883
 884     def load_build_state_json(self):
 885         """Load information about the state of previous builds."""
 886         if os.access(self.build_state_json, os.F_OK):
 887             with open(self.build_state_json, 'r') as f:
 888                 self.build_state = json.load(f)
 889         else:
 890             self.build_state = {}
 891         for k in ('host-libraries', 'compilers', 'glibcs'):
 892             if k not in self.build_state:
 893                 self.build_state[k] = {}
 894             if 'build-time' not in self.build_state[k]:
 895                 self.build_state[k]['build-time'] = ''
 896             if 'build-versions' not in self.build_state[k]:
 897                 self.build_state[k]['build-versions'] = {}
 898             if 'build-results' not in self.build_state[k]:
 899                 self.build_state[k]['build-results'] = {}
 900             if 'result-changes' not in self.build_state[k]:
 901                 self.build_state[k]['result-changes'] = {}
 902             if 'ever-passed' not in self.build_state[k]:
 903                 self.build_state[k]['ever-passed'] = []
 904
 905     def store_build_state_json(self):
 906         """Store information about the state of previous builds."""
 907         self.store_json(self.build_state, self.build_state_json)
 908
 909     def clear_last_build_state(self, action):
 910         """Clear information about the state of part of the build."""
 911         # We clear the last build time and versions when starting a
 912         # new build.  The results of the last build are kept around,
 913         # as comparison is still meaningful if this build is aborted
 914         # and a new one started.
 915         self.build_state[action]['build-time'] = ''
 916         self.build_state[action]['build-versions'] = {}
 917         self.store_build_state_json()
 918
 919     def update_build_state(self, action, build_time, build_versions):
 920         """Update the build state after a build."""
 921         build_time = build_time.replace(microsecond=0)
 922         self.build_state[action]['build-time'] = str(build_time)
 923         self.build_state[action]['build-versions'] = build_versions
 924         build_results = {}
 925         for log in self.status_log_list:
 926             with open(log, 'r') as f:
 927                 log_text = f.read()
 928             log_text = log_text.rstrip()
 929             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 930             result = m.group(1)
 931             test_name = m.group(2)
 932             assert test_name not in build_results
 933             build_results[test_name] = result
 934         old_build_results = self.build_state[action]['build-results']
 935         self.build_state[action]['build-results'] = build_results
 936         result_changes = {}
 937         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 938         for t in all_tests:
 939             if t in old_build_results:
 940                 old_res = old_build_results[t]
 941             else:
 942                 old_res = '(New test)'
 943             if t in build_results:
 944                 new_res = build_results[t]
 945             else:
 946                 new_res = '(Test removed)'
 947             if old_res != new_res:
 948                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 949         self.build_state[action]['result-changes'] = result_changes
 950         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 951                            if t in build_results}
 952         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 953         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 954                                                          new_passes)
 955         self.store_build_state_json()
 956
 957     def load_bot_config_json(self):
 958         """Load bot configuration."""
 959         with open(self.bot_config_json, 'r') as f:
 960             self.bot_config = json.load(f)
 961
 962     def part_build_old(self, action, delay):
 963         """Return whether the last build for a given action was at least a
 964         given number of seconds ago, or does not have a time recorded."""
 965         old_time_str = self.build_state[action]['build-time']
 966         if not old_time_str:
 967             return True
 968         old_time = datetime.datetime.strptime(old_time_str,
 969                                               '%Y-%m-%d %H:%M:%S')
 970         new_time = datetime.datetime.utcnow()
 971         delta = new_time - old_time
 972         return delta.total_seconds() >= delay
 973
 974     def bot_cycle(self):
 975         """Run a single round of checkout and builds."""
 976         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 977         self.load_bot_config_json()
 978         actions = ('host-libraries', 'compilers', 'glibcs')
 979         self.bot_run_self(['--replace-sources'], 'checkout')
 980         self.load_versions_json()
 981         if self.get_script_text() != self.script_text:
 982             print('Script changed, re-execing.')
 983             # On script change, all parts of the build should be rerun.
 984             for a in actions:
 985                 self.clear_last_build_state(a)
 986             self.exec_self()
 987         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 988                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
 989                                           'mig', 'gnumach', 'hurd'),
 990                             'glibcs': ('glibc',)}
 991         must_build = {}
 992         for a in actions:
 993             build_vers = self.build_state[a]['build-versions']
 994             must_build[a] = False
 995             if not self.build_state[a]['build-time']:
 996                 must_build[a] = True
 997             old_vers = {}
 998             new_vers = {}
 999             for c in check_components[a]:
1000                 if c in build_vers:
1001                     old_vers[c] = build_vers[c]
1002                 new_vers[c] = {'version': self.versions[c]['version'],
1003                                'revision': self.versions[c]['revision']}
1004             if new_vers == old_vers:
1005                 print('Versions for %s unchanged.' % a)
1006             else:
1007                 print('Versions changed or rebuild forced for %s.' % a)
1008                 if a == 'compilers' and not self.part_build_old(
1009                         a, self.bot_config['compilers-rebuild-delay']):
1010                     print('Not requiring rebuild of compilers this soon.')
1011                 else:
1012                     must_build[a] = True
1013         if must_build['host-libraries']:
1014             must_build['compilers'] = True
1015         if must_build['compilers']:
1016             must_build['glibcs'] = True
1017         for a in actions:
1018             if must_build[a]:
1019                 print('Must rebuild %s.' % a)
1020                 self.clear_last_build_state(a)
1021             else:
1022                 print('No need to rebuild %s.' % a)
1023         if os.access(self.logsdir, os.F_OK):
1024             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1025             shutil.copytree(self.logsdir, self.logsdir_old)
1026         for a in actions:
1027             if must_build[a]:
1028                 build_time = datetime.datetime.utcnow()
1029                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1030                 self.bot_run_self([], a)
1031                 self.load_build_state_json()
1032                 self.bot_build_mail(a, build_time)
1033         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1034
1035     def bot_build_mail(self, action, build_time):
1036         """Send email with the results of a build."""
1037         if not ('email-from' in self.bot_config and
1038                 'email-server' in self.bot_config and
1039                 'email-subject' in self.bot_config and
1040                 'email-to' in self.bot_config):
1041             if not self.email_warning:
1042                 print("Email not configured, not sending.")
1043                 self.email_warning = True
1044             return
1045
1046         build_time = build_time.replace(microsecond=0)
1047         subject = (self.bot_config['email-subject'] %
1048                    {'action': action,
1049                     'build-time': str(build_time)})
1050         results = self.build_state[action]['build-results']
1051         changes = self.build_state[action]['result-changes']
1052         ever_passed = set(self.build_state[action]['ever-passed'])
1053         versions = self.build_state[action]['build-versions']
1054         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1055         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1056         all_fails = {k for k in results if results[k] == 'FAIL'}
1057         if new_regressions:
1058             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1059             new_reg_text = ('New regressions:\n\n%s\n\n' %
1060                             '\n'.join(new_reg_list))
1061         else:
1062             new_reg_text = ''
1063         if all_regressions:
1064             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1065             all_reg_text = ('All regressions:\n\n%s\n\n' %
1066                             '\n'.join(all_reg_list))
1067         else:
1068             all_reg_text = ''
1069         if all_fails:
1070             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1071             all_fail_text = ('All failures:\n\n%s\n\n' %
1072                              '\n'.join(all_fail_list))
1073         else:
1074             all_fail_text = ''
1075         if changes:
1076             changes_list = sorted(changes.keys())
1077             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1078             changes_text = ('All changed results:\n\n%s\n\n' %
1079                             '\n'.join(changes_list))
1080         else:
1081             changes_text = ''
1082         results_text = (new_reg_text + all_reg_text + all_fail_text +
1083                         changes_text)
1084         if not results_text:
1085             results_text = 'Clean build with unchanged results.\n\n'
1086         versions_list = sorted(versions.keys())
1087         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1088                                           versions[k]['revision'])
1089                          for k in versions_list]
1090         versions_text = ('Component versions for this build:\n\n%s\n' %
1091                          '\n'.join(versions_list))
1092         body_text = results_text + versions_text
1093         msg = email.mime.text.MIMEText(body_text)
1094         msg['Subject'] = subject
1095         msg['From'] = self.bot_config['email-from']
1096         msg['To'] = self.bot_config['email-to']
1097         msg['Message-ID'] = email.utils.make_msgid()
1098         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1099         with smtplib.SMTP(self.bot_config['email-server']) as s:
1100             s.send_message(msg)
1101
1102     def bot_run_self(self, opts, action, check=True):
1103         """Run a copy of this script with given options."""
1104         cmd = [sys.executable, sys.argv[0], '--keep=none',
1105                '-j%d' % self.parallelism]
1106         cmd.extend(opts)
1107         cmd.extend([self.topdir, action])
1108         sys.stdout.flush()
1109         subprocess.run(cmd, check=check)
1110
1111     def bot(self):
1112         """Run repeated rounds of checkout and builds."""
1113         while True:
1114             self.load_bot_config_json()
1115             if not self.bot_config['run']:
1116                 print('Bot exiting by request.')
1117                 exit(0)
1118             self.bot_run_self([], 'bot-cycle', check=False)
1119             self.load_bot_config_json()
1120             if not self.bot_config['run']:
1121                 print('Bot exiting by request.')
1122                 exit(0)
1123             time.sleep(self.bot_config['delay'])
1124             if self.get_script_text() != self.script_text:
1125                 print('Script changed, bot re-execing.')
1126                 self.exec_self()
1127
1128
1129 class Config(object):
1130     """A configuration for building a compiler and associated libraries."""
1131
1132     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1133                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1134         """Initialize a Config object."""
1135         self.ctx = ctx
1136         self.arch = arch
1137         self.os = os_name
1138         self.variant = variant
1139         if variant is None:
1140             self.name = '%s-%s' % (arch, os_name)
1141         else:
1142             self.name = '%s-%s-%s' % (arch, os_name, variant)
1143         self.triplet = '%s-glibc-%s' % (arch, os_name)
1144         if gcc_cfg is None:
1145             self.gcc_cfg = []
1146         else:
1147             self.gcc_cfg = gcc_cfg
1148         if first_gcc_cfg is None:
1149             self.first_gcc_cfg = []
1150         else:
1151             self.first_gcc_cfg = first_gcc_cfg
1152         if glibcs is None:
1153             glibcs = [{'variant': variant}]
1154         if extra_glibcs is None:
1155             extra_glibcs = []
1156         glibcs = [Glibc(self, **g) for g in glibcs]
1157         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1158         self.all_glibcs = glibcs + extra_glibcs
1159         self.compiler_glibcs = glibcs
1160         self.installdir = ctx.compiler_installdir(self.name)
1161         self.bindir = ctx.compiler_bindir(self.name)
1162         self.sysroot = ctx.compiler_sysroot(self.name)
1163         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1164         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1165
1166     def component_builddir(self, component):
1167         """Return the directory to use for a (non-glibc) build."""
1168         return self.ctx.component_builddir('compilers', self.name, component)
1169
1170     def build(self):
1171         """Generate commands to build this compiler."""
1172         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1173                                       self.logsdir)
1174         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1175         cmdlist.add_command('check-host-libraries',
1176                             ['test', '-f',
1177                              os.path.join(self.ctx.host_libraries_installdir,
1178                                           'ok')])
1179         cmdlist.use_path(self.bindir)
1180         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1181                               ['--disable-gdb',
1182                                '--disable-libdecnumber',
1183                                '--disable-readline',
1184                                '--disable-sim'])
1185         if self.os.startswith('linux'):
1186             self.install_linux_headers(cmdlist)
1187         self.build_gcc(cmdlist, True)
1188         if self.os == 'gnu':
1189             self.install_gnumach_headers(cmdlist)
1190             self.build_cross_tool(cmdlist, 'mig', 'mig')
1191             self.install_hurd_headers(cmdlist)
1192         for g in self.compiler_glibcs:
1193             cmdlist.push_subdesc('glibc')
1194             cmdlist.push_subdesc(g.name)
1195             g.build_glibc(cmdlist, True)
1196             cmdlist.pop_subdesc()
1197             cmdlist.pop_subdesc()
1198         self.build_gcc(cmdlist, False)
1199         cmdlist.add_command('done', ['touch',
1200                                      os.path.join(self.installdir, 'ok')])
1201         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1202                                       self.logsdir)
1203
1204     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1205         """Build one cross tool."""
1206         srcdir = self.ctx.component_srcdir(tool_src)
1207         builddir = self.component_builddir(tool_build)
1208         cmdlist.push_subdesc(tool_build)
1209         cmdlist.create_use_dir(builddir)
1210         cfg_cmd = [os.path.join(srcdir, 'configure'),
1211                    '--prefix=%s' % self.installdir,
1212                    '--build=%s' % self.ctx.build_triplet,
1213                    '--host=%s' % self.ctx.build_triplet,
1214                    '--target=%s' % self.triplet,
1215                    '--with-sysroot=%s' % self.sysroot]
1216         if extra_opts:
1217             cfg_cmd.extend(extra_opts)
1218         cmdlist.add_command('configure', cfg_cmd)
1219         cmdlist.add_command('build', ['make'])
1220         # Parallel "make install" for GCC has race conditions that can
1221         # cause it to fail; see
1222         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1223         # problems are not known for binutils, but doing the
1224         # installation in parallel within a particular toolchain build
1225         # (as opposed to installation of one toolchain from
1226         # build-many-glibcs.py running in parallel to the installation
1227         # of other toolchains being built) is not known to be
1228         # significantly beneficial, so it is simplest just to disable
1229         # parallel install for cross tools here.
1230         cmdlist.add_command('install', ['make', '-j1', 'install'])
1231         cmdlist.cleanup_dir()
1232         cmdlist.pop_subdesc()
1233
1234     def install_linux_headers(self, cmdlist):
1235         """Install Linux kernel headers."""
1236         arch_map = {'aarch64': 'arm64',
1237                     'alpha': 'alpha',
1238                     'arm': 'arm',
1239                     'hppa': 'parisc',
1240                     'i486': 'x86',
1241                     'i586': 'x86',
1242                     'i686': 'x86',
1243                     'i786': 'x86',
1244                     'ia64': 'ia64',
1245                     'm68k': 'm68k',
1246                     'microblaze': 'microblaze',
1247                     'mips': 'mips',
1248                     'nios2': 'nios2',
1249                     'powerpc': 'powerpc',
1250                     's390': 's390',
1251                     'sh': 'sh',
1252                     'sparc': 'sparc',
1253                     'tile': 'tile',
1254                     'x86_64': 'x86'}
1255         linux_arch = None
1256         for k in arch_map:
1257             if self.arch.startswith(k):
1258                 linux_arch = arch_map[k]
1259                 break
1260         assert linux_arch is not None
1261         srcdir = self.ctx.component_srcdir('linux')
1262         builddir = self.component_builddir('linux')
1263         headers_dir = os.path.join(self.sysroot, 'usr')
1264         cmdlist.push_subdesc('linux')
1265         cmdlist.create_use_dir(builddir)
1266         cmdlist.add_command('install-headers',
1267                             ['make', '-C', srcdir, 'O=%s' % builddir,
1268                              'ARCH=%s' % linux_arch,
1269                              'INSTALL_HDR_PATH=%s' % headers_dir,
1270                              'headers_install'])
1271         cmdlist.cleanup_dir()
1272         cmdlist.pop_subdesc()
1273
1274     def install_gnumach_headers(self, cmdlist):
1275         """Install GNU Mach headers."""
1276         srcdir = self.ctx.component_srcdir('gnumach')
1277         builddir = self.component_builddir('gnumach')
1278         cmdlist.push_subdesc('gnumach')
1279         cmdlist.create_use_dir(builddir)
1280         cmdlist.add_command('configure',
1281                             [os.path.join(srcdir, 'configure'),
1282                              '--build=%s' % self.ctx.build_triplet,
1283                              '--host=%s' % self.triplet,
1284                              '--prefix=',
1285                              'CC=%s-gcc -nostdlib' % self.triplet])
1286         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1287                                         'install-data'])
1288         cmdlist.cleanup_dir()
1289         cmdlist.pop_subdesc()
1290
1291     def install_hurd_headers(self, cmdlist):
1292         """Install Hurd headers."""
1293         srcdir = self.ctx.component_srcdir('hurd')
1294         builddir = self.component_builddir('hurd')
1295         cmdlist.push_subdesc('hurd')
1296         cmdlist.create_use_dir(builddir)
1297         cmdlist.add_command('configure',
1298                             [os.path.join(srcdir, 'configure'),
1299                              '--build=%s' % self.ctx.build_triplet,
1300                              '--host=%s' % self.triplet,
1301                              '--prefix=',
1302                              '--disable-profile', '--without-parted',
1303                              'CC=%s-gcc -nostdlib' % self.triplet])
1304         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1305                                         'no_deps=t', 'install-headers'])
1306         cmdlist.cleanup_dir()
1307         cmdlist.pop_subdesc()
1308
1309     def build_gcc(self, cmdlist, bootstrap):
1310         """Build GCC."""
1311         # libsanitizer commonly breaks because of glibc header
1312         # changes, or on unusual targets.  libssp is of little
1313         # relevance with glibc's own stack checking support.
1314         cfg_opts = list(self.gcc_cfg)
1315         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1316         host_libs = self.ctx.host_libraries_installdir
1317         cfg_opts += ['--with-gmp=%s' % host_libs,
1318                      '--with-mpfr=%s' % host_libs,
1319                      '--with-mpc=%s' % host_libs]
1320         if bootstrap:
1321             tool_build = 'gcc-first'
1322             # Building a static-only, C-only compiler that is
1323             # sufficient to build glibc.  Various libraries and
1324             # features that may require libc headers must be disabled.
1325             # When configuring with a sysroot, --with-newlib is
1326             # required to define inhibit_libc (to stop some parts of
1327             # libgcc including libc headers); --without-headers is not
1328             # sufficient.
1329             cfg_opts += ['--enable-languages=c', '--disable-shared',
1330                          '--disable-threads',
1331                          '--disable-libatomic',
1332                          '--disable-decimal-float',
1333                          '--disable-libffi',
1334                          '--disable-libgomp',
1335                          '--disable-libitm',
1336                          '--disable-libmpx',
1337                          '--disable-libquadmath',
1338                          '--without-headers', '--with-newlib',
1339                          '--with-glibc-version=%s' % self.ctx.glibc_version
1340                          ]
1341             cfg_opts += self.first_gcc_cfg
1342         else:
1343             tool_build = 'gcc'
1344             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1345                          '--enable-threads']
1346             if self.os == 'gnu':
1347                 cfg_opts += ['--disable-libcilkrts']
1348         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1349
1350
1351 class Glibc(object):
1352     """A configuration for building glibc."""
1353
1354     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1355                  cfg=None, ccopts=None):
1356         """Initialize a Glibc object."""
1357         self.ctx = compiler.ctx
1358         self.compiler = compiler
1359         if arch is None:
1360             self.arch = compiler.arch
1361         else:
1362             self.arch = arch
1363         if os_name is None:
1364             self.os = compiler.os
1365         else:
1366             self.os = os_name
1367         self.variant = variant
1368         if variant is None:
1369             self.name = '%s-%s' % (self.arch, self.os)
1370         else:
1371             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1372         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1373         if cfg is None:
1374             self.cfg = []
1375         else:
1376             self.cfg = cfg
1377         self.ccopts = ccopts
1378
1379     def tool_name(self, tool):
1380         """Return the name of a cross-compilation tool."""
1381         ctool = '%s-%s' % (self.compiler.triplet, tool)
1382         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1383             ctool = '%s %s' % (ctool, self.ccopts)
1384         return ctool
1385
1386     def build(self):
1387         """Generate commands to build this glibc."""
1388         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1389         installdir = self.ctx.glibc_installdir(self.name)
1390         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1391         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1392         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1393         cmdlist.add_command('check-compilers',
1394                             ['test', '-f',
1395                              os.path.join(self.compiler.installdir, 'ok')])
1396         cmdlist.use_path(self.compiler.bindir)
1397         self.build_glibc(cmdlist, False)
1398         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1399                                       logsdir)
1400
1401     def build_glibc(self, cmdlist, for_compiler):
1402         """Generate commands to build this glibc, either as part of a compiler
1403         build or with the bootstrapped compiler (and in the latter case, run
1404         tests as well)."""
1405         srcdir = self.ctx.component_srcdir('glibc')
1406         if for_compiler:
1407             builddir = self.ctx.component_builddir('compilers',
1408                                                    self.compiler.name, 'glibc',
1409                                                    self.name)
1410             installdir = self.compiler.sysroot
1411             srcdir_copy = self.ctx.component_builddir('compilers',
1412                                                       self.compiler.name,
1413                                                       'glibc-src',
1414                                                       self.name)
1415         else:
1416             builddir = self.ctx.component_builddir('glibcs', self.name,
1417                                                    'glibc')
1418             installdir = self.ctx.glibc_installdir(self.name)
1419             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1420                                                       'glibc-src')
1421         cmdlist.create_use_dir(builddir)
1422         # glibc builds write into the source directory, and even if
1423         # not intentionally there is a risk of bugs that involve
1424         # writing into the working directory.  To avoid possible
1425         # concurrency issues, copy the source directory.
1426         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1427         use_usr = self.os != 'gnu'
1428         prefix = '/usr' if use_usr else ''
1429         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1430                    '--prefix=%s' % prefix,
1431                    '--enable-profile',
1432                    '--build=%s' % self.ctx.build_triplet,
1433                    '--host=%s' % self.triplet,
1434                    'CC=%s' % self.tool_name('gcc'),
1435                    'CXX=%s' % self.tool_name('g++'),
1436                    'AR=%s' % self.tool_name('ar'),
1437                    'AS=%s' % self.tool_name('as'),
1438                    'LD=%s' % self.tool_name('ld'),
1439                    'NM=%s' % self.tool_name('nm'),
1440                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1441                    'OBJDUMP=%s' % self.tool_name('objdump'),
1442                    'RANLIB=%s' % self.tool_name('ranlib'),
1443                    'READELF=%s' % self.tool_name('readelf'),
1444                    'STRIP=%s' % self.tool_name('strip')]
1445         if self.os == 'gnu':
1446             cfg_cmd += ['MIG=%s' % self.tool_name('mig')]
1447         cfg_cmd += self.cfg
1448         cmdlist.add_command('configure', cfg_cmd)
1449         cmdlist.add_command('build', ['make'])
1450         cmdlist.add_command('install', ['make', 'install',
1451                                         'install_root=%s' % installdir])
1452         # GCC uses paths such as lib/../lib64, so make sure lib
1453         # directories always exist.
1454         mkdir_cmd = ['mkdir', '-p',
1455                      os.path.join(installdir, 'lib')]
1456         if use_usr:
1457             mkdir_cmd += [os.path.join(installdir, 'usr', 'lib')]
1458         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1459         if not for_compiler:
1460             if self.ctx.strip:
1461                 cmdlist.add_command('strip',
1462                                     ['sh', '-c',
1463                                      ('%s $(find %s/lib* -name "*.so")' %
1464                                       (self.tool_name('strip'), installdir))])
1465             cmdlist.add_command('check', ['make', 'check'])
1466             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1467                                 always_run=True)
1468         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1469         cmdlist.cleanup_dir()
1470
1471
1472 class Command(object):
1473     """A command run in the build process."""
1474
1475     def __init__(self, desc, num, dir, path, command, always_run=False):
1476         """Initialize a Command object."""
1477         self.dir = dir
1478         self.path = path
1479         self.desc = desc
1480         trans = str.maketrans({' ': '-'})
1481         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1482         self.command = command
1483         self.always_run = always_run
1484
1485     @staticmethod
1486     def shell_make_quote_string(s):
1487         """Given a string not containing a newline, quote it for use by the
1488         shell and make."""
1489         assert '\n' not in s
1490         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1491             return s
1492         strans = str.maketrans({"'": "'\\''"})
1493         s = "'%s'" % s.translate(strans)
1494         mtrans = str.maketrans({'$': '$$'})
1495         return s.translate(mtrans)
1496
1497     @staticmethod
1498     def shell_make_quote_list(l, translate_make):
1499         """Given a list of strings not containing newlines, quote them for use
1500         by the shell and make, returning a single string.  If translate_make
1501         is true and the first string is 'make', change it to $(MAKE)."""
1502         l = [Command.shell_make_quote_string(s) for s in l]
1503         if translate_make and l[0] == 'make':
1504             l[0] = '$(MAKE)'
1505         return ' '.join(l)
1506
1507     def shell_make_quote(self):
1508         """Return this command quoted for the shell and make."""
1509         return self.shell_make_quote_list(self.command, True)
1510
1511
1512 class CommandList(object):
1513     """A list of commands run in the build process."""
1514
1515     def __init__(self, desc, keep):
1516         """Initialize a CommandList object."""
1517         self.cmdlist = []
1518         self.dir = None
1519         self.path = None
1520         self.desc = [desc]
1521         self.keep = keep
1522
1523     def desc_txt(self, desc):
1524         """Return the description to use for a command."""
1525         return '%s %s' % (' '.join(self.desc), desc)
1526
1527     def use_dir(self, dir):
1528         """Set the default directory for subsequent commands."""
1529         self.dir = dir
1530
1531     def use_path(self, path):
1532         """Set a directory to be prepended to the PATH for subsequent
1533         commands."""
1534         self.path = path
1535
1536     def push_subdesc(self, subdesc):
1537         """Set the default subdescription for subsequent commands (e.g., the
1538         name of a component being built, within the series of commands
1539         building it)."""
1540         self.desc.append(subdesc)
1541
1542     def pop_subdesc(self):
1543         """Pop a subdescription from the list of descriptions."""
1544         self.desc.pop()
1545
1546     def create_use_dir(self, dir):
1547         """Remove and recreate a directory and use it for subsequent
1548         commands."""
1549         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1550         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1551         self.use_dir(dir)
1552
1553     def create_copy_dir(self, src, dest):
1554         """Remove a directory and recreate it as a copy from the given
1555         source."""
1556         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1557         parent = os.path.dirname(dest)
1558         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1559         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1560
1561     def add_command_dir(self, desc, dir, command, always_run=False):
1562         """Add a command to run in a given directory."""
1563         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1564                       command, always_run)
1565         self.cmdlist.append(cmd)
1566
1567     def add_command(self, desc, command, always_run=False):
1568         """Add a command to run in the default directory."""
1569         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1570                       self.path, command, always_run)
1571         self.cmdlist.append(cmd)
1572
1573     def cleanup_dir(self, desc='cleanup', dir=None):
1574         """Clean up a build directory.  If no directory is specified, the
1575         default directory is cleaned up and ceases to be the default
1576         directory."""
1577         if dir is None:
1578             dir = self.dir
1579             self.use_dir(None)
1580         if self.keep != 'all':
1581             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1582                                  always_run=(self.keep == 'none'))
1583
1584     def makefile_commands(self, wrapper, logsdir):
1585         """Return the sequence of commands in the form of text for a Makefile.
1586         The given wrapper script takes arguments: base of logs for
1587         previous command, or empty; base of logs for this command;
1588         description; directory; PATH addition; the command itself."""
1589         # prev_base is the base of the name for logs of the previous
1590         # command that is not always-run (that is, a build command,
1591         # whose failure should stop subsequent build commands from
1592         # being run, as opposed to a cleanup command, which is run
1593         # even if previous commands failed).
1594         prev_base = ''
1595         cmds = []
1596         for c in self.cmdlist:
1597             ctxt = c.shell_make_quote()
1598             if prev_base and not c.always_run:
1599                 prev_log = os.path.join(logsdir, prev_base)
1600             else:
1601                 prev_log = ''
1602             this_log = os.path.join(logsdir, c.logbase)
1603             if not c.always_run:
1604                 prev_base = c.logbase
1605             if c.dir is None:
1606                 dir = ''
1607             else:
1608                 dir = c.dir
1609             if c.path is None:
1610                 path = ''
1611             else:
1612                 path = c.path
1613             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1614             prelim_txt = Command.shell_make_quote_list(prelims, False)
1615             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1616         return '\n'.join(cmds)
1617
1618     def status_logs(self, logsdir):
1619         """Return the list of log files with command status."""
1620         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1621                 for c in self.cmdlist]
1622
1623
1624 def get_parser():
1625     """Return an argument parser for this module."""
1626     parser = argparse.ArgumentParser(description=__doc__)
1627     parser.add_argument('-j', dest='parallelism',
1628                         help='Run this number of jobs in parallel',
1629                         type=int, default=os.cpu_count())
1630     parser.add_argument('--keep', dest='keep',
1631                         help='Whether to keep all build directories, '
1632                         'none or only those from failed builds',
1633                         default='none', choices=('none', 'all', 'failed'))
1634     parser.add_argument('--replace-sources', action='store_true',
1635                         help='Remove and replace source directories '
1636                         'with the wrong version of a component')
1637     parser.add_argument('--strip', action='store_true',
1638                         help='Strip installed glibc libraries')
1639     parser.add_argument('topdir',
1640                         help='Toplevel working directory')
1641     parser.add_argument('action',
1642                         help='What to do',
1643                         choices=('checkout', 'bot-cycle', 'bot',
1644                                  'host-libraries', 'compilers', 'glibcs'))
1645     parser.add_argument('configs',
1646                         help='Versions to check out or configurations to build',
1647                         nargs='*')
1648     return parser
1649
1650
1651 def main(argv):
1652     """The main entry point."""
1653     parser = get_parser()
1654     opts = parser.parse_args(argv)
1655     topdir = os.path.abspath(opts.topdir)
1656     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1657                   opts.strip, opts.action)
1658     ctx.run_builds(opts.action, opts.configs)
1659
1660
1661 if __name__ == '__main__':
1662     main(sys.argv[1:])