scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2022 Free Software Foundation, Inc.
   4 # Copyright The GNU Toolchain Authors.
   5 # This file is part of the GNU C Library.
   6 #
   7 # The GNU C Library is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU Lesser General Public
   9 # License as published by the Free Software Foundation; either
  10 # version 2.1 of the License, or (at your option) any later version.
  11 #
  12 # The GNU C Library is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 # Lesser General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Lesser General Public
  18 # License along with the GNU C Library; if not, see
  19 # <https://www.gnu.org/licenses/>.
  20
  21 """Build many configurations of glibc.
  22
  23 This script takes as arguments a directory name (containing a src
  24 subdirectory with sources of the relevant toolchain components) and a
  25 description of what to do: 'checkout', to check out sources into that
  26 directory, 'bot-cycle', to run a series of checkout and build steps,
  27 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  28 libraries required by the toolchain, 'compilers', to build
  29 cross-compilers for various configurations, or 'glibcs', to build
  30 glibc for various configurations and run the compilation parts of the
  31 testsuite.  Subsequent arguments name the versions of components to
  32 check out (<component>-<version), for 'checkout', or, for actions
  33 other than 'checkout' and 'bot-cycle', name configurations for which
  34 compilers or glibc are to be built.
  35
  36 The 'list-compilers' command prints the name of each available
  37 compiler configuration, without building anything.  The 'list-glibcs'
  38 command prints the name of each glibc compiler configuration, followed
  39 by the space, followed by the name of the compiler configuration used
  40 for building this glibc variant.
  41
  42 """
  43
  44 import argparse
  45 import datetime
  46 import email.mime.text
  47 import email.utils
  48 import json
  49 import os
  50 import re
  51 import shutil
  52 import smtplib
  53 import stat
  54 import subprocess
  55 import sys
  56 import time
  57 import urllib.request
  58
  59 try:
  60     subprocess.run
  61 except:
  62     class _CompletedProcess:
  63         def __init__(self, args, returncode, stdout=None, stderr=None):
  64             self.args = args
  65             self.returncode = returncode
  66             self.stdout = stdout
  67             self.stderr = stderr
  68
  69     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  70         assert(timeout is None)
  71         with subprocess.Popen(*popenargs, **kwargs) as process:
  72             try:
  73                 stdout, stderr = process.communicate(input)
  74             except:
  75                 process.kill()
  76                 process.wait()
  77                 raise
  78             returncode = process.poll()
  79             if check and returncode:
  80                 raise subprocess.CalledProcessError(returncode, popenargs)
  81         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  82
  83     subprocess.run = _run
  84
  85
  86 class Context(object):
  87     """The global state associated with builds in a given directory."""
  88
  89     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  90                  full_gcc, action, shallow=False):
  91         """Initialize the context."""
  92         self.topdir = topdir
  93         self.parallelism = parallelism
  94         self.keep = keep
  95         self.replace_sources = replace_sources
  96         self.strip = strip
  97         self.full_gcc = full_gcc
  98         self.shallow = shallow
  99         self.srcdir = os.path.join(topdir, 'src')
 100         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 101         self.build_state_json = os.path.join(topdir, 'build-state.json')
 102         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 103         self.installdir = os.path.join(topdir, 'install')
 104         self.host_libraries_installdir = os.path.join(self.installdir,
 105                                                       'host-libraries')
 106         self.builddir = os.path.join(topdir, 'build')
 107         self.logsdir = os.path.join(topdir, 'logs')
 108         self.logsdir_old = os.path.join(topdir, 'logs-old')
 109         self.makefile = os.path.join(self.builddir, 'Makefile')
 110         self.wrapper = os.path.join(self.builddir, 'wrapper')
 111         self.save_logs = os.path.join(self.builddir, 'save-logs')
 112         self.script_text = self.get_script_text()
 113         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 114             self.build_triplet = self.get_build_triplet()
 115             self.glibc_version = self.get_glibc_version()
 116         self.configs = {}
 117         self.glibc_configs = {}
 118         self.makefile_pieces = ['.PHONY: all\n']
 119         self.add_all_configs()
 120         self.load_versions_json()
 121         self.load_build_state_json()
 122         self.status_log_list = []
 123         self.email_warning = False
 124
 125     def get_script_text(self):
 126         """Return the text of this script."""
 127         with open(sys.argv[0], 'r') as f:
 128             return f.read()
 129
 130     def exec_self(self):
 131         """Re-execute this script with the same arguments."""
 132         sys.stdout.flush()
 133         os.execv(sys.executable, [sys.executable] + sys.argv)
 134
 135     def get_build_triplet(self):
 136         """Determine the build triplet with config.guess."""
 137         config_guess = os.path.join(self.component_srcdir('gcc'),
 138                                     'config.guess')
 139         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 140                                 check=True, universal_newlines=True).stdout
 141         return cg_out.rstrip()
 142
 143     def get_glibc_version(self):
 144         """Determine the glibc version number (major.minor)."""
 145         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 146         with open(version_h, 'r') as f:
 147             lines = f.readlines()
 148         starttext = '#define VERSION "'
 149         for l in lines:
 150             if l.startswith(starttext):
 151                 l = l[len(starttext):]
 152                 l = l.rstrip('"\n')
 153                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 154                 return '%s.%s' % m.group(1, 2)
 155         print('error: could not determine glibc version')
 156         exit(1)
 157
 158     def add_all_configs(self):
 159         """Add all known glibc build configurations."""
 160         self.add_config(arch='aarch64',
 161                         os_name='linux-gnu',
 162                         extra_glibcs=[{'variant': 'disable-multi-arch',
 163                                        'cfg': ['--disable-multi-arch']}])
 164         self.add_config(arch='aarch64_be',
 165                         os_name='linux-gnu')
 166         self.add_config(arch='arc',
 167                         os_name='linux-gnu',
 168                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 169         self.add_config(arch='arc',
 170                         os_name='linux-gnuhf',
 171                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
 172         self.add_config(arch='arceb',
 173                         os_name='linux-gnu',
 174                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 175         self.add_config(arch='alpha',
 176                         os_name='linux-gnu')
 177         self.add_config(arch='arm',
 178                         os_name='linux-gnueabi',
 179                         extra_glibcs=[{'variant': 'v4t',
 180                                        'ccopts': '-march=armv4t'}])
 181         self.add_config(arch='armeb',
 182                         os_name='linux-gnueabi')
 183         self.add_config(arch='armeb',
 184                         os_name='linux-gnueabi',
 185                         variant='be8',
 186                         gcc_cfg=['--with-arch=armv7-a'])
 187         self.add_config(arch='arm',
 188                         os_name='linux-gnueabihf',
 189                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 190                         extra_glibcs=[{'variant': 'v7a',
 191                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 192                                       {'variant': 'thumb',
 193                                        'ccopts':
 194                                        '-mthumb -march=armv7-a -mfpu=vfpv3'},
 195                                       {'variant': 'v7a-disable-multi-arch',
 196                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 197                                        'cfg': ['--disable-multi-arch']}])
 198         self.add_config(arch='armeb',
 199                         os_name='linux-gnueabihf',
 200                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 201         self.add_config(arch='armeb',
 202                         os_name='linux-gnueabihf',
 203                         variant='be8',
 204                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 205                                  '--with-fpu=vfpv3'])
 206         self.add_config(arch='csky',
 207                         os_name='linux-gnuabiv2',
 208                         variant='soft',
 209                         gcc_cfg=['--disable-multilib'])
 210         self.add_config(arch='csky',
 211                         os_name='linux-gnuabiv2',
 212                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 213         self.add_config(arch='hppa',
 214                         os_name='linux-gnu')
 215         self.add_config(arch='i686',
 216                         os_name='gnu')
 217         self.add_config(arch='ia64',
 218                         os_name='linux-gnu',
 219                         first_gcc_cfg=['--with-system-libunwind'],
 220                         binutils_cfg=['--enable-obsolete'])
 221         self.add_config(arch='m68k',
 222                         os_name='linux-gnu',
 223                         gcc_cfg=['--disable-multilib'])
 224         self.add_config(arch='m68k',
 225                         os_name='linux-gnu',
 226                         variant='coldfire',
 227                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 228         self.add_config(arch='m68k',
 229                         os_name='linux-gnu',
 230                         variant='coldfire-soft',
 231                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 232                                  '--disable-multilib'])
 233         self.add_config(arch='microblaze',
 234                         os_name='linux-gnu',
 235                         gcc_cfg=['--disable-multilib'])
 236         self.add_config(arch='microblazeel',
 237                         os_name='linux-gnu',
 238                         gcc_cfg=['--disable-multilib'])
 239         self.add_config(arch='mips64',
 240                         os_name='linux-gnu',
 241                         gcc_cfg=['--with-mips-plt'],
 242                         glibcs=[{'variant': 'n32'},
 243                                 {'arch': 'mips',
 244                                  'ccopts': '-mabi=32'},
 245                                 {'variant': 'n64',
 246                                  'ccopts': '-mabi=64'}])
 247         self.add_config(arch='mips64',
 248                         os_name='linux-gnu',
 249                         variant='soft',
 250                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 251                         glibcs=[{'variant': 'n32-soft'},
 252                                 {'variant': 'soft',
 253                                  'arch': 'mips',
 254                                  'ccopts': '-mabi=32'},
 255                                 {'variant': 'n64-soft',
 256                                  'ccopts': '-mabi=64'}])
 257         self.add_config(arch='mips64',
 258                         os_name='linux-gnu',
 259                         variant='nan2008',
 260                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 261                                  '--with-arch-64=mips64r2',
 262                                  '--with-arch-32=mips32r2'],
 263                         glibcs=[{'variant': 'n32-nan2008'},
 264                                 {'variant': 'nan2008',
 265                                  'arch': 'mips',
 266                                  'ccopts': '-mabi=32'},
 267                                 {'variant': 'n64-nan2008',
 268                                  'ccopts': '-mabi=64'}])
 269         self.add_config(arch='mips64',
 270                         os_name='linux-gnu',
 271                         variant='nan2008-soft',
 272                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 273                                  '--with-arch-64=mips64r2',
 274                                  '--with-arch-32=mips32r2',
 275                                  '--with-float=soft'],
 276                         glibcs=[{'variant': 'n32-nan2008-soft'},
 277                                 {'variant': 'nan2008-soft',
 278                                  'arch': 'mips',
 279                                  'ccopts': '-mabi=32'},
 280                                 {'variant': 'n64-nan2008-soft',
 281                                  'ccopts': '-mabi=64'}])
 282         self.add_config(arch='mips64el',
 283                         os_name='linux-gnu',
 284                         gcc_cfg=['--with-mips-plt'],
 285                         glibcs=[{'variant': 'n32'},
 286                                 {'arch': 'mipsel',
 287                                  'ccopts': '-mabi=32'},
 288                                 {'variant': 'n64',
 289                                  'ccopts': '-mabi=64'}])
 290         self.add_config(arch='mips64el',
 291                         os_name='linux-gnu',
 292                         variant='soft',
 293                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 294                         glibcs=[{'variant': 'n32-soft'},
 295                                 {'variant': 'soft',
 296                                  'arch': 'mipsel',
 297                                  'ccopts': '-mabi=32'},
 298                                 {'variant': 'n64-soft',
 299                                  'ccopts': '-mabi=64'}])
 300         self.add_config(arch='mips64el',
 301                         os_name='linux-gnu',
 302                         variant='nan2008',
 303                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 304                                  '--with-arch-64=mips64r2',
 305                                  '--with-arch-32=mips32r2'],
 306                         glibcs=[{'variant': 'n32-nan2008'},
 307                                 {'variant': 'nan2008',
 308                                  'arch': 'mipsel',
 309                                  'ccopts': '-mabi=32'},
 310                                 {'variant': 'n64-nan2008',
 311                                  'ccopts': '-mabi=64'}])
 312         self.add_config(arch='mips64el',
 313                         os_name='linux-gnu',
 314                         variant='nan2008-soft',
 315                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 316                                  '--with-arch-64=mips64r2',
 317                                  '--with-arch-32=mips32r2',
 318                                  '--with-float=soft'],
 319                         glibcs=[{'variant': 'n32-nan2008-soft'},
 320                                 {'variant': 'nan2008-soft',
 321                                  'arch': 'mipsel',
 322                                  'ccopts': '-mabi=32'},
 323                                 {'variant': 'n64-nan2008-soft',
 324                                  'ccopts': '-mabi=64'}])
 325         self.add_config(arch='mipsisa64r6el',
 326                         os_name='linux-gnu',
 327                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 328                                  '--with-arch-64=mips64r6',
 329                                  '--with-arch-32=mips32r6',
 330                                  '--with-float=hard'],
 331                         glibcs=[{'variant': 'n32'},
 332                                 {'arch': 'mipsisa32r6el',
 333                                  'ccopts': '-mabi=32'},
 334                                 {'variant': 'n64',
 335                                  'ccopts': '-mabi=64'}])
 336         self.add_config(arch='nios2',
 337                         os_name='linux-gnu')
 338         self.add_config(arch='or1k',
 339                         os_name='linux-gnu',
 340                         variant='soft',
 341                         gcc_cfg=['--with-multilib-list=mcmov'])
 342         self.add_config(arch='powerpc',
 343                         os_name='linux-gnu',
 344                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 345                         extra_glibcs=[{'variant': 'power4',
 346                                        'ccopts': '-mcpu=power4',
 347                                        'cfg': ['--with-cpu=power4']}])
 348         self.add_config(arch='powerpc',
 349                         os_name='linux-gnu',
 350                         variant='soft',
 351                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 352                                  '--enable-secureplt'])
 353         self.add_config(arch='powerpc64',
 354                         os_name='linux-gnu',
 355                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 356         self.add_config(arch='powerpc64le',
 357                         os_name='linux-gnu',
 358                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 359                         extra_glibcs=[{'variant': 'disable-multi-arch',
 360                                        'cfg': ['--disable-multi-arch']}])
 361         self.add_config(arch='riscv32',
 362                         os_name='linux-gnu',
 363                         variant='rv32imac-ilp32',
 364                         gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
 365                                  '--disable-multilib'])
 366         self.add_config(arch='riscv32',
 367                         os_name='linux-gnu',
 368                         variant='rv32imafdc-ilp32',
 369                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
 370                                  '--disable-multilib'])
 371         self.add_config(arch='riscv32',
 372                         os_name='linux-gnu',
 373                         variant='rv32imafdc-ilp32d',
 374                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
 375                                  '--disable-multilib'])
 376         self.add_config(arch='riscv64',
 377                         os_name='linux-gnu',
 378                         variant='rv64imac-lp64',
 379                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 380                                  '--disable-multilib'])
 381         self.add_config(arch='riscv64',
 382                         os_name='linux-gnu',
 383                         variant='rv64imafdc-lp64',
 384                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 385                                  '--disable-multilib'])
 386         self.add_config(arch='riscv64',
 387                         os_name='linux-gnu',
 388                         variant='rv64imafdc-lp64d',
 389                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 390                                  '--disable-multilib'])
 391         self.add_config(arch='s390x',
 392                         os_name='linux-gnu',
 393                         glibcs=[{},
 394                                 {'arch': 's390', 'ccopts': '-m31'}],
 395                         extra_glibcs=[{'variant': 'O3',
 396                                        'cflags': '-O3'}])
 397         self.add_config(arch='sh3',
 398                         os_name='linux-gnu')
 399         self.add_config(arch='sh3eb',
 400                         os_name='linux-gnu')
 401         self.add_config(arch='sh4',
 402                         os_name='linux-gnu')
 403         self.add_config(arch='sh4eb',
 404                         os_name='linux-gnu')
 405         self.add_config(arch='sh4',
 406                         os_name='linux-gnu',
 407                         variant='soft',
 408                         gcc_cfg=['--without-fp'])
 409         self.add_config(arch='sh4eb',
 410                         os_name='linux-gnu',
 411                         variant='soft',
 412                         gcc_cfg=['--without-fp'])
 413         self.add_config(arch='sparc64',
 414                         os_name='linux-gnu',
 415                         glibcs=[{},
 416                                 {'arch': 'sparcv9',
 417                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 418                         extra_glibcs=[{'variant': 'leon3',
 419                                        'arch' : 'sparcv8',
 420                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 421                                       {'variant': 'disable-multi-arch',
 422                                        'cfg': ['--disable-multi-arch']},
 423                                       {'variant': 'disable-multi-arch',
 424                                        'arch': 'sparcv9',
 425                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 426                                        'cfg': ['--disable-multi-arch']}])
 427         self.add_config(arch='x86_64',
 428                         os_name='linux-gnu',
 429                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 430                         glibcs=[{},
 431                                 {'variant': 'x32', 'ccopts': '-mx32'},
 432                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 433                         extra_glibcs=[{'variant': 'disable-multi-arch',
 434                                        'cfg': ['--disable-multi-arch']},
 435                                       {'variant': 'minimal',
 436                                        'cfg': ['--disable-multi-arch',
 437                                                '--disable-profile',
 438                                                '--disable-timezone-tools',
 439                                                '--disable-mathvec',
 440                                                '--disable-tunables',
 441                                                '--disable-crypt',
 442                                                '--disable-experimental-malloc',
 443                                                '--disable-build-nscd',
 444                                                '--disable-nscd']},
 445                                       {'variant': 'no-pie',
 446                                        'cfg': ['--disable-default-pie']},
 447                                       {'variant': 'x32-no-pie',
 448                                        'ccopts': '-mx32',
 449                                        'cfg': ['--disable-default-pie']},
 450                                       {'variant': 'no-pie',
 451                                        'arch': 'i686',
 452                                        'ccopts': '-m32 -march=i686',
 453                                        'cfg': ['--disable-default-pie']},
 454                                       {'variant': 'disable-multi-arch',
 455                                        'arch': 'i686',
 456                                        'ccopts': '-m32 -march=i686',
 457                                        'cfg': ['--disable-multi-arch']},
 458                                       {'arch': 'i486',
 459                                        'ccopts': '-m32 -march=i486'},
 460                                       {'arch': 'i586',
 461                                        'ccopts': '-m32 -march=i586'}])
 462
 463     def add_config(self, **args):
 464         """Add an individual build configuration."""
 465         cfg = Config(self, **args)
 466         if cfg.name in self.configs:
 467             print('error: duplicate config %s' % cfg.name)
 468             exit(1)
 469         self.configs[cfg.name] = cfg
 470         for c in cfg.all_glibcs:
 471             if c.name in self.glibc_configs:
 472                 print('error: duplicate glibc config %s' % c.name)
 473                 exit(1)
 474             self.glibc_configs[c.name] = c
 475
 476     def component_srcdir(self, component):
 477         """Return the source directory for a given component, e.g. gcc."""
 478         return os.path.join(self.srcdir, component)
 479
 480     def component_builddir(self, action, config, component, subconfig=None):
 481         """Return the directory to use for a build."""
 482         if config is None:
 483             # Host libraries.
 484             assert subconfig is None
 485             return os.path.join(self.builddir, action, component)
 486         if subconfig is None:
 487             return os.path.join(self.builddir, action, config, component)
 488         else:
 489             # glibc build as part of compiler build.
 490             return os.path.join(self.builddir, action, config, component,
 491                                 subconfig)
 492
 493     def compiler_installdir(self, config):
 494         """Return the directory in which to install a compiler."""
 495         return os.path.join(self.installdir, 'compilers', config)
 496
 497     def compiler_bindir(self, config):
 498         """Return the directory in which to find compiler binaries."""
 499         return os.path.join(self.compiler_installdir(config), 'bin')
 500
 501     def compiler_sysroot(self, config):
 502         """Return the sysroot directory for a compiler."""
 503         return os.path.join(self.compiler_installdir(config), 'sysroot')
 504
 505     def glibc_installdir(self, config):
 506         """Return the directory in which to install glibc."""
 507         return os.path.join(self.installdir, 'glibcs', config)
 508
 509     def run_builds(self, action, configs):
 510         """Run the requested builds."""
 511         if action == 'checkout':
 512             self.checkout(configs)
 513             return
 514         if action == 'bot-cycle':
 515             if configs:
 516                 print('error: configurations specified for bot-cycle')
 517                 exit(1)
 518             self.bot_cycle()
 519             return
 520         if action == 'bot':
 521             if configs:
 522                 print('error: configurations specified for bot')
 523                 exit(1)
 524             self.bot()
 525             return
 526         if action in ('host-libraries', 'list-compilers',
 527                       'list-glibcs') and configs:
 528             print('error: configurations specified for ' + action)
 529             exit(1)
 530         if action == 'list-compilers':
 531             for name in sorted(self.configs.keys()):
 532                 print(name)
 533             return
 534         if action == 'list-glibcs':
 535             for config in sorted(self.glibc_configs.values(),
 536                                  key=lambda c: c.name):
 537                 print(config.name, config.compiler.name)
 538             return
 539         self.clear_last_build_state(action)
 540         build_time = datetime.datetime.utcnow()
 541         if action == 'host-libraries':
 542             build_components = ('gmp', 'mpfr', 'mpc')
 543             old_components = ()
 544             old_versions = {}
 545             self.build_host_libraries()
 546         elif action == 'compilers':
 547             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 548                                 'gnumach', 'hurd')
 549             old_components = ('gmp', 'mpfr', 'mpc')
 550             old_versions = self.build_state['host-libraries']['build-versions']
 551             self.build_compilers(configs)
 552         else:
 553             build_components = ('glibc',)
 554             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 555                               'mig', 'gnumach', 'hurd')
 556             old_versions = self.build_state['compilers']['build-versions']
 557             if action == 'update-syscalls':
 558                 self.update_syscalls(configs)
 559             else:
 560                 self.build_glibcs(configs)
 561         self.write_files()
 562         self.do_build()
 563         if configs:
 564             # Partial build, do not update stored state.
 565             return
 566         build_versions = {}
 567         for k in build_components:
 568             if k in self.versions:
 569                 build_versions[k] = {'version': self.versions[k]['version'],
 570                                      'revision': self.versions[k]['revision']}
 571         for k in old_components:
 572             if k in old_versions:
 573                 build_versions[k] = {'version': old_versions[k]['version'],
 574                                      'revision': old_versions[k]['revision']}
 575         self.update_build_state(action, build_time, build_versions)
 576
 577     @staticmethod
 578     def remove_dirs(*args):
 579         """Remove directories and their contents if they exist."""
 580         for dir in args:
 581             shutil.rmtree(dir, ignore_errors=True)
 582
 583     @staticmethod
 584     def remove_recreate_dirs(*args):
 585         """Remove directories if they exist, and create them as empty."""
 586         Context.remove_dirs(*args)
 587         for dir in args:
 588             os.makedirs(dir, exist_ok=True)
 589
 590     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 591         """Add makefile text for a list of commands."""
 592         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 593         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 594                                     (target, target, target, commands))
 595         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 596
 597     def write_files(self):
 598         """Write out the Makefile and wrapper script."""
 599         mftext = ''.join(self.makefile_pieces)
 600         with open(self.makefile, 'w') as f:
 601             f.write(mftext)
 602         wrapper_text = (
 603             '#!/bin/sh\n'
 604             'prev_base=$1\n'
 605             'this_base=$2\n'
 606             'desc=$3\n'
 607             'dir=$4\n'
 608             'path=$5\n'
 609             'shift 5\n'
 610             'prev_status=$prev_base-status.txt\n'
 611             'this_status=$this_base-status.txt\n'
 612             'this_log=$this_base-log.txt\n'
 613             'date > "$this_log"\n'
 614             'echo >> "$this_log"\n'
 615             'echo "Description: $desc" >> "$this_log"\n'
 616             'printf "%s" "Command:" >> "$this_log"\n'
 617             'for word in "$@"; do\n'
 618             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 619             '    printf " %s" "$word"\n'
 620             '  else\n'
 621             '    printf " \'"\n'
 622             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 623             '    printf "\'"\n'
 624             '  fi\n'
 625             'done >> "$this_log"\n'
 626             'echo >> "$this_log"\n'
 627             'echo "Directory: $dir" >> "$this_log"\n'
 628             'echo "Path addition: $path" >> "$this_log"\n'
 629             'echo >> "$this_log"\n'
 630             'record_status ()\n'
 631             '{\n'
 632             '  echo >> "$this_log"\n'
 633             '  echo "$1: $desc" > "$this_status"\n'
 634             '  echo "$1: $desc" >> "$this_log"\n'
 635             '  echo >> "$this_log"\n'
 636             '  date >> "$this_log"\n'
 637             '  echo "$1: $desc"\n'
 638             '  exit 0\n'
 639             '}\n'
 640             'check_error ()\n'
 641             '{\n'
 642             '  if [ "$1" != "0" ]; then\n'
 643             '    record_status FAIL\n'
 644             '  fi\n'
 645             '}\n'
 646             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 647             '    record_status UNRESOLVED\n'
 648             'fi\n'
 649             'if [ "$dir" ]; then\n'
 650             '  cd "$dir"\n'
 651             '  check_error "$?"\n'
 652             'fi\n'
 653             'if [ "$path" ]; then\n'
 654             '  PATH=$path:$PATH\n'
 655             'fi\n'
 656             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 657             'check_error "$?"\n'
 658             'record_status PASS\n')
 659         with open(self.wrapper, 'w') as f:
 660             f.write(wrapper_text)
 661         # Mode 0o755.
 662         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 663                      stat.S_IROTH|stat.S_IXOTH)
 664         os.chmod(self.wrapper, mode_exec)
 665         save_logs_text = (
 666             '#!/bin/sh\n'
 667             'if ! [ -f tests.sum ]; then\n'
 668             '  echo "No test summary available."\n'
 669             '  exit 0\n'
 670             'fi\n'
 671             'save_file ()\n'
 672             '{\n'
 673             '  echo "Contents of $1:"\n'
 674             '  echo\n'
 675             '  cat "$1"\n'
 676             '  echo\n'
 677             '  echo "End of contents of $1."\n'
 678             '  echo\n'
 679             '}\n'
 680             'save_file tests.sum\n'
 681             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 682             'for t in $non_pass_tests; do\n'
 683             '  if [ -f "$t.out" ]; then\n'
 684             '    save_file "$t.out"\n'
 685             '  fi\n'
 686             'done\n')
 687         with open(self.save_logs, 'w') as f:
 688             f.write(save_logs_text)
 689         os.chmod(self.save_logs, mode_exec)
 690
 691     def do_build(self):
 692         """Do the actual build."""
 693         cmd = ['make', '-O', '-j%d' % self.parallelism]
 694         subprocess.run(cmd, cwd=self.builddir, check=True)
 695
 696     def build_host_libraries(self):
 697         """Build the host libraries."""
 698         installdir = self.host_libraries_installdir
 699         builddir = os.path.join(self.builddir, 'host-libraries')
 700         logsdir = os.path.join(self.logsdir, 'host-libraries')
 701         self.remove_recreate_dirs(installdir, builddir, logsdir)
 702         cmdlist = CommandList('host-libraries', self.keep)
 703         self.build_host_library(cmdlist, 'gmp')
 704         self.build_host_library(cmdlist, 'mpfr',
 705                                 ['--with-gmp=%s' % installdir])
 706         self.build_host_library(cmdlist, 'mpc',
 707                                 ['--with-gmp=%s' % installdir,
 708                                 '--with-mpfr=%s' % installdir])
 709         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 710         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 711
 712     def build_host_library(self, cmdlist, lib, extra_opts=None):
 713         """Build one host library."""
 714         srcdir = self.component_srcdir(lib)
 715         builddir = self.component_builddir('host-libraries', None, lib)
 716         installdir = self.host_libraries_installdir
 717         cmdlist.push_subdesc(lib)
 718         cmdlist.create_use_dir(builddir)
 719         cfg_cmd = [os.path.join(srcdir, 'configure'),
 720                    '--prefix=%s' % installdir,
 721                    '--disable-shared']
 722         if extra_opts:
 723             cfg_cmd.extend (extra_opts)
 724         cmdlist.add_command('configure', cfg_cmd)
 725         cmdlist.add_command('build', ['make'])
 726         cmdlist.add_command('check', ['make', 'check'])
 727         cmdlist.add_command('install', ['make', 'install'])
 728         cmdlist.cleanup_dir()
 729         cmdlist.pop_subdesc()
 730
 731     def build_compilers(self, configs):
 732         """Build the compilers."""
 733         if not configs:
 734             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 735             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 736             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 737             configs = sorted(self.configs.keys())
 738         for c in configs:
 739             self.configs[c].build()
 740
 741     def build_glibcs(self, configs):
 742         """Build the glibcs."""
 743         if not configs:
 744             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 745             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 746             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 747             configs = sorted(self.glibc_configs.keys())
 748         for c in configs:
 749             self.glibc_configs[c].build()
 750
 751     def update_syscalls(self, configs):
 752         """Update the glibc syscall lists."""
 753         if not configs:
 754             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 755             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 756             configs = sorted(self.glibc_configs.keys())
 757         for c in configs:
 758             self.glibc_configs[c].update_syscalls()
 759
 760     def load_versions_json(self):
 761         """Load information about source directory versions."""
 762         if not os.access(self.versions_json, os.F_OK):
 763             self.versions = {}
 764             return
 765         with open(self.versions_json, 'r') as f:
 766             self.versions = json.load(f)
 767
 768     def store_json(self, data, filename):
 769         """Store information in a JSON file."""
 770         filename_tmp = filename + '.tmp'
 771         with open(filename_tmp, 'w') as f:
 772             json.dump(data, f, indent=2, sort_keys=True)
 773         os.rename(filename_tmp, filename)
 774
 775     def store_versions_json(self):
 776         """Store information about source directory versions."""
 777         self.store_json(self.versions, self.versions_json)
 778
 779     def set_component_version(self, component, version, explicit, revision):
 780         """Set the version information for a component."""
 781         self.versions[component] = {'version': version,
 782                                     'explicit': explicit,
 783                                     'revision': revision}
 784         self.store_versions_json()
 785
 786     def checkout(self, versions):
 787         """Check out the desired component versions."""
 788         default_versions = {'binutils': 'vcs-2.38',
 789                             'gcc': 'vcs-11',
 790                             'glibc': 'vcs-mainline',
 791                             'gmp': '6.2.1',
 792                             'linux': '5.18',
 793                             'mpc': '1.2.1',
 794                             'mpfr': '4.1.0',
 795                             'mig': 'vcs-mainline',
 796                             'gnumach': 'vcs-mainline',
 797                             'hurd': 'vcs-mainline'}
 798         use_versions = {}
 799         explicit_versions = {}
 800         for v in versions:
 801             found_v = False
 802             for k in default_versions.keys():
 803                 kx = k + '-'
 804                 if v.startswith(kx):
 805                     vx = v[len(kx):]
 806                     if k in use_versions:
 807                         print('error: multiple versions for %s' % k)
 808                         exit(1)
 809                     use_versions[k] = vx
 810                     explicit_versions[k] = True
 811                     found_v = True
 812                     break
 813             if not found_v:
 814                 print('error: unknown component in %s' % v)
 815                 exit(1)
 816         for k in default_versions.keys():
 817             if k not in use_versions:
 818                 if k in self.versions and self.versions[k]['explicit']:
 819                     use_versions[k] = self.versions[k]['version']
 820                     explicit_versions[k] = True
 821                 else:
 822                     use_versions[k] = default_versions[k]
 823                     explicit_versions[k] = False
 824         os.makedirs(self.srcdir, exist_ok=True)
 825         for k in sorted(default_versions.keys()):
 826             update = os.access(self.component_srcdir(k), os.F_OK)
 827             v = use_versions[k]
 828             if (update and
 829                 k in self.versions and
 830                 v != self.versions[k]['version']):
 831                 if not self.replace_sources:
 832                     print('error: version of %s has changed from %s to %s, '
 833                           'use --replace-sources to check out again' %
 834                           (k, self.versions[k]['version'], v))
 835                     exit(1)
 836                 shutil.rmtree(self.component_srcdir(k))
 837                 update = False
 838             if v.startswith('vcs-'):
 839                 revision = self.checkout_vcs(k, v[4:], update)
 840             else:
 841                 self.checkout_tar(k, v, update)
 842                 revision = v
 843             self.set_component_version(k, v, explicit_versions[k], revision)
 844         if self.get_script_text() != self.script_text:
 845             # Rerun the checkout process in case the updated script
 846             # uses different default versions or new components.
 847             self.exec_self()
 848
 849     def checkout_vcs(self, component, version, update):
 850         """Check out the given version of the given component from version
 851         control.  Return a revision identifier."""
 852         if component == 'binutils':
 853             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 854             if version == 'mainline':
 855                 git_branch = 'master'
 856             else:
 857                 trans = str.maketrans({'.': '_'})
 858                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 859             return self.git_checkout(component, git_url, git_branch, update)
 860         elif component == 'gcc':
 861             if version == 'mainline':
 862                 branch = 'master'
 863             else:
 864                 branch = 'releases/gcc-%s' % version
 865             return self.gcc_checkout(branch, update)
 866         elif component == 'glibc':
 867             git_url = 'git://sourceware.org/git/glibc.git'
 868             if version == 'mainline':
 869                 git_branch = 'master'
 870             else:
 871                 git_branch = 'release/%s/master' % version
 872             r = self.git_checkout(component, git_url, git_branch, update)
 873             self.fix_glibc_timestamps()
 874             return r
 875         elif component == 'gnumach':
 876             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 877             git_branch = 'master'
 878             r = self.git_checkout(component, git_url, git_branch, update)
 879             subprocess.run(['autoreconf', '-i'],
 880                            cwd=self.component_srcdir(component), check=True)
 881             return r
 882         elif component == 'mig':
 883             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 884             git_branch = 'master'
 885             r = self.git_checkout(component, git_url, git_branch, update)
 886             subprocess.run(['autoreconf', '-i'],
 887                            cwd=self.component_srcdir(component), check=True)
 888             return r
 889         elif component == 'hurd':
 890             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 891             git_branch = 'master'
 892             r = self.git_checkout(component, git_url, git_branch, update)
 893             subprocess.run(['autoconf'],
 894                            cwd=self.component_srcdir(component), check=True)
 895             return r
 896         else:
 897             print('error: component %s coming from VCS' % component)
 898             exit(1)
 899
 900     def git_checkout(self, component, git_url, git_branch, update):
 901         """Check out a component from git.  Return a commit identifier."""
 902         if update:
 903             subprocess.run(['git', 'remote', 'prune', 'origin'],
 904                            cwd=self.component_srcdir(component), check=True)
 905             if self.replace_sources:
 906                 subprocess.run(['git', 'clean', '-dxfq'],
 907                                cwd=self.component_srcdir(component), check=True)
 908             subprocess.run(['git', 'pull', '-q'],
 909                            cwd=self.component_srcdir(component), check=True)
 910         else:
 911             if self.shallow:
 912                 depth_arg = ('--depth', '1')
 913             else:
 914                 depth_arg = ()
 915             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 916                             *depth_arg, git_url,
 917                             self.component_srcdir(component)], check=True)
 918         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 919                            cwd=self.component_srcdir(component),
 920                            stdout=subprocess.PIPE,
 921                            check=True, universal_newlines=True).stdout
 922         return r.rstrip()
 923
 924     def fix_glibc_timestamps(self):
 925         """Fix timestamps in a glibc checkout."""
 926         # Ensure that builds do not try to regenerate generated files
 927         # in the source tree.
 928         srcdir = self.component_srcdir('glibc')
 929         # These files have Makefile dependencies to regenerate them in
 930         # the source tree that may be active during a normal build.
 931         # Some other files have such dependencies but do not need to
 932         # be touched because nothing in a build depends on the files
 933         # in question.
 934         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 935             to_touch = os.path.join(srcdir, f)
 936             subprocess.run(['touch', '-c', to_touch], check=True)
 937         for dirpath, dirnames, filenames in os.walk(srcdir):
 938             for f in filenames:
 939                 if (f == 'configure' or
 940                     f == 'preconfigure' or
 941                     f.endswith('-kw.h')):
 942                     to_touch = os.path.join(dirpath, f)
 943                     subprocess.run(['touch', to_touch], check=True)
 944
 945     def gcc_checkout(self, branch, update):
 946         """Check out GCC from git.  Return the commit identifier."""
 947         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 948                      os.F_OK):
 949             if not self.replace_sources:
 950                 print('error: GCC has moved from SVN to git, use '
 951                       '--replace-sources to check out again')
 952                 exit(1)
 953             shutil.rmtree(self.component_srcdir('gcc'))
 954             update = False
 955         if not update:
 956             self.git_checkout('gcc', 'git://gcc.gnu.org/git/gcc.git',
 957                               branch, update)
 958         subprocess.run(['contrib/gcc_update', '--silent'],
 959                        cwd=self.component_srcdir('gcc'), check=True)
 960         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 961                            cwd=self.component_srcdir('gcc'),
 962                            stdout=subprocess.PIPE,
 963                            check=True, universal_newlines=True).stdout
 964         return r.rstrip()
 965
 966     def checkout_tar(self, component, version, update):
 967         """Check out the given version of the given component from a
 968         tarball."""
 969         if update:
 970             return
 971         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 972                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
 973                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 974                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
 975                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 976                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 977                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 978                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 979                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 980         if component not in url_map:
 981             print('error: component %s coming from tarball' % component)
 982             exit(1)
 983         version_major = version.split('.')[0]
 984         url = url_map[component] % {'version': version, 'major': version_major}
 985         filename = os.path.join(self.srcdir, url.split('/')[-1])
 986         response = urllib.request.urlopen(url)
 987         data = response.read()
 988         with open(filename, 'wb') as f:
 989             f.write(data)
 990         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 991                        check=True)
 992         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 993                   self.component_srcdir(component))
 994         os.remove(filename)
 995
 996     def load_build_state_json(self):
 997         """Load information about the state of previous builds."""
 998         if os.access(self.build_state_json, os.F_OK):
 999             with open(self.build_state_json, 'r') as f:
1000                 self.build_state = json.load(f)
1001         else:
1002             self.build_state = {}
1003         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
1004             if k not in self.build_state:
1005                 self.build_state[k] = {}
1006             if 'build-time' not in self.build_state[k]:
1007                 self.build_state[k]['build-time'] = ''
1008             if 'build-versions' not in self.build_state[k]:
1009                 self.build_state[k]['build-versions'] = {}
1010             if 'build-results' not in self.build_state[k]:
1011                 self.build_state[k]['build-results'] = {}
1012             if 'result-changes' not in self.build_state[k]:
1013                 self.build_state[k]['result-changes'] = {}
1014             if 'ever-passed' not in self.build_state[k]:
1015                 self.build_state[k]['ever-passed'] = []
1016
1017     def store_build_state_json(self):
1018         """Store information about the state of previous builds."""
1019         self.store_json(self.build_state, self.build_state_json)
1020
1021     def clear_last_build_state(self, action):
1022         """Clear information about the state of part of the build."""
1023         # We clear the last build time and versions when starting a
1024         # new build.  The results of the last build are kept around,
1025         # as comparison is still meaningful if this build is aborted
1026         # and a new one started.
1027         self.build_state[action]['build-time'] = ''
1028         self.build_state[action]['build-versions'] = {}
1029         self.store_build_state_json()
1030
1031     def update_build_state(self, action, build_time, build_versions):
1032         """Update the build state after a build."""
1033         build_time = build_time.replace(microsecond=0)
1034         self.build_state[action]['build-time'] = str(build_time)
1035         self.build_state[action]['build-versions'] = build_versions
1036         build_results = {}
1037         for log in self.status_log_list:
1038             with open(log, 'r') as f:
1039                 log_text = f.read()
1040             log_text = log_text.rstrip()
1041             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1042             result = m.group(1)
1043             test_name = m.group(2)
1044             assert test_name not in build_results
1045             build_results[test_name] = result
1046         old_build_results = self.build_state[action]['build-results']
1047         self.build_state[action]['build-results'] = build_results
1048         result_changes = {}
1049         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1050         for t in all_tests:
1051             if t in old_build_results:
1052                 old_res = old_build_results[t]
1053             else:
1054                 old_res = '(New test)'
1055             if t in build_results:
1056                 new_res = build_results[t]
1057             else:
1058                 new_res = '(Test removed)'
1059             if old_res != new_res:
1060                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1061         self.build_state[action]['result-changes'] = result_changes
1062         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1063                            if t in build_results}
1064         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1065         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1066                                                          new_passes)
1067         self.store_build_state_json()
1068
1069     def load_bot_config_json(self):
1070         """Load bot configuration."""
1071         with open(self.bot_config_json, 'r') as f:
1072             self.bot_config = json.load(f)
1073
1074     def part_build_old(self, action, delay):
1075         """Return whether the last build for a given action was at least a
1076         given number of seconds ago, or does not have a time recorded."""
1077         old_time_str = self.build_state[action]['build-time']
1078         if not old_time_str:
1079             return True
1080         old_time = datetime.datetime.strptime(old_time_str,
1081                                               '%Y-%m-%d %H:%M:%S')
1082         new_time = datetime.datetime.utcnow()
1083         delta = new_time - old_time
1084         return delta.total_seconds() >= delay
1085
1086     def bot_cycle(self):
1087         """Run a single round of checkout and builds."""
1088         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1089         self.load_bot_config_json()
1090         actions = ('host-libraries', 'compilers', 'glibcs')
1091         self.bot_run_self(['--replace-sources'], 'checkout')
1092         self.load_versions_json()
1093         if self.get_script_text() != self.script_text:
1094             print('Script changed, re-execing.')
1095             # On script change, all parts of the build should be rerun.
1096             for a in actions:
1097                 self.clear_last_build_state(a)
1098             self.exec_self()
1099         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1100                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1101                                           'mig', 'gnumach', 'hurd'),
1102                             'glibcs': ('glibc',)}
1103         must_build = {}
1104         for a in actions:
1105             build_vers = self.build_state[a]['build-versions']
1106             must_build[a] = False
1107             if not self.build_state[a]['build-time']:
1108                 must_build[a] = True
1109             old_vers = {}
1110             new_vers = {}
1111             for c in check_components[a]:
1112                 if c in build_vers:
1113                     old_vers[c] = build_vers[c]
1114                 new_vers[c] = {'version': self.versions[c]['version'],
1115                                'revision': self.versions[c]['revision']}
1116             if new_vers == old_vers:
1117                 print('Versions for %s unchanged.' % a)
1118             else:
1119                 print('Versions changed or rebuild forced for %s.' % a)
1120                 if a == 'compilers' and not self.part_build_old(
1121                         a, self.bot_config['compilers-rebuild-delay']):
1122                     print('Not requiring rebuild of compilers this soon.')
1123                 else:
1124                     must_build[a] = True
1125         if must_build['host-libraries']:
1126             must_build['compilers'] = True
1127         if must_build['compilers']:
1128             must_build['glibcs'] = True
1129         for a in actions:
1130             if must_build[a]:
1131                 print('Must rebuild %s.' % a)
1132                 self.clear_last_build_state(a)
1133             else:
1134                 print('No need to rebuild %s.' % a)
1135         if os.access(self.logsdir, os.F_OK):
1136             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1137             shutil.copytree(self.logsdir, self.logsdir_old)
1138         for a in actions:
1139             if must_build[a]:
1140                 build_time = datetime.datetime.utcnow()
1141                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1142                 self.bot_run_self([], a)
1143                 self.load_build_state_json()
1144                 self.bot_build_mail(a, build_time)
1145         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1146
1147     def bot_build_mail(self, action, build_time):
1148         """Send email with the results of a build."""
1149         if not ('email-from' in self.bot_config and
1150                 'email-server' in self.bot_config and
1151                 'email-subject' in self.bot_config and
1152                 'email-to' in self.bot_config):
1153             if not self.email_warning:
1154                 print("Email not configured, not sending.")
1155                 self.email_warning = True
1156             return
1157
1158         build_time = build_time.replace(microsecond=0)
1159         subject = (self.bot_config['email-subject'] %
1160                    {'action': action,
1161                     'build-time': str(build_time)})
1162         results = self.build_state[action]['build-results']
1163         changes = self.build_state[action]['result-changes']
1164         ever_passed = set(self.build_state[action]['ever-passed'])
1165         versions = self.build_state[action]['build-versions']
1166         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1167         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1168         all_fails = {k for k in results if results[k] == 'FAIL'}
1169         if new_regressions:
1170             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1171             new_reg_text = ('New regressions:\n\n%s\n\n' %
1172                             '\n'.join(new_reg_list))
1173         else:
1174             new_reg_text = ''
1175         if all_regressions:
1176             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1177             all_reg_text = ('All regressions:\n\n%s\n\n' %
1178                             '\n'.join(all_reg_list))
1179         else:
1180             all_reg_text = ''
1181         if all_fails:
1182             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1183             all_fail_text = ('All failures:\n\n%s\n\n' %
1184                              '\n'.join(all_fail_list))
1185         else:
1186             all_fail_text = ''
1187         if changes:
1188             changes_list = sorted(changes.keys())
1189             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1190             changes_text = ('All changed results:\n\n%s\n\n' %
1191                             '\n'.join(changes_list))
1192         else:
1193             changes_text = ''
1194         results_text = (new_reg_text + all_reg_text + all_fail_text +
1195                         changes_text)
1196         if not results_text:
1197             results_text = 'Clean build with unchanged results.\n\n'
1198         versions_list = sorted(versions.keys())
1199         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1200                                           versions[k]['revision'])
1201                          for k in versions_list]
1202         versions_text = ('Component versions for this build:\n\n%s\n' %
1203                          '\n'.join(versions_list))
1204         body_text = results_text + versions_text
1205         msg = email.mime.text.MIMEText(body_text)
1206         msg['Subject'] = subject
1207         msg['From'] = self.bot_config['email-from']
1208         msg['To'] = self.bot_config['email-to']
1209         msg['Message-ID'] = email.utils.make_msgid()
1210         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1211         with smtplib.SMTP(self.bot_config['email-server']) as s:
1212             s.send_message(msg)
1213
1214     def bot_run_self(self, opts, action, check=True):
1215         """Run a copy of this script with given options."""
1216         cmd = [sys.executable, sys.argv[0], '--keep=none',
1217                '-j%d' % self.parallelism]
1218         if self.full_gcc:
1219             cmd.append('--full-gcc')
1220         cmd.extend(opts)
1221         cmd.extend([self.topdir, action])
1222         sys.stdout.flush()
1223         subprocess.run(cmd, check=check)
1224
1225     def bot(self):
1226         """Run repeated rounds of checkout and builds."""
1227         while True:
1228             self.load_bot_config_json()
1229             if not self.bot_config['run']:
1230                 print('Bot exiting by request.')
1231                 exit(0)
1232             self.bot_run_self([], 'bot-cycle', check=False)
1233             self.load_bot_config_json()
1234             if not self.bot_config['run']:
1235                 print('Bot exiting by request.')
1236                 exit(0)
1237             time.sleep(self.bot_config['delay'])
1238             if self.get_script_text() != self.script_text:
1239                 print('Script changed, bot re-execing.')
1240                 self.exec_self()
1241
1242 class LinuxHeadersPolicyForBuild(object):
1243     """Names and directories for installing Linux headers.  Build variant."""
1244
1245     def __init__(self, config):
1246         self.arch = config.arch
1247         self.srcdir = config.ctx.component_srcdir('linux')
1248         self.builddir = config.component_builddir('linux')
1249         self.headers_dir = os.path.join(config.sysroot, 'usr')
1250
1251 class LinuxHeadersPolicyForUpdateSyscalls(object):
1252     """Names and directories for Linux headers.  update-syscalls variant."""
1253
1254     def __init__(self, glibc, headers_dir):
1255         self.arch = glibc.compiler.arch
1256         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1257         self.builddir = glibc.ctx.component_builddir(
1258             'update-syscalls', glibc.name, 'build-linux')
1259         self.headers_dir = headers_dir
1260
1261 def install_linux_headers(policy, cmdlist):
1262     """Install Linux kernel headers."""
1263     arch_map = {'aarch64': 'arm64',
1264                 'alpha': 'alpha',
1265                 'arc': 'arc',
1266                 'arm': 'arm',
1267                 'csky': 'csky',
1268                 'hppa': 'parisc',
1269                 'i486': 'x86',
1270                 'i586': 'x86',
1271                 'i686': 'x86',
1272                 'i786': 'x86',
1273                 'ia64': 'ia64',
1274                 'm68k': 'm68k',
1275                 'microblaze': 'microblaze',
1276                 'mips': 'mips',
1277                 'nios2': 'nios2',
1278                 'or1k': 'openrisc',
1279                 'powerpc': 'powerpc',
1280                 's390': 's390',
1281                 'riscv32': 'riscv',
1282                 'riscv64': 'riscv',
1283                 'sh': 'sh',
1284                 'sparc': 'sparc',
1285                 'x86_64': 'x86'}
1286     linux_arch = None
1287     for k in arch_map:
1288         if policy.arch.startswith(k):
1289             linux_arch = arch_map[k]
1290             break
1291     assert linux_arch is not None
1292     cmdlist.push_subdesc('linux')
1293     cmdlist.create_use_dir(policy.builddir)
1294     cmdlist.add_command('install-headers',
1295                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1296                          'ARCH=%s' % linux_arch,
1297                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1298                          'headers_install'])
1299     cmdlist.cleanup_dir()
1300     cmdlist.pop_subdesc()
1301
1302 class Config(object):
1303     """A configuration for building a compiler and associated libraries."""
1304
1305     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1306                  first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1307                  extra_glibcs=None):
1308         """Initialize a Config object."""
1309         self.ctx = ctx
1310         self.arch = arch
1311         self.os = os_name
1312         self.variant = variant
1313         if variant is None:
1314             self.name = '%s-%s' % (arch, os_name)
1315         else:
1316             self.name = '%s-%s-%s' % (arch, os_name, variant)
1317         self.triplet = '%s-glibc-%s' % (arch, os_name)
1318         if gcc_cfg is None:
1319             self.gcc_cfg = []
1320         else:
1321             self.gcc_cfg = gcc_cfg
1322         if first_gcc_cfg is None:
1323             self.first_gcc_cfg = []
1324         else:
1325             self.first_gcc_cfg = first_gcc_cfg
1326         if binutils_cfg is None:
1327             self.binutils_cfg = []
1328         else:
1329             self.binutils_cfg = binutils_cfg
1330         if glibcs is None:
1331             glibcs = [{'variant': variant}]
1332         if extra_glibcs is None:
1333             extra_glibcs = []
1334         glibcs = [Glibc(self, **g) for g in glibcs]
1335         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1336         self.all_glibcs = glibcs + extra_glibcs
1337         self.compiler_glibcs = glibcs
1338         self.installdir = ctx.compiler_installdir(self.name)
1339         self.bindir = ctx.compiler_bindir(self.name)
1340         self.sysroot = ctx.compiler_sysroot(self.name)
1341         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1342         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1343
1344     def component_builddir(self, component):
1345         """Return the directory to use for a (non-glibc) build."""
1346         return self.ctx.component_builddir('compilers', self.name, component)
1347
1348     def build(self):
1349         """Generate commands to build this compiler."""
1350         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1351                                       self.logsdir)
1352         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1353         cmdlist.add_command('check-host-libraries',
1354                             ['test', '-f',
1355                              os.path.join(self.ctx.host_libraries_installdir,
1356                                           'ok')])
1357         cmdlist.use_path(self.bindir)
1358         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1359                               ['--disable-gdb',
1360                                '--disable-gdbserver',
1361                                '--disable-libdecnumber',
1362                                '--disable-readline',
1363                                '--disable-sim'] + self.binutils_cfg)
1364         if self.os.startswith('linux'):
1365             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1366         self.build_gcc(cmdlist, True)
1367         if self.os == 'gnu':
1368             self.install_gnumach_headers(cmdlist)
1369             self.build_cross_tool(cmdlist, 'mig', 'mig')
1370             self.install_hurd_headers(cmdlist)
1371         for g in self.compiler_glibcs:
1372             cmdlist.push_subdesc('glibc')
1373             cmdlist.push_subdesc(g.name)
1374             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1375             cmdlist.pop_subdesc()
1376             cmdlist.pop_subdesc()
1377         self.build_gcc(cmdlist, False)
1378         cmdlist.add_command('done', ['touch',
1379                                      os.path.join(self.installdir, 'ok')])
1380         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1381                                       self.logsdir)
1382
1383     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1384         """Build one cross tool."""
1385         srcdir = self.ctx.component_srcdir(tool_src)
1386         builddir = self.component_builddir(tool_build)
1387         cmdlist.push_subdesc(tool_build)
1388         cmdlist.create_use_dir(builddir)
1389         cfg_cmd = [os.path.join(srcdir, 'configure'),
1390                    '--prefix=%s' % self.installdir,
1391                    '--build=%s' % self.ctx.build_triplet,
1392                    '--host=%s' % self.ctx.build_triplet,
1393                    '--target=%s' % self.triplet,
1394                    '--with-sysroot=%s' % self.sysroot]
1395         if extra_opts:
1396             cfg_cmd.extend(extra_opts)
1397         cmdlist.add_command('configure', cfg_cmd)
1398         cmdlist.add_command('build', ['make'])
1399         # Parallel "make install" for GCC has race conditions that can
1400         # cause it to fail; see
1401         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1402         # problems are not known for binutils, but doing the
1403         # installation in parallel within a particular toolchain build
1404         # (as opposed to installation of one toolchain from
1405         # build-many-glibcs.py running in parallel to the installation
1406         # of other toolchains being built) is not known to be
1407         # significantly beneficial, so it is simplest just to disable
1408         # parallel install for cross tools here.
1409         cmdlist.add_command('install', ['make', '-j1', 'install'])
1410         cmdlist.cleanup_dir()
1411         cmdlist.pop_subdesc()
1412
1413     def install_gnumach_headers(self, cmdlist):
1414         """Install GNU Mach headers."""
1415         srcdir = self.ctx.component_srcdir('gnumach')
1416         builddir = self.component_builddir('gnumach')
1417         cmdlist.push_subdesc('gnumach')
1418         cmdlist.create_use_dir(builddir)
1419         cmdlist.add_command('configure',
1420                             [os.path.join(srcdir, 'configure'),
1421                              '--build=%s' % self.ctx.build_triplet,
1422                              '--host=%s' % self.triplet,
1423                              '--prefix=',
1424                              'CC=%s-gcc -nostdlib' % self.triplet])
1425         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1426                                         'install-data'])
1427         cmdlist.cleanup_dir()
1428         cmdlist.pop_subdesc()
1429
1430     def install_hurd_headers(self, cmdlist):
1431         """Install Hurd headers."""
1432         srcdir = self.ctx.component_srcdir('hurd')
1433         builddir = self.component_builddir('hurd')
1434         cmdlist.push_subdesc('hurd')
1435         cmdlist.create_use_dir(builddir)
1436         cmdlist.add_command('configure',
1437                             [os.path.join(srcdir, 'configure'),
1438                              '--build=%s' % self.ctx.build_triplet,
1439                              '--host=%s' % self.triplet,
1440                              '--prefix=',
1441                              '--disable-profile', '--without-parted',
1442                              'CC=%s-gcc -nostdlib' % self.triplet])
1443         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1444                                         'no_deps=t', 'install-headers'])
1445         cmdlist.cleanup_dir()
1446         cmdlist.pop_subdesc()
1447
1448     def build_gcc(self, cmdlist, bootstrap):
1449         """Build GCC."""
1450         # libssp is of little relevance with glibc's own stack
1451         # checking support.  libcilkrts does not support GNU/Hurd (and
1452         # has been removed in GCC 8, so --disable-libcilkrts can be
1453         # removed once glibc no longer supports building with older
1454         # GCC versions).  --enable-initfini-array is enabled by default
1455         # in GCC 12, which can be removed when GCC 12 becomes the
1456         # minimum requirement.
1457         cfg_opts = list(self.gcc_cfg)
1458         cfg_opts += ['--enable-initfini-array']
1459         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1460         host_libs = self.ctx.host_libraries_installdir
1461         cfg_opts += ['--with-gmp=%s' % host_libs,
1462                      '--with-mpfr=%s' % host_libs,
1463                      '--with-mpc=%s' % host_libs]
1464         if bootstrap:
1465             tool_build = 'gcc-first'
1466             # Building a static-only, C-only compiler that is
1467             # sufficient to build glibc.  Various libraries and
1468             # features that may require libc headers must be disabled.
1469             # When configuring with a sysroot, --with-newlib is
1470             # required to define inhibit_libc (to stop some parts of
1471             # libgcc including libc headers); --without-headers is not
1472             # sufficient.
1473             cfg_opts += ['--enable-languages=c', '--disable-shared',
1474                          '--disable-threads',
1475                          '--disable-libatomic',
1476                          '--disable-decimal-float',
1477                          '--disable-libffi',
1478                          '--disable-libgomp',
1479                          '--disable-libitm',
1480                          '--disable-libmpx',
1481                          '--disable-libquadmath',
1482                          '--disable-libsanitizer',
1483                          '--without-headers', '--with-newlib',
1484                          '--with-glibc-version=%s' % self.ctx.glibc_version
1485                          ]
1486             cfg_opts += self.first_gcc_cfg
1487         else:
1488             tool_build = 'gcc'
1489             # libsanitizer commonly breaks because of glibc header
1490             # changes, or on unusual targets.  C++ pre-compiled
1491             # headers are not used during the glibc build and are
1492             # expensive to create.
1493             if not self.ctx.full_gcc:
1494                 cfg_opts += ['--disable-libsanitizer',
1495                              '--disable-libstdcxx-pch']
1496             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1497             cfg_opts += ['--enable-languages=%s' % langs,
1498                          '--enable-shared', '--enable-threads']
1499         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1500
1501 class GlibcPolicyDefault(object):
1502     """Build policy for glibc: common defaults."""
1503
1504     def __init__(self, glibc):
1505         self.srcdir = glibc.ctx.component_srcdir('glibc')
1506         self.use_usr = glibc.os != 'gnu'
1507         self.prefix = '/usr' if self.use_usr else ''
1508         self.configure_args = [
1509             '--prefix=%s' % self.prefix,
1510             '--enable-profile',
1511             '--build=%s' % glibc.ctx.build_triplet,
1512             '--host=%s' % glibc.triplet,
1513             'CC=%s' % glibc.tool_name('gcc'),
1514             'CXX=%s' % glibc.tool_name('g++'),
1515             'AR=%s' % glibc.tool_name('ar'),
1516             'AS=%s' % glibc.tool_name('as'),
1517             'LD=%s' % glibc.tool_name('ld'),
1518             'NM=%s' % glibc.tool_name('nm'),
1519             'OBJCOPY=%s' % glibc.tool_name('objcopy'),
1520             'OBJDUMP=%s' % glibc.tool_name('objdump'),
1521             'RANLIB=%s' % glibc.tool_name('ranlib'),
1522             'READELF=%s' % glibc.tool_name('readelf'),
1523             'STRIP=%s' % glibc.tool_name('strip'),
1524         ]
1525         if glibc.os == 'gnu':
1526             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1527         if glibc.cflags:
1528             self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1529             self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
1530         self.configure_args += glibc.cfg
1531
1532     def configure(self, cmdlist):
1533         """Invoked to add the configure command to the command list."""
1534         cmdlist.add_command('configure',
1535                             [os.path.join(self.srcdir, 'configure'),
1536                              *self.configure_args])
1537
1538     def extra_commands(self, cmdlist):
1539         """Invoked to inject additional commands (make check) after build."""
1540         pass
1541
1542 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1543     """Build policy for glibc during the compilers stage."""
1544
1545     def __init__(self, glibc):
1546         super().__init__(glibc)
1547         self.builddir = glibc.ctx.component_builddir(
1548             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1549         self.installdir = glibc.compiler.sysroot
1550
1551 class GlibcPolicyForBuild(GlibcPolicyDefault):
1552     """Build policy for glibc during the glibcs stage."""
1553
1554     def __init__(self, glibc):
1555         super().__init__(glibc)
1556         self.builddir = glibc.ctx.component_builddir(
1557             'glibcs', glibc.name, 'glibc')
1558         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1559         if glibc.ctx.strip:
1560             self.strip = glibc.tool_name('strip')
1561         else:
1562             self.strip = None
1563         self.save_logs = glibc.ctx.save_logs
1564
1565     def extra_commands(self, cmdlist):
1566         if self.strip:
1567             # Avoid stripping libc.so and libpthread.so, which are
1568             # linker scripts stored in /lib on Hurd.
1569             find_command = 'find %s/lib* -name "*.so*"' % self.installdir
1570             cmdlist.add_command('strip', ['sh', '-c', (
1571                 'set -e; for f in $(%s); do '
1572                 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1573                 'done' % (find_command, self.strip))])
1574         cmdlist.add_command('check', ['make', 'check'])
1575         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1576
1577 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1578     """Build policy for glibc during update-syscalls."""
1579
1580     def __init__(self, glibc):
1581         super().__init__(glibc)
1582         self.builddir = glibc.ctx.component_builddir(
1583             'update-syscalls', glibc.name, 'glibc')
1584         self.linuxdir = glibc.ctx.component_builddir(
1585             'update-syscalls', glibc.name, 'linux')
1586         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1587             glibc, self.linuxdir)
1588         self.configure_args.insert(
1589             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1590         # self.installdir not set because installation is not supported
1591
1592 class Glibc(object):
1593     """A configuration for building glibc."""
1594
1595     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1596                  cfg=None, ccopts=None, cflags=None):
1597         """Initialize a Glibc object."""
1598         self.ctx = compiler.ctx
1599         self.compiler = compiler
1600         if arch is None:
1601             self.arch = compiler.arch
1602         else:
1603             self.arch = arch
1604         if os_name is None:
1605             self.os = compiler.os
1606         else:
1607             self.os = os_name
1608         self.variant = variant
1609         if variant is None:
1610             self.name = '%s-%s' % (self.arch, self.os)
1611         else:
1612             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1613         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1614         if cfg is None:
1615             self.cfg = []
1616         else:
1617             self.cfg = cfg
1618         # ccopts contain ABI options and are passed to configure as CC / CXX.
1619         self.ccopts = ccopts
1620         # cflags contain non-ABI options like -g or -O and are passed to
1621         # configure as CFLAGS / CXXFLAGS.
1622         self.cflags = cflags
1623
1624     def tool_name(self, tool):
1625         """Return the name of a cross-compilation tool."""
1626         ctool = '%s-%s' % (self.compiler.triplet, tool)
1627         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1628             ctool = '%s %s' % (ctool, self.ccopts)
1629         return ctool
1630
1631     def build(self):
1632         """Generate commands to build this glibc."""
1633         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1634         installdir = self.ctx.glibc_installdir(self.name)
1635         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1636         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1637         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1638         cmdlist.add_command('check-compilers',
1639                             ['test', '-f',
1640                              os.path.join(self.compiler.installdir, 'ok')])
1641         cmdlist.use_path(self.compiler.bindir)
1642         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1643         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1644                                       logsdir)
1645
1646     def build_glibc(self, cmdlist, policy):
1647         """Generate commands to build this glibc, either as part of a compiler
1648         build or with the bootstrapped compiler (and in the latter case, run
1649         tests as well)."""
1650         cmdlist.create_use_dir(policy.builddir)
1651         policy.configure(cmdlist)
1652         cmdlist.add_command('build', ['make'])
1653         cmdlist.add_command('install', ['make', 'install',
1654                                         'install_root=%s' % policy.installdir])
1655         # GCC uses paths such as lib/../lib64, so make sure lib
1656         # directories always exist.
1657         mkdir_cmd = ['mkdir', '-p',
1658                      os.path.join(policy.installdir, 'lib')]
1659         if policy.use_usr:
1660             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1661         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1662         policy.extra_commands(cmdlist)
1663         cmdlist.cleanup_dir()
1664
1665     def update_syscalls(self):
1666         if self.os == 'gnu':
1667             # Hurd does not have system call tables that need updating.
1668             return
1669
1670         policy = GlibcPolicyForUpdateSyscalls(self)
1671         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1672         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1673         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1674         cmdlist.add_command('check-compilers',
1675                             ['test', '-f',
1676                              os.path.join(self.compiler.installdir, 'ok')])
1677         cmdlist.use_path(self.compiler.bindir)
1678
1679         install_linux_headers(policy.linux_policy, cmdlist)
1680
1681         cmdlist.create_use_dir(policy.builddir)
1682         policy.configure(cmdlist)
1683         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1684         cmdlist.cleanup_dir()
1685         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1686                                       cmdlist, logsdir)
1687
1688 class Command(object):
1689     """A command run in the build process."""
1690
1691     def __init__(self, desc, num, dir, path, command, always_run=False):
1692         """Initialize a Command object."""
1693         self.dir = dir
1694         self.path = path
1695         self.desc = desc
1696         trans = str.maketrans({' ': '-'})
1697         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1698         self.command = command
1699         self.always_run = always_run
1700
1701     @staticmethod
1702     def shell_make_quote_string(s):
1703         """Given a string not containing a newline, quote it for use by the
1704         shell and make."""
1705         assert '\n' not in s
1706         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1707             return s
1708         strans = str.maketrans({"'": "'\\''"})
1709         s = "'%s'" % s.translate(strans)
1710         mtrans = str.maketrans({'$': '$$'})
1711         return s.translate(mtrans)
1712
1713     @staticmethod
1714     def shell_make_quote_list(l, translate_make):
1715         """Given a list of strings not containing newlines, quote them for use
1716         by the shell and make, returning a single string.  If translate_make
1717         is true and the first string is 'make', change it to $(MAKE)."""
1718         l = [Command.shell_make_quote_string(s) for s in l]
1719         if translate_make and l[0] == 'make':
1720             l[0] = '$(MAKE)'
1721         return ' '.join(l)
1722
1723     def shell_make_quote(self):
1724         """Return this command quoted for the shell and make."""
1725         return self.shell_make_quote_list(self.command, True)
1726
1727
1728 class CommandList(object):
1729     """A list of commands run in the build process."""
1730
1731     def __init__(self, desc, keep):
1732         """Initialize a CommandList object."""
1733         self.cmdlist = []
1734         self.dir = None
1735         self.path = None
1736         self.desc = [desc]
1737         self.keep = keep
1738
1739     def desc_txt(self, desc):
1740         """Return the description to use for a command."""
1741         return '%s %s' % (' '.join(self.desc), desc)
1742
1743     def use_dir(self, dir):
1744         """Set the default directory for subsequent commands."""
1745         self.dir = dir
1746
1747     def use_path(self, path):
1748         """Set a directory to be prepended to the PATH for subsequent
1749         commands."""
1750         self.path = path
1751
1752     def push_subdesc(self, subdesc):
1753         """Set the default subdescription for subsequent commands (e.g., the
1754         name of a component being built, within the series of commands
1755         building it)."""
1756         self.desc.append(subdesc)
1757
1758     def pop_subdesc(self):
1759         """Pop a subdescription from the list of descriptions."""
1760         self.desc.pop()
1761
1762     def create_use_dir(self, dir):
1763         """Remove and recreate a directory and use it for subsequent
1764         commands."""
1765         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1766         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1767         self.use_dir(dir)
1768
1769     def add_command_dir(self, desc, dir, command, always_run=False):
1770         """Add a command to run in a given directory."""
1771         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1772                       command, always_run)
1773         self.cmdlist.append(cmd)
1774
1775     def add_command(self, desc, command, always_run=False):
1776         """Add a command to run in the default directory."""
1777         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1778                       self.path, command, always_run)
1779         self.cmdlist.append(cmd)
1780
1781     def cleanup_dir(self, desc='cleanup', dir=None):
1782         """Clean up a build directory.  If no directory is specified, the
1783         default directory is cleaned up and ceases to be the default
1784         directory."""
1785         if dir is None:
1786             dir = self.dir
1787             self.use_dir(None)
1788         if self.keep != 'all':
1789             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1790                                  always_run=(self.keep == 'none'))
1791
1792     def makefile_commands(self, wrapper, logsdir):
1793         """Return the sequence of commands in the form of text for a Makefile.
1794         The given wrapper script takes arguments: base of logs for
1795         previous command, or empty; base of logs for this command;
1796         description; directory; PATH addition; the command itself."""
1797         # prev_base is the base of the name for logs of the previous
1798         # command that is not always-run (that is, a build command,
1799         # whose failure should stop subsequent build commands from
1800         # being run, as opposed to a cleanup command, which is run
1801         # even if previous commands failed).
1802         prev_base = ''
1803         cmds = []
1804         for c in self.cmdlist:
1805             ctxt = c.shell_make_quote()
1806             if prev_base and not c.always_run:
1807                 prev_log = os.path.join(logsdir, prev_base)
1808             else:
1809                 prev_log = ''
1810             this_log = os.path.join(logsdir, c.logbase)
1811             if not c.always_run:
1812                 prev_base = c.logbase
1813             if c.dir is None:
1814                 dir = ''
1815             else:
1816                 dir = c.dir
1817             if c.path is None:
1818                 path = ''
1819             else:
1820                 path = c.path
1821             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1822             prelim_txt = Command.shell_make_quote_list(prelims, False)
1823             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1824         return '\n'.join(cmds)
1825
1826     def status_logs(self, logsdir):
1827         """Return the list of log files with command status."""
1828         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1829                 for c in self.cmdlist]
1830
1831
1832 def get_parser():
1833     """Return an argument parser for this module."""
1834     parser = argparse.ArgumentParser(description=__doc__)
1835     parser.add_argument('-j', dest='parallelism',
1836                         help='Run this number of jobs in parallel',
1837                         type=int, default=os.cpu_count())
1838     parser.add_argument('--keep', dest='keep',
1839                         help='Whether to keep all build directories, '
1840                         'none or only those from failed builds',
1841                         default='none', choices=('none', 'all', 'failed'))
1842     parser.add_argument('--replace-sources', action='store_true',
1843                         help='Remove and replace source directories '
1844                         'with the wrong version of a component')
1845     parser.add_argument('--strip', action='store_true',
1846                         help='Strip installed glibc libraries')
1847     parser.add_argument('--full-gcc', action='store_true',
1848                         help='Build GCC with all languages and libsanitizer')
1849     parser.add_argument('--shallow', action='store_true',
1850                         help='Do not download Git history during checkout')
1851     parser.add_argument('topdir',
1852                         help='Toplevel working directory')
1853     parser.add_argument('action',
1854                         help='What to do',
1855                         choices=('checkout', 'bot-cycle', 'bot',
1856                                  'host-libraries', 'compilers', 'glibcs',
1857                                  'update-syscalls', 'list-compilers',
1858                                  'list-glibcs'))
1859     parser.add_argument('configs',
1860                         help='Versions to check out or configurations to build',
1861                         nargs='*')
1862     return parser
1863
1864
1865 def main(argv):
1866     """The main entry point."""
1867     parser = get_parser()
1868     opts = parser.parse_args(argv)
1869     topdir = os.path.abspath(opts.topdir)
1870     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1871                   opts.strip, opts.full_gcc, opts.action,
1872                   shallow=opts.shallow)
1873     ctx.run_builds(opts.action, opts.configs)
1874
1875
1876 if __name__ == '__main__':
1877     main(sys.argv[1:])