scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2023 Free Software Foundation, Inc.
   4 # Copyright The GNU Toolchain Authors.
   5 # This file is part of the GNU C Library.
   6 #
   7 # The GNU C Library is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU Lesser General Public
   9 # License as published by the Free Software Foundation; either
  10 # version 2.1 of the License, or (at your option) any later version.
  11 #
  12 # The GNU C Library is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 # Lesser General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Lesser General Public
  18 # License along with the GNU C Library; if not, see
  19 # <https://www.gnu.org/licenses/>.
  20
  21 """Build many configurations of glibc.
  22
  23 This script takes as arguments a directory name (containing a src
  24 subdirectory with sources of the relevant toolchain components) and a
  25 description of what to do: 'checkout', to check out sources into that
  26 directory, 'bot-cycle', to run a series of checkout and build steps,
  27 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  28 libraries required by the toolchain, 'compilers', to build
  29 cross-compilers for various configurations, or 'glibcs', to build
  30 glibc for various configurations and run the compilation parts of the
  31 testsuite.  Subsequent arguments name the versions of components to
  32 check out (<component>-<version), for 'checkout', or, for actions
  33 other than 'checkout' and 'bot-cycle', name configurations for which
  34 compilers or glibc are to be built.
  35
  36 The 'list-compilers' command prints the name of each available
  37 compiler configuration, without building anything.  The 'list-glibcs'
  38 command prints the name of each glibc compiler configuration, followed
  39 by the space, followed by the name of the compiler configuration used
  40 for building this glibc variant.
  41
  42 """
  43
  44 import argparse
  45 import datetime
  46 import email.mime.text
  47 import email.utils
  48 import json
  49 import os
  50 import re
  51 import shutil
  52 import smtplib
  53 import stat
  54 import subprocess
  55 import sys
  56 import time
  57 import urllib.request
  58
  59 try:
  60     subprocess.run
  61 except:
  62     class _CompletedProcess:
  63         def __init__(self, args, returncode, stdout=None, stderr=None):
  64             self.args = args
  65             self.returncode = returncode
  66             self.stdout = stdout
  67             self.stderr = stderr
  68
  69     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  70         assert(timeout is None)
  71         with subprocess.Popen(*popenargs, **kwargs) as process:
  72             try:
  73                 stdout, stderr = process.communicate(input)
  74             except:
  75                 process.kill()
  76                 process.wait()
  77                 raise
  78             returncode = process.poll()
  79             if check and returncode:
  80                 raise subprocess.CalledProcessError(returncode, popenargs)
  81         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  82
  83     subprocess.run = _run
  84
  85
  86 class Context(object):
  87     """The global state associated with builds in a given directory."""
  88
  89     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  90                  full_gcc, action, shallow=False):
  91         """Initialize the context."""
  92         self.topdir = topdir
  93         self.parallelism = parallelism
  94         self.keep = keep
  95         self.replace_sources = replace_sources
  96         self.strip = strip
  97         self.full_gcc = full_gcc
  98         self.shallow = shallow
  99         self.srcdir = os.path.join(topdir, 'src')
 100         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 101         self.build_state_json = os.path.join(topdir, 'build-state.json')
 102         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 103         self.installdir = os.path.join(topdir, 'install')
 104         self.host_libraries_installdir = os.path.join(self.installdir,
 105                                                       'host-libraries')
 106         self.builddir = os.path.join(topdir, 'build')
 107         self.logsdir = os.path.join(topdir, 'logs')
 108         self.logsdir_old = os.path.join(topdir, 'logs-old')
 109         self.makefile = os.path.join(self.builddir, 'Makefile')
 110         self.wrapper = os.path.join(self.builddir, 'wrapper')
 111         self.save_logs = os.path.join(self.builddir, 'save-logs')
 112         self.script_text = self.get_script_text()
 113         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 114             self.build_triplet = self.get_build_triplet()
 115             self.glibc_version = self.get_glibc_version()
 116         self.configs = {}
 117         self.glibc_configs = {}
 118         self.makefile_pieces = ['.PHONY: all\n']
 119         self.add_all_configs()
 120         self.load_versions_json()
 121         self.load_build_state_json()
 122         self.status_log_list = []
 123         self.email_warning = False
 124
 125     def get_script_text(self):
 126         """Return the text of this script."""
 127         with open(sys.argv[0], 'r') as f:
 128             return f.read()
 129
 130     def exec_self(self):
 131         """Re-execute this script with the same arguments."""
 132         sys.stdout.flush()
 133         os.execv(sys.executable, [sys.executable] + sys.argv)
 134
 135     def get_build_triplet(self):
 136         """Determine the build triplet with config.guess."""
 137         config_guess = os.path.join(self.component_srcdir('gcc'),
 138                                     'config.guess')
 139         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 140                                 check=True, universal_newlines=True).stdout
 141         return cg_out.rstrip()
 142
 143     def get_glibc_version(self):
 144         """Determine the glibc version number (major.minor)."""
 145         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 146         with open(version_h, 'r') as f:
 147             lines = f.readlines()
 148         starttext = '#define VERSION "'
 149         for l in lines:
 150             if l.startswith(starttext):
 151                 l = l[len(starttext):]
 152                 l = l.rstrip('"\n')
 153                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 154                 return '%s.%s' % m.group(1, 2)
 155         print('error: could not determine glibc version')
 156         exit(1)
 157
 158     def add_all_configs(self):
 159         """Add all known glibc build configurations."""
 160         self.add_config(arch='aarch64',
 161                         os_name='linux-gnu',
 162                         extra_glibcs=[{'variant': 'disable-multi-arch',
 163                                        'cfg': ['--disable-multi-arch']}])
 164         self.add_config(arch='aarch64_be',
 165                         os_name='linux-gnu')
 166         self.add_config(arch='arc',
 167                         os_name='linux-gnu',
 168                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 169         self.add_config(arch='arc',
 170                         os_name='linux-gnuhf',
 171                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
 172         self.add_config(arch='arceb',
 173                         os_name='linux-gnu',
 174                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 175         self.add_config(arch='alpha',
 176                         os_name='linux-gnu')
 177         self.add_config(arch='arm',
 178                         os_name='linux-gnueabi',
 179                         extra_glibcs=[{'variant': 'v4t',
 180                                        'ccopts': '-march=armv4t'}])
 181         self.add_config(arch='armeb',
 182                         os_name='linux-gnueabi')
 183         self.add_config(arch='armeb',
 184                         os_name='linux-gnueabi',
 185                         variant='be8',
 186                         gcc_cfg=['--with-arch=armv7-a'])
 187         self.add_config(arch='arm',
 188                         os_name='linux-gnueabihf',
 189                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 190                         extra_glibcs=[{'variant': 'v7a',
 191                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 192                                       {'variant': 'thumb',
 193                                        'ccopts':
 194                                        '-mthumb -march=armv7-a -mfpu=vfpv3'},
 195                                       {'variant': 'v7a-disable-multi-arch',
 196                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 197                                        'cfg': ['--disable-multi-arch']}])
 198         self.add_config(arch='armeb',
 199                         os_name='linux-gnueabihf',
 200                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 201         self.add_config(arch='armeb',
 202                         os_name='linux-gnueabihf',
 203                         variant='be8',
 204                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 205                                  '--with-fpu=vfpv3'])
 206         self.add_config(arch='csky',
 207                         os_name='linux-gnuabiv2',
 208                         variant='soft',
 209                         gcc_cfg=['--disable-multilib'])
 210         self.add_config(arch='csky',
 211                         os_name='linux-gnuabiv2',
 212                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 213         self.add_config(arch='hppa',
 214                         os_name='linux-gnu')
 215         self.add_config(arch='i686',
 216                         os_name='gnu')
 217         self.add_config(arch='ia64',
 218                         os_name='linux-gnu',
 219                         first_gcc_cfg=['--with-system-libunwind'],
 220                         binutils_cfg=['--enable-obsolete'])
 221         self.add_config(arch='loongarch64',
 222                         os_name='linux-gnu',
 223                         variant='lp64d',
 224                         gcc_cfg=['--with-abi=lp64d','--disable-multilib'])
 225         self.add_config(arch='loongarch64',
 226                         os_name='linux-gnu',
 227                         variant='lp64s',
 228                         gcc_cfg=['--with-abi=lp64s','--disable-multilib'])
 229         self.add_config(arch='m68k',
 230                         os_name='linux-gnu',
 231                         gcc_cfg=['--disable-multilib'])
 232         self.add_config(arch='m68k',
 233                         os_name='linux-gnu',
 234                         variant='coldfire',
 235                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 236         self.add_config(arch='m68k',
 237                         os_name='linux-gnu',
 238                         variant='coldfire-soft',
 239                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 240                                  '--disable-multilib'])
 241         self.add_config(arch='microblaze',
 242                         os_name='linux-gnu',
 243                         gcc_cfg=['--disable-multilib'])
 244         self.add_config(arch='microblazeel',
 245                         os_name='linux-gnu',
 246                         gcc_cfg=['--disable-multilib'])
 247         self.add_config(arch='mips64',
 248                         os_name='linux-gnu',
 249                         gcc_cfg=['--with-mips-plt'],
 250                         glibcs=[{'variant': 'n32'},
 251                                 {'arch': 'mips',
 252                                  'ccopts': '-mabi=32'},
 253                                 {'variant': 'n64',
 254                                  'ccopts': '-mabi=64'}])
 255         self.add_config(arch='mips64',
 256                         os_name='linux-gnu',
 257                         variant='soft',
 258                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 259                         glibcs=[{'variant': 'n32-soft'},
 260                                 {'variant': 'soft',
 261                                  'arch': 'mips',
 262                                  'ccopts': '-mabi=32'},
 263                                 {'variant': 'n64-soft',
 264                                  'ccopts': '-mabi=64'}])
 265         self.add_config(arch='mips64',
 266                         os_name='linux-gnu',
 267                         variant='nan2008',
 268                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 269                                  '--with-arch-64=mips64r2',
 270                                  '--with-arch-32=mips32r2'],
 271                         glibcs=[{'variant': 'n32-nan2008'},
 272                                 {'variant': 'nan2008',
 273                                  'arch': 'mips',
 274                                  'ccopts': '-mabi=32'},
 275                                 {'variant': 'n64-nan2008',
 276                                  'ccopts': '-mabi=64'}])
 277         self.add_config(arch='mips64',
 278                         os_name='linux-gnu',
 279                         variant='nan2008-soft',
 280                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 281                                  '--with-arch-64=mips64r2',
 282                                  '--with-arch-32=mips32r2',
 283                                  '--with-float=soft'],
 284                         glibcs=[{'variant': 'n32-nan2008-soft'},
 285                                 {'variant': 'nan2008-soft',
 286                                  'arch': 'mips',
 287                                  'ccopts': '-mabi=32'},
 288                                 {'variant': 'n64-nan2008-soft',
 289                                  'ccopts': '-mabi=64'}])
 290         self.add_config(arch='mips64el',
 291                         os_name='linux-gnu',
 292                         gcc_cfg=['--with-mips-plt'],
 293                         glibcs=[{'variant': 'n32'},
 294                                 {'arch': 'mipsel',
 295                                  'ccopts': '-mabi=32'},
 296                                 {'variant': 'n64',
 297                                  'ccopts': '-mabi=64'}])
 298         self.add_config(arch='mips64el',
 299                         os_name='linux-gnu',
 300                         variant='soft',
 301                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 302                         glibcs=[{'variant': 'n32-soft'},
 303                                 {'variant': 'soft',
 304                                  'arch': 'mipsel',
 305                                  'ccopts': '-mabi=32'},
 306                                 {'variant': 'n64-soft',
 307                                  'ccopts': '-mabi=64'}])
 308         self.add_config(arch='mips64el',
 309                         os_name='linux-gnu',
 310                         variant='nan2008',
 311                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 312                                  '--with-arch-64=mips64r2',
 313                                  '--with-arch-32=mips32r2'],
 314                         glibcs=[{'variant': 'n32-nan2008'},
 315                                 {'variant': 'nan2008',
 316                                  'arch': 'mipsel',
 317                                  'ccopts': '-mabi=32'},
 318                                 {'variant': 'n64-nan2008',
 319                                  'ccopts': '-mabi=64'}])
 320         self.add_config(arch='mips64el',
 321                         os_name='linux-gnu',
 322                         variant='nan2008-soft',
 323                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 324                                  '--with-arch-64=mips64r2',
 325                                  '--with-arch-32=mips32r2',
 326                                  '--with-float=soft'],
 327                         glibcs=[{'variant': 'n32-nan2008-soft'},
 328                                 {'variant': 'nan2008-soft',
 329                                  'arch': 'mipsel',
 330                                  'ccopts': '-mabi=32'},
 331                                 {'variant': 'n64-nan2008-soft',
 332                                  'ccopts': '-mabi=64'}])
 333         self.add_config(arch='mipsisa64r6el',
 334                         os_name='linux-gnu',
 335                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 336                                  '--with-arch-64=mips64r6',
 337                                  '--with-arch-32=mips32r6',
 338                                  '--with-float=hard'],
 339                         glibcs=[{'variant': 'n32'},
 340                                 {'arch': 'mipsisa32r6el',
 341                                  'ccopts': '-mabi=32'},
 342                                 {'variant': 'n64',
 343                                  'ccopts': '-mabi=64'}])
 344         self.add_config(arch='nios2',
 345                         os_name='linux-gnu')
 346         self.add_config(arch='or1k',
 347                         os_name='linux-gnu',
 348                         variant='soft',
 349                         gcc_cfg=['--with-multilib-list=mcmov'])
 350         self.add_config(arch='powerpc',
 351                         os_name='linux-gnu',
 352                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 353                         extra_glibcs=[{'variant': 'power4',
 354                                        'ccopts': '-mcpu=power4',
 355                                        'cfg': ['--with-cpu=power4']}])
 356         self.add_config(arch='powerpc',
 357                         os_name='linux-gnu',
 358                         variant='soft',
 359                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 360                                  '--enable-secureplt'])
 361         self.add_config(arch='powerpc64',
 362                         os_name='linux-gnu',
 363                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 364         self.add_config(arch='powerpc64le',
 365                         os_name='linux-gnu',
 366                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 367                         extra_glibcs=[{'variant': 'disable-multi-arch',
 368                                        'cfg': ['--disable-multi-arch']}])
 369         self.add_config(arch='riscv32',
 370                         os_name='linux-gnu',
 371                         variant='rv32imac-ilp32',
 372                         gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
 373                                  '--disable-multilib'])
 374         self.add_config(arch='riscv32',
 375                         os_name='linux-gnu',
 376                         variant='rv32imafdc-ilp32',
 377                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
 378                                  '--disable-multilib'])
 379         self.add_config(arch='riscv32',
 380                         os_name='linux-gnu',
 381                         variant='rv32imafdc-ilp32d',
 382                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
 383                                  '--disable-multilib'])
 384         self.add_config(arch='riscv64',
 385                         os_name='linux-gnu',
 386                         variant='rv64imac-lp64',
 387                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 388                                  '--disable-multilib'])
 389         self.add_config(arch='riscv64',
 390                         os_name='linux-gnu',
 391                         variant='rv64imafdc-lp64',
 392                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 393                                  '--disable-multilib'])
 394         self.add_config(arch='riscv64',
 395                         os_name='linux-gnu',
 396                         variant='rv64imafdc-lp64d',
 397                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 398                                  '--disable-multilib'])
 399         self.add_config(arch='s390x',
 400                         os_name='linux-gnu',
 401                         glibcs=[{},
 402                                 {'arch': 's390', 'ccopts': '-m31'}],
 403                         extra_glibcs=[{'variant': 'O3',
 404                                        'cflags': '-O3'}])
 405         self.add_config(arch='sh3',
 406                         os_name='linux-gnu')
 407         self.add_config(arch='sh3eb',
 408                         os_name='linux-gnu')
 409         self.add_config(arch='sh4',
 410                         os_name='linux-gnu')
 411         self.add_config(arch='sh4eb',
 412                         os_name='linux-gnu')
 413         self.add_config(arch='sh4',
 414                         os_name='linux-gnu',
 415                         variant='soft',
 416                         gcc_cfg=['--without-fp'])
 417         self.add_config(arch='sh4eb',
 418                         os_name='linux-gnu',
 419                         variant='soft',
 420                         gcc_cfg=['--without-fp'])
 421         self.add_config(arch='sparc64',
 422                         os_name='linux-gnu',
 423                         glibcs=[{},
 424                                 {'arch': 'sparcv9',
 425                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 426                         extra_glibcs=[{'variant': 'leon3',
 427                                        'arch' : 'sparcv8',
 428                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 429                                       {'variant': 'disable-multi-arch',
 430                                        'cfg': ['--disable-multi-arch']},
 431                                       {'variant': 'disable-multi-arch',
 432                                        'arch': 'sparcv9',
 433                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 434                                        'cfg': ['--disable-multi-arch']}])
 435         self.add_config(arch='x86_64',
 436                         os_name='linux-gnu',
 437                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 438                         glibcs=[{},
 439                                 {'variant': 'x32', 'ccopts': '-mx32'},
 440                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 441                         extra_glibcs=[{'variant': 'disable-multi-arch',
 442                                        'cfg': ['--disable-multi-arch']},
 443                                       {'variant': 'minimal',
 444                                        'cfg': ['--disable-multi-arch',
 445                                                '--disable-profile',
 446                                                '--disable-timezone-tools',
 447                                                '--disable-mathvec',
 448                                                '--disable-crypt',
 449                                                '--disable-build-nscd',
 450                                                '--disable-nscd']},
 451                                       {'variant': 'no-pie',
 452                                        'cfg': ['--disable-default-pie']},
 453                                       {'variant': 'x32-no-pie',
 454                                        'ccopts': '-mx32',
 455                                        'cfg': ['--disable-default-pie']},
 456                                       {'variant': 'no-pie',
 457                                        'arch': 'i686',
 458                                        'ccopts': '-m32 -march=i686',
 459                                        'cfg': ['--disable-default-pie']},
 460                                       {'variant': 'disable-multi-arch',
 461                                        'arch': 'i686',
 462                                        'ccopts': '-m32 -march=i686',
 463                                        'cfg': ['--disable-multi-arch']},
 464                                       {'arch': 'i486',
 465                                        'ccopts': '-m32 -march=i486'},
 466                                       {'arch': 'i586',
 467                                        'ccopts': '-m32 -march=i586'},
 468                                       {'variant': 'enable-fortify-source',
 469                                        'cfg': ['--enable-fortify-source']}])
 470         self.add_config(arch='x86_64',
 471                         os_name='gnu',
 472                         gcc_cfg=['--disable-multilib'])
 473
 474     def add_config(self, **args):
 475         """Add an individual build configuration."""
 476         cfg = Config(self, **args)
 477         if cfg.name in self.configs:
 478             print('error: duplicate config %s' % cfg.name)
 479             exit(1)
 480         self.configs[cfg.name] = cfg
 481         for c in cfg.all_glibcs:
 482             if c.name in self.glibc_configs:
 483                 print('error: duplicate glibc config %s' % c.name)
 484                 exit(1)
 485             self.glibc_configs[c.name] = c
 486
 487     def component_srcdir(self, component):
 488         """Return the source directory for a given component, e.g. gcc."""
 489         return os.path.join(self.srcdir, component)
 490
 491     def component_builddir(self, action, config, component, subconfig=None):
 492         """Return the directory to use for a build."""
 493         if config is None:
 494             # Host libraries.
 495             assert subconfig is None
 496             return os.path.join(self.builddir, action, component)
 497         if subconfig is None:
 498             return os.path.join(self.builddir, action, config, component)
 499         else:
 500             # glibc build as part of compiler build.
 501             return os.path.join(self.builddir, action, config, component,
 502                                 subconfig)
 503
 504     def compiler_installdir(self, config):
 505         """Return the directory in which to install a compiler."""
 506         return os.path.join(self.installdir, 'compilers', config)
 507
 508     def compiler_bindir(self, config):
 509         """Return the directory in which to find compiler binaries."""
 510         return os.path.join(self.compiler_installdir(config), 'bin')
 511
 512     def compiler_sysroot(self, config):
 513         """Return the sysroot directory for a compiler."""
 514         return os.path.join(self.compiler_installdir(config), 'sysroot')
 515
 516     def glibc_installdir(self, config):
 517         """Return the directory in which to install glibc."""
 518         return os.path.join(self.installdir, 'glibcs', config)
 519
 520     def run_builds(self, action, configs):
 521         """Run the requested builds."""
 522         if action == 'checkout':
 523             self.checkout(configs)
 524             return
 525         if action == 'bot-cycle':
 526             if configs:
 527                 print('error: configurations specified for bot-cycle')
 528                 exit(1)
 529             self.bot_cycle()
 530             return
 531         if action == 'bot':
 532             if configs:
 533                 print('error: configurations specified for bot')
 534                 exit(1)
 535             self.bot()
 536             return
 537         if action in ('host-libraries', 'list-compilers',
 538                       'list-glibcs') and configs:
 539             print('error: configurations specified for ' + action)
 540             exit(1)
 541         if action == 'list-compilers':
 542             for name in sorted(self.configs.keys()):
 543                 print(name)
 544             return
 545         if action == 'list-glibcs':
 546             for config in sorted(self.glibc_configs.values(),
 547                                  key=lambda c: c.name):
 548                 print(config.name, config.compiler.name)
 549             return
 550         self.clear_last_build_state(action)
 551         build_time = datetime.datetime.utcnow()
 552         if action == 'host-libraries':
 553             build_components = ('gmp', 'mpfr', 'mpc')
 554             old_components = ()
 555             old_versions = {}
 556             self.build_host_libraries()
 557         elif action == 'compilers':
 558             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 559                                 'gnumach', 'hurd')
 560             old_components = ('gmp', 'mpfr', 'mpc')
 561             old_versions = self.build_state['host-libraries']['build-versions']
 562             self.build_compilers(configs)
 563         else:
 564             build_components = ('glibc',)
 565             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 566                               'mig', 'gnumach', 'hurd')
 567             old_versions = self.build_state['compilers']['build-versions']
 568             if action == 'update-syscalls':
 569                 self.update_syscalls(configs)
 570             else:
 571                 self.build_glibcs(configs)
 572         self.write_files()
 573         self.do_build()
 574         if configs:
 575             # Partial build, do not update stored state.
 576             return
 577         build_versions = {}
 578         for k in build_components:
 579             if k in self.versions:
 580                 build_versions[k] = {'version': self.versions[k]['version'],
 581                                      'revision': self.versions[k]['revision']}
 582         for k in old_components:
 583             if k in old_versions:
 584                 build_versions[k] = {'version': old_versions[k]['version'],
 585                                      'revision': old_versions[k]['revision']}
 586         self.update_build_state(action, build_time, build_versions)
 587
 588     @staticmethod
 589     def remove_dirs(*args):
 590         """Remove directories and their contents if they exist."""
 591         for dir in args:
 592             shutil.rmtree(dir, ignore_errors=True)
 593
 594     @staticmethod
 595     def remove_recreate_dirs(*args):
 596         """Remove directories if they exist, and create them as empty."""
 597         Context.remove_dirs(*args)
 598         for dir in args:
 599             os.makedirs(dir, exist_ok=True)
 600
 601     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 602         """Add makefile text for a list of commands."""
 603         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 604         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 605                                     (target, target, target, commands))
 606         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 607
 608     def write_files(self):
 609         """Write out the Makefile and wrapper script."""
 610         mftext = ''.join(self.makefile_pieces)
 611         with open(self.makefile, 'w') as f:
 612             f.write(mftext)
 613         wrapper_text = (
 614             '#!/bin/sh\n'
 615             'prev_base=$1\n'
 616             'this_base=$2\n'
 617             'desc=$3\n'
 618             'dir=$4\n'
 619             'path=$5\n'
 620             'shift 5\n'
 621             'prev_status=$prev_base-status.txt\n'
 622             'this_status=$this_base-status.txt\n'
 623             'this_log=$this_base-log.txt\n'
 624             'date > "$this_log"\n'
 625             'echo >> "$this_log"\n'
 626             'echo "Description: $desc" >> "$this_log"\n'
 627             'printf "%s" "Command:" >> "$this_log"\n'
 628             'for word in "$@"; do\n'
 629             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 630             '    printf " %s" "$word"\n'
 631             '  else\n'
 632             '    printf " \'"\n'
 633             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 634             '    printf "\'"\n'
 635             '  fi\n'
 636             'done >> "$this_log"\n'
 637             'echo >> "$this_log"\n'
 638             'echo "Directory: $dir" >> "$this_log"\n'
 639             'echo "Path addition: $path" >> "$this_log"\n'
 640             'echo >> "$this_log"\n'
 641             'record_status ()\n'
 642             '{\n'
 643             '  echo >> "$this_log"\n'
 644             '  echo "$1: $desc" > "$this_status"\n'
 645             '  echo "$1: $desc" >> "$this_log"\n'
 646             '  echo >> "$this_log"\n'
 647             '  date >> "$this_log"\n'
 648             '  echo "$1: $desc"\n'
 649             '  exit 0\n'
 650             '}\n'
 651             'check_error ()\n'
 652             '{\n'
 653             '  if [ "$1" != "0" ]; then\n'
 654             '    record_status FAIL\n'
 655             '  fi\n'
 656             '}\n'
 657             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 658             '    record_status UNRESOLVED\n'
 659             'fi\n'
 660             'if [ "$dir" ]; then\n'
 661             '  cd "$dir"\n'
 662             '  check_error "$?"\n'
 663             'fi\n'
 664             'if [ "$path" ]; then\n'
 665             '  PATH=$path:$PATH\n'
 666             'fi\n'
 667             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 668             'check_error "$?"\n'
 669             'record_status PASS\n')
 670         with open(self.wrapper, 'w') as f:
 671             f.write(wrapper_text)
 672         # Mode 0o755.
 673         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 674                      stat.S_IROTH|stat.S_IXOTH)
 675         os.chmod(self.wrapper, mode_exec)
 676         save_logs_text = (
 677             '#!/bin/sh\n'
 678             'if ! [ -f tests.sum ]; then\n'
 679             '  echo "No test summary available."\n'
 680             '  exit 0\n'
 681             'fi\n'
 682             'save_file ()\n'
 683             '{\n'
 684             '  echo "Contents of $1:"\n'
 685             '  echo\n'
 686             '  cat "$1"\n'
 687             '  echo\n'
 688             '  echo "End of contents of $1."\n'
 689             '  echo\n'
 690             '}\n'
 691             'save_file tests.sum\n'
 692             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 693             'for t in $non_pass_tests; do\n'
 694             '  if [ -f "$t.out" ]; then\n'
 695             '    save_file "$t.out"\n'
 696             '  fi\n'
 697             'done\n')
 698         with open(self.save_logs, 'w') as f:
 699             f.write(save_logs_text)
 700         os.chmod(self.save_logs, mode_exec)
 701
 702     def do_build(self):
 703         """Do the actual build."""
 704         cmd = ['make', '-O', '-j%d' % self.parallelism]
 705         subprocess.run(cmd, cwd=self.builddir, check=True)
 706
 707     def build_host_libraries(self):
 708         """Build the host libraries."""
 709         installdir = self.host_libraries_installdir
 710         builddir = os.path.join(self.builddir, 'host-libraries')
 711         logsdir = os.path.join(self.logsdir, 'host-libraries')
 712         self.remove_recreate_dirs(installdir, builddir, logsdir)
 713         cmdlist = CommandList('host-libraries', self.keep)
 714         self.build_host_library(cmdlist, 'gmp')
 715         self.build_host_library(cmdlist, 'mpfr',
 716                                 ['--with-gmp=%s' % installdir])
 717         self.build_host_library(cmdlist, 'mpc',
 718                                 ['--with-gmp=%s' % installdir,
 719                                 '--with-mpfr=%s' % installdir])
 720         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 721         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 722
 723     def build_host_library(self, cmdlist, lib, extra_opts=None):
 724         """Build one host library."""
 725         srcdir = self.component_srcdir(lib)
 726         builddir = self.component_builddir('host-libraries', None, lib)
 727         installdir = self.host_libraries_installdir
 728         cmdlist.push_subdesc(lib)
 729         cmdlist.create_use_dir(builddir)
 730         cfg_cmd = [os.path.join(srcdir, 'configure'),
 731                    '--prefix=%s' % installdir,
 732                    '--disable-shared']
 733         if extra_opts:
 734             cfg_cmd.extend (extra_opts)
 735         cmdlist.add_command('configure', cfg_cmd)
 736         cmdlist.add_command('build', ['make'])
 737         cmdlist.add_command('check', ['make', 'check'])
 738         cmdlist.add_command('install', ['make', 'install'])
 739         cmdlist.cleanup_dir()
 740         cmdlist.pop_subdesc()
 741
 742     def build_compilers(self, configs):
 743         """Build the compilers."""
 744         if not configs:
 745             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 746             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 747             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 748             configs = sorted(self.configs.keys())
 749         for c in configs:
 750             self.configs[c].build()
 751
 752     def build_glibcs(self, configs):
 753         """Build the glibcs."""
 754         if not configs:
 755             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 756             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 757             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 758             configs = sorted(self.glibc_configs.keys())
 759         for c in configs:
 760             self.glibc_configs[c].build()
 761
 762     def update_syscalls(self, configs):
 763         """Update the glibc syscall lists."""
 764         if not configs:
 765             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 766             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 767             configs = sorted(self.glibc_configs.keys())
 768         for c in configs:
 769             self.glibc_configs[c].update_syscalls()
 770
 771     def load_versions_json(self):
 772         """Load information about source directory versions."""
 773         if not os.access(self.versions_json, os.F_OK):
 774             self.versions = {}
 775             return
 776         with open(self.versions_json, 'r') as f:
 777             self.versions = json.load(f)
 778
 779     def store_json(self, data, filename):
 780         """Store information in a JSON file."""
 781         filename_tmp = filename + '.tmp'
 782         with open(filename_tmp, 'w') as f:
 783             json.dump(data, f, indent=2, sort_keys=True)
 784         os.rename(filename_tmp, filename)
 785
 786     def store_versions_json(self):
 787         """Store information about source directory versions."""
 788         self.store_json(self.versions, self.versions_json)
 789
 790     def set_component_version(self, component, version, explicit, revision):
 791         """Set the version information for a component."""
 792         self.versions[component] = {'version': version,
 793                                     'explicit': explicit,
 794                                     'revision': revision}
 795         self.store_versions_json()
 796
 797     def checkout(self, versions):
 798         """Check out the desired component versions."""
 799         default_versions = {'binutils': 'vcs-2.40',
 800                             'gcc': 'vcs-13',
 801                             'glibc': 'vcs-mainline',
 802                             'gmp': '6.2.1',
 803                             'linux': '6.4',
 804                             'mpc': '1.3.1',
 805                             'mpfr': '4.2.0',
 806                             'mig': 'vcs-mainline',
 807                             'gnumach': 'vcs-mainline',
 808                             'hurd': 'vcs-mainline'}
 809         use_versions = {}
 810         explicit_versions = {}
 811         for v in versions:
 812             found_v = False
 813             for k in default_versions.keys():
 814                 kx = k + '-'
 815                 if v.startswith(kx):
 816                     vx = v[len(kx):]
 817                     if k in use_versions:
 818                         print('error: multiple versions for %s' % k)
 819                         exit(1)
 820                     use_versions[k] = vx
 821                     explicit_versions[k] = True
 822                     found_v = True
 823                     break
 824             if not found_v:
 825                 print('error: unknown component in %s' % v)
 826                 exit(1)
 827         for k in default_versions.keys():
 828             if k not in use_versions:
 829                 if k in self.versions and self.versions[k]['explicit']:
 830                     use_versions[k] = self.versions[k]['version']
 831                     explicit_versions[k] = True
 832                 else:
 833                     use_versions[k] = default_versions[k]
 834                     explicit_versions[k] = False
 835         os.makedirs(self.srcdir, exist_ok=True)
 836         for k in sorted(default_versions.keys()):
 837             update = os.access(self.component_srcdir(k), os.F_OK)
 838             v = use_versions[k]
 839             if (update and
 840                 k in self.versions and
 841                 v != self.versions[k]['version']):
 842                 if not self.replace_sources:
 843                     print('error: version of %s has changed from %s to %s, '
 844                           'use --replace-sources to check out again' %
 845                           (k, self.versions[k]['version'], v))
 846                     exit(1)
 847                 shutil.rmtree(self.component_srcdir(k))
 848                 update = False
 849             if v.startswith('vcs-'):
 850                 revision = self.checkout_vcs(k, v[4:], update)
 851             else:
 852                 self.checkout_tar(k, v, update)
 853                 revision = v
 854             self.set_component_version(k, v, explicit_versions[k], revision)
 855         if self.get_script_text() != self.script_text:
 856             # Rerun the checkout process in case the updated script
 857             # uses different default versions or new components.
 858             self.exec_self()
 859
 860     def checkout_vcs(self, component, version, update):
 861         """Check out the given version of the given component from version
 862         control.  Return a revision identifier."""
 863         if component == 'binutils':
 864             git_url = 'https://sourceware.org/git/binutils-gdb.git'
 865             if version == 'mainline':
 866                 git_branch = 'master'
 867             else:
 868                 trans = str.maketrans({'.': '_'})
 869                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 870             return self.git_checkout(component, git_url, git_branch, update)
 871         elif component == 'gcc':
 872             if version == 'mainline':
 873                 branch = 'master'
 874             else:
 875                 branch = 'releases/gcc-%s' % version
 876             return self.gcc_checkout(branch, update)
 877         elif component == 'glibc':
 878             git_url = 'https://sourceware.org/git/glibc.git'
 879             if version == 'mainline':
 880                 git_branch = 'master'
 881             else:
 882                 git_branch = 'release/%s/master' % version
 883             r = self.git_checkout(component, git_url, git_branch, update)
 884             self.fix_glibc_timestamps()
 885             return r
 886         elif component == 'gnumach':
 887             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 888             git_branch = 'master'
 889             r = self.git_checkout(component, git_url, git_branch, update)
 890             subprocess.run(['autoreconf', '-i'],
 891                            cwd=self.component_srcdir(component), check=True)
 892             return r
 893         elif component == 'mig':
 894             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 895             git_branch = 'master'
 896             r = self.git_checkout(component, git_url, git_branch, update)
 897             subprocess.run(['autoreconf', '-i'],
 898                            cwd=self.component_srcdir(component), check=True)
 899             return r
 900         elif component == 'hurd':
 901             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 902             git_branch = 'master'
 903             r = self.git_checkout(component, git_url, git_branch, update)
 904             subprocess.run(['autoconf'],
 905                            cwd=self.component_srcdir(component), check=True)
 906             return r
 907         else:
 908             print('error: component %s coming from VCS' % component)
 909             exit(1)
 910
 911     def git_checkout(self, component, git_url, git_branch, update):
 912         """Check out a component from git.  Return a commit identifier."""
 913         if update:
 914             subprocess.run(['git', 'remote', 'prune', 'origin'],
 915                            cwd=self.component_srcdir(component), check=True)
 916             if self.replace_sources:
 917                 subprocess.run(['git', 'clean', '-dxfq'],
 918                                cwd=self.component_srcdir(component), check=True)
 919             subprocess.run(['git', 'pull', '-q'],
 920                            cwd=self.component_srcdir(component), check=True)
 921         else:
 922             if self.shallow:
 923                 depth_arg = ('--depth', '1')
 924             else:
 925                 depth_arg = ()
 926             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 927                             *depth_arg, git_url,
 928                             self.component_srcdir(component)], check=True)
 929         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 930                            cwd=self.component_srcdir(component),
 931                            stdout=subprocess.PIPE,
 932                            check=True, universal_newlines=True).stdout
 933         return r.rstrip()
 934
 935     def fix_glibc_timestamps(self):
 936         """Fix timestamps in a glibc checkout."""
 937         # Ensure that builds do not try to regenerate generated files
 938         # in the source tree.
 939         srcdir = self.component_srcdir('glibc')
 940         # These files have Makefile dependencies to regenerate them in
 941         # the source tree that may be active during a normal build.
 942         # Some other files have such dependencies but do not need to
 943         # be touched because nothing in a build depends on the files
 944         # in question.
 945         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 946             to_touch = os.path.join(srcdir, f)
 947             subprocess.run(['touch', '-c', to_touch], check=True)
 948         for dirpath, dirnames, filenames in os.walk(srcdir):
 949             for f in filenames:
 950                 if (f == 'configure' or
 951                     f == 'preconfigure' or
 952                     f.endswith('-kw.h')):
 953                     to_touch = os.path.join(dirpath, f)
 954                     subprocess.run(['touch', to_touch], check=True)
 955
 956     def gcc_checkout(self, branch, update):
 957         """Check out GCC from git.  Return the commit identifier."""
 958         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 959                      os.F_OK):
 960             if not self.replace_sources:
 961                 print('error: GCC has moved from SVN to git, use '
 962                       '--replace-sources to check out again')
 963                 exit(1)
 964             shutil.rmtree(self.component_srcdir('gcc'))
 965             update = False
 966         if not update:
 967             self.git_checkout('gcc', 'https://gcc.gnu.org/git/gcc.git',
 968                               branch, update)
 969         subprocess.run(['contrib/gcc_update', '--silent'],
 970                        cwd=self.component_srcdir('gcc'), check=True)
 971         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 972                            cwd=self.component_srcdir('gcc'),
 973                            stdout=subprocess.PIPE,
 974                            check=True, universal_newlines=True).stdout
 975         return r.rstrip()
 976
 977     def checkout_tar(self, component, version, update):
 978         """Check out the given version of the given component from a
 979         tarball."""
 980         if update:
 981             return
 982         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 983                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
 984                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 985                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
 986                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 987                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 988                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 989                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 990                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 991         if component not in url_map:
 992             print('error: component %s coming from tarball' % component)
 993             exit(1)
 994         version_major = version.split('.')[0]
 995         url = url_map[component] % {'version': version, 'major': version_major}
 996         filename = os.path.join(self.srcdir, url.split('/')[-1])
 997         response = urllib.request.urlopen(url)
 998         data = response.read()
 999         with open(filename, 'wb') as f:
1000             f.write(data)
1001         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
1002                        check=True)
1003         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
1004                   self.component_srcdir(component))
1005         os.remove(filename)
1006
1007     def load_build_state_json(self):
1008         """Load information about the state of previous builds."""
1009         if os.access(self.build_state_json, os.F_OK):
1010             with open(self.build_state_json, 'r') as f:
1011                 self.build_state = json.load(f)
1012         else:
1013             self.build_state = {}
1014         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
1015             if k not in self.build_state:
1016                 self.build_state[k] = {}
1017             if 'build-time' not in self.build_state[k]:
1018                 self.build_state[k]['build-time'] = ''
1019             if 'build-versions' not in self.build_state[k]:
1020                 self.build_state[k]['build-versions'] = {}
1021             if 'build-results' not in self.build_state[k]:
1022                 self.build_state[k]['build-results'] = {}
1023             if 'result-changes' not in self.build_state[k]:
1024                 self.build_state[k]['result-changes'] = {}
1025             if 'ever-passed' not in self.build_state[k]:
1026                 self.build_state[k]['ever-passed'] = []
1027
1028     def store_build_state_json(self):
1029         """Store information about the state of previous builds."""
1030         self.store_json(self.build_state, self.build_state_json)
1031
1032     def clear_last_build_state(self, action):
1033         """Clear information about the state of part of the build."""
1034         # We clear the last build time and versions when starting a
1035         # new build.  The results of the last build are kept around,
1036         # as comparison is still meaningful if this build is aborted
1037         # and a new one started.
1038         self.build_state[action]['build-time'] = ''
1039         self.build_state[action]['build-versions'] = {}
1040         self.store_build_state_json()
1041
1042     def update_build_state(self, action, build_time, build_versions):
1043         """Update the build state after a build."""
1044         build_time = build_time.replace(microsecond=0)
1045         self.build_state[action]['build-time'] = str(build_time)
1046         self.build_state[action]['build-versions'] = build_versions
1047         build_results = {}
1048         for log in self.status_log_list:
1049             with open(log, 'r') as f:
1050                 log_text = f.read()
1051             log_text = log_text.rstrip()
1052             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1053             result = m.group(1)
1054             test_name = m.group(2)
1055             assert test_name not in build_results
1056             build_results[test_name] = result
1057         old_build_results = self.build_state[action]['build-results']
1058         self.build_state[action]['build-results'] = build_results
1059         result_changes = {}
1060         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1061         for t in all_tests:
1062             if t in old_build_results:
1063                 old_res = old_build_results[t]
1064             else:
1065                 old_res = '(New test)'
1066             if t in build_results:
1067                 new_res = build_results[t]
1068             else:
1069                 new_res = '(Test removed)'
1070             if old_res != new_res:
1071                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1072         self.build_state[action]['result-changes'] = result_changes
1073         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1074                            if t in build_results}
1075         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1076         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1077                                                          new_passes)
1078         self.store_build_state_json()
1079
1080     def load_bot_config_json(self):
1081         """Load bot configuration."""
1082         with open(self.bot_config_json, 'r') as f:
1083             self.bot_config = json.load(f)
1084
1085     def part_build_old(self, action, delay):
1086         """Return whether the last build for a given action was at least a
1087         given number of seconds ago, or does not have a time recorded."""
1088         old_time_str = self.build_state[action]['build-time']
1089         if not old_time_str:
1090             return True
1091         old_time = datetime.datetime.strptime(old_time_str,
1092                                               '%Y-%m-%d %H:%M:%S')
1093         new_time = datetime.datetime.utcnow()
1094         delta = new_time - old_time
1095         return delta.total_seconds() >= delay
1096
1097     def bot_cycle(self):
1098         """Run a single round of checkout and builds."""
1099         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1100         self.load_bot_config_json()
1101         actions = ('host-libraries', 'compilers', 'glibcs')
1102         self.bot_run_self(['--replace-sources'], 'checkout')
1103         self.load_versions_json()
1104         if self.get_script_text() != self.script_text:
1105             print('Script changed, re-execing.')
1106             # On script change, all parts of the build should be rerun.
1107             for a in actions:
1108                 self.clear_last_build_state(a)
1109             self.exec_self()
1110         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1111                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1112                                           'mig', 'gnumach', 'hurd'),
1113                             'glibcs': ('glibc',)}
1114         must_build = {}
1115         for a in actions:
1116             build_vers = self.build_state[a]['build-versions']
1117             must_build[a] = False
1118             if not self.build_state[a]['build-time']:
1119                 must_build[a] = True
1120             old_vers = {}
1121             new_vers = {}
1122             for c in check_components[a]:
1123                 if c in build_vers:
1124                     old_vers[c] = build_vers[c]
1125                 new_vers[c] = {'version': self.versions[c]['version'],
1126                                'revision': self.versions[c]['revision']}
1127             if new_vers == old_vers:
1128                 print('Versions for %s unchanged.' % a)
1129             else:
1130                 print('Versions changed or rebuild forced for %s.' % a)
1131                 if a == 'compilers' and not self.part_build_old(
1132                         a, self.bot_config['compilers-rebuild-delay']):
1133                     print('Not requiring rebuild of compilers this soon.')
1134                 else:
1135                     must_build[a] = True
1136         if must_build['host-libraries']:
1137             must_build['compilers'] = True
1138         if must_build['compilers']:
1139             must_build['glibcs'] = True
1140         for a in actions:
1141             if must_build[a]:
1142                 print('Must rebuild %s.' % a)
1143                 self.clear_last_build_state(a)
1144             else:
1145                 print('No need to rebuild %s.' % a)
1146         if os.access(self.logsdir, os.F_OK):
1147             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1148             shutil.copytree(self.logsdir, self.logsdir_old)
1149         for a in actions:
1150             if must_build[a]:
1151                 build_time = datetime.datetime.utcnow()
1152                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1153                 self.bot_run_self([], a)
1154                 self.load_build_state_json()
1155                 self.bot_build_mail(a, build_time)
1156         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1157
1158     def bot_build_mail(self, action, build_time):
1159         """Send email with the results of a build."""
1160         if not ('email-from' in self.bot_config and
1161                 'email-server' in self.bot_config and
1162                 'email-subject' in self.bot_config and
1163                 'email-to' in self.bot_config):
1164             if not self.email_warning:
1165                 print("Email not configured, not sending.")
1166                 self.email_warning = True
1167             return
1168
1169         build_time = build_time.replace(microsecond=0)
1170         subject = (self.bot_config['email-subject'] %
1171                    {'action': action,
1172                     'build-time': str(build_time)})
1173         results = self.build_state[action]['build-results']
1174         changes = self.build_state[action]['result-changes']
1175         ever_passed = set(self.build_state[action]['ever-passed'])
1176         versions = self.build_state[action]['build-versions']
1177         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1178         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1179         all_fails = {k for k in results if results[k] == 'FAIL'}
1180         if new_regressions:
1181             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1182             new_reg_text = ('New regressions:\n\n%s\n\n' %
1183                             '\n'.join(new_reg_list))
1184         else:
1185             new_reg_text = ''
1186         if all_regressions:
1187             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1188             all_reg_text = ('All regressions:\n\n%s\n\n' %
1189                             '\n'.join(all_reg_list))
1190         else:
1191             all_reg_text = ''
1192         if all_fails:
1193             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1194             all_fail_text = ('All failures:\n\n%s\n\n' %
1195                              '\n'.join(all_fail_list))
1196         else:
1197             all_fail_text = ''
1198         if changes:
1199             changes_list = sorted(changes.keys())
1200             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1201             changes_text = ('All changed results:\n\n%s\n\n' %
1202                             '\n'.join(changes_list))
1203         else:
1204             changes_text = ''
1205         results_text = (new_reg_text + all_reg_text + all_fail_text +
1206                         changes_text)
1207         if not results_text:
1208             results_text = 'Clean build with unchanged results.\n\n'
1209         versions_list = sorted(versions.keys())
1210         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1211                                           versions[k]['revision'])
1212                          for k in versions_list]
1213         versions_text = ('Component versions for this build:\n\n%s\n' %
1214                          '\n'.join(versions_list))
1215         body_text = results_text + versions_text
1216         msg = email.mime.text.MIMEText(body_text)
1217         msg['Subject'] = subject
1218         msg['From'] = self.bot_config['email-from']
1219         msg['To'] = self.bot_config['email-to']
1220         msg['Message-ID'] = email.utils.make_msgid()
1221         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1222         with smtplib.SMTP(self.bot_config['email-server']) as s:
1223             s.send_message(msg)
1224
1225     def bot_run_self(self, opts, action, check=True):
1226         """Run a copy of this script with given options."""
1227         cmd = [sys.executable, sys.argv[0], '--keep=none',
1228                '-j%d' % self.parallelism]
1229         if self.full_gcc:
1230             cmd.append('--full-gcc')
1231         cmd.extend(opts)
1232         cmd.extend([self.topdir, action])
1233         sys.stdout.flush()
1234         subprocess.run(cmd, check=check)
1235
1236     def bot(self):
1237         """Run repeated rounds of checkout and builds."""
1238         while True:
1239             self.load_bot_config_json()
1240             if not self.bot_config['run']:
1241                 print('Bot exiting by request.')
1242                 exit(0)
1243             self.bot_run_self([], 'bot-cycle', check=False)
1244             self.load_bot_config_json()
1245             if not self.bot_config['run']:
1246                 print('Bot exiting by request.')
1247                 exit(0)
1248             time.sleep(self.bot_config['delay'])
1249             if self.get_script_text() != self.script_text:
1250                 print('Script changed, bot re-execing.')
1251                 self.exec_self()
1252
1253 class LinuxHeadersPolicyForBuild(object):
1254     """Names and directories for installing Linux headers.  Build variant."""
1255
1256     def __init__(self, config):
1257         self.arch = config.arch
1258         self.srcdir = config.ctx.component_srcdir('linux')
1259         self.builddir = config.component_builddir('linux')
1260         self.headers_dir = os.path.join(config.sysroot, 'usr')
1261
1262 class LinuxHeadersPolicyForUpdateSyscalls(object):
1263     """Names and directories for Linux headers.  update-syscalls variant."""
1264
1265     def __init__(self, glibc, headers_dir):
1266         self.arch = glibc.compiler.arch
1267         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1268         self.builddir = glibc.ctx.component_builddir(
1269             'update-syscalls', glibc.name, 'build-linux')
1270         self.headers_dir = headers_dir
1271
1272 def install_linux_headers(policy, cmdlist):
1273     """Install Linux kernel headers."""
1274     arch_map = {'aarch64': 'arm64',
1275                 'alpha': 'alpha',
1276                 'arc': 'arc',
1277                 'arm': 'arm',
1278                 'csky': 'csky',
1279                 'hppa': 'parisc',
1280                 'i486': 'x86',
1281                 'i586': 'x86',
1282                 'i686': 'x86',
1283                 'i786': 'x86',
1284                 'ia64': 'ia64',
1285                 'loongarch64': 'loongarch',
1286                 'm68k': 'm68k',
1287                 'microblaze': 'microblaze',
1288                 'mips': 'mips',
1289                 'nios2': 'nios2',
1290                 'or1k': 'openrisc',
1291                 'powerpc': 'powerpc',
1292                 's390': 's390',
1293                 'riscv32': 'riscv',
1294                 'riscv64': 'riscv',
1295                 'sh': 'sh',
1296                 'sparc': 'sparc',
1297                 'x86_64': 'x86'}
1298     linux_arch = None
1299     for k in arch_map:
1300         if policy.arch.startswith(k):
1301             linux_arch = arch_map[k]
1302             break
1303     assert linux_arch is not None
1304     cmdlist.push_subdesc('linux')
1305     cmdlist.create_use_dir(policy.builddir)
1306     cmdlist.add_command('install-headers',
1307                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1308                          'ARCH=%s' % linux_arch,
1309                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1310                          'headers_install'])
1311     cmdlist.cleanup_dir()
1312     cmdlist.pop_subdesc()
1313
1314 class Config(object):
1315     """A configuration for building a compiler and associated libraries."""
1316
1317     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1318                  first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1319                  extra_glibcs=None):
1320         """Initialize a Config object."""
1321         self.ctx = ctx
1322         self.arch = arch
1323         self.os = os_name
1324         self.variant = variant
1325         if variant is None:
1326             self.name = '%s-%s' % (arch, os_name)
1327         else:
1328             self.name = '%s-%s-%s' % (arch, os_name, variant)
1329         self.triplet = '%s-glibc-%s' % (arch, os_name)
1330         if gcc_cfg is None:
1331             self.gcc_cfg = []
1332         else:
1333             self.gcc_cfg = gcc_cfg
1334         if first_gcc_cfg is None:
1335             self.first_gcc_cfg = []
1336         else:
1337             self.first_gcc_cfg = first_gcc_cfg
1338         if binutils_cfg is None:
1339             self.binutils_cfg = []
1340         else:
1341             self.binutils_cfg = binutils_cfg
1342         if glibcs is None:
1343             glibcs = [{'variant': variant}]
1344         if extra_glibcs is None:
1345             extra_glibcs = []
1346         glibcs = [Glibc(self, **g) for g in glibcs]
1347         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1348         self.all_glibcs = glibcs + extra_glibcs
1349         self.compiler_glibcs = glibcs
1350         self.installdir = ctx.compiler_installdir(self.name)
1351         self.bindir = ctx.compiler_bindir(self.name)
1352         self.sysroot = ctx.compiler_sysroot(self.name)
1353         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1354         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1355
1356     def component_builddir(self, component):
1357         """Return the directory to use for a (non-glibc) build."""
1358         return self.ctx.component_builddir('compilers', self.name, component)
1359
1360     def build(self):
1361         """Generate commands to build this compiler."""
1362         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1363                                       self.logsdir)
1364         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1365         cmdlist.add_command('check-host-libraries',
1366                             ['test', '-f',
1367                              os.path.join(self.ctx.host_libraries_installdir,
1368                                           'ok')])
1369         cmdlist.use_path(self.bindir)
1370         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1371                               ['--disable-gdb',
1372                                '--disable-gdbserver',
1373                                '--disable-libdecnumber',
1374                                '--disable-readline',
1375                                '--disable-sim'] + self.binutils_cfg)
1376         if self.os.startswith('linux'):
1377             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1378         self.build_gcc(cmdlist, True)
1379         if self.os == 'gnu':
1380             self.install_gnumach_headers(cmdlist)
1381             self.build_cross_tool(cmdlist, 'mig', 'mig')
1382             self.install_hurd_headers(cmdlist)
1383         for g in self.compiler_glibcs:
1384             cmdlist.push_subdesc('glibc')
1385             cmdlist.push_subdesc(g.name)
1386             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1387             cmdlist.pop_subdesc()
1388             cmdlist.pop_subdesc()
1389         self.build_gcc(cmdlist, False)
1390         cmdlist.add_command('done', ['touch',
1391                                      os.path.join(self.installdir, 'ok')])
1392         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1393                                       self.logsdir)
1394
1395     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1396         """Build one cross tool."""
1397         srcdir = self.ctx.component_srcdir(tool_src)
1398         builddir = self.component_builddir(tool_build)
1399         cmdlist.push_subdesc(tool_build)
1400         cmdlist.create_use_dir(builddir)
1401         cfg_cmd = [os.path.join(srcdir, 'configure'),
1402                    '--prefix=%s' % self.installdir,
1403                    '--build=%s' % self.ctx.build_triplet,
1404                    '--host=%s' % self.ctx.build_triplet,
1405                    '--target=%s' % self.triplet,
1406                    '--with-sysroot=%s' % self.sysroot]
1407         if extra_opts:
1408             cfg_cmd.extend(extra_opts)
1409         cmdlist.add_command('configure', cfg_cmd)
1410         cmdlist.add_command('build', ['make'])
1411         # Parallel "make install" for GCC has race conditions that can
1412         # cause it to fail; see
1413         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1414         # problems are not known for binutils, but doing the
1415         # installation in parallel within a particular toolchain build
1416         # (as opposed to installation of one toolchain from
1417         # build-many-glibcs.py running in parallel to the installation
1418         # of other toolchains being built) is not known to be
1419         # significantly beneficial, so it is simplest just to disable
1420         # parallel install for cross tools here.
1421         cmdlist.add_command('install', ['make', '-j1', 'install'])
1422         cmdlist.cleanup_dir()
1423         cmdlist.pop_subdesc()
1424
1425     def install_gnumach_headers(self, cmdlist):
1426         """Install GNU Mach headers."""
1427         srcdir = self.ctx.component_srcdir('gnumach')
1428         builddir = self.component_builddir('gnumach')
1429         cmdlist.push_subdesc('gnumach')
1430         cmdlist.create_use_dir(builddir)
1431         cmdlist.add_command('configure',
1432                             [os.path.join(srcdir, 'configure'),
1433                              '--build=%s' % self.ctx.build_triplet,
1434                              '--host=%s' % self.triplet,
1435                              '--prefix=',
1436                              '--disable-user32',
1437                              'CC=%s-gcc -nostdlib' % self.triplet])
1438         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1439                                         'install-data'])
1440         cmdlist.cleanup_dir()
1441         cmdlist.pop_subdesc()
1442
1443     def install_hurd_headers(self, cmdlist):
1444         """Install Hurd headers."""
1445         srcdir = self.ctx.component_srcdir('hurd')
1446         builddir = self.component_builddir('hurd')
1447         cmdlist.push_subdesc('hurd')
1448         cmdlist.create_use_dir(builddir)
1449         cmdlist.add_command('configure',
1450                             [os.path.join(srcdir, 'configure'),
1451                              '--build=%s' % self.ctx.build_triplet,
1452                              '--host=%s' % self.triplet,
1453                              '--prefix=',
1454                              '--disable-profile', '--without-parted',
1455                              'CC=%s-gcc -nostdlib' % self.triplet])
1456         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1457                                         'no_deps=t', 'install-headers'])
1458         cmdlist.cleanup_dir()
1459         cmdlist.pop_subdesc()
1460
1461     def build_gcc(self, cmdlist, bootstrap):
1462         """Build GCC."""
1463         # libssp is of little relevance with glibc's own stack
1464         # checking support.  libcilkrts does not support GNU/Hurd (and
1465         # has been removed in GCC 8, so --disable-libcilkrts can be
1466         # removed once glibc no longer supports building with older
1467         # GCC versions).  --enable-initfini-array is enabled by default
1468         # in GCC 12, which can be removed when GCC 12 becomes the
1469         # minimum requirement.
1470         cfg_opts = list(self.gcc_cfg)
1471         cfg_opts += ['--enable-initfini-array']
1472         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1473         host_libs = self.ctx.host_libraries_installdir
1474         cfg_opts += ['--with-gmp=%s' % host_libs,
1475                      '--with-mpfr=%s' % host_libs,
1476                      '--with-mpc=%s' % host_libs]
1477         if bootstrap:
1478             tool_build = 'gcc-first'
1479             # Building a static-only, C-only compiler that is
1480             # sufficient to build glibc.  Various libraries and
1481             # features that may require libc headers must be disabled.
1482             # When configuring with a sysroot, --with-newlib is
1483             # required to define inhibit_libc (to stop some parts of
1484             # libgcc including libc headers); --without-headers is not
1485             # sufficient.
1486             cfg_opts += ['--enable-languages=c', '--disable-shared',
1487                          '--disable-threads',
1488                          '--disable-libatomic',
1489                          '--disable-decimal-float',
1490                          '--disable-gcov',
1491                          '--disable-libffi',
1492                          '--disable-libgomp',
1493                          '--disable-libitm',
1494                          '--disable-libmpx',
1495                          '--disable-libquadmath',
1496                          '--disable-libsanitizer',
1497                          '--without-headers', '--with-newlib',
1498                          '--with-glibc-version=%s' % self.ctx.glibc_version
1499                          ]
1500             cfg_opts += self.first_gcc_cfg
1501         else:
1502             tool_build = 'gcc'
1503             # libsanitizer commonly breaks because of glibc header
1504             # changes, or on unusual targets.  C++ pre-compiled
1505             # headers are not used during the glibc build and are
1506             # expensive to create.
1507             if not self.ctx.full_gcc:
1508                 cfg_opts += ['--disable-libsanitizer',
1509                              '--disable-libstdcxx-pch']
1510             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1511             cfg_opts += ['--enable-languages=%s' % langs,
1512                          '--enable-shared', '--enable-threads']
1513         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1514
1515 class GlibcPolicyDefault(object):
1516     """Build policy for glibc: common defaults."""
1517
1518     def __init__(self, glibc):
1519         self.srcdir = glibc.ctx.component_srcdir('glibc')
1520         self.use_usr = glibc.os != 'gnu'
1521         self.prefix = '/usr' if self.use_usr else ''
1522         self.configure_args = [
1523             '--prefix=%s' % self.prefix,
1524             '--enable-profile',
1525             '--build=%s' % glibc.ctx.build_triplet,
1526             '--host=%s' % glibc.triplet,
1527             'CC=%s' % glibc.tool_name('gcc'),
1528             'CXX=%s' % glibc.tool_name('g++'),
1529         ]
1530         if glibc.os == 'gnu':
1531             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1532         if glibc.cflags:
1533             self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1534             self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
1535         self.configure_args += glibc.cfg
1536
1537     def configure(self, cmdlist):
1538         """Invoked to add the configure command to the command list."""
1539         cmdlist.add_command('configure',
1540                             [os.path.join(self.srcdir, 'configure'),
1541                              *self.configure_args])
1542
1543     def extra_commands(self, cmdlist):
1544         """Invoked to inject additional commands (make check) after build."""
1545         pass
1546
1547 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1548     """Build policy for glibc during the compilers stage."""
1549
1550     def __init__(self, glibc):
1551         super().__init__(glibc)
1552         self.builddir = glibc.ctx.component_builddir(
1553             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1554         self.installdir = glibc.compiler.sysroot
1555
1556 class GlibcPolicyForBuild(GlibcPolicyDefault):
1557     """Build policy for glibc during the glibcs stage."""
1558
1559     def __init__(self, glibc):
1560         super().__init__(glibc)
1561         self.builddir = glibc.ctx.component_builddir(
1562             'glibcs', glibc.name, 'glibc')
1563         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1564         if glibc.ctx.strip:
1565             self.strip = glibc.tool_name('strip')
1566         else:
1567             self.strip = None
1568         self.save_logs = glibc.ctx.save_logs
1569
1570     def extra_commands(self, cmdlist):
1571         if self.strip:
1572             # Avoid stripping libc.so and libpthread.so, which are
1573             # linker scripts stored in /lib on Hurd.
1574             find_command = 'find %s/lib* -name "*.so*"' % self.installdir
1575             cmdlist.add_command('strip', ['sh', '-c', (
1576                 'set -e; for f in $(%s); do '
1577                 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1578                 'done' % (find_command, self.strip))])
1579         cmdlist.add_command('check', ['make', 'check'])
1580         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1581
1582 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1583     """Build policy for glibc during update-syscalls."""
1584
1585     def __init__(self, glibc):
1586         super().__init__(glibc)
1587         self.builddir = glibc.ctx.component_builddir(
1588             'update-syscalls', glibc.name, 'glibc')
1589         self.linuxdir = glibc.ctx.component_builddir(
1590             'update-syscalls', glibc.name, 'linux')
1591         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1592             glibc, self.linuxdir)
1593         self.configure_args.insert(
1594             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1595         # self.installdir not set because installation is not supported
1596
1597 class Glibc(object):
1598     """A configuration for building glibc."""
1599
1600     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1601                  cfg=None, ccopts=None, cflags=None):
1602         """Initialize a Glibc object."""
1603         self.ctx = compiler.ctx
1604         self.compiler = compiler
1605         if arch is None:
1606             self.arch = compiler.arch
1607         else:
1608             self.arch = arch
1609         if os_name is None:
1610             self.os = compiler.os
1611         else:
1612             self.os = os_name
1613         self.variant = variant
1614         if variant is None:
1615             self.name = '%s-%s' % (self.arch, self.os)
1616         else:
1617             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1618         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1619         if cfg is None:
1620             self.cfg = []
1621         else:
1622             self.cfg = cfg
1623         # ccopts contain ABI options and are passed to configure as CC / CXX.
1624         self.ccopts = ccopts
1625         # cflags contain non-ABI options like -g or -O and are passed to
1626         # configure as CFLAGS / CXXFLAGS.
1627         self.cflags = cflags
1628
1629     def tool_name(self, tool):
1630         """Return the name of a cross-compilation tool."""
1631         ctool = '%s-%s' % (self.compiler.triplet, tool)
1632         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1633             ctool = '%s %s' % (ctool, self.ccopts)
1634         return ctool
1635
1636     def build(self):
1637         """Generate commands to build this glibc."""
1638         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1639         installdir = self.ctx.glibc_installdir(self.name)
1640         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1641         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1642         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1643         cmdlist.add_command('check-compilers',
1644                             ['test', '-f',
1645                              os.path.join(self.compiler.installdir, 'ok')])
1646         cmdlist.use_path(self.compiler.bindir)
1647         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1648         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1649                                       logsdir)
1650
1651     def build_glibc(self, cmdlist, policy):
1652         """Generate commands to build this glibc, either as part of a compiler
1653         build or with the bootstrapped compiler (and in the latter case, run
1654         tests as well)."""
1655         cmdlist.create_use_dir(policy.builddir)
1656         policy.configure(cmdlist)
1657         cmdlist.add_command('build', ['make'])
1658         cmdlist.add_command('install', ['make', 'install',
1659                                         'install_root=%s' % policy.installdir])
1660         # GCC uses paths such as lib/../lib64, so make sure lib
1661         # directories always exist.
1662         mkdir_cmd = ['mkdir', '-p',
1663                      os.path.join(policy.installdir, 'lib')]
1664         if policy.use_usr:
1665             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1666         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1667         policy.extra_commands(cmdlist)
1668         cmdlist.cleanup_dir()
1669
1670     def update_syscalls(self):
1671         if self.os == 'gnu':
1672             # Hurd does not have system call tables that need updating.
1673             return
1674
1675         policy = GlibcPolicyForUpdateSyscalls(self)
1676         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1677         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1678         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1679         cmdlist.add_command('check-compilers',
1680                             ['test', '-f',
1681                              os.path.join(self.compiler.installdir, 'ok')])
1682         cmdlist.use_path(self.compiler.bindir)
1683
1684         install_linux_headers(policy.linux_policy, cmdlist)
1685
1686         cmdlist.create_use_dir(policy.builddir)
1687         policy.configure(cmdlist)
1688         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1689         cmdlist.cleanup_dir()
1690         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1691                                       cmdlist, logsdir)
1692
1693 class Command(object):
1694     """A command run in the build process."""
1695
1696     def __init__(self, desc, num, dir, path, command, always_run=False):
1697         """Initialize a Command object."""
1698         self.dir = dir
1699         self.path = path
1700         self.desc = desc
1701         trans = str.maketrans({' ': '-'})
1702         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1703         self.command = command
1704         self.always_run = always_run
1705
1706     @staticmethod
1707     def shell_make_quote_string(s):
1708         """Given a string not containing a newline, quote it for use by the
1709         shell and make."""
1710         assert '\n' not in s
1711         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1712             return s
1713         strans = str.maketrans({"'": "'\\''"})
1714         s = "'%s'" % s.translate(strans)
1715         mtrans = str.maketrans({'$': '$$'})
1716         return s.translate(mtrans)
1717
1718     @staticmethod
1719     def shell_make_quote_list(l, translate_make):
1720         """Given a list of strings not containing newlines, quote them for use
1721         by the shell and make, returning a single string.  If translate_make
1722         is true and the first string is 'make', change it to $(MAKE)."""
1723         l = [Command.shell_make_quote_string(s) for s in l]
1724         if translate_make and l[0] == 'make':
1725             l[0] = '$(MAKE)'
1726         return ' '.join(l)
1727
1728     def shell_make_quote(self):
1729         """Return this command quoted for the shell and make."""
1730         return self.shell_make_quote_list(self.command, True)
1731
1732
1733 class CommandList(object):
1734     """A list of commands run in the build process."""
1735
1736     def __init__(self, desc, keep):
1737         """Initialize a CommandList object."""
1738         self.cmdlist = []
1739         self.dir = None
1740         self.path = None
1741         self.desc = [desc]
1742         self.keep = keep
1743
1744     def desc_txt(self, desc):
1745         """Return the description to use for a command."""
1746         return '%s %s' % (' '.join(self.desc), desc)
1747
1748     def use_dir(self, dir):
1749         """Set the default directory for subsequent commands."""
1750         self.dir = dir
1751
1752     def use_path(self, path):
1753         """Set a directory to be prepended to the PATH for subsequent
1754         commands."""
1755         self.path = path
1756
1757     def push_subdesc(self, subdesc):
1758         """Set the default subdescription for subsequent commands (e.g., the
1759         name of a component being built, within the series of commands
1760         building it)."""
1761         self.desc.append(subdesc)
1762
1763     def pop_subdesc(self):
1764         """Pop a subdescription from the list of descriptions."""
1765         self.desc.pop()
1766
1767     def create_use_dir(self, dir):
1768         """Remove and recreate a directory and use it for subsequent
1769         commands."""
1770         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1771         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1772         self.use_dir(dir)
1773
1774     def add_command_dir(self, desc, dir, command, always_run=False):
1775         """Add a command to run in a given directory."""
1776         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1777                       command, always_run)
1778         self.cmdlist.append(cmd)
1779
1780     def add_command(self, desc, command, always_run=False):
1781         """Add a command to run in the default directory."""
1782         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1783                       self.path, command, always_run)
1784         self.cmdlist.append(cmd)
1785
1786     def cleanup_dir(self, desc='cleanup', dir=None):
1787         """Clean up a build directory.  If no directory is specified, the
1788         default directory is cleaned up and ceases to be the default
1789         directory."""
1790         if dir is None:
1791             dir = self.dir
1792             self.use_dir(None)
1793         if self.keep != 'all':
1794             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1795                                  always_run=(self.keep == 'none'))
1796
1797     def makefile_commands(self, wrapper, logsdir):
1798         """Return the sequence of commands in the form of text for a Makefile.
1799         The given wrapper script takes arguments: base of logs for
1800         previous command, or empty; base of logs for this command;
1801         description; directory; PATH addition; the command itself."""
1802         # prev_base is the base of the name for logs of the previous
1803         # command that is not always-run (that is, a build command,
1804         # whose failure should stop subsequent build commands from
1805         # being run, as opposed to a cleanup command, which is run
1806         # even if previous commands failed).
1807         prev_base = ''
1808         cmds = []
1809         for c in self.cmdlist:
1810             ctxt = c.shell_make_quote()
1811             if prev_base and not c.always_run:
1812                 prev_log = os.path.join(logsdir, prev_base)
1813             else:
1814                 prev_log = ''
1815             this_log = os.path.join(logsdir, c.logbase)
1816             if not c.always_run:
1817                 prev_base = c.logbase
1818             if c.dir is None:
1819                 dir = ''
1820             else:
1821                 dir = c.dir
1822             if c.path is None:
1823                 path = ''
1824             else:
1825                 path = c.path
1826             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1827             prelim_txt = Command.shell_make_quote_list(prelims, False)
1828             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1829         return '\n'.join(cmds)
1830
1831     def status_logs(self, logsdir):
1832         """Return the list of log files with command status."""
1833         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1834                 for c in self.cmdlist]
1835
1836
1837 def get_parser():
1838     """Return an argument parser for this module."""
1839     parser = argparse.ArgumentParser(description=__doc__)
1840     parser.add_argument('-j', dest='parallelism',
1841                         help='Run this number of jobs in parallel',
1842                         type=int, default=os.cpu_count())
1843     parser.add_argument('--keep', dest='keep',
1844                         help='Whether to keep all build directories, '
1845                         'none or only those from failed builds',
1846                         default='none', choices=('none', 'all', 'failed'))
1847     parser.add_argument('--replace-sources', action='store_true',
1848                         help='Remove and replace source directories '
1849                         'with the wrong version of a component')
1850     parser.add_argument('--strip', action='store_true',
1851                         help='Strip installed glibc libraries')
1852     parser.add_argument('--full-gcc', action='store_true',
1853                         help='Build GCC with all languages and libsanitizer')
1854     parser.add_argument('--shallow', action='store_true',
1855                         help='Do not download Git history during checkout')
1856     parser.add_argument('topdir',
1857                         help='Toplevel working directory')
1858     parser.add_argument('action',
1859                         help='What to do',
1860                         choices=('checkout', 'bot-cycle', 'bot',
1861                                  'host-libraries', 'compilers', 'glibcs',
1862                                  'update-syscalls', 'list-compilers',
1863                                  'list-glibcs'))
1864     parser.add_argument('configs',
1865                         help='Versions to check out or configurations to build',
1866                         nargs='*')
1867     return parser
1868
1869
1870 def main(argv):
1871     """The main entry point."""
1872     parser = get_parser()
1873     opts = parser.parse_args(argv)
1874     topdir = os.path.abspath(opts.topdir)
1875     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1876                   opts.strip, opts.full_gcc, opts.action,
1877                   shallow=opts.shallow)
1878     ctx.run_builds(opts.action, opts.configs)
1879
1880
1881 if __name__ == '__main__':
1882     main(sys.argv[1:])