scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2023 Free Software Foundation, Inc.
   4 # Copyright The GNU Toolchain Authors.
   5 # This file is part of the GNU C Library.
   6 #
   7 # The GNU C Library is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU Lesser General Public
   9 # License as published by the Free Software Foundation; either
  10 # version 2.1 of the License, or (at your option) any later version.
  11 #
  12 # The GNU C Library is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 # Lesser General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Lesser General Public
  18 # License along with the GNU C Library; if not, see
  19 # <https://www.gnu.org/licenses/>.
  20
  21 """Build many configurations of glibc.
  22
  23 This script takes as arguments a directory name (containing a src
  24 subdirectory with sources of the relevant toolchain components) and a
  25 description of what to do: 'checkout', to check out sources into that
  26 directory, 'bot-cycle', to run a series of checkout and build steps,
  27 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  28 libraries required by the toolchain, 'compilers', to build
  29 cross-compilers for various configurations, or 'glibcs', to build
  30 glibc for various configurations and run the compilation parts of the
  31 testsuite.  Subsequent arguments name the versions of components to
  32 check out (<component>-<version), for 'checkout', or, for actions
  33 other than 'checkout' and 'bot-cycle', name configurations for which
  34 compilers or glibc are to be built.
  35
  36 The 'list-compilers' command prints the name of each available
  37 compiler configuration, without building anything.  The 'list-glibcs'
  38 command prints the name of each glibc compiler configuration, followed
  39 by the space, followed by the name of the compiler configuration used
  40 for building this glibc variant.
  41
  42 """
  43
  44 import argparse
  45 import datetime
  46 import email.mime.text
  47 import email.utils
  48 import json
  49 import os
  50 import re
  51 import shutil
  52 import smtplib
  53 import stat
  54 import subprocess
  55 import sys
  56 import time
  57 import urllib.request
  58
  59 try:
  60     subprocess.run
  61 except:
  62     class _CompletedProcess:
  63         def __init__(self, args, returncode, stdout=None, stderr=None):
  64             self.args = args
  65             self.returncode = returncode
  66             self.stdout = stdout
  67             self.stderr = stderr
  68
  69     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  70         assert(timeout is None)
  71         with subprocess.Popen(*popenargs, **kwargs) as process:
  72             try:
  73                 stdout, stderr = process.communicate(input)
  74             except:
  75                 process.kill()
  76                 process.wait()
  77                 raise
  78             returncode = process.poll()
  79             if check and returncode:
  80                 raise subprocess.CalledProcessError(returncode, popenargs)
  81         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  82
  83     subprocess.run = _run
  84
  85
  86 class Context(object):
  87     """The global state associated with builds in a given directory."""
  88
  89     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  90                  full_gcc, action, shallow=False):
  91         """Initialize the context."""
  92         self.topdir = topdir
  93         self.parallelism = parallelism
  94         self.keep = keep
  95         self.replace_sources = replace_sources
  96         self.strip = strip
  97         self.full_gcc = full_gcc
  98         self.shallow = shallow
  99         self.srcdir = os.path.join(topdir, 'src')
 100         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 101         self.build_state_json = os.path.join(topdir, 'build-state.json')
 102         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 103         self.installdir = os.path.join(topdir, 'install')
 104         self.host_libraries_installdir = os.path.join(self.installdir,
 105                                                       'host-libraries')
 106         self.builddir = os.path.join(topdir, 'build')
 107         self.logsdir = os.path.join(topdir, 'logs')
 108         self.logsdir_old = os.path.join(topdir, 'logs-old')
 109         self.makefile = os.path.join(self.builddir, 'Makefile')
 110         self.wrapper = os.path.join(self.builddir, 'wrapper')
 111         self.save_logs = os.path.join(self.builddir, 'save-logs')
 112         self.script_text = self.get_script_text()
 113         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 114             self.build_triplet = self.get_build_triplet()
 115             self.glibc_version = self.get_glibc_version()
 116         self.configs = {}
 117         self.glibc_configs = {}
 118         self.makefile_pieces = ['.PHONY: all\n']
 119         self.add_all_configs()
 120         self.load_versions_json()
 121         self.load_build_state_json()
 122         self.status_log_list = []
 123         self.email_warning = False
 124
 125     def get_script_text(self):
 126         """Return the text of this script."""
 127         with open(sys.argv[0], 'r') as f:
 128             return f.read()
 129
 130     def exec_self(self):
 131         """Re-execute this script with the same arguments."""
 132         sys.stdout.flush()
 133         os.execv(sys.executable, [sys.executable] + sys.argv)
 134
 135     def get_build_triplet(self):
 136         """Determine the build triplet with config.guess."""
 137         config_guess = os.path.join(self.component_srcdir('gcc'),
 138                                     'config.guess')
 139         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 140                                 check=True, universal_newlines=True).stdout
 141         return cg_out.rstrip()
 142
 143     def get_glibc_version(self):
 144         """Determine the glibc version number (major.minor)."""
 145         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 146         with open(version_h, 'r') as f:
 147             lines = f.readlines()
 148         starttext = '#define VERSION "'
 149         for l in lines:
 150             if l.startswith(starttext):
 151                 l = l[len(starttext):]
 152                 l = l.rstrip('"\n')
 153                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 154                 return '%s.%s' % m.group(1, 2)
 155         print('error: could not determine glibc version')
 156         exit(1)
 157
 158     def add_all_configs(self):
 159         """Add all known glibc build configurations."""
 160         self.add_config(arch='aarch64',
 161                         os_name='linux-gnu',
 162                         extra_glibcs=[{'variant': 'disable-multi-arch',
 163                                        'cfg': ['--disable-multi-arch']}])
 164         self.add_config(arch='aarch64_be',
 165                         os_name='linux-gnu')
 166         self.add_config(arch='arc',
 167                         os_name='linux-gnu',
 168                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 169         self.add_config(arch='arc',
 170                         os_name='linux-gnuhf',
 171                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
 172         self.add_config(arch='arceb',
 173                         os_name='linux-gnu',
 174                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 175         self.add_config(arch='alpha',
 176                         os_name='linux-gnu')
 177         self.add_config(arch='arm',
 178                         os_name='linux-gnueabi',
 179                         extra_glibcs=[{'variant': 'v4t',
 180                                        'ccopts': '-march=armv4t'}])
 181         self.add_config(arch='armeb',
 182                         os_name='linux-gnueabi')
 183         self.add_config(arch='armeb',
 184                         os_name='linux-gnueabi',
 185                         variant='be8',
 186                         gcc_cfg=['--with-arch=armv7-a'])
 187         self.add_config(arch='arm',
 188                         os_name='linux-gnueabihf',
 189                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 190                         extra_glibcs=[{'variant': 'v7a',
 191                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 192                                       {'variant': 'thumb',
 193                                        'ccopts':
 194                                        '-mthumb -march=armv7-a -mfpu=vfpv3'},
 195                                       {'variant': 'v7a-disable-multi-arch',
 196                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 197                                        'cfg': ['--disable-multi-arch']}])
 198         self.add_config(arch='armeb',
 199                         os_name='linux-gnueabihf',
 200                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 201         self.add_config(arch='armeb',
 202                         os_name='linux-gnueabihf',
 203                         variant='be8',
 204                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 205                                  '--with-fpu=vfpv3'])
 206         self.add_config(arch='csky',
 207                         os_name='linux-gnuabiv2',
 208                         variant='soft',
 209                         gcc_cfg=['--disable-multilib'])
 210         self.add_config(arch='csky',
 211                         os_name='linux-gnuabiv2',
 212                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 213         self.add_config(arch='hppa',
 214                         os_name='linux-gnu')
 215         self.add_config(arch='i686',
 216                         os_name='gnu')
 217         self.add_config(arch='ia64',
 218                         os_name='linux-gnu',
 219                         first_gcc_cfg=['--with-system-libunwind'],
 220                         binutils_cfg=['--enable-obsolete'])
 221         self.add_config(arch='loongarch64',
 222                         os_name='linux-gnu',
 223                         variant='lp64d',
 224                         gcc_cfg=['--with-abi=lp64d','--disable-multilib'])
 225         self.add_config(arch='loongarch64',
 226                         os_name='linux-gnu',
 227                         variant='lp64s',
 228                         gcc_cfg=['--with-abi=lp64s','--disable-multilib'])
 229         self.add_config(arch='m68k',
 230                         os_name='linux-gnu',
 231                         gcc_cfg=['--disable-multilib'])
 232         self.add_config(arch='m68k',
 233                         os_name='linux-gnu',
 234                         variant='coldfire',
 235                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 236         self.add_config(arch='m68k',
 237                         os_name='linux-gnu',
 238                         variant='coldfire-soft',
 239                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 240                                  '--disable-multilib'])
 241         self.add_config(arch='microblaze',
 242                         os_name='linux-gnu',
 243                         gcc_cfg=['--disable-multilib'])
 244         self.add_config(arch='microblazeel',
 245                         os_name='linux-gnu',
 246                         gcc_cfg=['--disable-multilib'])
 247         self.add_config(arch='mips64',
 248                         os_name='linux-gnu',
 249                         gcc_cfg=['--with-mips-plt'],
 250                         glibcs=[{'variant': 'n32'},
 251                                 {'arch': 'mips',
 252                                  'ccopts': '-mabi=32'},
 253                                 {'variant': 'n64',
 254                                  'ccopts': '-mabi=64'}])
 255         self.add_config(arch='mips64',
 256                         os_name='linux-gnu',
 257                         variant='soft',
 258                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 259                         glibcs=[{'variant': 'n32-soft'},
 260                                 {'variant': 'soft',
 261                                  'arch': 'mips',
 262                                  'ccopts': '-mabi=32'},
 263                                 {'variant': 'n64-soft',
 264                                  'ccopts': '-mabi=64'}])
 265         self.add_config(arch='mips64',
 266                         os_name='linux-gnu',
 267                         variant='nan2008',
 268                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 269                                  '--with-arch-64=mips64r2',
 270                                  '--with-arch-32=mips32r2'],
 271                         glibcs=[{'variant': 'n32-nan2008'},
 272                                 {'variant': 'nan2008',
 273                                  'arch': 'mips',
 274                                  'ccopts': '-mabi=32'},
 275                                 {'variant': 'n64-nan2008',
 276                                  'ccopts': '-mabi=64'}])
 277         self.add_config(arch='mips64',
 278                         os_name='linux-gnu',
 279                         variant='nan2008-soft',
 280                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 281                                  '--with-arch-64=mips64r2',
 282                                  '--with-arch-32=mips32r2',
 283                                  '--with-float=soft'],
 284                         glibcs=[{'variant': 'n32-nan2008-soft'},
 285                                 {'variant': 'nan2008-soft',
 286                                  'arch': 'mips',
 287                                  'ccopts': '-mabi=32'},
 288                                 {'variant': 'n64-nan2008-soft',
 289                                  'ccopts': '-mabi=64'}])
 290         self.add_config(arch='mips64el',
 291                         os_name='linux-gnu',
 292                         gcc_cfg=['--with-mips-plt'],
 293                         glibcs=[{'variant': 'n32'},
 294                                 {'arch': 'mipsel',
 295                                  'ccopts': '-mabi=32'},
 296                                 {'variant': 'n64',
 297                                  'ccopts': '-mabi=64'}])
 298         self.add_config(arch='mips64el',
 299                         os_name='linux-gnu',
 300                         variant='soft',
 301                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 302                         glibcs=[{'variant': 'n32-soft'},
 303                                 {'variant': 'soft',
 304                                  'arch': 'mipsel',
 305                                  'ccopts': '-mabi=32'},
 306                                 {'variant': 'n64-soft',
 307                                  'ccopts': '-mabi=64'}])
 308         self.add_config(arch='mips64el',
 309                         os_name='linux-gnu',
 310                         variant='nan2008',
 311                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 312                                  '--with-arch-64=mips64r2',
 313                                  '--with-arch-32=mips32r2'],
 314                         glibcs=[{'variant': 'n32-nan2008'},
 315                                 {'variant': 'nan2008',
 316                                  'arch': 'mipsel',
 317                                  'ccopts': '-mabi=32'},
 318                                 {'variant': 'n64-nan2008',
 319                                  'ccopts': '-mabi=64'}])
 320         self.add_config(arch='mips64el',
 321                         os_name='linux-gnu',
 322                         variant='nan2008-soft',
 323                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 324                                  '--with-arch-64=mips64r2',
 325                                  '--with-arch-32=mips32r2',
 326                                  '--with-float=soft'],
 327                         glibcs=[{'variant': 'n32-nan2008-soft'},
 328                                 {'variant': 'nan2008-soft',
 329                                  'arch': 'mipsel',
 330                                  'ccopts': '-mabi=32'},
 331                                 {'variant': 'n64-nan2008-soft',
 332                                  'ccopts': '-mabi=64'}])
 333         self.add_config(arch='mipsisa64r6el',
 334                         os_name='linux-gnu',
 335                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 336                                  '--with-arch-64=mips64r6',
 337                                  '--with-arch-32=mips32r6',
 338                                  '--with-float=hard'],
 339                         glibcs=[{'variant': 'n32'},
 340                                 {'arch': 'mipsisa32r6el',
 341                                  'ccopts': '-mabi=32'},
 342                                 {'variant': 'n64',
 343                                  'ccopts': '-mabi=64'}])
 344         self.add_config(arch='nios2',
 345                         os_name='linux-gnu')
 346         self.add_config(arch='or1k',
 347                         os_name='linux-gnu',
 348                         variant='soft',
 349                         gcc_cfg=['--with-multilib-list=mcmov'])
 350         self.add_config(arch='powerpc',
 351                         os_name='linux-gnu',
 352                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 353                         extra_glibcs=[{'variant': 'power4',
 354                                        'ccopts': '-mcpu=power4',
 355                                        'cfg': ['--with-cpu=power4']}])
 356         self.add_config(arch='powerpc',
 357                         os_name='linux-gnu',
 358                         variant='soft',
 359                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 360                                  '--enable-secureplt'])
 361         self.add_config(arch='powerpc64',
 362                         os_name='linux-gnu',
 363                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 364         self.add_config(arch='powerpc64le',
 365                         os_name='linux-gnu',
 366                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 367                         extra_glibcs=[{'variant': 'disable-multi-arch',
 368                                        'cfg': ['--disable-multi-arch']}])
 369         self.add_config(arch='riscv32',
 370                         os_name='linux-gnu',
 371                         variant='rv32imac-ilp32',
 372                         gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
 373                                  '--disable-multilib'])
 374         self.add_config(arch='riscv32',
 375                         os_name='linux-gnu',
 376                         variant='rv32imafdc-ilp32',
 377                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
 378                                  '--disable-multilib'])
 379         self.add_config(arch='riscv32',
 380                         os_name='linux-gnu',
 381                         variant='rv32imafdc-ilp32d',
 382                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
 383                                  '--disable-multilib'])
 384         self.add_config(arch='riscv64',
 385                         os_name='linux-gnu',
 386                         variant='rv64imac-lp64',
 387                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 388                                  '--disable-multilib'])
 389         self.add_config(arch='riscv64',
 390                         os_name='linux-gnu',
 391                         variant='rv64imafdc-lp64',
 392                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 393                                  '--disable-multilib'])
 394         self.add_config(arch='riscv64',
 395                         os_name='linux-gnu',
 396                         variant='rv64imafdc-lp64d',
 397                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 398                                  '--disable-multilib'])
 399         self.add_config(arch='s390x',
 400                         os_name='linux-gnu',
 401                         glibcs=[{},
 402                                 {'arch': 's390', 'ccopts': '-m31'}],
 403                         extra_glibcs=[{'variant': 'O3',
 404                                        'cflags': '-O3'}])
 405         self.add_config(arch='sh3',
 406                         os_name='linux-gnu')
 407         self.add_config(arch='sh3eb',
 408                         os_name='linux-gnu')
 409         self.add_config(arch='sh4',
 410                         os_name='linux-gnu')
 411         self.add_config(arch='sh4eb',
 412                         os_name='linux-gnu')
 413         self.add_config(arch='sh4',
 414                         os_name='linux-gnu',
 415                         variant='soft',
 416                         gcc_cfg=['--without-fp'])
 417         self.add_config(arch='sh4eb',
 418                         os_name='linux-gnu',
 419                         variant='soft',
 420                         gcc_cfg=['--without-fp'])
 421         self.add_config(arch='sparc64',
 422                         os_name='linux-gnu',
 423                         glibcs=[{},
 424                                 {'arch': 'sparcv9',
 425                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 426                         extra_glibcs=[{'variant': 'leon3',
 427                                        'arch' : 'sparcv8',
 428                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 429                                       {'variant': 'disable-multi-arch',
 430                                        'cfg': ['--disable-multi-arch']},
 431                                       {'variant': 'disable-multi-arch',
 432                                        'arch': 'sparcv9',
 433                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 434                                        'cfg': ['--disable-multi-arch']}])
 435         self.add_config(arch='x86_64',
 436                         os_name='linux-gnu',
 437                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 438                         glibcs=[{},
 439                                 {'variant': 'x32', 'ccopts': '-mx32'},
 440                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 441                         extra_glibcs=[{'variant': 'disable-multi-arch',
 442                                        'cfg': ['--disable-multi-arch']},
 443                                       {'variant': 'minimal',
 444                                        'cfg': ['--disable-multi-arch',
 445                                                '--disable-profile',
 446                                                '--disable-timezone-tools',
 447                                                '--disable-mathvec',
 448                                                '--disable-crypt',
 449                                                '--disable-build-nscd',
 450                                                '--disable-nscd']},
 451                                       {'variant': 'no-pie',
 452                                        'cfg': ['--disable-default-pie']},
 453                                       {'variant': 'x32-no-pie',
 454                                        'ccopts': '-mx32',
 455                                        'cfg': ['--disable-default-pie']},
 456                                       {'variant': 'no-pie',
 457                                        'arch': 'i686',
 458                                        'ccopts': '-m32 -march=i686',
 459                                        'cfg': ['--disable-default-pie']},
 460                                       {'variant': 'disable-multi-arch',
 461                                        'arch': 'i686',
 462                                        'ccopts': '-m32 -march=i686',
 463                                        'cfg': ['--disable-multi-arch']},
 464                                       {'arch': 'i486',
 465                                        'ccopts': '-m32 -march=i486'},
 466                                       {'arch': 'i586',
 467                                        'ccopts': '-m32 -march=i586'}])
 468         self.add_config(arch='x86_64',
 469                         os_name='gnu',
 470                         gcc_cfg=['--disable-multilib'])
 471
 472     def add_config(self, **args):
 473         """Add an individual build configuration."""
 474         cfg = Config(self, **args)
 475         if cfg.name in self.configs:
 476             print('error: duplicate config %s' % cfg.name)
 477             exit(1)
 478         self.configs[cfg.name] = cfg
 479         for c in cfg.all_glibcs:
 480             if c.name in self.glibc_configs:
 481                 print('error: duplicate glibc config %s' % c.name)
 482                 exit(1)
 483             self.glibc_configs[c.name] = c
 484
 485     def component_srcdir(self, component):
 486         """Return the source directory for a given component, e.g. gcc."""
 487         return os.path.join(self.srcdir, component)
 488
 489     def component_builddir(self, action, config, component, subconfig=None):
 490         """Return the directory to use for a build."""
 491         if config is None:
 492             # Host libraries.
 493             assert subconfig is None
 494             return os.path.join(self.builddir, action, component)
 495         if subconfig is None:
 496             return os.path.join(self.builddir, action, config, component)
 497         else:
 498             # glibc build as part of compiler build.
 499             return os.path.join(self.builddir, action, config, component,
 500                                 subconfig)
 501
 502     def compiler_installdir(self, config):
 503         """Return the directory in which to install a compiler."""
 504         return os.path.join(self.installdir, 'compilers', config)
 505
 506     def compiler_bindir(self, config):
 507         """Return the directory in which to find compiler binaries."""
 508         return os.path.join(self.compiler_installdir(config), 'bin')
 509
 510     def compiler_sysroot(self, config):
 511         """Return the sysroot directory for a compiler."""
 512         return os.path.join(self.compiler_installdir(config), 'sysroot')
 513
 514     def glibc_installdir(self, config):
 515         """Return the directory in which to install glibc."""
 516         return os.path.join(self.installdir, 'glibcs', config)
 517
 518     def run_builds(self, action, configs):
 519         """Run the requested builds."""
 520         if action == 'checkout':
 521             self.checkout(configs)
 522             return
 523         if action == 'bot-cycle':
 524             if configs:
 525                 print('error: configurations specified for bot-cycle')
 526                 exit(1)
 527             self.bot_cycle()
 528             return
 529         if action == 'bot':
 530             if configs:
 531                 print('error: configurations specified for bot')
 532                 exit(1)
 533             self.bot()
 534             return
 535         if action in ('host-libraries', 'list-compilers',
 536                       'list-glibcs') and configs:
 537             print('error: configurations specified for ' + action)
 538             exit(1)
 539         if action == 'list-compilers':
 540             for name in sorted(self.configs.keys()):
 541                 print(name)
 542             return
 543         if action == 'list-glibcs':
 544             for config in sorted(self.glibc_configs.values(),
 545                                  key=lambda c: c.name):
 546                 print(config.name, config.compiler.name)
 547             return
 548         self.clear_last_build_state(action)
 549         build_time = datetime.datetime.utcnow()
 550         if action == 'host-libraries':
 551             build_components = ('gmp', 'mpfr', 'mpc')
 552             old_components = ()
 553             old_versions = {}
 554             self.build_host_libraries()
 555         elif action == 'compilers':
 556             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 557                                 'gnumach', 'hurd')
 558             old_components = ('gmp', 'mpfr', 'mpc')
 559             old_versions = self.build_state['host-libraries']['build-versions']
 560             self.build_compilers(configs)
 561         else:
 562             build_components = ('glibc',)
 563             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 564                               'mig', 'gnumach', 'hurd')
 565             old_versions = self.build_state['compilers']['build-versions']
 566             if action == 'update-syscalls':
 567                 self.update_syscalls(configs)
 568             else:
 569                 self.build_glibcs(configs)
 570         self.write_files()
 571         self.do_build()
 572         if configs:
 573             # Partial build, do not update stored state.
 574             return
 575         build_versions = {}
 576         for k in build_components:
 577             if k in self.versions:
 578                 build_versions[k] = {'version': self.versions[k]['version'],
 579                                      'revision': self.versions[k]['revision']}
 580         for k in old_components:
 581             if k in old_versions:
 582                 build_versions[k] = {'version': old_versions[k]['version'],
 583                                      'revision': old_versions[k]['revision']}
 584         self.update_build_state(action, build_time, build_versions)
 585
 586     @staticmethod
 587     def remove_dirs(*args):
 588         """Remove directories and their contents if they exist."""
 589         for dir in args:
 590             shutil.rmtree(dir, ignore_errors=True)
 591
 592     @staticmethod
 593     def remove_recreate_dirs(*args):
 594         """Remove directories if they exist, and create them as empty."""
 595         Context.remove_dirs(*args)
 596         for dir in args:
 597             os.makedirs(dir, exist_ok=True)
 598
 599     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 600         """Add makefile text for a list of commands."""
 601         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 602         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 603                                     (target, target, target, commands))
 604         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 605
 606     def write_files(self):
 607         """Write out the Makefile and wrapper script."""
 608         mftext = ''.join(self.makefile_pieces)
 609         with open(self.makefile, 'w') as f:
 610             f.write(mftext)
 611         wrapper_text = (
 612             '#!/bin/sh\n'
 613             'prev_base=$1\n'
 614             'this_base=$2\n'
 615             'desc=$3\n'
 616             'dir=$4\n'
 617             'path=$5\n'
 618             'shift 5\n'
 619             'prev_status=$prev_base-status.txt\n'
 620             'this_status=$this_base-status.txt\n'
 621             'this_log=$this_base-log.txt\n'
 622             'date > "$this_log"\n'
 623             'echo >> "$this_log"\n'
 624             'echo "Description: $desc" >> "$this_log"\n'
 625             'printf "%s" "Command:" >> "$this_log"\n'
 626             'for word in "$@"; do\n'
 627             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 628             '    printf " %s" "$word"\n'
 629             '  else\n'
 630             '    printf " \'"\n'
 631             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 632             '    printf "\'"\n'
 633             '  fi\n'
 634             'done >> "$this_log"\n'
 635             'echo >> "$this_log"\n'
 636             'echo "Directory: $dir" >> "$this_log"\n'
 637             'echo "Path addition: $path" >> "$this_log"\n'
 638             'echo >> "$this_log"\n'
 639             'record_status ()\n'
 640             '{\n'
 641             '  echo >> "$this_log"\n'
 642             '  echo "$1: $desc" > "$this_status"\n'
 643             '  echo "$1: $desc" >> "$this_log"\n'
 644             '  echo >> "$this_log"\n'
 645             '  date >> "$this_log"\n'
 646             '  echo "$1: $desc"\n'
 647             '  exit 0\n'
 648             '}\n'
 649             'check_error ()\n'
 650             '{\n'
 651             '  if [ "$1" != "0" ]; then\n'
 652             '    record_status FAIL\n'
 653             '  fi\n'
 654             '}\n'
 655             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 656             '    record_status UNRESOLVED\n'
 657             'fi\n'
 658             'if [ "$dir" ]; then\n'
 659             '  cd "$dir"\n'
 660             '  check_error "$?"\n'
 661             'fi\n'
 662             'if [ "$path" ]; then\n'
 663             '  PATH=$path:$PATH\n'
 664             'fi\n'
 665             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 666             'check_error "$?"\n'
 667             'record_status PASS\n')
 668         with open(self.wrapper, 'w') as f:
 669             f.write(wrapper_text)
 670         # Mode 0o755.
 671         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 672                      stat.S_IROTH|stat.S_IXOTH)
 673         os.chmod(self.wrapper, mode_exec)
 674         save_logs_text = (
 675             '#!/bin/sh\n'
 676             'if ! [ -f tests.sum ]; then\n'
 677             '  echo "No test summary available."\n'
 678             '  exit 0\n'
 679             'fi\n'
 680             'save_file ()\n'
 681             '{\n'
 682             '  echo "Contents of $1:"\n'
 683             '  echo\n'
 684             '  cat "$1"\n'
 685             '  echo\n'
 686             '  echo "End of contents of $1."\n'
 687             '  echo\n'
 688             '}\n'
 689             'save_file tests.sum\n'
 690             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 691             'for t in $non_pass_tests; do\n'
 692             '  if [ -f "$t.out" ]; then\n'
 693             '    save_file "$t.out"\n'
 694             '  fi\n'
 695             'done\n')
 696         with open(self.save_logs, 'w') as f:
 697             f.write(save_logs_text)
 698         os.chmod(self.save_logs, mode_exec)
 699
 700     def do_build(self):
 701         """Do the actual build."""
 702         cmd = ['make', '-O', '-j%d' % self.parallelism]
 703         subprocess.run(cmd, cwd=self.builddir, check=True)
 704
 705     def build_host_libraries(self):
 706         """Build the host libraries."""
 707         installdir = self.host_libraries_installdir
 708         builddir = os.path.join(self.builddir, 'host-libraries')
 709         logsdir = os.path.join(self.logsdir, 'host-libraries')
 710         self.remove_recreate_dirs(installdir, builddir, logsdir)
 711         cmdlist = CommandList('host-libraries', self.keep)
 712         self.build_host_library(cmdlist, 'gmp')
 713         self.build_host_library(cmdlist, 'mpfr',
 714                                 ['--with-gmp=%s' % installdir])
 715         self.build_host_library(cmdlist, 'mpc',
 716                                 ['--with-gmp=%s' % installdir,
 717                                 '--with-mpfr=%s' % installdir])
 718         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 719         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 720
 721     def build_host_library(self, cmdlist, lib, extra_opts=None):
 722         """Build one host library."""
 723         srcdir = self.component_srcdir(lib)
 724         builddir = self.component_builddir('host-libraries', None, lib)
 725         installdir = self.host_libraries_installdir
 726         cmdlist.push_subdesc(lib)
 727         cmdlist.create_use_dir(builddir)
 728         cfg_cmd = [os.path.join(srcdir, 'configure'),
 729                    '--prefix=%s' % installdir,
 730                    '--disable-shared']
 731         if extra_opts:
 732             cfg_cmd.extend (extra_opts)
 733         cmdlist.add_command('configure', cfg_cmd)
 734         cmdlist.add_command('build', ['make'])
 735         cmdlist.add_command('check', ['make', 'check'])
 736         cmdlist.add_command('install', ['make', 'install'])
 737         cmdlist.cleanup_dir()
 738         cmdlist.pop_subdesc()
 739
 740     def build_compilers(self, configs):
 741         """Build the compilers."""
 742         if not configs:
 743             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 744             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 745             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 746             configs = sorted(self.configs.keys())
 747         for c in configs:
 748             self.configs[c].build()
 749
 750     def build_glibcs(self, configs):
 751         """Build the glibcs."""
 752         if not configs:
 753             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 754             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 755             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 756             configs = sorted(self.glibc_configs.keys())
 757         for c in configs:
 758             self.glibc_configs[c].build()
 759
 760     def update_syscalls(self, configs):
 761         """Update the glibc syscall lists."""
 762         if not configs:
 763             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 764             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 765             configs = sorted(self.glibc_configs.keys())
 766         for c in configs:
 767             self.glibc_configs[c].update_syscalls()
 768
 769     def load_versions_json(self):
 770         """Load information about source directory versions."""
 771         if not os.access(self.versions_json, os.F_OK):
 772             self.versions = {}
 773             return
 774         with open(self.versions_json, 'r') as f:
 775             self.versions = json.load(f)
 776
 777     def store_json(self, data, filename):
 778         """Store information in a JSON file."""
 779         filename_tmp = filename + '.tmp'
 780         with open(filename_tmp, 'w') as f:
 781             json.dump(data, f, indent=2, sort_keys=True)
 782         os.rename(filename_tmp, filename)
 783
 784     def store_versions_json(self):
 785         """Store information about source directory versions."""
 786         self.store_json(self.versions, self.versions_json)
 787
 788     def set_component_version(self, component, version, explicit, revision):
 789         """Set the version information for a component."""
 790         self.versions[component] = {'version': version,
 791                                     'explicit': explicit,
 792                                     'revision': revision}
 793         self.store_versions_json()
 794
 795     def checkout(self, versions):
 796         """Check out the desired component versions."""
 797         default_versions = {'binutils': 'vcs-2.40',
 798                             'gcc': 'vcs-13',
 799                             'glibc': 'vcs-mainline',
 800                             'gmp': '6.2.1',
 801                             'linux': '6.3',
 802                             'mpc': '1.3.1',
 803                             'mpfr': '4.2.0',
 804                             'mig': 'vcs-mainline',
 805                             'gnumach': 'vcs-mainline',
 806                             'hurd': 'vcs-mainline'}
 807         use_versions = {}
 808         explicit_versions = {}
 809         for v in versions:
 810             found_v = False
 811             for k in default_versions.keys():
 812                 kx = k + '-'
 813                 if v.startswith(kx):
 814                     vx = v[len(kx):]
 815                     if k in use_versions:
 816                         print('error: multiple versions for %s' % k)
 817                         exit(1)
 818                     use_versions[k] = vx
 819                     explicit_versions[k] = True
 820                     found_v = True
 821                     break
 822             if not found_v:
 823                 print('error: unknown component in %s' % v)
 824                 exit(1)
 825         for k in default_versions.keys():
 826             if k not in use_versions:
 827                 if k in self.versions and self.versions[k]['explicit']:
 828                     use_versions[k] = self.versions[k]['version']
 829                     explicit_versions[k] = True
 830                 else:
 831                     use_versions[k] = default_versions[k]
 832                     explicit_versions[k] = False
 833         os.makedirs(self.srcdir, exist_ok=True)
 834         for k in sorted(default_versions.keys()):
 835             update = os.access(self.component_srcdir(k), os.F_OK)
 836             v = use_versions[k]
 837             if (update and
 838                 k in self.versions and
 839                 v != self.versions[k]['version']):
 840                 if not self.replace_sources:
 841                     print('error: version of %s has changed from %s to %s, '
 842                           'use --replace-sources to check out again' %
 843                           (k, self.versions[k]['version'], v))
 844                     exit(1)
 845                 shutil.rmtree(self.component_srcdir(k))
 846                 update = False
 847             if v.startswith('vcs-'):
 848                 revision = self.checkout_vcs(k, v[4:], update)
 849             else:
 850                 self.checkout_tar(k, v, update)
 851                 revision = v
 852             self.set_component_version(k, v, explicit_versions[k], revision)
 853         if self.get_script_text() != self.script_text:
 854             # Rerun the checkout process in case the updated script
 855             # uses different default versions or new components.
 856             self.exec_self()
 857
 858     def checkout_vcs(self, component, version, update):
 859         """Check out the given version of the given component from version
 860         control.  Return a revision identifier."""
 861         if component == 'binutils':
 862             git_url = 'https://sourceware.org/git/binutils-gdb.git'
 863             if version == 'mainline':
 864                 git_branch = 'master'
 865             else:
 866                 trans = str.maketrans({'.': '_'})
 867                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 868             return self.git_checkout(component, git_url, git_branch, update)
 869         elif component == 'gcc':
 870             if version == 'mainline':
 871                 branch = 'master'
 872             else:
 873                 branch = 'releases/gcc-%s' % version
 874             return self.gcc_checkout(branch, update)
 875         elif component == 'glibc':
 876             git_url = 'https://sourceware.org/git/glibc.git'
 877             if version == 'mainline':
 878                 git_branch = 'master'
 879             else:
 880                 git_branch = 'release/%s/master' % version
 881             r = self.git_checkout(component, git_url, git_branch, update)
 882             self.fix_glibc_timestamps()
 883             return r
 884         elif component == 'gnumach':
 885             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 886             git_branch = 'master'
 887             r = self.git_checkout(component, git_url, git_branch, update)
 888             subprocess.run(['autoreconf', '-i'],
 889                            cwd=self.component_srcdir(component), check=True)
 890             return r
 891         elif component == 'mig':
 892             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 893             git_branch = 'master'
 894             r = self.git_checkout(component, git_url, git_branch, update)
 895             subprocess.run(['autoreconf', '-i'],
 896                            cwd=self.component_srcdir(component), check=True)
 897             return r
 898         elif component == 'hurd':
 899             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 900             git_branch = 'master'
 901             r = self.git_checkout(component, git_url, git_branch, update)
 902             subprocess.run(['autoconf'],
 903                            cwd=self.component_srcdir(component), check=True)
 904             return r
 905         else:
 906             print('error: component %s coming from VCS' % component)
 907             exit(1)
 908
 909     def git_checkout(self, component, git_url, git_branch, update):
 910         """Check out a component from git.  Return a commit identifier."""
 911         if update:
 912             subprocess.run(['git', 'remote', 'prune', 'origin'],
 913                            cwd=self.component_srcdir(component), check=True)
 914             if self.replace_sources:
 915                 subprocess.run(['git', 'clean', '-dxfq'],
 916                                cwd=self.component_srcdir(component), check=True)
 917             subprocess.run(['git', 'pull', '-q'],
 918                            cwd=self.component_srcdir(component), check=True)
 919         else:
 920             if self.shallow:
 921                 depth_arg = ('--depth', '1')
 922             else:
 923                 depth_arg = ()
 924             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 925                             *depth_arg, git_url,
 926                             self.component_srcdir(component)], check=True)
 927         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 928                            cwd=self.component_srcdir(component),
 929                            stdout=subprocess.PIPE,
 930                            check=True, universal_newlines=True).stdout
 931         return r.rstrip()
 932
 933     def fix_glibc_timestamps(self):
 934         """Fix timestamps in a glibc checkout."""
 935         # Ensure that builds do not try to regenerate generated files
 936         # in the source tree.
 937         srcdir = self.component_srcdir('glibc')
 938         # These files have Makefile dependencies to regenerate them in
 939         # the source tree that may be active during a normal build.
 940         # Some other files have such dependencies but do not need to
 941         # be touched because nothing in a build depends on the files
 942         # in question.
 943         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 944             to_touch = os.path.join(srcdir, f)
 945             subprocess.run(['touch', '-c', to_touch], check=True)
 946         for dirpath, dirnames, filenames in os.walk(srcdir):
 947             for f in filenames:
 948                 if (f == 'configure' or
 949                     f == 'preconfigure' or
 950                     f.endswith('-kw.h')):
 951                     to_touch = os.path.join(dirpath, f)
 952                     subprocess.run(['touch', to_touch], check=True)
 953
 954     def gcc_checkout(self, branch, update):
 955         """Check out GCC from git.  Return the commit identifier."""
 956         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 957                      os.F_OK):
 958             if not self.replace_sources:
 959                 print('error: GCC has moved from SVN to git, use '
 960                       '--replace-sources to check out again')
 961                 exit(1)
 962             shutil.rmtree(self.component_srcdir('gcc'))
 963             update = False
 964         if not update:
 965             self.git_checkout('gcc', 'https://gcc.gnu.org/git/gcc.git',
 966                               branch, update)
 967         subprocess.run(['contrib/gcc_update', '--silent'],
 968                        cwd=self.component_srcdir('gcc'), check=True)
 969         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 970                            cwd=self.component_srcdir('gcc'),
 971                            stdout=subprocess.PIPE,
 972                            check=True, universal_newlines=True).stdout
 973         return r.rstrip()
 974
 975     def checkout_tar(self, component, version, update):
 976         """Check out the given version of the given component from a
 977         tarball."""
 978         if update:
 979             return
 980         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 981                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
 982                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 983                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
 984                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 985                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 986                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 987                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 988                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 989         if component not in url_map:
 990             print('error: component %s coming from tarball' % component)
 991             exit(1)
 992         version_major = version.split('.')[0]
 993         url = url_map[component] % {'version': version, 'major': version_major}
 994         filename = os.path.join(self.srcdir, url.split('/')[-1])
 995         response = urllib.request.urlopen(url)
 996         data = response.read()
 997         with open(filename, 'wb') as f:
 998             f.write(data)
 999         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
1000                        check=True)
1001         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
1002                   self.component_srcdir(component))
1003         os.remove(filename)
1004
1005     def load_build_state_json(self):
1006         """Load information about the state of previous builds."""
1007         if os.access(self.build_state_json, os.F_OK):
1008             with open(self.build_state_json, 'r') as f:
1009                 self.build_state = json.load(f)
1010         else:
1011             self.build_state = {}
1012         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
1013             if k not in self.build_state:
1014                 self.build_state[k] = {}
1015             if 'build-time' not in self.build_state[k]:
1016                 self.build_state[k]['build-time'] = ''
1017             if 'build-versions' not in self.build_state[k]:
1018                 self.build_state[k]['build-versions'] = {}
1019             if 'build-results' not in self.build_state[k]:
1020                 self.build_state[k]['build-results'] = {}
1021             if 'result-changes' not in self.build_state[k]:
1022                 self.build_state[k]['result-changes'] = {}
1023             if 'ever-passed' not in self.build_state[k]:
1024                 self.build_state[k]['ever-passed'] = []
1025
1026     def store_build_state_json(self):
1027         """Store information about the state of previous builds."""
1028         self.store_json(self.build_state, self.build_state_json)
1029
1030     def clear_last_build_state(self, action):
1031         """Clear information about the state of part of the build."""
1032         # We clear the last build time and versions when starting a
1033         # new build.  The results of the last build are kept around,
1034         # as comparison is still meaningful if this build is aborted
1035         # and a new one started.
1036         self.build_state[action]['build-time'] = ''
1037         self.build_state[action]['build-versions'] = {}
1038         self.store_build_state_json()
1039
1040     def update_build_state(self, action, build_time, build_versions):
1041         """Update the build state after a build."""
1042         build_time = build_time.replace(microsecond=0)
1043         self.build_state[action]['build-time'] = str(build_time)
1044         self.build_state[action]['build-versions'] = build_versions
1045         build_results = {}
1046         for log in self.status_log_list:
1047             with open(log, 'r') as f:
1048                 log_text = f.read()
1049             log_text = log_text.rstrip()
1050             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1051             result = m.group(1)
1052             test_name = m.group(2)
1053             assert test_name not in build_results
1054             build_results[test_name] = result
1055         old_build_results = self.build_state[action]['build-results']
1056         self.build_state[action]['build-results'] = build_results
1057         result_changes = {}
1058         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1059         for t in all_tests:
1060             if t in old_build_results:
1061                 old_res = old_build_results[t]
1062             else:
1063                 old_res = '(New test)'
1064             if t in build_results:
1065                 new_res = build_results[t]
1066             else:
1067                 new_res = '(Test removed)'
1068             if old_res != new_res:
1069                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1070         self.build_state[action]['result-changes'] = result_changes
1071         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1072                            if t in build_results}
1073         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1074         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1075                                                          new_passes)
1076         self.store_build_state_json()
1077
1078     def load_bot_config_json(self):
1079         """Load bot configuration."""
1080         with open(self.bot_config_json, 'r') as f:
1081             self.bot_config = json.load(f)
1082
1083     def part_build_old(self, action, delay):
1084         """Return whether the last build for a given action was at least a
1085         given number of seconds ago, or does not have a time recorded."""
1086         old_time_str = self.build_state[action]['build-time']
1087         if not old_time_str:
1088             return True
1089         old_time = datetime.datetime.strptime(old_time_str,
1090                                               '%Y-%m-%d %H:%M:%S')
1091         new_time = datetime.datetime.utcnow()
1092         delta = new_time - old_time
1093         return delta.total_seconds() >= delay
1094
1095     def bot_cycle(self):
1096         """Run a single round of checkout and builds."""
1097         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1098         self.load_bot_config_json()
1099         actions = ('host-libraries', 'compilers', 'glibcs')
1100         self.bot_run_self(['--replace-sources'], 'checkout')
1101         self.load_versions_json()
1102         if self.get_script_text() != self.script_text:
1103             print('Script changed, re-execing.')
1104             # On script change, all parts of the build should be rerun.
1105             for a in actions:
1106                 self.clear_last_build_state(a)
1107             self.exec_self()
1108         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1109                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1110                                           'mig', 'gnumach', 'hurd'),
1111                             'glibcs': ('glibc',)}
1112         must_build = {}
1113         for a in actions:
1114             build_vers = self.build_state[a]['build-versions']
1115             must_build[a] = False
1116             if not self.build_state[a]['build-time']:
1117                 must_build[a] = True
1118             old_vers = {}
1119             new_vers = {}
1120             for c in check_components[a]:
1121                 if c in build_vers:
1122                     old_vers[c] = build_vers[c]
1123                 new_vers[c] = {'version': self.versions[c]['version'],
1124                                'revision': self.versions[c]['revision']}
1125             if new_vers == old_vers:
1126                 print('Versions for %s unchanged.' % a)
1127             else:
1128                 print('Versions changed or rebuild forced for %s.' % a)
1129                 if a == 'compilers' and not self.part_build_old(
1130                         a, self.bot_config['compilers-rebuild-delay']):
1131                     print('Not requiring rebuild of compilers this soon.')
1132                 else:
1133                     must_build[a] = True
1134         if must_build['host-libraries']:
1135             must_build['compilers'] = True
1136         if must_build['compilers']:
1137             must_build['glibcs'] = True
1138         for a in actions:
1139             if must_build[a]:
1140                 print('Must rebuild %s.' % a)
1141                 self.clear_last_build_state(a)
1142             else:
1143                 print('No need to rebuild %s.' % a)
1144         if os.access(self.logsdir, os.F_OK):
1145             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1146             shutil.copytree(self.logsdir, self.logsdir_old)
1147         for a in actions:
1148             if must_build[a]:
1149                 build_time = datetime.datetime.utcnow()
1150                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1151                 self.bot_run_self([], a)
1152                 self.load_build_state_json()
1153                 self.bot_build_mail(a, build_time)
1154         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1155
1156     def bot_build_mail(self, action, build_time):
1157         """Send email with the results of a build."""
1158         if not ('email-from' in self.bot_config and
1159                 'email-server' in self.bot_config and
1160                 'email-subject' in self.bot_config and
1161                 'email-to' in self.bot_config):
1162             if not self.email_warning:
1163                 print("Email not configured, not sending.")
1164                 self.email_warning = True
1165             return
1166
1167         build_time = build_time.replace(microsecond=0)
1168         subject = (self.bot_config['email-subject'] %
1169                    {'action': action,
1170                     'build-time': str(build_time)})
1171         results = self.build_state[action]['build-results']
1172         changes = self.build_state[action]['result-changes']
1173         ever_passed = set(self.build_state[action]['ever-passed'])
1174         versions = self.build_state[action]['build-versions']
1175         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1176         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1177         all_fails = {k for k in results if results[k] == 'FAIL'}
1178         if new_regressions:
1179             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1180             new_reg_text = ('New regressions:\n\n%s\n\n' %
1181                             '\n'.join(new_reg_list))
1182         else:
1183             new_reg_text = ''
1184         if all_regressions:
1185             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1186             all_reg_text = ('All regressions:\n\n%s\n\n' %
1187                             '\n'.join(all_reg_list))
1188         else:
1189             all_reg_text = ''
1190         if all_fails:
1191             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1192             all_fail_text = ('All failures:\n\n%s\n\n' %
1193                              '\n'.join(all_fail_list))
1194         else:
1195             all_fail_text = ''
1196         if changes:
1197             changes_list = sorted(changes.keys())
1198             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1199             changes_text = ('All changed results:\n\n%s\n\n' %
1200                             '\n'.join(changes_list))
1201         else:
1202             changes_text = ''
1203         results_text = (new_reg_text + all_reg_text + all_fail_text +
1204                         changes_text)
1205         if not results_text:
1206             results_text = 'Clean build with unchanged results.\n\n'
1207         versions_list = sorted(versions.keys())
1208         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1209                                           versions[k]['revision'])
1210                          for k in versions_list]
1211         versions_text = ('Component versions for this build:\n\n%s\n' %
1212                          '\n'.join(versions_list))
1213         body_text = results_text + versions_text
1214         msg = email.mime.text.MIMEText(body_text)
1215         msg['Subject'] = subject
1216         msg['From'] = self.bot_config['email-from']
1217         msg['To'] = self.bot_config['email-to']
1218         msg['Message-ID'] = email.utils.make_msgid()
1219         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1220         with smtplib.SMTP(self.bot_config['email-server']) as s:
1221             s.send_message(msg)
1222
1223     def bot_run_self(self, opts, action, check=True):
1224         """Run a copy of this script with given options."""
1225         cmd = [sys.executable, sys.argv[0], '--keep=none',
1226                '-j%d' % self.parallelism]
1227         if self.full_gcc:
1228             cmd.append('--full-gcc')
1229         cmd.extend(opts)
1230         cmd.extend([self.topdir, action])
1231         sys.stdout.flush()
1232         subprocess.run(cmd, check=check)
1233
1234     def bot(self):
1235         """Run repeated rounds of checkout and builds."""
1236         while True:
1237             self.load_bot_config_json()
1238             if not self.bot_config['run']:
1239                 print('Bot exiting by request.')
1240                 exit(0)
1241             self.bot_run_self([], 'bot-cycle', check=False)
1242             self.load_bot_config_json()
1243             if not self.bot_config['run']:
1244                 print('Bot exiting by request.')
1245                 exit(0)
1246             time.sleep(self.bot_config['delay'])
1247             if self.get_script_text() != self.script_text:
1248                 print('Script changed, bot re-execing.')
1249                 self.exec_self()
1250
1251 class LinuxHeadersPolicyForBuild(object):
1252     """Names and directories for installing Linux headers.  Build variant."""
1253
1254     def __init__(self, config):
1255         self.arch = config.arch
1256         self.srcdir = config.ctx.component_srcdir('linux')
1257         self.builddir = config.component_builddir('linux')
1258         self.headers_dir = os.path.join(config.sysroot, 'usr')
1259
1260 class LinuxHeadersPolicyForUpdateSyscalls(object):
1261     """Names and directories for Linux headers.  update-syscalls variant."""
1262
1263     def __init__(self, glibc, headers_dir):
1264         self.arch = glibc.compiler.arch
1265         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1266         self.builddir = glibc.ctx.component_builddir(
1267             'update-syscalls', glibc.name, 'build-linux')
1268         self.headers_dir = headers_dir
1269
1270 def install_linux_headers(policy, cmdlist):
1271     """Install Linux kernel headers."""
1272     arch_map = {'aarch64': 'arm64',
1273                 'alpha': 'alpha',
1274                 'arc': 'arc',
1275                 'arm': 'arm',
1276                 'csky': 'csky',
1277                 'hppa': 'parisc',
1278                 'i486': 'x86',
1279                 'i586': 'x86',
1280                 'i686': 'x86',
1281                 'i786': 'x86',
1282                 'ia64': 'ia64',
1283                 'loongarch64': 'loongarch',
1284                 'm68k': 'm68k',
1285                 'microblaze': 'microblaze',
1286                 'mips': 'mips',
1287                 'nios2': 'nios2',
1288                 'or1k': 'openrisc',
1289                 'powerpc': 'powerpc',
1290                 's390': 's390',
1291                 'riscv32': 'riscv',
1292                 'riscv64': 'riscv',
1293                 'sh': 'sh',
1294                 'sparc': 'sparc',
1295                 'x86_64': 'x86'}
1296     linux_arch = None
1297     for k in arch_map:
1298         if policy.arch.startswith(k):
1299             linux_arch = arch_map[k]
1300             break
1301     assert linux_arch is not None
1302     cmdlist.push_subdesc('linux')
1303     cmdlist.create_use_dir(policy.builddir)
1304     cmdlist.add_command('install-headers',
1305                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1306                          'ARCH=%s' % linux_arch,
1307                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1308                          'headers_install'])
1309     cmdlist.cleanup_dir()
1310     cmdlist.pop_subdesc()
1311
1312 class Config(object):
1313     """A configuration for building a compiler and associated libraries."""
1314
1315     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1316                  first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1317                  extra_glibcs=None):
1318         """Initialize a Config object."""
1319         self.ctx = ctx
1320         self.arch = arch
1321         self.os = os_name
1322         self.variant = variant
1323         if variant is None:
1324             self.name = '%s-%s' % (arch, os_name)
1325         else:
1326             self.name = '%s-%s-%s' % (arch, os_name, variant)
1327         self.triplet = '%s-glibc-%s' % (arch, os_name)
1328         if gcc_cfg is None:
1329             self.gcc_cfg = []
1330         else:
1331             self.gcc_cfg = gcc_cfg
1332         if first_gcc_cfg is None:
1333             self.first_gcc_cfg = []
1334         else:
1335             self.first_gcc_cfg = first_gcc_cfg
1336         if binutils_cfg is None:
1337             self.binutils_cfg = []
1338         else:
1339             self.binutils_cfg = binutils_cfg
1340         if glibcs is None:
1341             glibcs = [{'variant': variant}]
1342         if extra_glibcs is None:
1343             extra_glibcs = []
1344         glibcs = [Glibc(self, **g) for g in glibcs]
1345         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1346         self.all_glibcs = glibcs + extra_glibcs
1347         self.compiler_glibcs = glibcs
1348         self.installdir = ctx.compiler_installdir(self.name)
1349         self.bindir = ctx.compiler_bindir(self.name)
1350         self.sysroot = ctx.compiler_sysroot(self.name)
1351         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1352         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1353
1354     def component_builddir(self, component):
1355         """Return the directory to use for a (non-glibc) build."""
1356         return self.ctx.component_builddir('compilers', self.name, component)
1357
1358     def build(self):
1359         """Generate commands to build this compiler."""
1360         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1361                                       self.logsdir)
1362         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1363         cmdlist.add_command('check-host-libraries',
1364                             ['test', '-f',
1365                              os.path.join(self.ctx.host_libraries_installdir,
1366                                           'ok')])
1367         cmdlist.use_path(self.bindir)
1368         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1369                               ['--disable-gdb',
1370                                '--disable-gdbserver',
1371                                '--disable-libdecnumber',
1372                                '--disable-readline',
1373                                '--disable-sim'] + self.binutils_cfg)
1374         if self.os.startswith('linux'):
1375             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1376         self.build_gcc(cmdlist, True)
1377         if self.os == 'gnu':
1378             self.install_gnumach_headers(cmdlist)
1379             self.build_cross_tool(cmdlist, 'mig', 'mig')
1380             self.install_hurd_headers(cmdlist)
1381         for g in self.compiler_glibcs:
1382             cmdlist.push_subdesc('glibc')
1383             cmdlist.push_subdesc(g.name)
1384             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1385             cmdlist.pop_subdesc()
1386             cmdlist.pop_subdesc()
1387         self.build_gcc(cmdlist, False)
1388         cmdlist.add_command('done', ['touch',
1389                                      os.path.join(self.installdir, 'ok')])
1390         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1391                                       self.logsdir)
1392
1393     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1394         """Build one cross tool."""
1395         srcdir = self.ctx.component_srcdir(tool_src)
1396         builddir = self.component_builddir(tool_build)
1397         cmdlist.push_subdesc(tool_build)
1398         cmdlist.create_use_dir(builddir)
1399         cfg_cmd = [os.path.join(srcdir, 'configure'),
1400                    '--prefix=%s' % self.installdir,
1401                    '--build=%s' % self.ctx.build_triplet,
1402                    '--host=%s' % self.ctx.build_triplet,
1403                    '--target=%s' % self.triplet,
1404                    '--with-sysroot=%s' % self.sysroot]
1405         if extra_opts:
1406             cfg_cmd.extend(extra_opts)
1407         cmdlist.add_command('configure', cfg_cmd)
1408         cmdlist.add_command('build', ['make'])
1409         # Parallel "make install" for GCC has race conditions that can
1410         # cause it to fail; see
1411         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1412         # problems are not known for binutils, but doing the
1413         # installation in parallel within a particular toolchain build
1414         # (as opposed to installation of one toolchain from
1415         # build-many-glibcs.py running in parallel to the installation
1416         # of other toolchains being built) is not known to be
1417         # significantly beneficial, so it is simplest just to disable
1418         # parallel install for cross tools here.
1419         cmdlist.add_command('install', ['make', '-j1', 'install'])
1420         cmdlist.cleanup_dir()
1421         cmdlist.pop_subdesc()
1422
1423     def install_gnumach_headers(self, cmdlist):
1424         """Install GNU Mach headers."""
1425         srcdir = self.ctx.component_srcdir('gnumach')
1426         builddir = self.component_builddir('gnumach')
1427         cmdlist.push_subdesc('gnumach')
1428         cmdlist.create_use_dir(builddir)
1429         cmdlist.add_command('configure',
1430                             [os.path.join(srcdir, 'configure'),
1431                              '--build=%s' % self.ctx.build_triplet,
1432                              '--host=%s' % self.triplet,
1433                              '--prefix=',
1434                              '--disable-user32',
1435                              'CC=%s-gcc -nostdlib' % self.triplet])
1436         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1437                                         'install-data'])
1438         cmdlist.cleanup_dir()
1439         cmdlist.pop_subdesc()
1440
1441     def install_hurd_headers(self, cmdlist):
1442         """Install Hurd headers."""
1443         srcdir = self.ctx.component_srcdir('hurd')
1444         builddir = self.component_builddir('hurd')
1445         cmdlist.push_subdesc('hurd')
1446         cmdlist.create_use_dir(builddir)
1447         cmdlist.add_command('configure',
1448                             [os.path.join(srcdir, 'configure'),
1449                              '--build=%s' % self.ctx.build_triplet,
1450                              '--host=%s' % self.triplet,
1451                              '--prefix=',
1452                              '--disable-profile', '--without-parted',
1453                              'CC=%s-gcc -nostdlib' % self.triplet])
1454         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1455                                         'no_deps=t', 'install-headers'])
1456         cmdlist.cleanup_dir()
1457         cmdlist.pop_subdesc()
1458
1459     def build_gcc(self, cmdlist, bootstrap):
1460         """Build GCC."""
1461         # libssp is of little relevance with glibc's own stack
1462         # checking support.  libcilkrts does not support GNU/Hurd (and
1463         # has been removed in GCC 8, so --disable-libcilkrts can be
1464         # removed once glibc no longer supports building with older
1465         # GCC versions).  --enable-initfini-array is enabled by default
1466         # in GCC 12, which can be removed when GCC 12 becomes the
1467         # minimum requirement.
1468         cfg_opts = list(self.gcc_cfg)
1469         cfg_opts += ['--enable-initfini-array']
1470         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1471         host_libs = self.ctx.host_libraries_installdir
1472         cfg_opts += ['--with-gmp=%s' % host_libs,
1473                      '--with-mpfr=%s' % host_libs,
1474                      '--with-mpc=%s' % host_libs]
1475         if bootstrap:
1476             tool_build = 'gcc-first'
1477             # Building a static-only, C-only compiler that is
1478             # sufficient to build glibc.  Various libraries and
1479             # features that may require libc headers must be disabled.
1480             # When configuring with a sysroot, --with-newlib is
1481             # required to define inhibit_libc (to stop some parts of
1482             # libgcc including libc headers); --without-headers is not
1483             # sufficient.
1484             cfg_opts += ['--enable-languages=c', '--disable-shared',
1485                          '--disable-threads',
1486                          '--disable-libatomic',
1487                          '--disable-decimal-float',
1488                          '--disable-gcov',
1489                          '--disable-libffi',
1490                          '--disable-libgomp',
1491                          '--disable-libitm',
1492                          '--disable-libmpx',
1493                          '--disable-libquadmath',
1494                          '--disable-libsanitizer',
1495                          '--without-headers', '--with-newlib',
1496                          '--with-glibc-version=%s' % self.ctx.glibc_version
1497                          ]
1498             cfg_opts += self.first_gcc_cfg
1499         else:
1500             tool_build = 'gcc'
1501             # libsanitizer commonly breaks because of glibc header
1502             # changes, or on unusual targets.  C++ pre-compiled
1503             # headers are not used during the glibc build and are
1504             # expensive to create.
1505             if not self.ctx.full_gcc:
1506                 cfg_opts += ['--disable-libsanitizer',
1507                              '--disable-libstdcxx-pch']
1508             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1509             cfg_opts += ['--enable-languages=%s' % langs,
1510                          '--enable-shared', '--enable-threads']
1511         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1512
1513 class GlibcPolicyDefault(object):
1514     """Build policy for glibc: common defaults."""
1515
1516     def __init__(self, glibc):
1517         self.srcdir = glibc.ctx.component_srcdir('glibc')
1518         self.use_usr = glibc.os != 'gnu'
1519         self.prefix = '/usr' if self.use_usr else ''
1520         self.configure_args = [
1521             '--prefix=%s' % self.prefix,
1522             '--enable-profile',
1523             '--build=%s' % glibc.ctx.build_triplet,
1524             '--host=%s' % glibc.triplet,
1525             'CC=%s' % glibc.tool_name('gcc'),
1526             'CXX=%s' % glibc.tool_name('g++'),
1527         ]
1528         if glibc.os == 'gnu':
1529             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1530         if glibc.cflags:
1531             self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1532             self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
1533         self.configure_args += glibc.cfg
1534
1535     def configure(self, cmdlist):
1536         """Invoked to add the configure command to the command list."""
1537         cmdlist.add_command('configure',
1538                             [os.path.join(self.srcdir, 'configure'),
1539                              *self.configure_args])
1540
1541     def extra_commands(self, cmdlist):
1542         """Invoked to inject additional commands (make check) after build."""
1543         pass
1544
1545 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1546     """Build policy for glibc during the compilers stage."""
1547
1548     def __init__(self, glibc):
1549         super().__init__(glibc)
1550         self.builddir = glibc.ctx.component_builddir(
1551             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1552         self.installdir = glibc.compiler.sysroot
1553
1554 class GlibcPolicyForBuild(GlibcPolicyDefault):
1555     """Build policy for glibc during the glibcs stage."""
1556
1557     def __init__(self, glibc):
1558         super().__init__(glibc)
1559         self.builddir = glibc.ctx.component_builddir(
1560             'glibcs', glibc.name, 'glibc')
1561         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1562         if glibc.ctx.strip:
1563             self.strip = glibc.tool_name('strip')
1564         else:
1565             self.strip = None
1566         self.save_logs = glibc.ctx.save_logs
1567
1568     def extra_commands(self, cmdlist):
1569         if self.strip:
1570             # Avoid stripping libc.so and libpthread.so, which are
1571             # linker scripts stored in /lib on Hurd.
1572             find_command = 'find %s/lib* -name "*.so*"' % self.installdir
1573             cmdlist.add_command('strip', ['sh', '-c', (
1574                 'set -e; for f in $(%s); do '
1575                 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1576                 'done' % (find_command, self.strip))])
1577         cmdlist.add_command('check', ['make', 'check'])
1578         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1579
1580 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1581     """Build policy for glibc during update-syscalls."""
1582
1583     def __init__(self, glibc):
1584         super().__init__(glibc)
1585         self.builddir = glibc.ctx.component_builddir(
1586             'update-syscalls', glibc.name, 'glibc')
1587         self.linuxdir = glibc.ctx.component_builddir(
1588             'update-syscalls', glibc.name, 'linux')
1589         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1590             glibc, self.linuxdir)
1591         self.configure_args.insert(
1592             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1593         # self.installdir not set because installation is not supported
1594
1595 class Glibc(object):
1596     """A configuration for building glibc."""
1597
1598     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1599                  cfg=None, ccopts=None, cflags=None):
1600         """Initialize a Glibc object."""
1601         self.ctx = compiler.ctx
1602         self.compiler = compiler
1603         if arch is None:
1604             self.arch = compiler.arch
1605         else:
1606             self.arch = arch
1607         if os_name is None:
1608             self.os = compiler.os
1609         else:
1610             self.os = os_name
1611         self.variant = variant
1612         if variant is None:
1613             self.name = '%s-%s' % (self.arch, self.os)
1614         else:
1615             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1616         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1617         if cfg is None:
1618             self.cfg = []
1619         else:
1620             self.cfg = cfg
1621         # ccopts contain ABI options and are passed to configure as CC / CXX.
1622         self.ccopts = ccopts
1623         # cflags contain non-ABI options like -g or -O and are passed to
1624         # configure as CFLAGS / CXXFLAGS.
1625         self.cflags = cflags
1626
1627     def tool_name(self, tool):
1628         """Return the name of a cross-compilation tool."""
1629         ctool = '%s-%s' % (self.compiler.triplet, tool)
1630         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1631             ctool = '%s %s' % (ctool, self.ccopts)
1632         return ctool
1633
1634     def build(self):
1635         """Generate commands to build this glibc."""
1636         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1637         installdir = self.ctx.glibc_installdir(self.name)
1638         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1639         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1640         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1641         cmdlist.add_command('check-compilers',
1642                             ['test', '-f',
1643                              os.path.join(self.compiler.installdir, 'ok')])
1644         cmdlist.use_path(self.compiler.bindir)
1645         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1646         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1647                                       logsdir)
1648
1649     def build_glibc(self, cmdlist, policy):
1650         """Generate commands to build this glibc, either as part of a compiler
1651         build or with the bootstrapped compiler (and in the latter case, run
1652         tests as well)."""
1653         cmdlist.create_use_dir(policy.builddir)
1654         policy.configure(cmdlist)
1655         cmdlist.add_command('build', ['make'])
1656         cmdlist.add_command('install', ['make', 'install',
1657                                         'install_root=%s' % policy.installdir])
1658         # GCC uses paths such as lib/../lib64, so make sure lib
1659         # directories always exist.
1660         mkdir_cmd = ['mkdir', '-p',
1661                      os.path.join(policy.installdir, 'lib')]
1662         if policy.use_usr:
1663             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1664         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1665         policy.extra_commands(cmdlist)
1666         cmdlist.cleanup_dir()
1667
1668     def update_syscalls(self):
1669         if self.os == 'gnu':
1670             # Hurd does not have system call tables that need updating.
1671             return
1672
1673         policy = GlibcPolicyForUpdateSyscalls(self)
1674         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1675         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1676         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1677         cmdlist.add_command('check-compilers',
1678                             ['test', '-f',
1679                              os.path.join(self.compiler.installdir, 'ok')])
1680         cmdlist.use_path(self.compiler.bindir)
1681
1682         install_linux_headers(policy.linux_policy, cmdlist)
1683
1684         cmdlist.create_use_dir(policy.builddir)
1685         policy.configure(cmdlist)
1686         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1687         cmdlist.cleanup_dir()
1688         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1689                                       cmdlist, logsdir)
1690
1691 class Command(object):
1692     """A command run in the build process."""
1693
1694     def __init__(self, desc, num, dir, path, command, always_run=False):
1695         """Initialize a Command object."""
1696         self.dir = dir
1697         self.path = path
1698         self.desc = desc
1699         trans = str.maketrans({' ': '-'})
1700         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1701         self.command = command
1702         self.always_run = always_run
1703
1704     @staticmethod
1705     def shell_make_quote_string(s):
1706         """Given a string not containing a newline, quote it for use by the
1707         shell and make."""
1708         assert '\n' not in s
1709         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1710             return s
1711         strans = str.maketrans({"'": "'\\''"})
1712         s = "'%s'" % s.translate(strans)
1713         mtrans = str.maketrans({'$': '$$'})
1714         return s.translate(mtrans)
1715
1716     @staticmethod
1717     def shell_make_quote_list(l, translate_make):
1718         """Given a list of strings not containing newlines, quote them for use
1719         by the shell and make, returning a single string.  If translate_make
1720         is true and the first string is 'make', change it to $(MAKE)."""
1721         l = [Command.shell_make_quote_string(s) for s in l]
1722         if translate_make and l[0] == 'make':
1723             l[0] = '$(MAKE)'
1724         return ' '.join(l)
1725
1726     def shell_make_quote(self):
1727         """Return this command quoted for the shell and make."""
1728         return self.shell_make_quote_list(self.command, True)
1729
1730
1731 class CommandList(object):
1732     """A list of commands run in the build process."""
1733
1734     def __init__(self, desc, keep):
1735         """Initialize a CommandList object."""
1736         self.cmdlist = []
1737         self.dir = None
1738         self.path = None
1739         self.desc = [desc]
1740         self.keep = keep
1741
1742     def desc_txt(self, desc):
1743         """Return the description to use for a command."""
1744         return '%s %s' % (' '.join(self.desc), desc)
1745
1746     def use_dir(self, dir):
1747         """Set the default directory for subsequent commands."""
1748         self.dir = dir
1749
1750     def use_path(self, path):
1751         """Set a directory to be prepended to the PATH for subsequent
1752         commands."""
1753         self.path = path
1754
1755     def push_subdesc(self, subdesc):
1756         """Set the default subdescription for subsequent commands (e.g., the
1757         name of a component being built, within the series of commands
1758         building it)."""
1759         self.desc.append(subdesc)
1760
1761     def pop_subdesc(self):
1762         """Pop a subdescription from the list of descriptions."""
1763         self.desc.pop()
1764
1765     def create_use_dir(self, dir):
1766         """Remove and recreate a directory and use it for subsequent
1767         commands."""
1768         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1769         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1770         self.use_dir(dir)
1771
1772     def add_command_dir(self, desc, dir, command, always_run=False):
1773         """Add a command to run in a given directory."""
1774         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1775                       command, always_run)
1776         self.cmdlist.append(cmd)
1777
1778     def add_command(self, desc, command, always_run=False):
1779         """Add a command to run in the default directory."""
1780         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1781                       self.path, command, always_run)
1782         self.cmdlist.append(cmd)
1783
1784     def cleanup_dir(self, desc='cleanup', dir=None):
1785         """Clean up a build directory.  If no directory is specified, the
1786         default directory is cleaned up and ceases to be the default
1787         directory."""
1788         if dir is None:
1789             dir = self.dir
1790             self.use_dir(None)
1791         if self.keep != 'all':
1792             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1793                                  always_run=(self.keep == 'none'))
1794
1795     def makefile_commands(self, wrapper, logsdir):
1796         """Return the sequence of commands in the form of text for a Makefile.
1797         The given wrapper script takes arguments: base of logs for
1798         previous command, or empty; base of logs for this command;
1799         description; directory; PATH addition; the command itself."""
1800         # prev_base is the base of the name for logs of the previous
1801         # command that is not always-run (that is, a build command,
1802         # whose failure should stop subsequent build commands from
1803         # being run, as opposed to a cleanup command, which is run
1804         # even if previous commands failed).
1805         prev_base = ''
1806         cmds = []
1807         for c in self.cmdlist:
1808             ctxt = c.shell_make_quote()
1809             if prev_base and not c.always_run:
1810                 prev_log = os.path.join(logsdir, prev_base)
1811             else:
1812                 prev_log = ''
1813             this_log = os.path.join(logsdir, c.logbase)
1814             if not c.always_run:
1815                 prev_base = c.logbase
1816             if c.dir is None:
1817                 dir = ''
1818             else:
1819                 dir = c.dir
1820             if c.path is None:
1821                 path = ''
1822             else:
1823                 path = c.path
1824             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1825             prelim_txt = Command.shell_make_quote_list(prelims, False)
1826             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1827         return '\n'.join(cmds)
1828
1829     def status_logs(self, logsdir):
1830         """Return the list of log files with command status."""
1831         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1832                 for c in self.cmdlist]
1833
1834
1835 def get_parser():
1836     """Return an argument parser for this module."""
1837     parser = argparse.ArgumentParser(description=__doc__)
1838     parser.add_argument('-j', dest='parallelism',
1839                         help='Run this number of jobs in parallel',
1840                         type=int, default=os.cpu_count())
1841     parser.add_argument('--keep', dest='keep',
1842                         help='Whether to keep all build directories, '
1843                         'none or only those from failed builds',
1844                         default='none', choices=('none', 'all', 'failed'))
1845     parser.add_argument('--replace-sources', action='store_true',
1846                         help='Remove and replace source directories '
1847                         'with the wrong version of a component')
1848     parser.add_argument('--strip', action='store_true',
1849                         help='Strip installed glibc libraries')
1850     parser.add_argument('--full-gcc', action='store_true',
1851                         help='Build GCC with all languages and libsanitizer')
1852     parser.add_argument('--shallow', action='store_true',
1853                         help='Do not download Git history during checkout')
1854     parser.add_argument('topdir',
1855                         help='Toplevel working directory')
1856     parser.add_argument('action',
1857                         help='What to do',
1858                         choices=('checkout', 'bot-cycle', 'bot',
1859                                  'host-libraries', 'compilers', 'glibcs',
1860                                  'update-syscalls', 'list-compilers',
1861                                  'list-glibcs'))
1862     parser.add_argument('configs',
1863                         help='Versions to check out or configurations to build',
1864                         nargs='*')
1865     return parser
1866
1867
1868 def main(argv):
1869     """The main entry point."""
1870     parser = get_parser()
1871     opts = parser.parse_args(argv)
1872     topdir = os.path.abspath(opts.topdir)
1873     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1874                   opts.strip, opts.full_gcc, opts.action,
1875                   shallow=opts.shallow)
1876     ctx.run_builds(opts.action, opts.configs)
1877
1878
1879 if __name__ == '__main__':
1880     main(sys.argv[1:])