scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2023 Free Software Foundation, Inc.
   4 # Copyright The GNU Toolchain Authors.
   5 # This file is part of the GNU C Library.
   6 #
   7 # The GNU C Library is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU Lesser General Public
   9 # License as published by the Free Software Foundation; either
  10 # version 2.1 of the License, or (at your option) any later version.
  11 #
  12 # The GNU C Library is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 # Lesser General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Lesser General Public
  18 # License along with the GNU C Library; if not, see
  19 # <https://www.gnu.org/licenses/>.
  20
  21 """Build many configurations of glibc.
  22
  23 This script takes as arguments a directory name (containing a src
  24 subdirectory with sources of the relevant toolchain components) and a
  25 description of what to do: 'checkout', to check out sources into that
  26 directory, 'bot-cycle', to run a series of checkout and build steps,
  27 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  28 libraries required by the toolchain, 'compilers', to build
  29 cross-compilers for various configurations, or 'glibcs', to build
  30 glibc for various configurations and run the compilation parts of the
  31 testsuite.  Subsequent arguments name the versions of components to
  32 check out (<component>-<version), for 'checkout', or, for actions
  33 other than 'checkout' and 'bot-cycle', name configurations for which
  34 compilers or glibc are to be built.
  35
  36 The 'list-compilers' command prints the name of each available
  37 compiler configuration, without building anything.  The 'list-glibcs'
  38 command prints the name of each glibc compiler configuration, followed
  39 by the space, followed by the name of the compiler configuration used
  40 for building this glibc variant.
  41
  42 """
  43
  44 import argparse
  45 import datetime
  46 import email.mime.text
  47 import email.utils
  48 import json
  49 import os
  50 import re
  51 import shutil
  52 import smtplib
  53 import stat
  54 import subprocess
  55 import sys
  56 import time
  57 import urllib.request
  58
  59 # This is a list of system utilities that are expected to be available
  60 # to this script, and, if a non-zero version is included, the minimum
  61 # version required to work with this sccript.
  62 def get_list_of_required_tools():
  63     global REQUIRED_TOOLS
  64     REQUIRED_TOOLS = {
  65         'awk'      : (get_version_awk,   (0,0,0)),
  66         'bison'    : (get_version,       (0,0)),
  67         'flex'     : (get_version,       (0,0,0)),
  68         'git'      : (get_version,       (1,8,3)),
  69         'make'     : (get_version,       (4,0)),
  70         'makeinfo' : (get_version,       (0,0)),
  71         'patch'    : (get_version,       (0,0,0)),
  72         'sed'      : (get_version,       (0,0)),
  73         'tar'      : (get_version,       (0,0,0)),
  74         'gzip'     : (get_version,       (0,0)),
  75         'bzip2'    : (get_version_bzip2, (0,0,0)),
  76         'xz'       : (get_version,       (0,0,0)),
  77     }
  78
  79 try:
  80     subprocess.run
  81 except:
  82     class _CompletedProcess:
  83         def __init__(self, args, returncode, stdout=None, stderr=None):
  84             self.args = args
  85             self.returncode = returncode
  86             self.stdout = stdout
  87             self.stderr = stderr
  88
  89     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  90         assert(timeout is None)
  91         with subprocess.Popen(*popenargs, **kwargs) as process:
  92             try:
  93                 stdout, stderr = process.communicate(input)
  94             except:
  95                 process.kill()
  96                 process.wait()
  97                 raise
  98             returncode = process.poll()
  99             if check and returncode:
 100                 raise subprocess.CalledProcessError(returncode, popenargs)
 101         return _CompletedProcess(popenargs, returncode, stdout, stderr)
 102
 103     subprocess.run = _run
 104
 105
 106 class Context(object):
 107     """The global state associated with builds in a given directory."""
 108
 109     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
 110                  full_gcc, action, shallow=False):
 111         """Initialize the context."""
 112         self.topdir = topdir
 113         self.parallelism = parallelism
 114         self.keep = keep
 115         self.replace_sources = replace_sources
 116         self.strip = strip
 117         self.full_gcc = full_gcc
 118         self.shallow = shallow
 119         self.srcdir = os.path.join(topdir, 'src')
 120         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 121         self.build_state_json = os.path.join(topdir, 'build-state.json')
 122         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 123         self.installdir = os.path.join(topdir, 'install')
 124         self.host_libraries_installdir = os.path.join(self.installdir,
 125                                                       'host-libraries')
 126         self.builddir = os.path.join(topdir, 'build')
 127         self.logsdir = os.path.join(topdir, 'logs')
 128         self.logsdir_old = os.path.join(topdir, 'logs-old')
 129         self.makefile = os.path.join(self.builddir, 'Makefile')
 130         self.wrapper = os.path.join(self.builddir, 'wrapper')
 131         self.save_logs = os.path.join(self.builddir, 'save-logs')
 132         self.script_text = self.get_script_text()
 133         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 134             self.build_triplet = self.get_build_triplet()
 135             self.glibc_version = self.get_glibc_version()
 136         self.configs = {}
 137         self.glibc_configs = {}
 138         self.makefile_pieces = ['.PHONY: all\n']
 139         self.add_all_configs()
 140         self.load_versions_json()
 141         self.load_build_state_json()
 142         self.status_log_list = []
 143         self.email_warning = False
 144
 145     def get_script_text(self):
 146         """Return the text of this script."""
 147         with open(sys.argv[0], 'r') as f:
 148             return f.read()
 149
 150     def exec_self(self):
 151         """Re-execute this script with the same arguments."""
 152         sys.stdout.flush()
 153         os.execv(sys.executable, [sys.executable] + sys.argv)
 154
 155     def get_build_triplet(self):
 156         """Determine the build triplet with config.guess."""
 157         config_guess = os.path.join(self.component_srcdir('gcc'),
 158                                     'config.guess')
 159         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 160                                 check=True, universal_newlines=True).stdout
 161         return cg_out.rstrip()
 162
 163     def get_glibc_version(self):
 164         """Determine the glibc version number (major.minor)."""
 165         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 166         with open(version_h, 'r') as f:
 167             lines = f.readlines()
 168         starttext = '#define VERSION "'
 169         for l in lines:
 170             if l.startswith(starttext):
 171                 l = l[len(starttext):]
 172                 l = l.rstrip('"\n')
 173                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 174                 return '%s.%s' % m.group(1, 2)
 175         print('error: could not determine glibc version')
 176         exit(1)
 177
 178     def add_all_configs(self):
 179         """Add all known glibc build configurations."""
 180         self.add_config(arch='aarch64',
 181                         os_name='linux-gnu',
 182                         extra_glibcs=[{'variant': 'disable-multi-arch',
 183                                        'cfg': ['--disable-multi-arch']}])
 184         self.add_config(arch='aarch64_be',
 185                         os_name='linux-gnu')
 186         self.add_config(arch='arc',
 187                         os_name='linux-gnu',
 188                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 189         self.add_config(arch='arc',
 190                         os_name='linux-gnuhf',
 191                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
 192         self.add_config(arch='arceb',
 193                         os_name='linux-gnu',
 194                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 195         self.add_config(arch='alpha',
 196                         os_name='linux-gnu')
 197         self.add_config(arch='arm',
 198                         os_name='linux-gnueabi',
 199                         extra_glibcs=[{'variant': 'v4t',
 200                                        'ccopts': '-march=armv4t'}])
 201         self.add_config(arch='armeb',
 202                         os_name='linux-gnueabi')
 203         self.add_config(arch='armeb',
 204                         os_name='linux-gnueabi',
 205                         variant='be8',
 206                         gcc_cfg=['--with-arch=armv7-a'])
 207         self.add_config(arch='arm',
 208                         os_name='linux-gnueabihf',
 209                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 210                         extra_glibcs=[{'variant': 'v7a',
 211                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 212                                       {'variant': 'thumb',
 213                                        'ccopts':
 214                                        '-mthumb -march=armv7-a -mfpu=vfpv3'},
 215                                       {'variant': 'v7a-disable-multi-arch',
 216                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 217                                        'cfg': ['--disable-multi-arch']}])
 218         self.add_config(arch='armeb',
 219                         os_name='linux-gnueabihf',
 220                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 221         self.add_config(arch='armeb',
 222                         os_name='linux-gnueabihf',
 223                         variant='be8',
 224                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 225                                  '--with-fpu=vfpv3'])
 226         self.add_config(arch='csky',
 227                         os_name='linux-gnuabiv2',
 228                         variant='soft',
 229                         gcc_cfg=['--disable-multilib'])
 230         self.add_config(arch='csky',
 231                         os_name='linux-gnuabiv2',
 232                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 233         self.add_config(arch='hppa',
 234                         os_name='linux-gnu')
 235         self.add_config(arch='i686',
 236                         os_name='gnu')
 237         self.add_config(arch='ia64',
 238                         os_name='linux-gnu',
 239                         first_gcc_cfg=['--with-system-libunwind'],
 240                         binutils_cfg=['--enable-obsolete'])
 241         self.add_config(arch='loongarch64',
 242                         os_name='linux-gnu',
 243                         variant='lp64d',
 244                         gcc_cfg=['--with-abi=lp64d','--disable-multilib'])
 245         self.add_config(arch='loongarch64',
 246                         os_name='linux-gnu',
 247                         variant='lp64s',
 248                         gcc_cfg=['--with-abi=lp64s','--disable-multilib'])
 249         self.add_config(arch='m68k',
 250                         os_name='linux-gnu',
 251                         gcc_cfg=['--disable-multilib'])
 252         self.add_config(arch='m68k',
 253                         os_name='linux-gnu',
 254                         variant='coldfire',
 255                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 256         self.add_config(arch='m68k',
 257                         os_name='linux-gnu',
 258                         variant='coldfire-soft',
 259                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 260                                  '--disable-multilib'])
 261         self.add_config(arch='microblaze',
 262                         os_name='linux-gnu',
 263                         gcc_cfg=['--disable-multilib'])
 264         self.add_config(arch='microblazeel',
 265                         os_name='linux-gnu',
 266                         gcc_cfg=['--disable-multilib'])
 267         self.add_config(arch='mips64',
 268                         os_name='linux-gnu',
 269                         gcc_cfg=['--with-mips-plt'],
 270                         glibcs=[{'variant': 'n32'},
 271                                 {'arch': 'mips',
 272                                  'ccopts': '-mabi=32'},
 273                                 {'variant': 'n64',
 274                                  'ccopts': '-mabi=64'}])
 275         self.add_config(arch='mips64',
 276                         os_name='linux-gnu',
 277                         variant='soft',
 278                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 279                         glibcs=[{'variant': 'n32-soft'},
 280                                 {'variant': 'soft',
 281                                  'arch': 'mips',
 282                                  'ccopts': '-mabi=32'},
 283                                 {'variant': 'n64-soft',
 284                                  'ccopts': '-mabi=64'}])
 285         self.add_config(arch='mips64',
 286                         os_name='linux-gnu',
 287                         variant='nan2008',
 288                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 289                                  '--with-arch-64=mips64r2',
 290                                  '--with-arch-32=mips32r2'],
 291                         glibcs=[{'variant': 'n32-nan2008'},
 292                                 {'variant': 'nan2008',
 293                                  'arch': 'mips',
 294                                  'ccopts': '-mabi=32'},
 295                                 {'variant': 'n64-nan2008',
 296                                  'ccopts': '-mabi=64'}])
 297         self.add_config(arch='mips64',
 298                         os_name='linux-gnu',
 299                         variant='nan2008-soft',
 300                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 301                                  '--with-arch-64=mips64r2',
 302                                  '--with-arch-32=mips32r2',
 303                                  '--with-float=soft'],
 304                         glibcs=[{'variant': 'n32-nan2008-soft'},
 305                                 {'variant': 'nan2008-soft',
 306                                  'arch': 'mips',
 307                                  'ccopts': '-mabi=32'},
 308                                 {'variant': 'n64-nan2008-soft',
 309                                  'ccopts': '-mabi=64'}])
 310         self.add_config(arch='mips64el',
 311                         os_name='linux-gnu',
 312                         gcc_cfg=['--with-mips-plt'],
 313                         glibcs=[{'variant': 'n32'},
 314                                 {'arch': 'mipsel',
 315                                  'ccopts': '-mabi=32'},
 316                                 {'variant': 'n64',
 317                                  'ccopts': '-mabi=64'}])
 318         self.add_config(arch='mips64el',
 319                         os_name='linux-gnu',
 320                         variant='soft',
 321                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 322                         glibcs=[{'variant': 'n32-soft'},
 323                                 {'variant': 'soft',
 324                                  'arch': 'mipsel',
 325                                  'ccopts': '-mabi=32'},
 326                                 {'variant': 'n64-soft',
 327                                  'ccopts': '-mabi=64'}])
 328         self.add_config(arch='mips64el',
 329                         os_name='linux-gnu',
 330                         variant='nan2008',
 331                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 332                                  '--with-arch-64=mips64r2',
 333                                  '--with-arch-32=mips32r2'],
 334                         glibcs=[{'variant': 'n32-nan2008'},
 335                                 {'variant': 'nan2008',
 336                                  'arch': 'mipsel',
 337                                  'ccopts': '-mabi=32'},
 338                                 {'variant': 'n64-nan2008',
 339                                  'ccopts': '-mabi=64'}])
 340         self.add_config(arch='mips64el',
 341                         os_name='linux-gnu',
 342                         variant='nan2008-soft',
 343                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 344                                  '--with-arch-64=mips64r2',
 345                                  '--with-arch-32=mips32r2',
 346                                  '--with-float=soft'],
 347                         glibcs=[{'variant': 'n32-nan2008-soft'},
 348                                 {'variant': 'nan2008-soft',
 349                                  'arch': 'mipsel',
 350                                  'ccopts': '-mabi=32'},
 351                                 {'variant': 'n64-nan2008-soft',
 352                                  'ccopts': '-mabi=64'}])
 353         self.add_config(arch='mipsisa64r6el',
 354                         os_name='linux-gnu',
 355                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 356                                  '--with-arch-64=mips64r6',
 357                                  '--with-arch-32=mips32r6',
 358                                  '--with-float=hard'],
 359                         glibcs=[{'variant': 'n32'},
 360                                 {'arch': 'mipsisa32r6el',
 361                                  'ccopts': '-mabi=32'},
 362                                 {'variant': 'n64',
 363                                  'ccopts': '-mabi=64'}])
 364         self.add_config(arch='nios2',
 365                         os_name='linux-gnu')
 366         self.add_config(arch='or1k',
 367                         os_name='linux-gnu',
 368                         variant='soft',
 369                         gcc_cfg=['--with-multilib-list=mcmov'])
 370         self.add_config(arch='powerpc',
 371                         os_name='linux-gnu',
 372                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 373                         extra_glibcs=[{'variant': 'power4',
 374                                        'ccopts': '-mcpu=power4',
 375                                        'cfg': ['--with-cpu=power4']}])
 376         self.add_config(arch='powerpc',
 377                         os_name='linux-gnu',
 378                         variant='soft',
 379                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 380                                  '--enable-secureplt'])
 381         self.add_config(arch='powerpc64',
 382                         os_name='linux-gnu',
 383                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 384         self.add_config(arch='powerpc64le',
 385                         os_name='linux-gnu',
 386                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 387                         extra_glibcs=[{'variant': 'disable-multi-arch',
 388                                        'cfg': ['--disable-multi-arch']}])
 389         self.add_config(arch='riscv32',
 390                         os_name='linux-gnu',
 391                         variant='rv32imac-ilp32',
 392                         gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
 393                                  '--disable-multilib'])
 394         self.add_config(arch='riscv32',
 395                         os_name='linux-gnu',
 396                         variant='rv32imafdc-ilp32',
 397                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
 398                                  '--disable-multilib'])
 399         self.add_config(arch='riscv32',
 400                         os_name='linux-gnu',
 401                         variant='rv32imafdc-ilp32d',
 402                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
 403                                  '--disable-multilib'])
 404         self.add_config(arch='riscv64',
 405                         os_name='linux-gnu',
 406                         variant='rv64imac-lp64',
 407                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 408                                  '--disable-multilib'])
 409         self.add_config(arch='riscv64',
 410                         os_name='linux-gnu',
 411                         variant='rv64imafdc-lp64',
 412                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 413                                  '--disable-multilib'])
 414         self.add_config(arch='riscv64',
 415                         os_name='linux-gnu',
 416                         variant='rv64imafdc-lp64d',
 417                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 418                                  '--disable-multilib'])
 419         self.add_config(arch='s390x',
 420                         os_name='linux-gnu',
 421                         glibcs=[{},
 422                                 {'arch': 's390', 'ccopts': '-m31'}],
 423                         extra_glibcs=[{'variant': 'O3',
 424                                        'cflags': '-O3'}])
 425         self.add_config(arch='sh3',
 426                         os_name='linux-gnu')
 427         self.add_config(arch='sh3eb',
 428                         os_name='linux-gnu')
 429         self.add_config(arch='sh4',
 430                         os_name='linux-gnu')
 431         self.add_config(arch='sh4eb',
 432                         os_name='linux-gnu')
 433         self.add_config(arch='sh4',
 434                         os_name='linux-gnu',
 435                         variant='soft',
 436                         gcc_cfg=['--without-fp'])
 437         self.add_config(arch='sh4eb',
 438                         os_name='linux-gnu',
 439                         variant='soft',
 440                         gcc_cfg=['--without-fp'])
 441         self.add_config(arch='sparc64',
 442                         os_name='linux-gnu',
 443                         glibcs=[{},
 444                                 {'arch': 'sparcv9',
 445                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 446                         extra_glibcs=[{'variant': 'leon3',
 447                                        'arch' : 'sparcv8',
 448                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 449                                       {'variant': 'disable-multi-arch',
 450                                        'cfg': ['--disable-multi-arch']},
 451                                       {'variant': 'disable-multi-arch',
 452                                        'arch': 'sparcv9',
 453                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 454                                        'cfg': ['--disable-multi-arch']}])
 455         self.add_config(arch='x86_64',
 456                         os_name='linux-gnu',
 457                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 458                         glibcs=[{},
 459                                 {'variant': 'x32', 'ccopts': '-mx32'},
 460                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 461                         extra_glibcs=[{'variant': 'disable-multi-arch',
 462                                        'cfg': ['--disable-multi-arch']},
 463                                       {'variant': 'minimal',
 464                                        'cfg': ['--disable-multi-arch',
 465                                                '--disable-profile',
 466                                                '--disable-timezone-tools',
 467                                                '--disable-mathvec',
 468                                                '--disable-build-nscd',
 469                                                '--disable-nscd']},
 470                                       {'variant': 'no-pie',
 471                                        'cfg': ['--disable-default-pie']},
 472                                       {'variant': 'x32-no-pie',
 473                                        'ccopts': '-mx32',
 474                                        'cfg': ['--disable-default-pie']},
 475                                       {'variant': 'no-pie',
 476                                        'arch': 'i686',
 477                                        'ccopts': '-m32 -march=i686',
 478                                        'cfg': ['--disable-default-pie']},
 479                                       {'variant': 'disable-multi-arch',
 480                                        'arch': 'i686',
 481                                        'ccopts': '-m32 -march=i686',
 482                                        'cfg': ['--disable-multi-arch']},
 483                                       {'arch': 'i486',
 484                                        'ccopts': '-m32 -march=i486'},
 485                                       {'arch': 'i586',
 486                                        'ccopts': '-m32 -march=i586'},
 487                                       {'variant': 'enable-fortify-source',
 488                                        'cfg': ['--enable-fortify-source']}])
 489         self.add_config(arch='x86_64',
 490                         os_name='gnu',
 491                         gcc_cfg=['--disable-multilib'])
 492
 493     def add_config(self, **args):
 494         """Add an individual build configuration."""
 495         cfg = Config(self, **args)
 496         if cfg.name in self.configs:
 497             print('error: duplicate config %s' % cfg.name)
 498             exit(1)
 499         self.configs[cfg.name] = cfg
 500         for c in cfg.all_glibcs:
 501             if c.name in self.glibc_configs:
 502                 print('error: duplicate glibc config %s' % c.name)
 503                 exit(1)
 504             self.glibc_configs[c.name] = c
 505
 506     def component_srcdir(self, component):
 507         """Return the source directory for a given component, e.g. gcc."""
 508         return os.path.join(self.srcdir, component)
 509
 510     def component_builddir(self, action, config, component, subconfig=None):
 511         """Return the directory to use for a build."""
 512         if config is None:
 513             # Host libraries.
 514             assert subconfig is None
 515             return os.path.join(self.builddir, action, component)
 516         if subconfig is None:
 517             return os.path.join(self.builddir, action, config, component)
 518         else:
 519             # glibc build as part of compiler build.
 520             return os.path.join(self.builddir, action, config, component,
 521                                 subconfig)
 522
 523     def compiler_installdir(self, config):
 524         """Return the directory in which to install a compiler."""
 525         return os.path.join(self.installdir, 'compilers', config)
 526
 527     def compiler_bindir(self, config):
 528         """Return the directory in which to find compiler binaries."""
 529         return os.path.join(self.compiler_installdir(config), 'bin')
 530
 531     def compiler_sysroot(self, config):
 532         """Return the sysroot directory for a compiler."""
 533         return os.path.join(self.compiler_installdir(config), 'sysroot')
 534
 535     def glibc_installdir(self, config):
 536         """Return the directory in which to install glibc."""
 537         return os.path.join(self.installdir, 'glibcs', config)
 538
 539     def run_builds(self, action, configs):
 540         """Run the requested builds."""
 541         if action == 'checkout':
 542             self.checkout(configs)
 543             return
 544         if action == 'bot-cycle':
 545             if configs:
 546                 print('error: configurations specified for bot-cycle')
 547                 exit(1)
 548             self.bot_cycle()
 549             return
 550         if action == 'bot':
 551             if configs:
 552                 print('error: configurations specified for bot')
 553                 exit(1)
 554             self.bot()
 555             return
 556         if action in ('host-libraries', 'list-compilers',
 557                       'list-glibcs') and configs:
 558             print('error: configurations specified for ' + action)
 559             exit(1)
 560         if action == 'list-compilers':
 561             for name in sorted(self.configs.keys()):
 562                 print(name)
 563             return
 564         if action == 'list-glibcs':
 565             for config in sorted(self.glibc_configs.values(),
 566                                  key=lambda c: c.name):
 567                 print(config.name, config.compiler.name)
 568             return
 569         self.clear_last_build_state(action)
 570         build_time = datetime.datetime.utcnow()
 571         if action == 'host-libraries':
 572             build_components = ('gmp', 'mpfr', 'mpc')
 573             old_components = ()
 574             old_versions = {}
 575             self.build_host_libraries()
 576         elif action == 'compilers':
 577             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 578                                 'gnumach', 'hurd')
 579             old_components = ('gmp', 'mpfr', 'mpc')
 580             old_versions = self.build_state['host-libraries']['build-versions']
 581             self.build_compilers(configs)
 582         else:
 583             build_components = ('glibc',)
 584             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 585                               'mig', 'gnumach', 'hurd')
 586             old_versions = self.build_state['compilers']['build-versions']
 587             if action == 'update-syscalls':
 588                 self.update_syscalls(configs)
 589             else:
 590                 self.build_glibcs(configs)
 591         self.write_files()
 592         self.do_build()
 593         if configs:
 594             # Partial build, do not update stored state.
 595             return
 596         build_versions = {}
 597         for k in build_components:
 598             if k in self.versions:
 599                 build_versions[k] = {'version': self.versions[k]['version'],
 600                                      'revision': self.versions[k]['revision']}
 601         for k in old_components:
 602             if k in old_versions:
 603                 build_versions[k] = {'version': old_versions[k]['version'],
 604                                      'revision': old_versions[k]['revision']}
 605         self.update_build_state(action, build_time, build_versions)
 606
 607     @staticmethod
 608     def remove_dirs(*args):
 609         """Remove directories and their contents if they exist."""
 610         for dir in args:
 611             shutil.rmtree(dir, ignore_errors=True)
 612
 613     @staticmethod
 614     def remove_recreate_dirs(*args):
 615         """Remove directories if they exist, and create them as empty."""
 616         Context.remove_dirs(*args)
 617         for dir in args:
 618             os.makedirs(dir, exist_ok=True)
 619
 620     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 621         """Add makefile text for a list of commands."""
 622         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 623         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 624                                     (target, target, target, commands))
 625         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 626
 627     def write_files(self):
 628         """Write out the Makefile and wrapper script."""
 629         mftext = ''.join(self.makefile_pieces)
 630         with open(self.makefile, 'w') as f:
 631             f.write(mftext)
 632         wrapper_text = (
 633             '#!/bin/sh\n'
 634             'prev_base=$1\n'
 635             'this_base=$2\n'
 636             'desc=$3\n'
 637             'dir=$4\n'
 638             'path=$5\n'
 639             'shift 5\n'
 640             'prev_status=$prev_base-status.txt\n'
 641             'this_status=$this_base-status.txt\n'
 642             'this_log=$this_base-log.txt\n'
 643             'date > "$this_log"\n'
 644             'echo >> "$this_log"\n'
 645             'echo "Description: $desc" >> "$this_log"\n'
 646             'printf "%s" "Command:" >> "$this_log"\n'
 647             'for word in "$@"; do\n'
 648             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 649             '    printf " %s" "$word"\n'
 650             '  else\n'
 651             '    printf " \'"\n'
 652             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 653             '    printf "\'"\n'
 654             '  fi\n'
 655             'done >> "$this_log"\n'
 656             'echo >> "$this_log"\n'
 657             'echo "Directory: $dir" >> "$this_log"\n'
 658             'echo "Path addition: $path" >> "$this_log"\n'
 659             'echo >> "$this_log"\n'
 660             'record_status ()\n'
 661             '{\n'
 662             '  echo >> "$this_log"\n'
 663             '  echo "$1: $desc" > "$this_status"\n'
 664             '  echo "$1: $desc" >> "$this_log"\n'
 665             '  echo >> "$this_log"\n'
 666             '  date >> "$this_log"\n'
 667             '  echo "$1: $desc"\n'
 668             '  exit 0\n'
 669             '}\n'
 670             'check_error ()\n'
 671             '{\n'
 672             '  if [ "$1" != "0" ]; then\n'
 673             '    record_status FAIL\n'
 674             '  fi\n'
 675             '}\n'
 676             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 677             '    record_status UNRESOLVED\n'
 678             'fi\n'
 679             'if [ "$dir" ]; then\n'
 680             '  cd "$dir"\n'
 681             '  check_error "$?"\n'
 682             'fi\n'
 683             'if [ "$path" ]; then\n'
 684             '  PATH=$path:$PATH\n'
 685             'fi\n'
 686             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 687             'check_error "$?"\n'
 688             'record_status PASS\n')
 689         with open(self.wrapper, 'w') as f:
 690             f.write(wrapper_text)
 691         # Mode 0o755.
 692         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 693                      stat.S_IROTH|stat.S_IXOTH)
 694         os.chmod(self.wrapper, mode_exec)
 695         save_logs_text = (
 696             '#!/bin/sh\n'
 697             'if ! [ -f tests.sum ]; then\n'
 698             '  echo "No test summary available."\n'
 699             '  exit 0\n'
 700             'fi\n'
 701             'save_file ()\n'
 702             '{\n'
 703             '  echo "Contents of $1:"\n'
 704             '  echo\n'
 705             '  cat "$1"\n'
 706             '  echo\n'
 707             '  echo "End of contents of $1."\n'
 708             '  echo\n'
 709             '}\n'
 710             'save_file tests.sum\n'
 711             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 712             'for t in $non_pass_tests; do\n'
 713             '  if [ -f "$t.out" ]; then\n'
 714             '    save_file "$t.out"\n'
 715             '  fi\n'
 716             'done\n')
 717         with open(self.save_logs, 'w') as f:
 718             f.write(save_logs_text)
 719         os.chmod(self.save_logs, mode_exec)
 720
 721     def do_build(self):
 722         """Do the actual build."""
 723         cmd = ['make', '-O', '-j%d' % self.parallelism]
 724         subprocess.run(cmd, cwd=self.builddir, check=True)
 725
 726     def build_host_libraries(self):
 727         """Build the host libraries."""
 728         installdir = self.host_libraries_installdir
 729         builddir = os.path.join(self.builddir, 'host-libraries')
 730         logsdir = os.path.join(self.logsdir, 'host-libraries')
 731         self.remove_recreate_dirs(installdir, builddir, logsdir)
 732         cmdlist = CommandList('host-libraries', self.keep)
 733         self.build_host_library(cmdlist, 'gmp')
 734         self.build_host_library(cmdlist, 'mpfr',
 735                                 ['--with-gmp=%s' % installdir])
 736         self.build_host_library(cmdlist, 'mpc',
 737                                 ['--with-gmp=%s' % installdir,
 738                                 '--with-mpfr=%s' % installdir])
 739         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 740         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 741
 742     def build_host_library(self, cmdlist, lib, extra_opts=None):
 743         """Build one host library."""
 744         srcdir = self.component_srcdir(lib)
 745         builddir = self.component_builddir('host-libraries', None, lib)
 746         installdir = self.host_libraries_installdir
 747         cmdlist.push_subdesc(lib)
 748         cmdlist.create_use_dir(builddir)
 749         cfg_cmd = [os.path.join(srcdir, 'configure'),
 750                    '--prefix=%s' % installdir,
 751                    '--disable-shared']
 752         if extra_opts:
 753             cfg_cmd.extend (extra_opts)
 754         cmdlist.add_command('configure', cfg_cmd)
 755         cmdlist.add_command('build', ['make'])
 756         cmdlist.add_command('check', ['make', 'check'])
 757         cmdlist.add_command('install', ['make', 'install'])
 758         cmdlist.cleanup_dir()
 759         cmdlist.pop_subdesc()
 760
 761     def build_compilers(self, configs):
 762         """Build the compilers."""
 763         if not configs:
 764             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 765             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 766             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 767             configs = sorted(self.configs.keys())
 768         for c in configs:
 769             self.configs[c].build()
 770
 771     def build_glibcs(self, configs):
 772         """Build the glibcs."""
 773         if not configs:
 774             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 775             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 776             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 777             configs = sorted(self.glibc_configs.keys())
 778         for c in configs:
 779             self.glibc_configs[c].build()
 780
 781     def update_syscalls(self, configs):
 782         """Update the glibc syscall lists."""
 783         if not configs:
 784             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 785             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 786             configs = sorted(self.glibc_configs.keys())
 787         for c in configs:
 788             self.glibc_configs[c].update_syscalls()
 789
 790     def load_versions_json(self):
 791         """Load information about source directory versions."""
 792         if not os.access(self.versions_json, os.F_OK):
 793             self.versions = {}
 794             return
 795         with open(self.versions_json, 'r') as f:
 796             self.versions = json.load(f)
 797
 798     def store_json(self, data, filename):
 799         """Store information in a JSON file."""
 800         filename_tmp = filename + '.tmp'
 801         with open(filename_tmp, 'w') as f:
 802             json.dump(data, f, indent=2, sort_keys=True)
 803         os.rename(filename_tmp, filename)
 804
 805     def store_versions_json(self):
 806         """Store information about source directory versions."""
 807         self.store_json(self.versions, self.versions_json)
 808
 809     def set_component_version(self, component, version, explicit, revision):
 810         """Set the version information for a component."""
 811         self.versions[component] = {'version': version,
 812                                     'explicit': explicit,
 813                                     'revision': revision}
 814         self.store_versions_json()
 815
 816     def checkout(self, versions):
 817         """Check out the desired component versions."""
 818         default_versions = {'binutils': 'vcs-2.41',
 819                             'gcc': 'vcs-13',
 820                             'glibc': 'vcs-mainline',
 821                             'gmp': '6.3.0',
 822                             'linux': '6.6',
 823                             'mpc': '1.3.1',
 824                             'mpfr': '4.2.1',
 825                             'mig': 'vcs-mainline',
 826                             'gnumach': 'vcs-mainline',
 827                             'hurd': 'vcs-mainline'}
 828         use_versions = {}
 829         explicit_versions = {}
 830         for v in versions:
 831             found_v = False
 832             for k in default_versions.keys():
 833                 kx = k + '-'
 834                 if v.startswith(kx):
 835                     vx = v[len(kx):]
 836                     if k in use_versions:
 837                         print('error: multiple versions for %s' % k)
 838                         exit(1)
 839                     use_versions[k] = vx
 840                     explicit_versions[k] = True
 841                     found_v = True
 842                     break
 843             if not found_v:
 844                 print('error: unknown component in %s' % v)
 845                 exit(1)
 846         for k in default_versions.keys():
 847             if k not in use_versions:
 848                 if k in self.versions and self.versions[k]['explicit']:
 849                     use_versions[k] = self.versions[k]['version']
 850                     explicit_versions[k] = True
 851                 else:
 852                     use_versions[k] = default_versions[k]
 853                     explicit_versions[k] = False
 854         os.makedirs(self.srcdir, exist_ok=True)
 855         for k in sorted(default_versions.keys()):
 856             update = os.access(self.component_srcdir(k), os.F_OK)
 857             v = use_versions[k]
 858             if (update and
 859                 k in self.versions and
 860                 v != self.versions[k]['version']):
 861                 if not self.replace_sources:
 862                     print('error: version of %s has changed from %s to %s, '
 863                           'use --replace-sources to check out again' %
 864                           (k, self.versions[k]['version'], v))
 865                     exit(1)
 866                 shutil.rmtree(self.component_srcdir(k))
 867                 update = False
 868             if v.startswith('vcs-'):
 869                 revision = self.checkout_vcs(k, v[4:], update)
 870             else:
 871                 self.checkout_tar(k, v, update)
 872                 revision = v
 873             self.set_component_version(k, v, explicit_versions[k], revision)
 874         if self.get_script_text() != self.script_text:
 875             # Rerun the checkout process in case the updated script
 876             # uses different default versions or new components.
 877             self.exec_self()
 878
 879     def checkout_vcs(self, component, version, update):
 880         """Check out the given version of the given component from version
 881         control.  Return a revision identifier."""
 882         if component == 'binutils':
 883             git_url = 'https://sourceware.org/git/binutils-gdb.git'
 884             if version == 'mainline':
 885                 git_branch = 'master'
 886             else:
 887                 trans = str.maketrans({'.': '_'})
 888                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 889             return self.git_checkout(component, git_url, git_branch, update)
 890         elif component == 'gcc':
 891             if version == 'mainline':
 892                 branch = 'master'
 893             else:
 894                 branch = 'releases/gcc-%s' % version
 895             return self.gcc_checkout(branch, update)
 896         elif component == 'glibc':
 897             git_url = 'https://sourceware.org/git/glibc.git'
 898             if version == 'mainline':
 899                 git_branch = 'master'
 900             else:
 901                 git_branch = 'release/%s/master' % version
 902             r = self.git_checkout(component, git_url, git_branch, update)
 903             self.fix_glibc_timestamps()
 904             return r
 905         elif component == 'gnumach':
 906             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 907             git_branch = 'master'
 908             r = self.git_checkout(component, git_url, git_branch, update)
 909             subprocess.run(['autoreconf', '-i'],
 910                            cwd=self.component_srcdir(component), check=True)
 911             return r
 912         elif component == 'mig':
 913             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 914             git_branch = 'master'
 915             r = self.git_checkout(component, git_url, git_branch, update)
 916             subprocess.run(['autoreconf', '-i'],
 917                            cwd=self.component_srcdir(component), check=True)
 918             return r
 919         elif component == 'hurd':
 920             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 921             git_branch = 'master'
 922             r = self.git_checkout(component, git_url, git_branch, update)
 923             subprocess.run(['autoconf'],
 924                            cwd=self.component_srcdir(component), check=True)
 925             return r
 926         else:
 927             print('error: component %s coming from VCS' % component)
 928             exit(1)
 929
 930     def git_checkout(self, component, git_url, git_branch, update):
 931         """Check out a component from git.  Return a commit identifier."""
 932         if update:
 933             subprocess.run(['git', 'remote', 'prune', 'origin'],
 934                            cwd=self.component_srcdir(component), check=True)
 935             if self.replace_sources:
 936                 subprocess.run(['git', 'clean', '-dxfq'],
 937                                cwd=self.component_srcdir(component), check=True)
 938             subprocess.run(['git', 'pull', '-q'],
 939                            cwd=self.component_srcdir(component), check=True)
 940         else:
 941             if self.shallow:
 942                 depth_arg = ('--depth', '1')
 943             else:
 944                 depth_arg = ()
 945             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 946                             *depth_arg, git_url,
 947                             self.component_srcdir(component)], check=True)
 948         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 949                            cwd=self.component_srcdir(component),
 950                            stdout=subprocess.PIPE,
 951                            check=True, universal_newlines=True).stdout
 952         return r.rstrip()
 953
 954     def fix_glibc_timestamps(self):
 955         """Fix timestamps in a glibc checkout."""
 956         # Ensure that builds do not try to regenerate generated files
 957         # in the source tree.
 958         srcdir = self.component_srcdir('glibc')
 959         # These files have Makefile dependencies to regenerate them in
 960         # the source tree that may be active during a normal build.
 961         # Some other files have such dependencies but do not need to
 962         # be touched because nothing in a build depends on the files
 963         # in question.
 964         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 965             to_touch = os.path.join(srcdir, f)
 966             subprocess.run(['touch', '-c', to_touch], check=True)
 967         for dirpath, dirnames, filenames in os.walk(srcdir):
 968             for f in filenames:
 969                 if (f == 'configure' or
 970                     f == 'preconfigure' or
 971                     f.endswith('-kw.h')):
 972                     to_touch = os.path.join(dirpath, f)
 973                     subprocess.run(['touch', to_touch], check=True)
 974
 975     def gcc_checkout(self, branch, update):
 976         """Check out GCC from git.  Return the commit identifier."""
 977         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 978                      os.F_OK):
 979             if not self.replace_sources:
 980                 print('error: GCC has moved from SVN to git, use '
 981                       '--replace-sources to check out again')
 982                 exit(1)
 983             shutil.rmtree(self.component_srcdir('gcc'))
 984             update = False
 985         if not update:
 986             self.git_checkout('gcc', 'https://gcc.gnu.org/git/gcc.git',
 987                               branch, update)
 988         subprocess.run(['contrib/gcc_update', '--silent'],
 989                        cwd=self.component_srcdir('gcc'), check=True)
 990         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 991                            cwd=self.component_srcdir('gcc'),
 992                            stdout=subprocess.PIPE,
 993                            check=True, universal_newlines=True).stdout
 994         return r.rstrip()
 995
 996     def checkout_tar(self, component, version, update):
 997         """Check out the given version of the given component from a
 998         tarball."""
 999         if update:
1000             return
1001         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
1002                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
1003                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
1004                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
1005                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
1006                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
1007                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
1008                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
1009                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
1010         if component not in url_map:
1011             print('error: component %s coming from tarball' % component)
1012             exit(1)
1013         version_major = version.split('.')[0]
1014         url = url_map[component] % {'version': version, 'major': version_major}
1015         filename = os.path.join(self.srcdir, url.split('/')[-1])
1016         response = urllib.request.urlopen(url)
1017         data = response.read()
1018         with open(filename, 'wb') as f:
1019             f.write(data)
1020         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
1021                        check=True)
1022         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
1023                   self.component_srcdir(component))
1024         os.remove(filename)
1025
1026     def load_build_state_json(self):
1027         """Load information about the state of previous builds."""
1028         if os.access(self.build_state_json, os.F_OK):
1029             with open(self.build_state_json, 'r') as f:
1030                 self.build_state = json.load(f)
1031         else:
1032             self.build_state = {}
1033         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
1034             if k not in self.build_state:
1035                 self.build_state[k] = {}
1036             if 'build-time' not in self.build_state[k]:
1037                 self.build_state[k]['build-time'] = ''
1038             if 'build-versions' not in self.build_state[k]:
1039                 self.build_state[k]['build-versions'] = {}
1040             if 'build-results' not in self.build_state[k]:
1041                 self.build_state[k]['build-results'] = {}
1042             if 'result-changes' not in self.build_state[k]:
1043                 self.build_state[k]['result-changes'] = {}
1044             if 'ever-passed' not in self.build_state[k]:
1045                 self.build_state[k]['ever-passed'] = []
1046
1047     def store_build_state_json(self):
1048         """Store information about the state of previous builds."""
1049         self.store_json(self.build_state, self.build_state_json)
1050
1051     def clear_last_build_state(self, action):
1052         """Clear information about the state of part of the build."""
1053         # We clear the last build time and versions when starting a
1054         # new build.  The results of the last build are kept around,
1055         # as comparison is still meaningful if this build is aborted
1056         # and a new one started.
1057         self.build_state[action]['build-time'] = ''
1058         self.build_state[action]['build-versions'] = {}
1059         self.store_build_state_json()
1060
1061     def update_build_state(self, action, build_time, build_versions):
1062         """Update the build state after a build."""
1063         build_time = build_time.replace(microsecond=0)
1064         self.build_state[action]['build-time'] = str(build_time)
1065         self.build_state[action]['build-versions'] = build_versions
1066         build_results = {}
1067         for log in self.status_log_list:
1068             with open(log, 'r') as f:
1069                 log_text = f.read()
1070             log_text = log_text.rstrip()
1071             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1072             result = m.group(1)
1073             test_name = m.group(2)
1074             assert test_name not in build_results
1075             build_results[test_name] = result
1076         old_build_results = self.build_state[action]['build-results']
1077         self.build_state[action]['build-results'] = build_results
1078         result_changes = {}
1079         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1080         for t in all_tests:
1081             if t in old_build_results:
1082                 old_res = old_build_results[t]
1083             else:
1084                 old_res = '(New test)'
1085             if t in build_results:
1086                 new_res = build_results[t]
1087             else:
1088                 new_res = '(Test removed)'
1089             if old_res != new_res:
1090                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1091         self.build_state[action]['result-changes'] = result_changes
1092         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1093                            if t in build_results}
1094         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1095         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1096                                                          new_passes)
1097         self.store_build_state_json()
1098
1099     def load_bot_config_json(self):
1100         """Load bot configuration."""
1101         with open(self.bot_config_json, 'r') as f:
1102             self.bot_config = json.load(f)
1103
1104     def part_build_old(self, action, delay):
1105         """Return whether the last build for a given action was at least a
1106         given number of seconds ago, or does not have a time recorded."""
1107         old_time_str = self.build_state[action]['build-time']
1108         if not old_time_str:
1109             return True
1110         old_time = datetime.datetime.strptime(old_time_str,
1111                                               '%Y-%m-%d %H:%M:%S')
1112         new_time = datetime.datetime.utcnow()
1113         delta = new_time - old_time
1114         return delta.total_seconds() >= delay
1115
1116     def bot_cycle(self):
1117         """Run a single round of checkout and builds."""
1118         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1119         self.load_bot_config_json()
1120         actions = ('host-libraries', 'compilers', 'glibcs')
1121         self.bot_run_self(['--replace-sources'], 'checkout')
1122         self.load_versions_json()
1123         if self.get_script_text() != self.script_text:
1124             print('Script changed, re-execing.')
1125             # On script change, all parts of the build should be rerun.
1126             for a in actions:
1127                 self.clear_last_build_state(a)
1128             self.exec_self()
1129         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1130                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1131                                           'mig', 'gnumach', 'hurd'),
1132                             'glibcs': ('glibc',)}
1133         must_build = {}
1134         for a in actions:
1135             build_vers = self.build_state[a]['build-versions']
1136             must_build[a] = False
1137             if not self.build_state[a]['build-time']:
1138                 must_build[a] = True
1139             old_vers = {}
1140             new_vers = {}
1141             for c in check_components[a]:
1142                 if c in build_vers:
1143                     old_vers[c] = build_vers[c]
1144                 new_vers[c] = {'version': self.versions[c]['version'],
1145                                'revision': self.versions[c]['revision']}
1146             if new_vers == old_vers:
1147                 print('Versions for %s unchanged.' % a)
1148             else:
1149                 print('Versions changed or rebuild forced for %s.' % a)
1150                 if a == 'compilers' and not self.part_build_old(
1151                         a, self.bot_config['compilers-rebuild-delay']):
1152                     print('Not requiring rebuild of compilers this soon.')
1153                 else:
1154                     must_build[a] = True
1155         if must_build['host-libraries']:
1156             must_build['compilers'] = True
1157         if must_build['compilers']:
1158             must_build['glibcs'] = True
1159         for a in actions:
1160             if must_build[a]:
1161                 print('Must rebuild %s.' % a)
1162                 self.clear_last_build_state(a)
1163             else:
1164                 print('No need to rebuild %s.' % a)
1165         if os.access(self.logsdir, os.F_OK):
1166             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1167             shutil.copytree(self.logsdir, self.logsdir_old)
1168         for a in actions:
1169             if must_build[a]:
1170                 build_time = datetime.datetime.utcnow()
1171                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1172                 self.bot_run_self([], a)
1173                 self.load_build_state_json()
1174                 self.bot_build_mail(a, build_time)
1175         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1176
1177     def bot_build_mail(self, action, build_time):
1178         """Send email with the results of a build."""
1179         if not ('email-from' in self.bot_config and
1180                 'email-server' in self.bot_config and
1181                 'email-subject' in self.bot_config and
1182                 'email-to' in self.bot_config):
1183             if not self.email_warning:
1184                 print("Email not configured, not sending.")
1185                 self.email_warning = True
1186             return
1187
1188         build_time = build_time.replace(microsecond=0)
1189         subject = (self.bot_config['email-subject'] %
1190                    {'action': action,
1191                     'build-time': str(build_time)})
1192         results = self.build_state[action]['build-results']
1193         changes = self.build_state[action]['result-changes']
1194         ever_passed = set(self.build_state[action]['ever-passed'])
1195         versions = self.build_state[action]['build-versions']
1196         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1197         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1198         all_fails = {k for k in results if results[k] == 'FAIL'}
1199         if new_regressions:
1200             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1201             new_reg_text = ('New regressions:\n\n%s\n\n' %
1202                             '\n'.join(new_reg_list))
1203         else:
1204             new_reg_text = ''
1205         if all_regressions:
1206             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1207             all_reg_text = ('All regressions:\n\n%s\n\n' %
1208                             '\n'.join(all_reg_list))
1209         else:
1210             all_reg_text = ''
1211         if all_fails:
1212             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1213             all_fail_text = ('All failures:\n\n%s\n\n' %
1214                              '\n'.join(all_fail_list))
1215         else:
1216             all_fail_text = ''
1217         if changes:
1218             changes_list = sorted(changes.keys())
1219             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1220             changes_text = ('All changed results:\n\n%s\n\n' %
1221                             '\n'.join(changes_list))
1222         else:
1223             changes_text = ''
1224         results_text = (new_reg_text + all_reg_text + all_fail_text +
1225                         changes_text)
1226         if not results_text:
1227             results_text = 'Clean build with unchanged results.\n\n'
1228         versions_list = sorted(versions.keys())
1229         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1230                                           versions[k]['revision'])
1231                          for k in versions_list]
1232         versions_text = ('Component versions for this build:\n\n%s\n' %
1233                          '\n'.join(versions_list))
1234         body_text = results_text + versions_text
1235         msg = email.mime.text.MIMEText(body_text)
1236         msg['Subject'] = subject
1237         msg['From'] = self.bot_config['email-from']
1238         msg['To'] = self.bot_config['email-to']
1239         msg['Message-ID'] = email.utils.make_msgid()
1240         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1241         with smtplib.SMTP(self.bot_config['email-server']) as s:
1242             s.send_message(msg)
1243
1244     def bot_run_self(self, opts, action, check=True):
1245         """Run a copy of this script with given options."""
1246         cmd = [sys.executable, sys.argv[0], '--keep=none',
1247                '-j%d' % self.parallelism]
1248         if self.full_gcc:
1249             cmd.append('--full-gcc')
1250         cmd.extend(opts)
1251         cmd.extend([self.topdir, action])
1252         sys.stdout.flush()
1253         subprocess.run(cmd, check=check)
1254
1255     def bot(self):
1256         """Run repeated rounds of checkout and builds."""
1257         while True:
1258             self.load_bot_config_json()
1259             if not self.bot_config['run']:
1260                 print('Bot exiting by request.')
1261                 exit(0)
1262             self.bot_run_self([], 'bot-cycle', check=False)
1263             self.load_bot_config_json()
1264             if not self.bot_config['run']:
1265                 print('Bot exiting by request.')
1266                 exit(0)
1267             time.sleep(self.bot_config['delay'])
1268             if self.get_script_text() != self.script_text:
1269                 print('Script changed, bot re-execing.')
1270                 self.exec_self()
1271
1272 class LinuxHeadersPolicyForBuild(object):
1273     """Names and directories for installing Linux headers.  Build variant."""
1274
1275     def __init__(self, config):
1276         self.arch = config.arch
1277         self.srcdir = config.ctx.component_srcdir('linux')
1278         self.builddir = config.component_builddir('linux')
1279         self.headers_dir = os.path.join(config.sysroot, 'usr')
1280
1281 class LinuxHeadersPolicyForUpdateSyscalls(object):
1282     """Names and directories for Linux headers.  update-syscalls variant."""
1283
1284     def __init__(self, glibc, headers_dir):
1285         self.arch = glibc.compiler.arch
1286         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1287         self.builddir = glibc.ctx.component_builddir(
1288             'update-syscalls', glibc.name, 'build-linux')
1289         self.headers_dir = headers_dir
1290
1291 def install_linux_headers(policy, cmdlist):
1292     """Install Linux kernel headers."""
1293     arch_map = {'aarch64': 'arm64',
1294                 'alpha': 'alpha',
1295                 'arc': 'arc',
1296                 'arm': 'arm',
1297                 'csky': 'csky',
1298                 'hppa': 'parisc',
1299                 'i486': 'x86',
1300                 'i586': 'x86',
1301                 'i686': 'x86',
1302                 'i786': 'x86',
1303                 'ia64': 'ia64',
1304                 'loongarch64': 'loongarch',
1305                 'm68k': 'm68k',
1306                 'microblaze': 'microblaze',
1307                 'mips': 'mips',
1308                 'nios2': 'nios2',
1309                 'or1k': 'openrisc',
1310                 'powerpc': 'powerpc',
1311                 's390': 's390',
1312                 'riscv32': 'riscv',
1313                 'riscv64': 'riscv',
1314                 'sh': 'sh',
1315                 'sparc': 'sparc',
1316                 'x86_64': 'x86'}
1317     linux_arch = None
1318     for k in arch_map:
1319         if policy.arch.startswith(k):
1320             linux_arch = arch_map[k]
1321             break
1322     assert linux_arch is not None
1323     cmdlist.push_subdesc('linux')
1324     cmdlist.create_use_dir(policy.builddir)
1325     cmdlist.add_command('install-headers',
1326                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1327                          'ARCH=%s' % linux_arch,
1328                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1329                          'headers_install'])
1330     cmdlist.cleanup_dir()
1331     cmdlist.pop_subdesc()
1332
1333 class Config(object):
1334     """A configuration for building a compiler and associated libraries."""
1335
1336     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1337                  first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1338                  extra_glibcs=None):
1339         """Initialize a Config object."""
1340         self.ctx = ctx
1341         self.arch = arch
1342         self.os = os_name
1343         self.variant = variant
1344         if variant is None:
1345             self.name = '%s-%s' % (arch, os_name)
1346         else:
1347             self.name = '%s-%s-%s' % (arch, os_name, variant)
1348         self.triplet = '%s-glibc-%s' % (arch, os_name)
1349         if gcc_cfg is None:
1350             self.gcc_cfg = []
1351         else:
1352             self.gcc_cfg = gcc_cfg
1353         if first_gcc_cfg is None:
1354             self.first_gcc_cfg = []
1355         else:
1356             self.first_gcc_cfg = first_gcc_cfg
1357         if binutils_cfg is None:
1358             self.binutils_cfg = []
1359         else:
1360             self.binutils_cfg = binutils_cfg
1361         if glibcs is None:
1362             glibcs = [{'variant': variant}]
1363         if extra_glibcs is None:
1364             extra_glibcs = []
1365         glibcs = [Glibc(self, **g) for g in glibcs]
1366         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1367         self.all_glibcs = glibcs + extra_glibcs
1368         self.compiler_glibcs = glibcs
1369         self.installdir = ctx.compiler_installdir(self.name)
1370         self.bindir = ctx.compiler_bindir(self.name)
1371         self.sysroot = ctx.compiler_sysroot(self.name)
1372         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1373         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1374
1375     def component_builddir(self, component):
1376         """Return the directory to use for a (non-glibc) build."""
1377         return self.ctx.component_builddir('compilers', self.name, component)
1378
1379     def build(self):
1380         """Generate commands to build this compiler."""
1381         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1382                                       self.logsdir)
1383         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1384         cmdlist.add_command('check-host-libraries',
1385                             ['test', '-f',
1386                              os.path.join(self.ctx.host_libraries_installdir,
1387                                           'ok')])
1388         cmdlist.use_path(self.bindir)
1389         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1390                               ['--disable-gdb',
1391                                '--disable-gdbserver',
1392                                '--disable-libdecnumber',
1393                                '--disable-readline',
1394                                '--disable-sim'] + self.binutils_cfg)
1395         if self.os.startswith('linux'):
1396             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1397         self.build_gcc(cmdlist, True)
1398         if self.os == 'gnu':
1399             self.install_gnumach_headers(cmdlist)
1400             self.build_cross_tool(cmdlist, 'mig', 'mig')
1401             self.install_hurd_headers(cmdlist)
1402         for g in self.compiler_glibcs:
1403             cmdlist.push_subdesc('glibc')
1404             cmdlist.push_subdesc(g.name)
1405             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1406             cmdlist.pop_subdesc()
1407             cmdlist.pop_subdesc()
1408         self.build_gcc(cmdlist, False)
1409         cmdlist.add_command('done', ['touch',
1410                                      os.path.join(self.installdir, 'ok')])
1411         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1412                                       self.logsdir)
1413
1414     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1415         """Build one cross tool."""
1416         srcdir = self.ctx.component_srcdir(tool_src)
1417         builddir = self.component_builddir(tool_build)
1418         cmdlist.push_subdesc(tool_build)
1419         cmdlist.create_use_dir(builddir)
1420         cfg_cmd = [os.path.join(srcdir, 'configure'),
1421                    '--prefix=%s' % self.installdir,
1422                    '--build=%s' % self.ctx.build_triplet,
1423                    '--host=%s' % self.ctx.build_triplet,
1424                    '--target=%s' % self.triplet,
1425                    '--with-sysroot=%s' % self.sysroot]
1426         if extra_opts:
1427             cfg_cmd.extend(extra_opts)
1428         cmdlist.add_command('configure', cfg_cmd)
1429         cmdlist.add_command('build', ['make'])
1430         # Parallel "make install" for GCC has race conditions that can
1431         # cause it to fail; see
1432         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1433         # problems are not known for binutils, but doing the
1434         # installation in parallel within a particular toolchain build
1435         # (as opposed to installation of one toolchain from
1436         # build-many-glibcs.py running in parallel to the installation
1437         # of other toolchains being built) is not known to be
1438         # significantly beneficial, so it is simplest just to disable
1439         # parallel install for cross tools here.
1440         cmdlist.add_command('install', ['make', '-j1', 'install'])
1441         cmdlist.cleanup_dir()
1442         cmdlist.pop_subdesc()
1443
1444     def install_gnumach_headers(self, cmdlist):
1445         """Install GNU Mach headers."""
1446         srcdir = self.ctx.component_srcdir('gnumach')
1447         builddir = self.component_builddir('gnumach')
1448         cmdlist.push_subdesc('gnumach')
1449         cmdlist.create_use_dir(builddir)
1450         cmdlist.add_command('configure',
1451                             [os.path.join(srcdir, 'configure'),
1452                              '--build=%s' % self.ctx.build_triplet,
1453                              '--host=%s' % self.triplet,
1454                              '--prefix=',
1455                              '--disable-user32',
1456                              'CC=%s-gcc -nostdlib' % self.triplet])
1457         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1458                                         'install-data'])
1459         cmdlist.cleanup_dir()
1460         cmdlist.pop_subdesc()
1461
1462     def install_hurd_headers(self, cmdlist):
1463         """Install Hurd headers."""
1464         srcdir = self.ctx.component_srcdir('hurd')
1465         builddir = self.component_builddir('hurd')
1466         cmdlist.push_subdesc('hurd')
1467         cmdlist.create_use_dir(builddir)
1468         cmdlist.add_command('configure',
1469                             [os.path.join(srcdir, 'configure'),
1470                              '--build=%s' % self.ctx.build_triplet,
1471                              '--host=%s' % self.triplet,
1472                              '--prefix=',
1473                              '--disable-profile', '--without-parted',
1474                              'CC=%s-gcc -nostdlib' % self.triplet])
1475         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1476                                         'no_deps=t', 'install-headers'])
1477         cmdlist.cleanup_dir()
1478         cmdlist.pop_subdesc()
1479
1480     def build_gcc(self, cmdlist, bootstrap):
1481         """Build GCC."""
1482         # libssp is of little relevance with glibc's own stack
1483         # checking support.  libcilkrts does not support GNU/Hurd (and
1484         # has been removed in GCC 8, so --disable-libcilkrts can be
1485         # removed once glibc no longer supports building with older
1486         # GCC versions).  --enable-initfini-array is enabled by default
1487         # in GCC 12, which can be removed when GCC 12 becomes the
1488         # minimum requirement.
1489         cfg_opts = list(self.gcc_cfg)
1490         cfg_opts += ['--enable-initfini-array']
1491         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1492         host_libs = self.ctx.host_libraries_installdir
1493         cfg_opts += ['--with-gmp=%s' % host_libs,
1494                      '--with-mpfr=%s' % host_libs,
1495                      '--with-mpc=%s' % host_libs]
1496         if bootstrap:
1497             tool_build = 'gcc-first'
1498             # Building a static-only, C-only compiler that is
1499             # sufficient to build glibc.  Various libraries and
1500             # features that may require libc headers must be disabled.
1501             # When configuring with a sysroot, --with-newlib is
1502             # required to define inhibit_libc (to stop some parts of
1503             # libgcc including libc headers); --without-headers is not
1504             # sufficient.
1505             cfg_opts += ['--enable-languages=c', '--disable-shared',
1506                          '--disable-threads',
1507                          '--disable-libatomic',
1508                          '--disable-decimal-float',
1509                          '--disable-gcov',
1510                          '--disable-libffi',
1511                          '--disable-libgomp',
1512                          '--disable-libitm',
1513                          '--disable-libmpx',
1514                          '--disable-libquadmath',
1515                          '--disable-libsanitizer',
1516                          '--without-headers', '--with-newlib',
1517                          '--with-glibc-version=%s' % self.ctx.glibc_version
1518                          ]
1519             cfg_opts += self.first_gcc_cfg
1520         else:
1521             tool_build = 'gcc'
1522             # libsanitizer commonly breaks because of glibc header
1523             # changes, or on unusual targets.  C++ pre-compiled
1524             # headers are not used during the glibc build and are
1525             # expensive to create.
1526             if not self.ctx.full_gcc:
1527                 cfg_opts += ['--disable-libsanitizer',
1528                              '--disable-libstdcxx-pch']
1529             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1530             cfg_opts += ['--enable-languages=%s' % langs,
1531                          '--enable-shared', '--enable-threads']
1532         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1533
1534 class GlibcPolicyDefault(object):
1535     """Build policy for glibc: common defaults."""
1536
1537     def __init__(self, glibc):
1538         self.srcdir = glibc.ctx.component_srcdir('glibc')
1539         self.use_usr = glibc.os != 'gnu'
1540         self.prefix = '/usr' if self.use_usr else ''
1541         self.configure_args = [
1542             '--prefix=%s' % self.prefix,
1543             '--enable-profile',
1544             '--build=%s' % glibc.ctx.build_triplet,
1545             '--host=%s' % glibc.triplet,
1546             'CC=%s' % glibc.tool_name('gcc'),
1547             'CXX=%s' % glibc.tool_name('g++'),
1548         ]
1549         if glibc.os == 'gnu':
1550             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1551         if glibc.cflags:
1552             self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1553             self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
1554         self.configure_args += glibc.cfg
1555
1556     def configure(self, cmdlist):
1557         """Invoked to add the configure command to the command list."""
1558         cmdlist.add_command('configure',
1559                             [os.path.join(self.srcdir, 'configure'),
1560                              *self.configure_args])
1561
1562     def extra_commands(self, cmdlist):
1563         """Invoked to inject additional commands (make check) after build."""
1564         pass
1565
1566 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1567     """Build policy for glibc during the compilers stage."""
1568
1569     def __init__(self, glibc):
1570         super().__init__(glibc)
1571         self.builddir = glibc.ctx.component_builddir(
1572             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1573         self.installdir = glibc.compiler.sysroot
1574
1575 class GlibcPolicyForBuild(GlibcPolicyDefault):
1576     """Build policy for glibc during the glibcs stage."""
1577
1578     def __init__(self, glibc):
1579         super().__init__(glibc)
1580         self.builddir = glibc.ctx.component_builddir(
1581             'glibcs', glibc.name, 'glibc')
1582         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1583         if glibc.ctx.strip:
1584             self.strip = glibc.tool_name('strip')
1585         else:
1586             self.strip = None
1587         self.save_logs = glibc.ctx.save_logs
1588
1589     def extra_commands(self, cmdlist):
1590         if self.strip:
1591             # Avoid stripping libc.so and libpthread.so, which are
1592             # linker scripts stored in /lib on Hurd.
1593             find_command = 'find %s/lib* -name "*.so*"' % self.installdir
1594             cmdlist.add_command('strip', ['sh', '-c', (
1595                 'set -e; for f in $(%s); do '
1596                 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1597                 'done' % (find_command, self.strip))])
1598         cmdlist.add_command('check', ['make', 'check'])
1599         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1600
1601 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1602     """Build policy for glibc during update-syscalls."""
1603
1604     def __init__(self, glibc):
1605         super().__init__(glibc)
1606         self.builddir = glibc.ctx.component_builddir(
1607             'update-syscalls', glibc.name, 'glibc')
1608         self.linuxdir = glibc.ctx.component_builddir(
1609             'update-syscalls', glibc.name, 'linux')
1610         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1611             glibc, self.linuxdir)
1612         self.configure_args.insert(
1613             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1614         # self.installdir not set because installation is not supported
1615
1616 class Glibc(object):
1617     """A configuration for building glibc."""
1618
1619     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1620                  cfg=None, ccopts=None, cflags=None):
1621         """Initialize a Glibc object."""
1622         self.ctx = compiler.ctx
1623         self.compiler = compiler
1624         if arch is None:
1625             self.arch = compiler.arch
1626         else:
1627             self.arch = arch
1628         if os_name is None:
1629             self.os = compiler.os
1630         else:
1631             self.os = os_name
1632         self.variant = variant
1633         if variant is None:
1634             self.name = '%s-%s' % (self.arch, self.os)
1635         else:
1636             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1637         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1638         if cfg is None:
1639             self.cfg = []
1640         else:
1641             self.cfg = cfg
1642         # ccopts contain ABI options and are passed to configure as CC / CXX.
1643         self.ccopts = ccopts
1644         # cflags contain non-ABI options like -g or -O and are passed to
1645         # configure as CFLAGS / CXXFLAGS.
1646         self.cflags = cflags
1647
1648     def tool_name(self, tool):
1649         """Return the name of a cross-compilation tool."""
1650         ctool = '%s-%s' % (self.compiler.triplet, tool)
1651         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1652             ctool = '%s %s' % (ctool, self.ccopts)
1653         return ctool
1654
1655     def build(self):
1656         """Generate commands to build this glibc."""
1657         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1658         installdir = self.ctx.glibc_installdir(self.name)
1659         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1660         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1661         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1662         cmdlist.add_command('check-compilers',
1663                             ['test', '-f',
1664                              os.path.join(self.compiler.installdir, 'ok')])
1665         cmdlist.use_path(self.compiler.bindir)
1666         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1667         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1668                                       logsdir)
1669
1670     def build_glibc(self, cmdlist, policy):
1671         """Generate commands to build this glibc, either as part of a compiler
1672         build or with the bootstrapped compiler (and in the latter case, run
1673         tests as well)."""
1674         cmdlist.create_use_dir(policy.builddir)
1675         policy.configure(cmdlist)
1676         cmdlist.add_command('build', ['make'])
1677         cmdlist.add_command('install', ['make', 'install',
1678                                         'install_root=%s' % policy.installdir])
1679         # GCC uses paths such as lib/../lib64, so make sure lib
1680         # directories always exist.
1681         mkdir_cmd = ['mkdir', '-p',
1682                      os.path.join(policy.installdir, 'lib')]
1683         if policy.use_usr:
1684             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1685         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1686         policy.extra_commands(cmdlist)
1687         cmdlist.cleanup_dir()
1688
1689     def update_syscalls(self):
1690         if self.os == 'gnu':
1691             # Hurd does not have system call tables that need updating.
1692             return
1693
1694         policy = GlibcPolicyForUpdateSyscalls(self)
1695         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1696         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1697         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1698         cmdlist.add_command('check-compilers',
1699                             ['test', '-f',
1700                              os.path.join(self.compiler.installdir, 'ok')])
1701         cmdlist.use_path(self.compiler.bindir)
1702
1703         install_linux_headers(policy.linux_policy, cmdlist)
1704
1705         cmdlist.create_use_dir(policy.builddir)
1706         policy.configure(cmdlist)
1707         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1708         cmdlist.cleanup_dir()
1709         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1710                                       cmdlist, logsdir)
1711
1712 class Command(object):
1713     """A command run in the build process."""
1714
1715     def __init__(self, desc, num, dir, path, command, always_run=False):
1716         """Initialize a Command object."""
1717         self.dir = dir
1718         self.path = path
1719         self.desc = desc
1720         trans = str.maketrans({' ': '-'})
1721         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1722         self.command = command
1723         self.always_run = always_run
1724
1725     @staticmethod
1726     def shell_make_quote_string(s):
1727         """Given a string not containing a newline, quote it for use by the
1728         shell and make."""
1729         assert '\n' not in s
1730         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1731             return s
1732         strans = str.maketrans({"'": "'\\''"})
1733         s = "'%s'" % s.translate(strans)
1734         mtrans = str.maketrans({'$': '$$'})
1735         return s.translate(mtrans)
1736
1737     @staticmethod
1738     def shell_make_quote_list(l, translate_make):
1739         """Given a list of strings not containing newlines, quote them for use
1740         by the shell and make, returning a single string.  If translate_make
1741         is true and the first string is 'make', change it to $(MAKE)."""
1742         l = [Command.shell_make_quote_string(s) for s in l]
1743         if translate_make and l[0] == 'make':
1744             l[0] = '$(MAKE)'
1745         return ' '.join(l)
1746
1747     def shell_make_quote(self):
1748         """Return this command quoted for the shell and make."""
1749         return self.shell_make_quote_list(self.command, True)
1750
1751
1752 class CommandList(object):
1753     """A list of commands run in the build process."""
1754
1755     def __init__(self, desc, keep):
1756         """Initialize a CommandList object."""
1757         self.cmdlist = []
1758         self.dir = None
1759         self.path = None
1760         self.desc = [desc]
1761         self.keep = keep
1762
1763     def desc_txt(self, desc):
1764         """Return the description to use for a command."""
1765         return '%s %s' % (' '.join(self.desc), desc)
1766
1767     def use_dir(self, dir):
1768         """Set the default directory for subsequent commands."""
1769         self.dir = dir
1770
1771     def use_path(self, path):
1772         """Set a directory to be prepended to the PATH for subsequent
1773         commands."""
1774         self.path = path
1775
1776     def push_subdesc(self, subdesc):
1777         """Set the default subdescription for subsequent commands (e.g., the
1778         name of a component being built, within the series of commands
1779         building it)."""
1780         self.desc.append(subdesc)
1781
1782     def pop_subdesc(self):
1783         """Pop a subdescription from the list of descriptions."""
1784         self.desc.pop()
1785
1786     def create_use_dir(self, dir):
1787         """Remove and recreate a directory and use it for subsequent
1788         commands."""
1789         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1790         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1791         self.use_dir(dir)
1792
1793     def add_command_dir(self, desc, dir, command, always_run=False):
1794         """Add a command to run in a given directory."""
1795         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1796                       command, always_run)
1797         self.cmdlist.append(cmd)
1798
1799     def add_command(self, desc, command, always_run=False):
1800         """Add a command to run in the default directory."""
1801         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1802                       self.path, command, always_run)
1803         self.cmdlist.append(cmd)
1804
1805     def cleanup_dir(self, desc='cleanup', dir=None):
1806         """Clean up a build directory.  If no directory is specified, the
1807         default directory is cleaned up and ceases to be the default
1808         directory."""
1809         if dir is None:
1810             dir = self.dir
1811             self.use_dir(None)
1812         if self.keep != 'all':
1813             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1814                                  always_run=(self.keep == 'none'))
1815
1816     def makefile_commands(self, wrapper, logsdir):
1817         """Return the sequence of commands in the form of text for a Makefile.
1818         The given wrapper script takes arguments: base of logs for
1819         previous command, or empty; base of logs for this command;
1820         description; directory; PATH addition; the command itself."""
1821         # prev_base is the base of the name for logs of the previous
1822         # command that is not always-run (that is, a build command,
1823         # whose failure should stop subsequent build commands from
1824         # being run, as opposed to a cleanup command, which is run
1825         # even if previous commands failed).
1826         prev_base = ''
1827         cmds = []
1828         for c in self.cmdlist:
1829             ctxt = c.shell_make_quote()
1830             if prev_base and not c.always_run:
1831                 prev_log = os.path.join(logsdir, prev_base)
1832             else:
1833                 prev_log = ''
1834             this_log = os.path.join(logsdir, c.logbase)
1835             if not c.always_run:
1836                 prev_base = c.logbase
1837             if c.dir is None:
1838                 dir = ''
1839             else:
1840                 dir = c.dir
1841             if c.path is None:
1842                 path = ''
1843             else:
1844                 path = c.path
1845             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1846             prelim_txt = Command.shell_make_quote_list(prelims, False)
1847             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1848         return '\n'.join(cmds)
1849
1850     def status_logs(self, logsdir):
1851         """Return the list of log files with command status."""
1852         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1853                 for c in self.cmdlist]
1854
1855
1856 def get_parser():
1857     """Return an argument parser for this module."""
1858     parser = argparse.ArgumentParser(description=__doc__)
1859     parser.add_argument('-j', dest='parallelism',
1860                         help='Run this number of jobs in parallel',
1861                         type=int, default=os.cpu_count())
1862     parser.add_argument('--keep', dest='keep',
1863                         help='Whether to keep all build directories, '
1864                         'none or only those from failed builds',
1865                         default='none', choices=('none', 'all', 'failed'))
1866     parser.add_argument('--replace-sources', action='store_true',
1867                         help='Remove and replace source directories '
1868                         'with the wrong version of a component')
1869     parser.add_argument('--strip', action='store_true',
1870                         help='Strip installed glibc libraries')
1871     parser.add_argument('--full-gcc', action='store_true',
1872                         help='Build GCC with all languages and libsanitizer')
1873     parser.add_argument('--shallow', action='store_true',
1874                         help='Do not download Git history during checkout')
1875     parser.add_argument('topdir',
1876                         help='Toplevel working directory')
1877     parser.add_argument('action',
1878                         help='What to do',
1879                         choices=('checkout', 'bot-cycle', 'bot',
1880                                  'host-libraries', 'compilers', 'glibcs',
1881                                  'update-syscalls', 'list-compilers',
1882                                  'list-glibcs'))
1883     parser.add_argument('configs',
1884                         help='Versions to check out or configurations to build',
1885                         nargs='*')
1886     return parser
1887
1888
1889 def get_version_common(progname,line,word,delchars,arg1):
1890     try:
1891         out = subprocess.run([progname, arg1],
1892                              stdout=subprocess.PIPE,
1893                              stderr=subprocess.DEVNULL,
1894                              stdin=subprocess.DEVNULL,
1895                              check=True, universal_newlines=True)
1896         v = out.stdout.splitlines()[line].split()[word]
1897         if delchars:
1898             v = v.replace(delchars,'')
1899         return [int(x) for x in v.split('.')]
1900     except:
1901         return 'missing';
1902
1903 def get_version_common_stderr(progname,line,word,delchars,arg1):
1904     try:
1905         out = subprocess.run([progname, arg1],
1906                              stdout=subprocess.DEVNULL,
1907                              stderr=subprocess.PIPE,
1908                              stdin=subprocess.DEVNULL,
1909                              check=True, universal_newlines=True)
1910         v = out.stderr.splitlines()[line].split()[word]
1911         if delchars:
1912             v = v.replace(delchars,'')
1913         return [int(x) for x in v.split('.')]
1914     except:
1915         return 'missing';
1916
1917 def get_version(progname):
1918     return get_version_common (progname, 0, -1, None, '--version');
1919
1920 def get_version_awk(progname):
1921     return get_version_common (progname, 0, 2, ',', '--version');
1922
1923 def get_version_bzip2(progname):
1924     return get_version_common_stderr (progname, 0, 6, ',', '-h');
1925
1926 def check_version(ver, req):
1927     for v, r in zip(ver, req):
1928         if v > r:
1929             return True
1930         if v < r:
1931             return False
1932     return True
1933
1934 def version_str(ver):
1935     return '.'.join([str (x) for x in ver])
1936
1937 def check_for_required_tools():
1938     get_list_of_required_tools()
1939     count_old_tools = 0
1940     count_missing_tools = 0
1941
1942     for k, v in REQUIRED_TOOLS.items():
1943         version = v[0](k)
1944         if version == 'missing':
1945             ok = 'missing'
1946         else:
1947             ok = 'ok' if check_version (version, v[1]) else 'old'
1948         if ok == 'old':
1949             if count_old_tools == 0:
1950                 print("One or more required tools are too old:")
1951             count_old_tools = count_old_tools + 1
1952             print('{:9}: {:3} (obtained=\"{}\" required=\"{}\")'.format(k, ok,
1953                     version_str(version), version_str(v[1])))
1954         if ok == 'missing':
1955             if count_missing_tools == 0:
1956                 print("One or more required tools are missing:")
1957             count_missing_tools = count_missing_tools + 1
1958             print('{:9}: {:3} (required=\"{}\")'.format(k, ok,
1959                     version_str(v[1])))
1960
1961     if count_old_tools > 0 or count_missing_tools > 0:
1962         exit (1);
1963
1964 def main(argv):
1965     """The main entry point."""
1966     check_for_required_tools();
1967     parser = get_parser()
1968     opts = parser.parse_args(argv)
1969     topdir = os.path.abspath(opts.topdir)
1970     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1971                   opts.strip, opts.full_gcc, opts.action,
1972                   shallow=opts.shallow)
1973     ctx.run_builds(opts.action, opts.configs)
1974
1975
1976 if __name__ == '__main__':
1977     main(sys.argv[1:])