scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2024 Free Software Foundation, Inc.
   4 # Copyright The GNU Toolchain Authors.
   5 # This file is part of the GNU C Library.
   6 #
   7 # The GNU C Library is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU Lesser General Public
   9 # License as published by the Free Software Foundation; either
  10 # version 2.1 of the License, or (at your option) any later version.
  11 #
  12 # The GNU C Library is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 # Lesser General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Lesser General Public
  18 # License along with the GNU C Library; if not, see
  19 # <https://www.gnu.org/licenses/>.
  20
  21 """Build many configurations of glibc.
  22
  23 This script takes as arguments a directory name (containing a src
  24 subdirectory with sources of the relevant toolchain components) and a
  25 description of what to do: 'checkout', to check out sources into that
  26 directory, 'bot-cycle', to run a series of checkout and build steps,
  27 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  28 libraries required by the toolchain, 'compilers', to build
  29 cross-compilers for various configurations, or 'glibcs', to build
  30 glibc for various configurations and run the compilation parts of the
  31 testsuite.  Subsequent arguments name the versions of components to
  32 check out (<component>-<version), for 'checkout', or, for actions
  33 other than 'checkout' and 'bot-cycle', name configurations for which
  34 compilers or glibc are to be built.
  35
  36 The 'list-compilers' command prints the name of each available
  37 compiler configuration, without building anything.  The 'list-glibcs'
  38 command prints the name of each glibc compiler configuration, followed
  39 by the space, followed by the name of the compiler configuration used
  40 for building this glibc variant.
  41
  42 """
  43
  44 import argparse
  45 import datetime
  46 import email.mime.text
  47 import email.utils
  48 import json
  49 import os
  50 import re
  51 import shutil
  52 import smtplib
  53 import stat
  54 import subprocess
  55 import sys
  56 import time
  57 import urllib.request
  58
  59 # This is a list of system utilities that are expected to be available
  60 # to this script, and, if a non-zero version is included, the minimum
  61 # version required to work with this sccript.
  62 def get_list_of_required_tools():
  63     global REQUIRED_TOOLS
  64     REQUIRED_TOOLS = {
  65         'awk'      : (get_version_awk,   (0,0,0)),
  66         'bison'    : (get_version,       (0,0)),
  67         'flex'     : (get_version,       (0,0,0)),
  68         'git'      : (get_version,       (1,8,3)),
  69         'make'     : (get_version,       (4,0)),
  70         'makeinfo' : (get_version,       (0,0)),
  71         'patch'    : (get_version,       (0,0,0)),
  72         'sed'      : (get_version,       (0,0)),
  73         'tar'      : (get_version,       (0,0,0)),
  74         'gzip'     : (get_version,       (0,0)),
  75         'bzip2'    : (get_version_bzip2, (0,0,0)),
  76         'xz'       : (get_version,       (0,0,0)),
  77     }
  78
  79 try:
  80     subprocess.run
  81 except:
  82     class _CompletedProcess:
  83         def __init__(self, args, returncode, stdout=None, stderr=None):
  84             self.args = args
  85             self.returncode = returncode
  86             self.stdout = stdout
  87             self.stderr = stderr
  88
  89     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  90         assert(timeout is None)
  91         with subprocess.Popen(*popenargs, **kwargs) as process:
  92             try:
  93                 stdout, stderr = process.communicate(input)
  94             except:
  95                 process.kill()
  96                 process.wait()
  97                 raise
  98             returncode = process.poll()
  99             if check and returncode:
 100                 raise subprocess.CalledProcessError(returncode, popenargs)
 101         return _CompletedProcess(popenargs, returncode, stdout, stderr)
 102
 103     subprocess.run = _run
 104
 105
 106 class Context(object):
 107     """The global state associated with builds in a given directory."""
 108
 109     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
 110                  full_gcc, action, shallow=False):
 111         """Initialize the context."""
 112         self.topdir = topdir
 113         self.parallelism = parallelism
 114         self.keep = keep
 115         self.replace_sources = replace_sources
 116         self.strip = strip
 117         self.full_gcc = full_gcc
 118         self.shallow = shallow
 119         self.srcdir = os.path.join(topdir, 'src')
 120         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 121         self.build_state_json = os.path.join(topdir, 'build-state.json')
 122         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 123         self.installdir = os.path.join(topdir, 'install')
 124         self.host_libraries_installdir = os.path.join(self.installdir,
 125                                                       'host-libraries')
 126         self.builddir = os.path.join(topdir, 'build')
 127         self.logsdir = os.path.join(topdir, 'logs')
 128         self.logsdir_old = os.path.join(topdir, 'logs-old')
 129         self.makefile = os.path.join(self.builddir, 'Makefile')
 130         self.wrapper = os.path.join(self.builddir, 'wrapper')
 131         self.save_logs = os.path.join(self.builddir, 'save-logs')
 132         self.script_text = self.get_script_text()
 133         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 134             self.build_triplet = self.get_build_triplet()
 135             self.glibc_version = self.get_glibc_version()
 136         self.configs = {}
 137         self.glibc_configs = {}
 138         self.makefile_pieces = ['.PHONY: all\n']
 139         self.add_all_configs()
 140         self.load_versions_json()
 141         self.load_build_state_json()
 142         self.status_log_list = []
 143         self.email_warning = False
 144
 145     def get_script_text(self):
 146         """Return the text of this script."""
 147         with open(sys.argv[0], 'r') as f:
 148             return f.read()
 149
 150     def exec_self(self):
 151         """Re-execute this script with the same arguments."""
 152         sys.stdout.flush()
 153         os.execv(sys.executable, [sys.executable] + sys.argv)
 154
 155     def get_build_triplet(self):
 156         """Determine the build triplet with config.guess."""
 157         config_guess = os.path.join(self.component_srcdir('gcc'),
 158                                     'config.guess')
 159         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 160                                 check=True, universal_newlines=True).stdout
 161         return cg_out.rstrip()
 162
 163     def get_glibc_version(self):
 164         """Determine the glibc version number (major.minor)."""
 165         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 166         with open(version_h, 'r') as f:
 167             lines = f.readlines()
 168         starttext = '#define VERSION "'
 169         for l in lines:
 170             if l.startswith(starttext):
 171                 l = l[len(starttext):]
 172                 l = l.rstrip('"\n')
 173                 m = re.fullmatch(r'([0-9]+)\.([0-9]+)[.0-9]*', l)
 174                 return '%s.%s' % m.group(1, 2)
 175         print('error: could not determine glibc version')
 176         exit(1)
 177
 178     def add_all_configs(self):
 179         """Add all known glibc build configurations."""
 180         self.add_config(arch='aarch64',
 181                         os_name='linux-gnu',
 182                         extra_glibcs=[{'variant': 'disable-multi-arch',
 183                                        'cfg': ['--disable-multi-arch']}])
 184         self.add_config(arch='aarch64_be',
 185                         os_name='linux-gnu')
 186         self.add_config(arch='arc',
 187                         os_name='linux-gnu',
 188                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 189         self.add_config(arch='arc',
 190                         os_name='linux-gnuhf',
 191                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
 192         self.add_config(arch='arceb',
 193                         os_name='linux-gnu',
 194                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 195         self.add_config(arch='alpha',
 196                         os_name='linux-gnu')
 197         self.add_config(arch='arm',
 198                         os_name='linux-gnueabi',
 199                         extra_glibcs=[{'variant': 'v4t',
 200                                        'ccopts': '-march=armv4t'}])
 201         self.add_config(arch='armeb',
 202                         os_name='linux-gnueabi')
 203         self.add_config(arch='armeb',
 204                         os_name='linux-gnueabi',
 205                         variant='be8',
 206                         gcc_cfg=['--with-arch=armv7-a'])
 207         self.add_config(arch='arm',
 208                         os_name='linux-gnueabihf',
 209                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 210                         extra_glibcs=[{'variant': 'v7a',
 211                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 212                                       {'variant': 'thumb',
 213                                        'ccopts':
 214                                        '-mthumb -march=armv7-a -mfpu=vfpv3'},
 215                                       {'variant': 'v7a-disable-multi-arch',
 216                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 217                                        'cfg': ['--disable-multi-arch']}])
 218         self.add_config(arch='armeb',
 219                         os_name='linux-gnueabihf',
 220                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 221         self.add_config(arch='armeb',
 222                         os_name='linux-gnueabihf',
 223                         variant='be8',
 224                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 225                                  '--with-fpu=vfpv3'])
 226         self.add_config(arch='csky',
 227                         os_name='linux-gnuabiv2',
 228                         variant='soft',
 229                         gcc_cfg=['--disable-multilib'])
 230         self.add_config(arch='csky',
 231                         os_name='linux-gnuabiv2',
 232                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 233         self.add_config(arch='hppa',
 234                         os_name='linux-gnu')
 235         self.add_config(arch='i686',
 236                         os_name='gnu')
 237         self.add_config(arch='loongarch64',
 238                         os_name='linux-gnu',
 239                         variant='lp64d',
 240                         gcc_cfg=['--with-abi=lp64d','--disable-multilib'])
 241         self.add_config(arch='loongarch64',
 242                         os_name='linux-gnu',
 243                         variant='lp64s',
 244                         gcc_cfg=['--with-abi=lp64s','--disable-multilib'])
 245         self.add_config(arch='m68k',
 246                         os_name='linux-gnu',
 247                         gcc_cfg=['--disable-multilib'])
 248         self.add_config(arch='m68k',
 249                         os_name='linux-gnu',
 250                         variant='coldfire',
 251                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 252         self.add_config(arch='m68k',
 253                         os_name='linux-gnu',
 254                         variant='coldfire-soft',
 255                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 256                                  '--disable-multilib'])
 257         self.add_config(arch='microblaze',
 258                         os_name='linux-gnu',
 259                         gcc_cfg=['--disable-multilib'])
 260         self.add_config(arch='microblazeel',
 261                         os_name='linux-gnu',
 262                         gcc_cfg=['--disable-multilib'])
 263         self.add_config(arch='mips64',
 264                         os_name='linux-gnu',
 265                         gcc_cfg=['--with-mips-plt'],
 266                         glibcs=[{'variant': 'n32'},
 267                                 {'arch': 'mips',
 268                                  'ccopts': '-mabi=32'},
 269                                 {'variant': 'n64',
 270                                  'ccopts': '-mabi=64'}])
 271         self.add_config(arch='mips64',
 272                         os_name='linux-gnu',
 273                         variant='soft',
 274                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 275                         glibcs=[{'variant': 'n32-soft'},
 276                                 {'variant': 'soft',
 277                                  'arch': 'mips',
 278                                  'ccopts': '-mabi=32'},
 279                                 {'variant': 'n64-soft',
 280                                  'ccopts': '-mabi=64'}])
 281         self.add_config(arch='mips64',
 282                         os_name='linux-gnu',
 283                         variant='nan2008',
 284                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 285                                  '--with-arch-64=mips64r2',
 286                                  '--with-arch-32=mips32r2'],
 287                         glibcs=[{'variant': 'n32-nan2008'},
 288                                 {'variant': 'nan2008',
 289                                  'arch': 'mips',
 290                                  'ccopts': '-mabi=32'},
 291                                 {'variant': 'n64-nan2008',
 292                                  'ccopts': '-mabi=64'}])
 293         self.add_config(arch='mips64',
 294                         os_name='linux-gnu',
 295                         variant='nan2008-soft',
 296                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 297                                  '--with-arch-64=mips64r2',
 298                                  '--with-arch-32=mips32r2',
 299                                  '--with-float=soft'],
 300                         glibcs=[{'variant': 'n32-nan2008-soft'},
 301                                 {'variant': 'nan2008-soft',
 302                                  'arch': 'mips',
 303                                  'ccopts': '-mabi=32'},
 304                                 {'variant': 'n64-nan2008-soft',
 305                                  'ccopts': '-mabi=64'}])
 306         self.add_config(arch='mips64el',
 307                         os_name='linux-gnu',
 308                         gcc_cfg=['--with-mips-plt'],
 309                         glibcs=[{'variant': 'n32'},
 310                                 {'arch': 'mipsel',
 311                                  'ccopts': '-mabi=32'},
 312                                 {'variant': 'n64',
 313                                  'ccopts': '-mabi=64'}])
 314         self.add_config(arch='mips64el',
 315                         os_name='linux-gnu',
 316                         variant='soft',
 317                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 318                         glibcs=[{'variant': 'n32-soft'},
 319                                 {'variant': 'soft',
 320                                  'arch': 'mipsel',
 321                                  'ccopts': '-mabi=32'},
 322                                 {'variant': 'n64-soft',
 323                                  'ccopts': '-mabi=64'}])
 324         self.add_config(arch='mips64el',
 325                         os_name='linux-gnu',
 326                         variant='nan2008',
 327                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 328                                  '--with-arch-64=mips64r2',
 329                                  '--with-arch-32=mips32r2'],
 330                         glibcs=[{'variant': 'n32-nan2008'},
 331                                 {'variant': 'nan2008',
 332                                  'arch': 'mipsel',
 333                                  'ccopts': '-mabi=32'},
 334                                 {'variant': 'n64-nan2008',
 335                                  'ccopts': '-mabi=64'}])
 336         self.add_config(arch='mips64el',
 337                         os_name='linux-gnu',
 338                         variant='nan2008-soft',
 339                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 340                                  '--with-arch-64=mips64r2',
 341                                  '--with-arch-32=mips32r2',
 342                                  '--with-float=soft'],
 343                         glibcs=[{'variant': 'n32-nan2008-soft'},
 344                                 {'variant': 'nan2008-soft',
 345                                  'arch': 'mipsel',
 346                                  'ccopts': '-mabi=32'},
 347                                 {'variant': 'n64-nan2008-soft',
 348                                  'ccopts': '-mabi=64'}])
 349         self.add_config(arch='mipsisa64r6el',
 350                         os_name='linux-gnu',
 351                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 352                                  '--with-arch-64=mips64r6',
 353                                  '--with-arch-32=mips32r6',
 354                                  '--with-float=hard'],
 355                         glibcs=[{'variant': 'n32'},
 356                                 {'arch': 'mipsisa32r6el',
 357                                  'ccopts': '-mabi=32'},
 358                                 {'variant': 'n64',
 359                                  'ccopts': '-mabi=64'}])
 360         self.add_config(arch='nios2',
 361                         os_name='linux-gnu',
 362                         gcc_cfg=['--enable-obsolete'])
 363         self.add_config(arch='or1k',
 364                         os_name='linux-gnu',
 365                         gcc_cfg=['--with-multilib-list=mcmov,mhard-float'],
 366                         glibcs=[{'variant': 'soft'},
 367                                 {'variant': 'hard', 'ccopts': '-mhard-float'}])
 368         self.add_config(arch='powerpc',
 369                         os_name='linux-gnu',
 370                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 371                         extra_glibcs=[{'variant': 'power4',
 372                                        'ccopts': '-mcpu=power4',
 373                                        'cfg': ['--with-cpu=power4']}])
 374         self.add_config(arch='powerpc',
 375                         os_name='linux-gnu',
 376                         variant='soft',
 377                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 378                                  '--enable-secureplt'])
 379         self.add_config(arch='powerpc64',
 380                         os_name='linux-gnu',
 381                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 382         self.add_config(arch='powerpc64le',
 383                         os_name='linux-gnu',
 384                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 385                         extra_glibcs=[{'variant': 'disable-multi-arch',
 386                                        'cfg': ['--disable-multi-arch']}])
 387         self.add_config(arch='riscv32',
 388                         os_name='linux-gnu',
 389                         variant='rv32imac-ilp32',
 390                         gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
 391                                  '--disable-multilib'])
 392         self.add_config(arch='riscv32',
 393                         os_name='linux-gnu',
 394                         variant='rv32imafdc-ilp32',
 395                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
 396                                  '--disable-multilib'])
 397         self.add_config(arch='riscv32',
 398                         os_name='linux-gnu',
 399                         variant='rv32imafdc-ilp32d',
 400                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
 401                                  '--disable-multilib'])
 402         self.add_config(arch='riscv64',
 403                         os_name='linux-gnu',
 404                         variant='rv64imac-lp64',
 405                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 406                                  '--disable-multilib'])
 407         self.add_config(arch='riscv64',
 408                         os_name='linux-gnu',
 409                         variant='rv64imafdc-lp64',
 410                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 411                                  '--disable-multilib'])
 412         self.add_config(arch='riscv64',
 413                         os_name='linux-gnu',
 414                         variant='rv64imafdc-lp64d',
 415                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 416                                  '--disable-multilib'])
 417         self.add_config(arch='s390x',
 418                         os_name='linux-gnu',
 419                         glibcs=[{},
 420                                 {'arch': 's390', 'ccopts': '-m31'}],
 421                         extra_glibcs=[{'variant': 'O3',
 422                                        'cflags': '-O3'},
 423                                       {'variant': 'zEC12',
 424                                        'ccopts': '-march=zEC12'},
 425                                       {'variant': 'z13',
 426                                        'ccopts': '-march=z13'},
 427                                       {'variant': 'z15',
 428                                        'ccopts': '-march=z15'},
 429                                       {'variant': 'zEC12-disable-multi-arch',
 430                                        'ccopts': '-march=zEC12',
 431                                        'cfg': ['--disable-multi-arch']},
 432                                       {'variant': 'z13-disable-multi-arch',
 433                                        'ccopts': '-march=z13',
 434                                        'cfg': ['--disable-multi-arch']},
 435                                       {'variant': 'z15-disable-multi-arch',
 436                                        'ccopts': '-march=z15',
 437                                        'cfg': ['--disable-multi-arch']}])
 438         self.add_config(arch='sh3',
 439                         os_name='linux-gnu')
 440         self.add_config(arch='sh3eb',
 441                         os_name='linux-gnu')
 442         self.add_config(arch='sh4',
 443                         os_name='linux-gnu')
 444         self.add_config(arch='sh4eb',
 445                         os_name='linux-gnu')
 446         self.add_config(arch='sh4',
 447                         os_name='linux-gnu',
 448                         variant='soft',
 449                         gcc_cfg=['--without-fp'])
 450         self.add_config(arch='sh4eb',
 451                         os_name='linux-gnu',
 452                         variant='soft',
 453                         gcc_cfg=['--without-fp'])
 454         self.add_config(arch='sparc64',
 455                         os_name='linux-gnu',
 456                         glibcs=[{'cfg' : ['--disable-default-pie']},
 457                                 {'cfg' : ['--disable-default-pie'],
 458                                  'arch': 'sparcv9',
 459                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 460                         extra_glibcs=[{'variant': 'leon3',
 461                                        'cfg' : ['--disable-default-pie'],
 462                                        'arch' : 'sparcv8',
 463                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 464                                       {'variant': 'disable-multi-arch',
 465                                        'cfg': ['--disable-multi-arch', '--disable-default-pie']},
 466                                       {'variant': 'disable-multi-arch',
 467                                        'arch': 'sparcv9',
 468                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 469                                        'cfg': ['--disable-multi-arch', '--disable-default-pie']}])
 470         self.add_config(arch='x86_64',
 471                         os_name='linux-gnu',
 472                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 473                         glibcs=[{},
 474                                 {'variant': 'x32', 'ccopts': '-mx32'},
 475                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 476                         extra_glibcs=[{'variant': 'disable-multi-arch',
 477                                        'cfg': ['--disable-multi-arch']},
 478                                       {'variant': 'minimal',
 479                                        'cfg': ['--disable-multi-arch',
 480                                                '--disable-profile',
 481                                                '--disable-timezone-tools',
 482                                                '--disable-mathvec',
 483                                                '--disable-build-nscd',
 484                                                '--disable-nscd']},
 485                                       {'variant': 'no-pie',
 486                                        'cfg': ['--disable-default-pie']},
 487                                       {'variant': 'x32-no-pie',
 488                                        'ccopts': '-mx32',
 489                                        'cfg': ['--disable-default-pie']},
 490                                       {'variant': 'no-pie',
 491                                        'arch': 'i686',
 492                                        'ccopts': '-m32 -march=i686',
 493                                        'cfg': ['--disable-default-pie']},
 494                                       {'variant': 'disable-multi-arch',
 495                                        'arch': 'i686',
 496                                        'ccopts': '-m32 -march=i686',
 497                                        'cfg': ['--disable-multi-arch']},
 498                                       {'arch': 'i486',
 499                                        'ccopts': '-m32 -march=i486'},
 500                                       {'arch': 'i586',
 501                                        'ccopts': '-m32 -march=i586'},
 502                                       {'variant': 'enable-fortify-source',
 503                                        'cfg': ['--enable-fortify-source']}])
 504         self.add_config(arch='x86_64',
 505                         os_name='gnu',
 506                         gcc_cfg=['--disable-multilib'])
 507
 508     def add_config(self, **args):
 509         """Add an individual build configuration."""
 510         cfg = Config(self, **args)
 511         if cfg.name in self.configs:
 512             print('error: duplicate config %s' % cfg.name)
 513             exit(1)
 514         self.configs[cfg.name] = cfg
 515         for c in cfg.all_glibcs:
 516             if c.name in self.glibc_configs:
 517                 print('error: duplicate glibc config %s' % c.name)
 518                 exit(1)
 519             self.glibc_configs[c.name] = c
 520
 521     def component_srcdir(self, component):
 522         """Return the source directory for a given component, e.g. gcc."""
 523         return os.path.join(self.srcdir, component)
 524
 525     def component_builddir(self, action, config, component, subconfig=None):
 526         """Return the directory to use for a build."""
 527         if config is None:
 528             # Host libraries.
 529             assert subconfig is None
 530             return os.path.join(self.builddir, action, component)
 531         if subconfig is None:
 532             return os.path.join(self.builddir, action, config, component)
 533         else:
 534             # glibc build as part of compiler build.
 535             return os.path.join(self.builddir, action, config, component,
 536                                 subconfig)
 537
 538     def compiler_installdir(self, config):
 539         """Return the directory in which to install a compiler."""
 540         return os.path.join(self.installdir, 'compilers', config)
 541
 542     def compiler_bindir(self, config):
 543         """Return the directory in which to find compiler binaries."""
 544         return os.path.join(self.compiler_installdir(config), 'bin')
 545
 546     def compiler_sysroot(self, config):
 547         """Return the sysroot directory for a compiler."""
 548         return os.path.join(self.compiler_installdir(config), 'sysroot')
 549
 550     def glibc_installdir(self, config):
 551         """Return the directory in which to install glibc."""
 552         return os.path.join(self.installdir, 'glibcs', config)
 553
 554     def run_builds(self, action, configs):
 555         """Run the requested builds."""
 556         if action == 'checkout':
 557             self.checkout(configs)
 558             return
 559         if action == 'bot-cycle':
 560             if configs:
 561                 print('error: configurations specified for bot-cycle')
 562                 exit(1)
 563             self.bot_cycle()
 564             return
 565         if action == 'bot':
 566             if configs:
 567                 print('error: configurations specified for bot')
 568                 exit(1)
 569             self.bot()
 570             return
 571         if action in ('host-libraries', 'list-compilers',
 572                       'list-glibcs') and configs:
 573             print('error: configurations specified for ' + action)
 574             exit(1)
 575         if action == 'list-compilers':
 576             for name in sorted(self.configs.keys()):
 577                 print(name)
 578             return
 579         if action == 'list-glibcs':
 580             for config in sorted(self.glibc_configs.values(),
 581                                  key=lambda c: c.name):
 582                 print(config.name, config.compiler.name)
 583             return
 584         self.clear_last_build_state(action)
 585         build_time = datetime.datetime.now(datetime.timezone.utc)
 586         if action == 'host-libraries':
 587             build_components = ('gmp', 'mpfr', 'mpc')
 588             old_components = ()
 589             old_versions = {}
 590             self.build_host_libraries()
 591         elif action == 'compilers':
 592             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 593                                 'gnumach', 'hurd')
 594             old_components = ('gmp', 'mpfr', 'mpc')
 595             old_versions = self.build_state['host-libraries']['build-versions']
 596             self.build_compilers(configs)
 597         else:
 598             build_components = ('glibc',)
 599             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 600                               'mig', 'gnumach', 'hurd')
 601             old_versions = self.build_state['compilers']['build-versions']
 602             if action == 'update-syscalls':
 603                 self.update_syscalls(configs)
 604             else:
 605                 self.build_glibcs(configs)
 606         self.write_files()
 607         self.do_build()
 608         if configs:
 609             # Partial build, do not update stored state.
 610             return
 611         build_versions = {}
 612         for k in build_components:
 613             if k in self.versions:
 614                 build_versions[k] = {'version': self.versions[k]['version'],
 615                                      'revision': self.versions[k]['revision']}
 616         for k in old_components:
 617             if k in old_versions:
 618                 build_versions[k] = {'version': old_versions[k]['version'],
 619                                      'revision': old_versions[k]['revision']}
 620         self.update_build_state(action, build_time, build_versions)
 621
 622     @staticmethod
 623     def remove_dirs(*args):
 624         """Remove directories and their contents if they exist."""
 625         for dir in args:
 626             shutil.rmtree(dir, ignore_errors=True)
 627
 628     @staticmethod
 629     def remove_recreate_dirs(*args):
 630         """Remove directories if they exist, and create them as empty."""
 631         Context.remove_dirs(*args)
 632         for dir in args:
 633             os.makedirs(dir, exist_ok=True)
 634
 635     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 636         """Add makefile text for a list of commands."""
 637         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 638         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 639                                     (target, target, target, commands))
 640         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 641
 642     def write_files(self):
 643         """Write out the Makefile and wrapper script."""
 644         mftext = ''.join(self.makefile_pieces)
 645         with open(self.makefile, 'w') as f:
 646             f.write(mftext)
 647         wrapper_text = (
 648             '#!/bin/sh\n'
 649             'prev_base=$1\n'
 650             'this_base=$2\n'
 651             'desc=$3\n'
 652             'dir=$4\n'
 653             'path=$5\n'
 654             'shift 5\n'
 655             'prev_status=$prev_base-status.txt\n'
 656             'this_status=$this_base-status.txt\n'
 657             'this_log=$this_base-log.txt\n'
 658             'date > "$this_log"\n'
 659             'echo >> "$this_log"\n'
 660             'echo "Description: $desc" >> "$this_log"\n'
 661             'printf "%s" "Command:" >> "$this_log"\n'
 662             'for word in "$@"; do\n'
 663             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 664             '    printf " %s" "$word"\n'
 665             '  else\n'
 666             '    printf " \'"\n'
 667             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 668             '    printf "\'"\n'
 669             '  fi\n'
 670             'done >> "$this_log"\n'
 671             'echo >> "$this_log"\n'
 672             'echo "Directory: $dir" >> "$this_log"\n'
 673             'echo "Path addition: $path" >> "$this_log"\n'
 674             'echo >> "$this_log"\n'
 675             'record_status ()\n'
 676             '{\n'
 677             '  echo >> "$this_log"\n'
 678             '  echo "$1: $desc" > "$this_status"\n'
 679             '  echo "$1: $desc" >> "$this_log"\n'
 680             '  echo >> "$this_log"\n'
 681             '  date >> "$this_log"\n'
 682             '  echo "$1: $desc"\n'
 683             '  exit 0\n'
 684             '}\n'
 685             'check_error ()\n'
 686             '{\n'
 687             '  if [ "$1" != "0" ]; then\n'
 688             '    record_status FAIL\n'
 689             '  fi\n'
 690             '}\n'
 691             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 692             '    record_status UNRESOLVED\n'
 693             'fi\n'
 694             'if [ "$dir" ]; then\n'
 695             '  cd "$dir"\n'
 696             '  check_error "$?"\n'
 697             'fi\n'
 698             'if [ "$path" ]; then\n'
 699             '  PATH=$path:$PATH\n'
 700             'fi\n'
 701             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 702             'check_error "$?"\n'
 703             'record_status PASS\n')
 704         with open(self.wrapper, 'w') as f:
 705             f.write(wrapper_text)
 706         # Mode 0o755.
 707         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 708                      stat.S_IROTH|stat.S_IXOTH)
 709         os.chmod(self.wrapper, mode_exec)
 710         save_logs_text = (
 711             '#!/bin/sh\n'
 712             'if ! [ -f tests.sum ]; then\n'
 713             '  echo "No test summary available."\n'
 714             '  exit 0\n'
 715             'fi\n'
 716             'save_file ()\n'
 717             '{\n'
 718             '  echo "Contents of $1:"\n'
 719             '  echo\n'
 720             '  cat "$1"\n'
 721             '  echo\n'
 722             '  echo "End of contents of $1."\n'
 723             '  echo\n'
 724             '}\n'
 725             'save_file tests.sum\n'
 726             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 727             'for t in $non_pass_tests; do\n'
 728             '  if [ -f "$t.out" ]; then\n'
 729             '    save_file "$t.out"\n'
 730             '  fi\n'
 731             'done\n')
 732         with open(self.save_logs, 'w') as f:
 733             f.write(save_logs_text)
 734         os.chmod(self.save_logs, mode_exec)
 735
 736     def do_build(self):
 737         """Do the actual build."""
 738         cmd = ['make', '-O', '-j%d' % self.parallelism]
 739         subprocess.run(cmd, cwd=self.builddir, check=True)
 740
 741     def build_host_libraries(self):
 742         """Build the host libraries."""
 743         installdir = self.host_libraries_installdir
 744         builddir = os.path.join(self.builddir, 'host-libraries')
 745         logsdir = os.path.join(self.logsdir, 'host-libraries')
 746         self.remove_recreate_dirs(installdir, builddir, logsdir)
 747         cmdlist = CommandList('host-libraries', self.keep)
 748         self.build_host_library(cmdlist, 'gmp')
 749         self.build_host_library(cmdlist, 'mpfr',
 750                                 ['--with-gmp=%s' % installdir])
 751         self.build_host_library(cmdlist, 'mpc',
 752                                 ['--with-gmp=%s' % installdir,
 753                                 '--with-mpfr=%s' % installdir])
 754         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 755         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 756
 757     def build_host_library(self, cmdlist, lib, extra_opts=None):
 758         """Build one host library."""
 759         srcdir = self.component_srcdir(lib)
 760         builddir = self.component_builddir('host-libraries', None, lib)
 761         installdir = self.host_libraries_installdir
 762         cmdlist.push_subdesc(lib)
 763         cmdlist.create_use_dir(builddir)
 764         cfg_cmd = [os.path.join(srcdir, 'configure'),
 765                    '--prefix=%s' % installdir,
 766                    '--disable-shared']
 767         if extra_opts:
 768             cfg_cmd.extend (extra_opts)
 769         cmdlist.add_command('configure', cfg_cmd)
 770         cmdlist.add_command('build', ['make'])
 771         cmdlist.add_command('check', ['make', 'check'])
 772         cmdlist.add_command('install', ['make', 'install'])
 773         cmdlist.cleanup_dir()
 774         cmdlist.pop_subdesc()
 775
 776     def build_compilers(self, configs):
 777         """Build the compilers."""
 778         if not configs:
 779             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 780             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 781             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 782             configs = sorted(self.configs.keys())
 783         for c in configs:
 784             self.configs[c].build()
 785
 786     def build_glibcs(self, configs):
 787         """Build the glibcs."""
 788         if not configs:
 789             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 790             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 791             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 792             configs = sorted(self.glibc_configs.keys())
 793         for c in configs:
 794             self.glibc_configs[c].build()
 795
 796     def update_syscalls(self, configs):
 797         """Update the glibc syscall lists."""
 798         if not configs:
 799             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 800             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 801             configs = sorted(self.glibc_configs.keys())
 802         for c in configs:
 803             self.glibc_configs[c].update_syscalls()
 804
 805     def load_versions_json(self):
 806         """Load information about source directory versions."""
 807         if not os.access(self.versions_json, os.F_OK):
 808             self.versions = {}
 809             return
 810         with open(self.versions_json, 'r') as f:
 811             self.versions = json.load(f)
 812
 813     def store_json(self, data, filename):
 814         """Store information in a JSON file."""
 815         filename_tmp = filename + '.tmp'
 816         with open(filename_tmp, 'w') as f:
 817             json.dump(data, f, indent=2, sort_keys=True)
 818         os.rename(filename_tmp, filename)
 819
 820     def store_versions_json(self):
 821         """Store information about source directory versions."""
 822         self.store_json(self.versions, self.versions_json)
 823
 824     def set_component_version(self, component, version, explicit, revision):
 825         """Set the version information for a component."""
 826         self.versions[component] = {'version': version,
 827                                     'explicit': explicit,
 828                                     'revision': revision}
 829         self.store_versions_json()
 830
 831     def checkout(self, versions):
 832         """Check out the desired component versions."""
 833         default_versions = {'binutils': 'vcs-2.42',
 834                             'gcc': 'vcs-13',
 835                             'glibc': 'vcs-mainline',
 836                             'gmp': '6.3.0',
 837                             'linux': '6.9',
 838                             'mpc': '1.3.1',
 839                             'mpfr': '4.2.1',
 840                             'mig': 'vcs-mainline',
 841                             'gnumach': 'vcs-mainline',
 842                             'hurd': 'vcs-mainline'}
 843         use_versions = {}
 844         explicit_versions = {}
 845         for v in versions:
 846             found_v = False
 847             for k in default_versions.keys():
 848                 kx = k + '-'
 849                 if v.startswith(kx):
 850                     vx = v[len(kx):]
 851                     if k in use_versions:
 852                         print('error: multiple versions for %s' % k)
 853                         exit(1)
 854                     use_versions[k] = vx
 855                     explicit_versions[k] = True
 856                     found_v = True
 857                     break
 858             if not found_v:
 859                 print('error: unknown component in %s' % v)
 860                 exit(1)
 861         for k in default_versions.keys():
 862             if k not in use_versions:
 863                 if k in self.versions and self.versions[k]['explicit']:
 864                     use_versions[k] = self.versions[k]['version']
 865                     explicit_versions[k] = True
 866                 else:
 867                     use_versions[k] = default_versions[k]
 868                     explicit_versions[k] = False
 869         os.makedirs(self.srcdir, exist_ok=True)
 870         for k in sorted(default_versions.keys()):
 871             update = os.access(self.component_srcdir(k), os.F_OK)
 872             v = use_versions[k]
 873             if (update and
 874                 k in self.versions and
 875                 v != self.versions[k]['version']):
 876                 if not self.replace_sources:
 877                     print('error: version of %s has changed from %s to %s, '
 878                           'use --replace-sources to check out again' %
 879                           (k, self.versions[k]['version'], v))
 880                     exit(1)
 881                 shutil.rmtree(self.component_srcdir(k))
 882                 update = False
 883             if v.startswith('vcs-'):
 884                 revision = self.checkout_vcs(k, v[4:], update)
 885             else:
 886                 self.checkout_tar(k, v, update)
 887                 revision = v
 888             self.set_component_version(k, v, explicit_versions[k], revision)
 889         if self.get_script_text() != self.script_text:
 890             # Rerun the checkout process in case the updated script
 891             # uses different default versions or new components.
 892             self.exec_self()
 893
 894     def checkout_vcs(self, component, version, update):
 895         """Check out the given version of the given component from version
 896         control.  Return a revision identifier."""
 897         if component == 'binutils':
 898             git_url = 'https://sourceware.org/git/binutils-gdb.git'
 899             if version == 'mainline':
 900                 git_branch = 'master'
 901             else:
 902                 trans = str.maketrans({'.': '_'})
 903                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 904             return self.git_checkout(component, git_url, git_branch, update)
 905         elif component == 'gcc':
 906             if version == 'mainline':
 907                 branch = 'master'
 908             else:
 909                 branch = 'releases/gcc-%s' % version
 910             return self.gcc_checkout(branch, update)
 911         elif component == 'glibc':
 912             git_url = 'https://sourceware.org/git/glibc.git'
 913             if version == 'mainline':
 914                 git_branch = 'master'
 915             else:
 916                 git_branch = 'release/%s/master' % version
 917             r = self.git_checkout(component, git_url, git_branch, update)
 918             self.fix_glibc_timestamps()
 919             return r
 920         elif component == 'gnumach':
 921             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 922             git_branch = 'master'
 923             r = self.git_checkout(component, git_url, git_branch, update)
 924             subprocess.run(['autoreconf', '-i'],
 925                            cwd=self.component_srcdir(component), check=True)
 926             return r
 927         elif component == 'mig':
 928             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 929             git_branch = 'master'
 930             r = self.git_checkout(component, git_url, git_branch, update)
 931             subprocess.run(['autoreconf', '-i'],
 932                            cwd=self.component_srcdir(component), check=True)
 933             return r
 934         elif component == 'hurd':
 935             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 936             git_branch = 'master'
 937             r = self.git_checkout(component, git_url, git_branch, update)
 938             subprocess.run(['autoconf'],
 939                            cwd=self.component_srcdir(component), check=True)
 940             return r
 941         else:
 942             print('error: component %s coming from VCS' % component)
 943             exit(1)
 944
 945     def git_checkout(self, component, git_url, git_branch, update):
 946         """Check out a component from git.  Return a commit identifier."""
 947         if update:
 948             subprocess.run(['git', 'remote', 'prune', 'origin'],
 949                            cwd=self.component_srcdir(component), check=True)
 950             if self.replace_sources:
 951                 subprocess.run(['git', 'clean', '-dxfq'],
 952                                cwd=self.component_srcdir(component), check=True)
 953             subprocess.run(['git', 'pull', '-q'],
 954                            cwd=self.component_srcdir(component), check=True)
 955         else:
 956             if self.shallow:
 957                 depth_arg = ('--depth', '1')
 958             else:
 959                 depth_arg = ()
 960             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 961                             *depth_arg, git_url,
 962                             self.component_srcdir(component)], check=True)
 963         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 964                            cwd=self.component_srcdir(component),
 965                            stdout=subprocess.PIPE,
 966                            check=True, universal_newlines=True).stdout
 967         return r.rstrip()
 968
 969     def fix_glibc_timestamps(self):
 970         """Fix timestamps in a glibc checkout."""
 971         # Ensure that builds do not try to regenerate generated files
 972         # in the source tree.
 973         srcdir = self.component_srcdir('glibc')
 974         # These files have Makefile dependencies to regenerate them in
 975         # the source tree that may be active during a normal build.
 976         # Some other files have such dependencies but do not need to
 977         # be touched because nothing in a build depends on the files
 978         # in question.
 979         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 980             to_touch = os.path.join(srcdir, f)
 981             subprocess.run(['touch', '-c', to_touch], check=True)
 982         for dirpath, dirnames, filenames in os.walk(srcdir):
 983             for f in filenames:
 984                 if (f == 'configure' or
 985                     f == 'preconfigure' or
 986                     f.endswith('-kw.h')):
 987                     to_touch = os.path.join(dirpath, f)
 988                     subprocess.run(['touch', to_touch], check=True)
 989
 990     def gcc_checkout(self, branch, update):
 991         """Check out GCC from git.  Return the commit identifier."""
 992         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 993                      os.F_OK):
 994             if not self.replace_sources:
 995                 print('error: GCC has moved from SVN to git, use '
 996                       '--replace-sources to check out again')
 997                 exit(1)
 998             shutil.rmtree(self.component_srcdir('gcc'))
 999             update = False
1000         if not update:
1001             self.git_checkout('gcc', 'https://gcc.gnu.org/git/gcc.git',
1002                               branch, update)
1003         subprocess.run(['contrib/gcc_update', '--silent'],
1004                        cwd=self.component_srcdir('gcc'), check=True)
1005         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
1006                            cwd=self.component_srcdir('gcc'),
1007                            stdout=subprocess.PIPE,
1008                            check=True, universal_newlines=True).stdout
1009         return r.rstrip()
1010
1011     def checkout_tar(self, component, version, update):
1012         """Check out the given version of the given component from a
1013         tarball."""
1014         if update:
1015             return
1016         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
1017                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
1018                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
1019                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
1020                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
1021                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
1022                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
1023                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
1024                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
1025         if component not in url_map:
1026             print('error: component %s coming from tarball' % component)
1027             exit(1)
1028         version_major = version.split('.')[0]
1029         url = url_map[component] % {'version': version, 'major': version_major}
1030         filename = os.path.join(self.srcdir, url.split('/')[-1])
1031         response = urllib.request.urlopen(url)
1032         data = response.read()
1033         with open(filename, 'wb') as f:
1034             f.write(data)
1035         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
1036                        check=True)
1037         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
1038                   self.component_srcdir(component))
1039         os.remove(filename)
1040
1041     def load_build_state_json(self):
1042         """Load information about the state of previous builds."""
1043         if os.access(self.build_state_json, os.F_OK):
1044             with open(self.build_state_json, 'r') as f:
1045                 self.build_state = json.load(f)
1046         else:
1047             self.build_state = {}
1048         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
1049             if k not in self.build_state:
1050                 self.build_state[k] = {}
1051             if 'build-time' not in self.build_state[k]:
1052                 self.build_state[k]['build-time'] = ''
1053             if 'build-versions' not in self.build_state[k]:
1054                 self.build_state[k]['build-versions'] = {}
1055             if 'build-results' not in self.build_state[k]:
1056                 self.build_state[k]['build-results'] = {}
1057             if 'result-changes' not in self.build_state[k]:
1058                 self.build_state[k]['result-changes'] = {}
1059             if 'ever-passed' not in self.build_state[k]:
1060                 self.build_state[k]['ever-passed'] = []
1061
1062     def store_build_state_json(self):
1063         """Store information about the state of previous builds."""
1064         self.store_json(self.build_state, self.build_state_json)
1065
1066     def clear_last_build_state(self, action):
1067         """Clear information about the state of part of the build."""
1068         # We clear the last build time and versions when starting a
1069         # new build.  The results of the last build are kept around,
1070         # as comparison is still meaningful if this build is aborted
1071         # and a new one started.
1072         self.build_state[action]['build-time'] = ''
1073         self.build_state[action]['build-versions'] = {}
1074         self.store_build_state_json()
1075
1076     def update_build_state(self, action, build_time, build_versions):
1077         """Update the build state after a build."""
1078         build_time = build_time.replace(microsecond=0)
1079         self.build_state[action]['build-time'] = build_time.strftime(
1080             '%Y-%m-%d %H:%M:%S')
1081         self.build_state[action]['build-versions'] = build_versions
1082         build_results = {}
1083         for log in self.status_log_list:
1084             with open(log, 'r') as f:
1085                 log_text = f.read()
1086             log_text = log_text.rstrip()
1087             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1088             result = m.group(1)
1089             test_name = m.group(2)
1090             assert test_name not in build_results
1091             build_results[test_name] = result
1092         old_build_results = self.build_state[action]['build-results']
1093         self.build_state[action]['build-results'] = build_results
1094         result_changes = {}
1095         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1096         for t in all_tests:
1097             if t in old_build_results:
1098                 old_res = old_build_results[t]
1099             else:
1100                 old_res = '(New test)'
1101             if t in build_results:
1102                 new_res = build_results[t]
1103             else:
1104                 new_res = '(Test removed)'
1105             if old_res != new_res:
1106                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1107         self.build_state[action]['result-changes'] = result_changes
1108         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1109                            if t in build_results}
1110         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1111         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1112                                                          new_passes)
1113         self.store_build_state_json()
1114
1115     def load_bot_config_json(self):
1116         """Load bot configuration."""
1117         with open(self.bot_config_json, 'r') as f:
1118             self.bot_config = json.load(f)
1119
1120     def part_build_old(self, action, delay):
1121         """Return whether the last build for a given action was at least a
1122         given number of seconds ago, or does not have a time recorded."""
1123         old_time_str = self.build_state[action]['build-time']
1124         if not old_time_str:
1125             return True
1126         old_time = datetime.datetime.strptime(
1127             old_time_str, '%Y-%m-%d %H:%M:%S').replace(
1128                 tzinfo=datetime.timezone.utc)
1129         new_time = datetime.datetime.now(datetime.timezone.utc)
1130         delta = new_time - old_time
1131         return delta.total_seconds() >= delay
1132
1133     def bot_cycle(self):
1134         """Run a single round of checkout and builds."""
1135         print('Bot cycle starting %s.'
1136               % str(datetime.datetime.now(datetime.timezone.utc)))
1137         self.load_bot_config_json()
1138         actions = ('host-libraries', 'compilers', 'glibcs')
1139         self.bot_run_self(['--replace-sources'], 'checkout')
1140         self.load_versions_json()
1141         if self.get_script_text() != self.script_text:
1142             print('Script changed, re-execing.')
1143             # On script change, all parts of the build should be rerun.
1144             for a in actions:
1145                 self.clear_last_build_state(a)
1146             self.exec_self()
1147         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1148                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1149                                           'mig', 'gnumach', 'hurd'),
1150                             'glibcs': ('glibc',)}
1151         must_build = {}
1152         for a in actions:
1153             build_vers = self.build_state[a]['build-versions']
1154             must_build[a] = False
1155             if not self.build_state[a]['build-time']:
1156                 must_build[a] = True
1157             old_vers = {}
1158             new_vers = {}
1159             for c in check_components[a]:
1160                 if c in build_vers:
1161                     old_vers[c] = build_vers[c]
1162                 new_vers[c] = {'version': self.versions[c]['version'],
1163                                'revision': self.versions[c]['revision']}
1164             if new_vers == old_vers:
1165                 print('Versions for %s unchanged.' % a)
1166             else:
1167                 print('Versions changed or rebuild forced for %s.' % a)
1168                 if a == 'compilers' and not self.part_build_old(
1169                         a, self.bot_config['compilers-rebuild-delay']):
1170                     print('Not requiring rebuild of compilers this soon.')
1171                 else:
1172                     must_build[a] = True
1173         if must_build['host-libraries']:
1174             must_build['compilers'] = True
1175         if must_build['compilers']:
1176             must_build['glibcs'] = True
1177         for a in actions:
1178             if must_build[a]:
1179                 print('Must rebuild %s.' % a)
1180                 self.clear_last_build_state(a)
1181             else:
1182                 print('No need to rebuild %s.' % a)
1183         if os.access(self.logsdir, os.F_OK):
1184             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1185             shutil.copytree(self.logsdir, self.logsdir_old)
1186         for a in actions:
1187             if must_build[a]:
1188                 build_time = datetime.datetime.now(datetime.timezone.utc)
1189                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1190                 self.bot_run_self([], a)
1191                 self.load_build_state_json()
1192                 self.bot_build_mail(a, build_time)
1193         print('Bot cycle done at %s.'
1194               % str(datetime.datetime.now(datetime.timezone.utc)))
1195
1196     def bot_build_mail(self, action, build_time):
1197         """Send email with the results of a build."""
1198         if not ('email-from' in self.bot_config and
1199                 'email-server' in self.bot_config and
1200                 'email-subject' in self.bot_config and
1201                 'email-to' in self.bot_config):
1202             if not self.email_warning:
1203                 print("Email not configured, not sending.")
1204                 self.email_warning = True
1205             return
1206
1207         build_time = build_time.replace(microsecond=0)
1208         subject = (self.bot_config['email-subject'] %
1209                    {'action': action,
1210                     'build-time': build_time.strftime('%Y-%m-%d %H:%M:%S')})
1211         results = self.build_state[action]['build-results']
1212         changes = self.build_state[action]['result-changes']
1213         ever_passed = set(self.build_state[action]['ever-passed'])
1214         versions = self.build_state[action]['build-versions']
1215         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1216         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1217         all_fails = {k for k in results if results[k] == 'FAIL'}
1218         if new_regressions:
1219             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1220             new_reg_text = ('New regressions:\n\n%s\n\n' %
1221                             '\n'.join(new_reg_list))
1222         else:
1223             new_reg_text = ''
1224         if all_regressions:
1225             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1226             all_reg_text = ('All regressions:\n\n%s\n\n' %
1227                             '\n'.join(all_reg_list))
1228         else:
1229             all_reg_text = ''
1230         if all_fails:
1231             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1232             all_fail_text = ('All failures:\n\n%s\n\n' %
1233                              '\n'.join(all_fail_list))
1234         else:
1235             all_fail_text = ''
1236         if changes:
1237             changes_list = sorted(changes.keys())
1238             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1239             changes_text = ('All changed results:\n\n%s\n\n' %
1240                             '\n'.join(changes_list))
1241         else:
1242             changes_text = ''
1243         results_text = (new_reg_text + all_reg_text + all_fail_text +
1244                         changes_text)
1245         if not results_text:
1246             results_text = 'Clean build with unchanged results.\n\n'
1247         versions_list = sorted(versions.keys())
1248         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1249                                           versions[k]['revision'])
1250                          for k in versions_list]
1251         versions_text = ('Component versions for this build:\n\n%s\n' %
1252                          '\n'.join(versions_list))
1253         body_text = results_text + versions_text
1254         msg = email.mime.text.MIMEText(body_text)
1255         msg['Subject'] = subject
1256         msg['From'] = self.bot_config['email-from']
1257         msg['To'] = self.bot_config['email-to']
1258         msg['Message-ID'] = email.utils.make_msgid()
1259         msg['Date'] = email.utils.format_datetime(
1260             datetime.datetime.now(datetime.timezone.utc))
1261         with smtplib.SMTP(self.bot_config['email-server']) as s:
1262             s.send_message(msg)
1263
1264     def bot_run_self(self, opts, action, check=True):
1265         """Run a copy of this script with given options."""
1266         cmd = [sys.executable, sys.argv[0], '--keep=none',
1267                '-j%d' % self.parallelism]
1268         if self.full_gcc:
1269             cmd.append('--full-gcc')
1270         cmd.extend(opts)
1271         cmd.extend([self.topdir, action])
1272         sys.stdout.flush()
1273         subprocess.run(cmd, check=check)
1274
1275     def bot(self):
1276         """Run repeated rounds of checkout and builds."""
1277         while True:
1278             self.load_bot_config_json()
1279             if not self.bot_config['run']:
1280                 print('Bot exiting by request.')
1281                 exit(0)
1282             self.bot_run_self([], 'bot-cycle', check=False)
1283             self.load_bot_config_json()
1284             if not self.bot_config['run']:
1285                 print('Bot exiting by request.')
1286                 exit(0)
1287             time.sleep(self.bot_config['delay'])
1288             if self.get_script_text() != self.script_text:
1289                 print('Script changed, bot re-execing.')
1290                 self.exec_self()
1291
1292 class LinuxHeadersPolicyForBuild(object):
1293     """Names and directories for installing Linux headers.  Build variant."""
1294
1295     def __init__(self, config):
1296         self.arch = config.arch
1297         self.srcdir = config.ctx.component_srcdir('linux')
1298         self.builddir = config.component_builddir('linux')
1299         self.headers_dir = os.path.join(config.sysroot, 'usr')
1300
1301 class LinuxHeadersPolicyForUpdateSyscalls(object):
1302     """Names and directories for Linux headers.  update-syscalls variant."""
1303
1304     def __init__(self, glibc, headers_dir):
1305         self.arch = glibc.compiler.arch
1306         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1307         self.builddir = glibc.ctx.component_builddir(
1308             'update-syscalls', glibc.name, 'build-linux')
1309         self.headers_dir = headers_dir
1310
1311 def install_linux_headers(policy, cmdlist):
1312     """Install Linux kernel headers."""
1313     arch_map = {'aarch64': 'arm64',
1314                 'alpha': 'alpha',
1315                 'arc': 'arc',
1316                 'arm': 'arm',
1317                 'csky': 'csky',
1318                 'hppa': 'parisc',
1319                 'i486': 'x86',
1320                 'i586': 'x86',
1321                 'i686': 'x86',
1322                 'i786': 'x86',
1323                 'loongarch64': 'loongarch',
1324                 'm68k': 'm68k',
1325                 'microblaze': 'microblaze',
1326                 'mips': 'mips',
1327                 'nios2': 'nios2',
1328                 'or1k': 'openrisc',
1329                 'powerpc': 'powerpc',
1330                 's390': 's390',
1331                 'riscv32': 'riscv',
1332                 'riscv64': 'riscv',
1333                 'sh': 'sh',
1334                 'sparc': 'sparc',
1335                 'x86_64': 'x86'}
1336     linux_arch = None
1337     for k in arch_map:
1338         if policy.arch.startswith(k):
1339             linux_arch = arch_map[k]
1340             break
1341     assert linux_arch is not None
1342     cmdlist.push_subdesc('linux')
1343     cmdlist.create_use_dir(policy.builddir)
1344     cmdlist.add_command('install-headers',
1345                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1346                          'ARCH=%s' % linux_arch,
1347                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1348                          'headers_install'])
1349     cmdlist.cleanup_dir()
1350     cmdlist.pop_subdesc()
1351
1352 class Config(object):
1353     """A configuration for building a compiler and associated libraries."""
1354
1355     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1356                  first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1357                  extra_glibcs=None):
1358         """Initialize a Config object."""
1359         self.ctx = ctx
1360         self.arch = arch
1361         self.os = os_name
1362         self.variant = variant
1363         if variant is None:
1364             self.name = '%s-%s' % (arch, os_name)
1365         else:
1366             self.name = '%s-%s-%s' % (arch, os_name, variant)
1367         self.triplet = '%s-glibc-%s' % (arch, os_name)
1368         if gcc_cfg is None:
1369             self.gcc_cfg = []
1370         else:
1371             self.gcc_cfg = gcc_cfg
1372         if first_gcc_cfg is None:
1373             self.first_gcc_cfg = []
1374         else:
1375             self.first_gcc_cfg = first_gcc_cfg
1376         if binutils_cfg is None:
1377             self.binutils_cfg = []
1378         else:
1379             self.binutils_cfg = binutils_cfg
1380         if glibcs is None:
1381             glibcs = [{'variant': variant}]
1382         if extra_glibcs is None:
1383             extra_glibcs = []
1384         glibcs = [Glibc(self, **g) for g in glibcs]
1385         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1386         self.all_glibcs = glibcs + extra_glibcs
1387         self.compiler_glibcs = glibcs
1388         self.installdir = ctx.compiler_installdir(self.name)
1389         self.bindir = ctx.compiler_bindir(self.name)
1390         self.sysroot = ctx.compiler_sysroot(self.name)
1391         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1392         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1393
1394     def component_builddir(self, component):
1395         """Return the directory to use for a (non-glibc) build."""
1396         return self.ctx.component_builddir('compilers', self.name, component)
1397
1398     def build(self):
1399         """Generate commands to build this compiler."""
1400         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1401                                       self.logsdir)
1402         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1403         cmdlist.add_command('check-host-libraries',
1404                             ['test', '-f',
1405                              os.path.join(self.ctx.host_libraries_installdir,
1406                                           'ok')])
1407         cmdlist.use_path(self.bindir)
1408         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1409                               ['--disable-gdb',
1410                                '--disable-gdbserver',
1411                                '--disable-libdecnumber',
1412                                '--disable-readline',
1413                                '--disable-sim'] + self.binutils_cfg)
1414         if self.os.startswith('linux'):
1415             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1416         self.build_gcc(cmdlist, True)
1417         if self.os == 'gnu':
1418             self.install_gnumach_headers(cmdlist)
1419             self.build_cross_tool(cmdlist, 'mig', 'mig')
1420             self.install_hurd_headers(cmdlist)
1421         for g in self.compiler_glibcs:
1422             cmdlist.push_subdesc('glibc')
1423             cmdlist.push_subdesc(g.name)
1424             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1425             cmdlist.pop_subdesc()
1426             cmdlist.pop_subdesc()
1427         self.build_gcc(cmdlist, False)
1428         cmdlist.add_command('done', ['touch',
1429                                      os.path.join(self.installdir, 'ok')])
1430         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1431                                       self.logsdir)
1432
1433     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1434         """Build one cross tool."""
1435         srcdir = self.ctx.component_srcdir(tool_src)
1436         builddir = self.component_builddir(tool_build)
1437         cmdlist.push_subdesc(tool_build)
1438         cmdlist.create_use_dir(builddir)
1439         cfg_cmd = [os.path.join(srcdir, 'configure'),
1440                    '--prefix=%s' % self.installdir,
1441                    '--build=%s' % self.ctx.build_triplet,
1442                    '--host=%s' % self.ctx.build_triplet,
1443                    '--target=%s' % self.triplet,
1444                    '--with-sysroot=%s' % self.sysroot]
1445         if extra_opts:
1446             cfg_cmd.extend(extra_opts)
1447         cmdlist.add_command('configure', cfg_cmd)
1448         cmdlist.add_command('build', ['make'])
1449         # Parallel "make install" for GCC has race conditions that can
1450         # cause it to fail; see
1451         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1452         # problems are not known for binutils, but doing the
1453         # installation in parallel within a particular toolchain build
1454         # (as opposed to installation of one toolchain from
1455         # build-many-glibcs.py running in parallel to the installation
1456         # of other toolchains being built) is not known to be
1457         # significantly beneficial, so it is simplest just to disable
1458         # parallel install for cross tools here.
1459         cmdlist.add_command('install', ['make', '-j1', 'install'])
1460         cmdlist.cleanup_dir()
1461         cmdlist.pop_subdesc()
1462
1463     def install_gnumach_headers(self, cmdlist):
1464         """Install GNU Mach headers."""
1465         srcdir = self.ctx.component_srcdir('gnumach')
1466         builddir = self.component_builddir('gnumach')
1467         cmdlist.push_subdesc('gnumach')
1468         cmdlist.create_use_dir(builddir)
1469         cmdlist.add_command('configure',
1470                             [os.path.join(srcdir, 'configure'),
1471                              '--build=%s' % self.ctx.build_triplet,
1472                              '--host=%s' % self.triplet,
1473                              '--prefix=',
1474                              '--disable-user32',
1475                              'CC=%s-gcc -nostdlib' % self.triplet])
1476         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1477                                         'install-data'])
1478         cmdlist.cleanup_dir()
1479         cmdlist.pop_subdesc()
1480
1481     def install_hurd_headers(self, cmdlist):
1482         """Install Hurd headers."""
1483         srcdir = self.ctx.component_srcdir('hurd')
1484         builddir = self.component_builddir('hurd')
1485         cmdlist.push_subdesc('hurd')
1486         cmdlist.create_use_dir(builddir)
1487         cmdlist.add_command('configure',
1488                             [os.path.join(srcdir, 'configure'),
1489                              '--build=%s' % self.ctx.build_triplet,
1490                              '--host=%s' % self.triplet,
1491                              '--prefix=',
1492                              '--disable-profile', '--without-parted',
1493                              'CC=%s-gcc -nostdlib' % self.triplet])
1494         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1495                                         'no_deps=t', 'install-headers'])
1496         cmdlist.cleanup_dir()
1497         cmdlist.pop_subdesc()
1498
1499     def build_gcc(self, cmdlist, bootstrap):
1500         """Build GCC."""
1501         # libssp is of little relevance with glibc's own stack
1502         # checking support.  libcilkrts does not support GNU/Hurd (and
1503         # has been removed in GCC 8, so --disable-libcilkrts can be
1504         # removed once glibc no longer supports building with older
1505         # GCC versions).  --enable-initfini-array is enabled by default
1506         # in GCC 12, which can be removed when GCC 12 becomes the
1507         # minimum requirement.
1508         cfg_opts = list(self.gcc_cfg)
1509         cfg_opts += ['--enable-initfini-array']
1510         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1511         host_libs = self.ctx.host_libraries_installdir
1512         cfg_opts += ['--with-gmp=%s' % host_libs,
1513                      '--with-mpfr=%s' % host_libs,
1514                      '--with-mpc=%s' % host_libs]
1515         if bootstrap:
1516             tool_build = 'gcc-first'
1517             # Building a static-only, C-only compiler that is
1518             # sufficient to build glibc.  Various libraries and
1519             # features that may require libc headers must be disabled.
1520             # When configuring with a sysroot, --with-newlib is
1521             # required to define inhibit_libc (to stop some parts of
1522             # libgcc including libc headers); --without-headers is not
1523             # sufficient.
1524             cfg_opts += ['--enable-languages=c', '--disable-shared',
1525                          '--disable-threads',
1526                          '--disable-libatomic',
1527                          '--disable-decimal-float',
1528                          '--disable-gcov',
1529                          '--disable-libffi',
1530                          '--disable-libgomp',
1531                          '--disable-libitm',
1532                          '--disable-libmpx',
1533                          '--disable-libquadmath',
1534                          '--disable-libsanitizer',
1535                          '--without-headers', '--with-newlib',
1536                          '--with-glibc-version=%s' % self.ctx.glibc_version
1537                          ]
1538             cfg_opts += self.first_gcc_cfg
1539         else:
1540             tool_build = 'gcc'
1541             # libsanitizer commonly breaks because of glibc header
1542             # changes, or on unusual targets.  C++ pre-compiled
1543             # headers are not used during the glibc build and are
1544             # expensive to create.
1545             if not self.ctx.full_gcc:
1546                 cfg_opts += ['--disable-libsanitizer',
1547                              '--disable-libstdcxx-pch']
1548             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1549             cfg_opts += ['--enable-languages=%s' % langs,
1550                          '--enable-shared', '--enable-threads']
1551         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1552
1553 class GlibcPolicyDefault(object):
1554     """Build policy for glibc: common defaults."""
1555
1556     def __init__(self, glibc):
1557         self.srcdir = glibc.ctx.component_srcdir('glibc')
1558         self.use_usr = glibc.os != 'gnu'
1559         self.prefix = '/usr' if self.use_usr else ''
1560         self.configure_args = [
1561             '--prefix=%s' % self.prefix,
1562             '--enable-profile',
1563             '--build=%s' % glibc.ctx.build_triplet,
1564             '--host=%s' % glibc.triplet,
1565             'CC=%s' % glibc.tool_name('gcc'),
1566             'CXX=%s' % glibc.tool_name('g++'),
1567         ]
1568         if glibc.os == 'gnu':
1569             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1570         if glibc.cflags:
1571             self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1572             self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
1573         self.configure_args += glibc.cfg
1574
1575     def configure(self, cmdlist):
1576         """Invoked to add the configure command to the command list."""
1577         cmdlist.add_command('configure',
1578                             [os.path.join(self.srcdir, 'configure'),
1579                              *self.configure_args])
1580
1581     def extra_commands(self, cmdlist):
1582         """Invoked to inject additional commands (make check) after build."""
1583         pass
1584
1585 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1586     """Build policy for glibc during the compilers stage."""
1587
1588     def __init__(self, glibc):
1589         super().__init__(glibc)
1590         self.builddir = glibc.ctx.component_builddir(
1591             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1592         self.installdir = glibc.compiler.sysroot
1593
1594 class GlibcPolicyForBuild(GlibcPolicyDefault):
1595     """Build policy for glibc during the glibcs stage."""
1596
1597     def __init__(self, glibc):
1598         super().__init__(glibc)
1599         self.builddir = glibc.ctx.component_builddir(
1600             'glibcs', glibc.name, 'glibc')
1601         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1602         if glibc.ctx.strip:
1603             self.strip = glibc.tool_name('strip')
1604         else:
1605             self.strip = None
1606         self.save_logs = glibc.ctx.save_logs
1607
1608     def extra_commands(self, cmdlist):
1609         if self.strip:
1610             # Avoid stripping libc.so and libpthread.so, which are
1611             # linker scripts stored in /lib on Hurd.
1612             find_command = 'find %s/lib* -name "*.so*"' % self.installdir
1613             cmdlist.add_command('strip', ['sh', '-c', (
1614                 'set -e; for f in $(%s); do '
1615                 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1616                 'done' % (find_command, self.strip))])
1617         cmdlist.add_command('check', ['make', 'check'])
1618         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1619
1620 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1621     """Build policy for glibc during update-syscalls."""
1622
1623     def __init__(self, glibc):
1624         super().__init__(glibc)
1625         self.builddir = glibc.ctx.component_builddir(
1626             'update-syscalls', glibc.name, 'glibc')
1627         self.linuxdir = glibc.ctx.component_builddir(
1628             'update-syscalls', glibc.name, 'linux')
1629         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1630             glibc, self.linuxdir)
1631         self.configure_args.insert(
1632             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1633         # self.installdir not set because installation is not supported
1634
1635 class Glibc(object):
1636     """A configuration for building glibc."""
1637
1638     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1639                  cfg=None, ccopts=None, cflags=None):
1640         """Initialize a Glibc object."""
1641         self.ctx = compiler.ctx
1642         self.compiler = compiler
1643         if arch is None:
1644             self.arch = compiler.arch
1645         else:
1646             self.arch = arch
1647         if os_name is None:
1648             self.os = compiler.os
1649         else:
1650             self.os = os_name
1651         self.variant = variant
1652         if variant is None:
1653             self.name = '%s-%s' % (self.arch, self.os)
1654         else:
1655             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1656         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1657         if cfg is None:
1658             self.cfg = []
1659         else:
1660             self.cfg = cfg
1661         # ccopts contain ABI options and are passed to configure as CC / CXX.
1662         self.ccopts = ccopts
1663         # cflags contain non-ABI options like -g or -O and are passed to
1664         # configure as CFLAGS / CXXFLAGS.
1665         self.cflags = cflags
1666
1667     def tool_name(self, tool):
1668         """Return the name of a cross-compilation tool."""
1669         ctool = '%s-%s' % (self.compiler.triplet, tool)
1670         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1671             ctool = '%s %s' % (ctool, self.ccopts)
1672         return ctool
1673
1674     def build(self):
1675         """Generate commands to build this glibc."""
1676         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1677         installdir = self.ctx.glibc_installdir(self.name)
1678         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1679         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1680         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1681         cmdlist.add_command('check-compilers',
1682                             ['test', '-f',
1683                              os.path.join(self.compiler.installdir, 'ok')])
1684         cmdlist.use_path(self.compiler.bindir)
1685         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1686         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1687                                       logsdir)
1688
1689     def build_glibc(self, cmdlist, policy):
1690         """Generate commands to build this glibc, either as part of a compiler
1691         build or with the bootstrapped compiler (and in the latter case, run
1692         tests as well)."""
1693         cmdlist.create_use_dir(policy.builddir)
1694         policy.configure(cmdlist)
1695         cmdlist.add_command('build', ['make'])
1696         cmdlist.add_command('install', ['make', 'install',
1697                                         'install_root=%s' % policy.installdir])
1698         # GCC uses paths such as lib/../lib64, so make sure lib
1699         # directories always exist.
1700         mkdir_cmd = ['mkdir', '-p',
1701                      os.path.join(policy.installdir, 'lib')]
1702         if policy.use_usr:
1703             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1704         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1705         policy.extra_commands(cmdlist)
1706         cmdlist.cleanup_dir()
1707
1708     def update_syscalls(self):
1709         if self.os == 'gnu':
1710             # Hurd does not have system call tables that need updating.
1711             return
1712
1713         policy = GlibcPolicyForUpdateSyscalls(self)
1714         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1715         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1716         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1717         cmdlist.add_command('check-compilers',
1718                             ['test', '-f',
1719                              os.path.join(self.compiler.installdir, 'ok')])
1720         cmdlist.use_path(self.compiler.bindir)
1721
1722         install_linux_headers(policy.linux_policy, cmdlist)
1723
1724         cmdlist.create_use_dir(policy.builddir)
1725         policy.configure(cmdlist)
1726         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1727         cmdlist.cleanup_dir()
1728         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1729                                       cmdlist, logsdir)
1730
1731 class Command(object):
1732     """A command run in the build process."""
1733
1734     def __init__(self, desc, num, dir, path, command, always_run=False):
1735         """Initialize a Command object."""
1736         self.dir = dir
1737         self.path = path
1738         self.desc = desc
1739         trans = str.maketrans({' ': '-'})
1740         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1741         self.command = command
1742         self.always_run = always_run
1743
1744     @staticmethod
1745     def shell_make_quote_string(s):
1746         """Given a string not containing a newline, quote it for use by the
1747         shell and make."""
1748         assert '\n' not in s
1749         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1750             return s
1751         strans = str.maketrans({"'": "'\\''"})
1752         s = "'%s'" % s.translate(strans)
1753         mtrans = str.maketrans({'$': '$$'})
1754         return s.translate(mtrans)
1755
1756     @staticmethod
1757     def shell_make_quote_list(l, translate_make):
1758         """Given a list of strings not containing newlines, quote them for use
1759         by the shell and make, returning a single string.  If translate_make
1760         is true and the first string is 'make', change it to $(MAKE)."""
1761         l = [Command.shell_make_quote_string(s) for s in l]
1762         if translate_make and l[0] == 'make':
1763             l[0] = '$(MAKE)'
1764         return ' '.join(l)
1765
1766     def shell_make_quote(self):
1767         """Return this command quoted for the shell and make."""
1768         return self.shell_make_quote_list(self.command, True)
1769
1770
1771 class CommandList(object):
1772     """A list of commands run in the build process."""
1773
1774     def __init__(self, desc, keep):
1775         """Initialize a CommandList object."""
1776         self.cmdlist = []
1777         self.dir = None
1778         self.path = None
1779         self.desc = [desc]
1780         self.keep = keep
1781
1782     def desc_txt(self, desc):
1783         """Return the description to use for a command."""
1784         return '%s %s' % (' '.join(self.desc), desc)
1785
1786     def use_dir(self, dir):
1787         """Set the default directory for subsequent commands."""
1788         self.dir = dir
1789
1790     def use_path(self, path):
1791         """Set a directory to be prepended to the PATH for subsequent
1792         commands."""
1793         self.path = path
1794
1795     def push_subdesc(self, subdesc):
1796         """Set the default subdescription for subsequent commands (e.g., the
1797         name of a component being built, within the series of commands
1798         building it)."""
1799         self.desc.append(subdesc)
1800
1801     def pop_subdesc(self):
1802         """Pop a subdescription from the list of descriptions."""
1803         self.desc.pop()
1804
1805     def create_use_dir(self, dir):
1806         """Remove and recreate a directory and use it for subsequent
1807         commands."""
1808         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1809         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1810         self.use_dir(dir)
1811
1812     def add_command_dir(self, desc, dir, command, always_run=False):
1813         """Add a command to run in a given directory."""
1814         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1815                       command, always_run)
1816         self.cmdlist.append(cmd)
1817
1818     def add_command(self, desc, command, always_run=False):
1819         """Add a command to run in the default directory."""
1820         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1821                       self.path, command, always_run)
1822         self.cmdlist.append(cmd)
1823
1824     def cleanup_dir(self, desc='cleanup', dir=None):
1825         """Clean up a build directory.  If no directory is specified, the
1826         default directory is cleaned up and ceases to be the default
1827         directory."""
1828         if dir is None:
1829             dir = self.dir
1830             self.use_dir(None)
1831         if self.keep != 'all':
1832             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1833                                  always_run=(self.keep == 'none'))
1834
1835     def makefile_commands(self, wrapper, logsdir):
1836         """Return the sequence of commands in the form of text for a Makefile.
1837         The given wrapper script takes arguments: base of logs for
1838         previous command, or empty; base of logs for this command;
1839         description; directory; PATH addition; the command itself."""
1840         # prev_base is the base of the name for logs of the previous
1841         # command that is not always-run (that is, a build command,
1842         # whose failure should stop subsequent build commands from
1843         # being run, as opposed to a cleanup command, which is run
1844         # even if previous commands failed).
1845         prev_base = ''
1846         cmds = []
1847         for c in self.cmdlist:
1848             ctxt = c.shell_make_quote()
1849             if prev_base and not c.always_run:
1850                 prev_log = os.path.join(logsdir, prev_base)
1851             else:
1852                 prev_log = ''
1853             this_log = os.path.join(logsdir, c.logbase)
1854             if not c.always_run:
1855                 prev_base = c.logbase
1856             if c.dir is None:
1857                 dir = ''
1858             else:
1859                 dir = c.dir
1860             if c.path is None:
1861                 path = ''
1862             else:
1863                 path = c.path
1864             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1865             prelim_txt = Command.shell_make_quote_list(prelims, False)
1866             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1867         return '\n'.join(cmds)
1868
1869     def status_logs(self, logsdir):
1870         """Return the list of log files with command status."""
1871         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1872                 for c in self.cmdlist]
1873
1874
1875 def get_parser():
1876     """Return an argument parser for this module."""
1877     parser = argparse.ArgumentParser(description=__doc__)
1878     parser.add_argument('-j', dest='parallelism',
1879                         help='Run this number of jobs in parallel',
1880                         type=int, default=os.cpu_count())
1881     parser.add_argument('--keep', dest='keep',
1882                         help='Whether to keep all build directories, '
1883                         'none or only those from failed builds',
1884                         default='none', choices=('none', 'all', 'failed'))
1885     parser.add_argument('--replace-sources', action='store_true',
1886                         help='Remove and replace source directories '
1887                         'with the wrong version of a component')
1888     parser.add_argument('--strip', action='store_true',
1889                         help='Strip installed glibc libraries')
1890     parser.add_argument('--full-gcc', action='store_true',
1891                         help='Build GCC with all languages and libsanitizer')
1892     parser.add_argument('--shallow', action='store_true',
1893                         help='Do not download Git history during checkout')
1894     parser.add_argument('topdir',
1895                         help='Toplevel working directory')
1896     parser.add_argument('action',
1897                         help='What to do',
1898                         choices=('checkout', 'bot-cycle', 'bot',
1899                                  'host-libraries', 'compilers', 'glibcs',
1900                                  'update-syscalls', 'list-compilers',
1901                                  'list-glibcs'))
1902     parser.add_argument('configs',
1903                         help='Versions to check out or configurations to build',
1904                         nargs='*')
1905     return parser
1906
1907
1908 def get_version_common(progname,line,word,arg1):
1909     try:
1910         out = subprocess.run([progname, arg1],
1911                              stdout=subprocess.PIPE,
1912                              stderr=subprocess.DEVNULL,
1913                              stdin=subprocess.DEVNULL,
1914                              check=True, universal_newlines=True)
1915         v = out.stdout.splitlines()[line].split()[word]
1916         v = re.match(r'[0-9]+(.[0-9]+)*', v).group()
1917         return [int(x) for x in v.split('.')]
1918     except:
1919         return 'missing';
1920
1921 def get_version_common_stderr(progname,line,word,arg1):
1922     try:
1923         out = subprocess.run([progname, arg1],
1924                              stdout=subprocess.DEVNULL,
1925                              stderr=subprocess.PIPE,
1926                              stdin=subprocess.DEVNULL,
1927                              check=True, universal_newlines=True)
1928         v = out.stderr.splitlines()[line].split()[word]
1929         v = re.match(r'[0-9]+(.[0-9]+)*', v).group()
1930         return [int(x) for x in v.split('.')]
1931     except:
1932         return 'missing';
1933
1934 def get_version(progname):
1935     return get_version_common(progname, 0, -1, '--version');
1936
1937 def get_version_awk(progname):
1938     return get_version_common(progname, 0, 2, '--version');
1939
1940 def get_version_bzip2(progname):
1941     return get_version_common_stderr(progname, 0, 6, '-h');
1942
1943 def check_version(ver, req):
1944     for v, r in zip(ver, req):
1945         if v > r:
1946             return True
1947         if v < r:
1948             return False
1949     return True
1950
1951 def version_str(ver):
1952     return '.'.join([str (x) for x in ver])
1953
1954 def check_for_required_tools():
1955     get_list_of_required_tools()
1956     count_old_tools = 0
1957     count_missing_tools = 0
1958
1959     for k, v in REQUIRED_TOOLS.items():
1960         version = v[0](k)
1961         if version == 'missing':
1962             ok = 'missing'
1963         else:
1964             ok = 'ok' if check_version (version, v[1]) else 'old'
1965         if ok == 'old':
1966             if count_old_tools == 0:
1967                 print("One or more required tools are too old:")
1968             count_old_tools = count_old_tools + 1
1969             print('{:9}: {:3} (obtained=\"{}\" required=\"{}\")'.format(k, ok,
1970                     version_str(version), version_str(v[1])))
1971         if ok == 'missing':
1972             if count_missing_tools == 0:
1973                 print("One or more required tools are missing:")
1974             count_missing_tools = count_missing_tools + 1
1975             print('{:9}: {:3} (required=\"{}\")'.format(k, ok,
1976                     version_str(v[1])))
1977
1978     if count_old_tools > 0 or count_missing_tools > 0:
1979         exit (1);
1980
1981 def main(argv):
1982     """The main entry point."""
1983     check_for_required_tools();
1984     parser = get_parser()
1985     opts = parser.parse_args(argv)
1986     topdir = os.path.abspath(opts.topdir)
1987     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1988                   opts.strip, opts.full_gcc, opts.action,
1989                   shallow=opts.shallow)
1990     ctx.run_builds(opts.action, opts.configs)
1991
1992
1993 if __name__ == '__main__':
1994     main(sys.argv[1:])