scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2024 Free Software Foundation, Inc.
   4 # Copyright The GNU Toolchain Authors.
   5 # This file is part of the GNU C Library.
   6 #
   7 # The GNU C Library is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU Lesser General Public
   9 # License as published by the Free Software Foundation; either
  10 # version 2.1 of the License, or (at your option) any later version.
  11 #
  12 # The GNU C Library is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 # Lesser General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Lesser General Public
  18 # License along with the GNU C Library; if not, see
  19 # <https://www.gnu.org/licenses/>.
  20
  21 """Build many configurations of glibc.
  22
  23 This script takes as arguments a directory name (containing a src
  24 subdirectory with sources of the relevant toolchain components) and a
  25 description of what to do: 'checkout', to check out sources into that
  26 directory, 'bot-cycle', to run a series of checkout and build steps,
  27 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  28 libraries required by the toolchain, 'compilers', to build
  29 cross-compilers for various configurations, or 'glibcs', to build
  30 glibc for various configurations and run the compilation parts of the
  31 testsuite.  Subsequent arguments name the versions of components to
  32 check out (<component>-<version), for 'checkout', or, for actions
  33 other than 'checkout' and 'bot-cycle', name configurations for which
  34 compilers or glibc are to be built.
  35
  36 The 'list-compilers' command prints the name of each available
  37 compiler configuration, without building anything.  The 'list-glibcs'
  38 command prints the name of each glibc compiler configuration, followed
  39 by the space, followed by the name of the compiler configuration used
  40 for building this glibc variant.
  41
  42 """
  43
  44 import argparse
  45 import datetime
  46 import email.mime.text
  47 import email.utils
  48 import json
  49 import os
  50 import re
  51 import shutil
  52 import smtplib
  53 import stat
  54 import subprocess
  55 import sys
  56 import time
  57 import urllib.request
  58
  59 # This is a list of system utilities that are expected to be available
  60 # to this script, and, if a non-zero version is included, the minimum
  61 # version required to work with this sccript.
  62 def get_list_of_required_tools():
  63     global REQUIRED_TOOLS
  64     REQUIRED_TOOLS = {
  65         'awk'      : (get_version_awk,   (0,0,0)),
  66         'bison'    : (get_version,       (0,0)),
  67         'flex'     : (get_version,       (0,0,0)),
  68         'git'      : (get_version,       (1,8,3)),
  69         'make'     : (get_version,       (4,0)),
  70         'makeinfo' : (get_version,       (0,0)),
  71         'patch'    : (get_version,       (0,0,0)),
  72         'sed'      : (get_version,       (0,0)),
  73         'tar'      : (get_version,       (0,0,0)),
  74         'gzip'     : (get_version,       (0,0)),
  75         'bzip2'    : (get_version_bzip2, (0,0,0)),
  76         'xz'       : (get_version,       (0,0,0)),
  77     }
  78
  79 try:
  80     subprocess.run
  81 except:
  82     class _CompletedProcess:
  83         def __init__(self, args, returncode, stdout=None, stderr=None):
  84             self.args = args
  85             self.returncode = returncode
  86             self.stdout = stdout
  87             self.stderr = stderr
  88
  89     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  90         assert(timeout is None)
  91         with subprocess.Popen(*popenargs, **kwargs) as process:
  92             try:
  93                 stdout, stderr = process.communicate(input)
  94             except:
  95                 process.kill()
  96                 process.wait()
  97                 raise
  98             returncode = process.poll()
  99             if check and returncode:
 100                 raise subprocess.CalledProcessError(returncode, popenargs)
 101         return _CompletedProcess(popenargs, returncode, stdout, stderr)
 102
 103     subprocess.run = _run
 104
 105
 106 class Context(object):
 107     """The global state associated with builds in a given directory."""
 108
 109     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
 110                  full_gcc, action, shallow=False):
 111         """Initialize the context."""
 112         self.topdir = topdir
 113         self.parallelism = parallelism
 114         self.keep = keep
 115         self.replace_sources = replace_sources
 116         self.strip = strip
 117         self.full_gcc = full_gcc
 118         self.shallow = shallow
 119         self.srcdir = os.path.join(topdir, 'src')
 120         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 121         self.build_state_json = os.path.join(topdir, 'build-state.json')
 122         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 123         self.installdir = os.path.join(topdir, 'install')
 124         self.host_libraries_installdir = os.path.join(self.installdir,
 125                                                       'host-libraries')
 126         self.builddir = os.path.join(topdir, 'build')
 127         self.logsdir = os.path.join(topdir, 'logs')
 128         self.logsdir_old = os.path.join(topdir, 'logs-old')
 129         self.makefile = os.path.join(self.builddir, 'Makefile')
 130         self.wrapper = os.path.join(self.builddir, 'wrapper')
 131         self.save_logs = os.path.join(self.builddir, 'save-logs')
 132         self.script_text = self.get_script_text()
 133         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 134             self.build_triplet = self.get_build_triplet()
 135             self.glibc_version = self.get_glibc_version()
 136         self.configs = {}
 137         self.glibc_configs = {}
 138         self.makefile_pieces = ['.PHONY: all\n']
 139         self.add_all_configs()
 140         self.load_versions_json()
 141         self.load_build_state_json()
 142         self.status_log_list = []
 143         self.email_warning = False
 144
 145     def get_script_text(self):
 146         """Return the text of this script."""
 147         with open(sys.argv[0], 'r') as f:
 148             return f.read()
 149
 150     def exec_self(self):
 151         """Re-execute this script with the same arguments."""
 152         sys.stdout.flush()
 153         os.execv(sys.executable, [sys.executable] + sys.argv)
 154
 155     def get_build_triplet(self):
 156         """Determine the build triplet with config.guess."""
 157         config_guess = os.path.join(self.component_srcdir('gcc'),
 158                                     'config.guess')
 159         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 160                                 check=True, universal_newlines=True).stdout
 161         return cg_out.rstrip()
 162
 163     def get_glibc_version(self):
 164         """Determine the glibc version number (major.minor)."""
 165         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 166         with open(version_h, 'r') as f:
 167             lines = f.readlines()
 168         starttext = '#define VERSION "'
 169         for l in lines:
 170             if l.startswith(starttext):
 171                 l = l[len(starttext):]
 172                 l = l.rstrip('"\n')
 173                 m = re.fullmatch(r'([0-9]+)\.([0-9]+)[.0-9]*', l)
 174                 return '%s.%s' % m.group(1, 2)
 175         print('error: could not determine glibc version')
 176         exit(1)
 177
 178     def add_all_configs(self):
 179         """Add all known glibc build configurations."""
 180         self.add_config(arch='aarch64',
 181                         os_name='linux-gnu',
 182                         extra_glibcs=[{'variant': 'disable-multi-arch',
 183                                        'cfg': ['--disable-multi-arch']}])
 184         self.add_config(arch='aarch64_be',
 185                         os_name='linux-gnu')
 186         self.add_config(arch='arc',
 187                         os_name='linux-gnu',
 188                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 189         self.add_config(arch='arc',
 190                         os_name='linux-gnuhf',
 191                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
 192         self.add_config(arch='arceb',
 193                         os_name='linux-gnu',
 194                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 195         self.add_config(arch='alpha',
 196                         os_name='linux-gnu')
 197         self.add_config(arch='arm',
 198                         os_name='linux-gnueabi',
 199                         extra_glibcs=[{'variant': 'v4t',
 200                                        'ccopts': '-march=armv4t'}])
 201         self.add_config(arch='armeb',
 202                         os_name='linux-gnueabi')
 203         self.add_config(arch='armeb',
 204                         os_name='linux-gnueabi',
 205                         variant='be8',
 206                         gcc_cfg=['--with-arch=armv7-a'])
 207         self.add_config(arch='arm',
 208                         os_name='linux-gnueabihf',
 209                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 210                         extra_glibcs=[{'variant': 'v7a',
 211                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 212                                       {'variant': 'thumb',
 213                                        'ccopts':
 214                                        '-mthumb -march=armv7-a -mfpu=vfpv3'},
 215                                       {'variant': 'v7a-disable-multi-arch',
 216                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 217                                        'cfg': ['--disable-multi-arch']}])
 218         self.add_config(arch='armeb',
 219                         os_name='linux-gnueabihf',
 220                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 221         self.add_config(arch='armeb',
 222                         os_name='linux-gnueabihf',
 223                         variant='be8',
 224                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 225                                  '--with-fpu=vfpv3'])
 226         self.add_config(arch='csky',
 227                         os_name='linux-gnuabiv2',
 228                         variant='soft',
 229                         gcc_cfg=['--disable-multilib'])
 230         self.add_config(arch='csky',
 231                         os_name='linux-gnuabiv2',
 232                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 233         self.add_config(arch='hppa',
 234                         os_name='linux-gnu')
 235         self.add_config(arch='i686',
 236                         os_name='gnu')
 237         self.add_config(arch='loongarch64',
 238                         os_name='linux-gnu',
 239                         variant='lp64d',
 240                         gcc_cfg=['--with-abi=lp64d','--disable-multilib'])
 241         self.add_config(arch='loongarch64',
 242                         os_name='linux-gnu',
 243                         variant='lp64s',
 244                         gcc_cfg=['--with-abi=lp64s','--disable-multilib'])
 245         self.add_config(arch='m68k',
 246                         os_name='linux-gnu',
 247                         gcc_cfg=['--disable-multilib'])
 248         self.add_config(arch='m68k',
 249                         os_name='linux-gnu',
 250                         variant='coldfire',
 251                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 252         self.add_config(arch='m68k',
 253                         os_name='linux-gnu',
 254                         variant='coldfire-soft',
 255                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 256                                  '--disable-multilib'])
 257         self.add_config(arch='microblaze',
 258                         os_name='linux-gnu',
 259                         gcc_cfg=['--disable-multilib'])
 260         self.add_config(arch='microblazeel',
 261                         os_name='linux-gnu',
 262                         gcc_cfg=['--disable-multilib'])
 263         self.add_config(arch='mips64',
 264                         os_name='linux-gnu',
 265                         gcc_cfg=['--with-mips-plt'],
 266                         glibcs=[{'variant': 'n32'},
 267                                 {'arch': 'mips',
 268                                  'ccopts': '-mabi=32'},
 269                                 {'variant': 'n64',
 270                                  'ccopts': '-mabi=64'}])
 271         self.add_config(arch='mips64',
 272                         os_name='linux-gnu',
 273                         variant='soft',
 274                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 275                         glibcs=[{'variant': 'n32-soft'},
 276                                 {'variant': 'soft',
 277                                  'arch': 'mips',
 278                                  'ccopts': '-mabi=32'},
 279                                 {'variant': 'n64-soft',
 280                                  'ccopts': '-mabi=64'}])
 281         self.add_config(arch='mips64',
 282                         os_name='linux-gnu',
 283                         variant='nan2008',
 284                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 285                                  '--with-arch-64=mips64r2',
 286                                  '--with-arch-32=mips32r2'],
 287                         glibcs=[{'variant': 'n32-nan2008'},
 288                                 {'variant': 'nan2008',
 289                                  'arch': 'mips',
 290                                  'ccopts': '-mabi=32'},
 291                                 {'variant': 'n64-nan2008',
 292                                  'ccopts': '-mabi=64'}])
 293         self.add_config(arch='mips64',
 294                         os_name='linux-gnu',
 295                         variant='nan2008-soft',
 296                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 297                                  '--with-arch-64=mips64r2',
 298                                  '--with-arch-32=mips32r2',
 299                                  '--with-float=soft'],
 300                         glibcs=[{'variant': 'n32-nan2008-soft'},
 301                                 {'variant': 'nan2008-soft',
 302                                  'arch': 'mips',
 303                                  'ccopts': '-mabi=32'},
 304                                 {'variant': 'n64-nan2008-soft',
 305                                  'ccopts': '-mabi=64'}])
 306         self.add_config(arch='mips64el',
 307                         os_name='linux-gnu',
 308                         gcc_cfg=['--with-mips-plt'],
 309                         glibcs=[{'variant': 'n32'},
 310                                 {'arch': 'mipsel',
 311                                  'ccopts': '-mabi=32'},
 312                                 {'variant': 'n64',
 313                                  'ccopts': '-mabi=64'}])
 314         self.add_config(arch='mips64el',
 315                         os_name='linux-gnu',
 316                         variant='soft',
 317                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 318                         glibcs=[{'variant': 'n32-soft'},
 319                                 {'variant': 'soft',
 320                                  'arch': 'mipsel',
 321                                  'ccopts': '-mabi=32'},
 322                                 {'variant': 'n64-soft',
 323                                  'ccopts': '-mabi=64'}])
 324         self.add_config(arch='mips64el',
 325                         os_name='linux-gnu',
 326                         variant='nan2008',
 327                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 328                                  '--with-arch-64=mips64r2',
 329                                  '--with-arch-32=mips32r2'],
 330                         glibcs=[{'variant': 'n32-nan2008'},
 331                                 {'variant': 'nan2008',
 332                                  'arch': 'mipsel',
 333                                  'ccopts': '-mabi=32'},
 334                                 {'variant': 'n64-nan2008',
 335                                  'ccopts': '-mabi=64'}])
 336         self.add_config(arch='mips64el',
 337                         os_name='linux-gnu',
 338                         variant='nan2008-soft',
 339                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 340                                  '--with-arch-64=mips64r2',
 341                                  '--with-arch-32=mips32r2',
 342                                  '--with-float=soft'],
 343                         glibcs=[{'variant': 'n32-nan2008-soft'},
 344                                 {'variant': 'nan2008-soft',
 345                                  'arch': 'mipsel',
 346                                  'ccopts': '-mabi=32'},
 347                                 {'variant': 'n64-nan2008-soft',
 348                                  'ccopts': '-mabi=64'}])
 349         self.add_config(arch='mipsisa64r6el',
 350                         os_name='linux-gnu',
 351                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 352                                  '--with-arch-64=mips64r6',
 353                                  '--with-arch-32=mips32r6',
 354                                  '--with-float=hard'],
 355                         glibcs=[{'variant': 'n32'},
 356                                 {'arch': 'mipsisa32r6el',
 357                                  'ccopts': '-mabi=32'},
 358                                 {'variant': 'n64',
 359                                  'ccopts': '-mabi=64'}])
 360         self.add_config(arch='nios2',
 361                         os_name='linux-gnu')
 362         self.add_config(arch='or1k',
 363                         os_name='linux-gnu',
 364                         variant='soft',
 365                         gcc_cfg=['--with-multilib-list=mcmov'])
 366         self.add_config(arch='powerpc',
 367                         os_name='linux-gnu',
 368                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 369                         extra_glibcs=[{'variant': 'power4',
 370                                        'ccopts': '-mcpu=power4',
 371                                        'cfg': ['--with-cpu=power4']}])
 372         self.add_config(arch='powerpc',
 373                         os_name='linux-gnu',
 374                         variant='soft',
 375                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 376                                  '--enable-secureplt'])
 377         self.add_config(arch='powerpc64',
 378                         os_name='linux-gnu',
 379                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 380         self.add_config(arch='powerpc64le',
 381                         os_name='linux-gnu',
 382                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 383                         extra_glibcs=[{'variant': 'disable-multi-arch',
 384                                        'cfg': ['--disable-multi-arch']}])
 385         self.add_config(arch='riscv32',
 386                         os_name='linux-gnu',
 387                         variant='rv32imac-ilp32',
 388                         gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
 389                                  '--disable-multilib'])
 390         self.add_config(arch='riscv32',
 391                         os_name='linux-gnu',
 392                         variant='rv32imafdc-ilp32',
 393                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
 394                                  '--disable-multilib'])
 395         self.add_config(arch='riscv32',
 396                         os_name='linux-gnu',
 397                         variant='rv32imafdc-ilp32d',
 398                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
 399                                  '--disable-multilib'])
 400         self.add_config(arch='riscv64',
 401                         os_name='linux-gnu',
 402                         variant='rv64imac-lp64',
 403                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 404                                  '--disable-multilib'])
 405         self.add_config(arch='riscv64',
 406                         os_name='linux-gnu',
 407                         variant='rv64imafdc-lp64',
 408                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 409                                  '--disable-multilib'])
 410         self.add_config(arch='riscv64',
 411                         os_name='linux-gnu',
 412                         variant='rv64imafdc-lp64d',
 413                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 414                                  '--disable-multilib'])
 415         self.add_config(arch='s390x',
 416                         os_name='linux-gnu',
 417                         glibcs=[{},
 418                                 {'arch': 's390', 'ccopts': '-m31'}],
 419                         extra_glibcs=[{'variant': 'O3',
 420                                        'cflags': '-O3'}])
 421         self.add_config(arch='sh3',
 422                         os_name='linux-gnu')
 423         self.add_config(arch='sh3eb',
 424                         os_name='linux-gnu')
 425         self.add_config(arch='sh4',
 426                         os_name='linux-gnu')
 427         self.add_config(arch='sh4eb',
 428                         os_name='linux-gnu')
 429         self.add_config(arch='sh4',
 430                         os_name='linux-gnu',
 431                         variant='soft',
 432                         gcc_cfg=['--without-fp'])
 433         self.add_config(arch='sh4eb',
 434                         os_name='linux-gnu',
 435                         variant='soft',
 436                         gcc_cfg=['--without-fp'])
 437         self.add_config(arch='sparc64',
 438                         os_name='linux-gnu',
 439                         glibcs=[{},
 440                                 {'arch': 'sparcv9',
 441                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 442                         extra_glibcs=[{'variant': 'leon3',
 443                                        'arch' : 'sparcv8',
 444                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 445                                       {'variant': 'disable-multi-arch',
 446                                        'cfg': ['--disable-multi-arch']},
 447                                       {'variant': 'disable-multi-arch',
 448                                        'arch': 'sparcv9',
 449                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 450                                        'cfg': ['--disable-multi-arch']}])
 451         self.add_config(arch='x86_64',
 452                         os_name='linux-gnu',
 453                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 454                         glibcs=[{},
 455                                 {'variant': 'x32', 'ccopts': '-mx32'},
 456                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 457                         extra_glibcs=[{'variant': 'disable-multi-arch',
 458                                        'cfg': ['--disable-multi-arch']},
 459                                       {'variant': 'minimal',
 460                                        'cfg': ['--disable-multi-arch',
 461                                                '--disable-profile',
 462                                                '--disable-timezone-tools',
 463                                                '--disable-mathvec',
 464                                                '--disable-build-nscd',
 465                                                '--disable-nscd']},
 466                                       {'variant': 'no-pie',
 467                                        'cfg': ['--disable-default-pie']},
 468                                       {'variant': 'x32-no-pie',
 469                                        'ccopts': '-mx32',
 470                                        'cfg': ['--disable-default-pie']},
 471                                       {'variant': 'no-pie',
 472                                        'arch': 'i686',
 473                                        'ccopts': '-m32 -march=i686',
 474                                        'cfg': ['--disable-default-pie']},
 475                                       {'variant': 'disable-multi-arch',
 476                                        'arch': 'i686',
 477                                        'ccopts': '-m32 -march=i686',
 478                                        'cfg': ['--disable-multi-arch']},
 479                                       {'arch': 'i486',
 480                                        'ccopts': '-m32 -march=i486'},
 481                                       {'arch': 'i586',
 482                                        'ccopts': '-m32 -march=i586'},
 483                                       {'variant': 'enable-fortify-source',
 484                                        'cfg': ['--enable-fortify-source']}])
 485         self.add_config(arch='x86_64',
 486                         os_name='gnu',
 487                         gcc_cfg=['--disable-multilib'])
 488
 489     def add_config(self, **args):
 490         """Add an individual build configuration."""
 491         cfg = Config(self, **args)
 492         if cfg.name in self.configs:
 493             print('error: duplicate config %s' % cfg.name)
 494             exit(1)
 495         self.configs[cfg.name] = cfg
 496         for c in cfg.all_glibcs:
 497             if c.name in self.glibc_configs:
 498                 print('error: duplicate glibc config %s' % c.name)
 499                 exit(1)
 500             self.glibc_configs[c.name] = c
 501
 502     def component_srcdir(self, component):
 503         """Return the source directory for a given component, e.g. gcc."""
 504         return os.path.join(self.srcdir, component)
 505
 506     def component_builddir(self, action, config, component, subconfig=None):
 507         """Return the directory to use for a build."""
 508         if config is None:
 509             # Host libraries.
 510             assert subconfig is None
 511             return os.path.join(self.builddir, action, component)
 512         if subconfig is None:
 513             return os.path.join(self.builddir, action, config, component)
 514         else:
 515             # glibc build as part of compiler build.
 516             return os.path.join(self.builddir, action, config, component,
 517                                 subconfig)
 518
 519     def compiler_installdir(self, config):
 520         """Return the directory in which to install a compiler."""
 521         return os.path.join(self.installdir, 'compilers', config)
 522
 523     def compiler_bindir(self, config):
 524         """Return the directory in which to find compiler binaries."""
 525         return os.path.join(self.compiler_installdir(config), 'bin')
 526
 527     def compiler_sysroot(self, config):
 528         """Return the sysroot directory for a compiler."""
 529         return os.path.join(self.compiler_installdir(config), 'sysroot')
 530
 531     def glibc_installdir(self, config):
 532         """Return the directory in which to install glibc."""
 533         return os.path.join(self.installdir, 'glibcs', config)
 534
 535     def run_builds(self, action, configs):
 536         """Run the requested builds."""
 537         if action == 'checkout':
 538             self.checkout(configs)
 539             return
 540         if action == 'bot-cycle':
 541             if configs:
 542                 print('error: configurations specified for bot-cycle')
 543                 exit(1)
 544             self.bot_cycle()
 545             return
 546         if action == 'bot':
 547             if configs:
 548                 print('error: configurations specified for bot')
 549                 exit(1)
 550             self.bot()
 551             return
 552         if action in ('host-libraries', 'list-compilers',
 553                       'list-glibcs') and configs:
 554             print('error: configurations specified for ' + action)
 555             exit(1)
 556         if action == 'list-compilers':
 557             for name in sorted(self.configs.keys()):
 558                 print(name)
 559             return
 560         if action == 'list-glibcs':
 561             for config in sorted(self.glibc_configs.values(),
 562                                  key=lambda c: c.name):
 563                 print(config.name, config.compiler.name)
 564             return
 565         self.clear_last_build_state(action)
 566         build_time = datetime.datetime.now(datetime.timezone.utc)
 567         if action == 'host-libraries':
 568             build_components = ('gmp', 'mpfr', 'mpc')
 569             old_components = ()
 570             old_versions = {}
 571             self.build_host_libraries()
 572         elif action == 'compilers':
 573             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 574                                 'gnumach', 'hurd')
 575             old_components = ('gmp', 'mpfr', 'mpc')
 576             old_versions = self.build_state['host-libraries']['build-versions']
 577             self.build_compilers(configs)
 578         else:
 579             build_components = ('glibc',)
 580             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 581                               'mig', 'gnumach', 'hurd')
 582             old_versions = self.build_state['compilers']['build-versions']
 583             if action == 'update-syscalls':
 584                 self.update_syscalls(configs)
 585             else:
 586                 self.build_glibcs(configs)
 587         self.write_files()
 588         self.do_build()
 589         if configs:
 590             # Partial build, do not update stored state.
 591             return
 592         build_versions = {}
 593         for k in build_components:
 594             if k in self.versions:
 595                 build_versions[k] = {'version': self.versions[k]['version'],
 596                                      'revision': self.versions[k]['revision']}
 597         for k in old_components:
 598             if k in old_versions:
 599                 build_versions[k] = {'version': old_versions[k]['version'],
 600                                      'revision': old_versions[k]['revision']}
 601         self.update_build_state(action, build_time, build_versions)
 602
 603     @staticmethod
 604     def remove_dirs(*args):
 605         """Remove directories and their contents if they exist."""
 606         for dir in args:
 607             shutil.rmtree(dir, ignore_errors=True)
 608
 609     @staticmethod
 610     def remove_recreate_dirs(*args):
 611         """Remove directories if they exist, and create them as empty."""
 612         Context.remove_dirs(*args)
 613         for dir in args:
 614             os.makedirs(dir, exist_ok=True)
 615
 616     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 617         """Add makefile text for a list of commands."""
 618         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 619         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 620                                     (target, target, target, commands))
 621         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 622
 623     def write_files(self):
 624         """Write out the Makefile and wrapper script."""
 625         mftext = ''.join(self.makefile_pieces)
 626         with open(self.makefile, 'w') as f:
 627             f.write(mftext)
 628         wrapper_text = (
 629             '#!/bin/sh\n'
 630             'prev_base=$1\n'
 631             'this_base=$2\n'
 632             'desc=$3\n'
 633             'dir=$4\n'
 634             'path=$5\n'
 635             'shift 5\n'
 636             'prev_status=$prev_base-status.txt\n'
 637             'this_status=$this_base-status.txt\n'
 638             'this_log=$this_base-log.txt\n'
 639             'date > "$this_log"\n'
 640             'echo >> "$this_log"\n'
 641             'echo "Description: $desc" >> "$this_log"\n'
 642             'printf "%s" "Command:" >> "$this_log"\n'
 643             'for word in "$@"; do\n'
 644             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 645             '    printf " %s" "$word"\n'
 646             '  else\n'
 647             '    printf " \'"\n'
 648             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 649             '    printf "\'"\n'
 650             '  fi\n'
 651             'done >> "$this_log"\n'
 652             'echo >> "$this_log"\n'
 653             'echo "Directory: $dir" >> "$this_log"\n'
 654             'echo "Path addition: $path" >> "$this_log"\n'
 655             'echo >> "$this_log"\n'
 656             'record_status ()\n'
 657             '{\n'
 658             '  echo >> "$this_log"\n'
 659             '  echo "$1: $desc" > "$this_status"\n'
 660             '  echo "$1: $desc" >> "$this_log"\n'
 661             '  echo >> "$this_log"\n'
 662             '  date >> "$this_log"\n'
 663             '  echo "$1: $desc"\n'
 664             '  exit 0\n'
 665             '}\n'
 666             'check_error ()\n'
 667             '{\n'
 668             '  if [ "$1" != "0" ]; then\n'
 669             '    record_status FAIL\n'
 670             '  fi\n'
 671             '}\n'
 672             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 673             '    record_status UNRESOLVED\n'
 674             'fi\n'
 675             'if [ "$dir" ]; then\n'
 676             '  cd "$dir"\n'
 677             '  check_error "$?"\n'
 678             'fi\n'
 679             'if [ "$path" ]; then\n'
 680             '  PATH=$path:$PATH\n'
 681             'fi\n'
 682             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 683             'check_error "$?"\n'
 684             'record_status PASS\n')
 685         with open(self.wrapper, 'w') as f:
 686             f.write(wrapper_text)
 687         # Mode 0o755.
 688         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 689                      stat.S_IROTH|stat.S_IXOTH)
 690         os.chmod(self.wrapper, mode_exec)
 691         save_logs_text = (
 692             '#!/bin/sh\n'
 693             'if ! [ -f tests.sum ]; then\n'
 694             '  echo "No test summary available."\n'
 695             '  exit 0\n'
 696             'fi\n'
 697             'save_file ()\n'
 698             '{\n'
 699             '  echo "Contents of $1:"\n'
 700             '  echo\n'
 701             '  cat "$1"\n'
 702             '  echo\n'
 703             '  echo "End of contents of $1."\n'
 704             '  echo\n'
 705             '}\n'
 706             'save_file tests.sum\n'
 707             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 708             'for t in $non_pass_tests; do\n'
 709             '  if [ -f "$t.out" ]; then\n'
 710             '    save_file "$t.out"\n'
 711             '  fi\n'
 712             'done\n')
 713         with open(self.save_logs, 'w') as f:
 714             f.write(save_logs_text)
 715         os.chmod(self.save_logs, mode_exec)
 716
 717     def do_build(self):
 718         """Do the actual build."""
 719         cmd = ['make', '-O', '-j%d' % self.parallelism]
 720         subprocess.run(cmd, cwd=self.builddir, check=True)
 721
 722     def build_host_libraries(self):
 723         """Build the host libraries."""
 724         installdir = self.host_libraries_installdir
 725         builddir = os.path.join(self.builddir, 'host-libraries')
 726         logsdir = os.path.join(self.logsdir, 'host-libraries')
 727         self.remove_recreate_dirs(installdir, builddir, logsdir)
 728         cmdlist = CommandList('host-libraries', self.keep)
 729         self.build_host_library(cmdlist, 'gmp')
 730         self.build_host_library(cmdlist, 'mpfr',
 731                                 ['--with-gmp=%s' % installdir])
 732         self.build_host_library(cmdlist, 'mpc',
 733                                 ['--with-gmp=%s' % installdir,
 734                                 '--with-mpfr=%s' % installdir])
 735         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 736         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 737
 738     def build_host_library(self, cmdlist, lib, extra_opts=None):
 739         """Build one host library."""
 740         srcdir = self.component_srcdir(lib)
 741         builddir = self.component_builddir('host-libraries', None, lib)
 742         installdir = self.host_libraries_installdir
 743         cmdlist.push_subdesc(lib)
 744         cmdlist.create_use_dir(builddir)
 745         cfg_cmd = [os.path.join(srcdir, 'configure'),
 746                    '--prefix=%s' % installdir,
 747                    '--disable-shared']
 748         if extra_opts:
 749             cfg_cmd.extend (extra_opts)
 750         cmdlist.add_command('configure', cfg_cmd)
 751         cmdlist.add_command('build', ['make'])
 752         cmdlist.add_command('check', ['make', 'check'])
 753         cmdlist.add_command('install', ['make', 'install'])
 754         cmdlist.cleanup_dir()
 755         cmdlist.pop_subdesc()
 756
 757     def build_compilers(self, configs):
 758         """Build the compilers."""
 759         if not configs:
 760             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 761             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 762             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 763             configs = sorted(self.configs.keys())
 764         for c in configs:
 765             self.configs[c].build()
 766
 767     def build_glibcs(self, configs):
 768         """Build the glibcs."""
 769         if not configs:
 770             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 771             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 772             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 773             configs = sorted(self.glibc_configs.keys())
 774         for c in configs:
 775             self.glibc_configs[c].build()
 776
 777     def update_syscalls(self, configs):
 778         """Update the glibc syscall lists."""
 779         if not configs:
 780             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 781             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 782             configs = sorted(self.glibc_configs.keys())
 783         for c in configs:
 784             self.glibc_configs[c].update_syscalls()
 785
 786     def load_versions_json(self):
 787         """Load information about source directory versions."""
 788         if not os.access(self.versions_json, os.F_OK):
 789             self.versions = {}
 790             return
 791         with open(self.versions_json, 'r') as f:
 792             self.versions = json.load(f)
 793
 794     def store_json(self, data, filename):
 795         """Store information in a JSON file."""
 796         filename_tmp = filename + '.tmp'
 797         with open(filename_tmp, 'w') as f:
 798             json.dump(data, f, indent=2, sort_keys=True)
 799         os.rename(filename_tmp, filename)
 800
 801     def store_versions_json(self):
 802         """Store information about source directory versions."""
 803         self.store_json(self.versions, self.versions_json)
 804
 805     def set_component_version(self, component, version, explicit, revision):
 806         """Set the version information for a component."""
 807         self.versions[component] = {'version': version,
 808                                     'explicit': explicit,
 809                                     'revision': revision}
 810         self.store_versions_json()
 811
 812     def checkout(self, versions):
 813         """Check out the desired component versions."""
 814         default_versions = {'binutils': 'vcs-2.41',
 815                             'gcc': 'vcs-13',
 816                             'glibc': 'vcs-mainline',
 817                             'gmp': '6.3.0',
 818                             'linux': '6.6',
 819                             'mpc': '1.3.1',
 820                             'mpfr': '4.2.1',
 821                             'mig': 'vcs-mainline',
 822                             'gnumach': 'vcs-mainline',
 823                             'hurd': 'vcs-mainline'}
 824         use_versions = {}
 825         explicit_versions = {}
 826         for v in versions:
 827             found_v = False
 828             for k in default_versions.keys():
 829                 kx = k + '-'
 830                 if v.startswith(kx):
 831                     vx = v[len(kx):]
 832                     if k in use_versions:
 833                         print('error: multiple versions for %s' % k)
 834                         exit(1)
 835                     use_versions[k] = vx
 836                     explicit_versions[k] = True
 837                     found_v = True
 838                     break
 839             if not found_v:
 840                 print('error: unknown component in %s' % v)
 841                 exit(1)
 842         for k in default_versions.keys():
 843             if k not in use_versions:
 844                 if k in self.versions and self.versions[k]['explicit']:
 845                     use_versions[k] = self.versions[k]['version']
 846                     explicit_versions[k] = True
 847                 else:
 848                     use_versions[k] = default_versions[k]
 849                     explicit_versions[k] = False
 850         os.makedirs(self.srcdir, exist_ok=True)
 851         for k in sorted(default_versions.keys()):
 852             update = os.access(self.component_srcdir(k), os.F_OK)
 853             v = use_versions[k]
 854             if (update and
 855                 k in self.versions and
 856                 v != self.versions[k]['version']):
 857                 if not self.replace_sources:
 858                     print('error: version of %s has changed from %s to %s, '
 859                           'use --replace-sources to check out again' %
 860                           (k, self.versions[k]['version'], v))
 861                     exit(1)
 862                 shutil.rmtree(self.component_srcdir(k))
 863                 update = False
 864             if v.startswith('vcs-'):
 865                 revision = self.checkout_vcs(k, v[4:], update)
 866             else:
 867                 self.checkout_tar(k, v, update)
 868                 revision = v
 869             self.set_component_version(k, v, explicit_versions[k], revision)
 870         if self.get_script_text() != self.script_text:
 871             # Rerun the checkout process in case the updated script
 872             # uses different default versions or new components.
 873             self.exec_self()
 874
 875     def checkout_vcs(self, component, version, update):
 876         """Check out the given version of the given component from version
 877         control.  Return a revision identifier."""
 878         if component == 'binutils':
 879             git_url = 'https://sourceware.org/git/binutils-gdb.git'
 880             if version == 'mainline':
 881                 git_branch = 'master'
 882             else:
 883                 trans = str.maketrans({'.': '_'})
 884                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 885             return self.git_checkout(component, git_url, git_branch, update)
 886         elif component == 'gcc':
 887             if version == 'mainline':
 888                 branch = 'master'
 889             else:
 890                 branch = 'releases/gcc-%s' % version
 891             return self.gcc_checkout(branch, update)
 892         elif component == 'glibc':
 893             git_url = 'https://sourceware.org/git/glibc.git'
 894             if version == 'mainline':
 895                 git_branch = 'master'
 896             else:
 897                 git_branch = 'release/%s/master' % version
 898             r = self.git_checkout(component, git_url, git_branch, update)
 899             self.fix_glibc_timestamps()
 900             return r
 901         elif component == 'gnumach':
 902             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 903             git_branch = 'master'
 904             r = self.git_checkout(component, git_url, git_branch, update)
 905             subprocess.run(['autoreconf', '-i'],
 906                            cwd=self.component_srcdir(component), check=True)
 907             return r
 908         elif component == 'mig':
 909             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 910             git_branch = 'master'
 911             r = self.git_checkout(component, git_url, git_branch, update)
 912             subprocess.run(['autoreconf', '-i'],
 913                            cwd=self.component_srcdir(component), check=True)
 914             return r
 915         elif component == 'hurd':
 916             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 917             git_branch = 'master'
 918             r = self.git_checkout(component, git_url, git_branch, update)
 919             subprocess.run(['autoconf'],
 920                            cwd=self.component_srcdir(component), check=True)
 921             return r
 922         else:
 923             print('error: component %s coming from VCS' % component)
 924             exit(1)
 925
 926     def git_checkout(self, component, git_url, git_branch, update):
 927         """Check out a component from git.  Return a commit identifier."""
 928         if update:
 929             subprocess.run(['git', 'remote', 'prune', 'origin'],
 930                            cwd=self.component_srcdir(component), check=True)
 931             if self.replace_sources:
 932                 subprocess.run(['git', 'clean', '-dxfq'],
 933                                cwd=self.component_srcdir(component), check=True)
 934             subprocess.run(['git', 'pull', '-q'],
 935                            cwd=self.component_srcdir(component), check=True)
 936         else:
 937             if self.shallow:
 938                 depth_arg = ('--depth', '1')
 939             else:
 940                 depth_arg = ()
 941             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 942                             *depth_arg, git_url,
 943                             self.component_srcdir(component)], check=True)
 944         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 945                            cwd=self.component_srcdir(component),
 946                            stdout=subprocess.PIPE,
 947                            check=True, universal_newlines=True).stdout
 948         return r.rstrip()
 949
 950     def fix_glibc_timestamps(self):
 951         """Fix timestamps in a glibc checkout."""
 952         # Ensure that builds do not try to regenerate generated files
 953         # in the source tree.
 954         srcdir = self.component_srcdir('glibc')
 955         # These files have Makefile dependencies to regenerate them in
 956         # the source tree that may be active during a normal build.
 957         # Some other files have such dependencies but do not need to
 958         # be touched because nothing in a build depends on the files
 959         # in question.
 960         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 961             to_touch = os.path.join(srcdir, f)
 962             subprocess.run(['touch', '-c', to_touch], check=True)
 963         for dirpath, dirnames, filenames in os.walk(srcdir):
 964             for f in filenames:
 965                 if (f == 'configure' or
 966                     f == 'preconfigure' or
 967                     f.endswith('-kw.h')):
 968                     to_touch = os.path.join(dirpath, f)
 969                     subprocess.run(['touch', to_touch], check=True)
 970
 971     def gcc_checkout(self, branch, update):
 972         """Check out GCC from git.  Return the commit identifier."""
 973         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 974                      os.F_OK):
 975             if not self.replace_sources:
 976                 print('error: GCC has moved from SVN to git, use '
 977                       '--replace-sources to check out again')
 978                 exit(1)
 979             shutil.rmtree(self.component_srcdir('gcc'))
 980             update = False
 981         if not update:
 982             self.git_checkout('gcc', 'https://gcc.gnu.org/git/gcc.git',
 983                               branch, update)
 984         subprocess.run(['contrib/gcc_update', '--silent'],
 985                        cwd=self.component_srcdir('gcc'), check=True)
 986         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 987                            cwd=self.component_srcdir('gcc'),
 988                            stdout=subprocess.PIPE,
 989                            check=True, universal_newlines=True).stdout
 990         return r.rstrip()
 991
 992     def checkout_tar(self, component, version, update):
 993         """Check out the given version of the given component from a
 994         tarball."""
 995         if update:
 996             return
 997         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 998                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
 999                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
1000                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
1001                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
1002                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
1003                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
1004                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
1005                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
1006         if component not in url_map:
1007             print('error: component %s coming from tarball' % component)
1008             exit(1)
1009         version_major = version.split('.')[0]
1010         url = url_map[component] % {'version': version, 'major': version_major}
1011         filename = os.path.join(self.srcdir, url.split('/')[-1])
1012         response = urllib.request.urlopen(url)
1013         data = response.read()
1014         with open(filename, 'wb') as f:
1015             f.write(data)
1016         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
1017                        check=True)
1018         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
1019                   self.component_srcdir(component))
1020         os.remove(filename)
1021
1022     def load_build_state_json(self):
1023         """Load information about the state of previous builds."""
1024         if os.access(self.build_state_json, os.F_OK):
1025             with open(self.build_state_json, 'r') as f:
1026                 self.build_state = json.load(f)
1027         else:
1028             self.build_state = {}
1029         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
1030             if k not in self.build_state:
1031                 self.build_state[k] = {}
1032             if 'build-time' not in self.build_state[k]:
1033                 self.build_state[k]['build-time'] = ''
1034             if 'build-versions' not in self.build_state[k]:
1035                 self.build_state[k]['build-versions'] = {}
1036             if 'build-results' not in self.build_state[k]:
1037                 self.build_state[k]['build-results'] = {}
1038             if 'result-changes' not in self.build_state[k]:
1039                 self.build_state[k]['result-changes'] = {}
1040             if 'ever-passed' not in self.build_state[k]:
1041                 self.build_state[k]['ever-passed'] = []
1042
1043     def store_build_state_json(self):
1044         """Store information about the state of previous builds."""
1045         self.store_json(self.build_state, self.build_state_json)
1046
1047     def clear_last_build_state(self, action):
1048         """Clear information about the state of part of the build."""
1049         # We clear the last build time and versions when starting a
1050         # new build.  The results of the last build are kept around,
1051         # as comparison is still meaningful if this build is aborted
1052         # and a new one started.
1053         self.build_state[action]['build-time'] = ''
1054         self.build_state[action]['build-versions'] = {}
1055         self.store_build_state_json()
1056
1057     def update_build_state(self, action, build_time, build_versions):
1058         """Update the build state after a build."""
1059         build_time = build_time.replace(microsecond=0)
1060         self.build_state[action]['build-time'] = str(build_time)
1061         self.build_state[action]['build-versions'] = build_versions
1062         build_results = {}
1063         for log in self.status_log_list:
1064             with open(log, 'r') as f:
1065                 log_text = f.read()
1066             log_text = log_text.rstrip()
1067             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1068             result = m.group(1)
1069             test_name = m.group(2)
1070             assert test_name not in build_results
1071             build_results[test_name] = result
1072         old_build_results = self.build_state[action]['build-results']
1073         self.build_state[action]['build-results'] = build_results
1074         result_changes = {}
1075         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1076         for t in all_tests:
1077             if t in old_build_results:
1078                 old_res = old_build_results[t]
1079             else:
1080                 old_res = '(New test)'
1081             if t in build_results:
1082                 new_res = build_results[t]
1083             else:
1084                 new_res = '(Test removed)'
1085             if old_res != new_res:
1086                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1087         self.build_state[action]['result-changes'] = result_changes
1088         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1089                            if t in build_results}
1090         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1091         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1092                                                          new_passes)
1093         self.store_build_state_json()
1094
1095     def load_bot_config_json(self):
1096         """Load bot configuration."""
1097         with open(self.bot_config_json, 'r') as f:
1098             self.bot_config = json.load(f)
1099
1100     def part_build_old(self, action, delay):
1101         """Return whether the last build for a given action was at least a
1102         given number of seconds ago, or does not have a time recorded."""
1103         old_time_str = self.build_state[action]['build-time']
1104         if not old_time_str:
1105             return True
1106         old_time = datetime.datetime.strptime(old_time_str,
1107                                               '%Y-%m-%d %H:%M:%S')
1108         new_time = datetime.datetime.utcnow()
1109         delta = new_time - old_time
1110         return delta.total_seconds() >= delay
1111
1112     def bot_cycle(self):
1113         """Run a single round of checkout and builds."""
1114         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1115         self.load_bot_config_json()
1116         actions = ('host-libraries', 'compilers', 'glibcs')
1117         self.bot_run_self(['--replace-sources'], 'checkout')
1118         self.load_versions_json()
1119         if self.get_script_text() != self.script_text:
1120             print('Script changed, re-execing.')
1121             # On script change, all parts of the build should be rerun.
1122             for a in actions:
1123                 self.clear_last_build_state(a)
1124             self.exec_self()
1125         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1126                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1127                                           'mig', 'gnumach', 'hurd'),
1128                             'glibcs': ('glibc',)}
1129         must_build = {}
1130         for a in actions:
1131             build_vers = self.build_state[a]['build-versions']
1132             must_build[a] = False
1133             if not self.build_state[a]['build-time']:
1134                 must_build[a] = True
1135             old_vers = {}
1136             new_vers = {}
1137             for c in check_components[a]:
1138                 if c in build_vers:
1139                     old_vers[c] = build_vers[c]
1140                 new_vers[c] = {'version': self.versions[c]['version'],
1141                                'revision': self.versions[c]['revision']}
1142             if new_vers == old_vers:
1143                 print('Versions for %s unchanged.' % a)
1144             else:
1145                 print('Versions changed or rebuild forced for %s.' % a)
1146                 if a == 'compilers' and not self.part_build_old(
1147                         a, self.bot_config['compilers-rebuild-delay']):
1148                     print('Not requiring rebuild of compilers this soon.')
1149                 else:
1150                     must_build[a] = True
1151         if must_build['host-libraries']:
1152             must_build['compilers'] = True
1153         if must_build['compilers']:
1154             must_build['glibcs'] = True
1155         for a in actions:
1156             if must_build[a]:
1157                 print('Must rebuild %s.' % a)
1158                 self.clear_last_build_state(a)
1159             else:
1160                 print('No need to rebuild %s.' % a)
1161         if os.access(self.logsdir, os.F_OK):
1162             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1163             shutil.copytree(self.logsdir, self.logsdir_old)
1164         for a in actions:
1165             if must_build[a]:
1166                 build_time = datetime.datetime.utcnow()
1167                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1168                 self.bot_run_self([], a)
1169                 self.load_build_state_json()
1170                 self.bot_build_mail(a, build_time)
1171         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1172
1173     def bot_build_mail(self, action, build_time):
1174         """Send email with the results of a build."""
1175         if not ('email-from' in self.bot_config and
1176                 'email-server' in self.bot_config and
1177                 'email-subject' in self.bot_config and
1178                 'email-to' in self.bot_config):
1179             if not self.email_warning:
1180                 print("Email not configured, not sending.")
1181                 self.email_warning = True
1182             return
1183
1184         build_time = build_time.replace(microsecond=0)
1185         subject = (self.bot_config['email-subject'] %
1186                    {'action': action,
1187                     'build-time': str(build_time)})
1188         results = self.build_state[action]['build-results']
1189         changes = self.build_state[action]['result-changes']
1190         ever_passed = set(self.build_state[action]['ever-passed'])
1191         versions = self.build_state[action]['build-versions']
1192         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1193         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1194         all_fails = {k for k in results if results[k] == 'FAIL'}
1195         if new_regressions:
1196             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1197             new_reg_text = ('New regressions:\n\n%s\n\n' %
1198                             '\n'.join(new_reg_list))
1199         else:
1200             new_reg_text = ''
1201         if all_regressions:
1202             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1203             all_reg_text = ('All regressions:\n\n%s\n\n' %
1204                             '\n'.join(all_reg_list))
1205         else:
1206             all_reg_text = ''
1207         if all_fails:
1208             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1209             all_fail_text = ('All failures:\n\n%s\n\n' %
1210                              '\n'.join(all_fail_list))
1211         else:
1212             all_fail_text = ''
1213         if changes:
1214             changes_list = sorted(changes.keys())
1215             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1216             changes_text = ('All changed results:\n\n%s\n\n' %
1217                             '\n'.join(changes_list))
1218         else:
1219             changes_text = ''
1220         results_text = (new_reg_text + all_reg_text + all_fail_text +
1221                         changes_text)
1222         if not results_text:
1223             results_text = 'Clean build with unchanged results.\n\n'
1224         versions_list = sorted(versions.keys())
1225         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1226                                           versions[k]['revision'])
1227                          for k in versions_list]
1228         versions_text = ('Component versions for this build:\n\n%s\n' %
1229                          '\n'.join(versions_list))
1230         body_text = results_text + versions_text
1231         msg = email.mime.text.MIMEText(body_text)
1232         msg['Subject'] = subject
1233         msg['From'] = self.bot_config['email-from']
1234         msg['To'] = self.bot_config['email-to']
1235         msg['Message-ID'] = email.utils.make_msgid()
1236         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1237         with smtplib.SMTP(self.bot_config['email-server']) as s:
1238             s.send_message(msg)
1239
1240     def bot_run_self(self, opts, action, check=True):
1241         """Run a copy of this script with given options."""
1242         cmd = [sys.executable, sys.argv[0], '--keep=none',
1243                '-j%d' % self.parallelism]
1244         if self.full_gcc:
1245             cmd.append('--full-gcc')
1246         cmd.extend(opts)
1247         cmd.extend([self.topdir, action])
1248         sys.stdout.flush()
1249         subprocess.run(cmd, check=check)
1250
1251     def bot(self):
1252         """Run repeated rounds of checkout and builds."""
1253         while True:
1254             self.load_bot_config_json()
1255             if not self.bot_config['run']:
1256                 print('Bot exiting by request.')
1257                 exit(0)
1258             self.bot_run_self([], 'bot-cycle', check=False)
1259             self.load_bot_config_json()
1260             if not self.bot_config['run']:
1261                 print('Bot exiting by request.')
1262                 exit(0)
1263             time.sleep(self.bot_config['delay'])
1264             if self.get_script_text() != self.script_text:
1265                 print('Script changed, bot re-execing.')
1266                 self.exec_self()
1267
1268 class LinuxHeadersPolicyForBuild(object):
1269     """Names and directories for installing Linux headers.  Build variant."""
1270
1271     def __init__(self, config):
1272         self.arch = config.arch
1273         self.srcdir = config.ctx.component_srcdir('linux')
1274         self.builddir = config.component_builddir('linux')
1275         self.headers_dir = os.path.join(config.sysroot, 'usr')
1276
1277 class LinuxHeadersPolicyForUpdateSyscalls(object):
1278     """Names and directories for Linux headers.  update-syscalls variant."""
1279
1280     def __init__(self, glibc, headers_dir):
1281         self.arch = glibc.compiler.arch
1282         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1283         self.builddir = glibc.ctx.component_builddir(
1284             'update-syscalls', glibc.name, 'build-linux')
1285         self.headers_dir = headers_dir
1286
1287 def install_linux_headers(policy, cmdlist):
1288     """Install Linux kernel headers."""
1289     arch_map = {'aarch64': 'arm64',
1290                 'alpha': 'alpha',
1291                 'arc': 'arc',
1292                 'arm': 'arm',
1293                 'csky': 'csky',
1294                 'hppa': 'parisc',
1295                 'i486': 'x86',
1296                 'i586': 'x86',
1297                 'i686': 'x86',
1298                 'i786': 'x86',
1299                 'loongarch64': 'loongarch',
1300                 'm68k': 'm68k',
1301                 'microblaze': 'microblaze',
1302                 'mips': 'mips',
1303                 'nios2': 'nios2',
1304                 'or1k': 'openrisc',
1305                 'powerpc': 'powerpc',
1306                 's390': 's390',
1307                 'riscv32': 'riscv',
1308                 'riscv64': 'riscv',
1309                 'sh': 'sh',
1310                 'sparc': 'sparc',
1311                 'x86_64': 'x86'}
1312     linux_arch = None
1313     for k in arch_map:
1314         if policy.arch.startswith(k):
1315             linux_arch = arch_map[k]
1316             break
1317     assert linux_arch is not None
1318     cmdlist.push_subdesc('linux')
1319     cmdlist.create_use_dir(policy.builddir)
1320     cmdlist.add_command('install-headers',
1321                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1322                          'ARCH=%s' % linux_arch,
1323                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1324                          'headers_install'])
1325     cmdlist.cleanup_dir()
1326     cmdlist.pop_subdesc()
1327
1328 class Config(object):
1329     """A configuration for building a compiler and associated libraries."""
1330
1331     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1332                  first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1333                  extra_glibcs=None):
1334         """Initialize a Config object."""
1335         self.ctx = ctx
1336         self.arch = arch
1337         self.os = os_name
1338         self.variant = variant
1339         if variant is None:
1340             self.name = '%s-%s' % (arch, os_name)
1341         else:
1342             self.name = '%s-%s-%s' % (arch, os_name, variant)
1343         self.triplet = '%s-glibc-%s' % (arch, os_name)
1344         if gcc_cfg is None:
1345             self.gcc_cfg = []
1346         else:
1347             self.gcc_cfg = gcc_cfg
1348         if first_gcc_cfg is None:
1349             self.first_gcc_cfg = []
1350         else:
1351             self.first_gcc_cfg = first_gcc_cfg
1352         if binutils_cfg is None:
1353             self.binutils_cfg = []
1354         else:
1355             self.binutils_cfg = binutils_cfg
1356         if glibcs is None:
1357             glibcs = [{'variant': variant}]
1358         if extra_glibcs is None:
1359             extra_glibcs = []
1360         glibcs = [Glibc(self, **g) for g in glibcs]
1361         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1362         self.all_glibcs = glibcs + extra_glibcs
1363         self.compiler_glibcs = glibcs
1364         self.installdir = ctx.compiler_installdir(self.name)
1365         self.bindir = ctx.compiler_bindir(self.name)
1366         self.sysroot = ctx.compiler_sysroot(self.name)
1367         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1368         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1369
1370     def component_builddir(self, component):
1371         """Return the directory to use for a (non-glibc) build."""
1372         return self.ctx.component_builddir('compilers', self.name, component)
1373
1374     def build(self):
1375         """Generate commands to build this compiler."""
1376         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1377                                       self.logsdir)
1378         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1379         cmdlist.add_command('check-host-libraries',
1380                             ['test', '-f',
1381                              os.path.join(self.ctx.host_libraries_installdir,
1382                                           'ok')])
1383         cmdlist.use_path(self.bindir)
1384         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1385                               ['--disable-gdb',
1386                                '--disable-gdbserver',
1387                                '--disable-libdecnumber',
1388                                '--disable-readline',
1389                                '--disable-sim'] + self.binutils_cfg)
1390         if self.os.startswith('linux'):
1391             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1392         self.build_gcc(cmdlist, True)
1393         if self.os == 'gnu':
1394             self.install_gnumach_headers(cmdlist)
1395             self.build_cross_tool(cmdlist, 'mig', 'mig')
1396             self.install_hurd_headers(cmdlist)
1397         for g in self.compiler_glibcs:
1398             cmdlist.push_subdesc('glibc')
1399             cmdlist.push_subdesc(g.name)
1400             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1401             cmdlist.pop_subdesc()
1402             cmdlist.pop_subdesc()
1403         self.build_gcc(cmdlist, False)
1404         cmdlist.add_command('done', ['touch',
1405                                      os.path.join(self.installdir, 'ok')])
1406         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1407                                       self.logsdir)
1408
1409     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1410         """Build one cross tool."""
1411         srcdir = self.ctx.component_srcdir(tool_src)
1412         builddir = self.component_builddir(tool_build)
1413         cmdlist.push_subdesc(tool_build)
1414         cmdlist.create_use_dir(builddir)
1415         cfg_cmd = [os.path.join(srcdir, 'configure'),
1416                    '--prefix=%s' % self.installdir,
1417                    '--build=%s' % self.ctx.build_triplet,
1418                    '--host=%s' % self.ctx.build_triplet,
1419                    '--target=%s' % self.triplet,
1420                    '--with-sysroot=%s' % self.sysroot]
1421         if extra_opts:
1422             cfg_cmd.extend(extra_opts)
1423         cmdlist.add_command('configure', cfg_cmd)
1424         cmdlist.add_command('build', ['make'])
1425         # Parallel "make install" for GCC has race conditions that can
1426         # cause it to fail; see
1427         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1428         # problems are not known for binutils, but doing the
1429         # installation in parallel within a particular toolchain build
1430         # (as opposed to installation of one toolchain from
1431         # build-many-glibcs.py running in parallel to the installation
1432         # of other toolchains being built) is not known to be
1433         # significantly beneficial, so it is simplest just to disable
1434         # parallel install for cross tools here.
1435         cmdlist.add_command('install', ['make', '-j1', 'install'])
1436         cmdlist.cleanup_dir()
1437         cmdlist.pop_subdesc()
1438
1439     def install_gnumach_headers(self, cmdlist):
1440         """Install GNU Mach headers."""
1441         srcdir = self.ctx.component_srcdir('gnumach')
1442         builddir = self.component_builddir('gnumach')
1443         cmdlist.push_subdesc('gnumach')
1444         cmdlist.create_use_dir(builddir)
1445         cmdlist.add_command('configure',
1446                             [os.path.join(srcdir, 'configure'),
1447                              '--build=%s' % self.ctx.build_triplet,
1448                              '--host=%s' % self.triplet,
1449                              '--prefix=',
1450                              '--disable-user32',
1451                              'CC=%s-gcc -nostdlib' % self.triplet])
1452         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1453                                         'install-data'])
1454         cmdlist.cleanup_dir()
1455         cmdlist.pop_subdesc()
1456
1457     def install_hurd_headers(self, cmdlist):
1458         """Install Hurd headers."""
1459         srcdir = self.ctx.component_srcdir('hurd')
1460         builddir = self.component_builddir('hurd')
1461         cmdlist.push_subdesc('hurd')
1462         cmdlist.create_use_dir(builddir)
1463         cmdlist.add_command('configure',
1464                             [os.path.join(srcdir, 'configure'),
1465                              '--build=%s' % self.ctx.build_triplet,
1466                              '--host=%s' % self.triplet,
1467                              '--prefix=',
1468                              '--disable-profile', '--without-parted',
1469                              'CC=%s-gcc -nostdlib' % self.triplet])
1470         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1471                                         'no_deps=t', 'install-headers'])
1472         cmdlist.cleanup_dir()
1473         cmdlist.pop_subdesc()
1474
1475     def build_gcc(self, cmdlist, bootstrap):
1476         """Build GCC."""
1477         # libssp is of little relevance with glibc's own stack
1478         # checking support.  libcilkrts does not support GNU/Hurd (and
1479         # has been removed in GCC 8, so --disable-libcilkrts can be
1480         # removed once glibc no longer supports building with older
1481         # GCC versions).  --enable-initfini-array is enabled by default
1482         # in GCC 12, which can be removed when GCC 12 becomes the
1483         # minimum requirement.
1484         cfg_opts = list(self.gcc_cfg)
1485         cfg_opts += ['--enable-initfini-array']
1486         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1487         host_libs = self.ctx.host_libraries_installdir
1488         cfg_opts += ['--with-gmp=%s' % host_libs,
1489                      '--with-mpfr=%s' % host_libs,
1490                      '--with-mpc=%s' % host_libs]
1491         if bootstrap:
1492             tool_build = 'gcc-first'
1493             # Building a static-only, C-only compiler that is
1494             # sufficient to build glibc.  Various libraries and
1495             # features that may require libc headers must be disabled.
1496             # When configuring with a sysroot, --with-newlib is
1497             # required to define inhibit_libc (to stop some parts of
1498             # libgcc including libc headers); --without-headers is not
1499             # sufficient.
1500             cfg_opts += ['--enable-languages=c', '--disable-shared',
1501                          '--disable-threads',
1502                          '--disable-libatomic',
1503                          '--disable-decimal-float',
1504                          '--disable-gcov',
1505                          '--disable-libffi',
1506                          '--disable-libgomp',
1507                          '--disable-libitm',
1508                          '--disable-libmpx',
1509                          '--disable-libquadmath',
1510                          '--disable-libsanitizer',
1511                          '--without-headers', '--with-newlib',
1512                          '--with-glibc-version=%s' % self.ctx.glibc_version
1513                          ]
1514             cfg_opts += self.first_gcc_cfg
1515         else:
1516             tool_build = 'gcc'
1517             # libsanitizer commonly breaks because of glibc header
1518             # changes, or on unusual targets.  C++ pre-compiled
1519             # headers are not used during the glibc build and are
1520             # expensive to create.
1521             if not self.ctx.full_gcc:
1522                 cfg_opts += ['--disable-libsanitizer',
1523                              '--disable-libstdcxx-pch']
1524             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1525             cfg_opts += ['--enable-languages=%s' % langs,
1526                          '--enable-shared', '--enable-threads']
1527         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1528
1529 class GlibcPolicyDefault(object):
1530     """Build policy for glibc: common defaults."""
1531
1532     def __init__(self, glibc):
1533         self.srcdir = glibc.ctx.component_srcdir('glibc')
1534         self.use_usr = glibc.os != 'gnu'
1535         self.prefix = '/usr' if self.use_usr else ''
1536         self.configure_args = [
1537             '--prefix=%s' % self.prefix,
1538             '--enable-profile',
1539             '--build=%s' % glibc.ctx.build_triplet,
1540             '--host=%s' % glibc.triplet,
1541             'CC=%s' % glibc.tool_name('gcc'),
1542             'CXX=%s' % glibc.tool_name('g++'),
1543         ]
1544         if glibc.os == 'gnu':
1545             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1546         if glibc.cflags:
1547             self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1548             self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
1549         self.configure_args += glibc.cfg
1550
1551     def configure(self, cmdlist):
1552         """Invoked to add the configure command to the command list."""
1553         cmdlist.add_command('configure',
1554                             [os.path.join(self.srcdir, 'configure'),
1555                              *self.configure_args])
1556
1557     def extra_commands(self, cmdlist):
1558         """Invoked to inject additional commands (make check) after build."""
1559         pass
1560
1561 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1562     """Build policy for glibc during the compilers stage."""
1563
1564     def __init__(self, glibc):
1565         super().__init__(glibc)
1566         self.builddir = glibc.ctx.component_builddir(
1567             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1568         self.installdir = glibc.compiler.sysroot
1569
1570 class GlibcPolicyForBuild(GlibcPolicyDefault):
1571     """Build policy for glibc during the glibcs stage."""
1572
1573     def __init__(self, glibc):
1574         super().__init__(glibc)
1575         self.builddir = glibc.ctx.component_builddir(
1576             'glibcs', glibc.name, 'glibc')
1577         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1578         if glibc.ctx.strip:
1579             self.strip = glibc.tool_name('strip')
1580         else:
1581             self.strip = None
1582         self.save_logs = glibc.ctx.save_logs
1583
1584     def extra_commands(self, cmdlist):
1585         if self.strip:
1586             # Avoid stripping libc.so and libpthread.so, which are
1587             # linker scripts stored in /lib on Hurd.
1588             find_command = 'find %s/lib* -name "*.so*"' % self.installdir
1589             cmdlist.add_command('strip', ['sh', '-c', (
1590                 'set -e; for f in $(%s); do '
1591                 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1592                 'done' % (find_command, self.strip))])
1593         cmdlist.add_command('check', ['make', 'check'])
1594         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1595
1596 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1597     """Build policy for glibc during update-syscalls."""
1598
1599     def __init__(self, glibc):
1600         super().__init__(glibc)
1601         self.builddir = glibc.ctx.component_builddir(
1602             'update-syscalls', glibc.name, 'glibc')
1603         self.linuxdir = glibc.ctx.component_builddir(
1604             'update-syscalls', glibc.name, 'linux')
1605         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1606             glibc, self.linuxdir)
1607         self.configure_args.insert(
1608             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1609         # self.installdir not set because installation is not supported
1610
1611 class Glibc(object):
1612     """A configuration for building glibc."""
1613
1614     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1615                  cfg=None, ccopts=None, cflags=None):
1616         """Initialize a Glibc object."""
1617         self.ctx = compiler.ctx
1618         self.compiler = compiler
1619         if arch is None:
1620             self.arch = compiler.arch
1621         else:
1622             self.arch = arch
1623         if os_name is None:
1624             self.os = compiler.os
1625         else:
1626             self.os = os_name
1627         self.variant = variant
1628         if variant is None:
1629             self.name = '%s-%s' % (self.arch, self.os)
1630         else:
1631             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1632         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1633         if cfg is None:
1634             self.cfg = []
1635         else:
1636             self.cfg = cfg
1637         # ccopts contain ABI options and are passed to configure as CC / CXX.
1638         self.ccopts = ccopts
1639         # cflags contain non-ABI options like -g or -O and are passed to
1640         # configure as CFLAGS / CXXFLAGS.
1641         self.cflags = cflags
1642
1643     def tool_name(self, tool):
1644         """Return the name of a cross-compilation tool."""
1645         ctool = '%s-%s' % (self.compiler.triplet, tool)
1646         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1647             ctool = '%s %s' % (ctool, self.ccopts)
1648         return ctool
1649
1650     def build(self):
1651         """Generate commands to build this glibc."""
1652         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1653         installdir = self.ctx.glibc_installdir(self.name)
1654         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1655         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1656         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1657         cmdlist.add_command('check-compilers',
1658                             ['test', '-f',
1659                              os.path.join(self.compiler.installdir, 'ok')])
1660         cmdlist.use_path(self.compiler.bindir)
1661         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1662         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1663                                       logsdir)
1664
1665     def build_glibc(self, cmdlist, policy):
1666         """Generate commands to build this glibc, either as part of a compiler
1667         build or with the bootstrapped compiler (and in the latter case, run
1668         tests as well)."""
1669         cmdlist.create_use_dir(policy.builddir)
1670         policy.configure(cmdlist)
1671         cmdlist.add_command('build', ['make'])
1672         cmdlist.add_command('install', ['make', 'install',
1673                                         'install_root=%s' % policy.installdir])
1674         # GCC uses paths such as lib/../lib64, so make sure lib
1675         # directories always exist.
1676         mkdir_cmd = ['mkdir', '-p',
1677                      os.path.join(policy.installdir, 'lib')]
1678         if policy.use_usr:
1679             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1680         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1681         policy.extra_commands(cmdlist)
1682         cmdlist.cleanup_dir()
1683
1684     def update_syscalls(self):
1685         if self.os == 'gnu':
1686             # Hurd does not have system call tables that need updating.
1687             return
1688
1689         policy = GlibcPolicyForUpdateSyscalls(self)
1690         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1691         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1692         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1693         cmdlist.add_command('check-compilers',
1694                             ['test', '-f',
1695                              os.path.join(self.compiler.installdir, 'ok')])
1696         cmdlist.use_path(self.compiler.bindir)
1697
1698         install_linux_headers(policy.linux_policy, cmdlist)
1699
1700         cmdlist.create_use_dir(policy.builddir)
1701         policy.configure(cmdlist)
1702         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1703         cmdlist.cleanup_dir()
1704         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1705                                       cmdlist, logsdir)
1706
1707 class Command(object):
1708     """A command run in the build process."""
1709
1710     def __init__(self, desc, num, dir, path, command, always_run=False):
1711         """Initialize a Command object."""
1712         self.dir = dir
1713         self.path = path
1714         self.desc = desc
1715         trans = str.maketrans({' ': '-'})
1716         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1717         self.command = command
1718         self.always_run = always_run
1719
1720     @staticmethod
1721     def shell_make_quote_string(s):
1722         """Given a string not containing a newline, quote it for use by the
1723         shell and make."""
1724         assert '\n' not in s
1725         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1726             return s
1727         strans = str.maketrans({"'": "'\\''"})
1728         s = "'%s'" % s.translate(strans)
1729         mtrans = str.maketrans({'$': '$$'})
1730         return s.translate(mtrans)
1731
1732     @staticmethod
1733     def shell_make_quote_list(l, translate_make):
1734         """Given a list of strings not containing newlines, quote them for use
1735         by the shell and make, returning a single string.  If translate_make
1736         is true and the first string is 'make', change it to $(MAKE)."""
1737         l = [Command.shell_make_quote_string(s) for s in l]
1738         if translate_make and l[0] == 'make':
1739             l[0] = '$(MAKE)'
1740         return ' '.join(l)
1741
1742     def shell_make_quote(self):
1743         """Return this command quoted for the shell and make."""
1744         return self.shell_make_quote_list(self.command, True)
1745
1746
1747 class CommandList(object):
1748     """A list of commands run in the build process."""
1749
1750     def __init__(self, desc, keep):
1751         """Initialize a CommandList object."""
1752         self.cmdlist = []
1753         self.dir = None
1754         self.path = None
1755         self.desc = [desc]
1756         self.keep = keep
1757
1758     def desc_txt(self, desc):
1759         """Return the description to use for a command."""
1760         return '%s %s' % (' '.join(self.desc), desc)
1761
1762     def use_dir(self, dir):
1763         """Set the default directory for subsequent commands."""
1764         self.dir = dir
1765
1766     def use_path(self, path):
1767         """Set a directory to be prepended to the PATH for subsequent
1768         commands."""
1769         self.path = path
1770
1771     def push_subdesc(self, subdesc):
1772         """Set the default subdescription for subsequent commands (e.g., the
1773         name of a component being built, within the series of commands
1774         building it)."""
1775         self.desc.append(subdesc)
1776
1777     def pop_subdesc(self):
1778         """Pop a subdescription from the list of descriptions."""
1779         self.desc.pop()
1780
1781     def create_use_dir(self, dir):
1782         """Remove and recreate a directory and use it for subsequent
1783         commands."""
1784         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1785         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1786         self.use_dir(dir)
1787
1788     def add_command_dir(self, desc, dir, command, always_run=False):
1789         """Add a command to run in a given directory."""
1790         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1791                       command, always_run)
1792         self.cmdlist.append(cmd)
1793
1794     def add_command(self, desc, command, always_run=False):
1795         """Add a command to run in the default directory."""
1796         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1797                       self.path, command, always_run)
1798         self.cmdlist.append(cmd)
1799
1800     def cleanup_dir(self, desc='cleanup', dir=None):
1801         """Clean up a build directory.  If no directory is specified, the
1802         default directory is cleaned up and ceases to be the default
1803         directory."""
1804         if dir is None:
1805             dir = self.dir
1806             self.use_dir(None)
1807         if self.keep != 'all':
1808             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1809                                  always_run=(self.keep == 'none'))
1810
1811     def makefile_commands(self, wrapper, logsdir):
1812         """Return the sequence of commands in the form of text for a Makefile.
1813         The given wrapper script takes arguments: base of logs for
1814         previous command, or empty; base of logs for this command;
1815         description; directory; PATH addition; the command itself."""
1816         # prev_base is the base of the name for logs of the previous
1817         # command that is not always-run (that is, a build command,
1818         # whose failure should stop subsequent build commands from
1819         # being run, as opposed to a cleanup command, which is run
1820         # even if previous commands failed).
1821         prev_base = ''
1822         cmds = []
1823         for c in self.cmdlist:
1824             ctxt = c.shell_make_quote()
1825             if prev_base and not c.always_run:
1826                 prev_log = os.path.join(logsdir, prev_base)
1827             else:
1828                 prev_log = ''
1829             this_log = os.path.join(logsdir, c.logbase)
1830             if not c.always_run:
1831                 prev_base = c.logbase
1832             if c.dir is None:
1833                 dir = ''
1834             else:
1835                 dir = c.dir
1836             if c.path is None:
1837                 path = ''
1838             else:
1839                 path = c.path
1840             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1841             prelim_txt = Command.shell_make_quote_list(prelims, False)
1842             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1843         return '\n'.join(cmds)
1844
1845     def status_logs(self, logsdir):
1846         """Return the list of log files with command status."""
1847         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1848                 for c in self.cmdlist]
1849
1850
1851 def get_parser():
1852     """Return an argument parser for this module."""
1853     parser = argparse.ArgumentParser(description=__doc__)
1854     parser.add_argument('-j', dest='parallelism',
1855                         help='Run this number of jobs in parallel',
1856                         type=int, default=os.cpu_count())
1857     parser.add_argument('--keep', dest='keep',
1858                         help='Whether to keep all build directories, '
1859                         'none or only those from failed builds',
1860                         default='none', choices=('none', 'all', 'failed'))
1861     parser.add_argument('--replace-sources', action='store_true',
1862                         help='Remove and replace source directories '
1863                         'with the wrong version of a component')
1864     parser.add_argument('--strip', action='store_true',
1865                         help='Strip installed glibc libraries')
1866     parser.add_argument('--full-gcc', action='store_true',
1867                         help='Build GCC with all languages and libsanitizer')
1868     parser.add_argument('--shallow', action='store_true',
1869                         help='Do not download Git history during checkout')
1870     parser.add_argument('topdir',
1871                         help='Toplevel working directory')
1872     parser.add_argument('action',
1873                         help='What to do',
1874                         choices=('checkout', 'bot-cycle', 'bot',
1875                                  'host-libraries', 'compilers', 'glibcs',
1876                                  'update-syscalls', 'list-compilers',
1877                                  'list-glibcs'))
1878     parser.add_argument('configs',
1879                         help='Versions to check out or configurations to build',
1880                         nargs='*')
1881     return parser
1882
1883
1884 def get_version_common(progname,line,word,delchars,arg1):
1885     try:
1886         out = subprocess.run([progname, arg1],
1887                              stdout=subprocess.PIPE,
1888                              stderr=subprocess.DEVNULL,
1889                              stdin=subprocess.DEVNULL,
1890                              check=True, universal_newlines=True)
1891         v = out.stdout.splitlines()[line].split()[word]
1892         if delchars:
1893             v = v.replace(delchars,'')
1894         return [int(x) for x in v.split('.')]
1895     except:
1896         return 'missing';
1897
1898 def get_version_common_stderr(progname,line,word,delchars,arg1):
1899     try:
1900         out = subprocess.run([progname, arg1],
1901                              stdout=subprocess.DEVNULL,
1902                              stderr=subprocess.PIPE,
1903                              stdin=subprocess.DEVNULL,
1904                              check=True, universal_newlines=True)
1905         v = out.stderr.splitlines()[line].split()[word]
1906         if delchars:
1907             v = v.replace(delchars,'')
1908         return [int(x) for x in v.split('.')]
1909     except:
1910         return 'missing';
1911
1912 def get_version(progname):
1913     return get_version_common (progname, 0, -1, None, '--version');
1914
1915 def get_version_awk(progname):
1916     return get_version_common (progname, 0, 2, ',', '--version');
1917
1918 def get_version_bzip2(progname):
1919     return get_version_common_stderr (progname, 0, 6, ',', '-h');
1920
1921 def check_version(ver, req):
1922     for v, r in zip(ver, req):
1923         if v > r:
1924             return True
1925         if v < r:
1926             return False
1927     return True
1928
1929 def version_str(ver):
1930     return '.'.join([str (x) for x in ver])
1931
1932 def check_for_required_tools():
1933     get_list_of_required_tools()
1934     count_old_tools = 0
1935     count_missing_tools = 0
1936
1937     for k, v in REQUIRED_TOOLS.items():
1938         version = v[0](k)
1939         if version == 'missing':
1940             ok = 'missing'
1941         else:
1942             ok = 'ok' if check_version (version, v[1]) else 'old'
1943         if ok == 'old':
1944             if count_old_tools == 0:
1945                 print("One or more required tools are too old:")
1946             count_old_tools = count_old_tools + 1
1947             print('{:9}: {:3} (obtained=\"{}\" required=\"{}\")'.format(k, ok,
1948                     version_str(version), version_str(v[1])))
1949         if ok == 'missing':
1950             if count_missing_tools == 0:
1951                 print("One or more required tools are missing:")
1952             count_missing_tools = count_missing_tools + 1
1953             print('{:9}: {:3} (required=\"{}\")'.format(k, ok,
1954                     version_str(v[1])))
1955
1956     if count_old_tools > 0 or count_missing_tools > 0:
1957         exit (1);
1958
1959 def main(argv):
1960     """The main entry point."""
1961     check_for_required_tools();
1962     parser = get_parser()
1963     opts = parser.parse_args(argv)
1964     topdir = os.path.abspath(opts.topdir)
1965     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1966                   opts.strip, opts.full_gcc, opts.action,
1967                   shallow=opts.shallow)
1968     ctx.run_builds(opts.action, opts.configs)
1969
1970
1971 if __name__ == '__main__':
1972     main(sys.argv[1:])