scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2021 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <https://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 The 'list-compilers' command prints the name of each available
  36 compiler configuration, without building anything.  The 'list-glibcs'
  37 command prints the name of each glibc compiler configuration, followed
  38 by the space, followed by the name of the compiler configuration used
  39 for building this glibc variant.
  40
  41 """
  42
  43 import argparse
  44 import datetime
  45 import email.mime.text
  46 import email.utils
  47 import json
  48 import os
  49 import re
  50 import shutil
  51 import smtplib
  52 import stat
  53 import subprocess
  54 import sys
  55 import time
  56 import urllib.request
  57
  58 try:
  59     subprocess.run
  60 except:
  61     class _CompletedProcess:
  62         def __init__(self, args, returncode, stdout=None, stderr=None):
  63             self.args = args
  64             self.returncode = returncode
  65             self.stdout = stdout
  66             self.stderr = stderr
  67
  68     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  69         assert(timeout is None)
  70         with subprocess.Popen(*popenargs, **kwargs) as process:
  71             try:
  72                 stdout, stderr = process.communicate(input)
  73             except:
  74                 process.kill()
  75                 process.wait()
  76                 raise
  77             returncode = process.poll()
  78             if check and returncode:
  79                 raise subprocess.CalledProcessError(returncode, popenargs)
  80         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  81
  82     subprocess.run = _run
  83
  84
  85 class Context(object):
  86     """The global state associated with builds in a given directory."""
  87
  88     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  89                  full_gcc, action, shallow=False):
  90         """Initialize the context."""
  91         self.topdir = topdir
  92         self.parallelism = parallelism
  93         self.keep = keep
  94         self.replace_sources = replace_sources
  95         self.strip = strip
  96         self.full_gcc = full_gcc
  97         self.shallow = shallow
  98         self.srcdir = os.path.join(topdir, 'src')
  99         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 100         self.build_state_json = os.path.join(topdir, 'build-state.json')
 101         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 102         self.installdir = os.path.join(topdir, 'install')
 103         self.host_libraries_installdir = os.path.join(self.installdir,
 104                                                       'host-libraries')
 105         self.builddir = os.path.join(topdir, 'build')
 106         self.logsdir = os.path.join(topdir, 'logs')
 107         self.logsdir_old = os.path.join(topdir, 'logs-old')
 108         self.makefile = os.path.join(self.builddir, 'Makefile')
 109         self.wrapper = os.path.join(self.builddir, 'wrapper')
 110         self.save_logs = os.path.join(self.builddir, 'save-logs')
 111         self.script_text = self.get_script_text()
 112         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 113             self.build_triplet = self.get_build_triplet()
 114             self.glibc_version = self.get_glibc_version()
 115         self.configs = {}
 116         self.glibc_configs = {}
 117         self.makefile_pieces = ['.PHONY: all\n']
 118         self.add_all_configs()
 119         self.load_versions_json()
 120         self.load_build_state_json()
 121         self.status_log_list = []
 122         self.email_warning = False
 123
 124     def get_script_text(self):
 125         """Return the text of this script."""
 126         with open(sys.argv[0], 'r') as f:
 127             return f.read()
 128
 129     def exec_self(self):
 130         """Re-execute this script with the same arguments."""
 131         sys.stdout.flush()
 132         os.execv(sys.executable, [sys.executable] + sys.argv)
 133
 134     def get_build_triplet(self):
 135         """Determine the build triplet with config.guess."""
 136         config_guess = os.path.join(self.component_srcdir('gcc'),
 137                                     'config.guess')
 138         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 139                                 check=True, universal_newlines=True).stdout
 140         return cg_out.rstrip()
 141
 142     def get_glibc_version(self):
 143         """Determine the glibc version number (major.minor)."""
 144         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 145         with open(version_h, 'r') as f:
 146             lines = f.readlines()
 147         starttext = '#define VERSION "'
 148         for l in lines:
 149             if l.startswith(starttext):
 150                 l = l[len(starttext):]
 151                 l = l.rstrip('"\n')
 152                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 153                 return '%s.%s' % m.group(1, 2)
 154         print('error: could not determine glibc version')
 155         exit(1)
 156
 157     def add_all_configs(self):
 158         """Add all known glibc build configurations."""
 159         self.add_config(arch='aarch64',
 160                         os_name='linux-gnu',
 161                         extra_glibcs=[{'variant': 'disable-multi-arch',
 162                                        'cfg': ['--disable-multi-arch']}])
 163         self.add_config(arch='aarch64_be',
 164                         os_name='linux-gnu')
 165         self.add_config(arch='arc',
 166                         os_name='linux-gnu',
 167                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 168         self.add_config(arch='arc',
 169                         os_name='linux-gnuhf',
 170                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
 171         self.add_config(arch='arceb',
 172                         os_name='linux-gnu',
 173                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 174         self.add_config(arch='alpha',
 175                         os_name='linux-gnu')
 176         self.add_config(arch='arm',
 177                         os_name='linux-gnueabi',
 178                         extra_glibcs=[{'variant': 'v4t',
 179                                        'ccopts': '-march=armv4t'}])
 180         self.add_config(arch='armeb',
 181                         os_name='linux-gnueabi')
 182         self.add_config(arch='armeb',
 183                         os_name='linux-gnueabi',
 184                         variant='be8',
 185                         gcc_cfg=['--with-arch=armv7-a'])
 186         self.add_config(arch='arm',
 187                         os_name='linux-gnueabihf',
 188                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 189                         extra_glibcs=[{'variant': 'v7a',
 190                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 191                                       {'variant': 'thumb',
 192                                        'ccopts':
 193                                        '-mthumb -march=armv7-a -mfpu=vfpv3'},
 194                                       {'variant': 'v7a-disable-multi-arch',
 195                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 196                                        'cfg': ['--disable-multi-arch']}])
 197         self.add_config(arch='armeb',
 198                         os_name='linux-gnueabihf',
 199                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 200         self.add_config(arch='armeb',
 201                         os_name='linux-gnueabihf',
 202                         variant='be8',
 203                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 204                                  '--with-fpu=vfpv3'])
 205         self.add_config(arch='csky',
 206                         os_name='linux-gnuabiv2',
 207                         variant='soft',
 208                         gcc_cfg=['--disable-multilib'])
 209         self.add_config(arch='csky',
 210                         os_name='linux-gnuabiv2',
 211                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 212         self.add_config(arch='hppa',
 213                         os_name='linux-gnu')
 214         self.add_config(arch='i686',
 215                         os_name='gnu')
 216         self.add_config(arch='ia64',
 217                         os_name='linux-gnu',
 218                         first_gcc_cfg=['--with-system-libunwind'],
 219                         binutils_cfg=['--enable-obsolete'])
 220         self.add_config(arch='m68k',
 221                         os_name='linux-gnu',
 222                         gcc_cfg=['--disable-multilib'])
 223         self.add_config(arch='m68k',
 224                         os_name='linux-gnu',
 225                         variant='coldfire',
 226                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 227         self.add_config(arch='m68k',
 228                         os_name='linux-gnu',
 229                         variant='coldfire-soft',
 230                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 231                                  '--disable-multilib'])
 232         self.add_config(arch='microblaze',
 233                         os_name='linux-gnu',
 234                         gcc_cfg=['--disable-multilib'])
 235         self.add_config(arch='microblazeel',
 236                         os_name='linux-gnu',
 237                         gcc_cfg=['--disable-multilib'])
 238         self.add_config(arch='mips64',
 239                         os_name='linux-gnu',
 240                         gcc_cfg=['--with-mips-plt'],
 241                         glibcs=[{'variant': 'n32'},
 242                                 {'arch': 'mips',
 243                                  'ccopts': '-mabi=32'},
 244                                 {'variant': 'n64',
 245                                  'ccopts': '-mabi=64'}])
 246         self.add_config(arch='mips64',
 247                         os_name='linux-gnu',
 248                         variant='soft',
 249                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 250                         glibcs=[{'variant': 'n32-soft'},
 251                                 {'variant': 'soft',
 252                                  'arch': 'mips',
 253                                  'ccopts': '-mabi=32'},
 254                                 {'variant': 'n64-soft',
 255                                  'ccopts': '-mabi=64'}])
 256         self.add_config(arch='mips64',
 257                         os_name='linux-gnu',
 258                         variant='nan2008',
 259                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 260                                  '--with-arch-64=mips64r2',
 261                                  '--with-arch-32=mips32r2'],
 262                         glibcs=[{'variant': 'n32-nan2008'},
 263                                 {'variant': 'nan2008',
 264                                  'arch': 'mips',
 265                                  'ccopts': '-mabi=32'},
 266                                 {'variant': 'n64-nan2008',
 267                                  'ccopts': '-mabi=64'}])
 268         self.add_config(arch='mips64',
 269                         os_name='linux-gnu',
 270                         variant='nan2008-soft',
 271                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 272                                  '--with-arch-64=mips64r2',
 273                                  '--with-arch-32=mips32r2',
 274                                  '--with-float=soft'],
 275                         glibcs=[{'variant': 'n32-nan2008-soft'},
 276                                 {'variant': 'nan2008-soft',
 277                                  'arch': 'mips',
 278                                  'ccopts': '-mabi=32'},
 279                                 {'variant': 'n64-nan2008-soft',
 280                                  'ccopts': '-mabi=64'}])
 281         self.add_config(arch='mips64el',
 282                         os_name='linux-gnu',
 283                         gcc_cfg=['--with-mips-plt'],
 284                         glibcs=[{'variant': 'n32'},
 285                                 {'arch': 'mipsel',
 286                                  'ccopts': '-mabi=32'},
 287                                 {'variant': 'n64',
 288                                  'ccopts': '-mabi=64'}])
 289         self.add_config(arch='mips64el',
 290                         os_name='linux-gnu',
 291                         variant='soft',
 292                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 293                         glibcs=[{'variant': 'n32-soft'},
 294                                 {'variant': 'soft',
 295                                  'arch': 'mipsel',
 296                                  'ccopts': '-mabi=32'},
 297                                 {'variant': 'n64-soft',
 298                                  'ccopts': '-mabi=64'}])
 299         self.add_config(arch='mips64el',
 300                         os_name='linux-gnu',
 301                         variant='nan2008',
 302                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 303                                  '--with-arch-64=mips64r2',
 304                                  '--with-arch-32=mips32r2'],
 305                         glibcs=[{'variant': 'n32-nan2008'},
 306                                 {'variant': 'nan2008',
 307                                  'arch': 'mipsel',
 308                                  'ccopts': '-mabi=32'},
 309                                 {'variant': 'n64-nan2008',
 310                                  'ccopts': '-mabi=64'}])
 311         self.add_config(arch='mips64el',
 312                         os_name='linux-gnu',
 313                         variant='nan2008-soft',
 314                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 315                                  '--with-arch-64=mips64r2',
 316                                  '--with-arch-32=mips32r2',
 317                                  '--with-float=soft'],
 318                         glibcs=[{'variant': 'n32-nan2008-soft'},
 319                                 {'variant': 'nan2008-soft',
 320                                  'arch': 'mipsel',
 321                                  'ccopts': '-mabi=32'},
 322                                 {'variant': 'n64-nan2008-soft',
 323                                  'ccopts': '-mabi=64'}])
 324         self.add_config(arch='mipsisa64r6el',
 325                         os_name='linux-gnu',
 326                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 327                                  '--with-arch-64=mips64r6',
 328                                  '--with-arch-32=mips32r6',
 329                                  '--with-float=hard'],
 330                         glibcs=[{'variant': 'n32'},
 331                                 {'arch': 'mipsisa32r6el',
 332                                  'ccopts': '-mabi=32'},
 333                                 {'variant': 'n64',
 334                                  'ccopts': '-mabi=64'}])
 335         self.add_config(arch='nios2',
 336                         os_name='linux-gnu')
 337         self.add_config(arch='powerpc',
 338                         os_name='linux-gnu',
 339                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 340                         extra_glibcs=[{'variant': 'power4',
 341                                        'ccopts': '-mcpu=power4',
 342                                        'cfg': ['--with-cpu=power4']}])
 343         self.add_config(arch='powerpc',
 344                         os_name='linux-gnu',
 345                         variant='soft',
 346                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 347                                  '--enable-secureplt'])
 348         self.add_config(arch='powerpc64',
 349                         os_name='linux-gnu',
 350                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 351         self.add_config(arch='powerpc64le',
 352                         os_name='linux-gnu',
 353                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 354         self.add_config(arch='riscv32',
 355                         os_name='linux-gnu',
 356                         variant='rv32imac-ilp32',
 357                         gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
 358                                  '--disable-multilib'])
 359         self.add_config(arch='riscv32',
 360                         os_name='linux-gnu',
 361                         variant='rv32imafdc-ilp32',
 362                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
 363                                  '--disable-multilib'])
 364         self.add_config(arch='riscv32',
 365                         os_name='linux-gnu',
 366                         variant='rv32imafdc-ilp32d',
 367                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
 368                                  '--disable-multilib'])
 369         self.add_config(arch='riscv64',
 370                         os_name='linux-gnu',
 371                         variant='rv64imac-lp64',
 372                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 373                                  '--disable-multilib'])
 374         self.add_config(arch='riscv64',
 375                         os_name='linux-gnu',
 376                         variant='rv64imafdc-lp64',
 377                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 378                                  '--disable-multilib'])
 379         self.add_config(arch='riscv64',
 380                         os_name='linux-gnu',
 381                         variant='rv64imafdc-lp64d',
 382                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 383                                  '--disable-multilib'])
 384         self.add_config(arch='s390x',
 385                         os_name='linux-gnu',
 386                         glibcs=[{},
 387                                 {'arch': 's390', 'ccopts': '-m31'}],
 388                         extra_glibcs=[{'variant': 'O3',
 389                                        'cflags': '-O3'}])
 390         self.add_config(arch='sh3',
 391                         os_name='linux-gnu')
 392         self.add_config(arch='sh3eb',
 393                         os_name='linux-gnu')
 394         self.add_config(arch='sh4',
 395                         os_name='linux-gnu')
 396         self.add_config(arch='sh4eb',
 397                         os_name='linux-gnu')
 398         self.add_config(arch='sh4',
 399                         os_name='linux-gnu',
 400                         variant='soft',
 401                         gcc_cfg=['--without-fp'])
 402         self.add_config(arch='sh4eb',
 403                         os_name='linux-gnu',
 404                         variant='soft',
 405                         gcc_cfg=['--without-fp'])
 406         self.add_config(arch='sparc64',
 407                         os_name='linux-gnu',
 408                         glibcs=[{},
 409                                 {'arch': 'sparcv9',
 410                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 411                         extra_glibcs=[{'variant': 'leon3',
 412                                        'arch' : 'sparcv8',
 413                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 414                                       {'variant': 'disable-multi-arch',
 415                                        'cfg': ['--disable-multi-arch']},
 416                                       {'variant': 'disable-multi-arch',
 417                                        'arch': 'sparcv9',
 418                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 419                                        'cfg': ['--disable-multi-arch']}])
 420         self.add_config(arch='x86_64',
 421                         os_name='linux-gnu',
 422                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 423                         glibcs=[{},
 424                                 {'variant': 'x32', 'ccopts': '-mx32'},
 425                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 426                         extra_glibcs=[{'variant': 'disable-multi-arch',
 427                                        'cfg': ['--disable-multi-arch']},
 428                                       {'variant': 'minimal',
 429                                        'cfg': ['--disable-multi-arch',
 430                                                '--disable-profile',
 431                                                '--disable-timezone-tools',
 432                                                '--disable-mathvec',
 433                                                '--disable-tunables',
 434                                                '--disable-crypt',
 435                                                '--disable-experimental-malloc',
 436                                                '--disable-build-nscd',
 437                                                '--disable-nscd']},
 438                                       {'variant': 'static-pie',
 439                                        'cfg': ['--enable-static-pie']},
 440                                       {'variant': 'x32-static-pie',
 441                                        'ccopts': '-mx32',
 442                                        'cfg': ['--enable-static-pie']},
 443                                       {'variant': 'static-pie',
 444                                        'arch': 'i686',
 445                                        'ccopts': '-m32 -march=i686',
 446                                        'cfg': ['--enable-static-pie']},
 447                                       {'variant': 'disable-multi-arch',
 448                                        'arch': 'i686',
 449                                        'ccopts': '-m32 -march=i686',
 450                                        'cfg': ['--disable-multi-arch']},
 451                                       {'arch': 'i486',
 452                                        'ccopts': '-m32 -march=i486'},
 453                                       {'arch': 'i586',
 454                                        'ccopts': '-m32 -march=i586'}])
 455
 456     def add_config(self, **args):
 457         """Add an individual build configuration."""
 458         cfg = Config(self, **args)
 459         if cfg.name in self.configs:
 460             print('error: duplicate config %s' % cfg.name)
 461             exit(1)
 462         self.configs[cfg.name] = cfg
 463         for c in cfg.all_glibcs:
 464             if c.name in self.glibc_configs:
 465                 print('error: duplicate glibc config %s' % c.name)
 466                 exit(1)
 467             self.glibc_configs[c.name] = c
 468
 469     def component_srcdir(self, component):
 470         """Return the source directory for a given component, e.g. gcc."""
 471         return os.path.join(self.srcdir, component)
 472
 473     def component_builddir(self, action, config, component, subconfig=None):
 474         """Return the directory to use for a build."""
 475         if config is None:
 476             # Host libraries.
 477             assert subconfig is None
 478             return os.path.join(self.builddir, action, component)
 479         if subconfig is None:
 480             return os.path.join(self.builddir, action, config, component)
 481         else:
 482             # glibc build as part of compiler build.
 483             return os.path.join(self.builddir, action, config, component,
 484                                 subconfig)
 485
 486     def compiler_installdir(self, config):
 487         """Return the directory in which to install a compiler."""
 488         return os.path.join(self.installdir, 'compilers', config)
 489
 490     def compiler_bindir(self, config):
 491         """Return the directory in which to find compiler binaries."""
 492         return os.path.join(self.compiler_installdir(config), 'bin')
 493
 494     def compiler_sysroot(self, config):
 495         """Return the sysroot directory for a compiler."""
 496         return os.path.join(self.compiler_installdir(config), 'sysroot')
 497
 498     def glibc_installdir(self, config):
 499         """Return the directory in which to install glibc."""
 500         return os.path.join(self.installdir, 'glibcs', config)
 501
 502     def run_builds(self, action, configs):
 503         """Run the requested builds."""
 504         if action == 'checkout':
 505             self.checkout(configs)
 506             return
 507         if action == 'bot-cycle':
 508             if configs:
 509                 print('error: configurations specified for bot-cycle')
 510                 exit(1)
 511             self.bot_cycle()
 512             return
 513         if action == 'bot':
 514             if configs:
 515                 print('error: configurations specified for bot')
 516                 exit(1)
 517             self.bot()
 518             return
 519         if action in ('host-libraries', 'list-compilers',
 520                       'list-glibcs') and configs:
 521             print('error: configurations specified for ' + action)
 522             exit(1)
 523         if action == 'list-compilers':
 524             for name in sorted(self.configs.keys()):
 525                 print(name)
 526             return
 527         if action == 'list-glibcs':
 528             for config in sorted(self.glibc_configs.values(),
 529                                  key=lambda c: c.name):
 530                 print(config.name, config.compiler.name)
 531             return
 532         self.clear_last_build_state(action)
 533         build_time = datetime.datetime.utcnow()
 534         if action == 'host-libraries':
 535             build_components = ('gmp', 'mpfr', 'mpc')
 536             old_components = ()
 537             old_versions = {}
 538             self.build_host_libraries()
 539         elif action == 'compilers':
 540             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 541                                 'gnumach', 'hurd')
 542             old_components = ('gmp', 'mpfr', 'mpc')
 543             old_versions = self.build_state['host-libraries']['build-versions']
 544             self.build_compilers(configs)
 545         else:
 546             build_components = ('glibc',)
 547             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 548                               'mig', 'gnumach', 'hurd')
 549             old_versions = self.build_state['compilers']['build-versions']
 550             if action == 'update-syscalls':
 551                 self.update_syscalls(configs)
 552             else:
 553                 self.build_glibcs(configs)
 554         self.write_files()
 555         self.do_build()
 556         if configs:
 557             # Partial build, do not update stored state.
 558             return
 559         build_versions = {}
 560         for k in build_components:
 561             if k in self.versions:
 562                 build_versions[k] = {'version': self.versions[k]['version'],
 563                                      'revision': self.versions[k]['revision']}
 564         for k in old_components:
 565             if k in old_versions:
 566                 build_versions[k] = {'version': old_versions[k]['version'],
 567                                      'revision': old_versions[k]['revision']}
 568         self.update_build_state(action, build_time, build_versions)
 569
 570     @staticmethod
 571     def remove_dirs(*args):
 572         """Remove directories and their contents if they exist."""
 573         for dir in args:
 574             shutil.rmtree(dir, ignore_errors=True)
 575
 576     @staticmethod
 577     def remove_recreate_dirs(*args):
 578         """Remove directories if they exist, and create them as empty."""
 579         Context.remove_dirs(*args)
 580         for dir in args:
 581             os.makedirs(dir, exist_ok=True)
 582
 583     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 584         """Add makefile text for a list of commands."""
 585         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 586         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 587                                     (target, target, target, commands))
 588         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 589
 590     def write_files(self):
 591         """Write out the Makefile and wrapper script."""
 592         mftext = ''.join(self.makefile_pieces)
 593         with open(self.makefile, 'w') as f:
 594             f.write(mftext)
 595         wrapper_text = (
 596             '#!/bin/sh\n'
 597             'prev_base=$1\n'
 598             'this_base=$2\n'
 599             'desc=$3\n'
 600             'dir=$4\n'
 601             'path=$5\n'
 602             'shift 5\n'
 603             'prev_status=$prev_base-status.txt\n'
 604             'this_status=$this_base-status.txt\n'
 605             'this_log=$this_base-log.txt\n'
 606             'date > "$this_log"\n'
 607             'echo >> "$this_log"\n'
 608             'echo "Description: $desc" >> "$this_log"\n'
 609             'printf "%s" "Command:" >> "$this_log"\n'
 610             'for word in "$@"; do\n'
 611             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 612             '    printf " %s" "$word"\n'
 613             '  else\n'
 614             '    printf " \'"\n'
 615             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 616             '    printf "\'"\n'
 617             '  fi\n'
 618             'done >> "$this_log"\n'
 619             'echo >> "$this_log"\n'
 620             'echo "Directory: $dir" >> "$this_log"\n'
 621             'echo "Path addition: $path" >> "$this_log"\n'
 622             'echo >> "$this_log"\n'
 623             'record_status ()\n'
 624             '{\n'
 625             '  echo >> "$this_log"\n'
 626             '  echo "$1: $desc" > "$this_status"\n'
 627             '  echo "$1: $desc" >> "$this_log"\n'
 628             '  echo >> "$this_log"\n'
 629             '  date >> "$this_log"\n'
 630             '  echo "$1: $desc"\n'
 631             '  exit 0\n'
 632             '}\n'
 633             'check_error ()\n'
 634             '{\n'
 635             '  if [ "$1" != "0" ]; then\n'
 636             '    record_status FAIL\n'
 637             '  fi\n'
 638             '}\n'
 639             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 640             '    record_status UNRESOLVED\n'
 641             'fi\n'
 642             'if [ "$dir" ]; then\n'
 643             '  cd "$dir"\n'
 644             '  check_error "$?"\n'
 645             'fi\n'
 646             'if [ "$path" ]; then\n'
 647             '  PATH=$path:$PATH\n'
 648             'fi\n'
 649             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 650             'check_error "$?"\n'
 651             'record_status PASS\n')
 652         with open(self.wrapper, 'w') as f:
 653             f.write(wrapper_text)
 654         # Mode 0o755.
 655         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 656                      stat.S_IROTH|stat.S_IXOTH)
 657         os.chmod(self.wrapper, mode_exec)
 658         save_logs_text = (
 659             '#!/bin/sh\n'
 660             'if ! [ -f tests.sum ]; then\n'
 661             '  echo "No test summary available."\n'
 662             '  exit 0\n'
 663             'fi\n'
 664             'save_file ()\n'
 665             '{\n'
 666             '  echo "Contents of $1:"\n'
 667             '  echo\n'
 668             '  cat "$1"\n'
 669             '  echo\n'
 670             '  echo "End of contents of $1."\n'
 671             '  echo\n'
 672             '}\n'
 673             'save_file tests.sum\n'
 674             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 675             'for t in $non_pass_tests; do\n'
 676             '  if [ -f "$t.out" ]; then\n'
 677             '    save_file "$t.out"\n'
 678             '  fi\n'
 679             'done\n')
 680         with open(self.save_logs, 'w') as f:
 681             f.write(save_logs_text)
 682         os.chmod(self.save_logs, mode_exec)
 683
 684     def do_build(self):
 685         """Do the actual build."""
 686         cmd = ['make', '-O', '-j%d' % self.parallelism]
 687         subprocess.run(cmd, cwd=self.builddir, check=True)
 688
 689     def build_host_libraries(self):
 690         """Build the host libraries."""
 691         installdir = self.host_libraries_installdir
 692         builddir = os.path.join(self.builddir, 'host-libraries')
 693         logsdir = os.path.join(self.logsdir, 'host-libraries')
 694         self.remove_recreate_dirs(installdir, builddir, logsdir)
 695         cmdlist = CommandList('host-libraries', self.keep)
 696         self.build_host_library(cmdlist, 'gmp')
 697         self.build_host_library(cmdlist, 'mpfr',
 698                                 ['--with-gmp=%s' % installdir])
 699         self.build_host_library(cmdlist, 'mpc',
 700                                 ['--with-gmp=%s' % installdir,
 701                                 '--with-mpfr=%s' % installdir])
 702         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 703         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 704
 705     def build_host_library(self, cmdlist, lib, extra_opts=None):
 706         """Build one host library."""
 707         srcdir = self.component_srcdir(lib)
 708         builddir = self.component_builddir('host-libraries', None, lib)
 709         installdir = self.host_libraries_installdir
 710         cmdlist.push_subdesc(lib)
 711         cmdlist.create_use_dir(builddir)
 712         cfg_cmd = [os.path.join(srcdir, 'configure'),
 713                    '--prefix=%s' % installdir,
 714                    '--disable-shared']
 715         if extra_opts:
 716             cfg_cmd.extend (extra_opts)
 717         cmdlist.add_command('configure', cfg_cmd)
 718         cmdlist.add_command('build', ['make'])
 719         cmdlist.add_command('check', ['make', 'check'])
 720         cmdlist.add_command('install', ['make', 'install'])
 721         cmdlist.cleanup_dir()
 722         cmdlist.pop_subdesc()
 723
 724     def build_compilers(self, configs):
 725         """Build the compilers."""
 726         if not configs:
 727             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 728             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 729             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 730             configs = sorted(self.configs.keys())
 731         for c in configs:
 732             self.configs[c].build()
 733
 734     def build_glibcs(self, configs):
 735         """Build the glibcs."""
 736         if not configs:
 737             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 738             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 739             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 740             configs = sorted(self.glibc_configs.keys())
 741         for c in configs:
 742             self.glibc_configs[c].build()
 743
 744     def update_syscalls(self, configs):
 745         """Update the glibc syscall lists."""
 746         if not configs:
 747             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 748             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 749             configs = sorted(self.glibc_configs.keys())
 750         for c in configs:
 751             self.glibc_configs[c].update_syscalls()
 752
 753     def load_versions_json(self):
 754         """Load information about source directory versions."""
 755         if not os.access(self.versions_json, os.F_OK):
 756             self.versions = {}
 757             return
 758         with open(self.versions_json, 'r') as f:
 759             self.versions = json.load(f)
 760
 761     def store_json(self, data, filename):
 762         """Store information in a JSON file."""
 763         filename_tmp = filename + '.tmp'
 764         with open(filename_tmp, 'w') as f:
 765             json.dump(data, f, indent=2, sort_keys=True)
 766         os.rename(filename_tmp, filename)
 767
 768     def store_versions_json(self):
 769         """Store information about source directory versions."""
 770         self.store_json(self.versions, self.versions_json)
 771
 772     def set_component_version(self, component, version, explicit, revision):
 773         """Set the version information for a component."""
 774         self.versions[component] = {'version': version,
 775                                     'explicit': explicit,
 776                                     'revision': revision}
 777         self.store_versions_json()
 778
 779     def checkout(self, versions):
 780         """Check out the desired component versions."""
 781         default_versions = {'binutils': 'vcs-2.37',
 782                             'gcc': 'vcs-11',
 783                             'glibc': 'vcs-mainline',
 784                             'gmp': '6.2.1',
 785                             'linux': '5.13',
 786                             'mpc': '1.2.1',
 787                             'mpfr': '4.1.0',
 788                             'mig': 'vcs-mainline',
 789                             'gnumach': 'vcs-mainline',
 790                             'hurd': 'vcs-mainline'}
 791         use_versions = {}
 792         explicit_versions = {}
 793         for v in versions:
 794             found_v = False
 795             for k in default_versions.keys():
 796                 kx = k + '-'
 797                 if v.startswith(kx):
 798                     vx = v[len(kx):]
 799                     if k in use_versions:
 800                         print('error: multiple versions for %s' % k)
 801                         exit(1)
 802                     use_versions[k] = vx
 803                     explicit_versions[k] = True
 804                     found_v = True
 805                     break
 806             if not found_v:
 807                 print('error: unknown component in %s' % v)
 808                 exit(1)
 809         for k in default_versions.keys():
 810             if k not in use_versions:
 811                 if k in self.versions and self.versions[k]['explicit']:
 812                     use_versions[k] = self.versions[k]['version']
 813                     explicit_versions[k] = True
 814                 else:
 815                     use_versions[k] = default_versions[k]
 816                     explicit_versions[k] = False
 817         os.makedirs(self.srcdir, exist_ok=True)
 818         for k in sorted(default_versions.keys()):
 819             update = os.access(self.component_srcdir(k), os.F_OK)
 820             v = use_versions[k]
 821             if (update and
 822                 k in self.versions and
 823                 v != self.versions[k]['version']):
 824                 if not self.replace_sources:
 825                     print('error: version of %s has changed from %s to %s, '
 826                           'use --replace-sources to check out again' %
 827                           (k, self.versions[k]['version'], v))
 828                     exit(1)
 829                 shutil.rmtree(self.component_srcdir(k))
 830                 update = False
 831             if v.startswith('vcs-'):
 832                 revision = self.checkout_vcs(k, v[4:], update)
 833             else:
 834                 self.checkout_tar(k, v, update)
 835                 revision = v
 836             self.set_component_version(k, v, explicit_versions[k], revision)
 837         if self.get_script_text() != self.script_text:
 838             # Rerun the checkout process in case the updated script
 839             # uses different default versions or new components.
 840             self.exec_self()
 841
 842     def checkout_vcs(self, component, version, update):
 843         """Check out the given version of the given component from version
 844         control.  Return a revision identifier."""
 845         if component == 'binutils':
 846             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 847             if version == 'mainline':
 848                 git_branch = 'master'
 849             else:
 850                 trans = str.maketrans({'.': '_'})
 851                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 852             return self.git_checkout(component, git_url, git_branch, update)
 853         elif component == 'gcc':
 854             if version == 'mainline':
 855                 branch = 'master'
 856             else:
 857                 branch = 'releases/gcc-%s' % version
 858             return self.gcc_checkout(branch, update)
 859         elif component == 'glibc':
 860             git_url = 'git://sourceware.org/git/glibc.git'
 861             if version == 'mainline':
 862                 git_branch = 'master'
 863             else:
 864                 git_branch = 'release/%s/master' % version
 865             r = self.git_checkout(component, git_url, git_branch, update)
 866             self.fix_glibc_timestamps()
 867             return r
 868         elif component == 'gnumach':
 869             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 870             git_branch = 'master'
 871             r = self.git_checkout(component, git_url, git_branch, update)
 872             subprocess.run(['autoreconf', '-i'],
 873                            cwd=self.component_srcdir(component), check=True)
 874             return r
 875         elif component == 'mig':
 876             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 877             git_branch = 'master'
 878             r = self.git_checkout(component, git_url, git_branch, update)
 879             subprocess.run(['autoreconf', '-i'],
 880                            cwd=self.component_srcdir(component), check=True)
 881             return r
 882         elif component == 'hurd':
 883             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 884             git_branch = 'master'
 885             r = self.git_checkout(component, git_url, git_branch, update)
 886             subprocess.run(['autoconf'],
 887                            cwd=self.component_srcdir(component), check=True)
 888             return r
 889         else:
 890             print('error: component %s coming from VCS' % component)
 891             exit(1)
 892
 893     def git_checkout(self, component, git_url, git_branch, update):
 894         """Check out a component from git.  Return a commit identifier."""
 895         if update:
 896             subprocess.run(['git', 'remote', 'prune', 'origin'],
 897                            cwd=self.component_srcdir(component), check=True)
 898             if self.replace_sources:
 899                 subprocess.run(['git', 'clean', '-dxfq'],
 900                                cwd=self.component_srcdir(component), check=True)
 901             subprocess.run(['git', 'pull', '-q'],
 902                            cwd=self.component_srcdir(component), check=True)
 903         else:
 904             if self.shallow:
 905                 depth_arg = ('--depth', '1')
 906             else:
 907                 depth_arg = ()
 908             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 909                             *depth_arg, git_url,
 910                             self.component_srcdir(component)], check=True)
 911         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 912                            cwd=self.component_srcdir(component),
 913                            stdout=subprocess.PIPE,
 914                            check=True, universal_newlines=True).stdout
 915         return r.rstrip()
 916
 917     def fix_glibc_timestamps(self):
 918         """Fix timestamps in a glibc checkout."""
 919         # Ensure that builds do not try to regenerate generated files
 920         # in the source tree.
 921         srcdir = self.component_srcdir('glibc')
 922         # These files have Makefile dependencies to regenerate them in
 923         # the source tree that may be active during a normal build.
 924         # Some other files have such dependencies but do not need to
 925         # be touched because nothing in a build depends on the files
 926         # in question.
 927         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 928             to_touch = os.path.join(srcdir, f)
 929             subprocess.run(['touch', '-c', to_touch], check=True)
 930         for dirpath, dirnames, filenames in os.walk(srcdir):
 931             for f in filenames:
 932                 if (f == 'configure' or
 933                     f == 'preconfigure' or
 934                     f.endswith('-kw.h')):
 935                     to_touch = os.path.join(dirpath, f)
 936                     subprocess.run(['touch', to_touch], check=True)
 937
 938     def gcc_checkout(self, branch, update):
 939         """Check out GCC from git.  Return the commit identifier."""
 940         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 941                      os.F_OK):
 942             if not self.replace_sources:
 943                 print('error: GCC has moved from SVN to git, use '
 944                       '--replace-sources to check out again')
 945                 exit(1)
 946             shutil.rmtree(self.component_srcdir('gcc'))
 947             update = False
 948         if not update:
 949             self.git_checkout('gcc', 'git://gcc.gnu.org/git/gcc.git',
 950                               branch, update)
 951         subprocess.run(['contrib/gcc_update', '--silent'],
 952                        cwd=self.component_srcdir('gcc'), check=True)
 953         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 954                            cwd=self.component_srcdir('gcc'),
 955                            stdout=subprocess.PIPE,
 956                            check=True, universal_newlines=True).stdout
 957         return r.rstrip()
 958
 959     def checkout_tar(self, component, version, update):
 960         """Check out the given version of the given component from a
 961         tarball."""
 962         if update:
 963             return
 964         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 965                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
 966                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 967                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
 968                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 969                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 970                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 971                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 972                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 973         if component not in url_map:
 974             print('error: component %s coming from tarball' % component)
 975             exit(1)
 976         version_major = version.split('.')[0]
 977         url = url_map[component] % {'version': version, 'major': version_major}
 978         filename = os.path.join(self.srcdir, url.split('/')[-1])
 979         response = urllib.request.urlopen(url)
 980         data = response.read()
 981         with open(filename, 'wb') as f:
 982             f.write(data)
 983         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 984                        check=True)
 985         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 986                   self.component_srcdir(component))
 987         os.remove(filename)
 988
 989     def load_build_state_json(self):
 990         """Load information about the state of previous builds."""
 991         if os.access(self.build_state_json, os.F_OK):
 992             with open(self.build_state_json, 'r') as f:
 993                 self.build_state = json.load(f)
 994         else:
 995             self.build_state = {}
 996         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
 997             if k not in self.build_state:
 998                 self.build_state[k] = {}
 999             if 'build-time' not in self.build_state[k]:
1000                 self.build_state[k]['build-time'] = ''
1001             if 'build-versions' not in self.build_state[k]:
1002                 self.build_state[k]['build-versions'] = {}
1003             if 'build-results' not in self.build_state[k]:
1004                 self.build_state[k]['build-results'] = {}
1005             if 'result-changes' not in self.build_state[k]:
1006                 self.build_state[k]['result-changes'] = {}
1007             if 'ever-passed' not in self.build_state[k]:
1008                 self.build_state[k]['ever-passed'] = []
1009
1010     def store_build_state_json(self):
1011         """Store information about the state of previous builds."""
1012         self.store_json(self.build_state, self.build_state_json)
1013
1014     def clear_last_build_state(self, action):
1015         """Clear information about the state of part of the build."""
1016         # We clear the last build time and versions when starting a
1017         # new build.  The results of the last build are kept around,
1018         # as comparison is still meaningful if this build is aborted
1019         # and a new one started.
1020         self.build_state[action]['build-time'] = ''
1021         self.build_state[action]['build-versions'] = {}
1022         self.store_build_state_json()
1023
1024     def update_build_state(self, action, build_time, build_versions):
1025         """Update the build state after a build."""
1026         build_time = build_time.replace(microsecond=0)
1027         self.build_state[action]['build-time'] = str(build_time)
1028         self.build_state[action]['build-versions'] = build_versions
1029         build_results = {}
1030         for log in self.status_log_list:
1031             with open(log, 'r') as f:
1032                 log_text = f.read()
1033             log_text = log_text.rstrip()
1034             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1035             result = m.group(1)
1036             test_name = m.group(2)
1037             assert test_name not in build_results
1038             build_results[test_name] = result
1039         old_build_results = self.build_state[action]['build-results']
1040         self.build_state[action]['build-results'] = build_results
1041         result_changes = {}
1042         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1043         for t in all_tests:
1044             if t in old_build_results:
1045                 old_res = old_build_results[t]
1046             else:
1047                 old_res = '(New test)'
1048             if t in build_results:
1049                 new_res = build_results[t]
1050             else:
1051                 new_res = '(Test removed)'
1052             if old_res != new_res:
1053                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1054         self.build_state[action]['result-changes'] = result_changes
1055         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1056                            if t in build_results}
1057         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1058         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1059                                                          new_passes)
1060         self.store_build_state_json()
1061
1062     def load_bot_config_json(self):
1063         """Load bot configuration."""
1064         with open(self.bot_config_json, 'r') as f:
1065             self.bot_config = json.load(f)
1066
1067     def part_build_old(self, action, delay):
1068         """Return whether the last build for a given action was at least a
1069         given number of seconds ago, or does not have a time recorded."""
1070         old_time_str = self.build_state[action]['build-time']
1071         if not old_time_str:
1072             return True
1073         old_time = datetime.datetime.strptime(old_time_str,
1074                                               '%Y-%m-%d %H:%M:%S')
1075         new_time = datetime.datetime.utcnow()
1076         delta = new_time - old_time
1077         return delta.total_seconds() >= delay
1078
1079     def bot_cycle(self):
1080         """Run a single round of checkout and builds."""
1081         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1082         self.load_bot_config_json()
1083         actions = ('host-libraries', 'compilers', 'glibcs')
1084         self.bot_run_self(['--replace-sources'], 'checkout')
1085         self.load_versions_json()
1086         if self.get_script_text() != self.script_text:
1087             print('Script changed, re-execing.')
1088             # On script change, all parts of the build should be rerun.
1089             for a in actions:
1090                 self.clear_last_build_state(a)
1091             self.exec_self()
1092         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1093                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1094                                           'mig', 'gnumach', 'hurd'),
1095                             'glibcs': ('glibc',)}
1096         must_build = {}
1097         for a in actions:
1098             build_vers = self.build_state[a]['build-versions']
1099             must_build[a] = False
1100             if not self.build_state[a]['build-time']:
1101                 must_build[a] = True
1102             old_vers = {}
1103             new_vers = {}
1104             for c in check_components[a]:
1105                 if c in build_vers:
1106                     old_vers[c] = build_vers[c]
1107                 new_vers[c] = {'version': self.versions[c]['version'],
1108                                'revision': self.versions[c]['revision']}
1109             if new_vers == old_vers:
1110                 print('Versions for %s unchanged.' % a)
1111             else:
1112                 print('Versions changed or rebuild forced for %s.' % a)
1113                 if a == 'compilers' and not self.part_build_old(
1114                         a, self.bot_config['compilers-rebuild-delay']):
1115                     print('Not requiring rebuild of compilers this soon.')
1116                 else:
1117                     must_build[a] = True
1118         if must_build['host-libraries']:
1119             must_build['compilers'] = True
1120         if must_build['compilers']:
1121             must_build['glibcs'] = True
1122         for a in actions:
1123             if must_build[a]:
1124                 print('Must rebuild %s.' % a)
1125                 self.clear_last_build_state(a)
1126             else:
1127                 print('No need to rebuild %s.' % a)
1128         if os.access(self.logsdir, os.F_OK):
1129             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1130             shutil.copytree(self.logsdir, self.logsdir_old)
1131         for a in actions:
1132             if must_build[a]:
1133                 build_time = datetime.datetime.utcnow()
1134                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1135                 self.bot_run_self([], a)
1136                 self.load_build_state_json()
1137                 self.bot_build_mail(a, build_time)
1138         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1139
1140     def bot_build_mail(self, action, build_time):
1141         """Send email with the results of a build."""
1142         if not ('email-from' in self.bot_config and
1143                 'email-server' in self.bot_config and
1144                 'email-subject' in self.bot_config and
1145                 'email-to' in self.bot_config):
1146             if not self.email_warning:
1147                 print("Email not configured, not sending.")
1148                 self.email_warning = True
1149             return
1150
1151         build_time = build_time.replace(microsecond=0)
1152         subject = (self.bot_config['email-subject'] %
1153                    {'action': action,
1154                     'build-time': str(build_time)})
1155         results = self.build_state[action]['build-results']
1156         changes = self.build_state[action]['result-changes']
1157         ever_passed = set(self.build_state[action]['ever-passed'])
1158         versions = self.build_state[action]['build-versions']
1159         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1160         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1161         all_fails = {k for k in results if results[k] == 'FAIL'}
1162         if new_regressions:
1163             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1164             new_reg_text = ('New regressions:\n\n%s\n\n' %
1165                             '\n'.join(new_reg_list))
1166         else:
1167             new_reg_text = ''
1168         if all_regressions:
1169             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1170             all_reg_text = ('All regressions:\n\n%s\n\n' %
1171                             '\n'.join(all_reg_list))
1172         else:
1173             all_reg_text = ''
1174         if all_fails:
1175             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1176             all_fail_text = ('All failures:\n\n%s\n\n' %
1177                              '\n'.join(all_fail_list))
1178         else:
1179             all_fail_text = ''
1180         if changes:
1181             changes_list = sorted(changes.keys())
1182             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1183             changes_text = ('All changed results:\n\n%s\n\n' %
1184                             '\n'.join(changes_list))
1185         else:
1186             changes_text = ''
1187         results_text = (new_reg_text + all_reg_text + all_fail_text +
1188                         changes_text)
1189         if not results_text:
1190             results_text = 'Clean build with unchanged results.\n\n'
1191         versions_list = sorted(versions.keys())
1192         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1193                                           versions[k]['revision'])
1194                          for k in versions_list]
1195         versions_text = ('Component versions for this build:\n\n%s\n' %
1196                          '\n'.join(versions_list))
1197         body_text = results_text + versions_text
1198         msg = email.mime.text.MIMEText(body_text)
1199         msg['Subject'] = subject
1200         msg['From'] = self.bot_config['email-from']
1201         msg['To'] = self.bot_config['email-to']
1202         msg['Message-ID'] = email.utils.make_msgid()
1203         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1204         with smtplib.SMTP(self.bot_config['email-server']) as s:
1205             s.send_message(msg)
1206
1207     def bot_run_self(self, opts, action, check=True):
1208         """Run a copy of this script with given options."""
1209         cmd = [sys.executable, sys.argv[0], '--keep=none',
1210                '-j%d' % self.parallelism]
1211         if self.full_gcc:
1212             cmd.append('--full-gcc')
1213         cmd.extend(opts)
1214         cmd.extend([self.topdir, action])
1215         sys.stdout.flush()
1216         subprocess.run(cmd, check=check)
1217
1218     def bot(self):
1219         """Run repeated rounds of checkout and builds."""
1220         while True:
1221             self.load_bot_config_json()
1222             if not self.bot_config['run']:
1223                 print('Bot exiting by request.')
1224                 exit(0)
1225             self.bot_run_self([], 'bot-cycle', check=False)
1226             self.load_bot_config_json()
1227             if not self.bot_config['run']:
1228                 print('Bot exiting by request.')
1229                 exit(0)
1230             time.sleep(self.bot_config['delay'])
1231             if self.get_script_text() != self.script_text:
1232                 print('Script changed, bot re-execing.')
1233                 self.exec_self()
1234
1235 class LinuxHeadersPolicyForBuild(object):
1236     """Names and directories for installing Linux headers.  Build variant."""
1237
1238     def __init__(self, config):
1239         self.arch = config.arch
1240         self.srcdir = config.ctx.component_srcdir('linux')
1241         self.builddir = config.component_builddir('linux')
1242         self.headers_dir = os.path.join(config.sysroot, 'usr')
1243
1244 class LinuxHeadersPolicyForUpdateSyscalls(object):
1245     """Names and directories for Linux headers.  update-syscalls variant."""
1246
1247     def __init__(self, glibc, headers_dir):
1248         self.arch = glibc.compiler.arch
1249         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1250         self.builddir = glibc.ctx.component_builddir(
1251             'update-syscalls', glibc.name, 'build-linux')
1252         self.headers_dir = headers_dir
1253
1254 def install_linux_headers(policy, cmdlist):
1255     """Install Linux kernel headers."""
1256     arch_map = {'aarch64': 'arm64',
1257                 'alpha': 'alpha',
1258                 'arc': 'arc',
1259                 'arm': 'arm',
1260                 'csky': 'csky',
1261                 'hppa': 'parisc',
1262                 'i486': 'x86',
1263                 'i586': 'x86',
1264                 'i686': 'x86',
1265                 'i786': 'x86',
1266                 'ia64': 'ia64',
1267                 'm68k': 'm68k',
1268                 'microblaze': 'microblaze',
1269                 'mips': 'mips',
1270                 'nios2': 'nios2',
1271                 'powerpc': 'powerpc',
1272                 's390': 's390',
1273                 'riscv32': 'riscv',
1274                 'riscv64': 'riscv',
1275                 'sh': 'sh',
1276                 'sparc': 'sparc',
1277                 'x86_64': 'x86'}
1278     linux_arch = None
1279     for k in arch_map:
1280         if policy.arch.startswith(k):
1281             linux_arch = arch_map[k]
1282             break
1283     assert linux_arch is not None
1284     cmdlist.push_subdesc('linux')
1285     cmdlist.create_use_dir(policy.builddir)
1286     cmdlist.add_command('install-headers',
1287                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1288                          'ARCH=%s' % linux_arch,
1289                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1290                          'headers_install'])
1291     cmdlist.cleanup_dir()
1292     cmdlist.pop_subdesc()
1293
1294 class Config(object):
1295     """A configuration for building a compiler and associated libraries."""
1296
1297     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1298                  first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1299                  extra_glibcs=None):
1300         """Initialize a Config object."""
1301         self.ctx = ctx
1302         self.arch = arch
1303         self.os = os_name
1304         self.variant = variant
1305         if variant is None:
1306             self.name = '%s-%s' % (arch, os_name)
1307         else:
1308             self.name = '%s-%s-%s' % (arch, os_name, variant)
1309         self.triplet = '%s-glibc-%s' % (arch, os_name)
1310         if gcc_cfg is None:
1311             self.gcc_cfg = []
1312         else:
1313             self.gcc_cfg = gcc_cfg
1314         if first_gcc_cfg is None:
1315             self.first_gcc_cfg = []
1316         else:
1317             self.first_gcc_cfg = first_gcc_cfg
1318         if binutils_cfg is None:
1319             self.binutils_cfg = []
1320         else:
1321             self.binutils_cfg = binutils_cfg
1322         if glibcs is None:
1323             glibcs = [{'variant': variant}]
1324         if extra_glibcs is None:
1325             extra_glibcs = []
1326         glibcs = [Glibc(self, **g) for g in glibcs]
1327         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1328         self.all_glibcs = glibcs + extra_glibcs
1329         self.compiler_glibcs = glibcs
1330         self.installdir = ctx.compiler_installdir(self.name)
1331         self.bindir = ctx.compiler_bindir(self.name)
1332         self.sysroot = ctx.compiler_sysroot(self.name)
1333         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1334         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1335
1336     def component_builddir(self, component):
1337         """Return the directory to use for a (non-glibc) build."""
1338         return self.ctx.component_builddir('compilers', self.name, component)
1339
1340     def build(self):
1341         """Generate commands to build this compiler."""
1342         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1343                                       self.logsdir)
1344         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1345         cmdlist.add_command('check-host-libraries',
1346                             ['test', '-f',
1347                              os.path.join(self.ctx.host_libraries_installdir,
1348                                           'ok')])
1349         cmdlist.use_path(self.bindir)
1350         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1351                               ['--disable-gdb',
1352                                '--disable-gdbserver',
1353                                '--disable-libdecnumber',
1354                                '--disable-readline',
1355                                '--disable-sim'] + self.binutils_cfg)
1356         if self.os.startswith('linux'):
1357             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1358         self.build_gcc(cmdlist, True)
1359         if self.os == 'gnu':
1360             self.install_gnumach_headers(cmdlist)
1361             self.build_cross_tool(cmdlist, 'mig', 'mig')
1362             self.install_hurd_headers(cmdlist)
1363         for g in self.compiler_glibcs:
1364             cmdlist.push_subdesc('glibc')
1365             cmdlist.push_subdesc(g.name)
1366             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1367             cmdlist.pop_subdesc()
1368             cmdlist.pop_subdesc()
1369         self.build_gcc(cmdlist, False)
1370         cmdlist.add_command('done', ['touch',
1371                                      os.path.join(self.installdir, 'ok')])
1372         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1373                                       self.logsdir)
1374
1375     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1376         """Build one cross tool."""
1377         srcdir = self.ctx.component_srcdir(tool_src)
1378         builddir = self.component_builddir(tool_build)
1379         cmdlist.push_subdesc(tool_build)
1380         cmdlist.create_use_dir(builddir)
1381         cfg_cmd = [os.path.join(srcdir, 'configure'),
1382                    '--prefix=%s' % self.installdir,
1383                    '--build=%s' % self.ctx.build_triplet,
1384                    '--host=%s' % self.ctx.build_triplet,
1385                    '--target=%s' % self.triplet,
1386                    '--with-sysroot=%s' % self.sysroot]
1387         if extra_opts:
1388             cfg_cmd.extend(extra_opts)
1389         cmdlist.add_command('configure', cfg_cmd)
1390         cmdlist.add_command('build', ['make'])
1391         # Parallel "make install" for GCC has race conditions that can
1392         # cause it to fail; see
1393         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1394         # problems are not known for binutils, but doing the
1395         # installation in parallel within a particular toolchain build
1396         # (as opposed to installation of one toolchain from
1397         # build-many-glibcs.py running in parallel to the installation
1398         # of other toolchains being built) is not known to be
1399         # significantly beneficial, so it is simplest just to disable
1400         # parallel install for cross tools here.
1401         cmdlist.add_command('install', ['make', '-j1', 'install'])
1402         cmdlist.cleanup_dir()
1403         cmdlist.pop_subdesc()
1404
1405     def install_gnumach_headers(self, cmdlist):
1406         """Install GNU Mach headers."""
1407         srcdir = self.ctx.component_srcdir('gnumach')
1408         builddir = self.component_builddir('gnumach')
1409         cmdlist.push_subdesc('gnumach')
1410         cmdlist.create_use_dir(builddir)
1411         cmdlist.add_command('configure',
1412                             [os.path.join(srcdir, 'configure'),
1413                              '--build=%s' % self.ctx.build_triplet,
1414                              '--host=%s' % self.triplet,
1415                              '--prefix=',
1416                              'CC=%s-gcc -nostdlib' % self.triplet])
1417         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1418                                         'install-data'])
1419         cmdlist.cleanup_dir()
1420         cmdlist.pop_subdesc()
1421
1422     def install_hurd_headers(self, cmdlist):
1423         """Install Hurd headers."""
1424         srcdir = self.ctx.component_srcdir('hurd')
1425         builddir = self.component_builddir('hurd')
1426         cmdlist.push_subdesc('hurd')
1427         cmdlist.create_use_dir(builddir)
1428         cmdlist.add_command('configure',
1429                             [os.path.join(srcdir, 'configure'),
1430                              '--build=%s' % self.ctx.build_triplet,
1431                              '--host=%s' % self.triplet,
1432                              '--prefix=',
1433                              '--disable-profile', '--without-parted',
1434                              'CC=%s-gcc -nostdlib' % self.triplet])
1435         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1436                                         'no_deps=t', 'install-headers'])
1437         cmdlist.cleanup_dir()
1438         cmdlist.pop_subdesc()
1439
1440     def build_gcc(self, cmdlist, bootstrap):
1441         """Build GCC."""
1442         # libssp is of little relevance with glibc's own stack
1443         # checking support.  libcilkrts does not support GNU/Hurd (and
1444         # has been removed in GCC 8, so --disable-libcilkrts can be
1445         # removed once glibc no longer supports building with older
1446         # GCC versions).
1447         cfg_opts = list(self.gcc_cfg)
1448         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1449         host_libs = self.ctx.host_libraries_installdir
1450         cfg_opts += ['--with-gmp=%s' % host_libs,
1451                      '--with-mpfr=%s' % host_libs,
1452                      '--with-mpc=%s' % host_libs]
1453         if bootstrap:
1454             tool_build = 'gcc-first'
1455             # Building a static-only, C-only compiler that is
1456             # sufficient to build glibc.  Various libraries and
1457             # features that may require libc headers must be disabled.
1458             # When configuring with a sysroot, --with-newlib is
1459             # required to define inhibit_libc (to stop some parts of
1460             # libgcc including libc headers); --without-headers is not
1461             # sufficient.
1462             cfg_opts += ['--enable-languages=c', '--disable-shared',
1463                          '--disable-threads',
1464                          '--disable-libatomic',
1465                          '--disable-decimal-float',
1466                          '--disable-libffi',
1467                          '--disable-libgomp',
1468                          '--disable-libitm',
1469                          '--disable-libmpx',
1470                          '--disable-libquadmath',
1471                          '--disable-libsanitizer',
1472                          '--without-headers', '--with-newlib',
1473                          '--with-glibc-version=%s' % self.ctx.glibc_version
1474                          ]
1475             cfg_opts += self.first_gcc_cfg
1476         else:
1477             tool_build = 'gcc'
1478             # libsanitizer commonly breaks because of glibc header
1479             # changes, or on unusual targets.  C++ pre-compiled
1480             # headers are not used during the glibc build and are
1481             # expensive to create.
1482             if not self.ctx.full_gcc:
1483                 cfg_opts += ['--disable-libsanitizer',
1484                              '--disable-libstdcxx-pch']
1485             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1486             cfg_opts += ['--enable-languages=%s' % langs,
1487                          '--enable-shared', '--enable-threads']
1488         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1489
1490 class GlibcPolicyDefault(object):
1491     """Build policy for glibc: common defaults."""
1492
1493     def __init__(self, glibc):
1494         self.srcdir = glibc.ctx.component_srcdir('glibc')
1495         self.use_usr = glibc.os != 'gnu'
1496         self.prefix = '/usr' if self.use_usr else ''
1497         self.configure_args = [
1498             '--prefix=%s' % self.prefix,
1499             '--enable-profile',
1500             '--build=%s' % glibc.ctx.build_triplet,
1501             '--host=%s' % glibc.triplet,
1502             'CC=%s' % glibc.tool_name('gcc'),
1503             'CXX=%s' % glibc.tool_name('g++'),
1504             'AR=%s' % glibc.tool_name('ar'),
1505             'AS=%s' % glibc.tool_name('as'),
1506             'LD=%s' % glibc.tool_name('ld'),
1507             'NM=%s' % glibc.tool_name('nm'),
1508             'OBJCOPY=%s' % glibc.tool_name('objcopy'),
1509             'OBJDUMP=%s' % glibc.tool_name('objdump'),
1510             'RANLIB=%s' % glibc.tool_name('ranlib'),
1511             'READELF=%s' % glibc.tool_name('readelf'),
1512             'STRIP=%s' % glibc.tool_name('strip'),
1513         ]
1514         if glibc.os == 'gnu':
1515             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1516         if glibc.cflags:
1517             self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1518             self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
1519         self.configure_args += glibc.cfg
1520
1521     def configure(self, cmdlist):
1522         """Invoked to add the configure command to the command list."""
1523         cmdlist.add_command('configure',
1524                             [os.path.join(self.srcdir, 'configure'),
1525                              *self.configure_args])
1526
1527     def extra_commands(self, cmdlist):
1528         """Invoked to inject additional commands (make check) after build."""
1529         pass
1530
1531 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1532     """Build policy for glibc during the compilers stage."""
1533
1534     def __init__(self, glibc):
1535         super().__init__(glibc)
1536         self.builddir = glibc.ctx.component_builddir(
1537             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1538         self.installdir = glibc.compiler.sysroot
1539
1540 class GlibcPolicyForBuild(GlibcPolicyDefault):
1541     """Build policy for glibc during the glibcs stage."""
1542
1543     def __init__(self, glibc):
1544         super().__init__(glibc)
1545         self.builddir = glibc.ctx.component_builddir(
1546             'glibcs', glibc.name, 'glibc')
1547         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1548         if glibc.ctx.strip:
1549             self.strip = glibc.tool_name('strip')
1550         else:
1551             self.strip = None
1552         self.save_logs = glibc.ctx.save_logs
1553
1554     def extra_commands(self, cmdlist):
1555         if self.strip:
1556             # Avoid picking up libc.so and libpthread.so, which are
1557             # linker scripts stored in /lib on Hurd.  libc and
1558             # libpthread are still stripped via their libc-X.YY.so
1559             # implementation files.
1560             find_command = (('find %s/lib* -name "*.so"'
1561                              + r' \! -name libc.so \! -name libpthread.so')
1562                             % self.installdir)
1563             cmdlist.add_command('strip', ['sh', '-c', ('%s $(%s)' %
1564                                   (self.strip, find_command))])
1565         cmdlist.add_command('check', ['make', 'check'])
1566         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1567
1568 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1569     """Build policy for glibc during update-syscalls."""
1570
1571     def __init__(self, glibc):
1572         super().__init__(glibc)
1573         self.builddir = glibc.ctx.component_builddir(
1574             'update-syscalls', glibc.name, 'glibc')
1575         self.linuxdir = glibc.ctx.component_builddir(
1576             'update-syscalls', glibc.name, 'linux')
1577         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1578             glibc, self.linuxdir)
1579         self.configure_args.insert(
1580             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1581         # self.installdir not set because installation is not supported
1582
1583 class Glibc(object):
1584     """A configuration for building glibc."""
1585
1586     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1587                  cfg=None, ccopts=None, cflags=None):
1588         """Initialize a Glibc object."""
1589         self.ctx = compiler.ctx
1590         self.compiler = compiler
1591         if arch is None:
1592             self.arch = compiler.arch
1593         else:
1594             self.arch = arch
1595         if os_name is None:
1596             self.os = compiler.os
1597         else:
1598             self.os = os_name
1599         self.variant = variant
1600         if variant is None:
1601             self.name = '%s-%s' % (self.arch, self.os)
1602         else:
1603             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1604         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1605         if cfg is None:
1606             self.cfg = []
1607         else:
1608             self.cfg = cfg
1609         # ccopts contain ABI options and are passed to configure as CC / CXX.
1610         self.ccopts = ccopts
1611         # cflags contain non-ABI options like -g or -O and are passed to
1612         # configure as CFLAGS / CXXFLAGS.
1613         self.cflags = cflags
1614
1615     def tool_name(self, tool):
1616         """Return the name of a cross-compilation tool."""
1617         ctool = '%s-%s' % (self.compiler.triplet, tool)
1618         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1619             ctool = '%s %s' % (ctool, self.ccopts)
1620         return ctool
1621
1622     def build(self):
1623         """Generate commands to build this glibc."""
1624         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1625         installdir = self.ctx.glibc_installdir(self.name)
1626         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1627         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1628         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1629         cmdlist.add_command('check-compilers',
1630                             ['test', '-f',
1631                              os.path.join(self.compiler.installdir, 'ok')])
1632         cmdlist.use_path(self.compiler.bindir)
1633         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1634         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1635                                       logsdir)
1636
1637     def build_glibc(self, cmdlist, policy):
1638         """Generate commands to build this glibc, either as part of a compiler
1639         build or with the bootstrapped compiler (and in the latter case, run
1640         tests as well)."""
1641         cmdlist.create_use_dir(policy.builddir)
1642         policy.configure(cmdlist)
1643         cmdlist.add_command('build', ['make'])
1644         cmdlist.add_command('install', ['make', 'install',
1645                                         'install_root=%s' % policy.installdir])
1646         # GCC uses paths such as lib/../lib64, so make sure lib
1647         # directories always exist.
1648         mkdir_cmd = ['mkdir', '-p',
1649                      os.path.join(policy.installdir, 'lib')]
1650         if policy.use_usr:
1651             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1652         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1653         policy.extra_commands(cmdlist)
1654         cmdlist.cleanup_dir()
1655
1656     def update_syscalls(self):
1657         if self.os == 'gnu':
1658             # Hurd does not have system call tables that need updating.
1659             return
1660
1661         policy = GlibcPolicyForUpdateSyscalls(self)
1662         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1663         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1664         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1665         cmdlist.add_command('check-compilers',
1666                             ['test', '-f',
1667                              os.path.join(self.compiler.installdir, 'ok')])
1668         cmdlist.use_path(self.compiler.bindir)
1669
1670         install_linux_headers(policy.linux_policy, cmdlist)
1671
1672         cmdlist.create_use_dir(policy.builddir)
1673         policy.configure(cmdlist)
1674         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1675         cmdlist.cleanup_dir()
1676         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1677                                       cmdlist, logsdir)
1678
1679 class Command(object):
1680     """A command run in the build process."""
1681
1682     def __init__(self, desc, num, dir, path, command, always_run=False):
1683         """Initialize a Command object."""
1684         self.dir = dir
1685         self.path = path
1686         self.desc = desc
1687         trans = str.maketrans({' ': '-'})
1688         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1689         self.command = command
1690         self.always_run = always_run
1691
1692     @staticmethod
1693     def shell_make_quote_string(s):
1694         """Given a string not containing a newline, quote it for use by the
1695         shell and make."""
1696         assert '\n' not in s
1697         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1698             return s
1699         strans = str.maketrans({"'": "'\\''"})
1700         s = "'%s'" % s.translate(strans)
1701         mtrans = str.maketrans({'$': '$$'})
1702         return s.translate(mtrans)
1703
1704     @staticmethod
1705     def shell_make_quote_list(l, translate_make):
1706         """Given a list of strings not containing newlines, quote them for use
1707         by the shell and make, returning a single string.  If translate_make
1708         is true and the first string is 'make', change it to $(MAKE)."""
1709         l = [Command.shell_make_quote_string(s) for s in l]
1710         if translate_make and l[0] == 'make':
1711             l[0] = '$(MAKE)'
1712         return ' '.join(l)
1713
1714     def shell_make_quote(self):
1715         """Return this command quoted for the shell and make."""
1716         return self.shell_make_quote_list(self.command, True)
1717
1718
1719 class CommandList(object):
1720     """A list of commands run in the build process."""
1721
1722     def __init__(self, desc, keep):
1723         """Initialize a CommandList object."""
1724         self.cmdlist = []
1725         self.dir = None
1726         self.path = None
1727         self.desc = [desc]
1728         self.keep = keep
1729
1730     def desc_txt(self, desc):
1731         """Return the description to use for a command."""
1732         return '%s %s' % (' '.join(self.desc), desc)
1733
1734     def use_dir(self, dir):
1735         """Set the default directory for subsequent commands."""
1736         self.dir = dir
1737
1738     def use_path(self, path):
1739         """Set a directory to be prepended to the PATH for subsequent
1740         commands."""
1741         self.path = path
1742
1743     def push_subdesc(self, subdesc):
1744         """Set the default subdescription for subsequent commands (e.g., the
1745         name of a component being built, within the series of commands
1746         building it)."""
1747         self.desc.append(subdesc)
1748
1749     def pop_subdesc(self):
1750         """Pop a subdescription from the list of descriptions."""
1751         self.desc.pop()
1752
1753     def create_use_dir(self, dir):
1754         """Remove and recreate a directory and use it for subsequent
1755         commands."""
1756         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1757         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1758         self.use_dir(dir)
1759
1760     def add_command_dir(self, desc, dir, command, always_run=False):
1761         """Add a command to run in a given directory."""
1762         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1763                       command, always_run)
1764         self.cmdlist.append(cmd)
1765
1766     def add_command(self, desc, command, always_run=False):
1767         """Add a command to run in the default directory."""
1768         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1769                       self.path, command, always_run)
1770         self.cmdlist.append(cmd)
1771
1772     def cleanup_dir(self, desc='cleanup', dir=None):
1773         """Clean up a build directory.  If no directory is specified, the
1774         default directory is cleaned up and ceases to be the default
1775         directory."""
1776         if dir is None:
1777             dir = self.dir
1778             self.use_dir(None)
1779         if self.keep != 'all':
1780             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1781                                  always_run=(self.keep == 'none'))
1782
1783     def makefile_commands(self, wrapper, logsdir):
1784         """Return the sequence of commands in the form of text for a Makefile.
1785         The given wrapper script takes arguments: base of logs for
1786         previous command, or empty; base of logs for this command;
1787         description; directory; PATH addition; the command itself."""
1788         # prev_base is the base of the name for logs of the previous
1789         # command that is not always-run (that is, a build command,
1790         # whose failure should stop subsequent build commands from
1791         # being run, as opposed to a cleanup command, which is run
1792         # even if previous commands failed).
1793         prev_base = ''
1794         cmds = []
1795         for c in self.cmdlist:
1796             ctxt = c.shell_make_quote()
1797             if prev_base and not c.always_run:
1798                 prev_log = os.path.join(logsdir, prev_base)
1799             else:
1800                 prev_log = ''
1801             this_log = os.path.join(logsdir, c.logbase)
1802             if not c.always_run:
1803                 prev_base = c.logbase
1804             if c.dir is None:
1805                 dir = ''
1806             else:
1807                 dir = c.dir
1808             if c.path is None:
1809                 path = ''
1810             else:
1811                 path = c.path
1812             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1813             prelim_txt = Command.shell_make_quote_list(prelims, False)
1814             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1815         return '\n'.join(cmds)
1816
1817     def status_logs(self, logsdir):
1818         """Return the list of log files with command status."""
1819         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1820                 for c in self.cmdlist]
1821
1822
1823 def get_parser():
1824     """Return an argument parser for this module."""
1825     parser = argparse.ArgumentParser(description=__doc__)
1826     parser.add_argument('-j', dest='parallelism',
1827                         help='Run this number of jobs in parallel',
1828                         type=int, default=os.cpu_count())
1829     parser.add_argument('--keep', dest='keep',
1830                         help='Whether to keep all build directories, '
1831                         'none or only those from failed builds',
1832                         default='none', choices=('none', 'all', 'failed'))
1833     parser.add_argument('--replace-sources', action='store_true',
1834                         help='Remove and replace source directories '
1835                         'with the wrong version of a component')
1836     parser.add_argument('--strip', action='store_true',
1837                         help='Strip installed glibc libraries')
1838     parser.add_argument('--full-gcc', action='store_true',
1839                         help='Build GCC with all languages and libsanitizer')
1840     parser.add_argument('--shallow', action='store_true',
1841                         help='Do not download Git history during checkout')
1842     parser.add_argument('topdir',
1843                         help='Toplevel working directory')
1844     parser.add_argument('action',
1845                         help='What to do',
1846                         choices=('checkout', 'bot-cycle', 'bot',
1847                                  'host-libraries', 'compilers', 'glibcs',
1848                                  'update-syscalls', 'list-compilers',
1849                                  'list-glibcs'))
1850     parser.add_argument('configs',
1851                         help='Versions to check out or configurations to build',
1852                         nargs='*')
1853     return parser
1854
1855
1856 def main(argv):
1857     """The main entry point."""
1858     parser = get_parser()
1859     opts = parser.parse_args(argv)
1860     topdir = os.path.abspath(opts.topdir)
1861     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1862                   opts.strip, opts.full_gcc, opts.action,
1863                   shallow=opts.shallow)
1864     ctx.run_builds(opts.action, opts.configs)
1865
1866
1867 if __name__ == '__main__':
1868     main(sys.argv[1:])