scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2020 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <https://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 The 'list-compilers' command prints the name of each available
  36 compiler configuration, without building anything.  The 'list-glibcs'
  37 command prints the name of each glibc compiler configuration, followed
  38 by the space, followed by the name of the compiler configuration used
  39 for building this glibc variant.
  40
  41 """
  42
  43 import argparse
  44 import datetime
  45 import email.mime.text
  46 import email.utils
  47 import json
  48 import os
  49 import re
  50 import shutil
  51 import smtplib
  52 import stat
  53 import subprocess
  54 import sys
  55 import time
  56 import urllib.request
  57
  58 try:
  59     subprocess.run
  60 except:
  61     class _CompletedProcess:
  62         def __init__(self, args, returncode, stdout=None, stderr=None):
  63             self.args = args
  64             self.returncode = returncode
  65             self.stdout = stdout
  66             self.stderr = stderr
  67
  68     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  69         assert(timeout is None)
  70         with subprocess.Popen(*popenargs, **kwargs) as process:
  71             try:
  72                 stdout, stderr = process.communicate(input)
  73             except:
  74                 process.kill()
  75                 process.wait()
  76                 raise
  77             returncode = process.poll()
  78             if check and returncode:
  79                 raise subprocess.CalledProcessError(returncode, popenargs)
  80         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  81
  82     subprocess.run = _run
  83
  84
  85 class Context(object):
  86     """The global state associated with builds in a given directory."""
  87
  88     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  89                  full_gcc, action, shallow=False):
  90         """Initialize the context."""
  91         self.topdir = topdir
  92         self.parallelism = parallelism
  93         self.keep = keep
  94         self.replace_sources = replace_sources
  95         self.strip = strip
  96         self.full_gcc = full_gcc
  97         self.shallow = shallow
  98         self.srcdir = os.path.join(topdir, 'src')
  99         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 100         self.build_state_json = os.path.join(topdir, 'build-state.json')
 101         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 102         self.installdir = os.path.join(topdir, 'install')
 103         self.host_libraries_installdir = os.path.join(self.installdir,
 104                                                       'host-libraries')
 105         self.builddir = os.path.join(topdir, 'build')
 106         self.logsdir = os.path.join(topdir, 'logs')
 107         self.logsdir_old = os.path.join(topdir, 'logs-old')
 108         self.makefile = os.path.join(self.builddir, 'Makefile')
 109         self.wrapper = os.path.join(self.builddir, 'wrapper')
 110         self.save_logs = os.path.join(self.builddir, 'save-logs')
 111         self.script_text = self.get_script_text()
 112         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 113             self.build_triplet = self.get_build_triplet()
 114             self.glibc_version = self.get_glibc_version()
 115         self.configs = {}
 116         self.glibc_configs = {}
 117         self.makefile_pieces = ['.PHONY: all\n']
 118         self.add_all_configs()
 119         self.load_versions_json()
 120         self.load_build_state_json()
 121         self.status_log_list = []
 122         self.email_warning = False
 123
 124     def get_script_text(self):
 125         """Return the text of this script."""
 126         with open(sys.argv[0], 'r') as f:
 127             return f.read()
 128
 129     def exec_self(self):
 130         """Re-execute this script with the same arguments."""
 131         sys.stdout.flush()
 132         os.execv(sys.executable, [sys.executable] + sys.argv)
 133
 134     def get_build_triplet(self):
 135         """Determine the build triplet with config.guess."""
 136         config_guess = os.path.join(self.component_srcdir('gcc'),
 137                                     'config.guess')
 138         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 139                                 check=True, universal_newlines=True).stdout
 140         return cg_out.rstrip()
 141
 142     def get_glibc_version(self):
 143         """Determine the glibc version number (major.minor)."""
 144         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 145         with open(version_h, 'r') as f:
 146             lines = f.readlines()
 147         starttext = '#define VERSION "'
 148         for l in lines:
 149             if l.startswith(starttext):
 150                 l = l[len(starttext):]
 151                 l = l.rstrip('"\n')
 152                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 153                 return '%s.%s' % m.group(1, 2)
 154         print('error: could not determine glibc version')
 155         exit(1)
 156
 157     def add_all_configs(self):
 158         """Add all known glibc build configurations."""
 159         self.add_config(arch='aarch64',
 160                         os_name='linux-gnu',
 161                         extra_glibcs=[{'variant': 'disable-multi-arch',
 162                                        'cfg': ['--disable-multi-arch']}])
 163         self.add_config(arch='aarch64_be',
 164                         os_name='linux-gnu')
 165         self.add_config(arch='alpha',
 166                         os_name='linux-gnu')
 167         self.add_config(arch='arm',
 168                         os_name='linux-gnueabi',
 169                         extra_glibcs=[{'variant': 'v4t',
 170                                        'ccopts': '-march=armv4t'}])
 171         self.add_config(arch='armeb',
 172                         os_name='linux-gnueabi')
 173         self.add_config(arch='armeb',
 174                         os_name='linux-gnueabi',
 175                         variant='be8',
 176                         gcc_cfg=['--with-arch=armv7-a'])
 177         self.add_config(arch='arm',
 178                         os_name='linux-gnueabihf',
 179                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 180                         extra_glibcs=[{'variant': 'v7a',
 181                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 182                                       {'variant': 'v7a-disable-multi-arch',
 183                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 184                                        'cfg': ['--disable-multi-arch']}])
 185         self.add_config(arch='armeb',
 186                         os_name='linux-gnueabihf',
 187                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 188         self.add_config(arch='armeb',
 189                         os_name='linux-gnueabihf',
 190                         variant='be8',
 191                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 192                                  '--with-fpu=vfpv3'])
 193         self.add_config(arch='csky',
 194                         os_name='linux-gnuabiv2',
 195                         variant='soft',
 196                         gcc_cfg=['--disable-multilib'])
 197         self.add_config(arch='csky',
 198                         os_name='linux-gnuabiv2',
 199                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 200         self.add_config(arch='hppa',
 201                         os_name='linux-gnu')
 202         self.add_config(arch='i686',
 203                         os_name='gnu')
 204         self.add_config(arch='ia64',
 205                         os_name='linux-gnu',
 206                         first_gcc_cfg=['--with-system-libunwind'])
 207         self.add_config(arch='m68k',
 208                         os_name='linux-gnu',
 209                         gcc_cfg=['--disable-multilib'])
 210         self.add_config(arch='m68k',
 211                         os_name='linux-gnu',
 212                         variant='coldfire',
 213                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 214         self.add_config(arch='m68k',
 215                         os_name='linux-gnu',
 216                         variant='coldfire-soft',
 217                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 218                                  '--disable-multilib'])
 219         self.add_config(arch='microblaze',
 220                         os_name='linux-gnu',
 221                         gcc_cfg=['--disable-multilib'])
 222         self.add_config(arch='microblazeel',
 223                         os_name='linux-gnu',
 224                         gcc_cfg=['--disable-multilib'])
 225         self.add_config(arch='mips64',
 226                         os_name='linux-gnu',
 227                         gcc_cfg=['--with-mips-plt'],
 228                         glibcs=[{'variant': 'n32'},
 229                                 {'arch': 'mips',
 230                                  'ccopts': '-mabi=32'},
 231                                 {'variant': 'n64',
 232                                  'ccopts': '-mabi=64'}])
 233         self.add_config(arch='mips64',
 234                         os_name='linux-gnu',
 235                         variant='soft',
 236                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 237                         glibcs=[{'variant': 'n32-soft'},
 238                                 {'variant': 'soft',
 239                                  'arch': 'mips',
 240                                  'ccopts': '-mabi=32'},
 241                                 {'variant': 'n64-soft',
 242                                  'ccopts': '-mabi=64'}])
 243         self.add_config(arch='mips64',
 244                         os_name='linux-gnu',
 245                         variant='nan2008',
 246                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 247                                  '--with-arch-64=mips64r2',
 248                                  '--with-arch-32=mips32r2'],
 249                         glibcs=[{'variant': 'n32-nan2008'},
 250                                 {'variant': 'nan2008',
 251                                  'arch': 'mips',
 252                                  'ccopts': '-mabi=32'},
 253                                 {'variant': 'n64-nan2008',
 254                                  'ccopts': '-mabi=64'}])
 255         self.add_config(arch='mips64',
 256                         os_name='linux-gnu',
 257                         variant='nan2008-soft',
 258                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 259                                  '--with-arch-64=mips64r2',
 260                                  '--with-arch-32=mips32r2',
 261                                  '--with-float=soft'],
 262                         glibcs=[{'variant': 'n32-nan2008-soft'},
 263                                 {'variant': 'nan2008-soft',
 264                                  'arch': 'mips',
 265                                  'ccopts': '-mabi=32'},
 266                                 {'variant': 'n64-nan2008-soft',
 267                                  'ccopts': '-mabi=64'}])
 268         self.add_config(arch='mips64el',
 269                         os_name='linux-gnu',
 270                         gcc_cfg=['--with-mips-plt'],
 271                         glibcs=[{'variant': 'n32'},
 272                                 {'arch': 'mipsel',
 273                                  'ccopts': '-mabi=32'},
 274                                 {'variant': 'n64',
 275                                  'ccopts': '-mabi=64'}])
 276         self.add_config(arch='mips64el',
 277                         os_name='linux-gnu',
 278                         variant='soft',
 279                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 280                         glibcs=[{'variant': 'n32-soft'},
 281                                 {'variant': 'soft',
 282                                  'arch': 'mipsel',
 283                                  'ccopts': '-mabi=32'},
 284                                 {'variant': 'n64-soft',
 285                                  'ccopts': '-mabi=64'}])
 286         self.add_config(arch='mips64el',
 287                         os_name='linux-gnu',
 288                         variant='nan2008',
 289                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 290                                  '--with-arch-64=mips64r2',
 291                                  '--with-arch-32=mips32r2'],
 292                         glibcs=[{'variant': 'n32-nan2008'},
 293                                 {'variant': 'nan2008',
 294                                  'arch': 'mipsel',
 295                                  'ccopts': '-mabi=32'},
 296                                 {'variant': 'n64-nan2008',
 297                                  'ccopts': '-mabi=64'}])
 298         self.add_config(arch='mips64el',
 299                         os_name='linux-gnu',
 300                         variant='nan2008-soft',
 301                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 302                                  '--with-arch-64=mips64r2',
 303                                  '--with-arch-32=mips32r2',
 304                                  '--with-float=soft'],
 305                         glibcs=[{'variant': 'n32-nan2008-soft'},
 306                                 {'variant': 'nan2008-soft',
 307                                  'arch': 'mipsel',
 308                                  'ccopts': '-mabi=32'},
 309                                 {'variant': 'n64-nan2008-soft',
 310                                  'ccopts': '-mabi=64'}])
 311         self.add_config(arch='mipsisa64r6el',
 312                         os_name='linux-gnu',
 313                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 314                                  '--with-arch-64=mips64r6',
 315                                  '--with-arch-32=mips32r6',
 316                                  '--with-float=hard'],
 317                         glibcs=[{'variant': 'n32'},
 318                                 {'arch': 'mipsisa32r6el',
 319                                  'ccopts': '-mabi=32'},
 320                                 {'variant': 'n64',
 321                                  'ccopts': '-mabi=64'}])
 322         self.add_config(arch='nios2',
 323                         os_name='linux-gnu')
 324         self.add_config(arch='powerpc',
 325                         os_name='linux-gnu',
 326                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 327                         extra_glibcs=[{'variant': 'power4',
 328                                        'ccopts': '-mcpu=power4',
 329                                        'cfg': ['--with-cpu=power4']}])
 330         self.add_config(arch='powerpc',
 331                         os_name='linux-gnu',
 332                         variant='soft',
 333                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 334                                  '--enable-secureplt'])
 335         self.add_config(arch='powerpc64',
 336                         os_name='linux-gnu',
 337                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 338         self.add_config(arch='powerpc64le',
 339                         os_name='linux-gnu',
 340                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 341         self.add_config(arch='riscv64',
 342                         os_name='linux-gnu',
 343                         variant='rv64imac-lp64',
 344                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 345                                  '--disable-multilib'])
 346         self.add_config(arch='riscv64',
 347                         os_name='linux-gnu',
 348                         variant='rv64imafdc-lp64',
 349                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 350                                  '--disable-multilib'])
 351         self.add_config(arch='riscv64',
 352                         os_name='linux-gnu',
 353                         variant='rv64imafdc-lp64d',
 354                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 355                                  '--disable-multilib'])
 356         self.add_config(arch='s390x',
 357                         os_name='linux-gnu',
 358                         glibcs=[{},
 359                                 {'arch': 's390', 'ccopts': '-m31'}])
 360         self.add_config(arch='sh3',
 361                         os_name='linux-gnu')
 362         self.add_config(arch='sh3eb',
 363                         os_name='linux-gnu')
 364         self.add_config(arch='sh4',
 365                         os_name='linux-gnu')
 366         self.add_config(arch='sh4eb',
 367                         os_name='linux-gnu')
 368         self.add_config(arch='sh4',
 369                         os_name='linux-gnu',
 370                         variant='soft',
 371                         gcc_cfg=['--without-fp'])
 372         self.add_config(arch='sh4eb',
 373                         os_name='linux-gnu',
 374                         variant='soft',
 375                         gcc_cfg=['--without-fp'])
 376         self.add_config(arch='sparc64',
 377                         os_name='linux-gnu',
 378                         glibcs=[{},
 379                                 {'arch': 'sparcv9',
 380                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 381                         extra_glibcs=[{'variant': 'leon3',
 382                                        'arch' : 'sparcv8',
 383                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 384                                       {'variant': 'disable-multi-arch',
 385                                        'cfg': ['--disable-multi-arch']},
 386                                       {'variant': 'disable-multi-arch',
 387                                        'arch': 'sparcv9',
 388                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 389                                        'cfg': ['--disable-multi-arch']}])
 390         self.add_config(arch='x86_64',
 391                         os_name='linux-gnu',
 392                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 393                         glibcs=[{},
 394                                 {'variant': 'x32', 'ccopts': '-mx32'},
 395                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 396                         extra_glibcs=[{'variant': 'disable-multi-arch',
 397                                        'cfg': ['--disable-multi-arch']},
 398                                       {'variant': 'enable-obsolete',
 399                                        'cfg': ['--enable-obsolete-rpc',
 400                                                '--enable-obsolete-nsl']},
 401                                       {'variant': 'static-pie',
 402                                        'cfg': ['--enable-static-pie']},
 403                                       {'variant': 'x32-static-pie',
 404                                        'ccopts': '-mx32',
 405                                        'cfg': ['--enable-static-pie']},
 406                                       {'variant': 'static-pie',
 407                                        'arch': 'i686',
 408                                        'ccopts': '-m32 -march=i686',
 409                                        'cfg': ['--enable-static-pie']},
 410                                       {'variant': 'disable-multi-arch',
 411                                        'arch': 'i686',
 412                                        'ccopts': '-m32 -march=i686',
 413                                        'cfg': ['--disable-multi-arch']},
 414                                       {'variant': 'enable-obsolete',
 415                                        'arch': 'i686',
 416                                        'ccopts': '-m32 -march=i686',
 417                                        'cfg': ['--enable-obsolete-rpc',
 418                                                '--enable-obsolete-nsl']},
 419                                       {'arch': 'i486',
 420                                        'ccopts': '-m32 -march=i486'},
 421                                       {'arch': 'i586',
 422                                        'ccopts': '-m32 -march=i586'}])
 423
 424     def add_config(self, **args):
 425         """Add an individual build configuration."""
 426         cfg = Config(self, **args)
 427         if cfg.name in self.configs:
 428             print('error: duplicate config %s' % cfg.name)
 429             exit(1)
 430         self.configs[cfg.name] = cfg
 431         for c in cfg.all_glibcs:
 432             if c.name in self.glibc_configs:
 433                 print('error: duplicate glibc config %s' % c.name)
 434                 exit(1)
 435             self.glibc_configs[c.name] = c
 436
 437     def component_srcdir(self, component):
 438         """Return the source directory for a given component, e.g. gcc."""
 439         return os.path.join(self.srcdir, component)
 440
 441     def component_builddir(self, action, config, component, subconfig=None):
 442         """Return the directory to use for a build."""
 443         if config is None:
 444             # Host libraries.
 445             assert subconfig is None
 446             return os.path.join(self.builddir, action, component)
 447         if subconfig is None:
 448             return os.path.join(self.builddir, action, config, component)
 449         else:
 450             # glibc build as part of compiler build.
 451             return os.path.join(self.builddir, action, config, component,
 452                                 subconfig)
 453
 454     def compiler_installdir(self, config):
 455         """Return the directory in which to install a compiler."""
 456         return os.path.join(self.installdir, 'compilers', config)
 457
 458     def compiler_bindir(self, config):
 459         """Return the directory in which to find compiler binaries."""
 460         return os.path.join(self.compiler_installdir(config), 'bin')
 461
 462     def compiler_sysroot(self, config):
 463         """Return the sysroot directory for a compiler."""
 464         return os.path.join(self.compiler_installdir(config), 'sysroot')
 465
 466     def glibc_installdir(self, config):
 467         """Return the directory in which to install glibc."""
 468         return os.path.join(self.installdir, 'glibcs', config)
 469
 470     def run_builds(self, action, configs):
 471         """Run the requested builds."""
 472         if action == 'checkout':
 473             self.checkout(configs)
 474             return
 475         if action == 'bot-cycle':
 476             if configs:
 477                 print('error: configurations specified for bot-cycle')
 478                 exit(1)
 479             self.bot_cycle()
 480             return
 481         if action == 'bot':
 482             if configs:
 483                 print('error: configurations specified for bot')
 484                 exit(1)
 485             self.bot()
 486             return
 487         if action in ('host-libraries', 'list-compilers',
 488                       'list-glibcs') and configs:
 489             print('error: configurations specified for ' + action)
 490             exit(1)
 491         if action == 'list-compilers':
 492             for name in sorted(self.configs.keys()):
 493                 print(name)
 494             return
 495         if action == 'list-glibcs':
 496             for config in sorted(self.glibc_configs.values(),
 497                                  key=lambda c: c.name):
 498                 print(config.name, config.compiler.name)
 499             return
 500         self.clear_last_build_state(action)
 501         build_time = datetime.datetime.utcnow()
 502         if action == 'host-libraries':
 503             build_components = ('gmp', 'mpfr', 'mpc')
 504             old_components = ()
 505             old_versions = {}
 506             self.build_host_libraries()
 507         elif action == 'compilers':
 508             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 509                                 'gnumach', 'hurd')
 510             old_components = ('gmp', 'mpfr', 'mpc')
 511             old_versions = self.build_state['host-libraries']['build-versions']
 512             self.build_compilers(configs)
 513         else:
 514             build_components = ('glibc',)
 515             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 516                               'mig', 'gnumach', 'hurd')
 517             old_versions = self.build_state['compilers']['build-versions']
 518             if action == 'update-syscalls':
 519                 self.update_syscalls(configs)
 520             else:
 521                 self.build_glibcs(configs)
 522         self.write_files()
 523         self.do_build()
 524         if configs:
 525             # Partial build, do not update stored state.
 526             return
 527         build_versions = {}
 528         for k in build_components:
 529             if k in self.versions:
 530                 build_versions[k] = {'version': self.versions[k]['version'],
 531                                      'revision': self.versions[k]['revision']}
 532         for k in old_components:
 533             if k in old_versions:
 534                 build_versions[k] = {'version': old_versions[k]['version'],
 535                                      'revision': old_versions[k]['revision']}
 536         self.update_build_state(action, build_time, build_versions)
 537
 538     @staticmethod
 539     def remove_dirs(*args):
 540         """Remove directories and their contents if they exist."""
 541         for dir in args:
 542             shutil.rmtree(dir, ignore_errors=True)
 543
 544     @staticmethod
 545     def remove_recreate_dirs(*args):
 546         """Remove directories if they exist, and create them as empty."""
 547         Context.remove_dirs(*args)
 548         for dir in args:
 549             os.makedirs(dir, exist_ok=True)
 550
 551     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 552         """Add makefile text for a list of commands."""
 553         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 554         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 555                                     (target, target, target, commands))
 556         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 557
 558     def write_files(self):
 559         """Write out the Makefile and wrapper script."""
 560         mftext = ''.join(self.makefile_pieces)
 561         with open(self.makefile, 'w') as f:
 562             f.write(mftext)
 563         wrapper_text = (
 564             '#!/bin/sh\n'
 565             'prev_base=$1\n'
 566             'this_base=$2\n'
 567             'desc=$3\n'
 568             'dir=$4\n'
 569             'path=$5\n'
 570             'shift 5\n'
 571             'prev_status=$prev_base-status.txt\n'
 572             'this_status=$this_base-status.txt\n'
 573             'this_log=$this_base-log.txt\n'
 574             'date > "$this_log"\n'
 575             'echo >> "$this_log"\n'
 576             'echo "Description: $desc" >> "$this_log"\n'
 577             'printf "%s" "Command:" >> "$this_log"\n'
 578             'for word in "$@"; do\n'
 579             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 580             '    printf " %s" "$word"\n'
 581             '  else\n'
 582             '    printf " \'"\n'
 583             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 584             '    printf "\'"\n'
 585             '  fi\n'
 586             'done >> "$this_log"\n'
 587             'echo >> "$this_log"\n'
 588             'echo "Directory: $dir" >> "$this_log"\n'
 589             'echo "Path addition: $path" >> "$this_log"\n'
 590             'echo >> "$this_log"\n'
 591             'record_status ()\n'
 592             '{\n'
 593             '  echo >> "$this_log"\n'
 594             '  echo "$1: $desc" > "$this_status"\n'
 595             '  echo "$1: $desc" >> "$this_log"\n'
 596             '  echo >> "$this_log"\n'
 597             '  date >> "$this_log"\n'
 598             '  echo "$1: $desc"\n'
 599             '  exit 0\n'
 600             '}\n'
 601             'check_error ()\n'
 602             '{\n'
 603             '  if [ "$1" != "0" ]; then\n'
 604             '    record_status FAIL\n'
 605             '  fi\n'
 606             '}\n'
 607             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 608             '    record_status UNRESOLVED\n'
 609             'fi\n'
 610             'if [ "$dir" ]; then\n'
 611             '  cd "$dir"\n'
 612             '  check_error "$?"\n'
 613             'fi\n'
 614             'if [ "$path" ]; then\n'
 615             '  PATH=$path:$PATH\n'
 616             'fi\n'
 617             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 618             'check_error "$?"\n'
 619             'record_status PASS\n')
 620         with open(self.wrapper, 'w') as f:
 621             f.write(wrapper_text)
 622         # Mode 0o755.
 623         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 624                      stat.S_IROTH|stat.S_IXOTH)
 625         os.chmod(self.wrapper, mode_exec)
 626         save_logs_text = (
 627             '#!/bin/sh\n'
 628             'if ! [ -f tests.sum ]; then\n'
 629             '  echo "No test summary available."\n'
 630             '  exit 0\n'
 631             'fi\n'
 632             'save_file ()\n'
 633             '{\n'
 634             '  echo "Contents of $1:"\n'
 635             '  echo\n'
 636             '  cat "$1"\n'
 637             '  echo\n'
 638             '  echo "End of contents of $1."\n'
 639             '  echo\n'
 640             '}\n'
 641             'save_file tests.sum\n'
 642             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 643             'for t in $non_pass_tests; do\n'
 644             '  if [ -f "$t.out" ]; then\n'
 645             '    save_file "$t.out"\n'
 646             '  fi\n'
 647             'done\n')
 648         with open(self.save_logs, 'w') as f:
 649             f.write(save_logs_text)
 650         os.chmod(self.save_logs, mode_exec)
 651
 652     def do_build(self):
 653         """Do the actual build."""
 654         cmd = ['make', '-j%d' % self.parallelism]
 655         subprocess.run(cmd, cwd=self.builddir, check=True)
 656
 657     def build_host_libraries(self):
 658         """Build the host libraries."""
 659         installdir = self.host_libraries_installdir
 660         builddir = os.path.join(self.builddir, 'host-libraries')
 661         logsdir = os.path.join(self.logsdir, 'host-libraries')
 662         self.remove_recreate_dirs(installdir, builddir, logsdir)
 663         cmdlist = CommandList('host-libraries', self.keep)
 664         self.build_host_library(cmdlist, 'gmp')
 665         self.build_host_library(cmdlist, 'mpfr',
 666                                 ['--with-gmp=%s' % installdir])
 667         self.build_host_library(cmdlist, 'mpc',
 668                                 ['--with-gmp=%s' % installdir,
 669                                 '--with-mpfr=%s' % installdir])
 670         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 671         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 672
 673     def build_host_library(self, cmdlist, lib, extra_opts=None):
 674         """Build one host library."""
 675         srcdir = self.component_srcdir(lib)
 676         builddir = self.component_builddir('host-libraries', None, lib)
 677         installdir = self.host_libraries_installdir
 678         cmdlist.push_subdesc(lib)
 679         cmdlist.create_use_dir(builddir)
 680         cfg_cmd = [os.path.join(srcdir, 'configure'),
 681                    '--prefix=%s' % installdir,
 682                    '--disable-shared']
 683         if extra_opts:
 684             cfg_cmd.extend (extra_opts)
 685         cmdlist.add_command('configure', cfg_cmd)
 686         cmdlist.add_command('build', ['make'])
 687         cmdlist.add_command('check', ['make', 'check'])
 688         cmdlist.add_command('install', ['make', 'install'])
 689         cmdlist.cleanup_dir()
 690         cmdlist.pop_subdesc()
 691
 692     def build_compilers(self, configs):
 693         """Build the compilers."""
 694         if not configs:
 695             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 696             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 697             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 698             configs = sorted(self.configs.keys())
 699         for c in configs:
 700             self.configs[c].build()
 701
 702     def build_glibcs(self, configs):
 703         """Build the glibcs."""
 704         if not configs:
 705             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 706             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 707             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 708             configs = sorted(self.glibc_configs.keys())
 709         for c in configs:
 710             self.glibc_configs[c].build()
 711
 712     def update_syscalls(self, configs):
 713         """Update the glibc syscall lists."""
 714         if not configs:
 715             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 716             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 717             configs = sorted(self.glibc_configs.keys())
 718         for c in configs:
 719             self.glibc_configs[c].update_syscalls()
 720
 721     def load_versions_json(self):
 722         """Load information about source directory versions."""
 723         if not os.access(self.versions_json, os.F_OK):
 724             self.versions = {}
 725             return
 726         with open(self.versions_json, 'r') as f:
 727             self.versions = json.load(f)
 728
 729     def store_json(self, data, filename):
 730         """Store information in a JSON file."""
 731         filename_tmp = filename + '.tmp'
 732         with open(filename_tmp, 'w') as f:
 733             json.dump(data, f, indent=2, sort_keys=True)
 734         os.rename(filename_tmp, filename)
 735
 736     def store_versions_json(self):
 737         """Store information about source directory versions."""
 738         self.store_json(self.versions, self.versions_json)
 739
 740     def set_component_version(self, component, version, explicit, revision):
 741         """Set the version information for a component."""
 742         self.versions[component] = {'version': version,
 743                                     'explicit': explicit,
 744                                     'revision': revision}
 745         self.store_versions_json()
 746
 747     def checkout(self, versions):
 748         """Check out the desired component versions."""
 749         default_versions = {'binutils': 'vcs-2.34',
 750                             'gcc': 'vcs-9',
 751                             'glibc': 'vcs-mainline',
 752                             'gmp': '6.1.2',
 753                             'linux': '5.5',
 754                             'mpc': '1.1.0',
 755                             'mpfr': '4.0.2',
 756                             'mig': 'vcs-mainline',
 757                             'gnumach': 'vcs-mainline',
 758                             'hurd': 'vcs-mainline'}
 759         use_versions = {}
 760         explicit_versions = {}
 761         for v in versions:
 762             found_v = False
 763             for k in default_versions.keys():
 764                 kx = k + '-'
 765                 if v.startswith(kx):
 766                     vx = v[len(kx):]
 767                     if k in use_versions:
 768                         print('error: multiple versions for %s' % k)
 769                         exit(1)
 770                     use_versions[k] = vx
 771                     explicit_versions[k] = True
 772                     found_v = True
 773                     break
 774             if not found_v:
 775                 print('error: unknown component in %s' % v)
 776                 exit(1)
 777         for k in default_versions.keys():
 778             if k not in use_versions:
 779                 if k in self.versions and self.versions[k]['explicit']:
 780                     use_versions[k] = self.versions[k]['version']
 781                     explicit_versions[k] = True
 782                 else:
 783                     use_versions[k] = default_versions[k]
 784                     explicit_versions[k] = False
 785         os.makedirs(self.srcdir, exist_ok=True)
 786         for k in sorted(default_versions.keys()):
 787             update = os.access(self.component_srcdir(k), os.F_OK)
 788             v = use_versions[k]
 789             if (update and
 790                 k in self.versions and
 791                 v != self.versions[k]['version']):
 792                 if not self.replace_sources:
 793                     print('error: version of %s has changed from %s to %s, '
 794                           'use --replace-sources to check out again' %
 795                           (k, self.versions[k]['version'], v))
 796                     exit(1)
 797                 shutil.rmtree(self.component_srcdir(k))
 798                 update = False
 799             if v.startswith('vcs-'):
 800                 revision = self.checkout_vcs(k, v[4:], update)
 801             else:
 802                 self.checkout_tar(k, v, update)
 803                 revision = v
 804             self.set_component_version(k, v, explicit_versions[k], revision)
 805         if self.get_script_text() != self.script_text:
 806             # Rerun the checkout process in case the updated script
 807             # uses different default versions or new components.
 808             self.exec_self()
 809
 810     def checkout_vcs(self, component, version, update):
 811         """Check out the given version of the given component from version
 812         control.  Return a revision identifier."""
 813         if component == 'binutils':
 814             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 815             if version == 'mainline':
 816                 git_branch = 'master'
 817             else:
 818                 trans = str.maketrans({'.': '_'})
 819                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 820             return self.git_checkout(component, git_url, git_branch, update)
 821         elif component == 'gcc':
 822             if version == 'mainline':
 823                 branch = 'master'
 824             else:
 825                 branch = 'releases/gcc-%s' % version
 826             return self.gcc_checkout(branch, update)
 827         elif component == 'glibc':
 828             git_url = 'git://sourceware.org/git/glibc.git'
 829             if version == 'mainline':
 830                 git_branch = 'master'
 831             else:
 832                 git_branch = 'release/%s/master' % version
 833             r = self.git_checkout(component, git_url, git_branch, update)
 834             self.fix_glibc_timestamps()
 835             return r
 836         elif component == 'gnumach':
 837             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 838             git_branch = 'master'
 839             r = self.git_checkout(component, git_url, git_branch, update)
 840             subprocess.run(['autoreconf', '-i'],
 841                            cwd=self.component_srcdir(component), check=True)
 842             return r
 843         elif component == 'mig':
 844             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 845             git_branch = 'master'
 846             r = self.git_checkout(component, git_url, git_branch, update)
 847             subprocess.run(['autoreconf', '-i'],
 848                            cwd=self.component_srcdir(component), check=True)
 849             return r
 850         elif component == 'hurd':
 851             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 852             git_branch = 'master'
 853             r = self.git_checkout(component, git_url, git_branch, update)
 854             subprocess.run(['autoconf'],
 855                            cwd=self.component_srcdir(component), check=True)
 856             return r
 857         else:
 858             print('error: component %s coming from VCS' % component)
 859             exit(1)
 860
 861     def git_checkout(self, component, git_url, git_branch, update):
 862         """Check out a component from git.  Return a commit identifier."""
 863         if update:
 864             subprocess.run(['git', 'remote', 'prune', 'origin'],
 865                            cwd=self.component_srcdir(component), check=True)
 866             if self.replace_sources:
 867                 subprocess.run(['git', 'clean', '-dxfq'],
 868                                cwd=self.component_srcdir(component), check=True)
 869             subprocess.run(['git', 'pull', '-q'],
 870                            cwd=self.component_srcdir(component), check=True)
 871         else:
 872             if self.shallow:
 873                 depth_arg = ('--depth', '1')
 874             else:
 875                 depth_arg = ()
 876             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 877                             *depth_arg, git_url,
 878                             self.component_srcdir(component)], check=True)
 879         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 880                            cwd=self.component_srcdir(component),
 881                            stdout=subprocess.PIPE,
 882                            check=True, universal_newlines=True).stdout
 883         return r.rstrip()
 884
 885     def fix_glibc_timestamps(self):
 886         """Fix timestamps in a glibc checkout."""
 887         # Ensure that builds do not try to regenerate generated files
 888         # in the source tree.
 889         srcdir = self.component_srcdir('glibc')
 890         # These files have Makefile dependencies to regenerate them in
 891         # the source tree that may be active during a normal build.
 892         # Some other files have such dependencies but do not need to
 893         # be touched because nothing in a build depends on the files
 894         # in question.
 895         for f in ('sysdeps/gnu/errlist.c',
 896                   'sysdeps/mach/hurd/bits/errno.h'):
 897             to_touch = os.path.join(srcdir, f)
 898             subprocess.run(['touch', '-c', to_touch], check=True)
 899         for dirpath, dirnames, filenames in os.walk(srcdir):
 900             for f in filenames:
 901                 if (f == 'configure' or
 902                     f == 'preconfigure' or
 903                     f.endswith('-kw.h')):
 904                     to_touch = os.path.join(dirpath, f)
 905                     subprocess.run(['touch', to_touch], check=True)
 906
 907     def gcc_checkout(self, branch, update):
 908         """Check out GCC from git.  Return the commit identifier."""
 909         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 910                      os.F_OK):
 911             if not self.replace_sources:
 912                 print('error: GCC has moved from SVN to git, use '
 913                       '--replace-sources to check out again')
 914                 exit(1)
 915             shutil.rmtree(self.component_srcdir('gcc'))
 916             update = False
 917         if not update:
 918             self.git_checkout('gcc', 'git://gcc.gnu.org/git/gcc.git',
 919                               branch, update)
 920         subprocess.run(['contrib/gcc_update', '--silent'],
 921                        cwd=self.component_srcdir('gcc'), check=True)
 922         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 923                            cwd=self.component_srcdir('gcc'),
 924                            stdout=subprocess.PIPE,
 925                            check=True, universal_newlines=True).stdout
 926         return r.rstrip()
 927
 928     def checkout_tar(self, component, version, update):
 929         """Check out the given version of the given component from a
 930         tarball."""
 931         if update:
 932             return
 933         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 934                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
 935                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 936                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
 937                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 938                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 939                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 940                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 941                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 942         if component not in url_map:
 943             print('error: component %s coming from tarball' % component)
 944             exit(1)
 945         version_major = version.split('.')[0]
 946         url = url_map[component] % {'version': version, 'major': version_major}
 947         filename = os.path.join(self.srcdir, url.split('/')[-1])
 948         response = urllib.request.urlopen(url)
 949         data = response.read()
 950         with open(filename, 'wb') as f:
 951             f.write(data)
 952         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 953                        check=True)
 954         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 955                   self.component_srcdir(component))
 956         os.remove(filename)
 957
 958     def load_build_state_json(self):
 959         """Load information about the state of previous builds."""
 960         if os.access(self.build_state_json, os.F_OK):
 961             with open(self.build_state_json, 'r') as f:
 962                 self.build_state = json.load(f)
 963         else:
 964             self.build_state = {}
 965         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
 966             if k not in self.build_state:
 967                 self.build_state[k] = {}
 968             if 'build-time' not in self.build_state[k]:
 969                 self.build_state[k]['build-time'] = ''
 970             if 'build-versions' not in self.build_state[k]:
 971                 self.build_state[k]['build-versions'] = {}
 972             if 'build-results' not in self.build_state[k]:
 973                 self.build_state[k]['build-results'] = {}
 974             if 'result-changes' not in self.build_state[k]:
 975                 self.build_state[k]['result-changes'] = {}
 976             if 'ever-passed' not in self.build_state[k]:
 977                 self.build_state[k]['ever-passed'] = []
 978
 979     def store_build_state_json(self):
 980         """Store information about the state of previous builds."""
 981         self.store_json(self.build_state, self.build_state_json)
 982
 983     def clear_last_build_state(self, action):
 984         """Clear information about the state of part of the build."""
 985         # We clear the last build time and versions when starting a
 986         # new build.  The results of the last build are kept around,
 987         # as comparison is still meaningful if this build is aborted
 988         # and a new one started.
 989         self.build_state[action]['build-time'] = ''
 990         self.build_state[action]['build-versions'] = {}
 991         self.store_build_state_json()
 992
 993     def update_build_state(self, action, build_time, build_versions):
 994         """Update the build state after a build."""
 995         build_time = build_time.replace(microsecond=0)
 996         self.build_state[action]['build-time'] = str(build_time)
 997         self.build_state[action]['build-versions'] = build_versions
 998         build_results = {}
 999         for log in self.status_log_list:
1000             with open(log, 'r') as f:
1001                 log_text = f.read()
1002             log_text = log_text.rstrip()
1003             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1004             result = m.group(1)
1005             test_name = m.group(2)
1006             assert test_name not in build_results
1007             build_results[test_name] = result
1008         old_build_results = self.build_state[action]['build-results']
1009         self.build_state[action]['build-results'] = build_results
1010         result_changes = {}
1011         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1012         for t in all_tests:
1013             if t in old_build_results:
1014                 old_res = old_build_results[t]
1015             else:
1016                 old_res = '(New test)'
1017             if t in build_results:
1018                 new_res = build_results[t]
1019             else:
1020                 new_res = '(Test removed)'
1021             if old_res != new_res:
1022                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1023         self.build_state[action]['result-changes'] = result_changes
1024         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1025                            if t in build_results}
1026         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1027         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1028                                                          new_passes)
1029         self.store_build_state_json()
1030
1031     def load_bot_config_json(self):
1032         """Load bot configuration."""
1033         with open(self.bot_config_json, 'r') as f:
1034             self.bot_config = json.load(f)
1035
1036     def part_build_old(self, action, delay):
1037         """Return whether the last build for a given action was at least a
1038         given number of seconds ago, or does not have a time recorded."""
1039         old_time_str = self.build_state[action]['build-time']
1040         if not old_time_str:
1041             return True
1042         old_time = datetime.datetime.strptime(old_time_str,
1043                                               '%Y-%m-%d %H:%M:%S')
1044         new_time = datetime.datetime.utcnow()
1045         delta = new_time - old_time
1046         return delta.total_seconds() >= delay
1047
1048     def bot_cycle(self):
1049         """Run a single round of checkout and builds."""
1050         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1051         self.load_bot_config_json()
1052         actions = ('host-libraries', 'compilers', 'glibcs')
1053         self.bot_run_self(['--replace-sources'], 'checkout')
1054         self.load_versions_json()
1055         if self.get_script_text() != self.script_text:
1056             print('Script changed, re-execing.')
1057             # On script change, all parts of the build should be rerun.
1058             for a in actions:
1059                 self.clear_last_build_state(a)
1060             self.exec_self()
1061         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1062                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1063                                           'mig', 'gnumach', 'hurd'),
1064                             'glibcs': ('glibc',)}
1065         must_build = {}
1066         for a in actions:
1067             build_vers = self.build_state[a]['build-versions']
1068             must_build[a] = False
1069             if not self.build_state[a]['build-time']:
1070                 must_build[a] = True
1071             old_vers = {}
1072             new_vers = {}
1073             for c in check_components[a]:
1074                 if c in build_vers:
1075                     old_vers[c] = build_vers[c]
1076                 new_vers[c] = {'version': self.versions[c]['version'],
1077                                'revision': self.versions[c]['revision']}
1078             if new_vers == old_vers:
1079                 print('Versions for %s unchanged.' % a)
1080             else:
1081                 print('Versions changed or rebuild forced for %s.' % a)
1082                 if a == 'compilers' and not self.part_build_old(
1083                         a, self.bot_config['compilers-rebuild-delay']):
1084                     print('Not requiring rebuild of compilers this soon.')
1085                 else:
1086                     must_build[a] = True
1087         if must_build['host-libraries']:
1088             must_build['compilers'] = True
1089         if must_build['compilers']:
1090             must_build['glibcs'] = True
1091         for a in actions:
1092             if must_build[a]:
1093                 print('Must rebuild %s.' % a)
1094                 self.clear_last_build_state(a)
1095             else:
1096                 print('No need to rebuild %s.' % a)
1097         if os.access(self.logsdir, os.F_OK):
1098             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1099             shutil.copytree(self.logsdir, self.logsdir_old)
1100         for a in actions:
1101             if must_build[a]:
1102                 build_time = datetime.datetime.utcnow()
1103                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1104                 self.bot_run_self([], a)
1105                 self.load_build_state_json()
1106                 self.bot_build_mail(a, build_time)
1107         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1108
1109     def bot_build_mail(self, action, build_time):
1110         """Send email with the results of a build."""
1111         if not ('email-from' in self.bot_config and
1112                 'email-server' in self.bot_config and
1113                 'email-subject' in self.bot_config and
1114                 'email-to' in self.bot_config):
1115             if not self.email_warning:
1116                 print("Email not configured, not sending.")
1117                 self.email_warning = True
1118             return
1119
1120         build_time = build_time.replace(microsecond=0)
1121         subject = (self.bot_config['email-subject'] %
1122                    {'action': action,
1123                     'build-time': str(build_time)})
1124         results = self.build_state[action]['build-results']
1125         changes = self.build_state[action]['result-changes']
1126         ever_passed = set(self.build_state[action]['ever-passed'])
1127         versions = self.build_state[action]['build-versions']
1128         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1129         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1130         all_fails = {k for k in results if results[k] == 'FAIL'}
1131         if new_regressions:
1132             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1133             new_reg_text = ('New regressions:\n\n%s\n\n' %
1134                             '\n'.join(new_reg_list))
1135         else:
1136             new_reg_text = ''
1137         if all_regressions:
1138             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1139             all_reg_text = ('All regressions:\n\n%s\n\n' %
1140                             '\n'.join(all_reg_list))
1141         else:
1142             all_reg_text = ''
1143         if all_fails:
1144             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1145             all_fail_text = ('All failures:\n\n%s\n\n' %
1146                              '\n'.join(all_fail_list))
1147         else:
1148             all_fail_text = ''
1149         if changes:
1150             changes_list = sorted(changes.keys())
1151             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1152             changes_text = ('All changed results:\n\n%s\n\n' %
1153                             '\n'.join(changes_list))
1154         else:
1155             changes_text = ''
1156         results_text = (new_reg_text + all_reg_text + all_fail_text +
1157                         changes_text)
1158         if not results_text:
1159             results_text = 'Clean build with unchanged results.\n\n'
1160         versions_list = sorted(versions.keys())
1161         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1162                                           versions[k]['revision'])
1163                          for k in versions_list]
1164         versions_text = ('Component versions for this build:\n\n%s\n' %
1165                          '\n'.join(versions_list))
1166         body_text = results_text + versions_text
1167         msg = email.mime.text.MIMEText(body_text)
1168         msg['Subject'] = subject
1169         msg['From'] = self.bot_config['email-from']
1170         msg['To'] = self.bot_config['email-to']
1171         msg['Message-ID'] = email.utils.make_msgid()
1172         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1173         with smtplib.SMTP(self.bot_config['email-server']) as s:
1174             s.send_message(msg)
1175
1176     def bot_run_self(self, opts, action, check=True):
1177         """Run a copy of this script with given options."""
1178         cmd = [sys.executable, sys.argv[0], '--keep=none',
1179                '-j%d' % self.parallelism]
1180         if self.full_gcc:
1181             cmd.append('--full-gcc')
1182         cmd.extend(opts)
1183         cmd.extend([self.topdir, action])
1184         sys.stdout.flush()
1185         subprocess.run(cmd, check=check)
1186
1187     def bot(self):
1188         """Run repeated rounds of checkout and builds."""
1189         while True:
1190             self.load_bot_config_json()
1191             if not self.bot_config['run']:
1192                 print('Bot exiting by request.')
1193                 exit(0)
1194             self.bot_run_self([], 'bot-cycle', check=False)
1195             self.load_bot_config_json()
1196             if not self.bot_config['run']:
1197                 print('Bot exiting by request.')
1198                 exit(0)
1199             time.sleep(self.bot_config['delay'])
1200             if self.get_script_text() != self.script_text:
1201                 print('Script changed, bot re-execing.')
1202                 self.exec_self()
1203
1204 class LinuxHeadersPolicyForBuild(object):
1205     """Names and directories for installing Linux headers.  Build variant."""
1206
1207     def __init__(self, config):
1208         self.arch = config.arch
1209         self.srcdir = config.ctx.component_srcdir('linux')
1210         self.builddir = config.component_builddir('linux')
1211         self.headers_dir = os.path.join(config.sysroot, 'usr')
1212
1213 class LinuxHeadersPolicyForUpdateSyscalls(object):
1214     """Names and directories for Linux headers.  update-syscalls variant."""
1215
1216     def __init__(self, glibc, headers_dir):
1217         self.arch = glibc.compiler.arch
1218         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1219         self.builddir = glibc.ctx.component_builddir(
1220             'update-syscalls', glibc.name, 'build-linux')
1221         self.headers_dir = headers_dir
1222
1223 def install_linux_headers(policy, cmdlist):
1224     """Install Linux kernel headers."""
1225     arch_map = {'aarch64': 'arm64',
1226                 'alpha': 'alpha',
1227                 'arm': 'arm',
1228                 'csky': 'csky',
1229                 'hppa': 'parisc',
1230                 'i486': 'x86',
1231                 'i586': 'x86',
1232                 'i686': 'x86',
1233                 'i786': 'x86',
1234                 'ia64': 'ia64',
1235                 'm68k': 'm68k',
1236                 'microblaze': 'microblaze',
1237                 'mips': 'mips',
1238                 'nios2': 'nios2',
1239                 'powerpc': 'powerpc',
1240                 's390': 's390',
1241                 'riscv32': 'riscv',
1242                 'riscv64': 'riscv',
1243                 'sh': 'sh',
1244                 'sparc': 'sparc',
1245                 'x86_64': 'x86'}
1246     linux_arch = None
1247     for k in arch_map:
1248         if policy.arch.startswith(k):
1249             linux_arch = arch_map[k]
1250             break
1251     assert linux_arch is not None
1252     cmdlist.push_subdesc('linux')
1253     cmdlist.create_use_dir(policy.builddir)
1254     cmdlist.add_command('install-headers',
1255                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1256                          'ARCH=%s' % linux_arch,
1257                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1258                          'headers_install'])
1259     cmdlist.cleanup_dir()
1260     cmdlist.pop_subdesc()
1261
1262 class Config(object):
1263     """A configuration for building a compiler and associated libraries."""
1264
1265     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1266                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1267         """Initialize a Config object."""
1268         self.ctx = ctx
1269         self.arch = arch
1270         self.os = os_name
1271         self.variant = variant
1272         if variant is None:
1273             self.name = '%s-%s' % (arch, os_name)
1274         else:
1275             self.name = '%s-%s-%s' % (arch, os_name, variant)
1276         self.triplet = '%s-glibc-%s' % (arch, os_name)
1277         if gcc_cfg is None:
1278             self.gcc_cfg = []
1279         else:
1280             self.gcc_cfg = gcc_cfg
1281         if first_gcc_cfg is None:
1282             self.first_gcc_cfg = []
1283         else:
1284             self.first_gcc_cfg = first_gcc_cfg
1285         if glibcs is None:
1286             glibcs = [{'variant': variant}]
1287         if extra_glibcs is None:
1288             extra_glibcs = []
1289         glibcs = [Glibc(self, **g) for g in glibcs]
1290         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1291         self.all_glibcs = glibcs + extra_glibcs
1292         self.compiler_glibcs = glibcs
1293         self.installdir = ctx.compiler_installdir(self.name)
1294         self.bindir = ctx.compiler_bindir(self.name)
1295         self.sysroot = ctx.compiler_sysroot(self.name)
1296         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1297         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1298
1299     def component_builddir(self, component):
1300         """Return the directory to use for a (non-glibc) build."""
1301         return self.ctx.component_builddir('compilers', self.name, component)
1302
1303     def build(self):
1304         """Generate commands to build this compiler."""
1305         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1306                                       self.logsdir)
1307         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1308         cmdlist.add_command('check-host-libraries',
1309                             ['test', '-f',
1310                              os.path.join(self.ctx.host_libraries_installdir,
1311                                           'ok')])
1312         cmdlist.use_path(self.bindir)
1313         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1314                               ['--disable-gdb',
1315                                '--disable-gdbserver',
1316                                '--disable-libdecnumber',
1317                                '--disable-readline',
1318                                '--disable-sim'])
1319         if self.os.startswith('linux'):
1320             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1321         self.build_gcc(cmdlist, True)
1322         if self.os == 'gnu':
1323             self.install_gnumach_headers(cmdlist)
1324             self.build_cross_tool(cmdlist, 'mig', 'mig')
1325             self.install_hurd_headers(cmdlist)
1326         for g in self.compiler_glibcs:
1327             cmdlist.push_subdesc('glibc')
1328             cmdlist.push_subdesc(g.name)
1329             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1330             cmdlist.pop_subdesc()
1331             cmdlist.pop_subdesc()
1332         self.build_gcc(cmdlist, False)
1333         cmdlist.add_command('done', ['touch',
1334                                      os.path.join(self.installdir, 'ok')])
1335         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1336                                       self.logsdir)
1337
1338     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1339         """Build one cross tool."""
1340         srcdir = self.ctx.component_srcdir(tool_src)
1341         builddir = self.component_builddir(tool_build)
1342         cmdlist.push_subdesc(tool_build)
1343         cmdlist.create_use_dir(builddir)
1344         cfg_cmd = [os.path.join(srcdir, 'configure'),
1345                    '--prefix=%s' % self.installdir,
1346                    '--build=%s' % self.ctx.build_triplet,
1347                    '--host=%s' % self.ctx.build_triplet,
1348                    '--target=%s' % self.triplet,
1349                    '--with-sysroot=%s' % self.sysroot]
1350         if extra_opts:
1351             cfg_cmd.extend(extra_opts)
1352         cmdlist.add_command('configure', cfg_cmd)
1353         cmdlist.add_command('build', ['make'])
1354         # Parallel "make install" for GCC has race conditions that can
1355         # cause it to fail; see
1356         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1357         # problems are not known for binutils, but doing the
1358         # installation in parallel within a particular toolchain build
1359         # (as opposed to installation of one toolchain from
1360         # build-many-glibcs.py running in parallel to the installation
1361         # of other toolchains being built) is not known to be
1362         # significantly beneficial, so it is simplest just to disable
1363         # parallel install for cross tools here.
1364         cmdlist.add_command('install', ['make', '-j1', 'install'])
1365         cmdlist.cleanup_dir()
1366         cmdlist.pop_subdesc()
1367
1368     def install_gnumach_headers(self, cmdlist):
1369         """Install GNU Mach headers."""
1370         srcdir = self.ctx.component_srcdir('gnumach')
1371         builddir = self.component_builddir('gnumach')
1372         cmdlist.push_subdesc('gnumach')
1373         cmdlist.create_use_dir(builddir)
1374         cmdlist.add_command('configure',
1375                             [os.path.join(srcdir, 'configure'),
1376                              '--build=%s' % self.ctx.build_triplet,
1377                              '--host=%s' % self.triplet,
1378                              '--prefix=',
1379                              'CC=%s-gcc -nostdlib' % self.triplet])
1380         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1381                                         'install-data'])
1382         cmdlist.cleanup_dir()
1383         cmdlist.pop_subdesc()
1384
1385     def install_hurd_headers(self, cmdlist):
1386         """Install Hurd headers."""
1387         srcdir = self.ctx.component_srcdir('hurd')
1388         builddir = self.component_builddir('hurd')
1389         cmdlist.push_subdesc('hurd')
1390         cmdlist.create_use_dir(builddir)
1391         cmdlist.add_command('configure',
1392                             [os.path.join(srcdir, 'configure'),
1393                              '--build=%s' % self.ctx.build_triplet,
1394                              '--host=%s' % self.triplet,
1395                              '--prefix=',
1396                              '--disable-profile', '--without-parted',
1397                              'CC=%s-gcc -nostdlib' % self.triplet])
1398         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1399                                         'no_deps=t', 'install-headers'])
1400         cmdlist.cleanup_dir()
1401         cmdlist.pop_subdesc()
1402
1403     def build_gcc(self, cmdlist, bootstrap):
1404         """Build GCC."""
1405         # libssp is of little relevance with glibc's own stack
1406         # checking support.  libcilkrts does not support GNU/Hurd (and
1407         # has been removed in GCC 8, so --disable-libcilkrts can be
1408         # removed once glibc no longer supports building with older
1409         # GCC versions).
1410         cfg_opts = list(self.gcc_cfg)
1411         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1412         host_libs = self.ctx.host_libraries_installdir
1413         cfg_opts += ['--with-gmp=%s' % host_libs,
1414                      '--with-mpfr=%s' % host_libs,
1415                      '--with-mpc=%s' % host_libs]
1416         if bootstrap:
1417             tool_build = 'gcc-first'
1418             # Building a static-only, C-only compiler that is
1419             # sufficient to build glibc.  Various libraries and
1420             # features that may require libc headers must be disabled.
1421             # When configuring with a sysroot, --with-newlib is
1422             # required to define inhibit_libc (to stop some parts of
1423             # libgcc including libc headers); --without-headers is not
1424             # sufficient.
1425             cfg_opts += ['--enable-languages=c', '--disable-shared',
1426                          '--disable-threads',
1427                          '--disable-libatomic',
1428                          '--disable-decimal-float',
1429                          '--disable-libffi',
1430                          '--disable-libgomp',
1431                          '--disable-libitm',
1432                          '--disable-libmpx',
1433                          '--disable-libquadmath',
1434                          '--disable-libsanitizer',
1435                          '--without-headers', '--with-newlib',
1436                          '--with-glibc-version=%s' % self.ctx.glibc_version
1437                          ]
1438             cfg_opts += self.first_gcc_cfg
1439         else:
1440             tool_build = 'gcc'
1441             # libsanitizer commonly breaks because of glibc header
1442             # changes, or on unusual targets.  C++ pre-compiled
1443             # headers are not used during the glibc build and are
1444             # expensive to create.
1445             if not self.ctx.full_gcc:
1446                 cfg_opts += ['--disable-libsanitizer',
1447                              '--disable-libstdcxx-pch']
1448             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1449             cfg_opts += ['--enable-languages=%s' % langs,
1450                          '--enable-shared', '--enable-threads']
1451         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1452
1453 class GlibcPolicyDefault(object):
1454     """Build policy for glibc: common defaults."""
1455
1456     def __init__(self, glibc):
1457         self.srcdir = glibc.ctx.component_srcdir('glibc')
1458         self.use_usr = glibc.os != 'gnu'
1459         self.prefix = '/usr' if self.use_usr else ''
1460         self.configure_args = [
1461             '--prefix=%s' % self.prefix,
1462             '--enable-profile',
1463             '--build=%s' % glibc.ctx.build_triplet,
1464             '--host=%s' % glibc.triplet,
1465             'CC=%s' % glibc.tool_name('gcc'),
1466             'CXX=%s' % glibc.tool_name('g++'),
1467             'AR=%s' % glibc.tool_name('ar'),
1468             'AS=%s' % glibc.tool_name('as'),
1469             'LD=%s' % glibc.tool_name('ld'),
1470             'NM=%s' % glibc.tool_name('nm'),
1471             'OBJCOPY=%s' % glibc.tool_name('objcopy'),
1472             'OBJDUMP=%s' % glibc.tool_name('objdump'),
1473             'RANLIB=%s' % glibc.tool_name('ranlib'),
1474             'READELF=%s' % glibc.tool_name('readelf'),
1475             'STRIP=%s' % glibc.tool_name('strip'),
1476         ]
1477         if glibc.os == 'gnu':
1478             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1479         self.configure_args += glibc.cfg
1480
1481     def configure(self, cmdlist):
1482         """Invoked to add the configure command to the command list."""
1483         cmdlist.add_command('configure',
1484                             [os.path.join(self.srcdir, 'configure'),
1485                              *self.configure_args])
1486
1487     def extra_commands(self, cmdlist):
1488         """Invoked to inject additional commands (make check) after build."""
1489         pass
1490
1491 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1492     """Build policy for glibc during the compilers stage."""
1493
1494     def __init__(self, glibc):
1495         super().__init__(glibc)
1496         self.builddir = glibc.ctx.component_builddir(
1497             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1498         self.installdir = glibc.compiler.sysroot
1499
1500 class GlibcPolicyForBuild(GlibcPolicyDefault):
1501     """Build policy for glibc during the glibcs stage."""
1502
1503     def __init__(self, glibc):
1504         super().__init__(glibc)
1505         self.builddir = glibc.ctx.component_builddir(
1506             'glibcs', glibc.name, 'glibc')
1507         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1508         if glibc.ctx.strip:
1509             self.strip = glibc.tool_name('strip')
1510         else:
1511             self.strip = None
1512         self.save_logs = glibc.ctx.save_logs
1513
1514     def extra_commands(self, cmdlist):
1515         if self.strip:
1516             # Avoid picking up libc.so and libpthread.so, which are
1517             # linker scripts stored in /lib on Hurd.  libc and
1518             # libpthread are still stripped via their libc-X.YY.so
1519             # implementation files.
1520             find_command = (('find %s/lib* -name "*.so"'
1521                              + r' \! -name libc.so \! -name libpthread.so')
1522                             % self.installdir)
1523             cmdlist.add_command('strip', ['sh', '-c', ('%s $(%s)' %
1524                                   (self.strip, find_command))])
1525         cmdlist.add_command('check', ['make', 'check'])
1526         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1527
1528 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1529     """Build policy for glibc during update-syscalls."""
1530
1531     def __init__(self, glibc):
1532         super().__init__(glibc)
1533         self.builddir = glibc.ctx.component_builddir(
1534             'update-syscalls', glibc.name, 'glibc')
1535         self.linuxdir = glibc.ctx.component_builddir(
1536             'update-syscalls', glibc.name, 'linux')
1537         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1538             glibc, self.linuxdir)
1539         self.configure_args.insert(
1540             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1541         # self.installdir not set because installation is not supported
1542
1543 class Glibc(object):
1544     """A configuration for building glibc."""
1545
1546     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1547                  cfg=None, ccopts=None):
1548         """Initialize a Glibc object."""
1549         self.ctx = compiler.ctx
1550         self.compiler = compiler
1551         if arch is None:
1552             self.arch = compiler.arch
1553         else:
1554             self.arch = arch
1555         if os_name is None:
1556             self.os = compiler.os
1557         else:
1558             self.os = os_name
1559         self.variant = variant
1560         if variant is None:
1561             self.name = '%s-%s' % (self.arch, self.os)
1562         else:
1563             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1564         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1565         if cfg is None:
1566             self.cfg = []
1567         else:
1568             self.cfg = cfg
1569         self.ccopts = ccopts
1570
1571     def tool_name(self, tool):
1572         """Return the name of a cross-compilation tool."""
1573         ctool = '%s-%s' % (self.compiler.triplet, tool)
1574         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1575             ctool = '%s %s' % (ctool, self.ccopts)
1576         return ctool
1577
1578     def build(self):
1579         """Generate commands to build this glibc."""
1580         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1581         installdir = self.ctx.glibc_installdir(self.name)
1582         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1583         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1584         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1585         cmdlist.add_command('check-compilers',
1586                             ['test', '-f',
1587                              os.path.join(self.compiler.installdir, 'ok')])
1588         cmdlist.use_path(self.compiler.bindir)
1589         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1590         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1591                                       logsdir)
1592
1593     def build_glibc(self, cmdlist, policy):
1594         """Generate commands to build this glibc, either as part of a compiler
1595         build or with the bootstrapped compiler (and in the latter case, run
1596         tests as well)."""
1597         cmdlist.create_use_dir(policy.builddir)
1598         policy.configure(cmdlist)
1599         cmdlist.add_command('build', ['make'])
1600         cmdlist.add_command('install', ['make', 'install',
1601                                         'install_root=%s' % policy.installdir])
1602         # GCC uses paths such as lib/../lib64, so make sure lib
1603         # directories always exist.
1604         mkdir_cmd = ['mkdir', '-p',
1605                      os.path.join(policy.installdir, 'lib')]
1606         if policy.use_usr:
1607             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1608         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1609         policy.extra_commands(cmdlist)
1610         cmdlist.cleanup_dir()
1611
1612     def update_syscalls(self):
1613         if self.os == 'gnu':
1614             # Hurd does not have system call tables that need updating.
1615             return
1616
1617         policy = GlibcPolicyForUpdateSyscalls(self)
1618         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1619         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1620         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1621         cmdlist.add_command('check-compilers',
1622                             ['test', '-f',
1623                              os.path.join(self.compiler.installdir, 'ok')])
1624         cmdlist.use_path(self.compiler.bindir)
1625
1626         install_linux_headers(policy.linux_policy, cmdlist)
1627
1628         cmdlist.create_use_dir(policy.builddir)
1629         policy.configure(cmdlist)
1630         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1631         cmdlist.cleanup_dir()
1632         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1633                                       cmdlist, logsdir)
1634
1635 class Command(object):
1636     """A command run in the build process."""
1637
1638     def __init__(self, desc, num, dir, path, command, always_run=False):
1639         """Initialize a Command object."""
1640         self.dir = dir
1641         self.path = path
1642         self.desc = desc
1643         trans = str.maketrans({' ': '-'})
1644         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1645         self.command = command
1646         self.always_run = always_run
1647
1648     @staticmethod
1649     def shell_make_quote_string(s):
1650         """Given a string not containing a newline, quote it for use by the
1651         shell and make."""
1652         assert '\n' not in s
1653         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1654             return s
1655         strans = str.maketrans({"'": "'\\''"})
1656         s = "'%s'" % s.translate(strans)
1657         mtrans = str.maketrans({'$': '$$'})
1658         return s.translate(mtrans)
1659
1660     @staticmethod
1661     def shell_make_quote_list(l, translate_make):
1662         """Given a list of strings not containing newlines, quote them for use
1663         by the shell and make, returning a single string.  If translate_make
1664         is true and the first string is 'make', change it to $(MAKE)."""
1665         l = [Command.shell_make_quote_string(s) for s in l]
1666         if translate_make and l[0] == 'make':
1667             l[0] = '$(MAKE)'
1668         return ' '.join(l)
1669
1670     def shell_make_quote(self):
1671         """Return this command quoted for the shell and make."""
1672         return self.shell_make_quote_list(self.command, True)
1673
1674
1675 class CommandList(object):
1676     """A list of commands run in the build process."""
1677
1678     def __init__(self, desc, keep):
1679         """Initialize a CommandList object."""
1680         self.cmdlist = []
1681         self.dir = None
1682         self.path = None
1683         self.desc = [desc]
1684         self.keep = keep
1685
1686     def desc_txt(self, desc):
1687         """Return the description to use for a command."""
1688         return '%s %s' % (' '.join(self.desc), desc)
1689
1690     def use_dir(self, dir):
1691         """Set the default directory for subsequent commands."""
1692         self.dir = dir
1693
1694     def use_path(self, path):
1695         """Set a directory to be prepended to the PATH for subsequent
1696         commands."""
1697         self.path = path
1698
1699     def push_subdesc(self, subdesc):
1700         """Set the default subdescription for subsequent commands (e.g., the
1701         name of a component being built, within the series of commands
1702         building it)."""
1703         self.desc.append(subdesc)
1704
1705     def pop_subdesc(self):
1706         """Pop a subdescription from the list of descriptions."""
1707         self.desc.pop()
1708
1709     def create_use_dir(self, dir):
1710         """Remove and recreate a directory and use it for subsequent
1711         commands."""
1712         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1713         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1714         self.use_dir(dir)
1715
1716     def add_command_dir(self, desc, dir, command, always_run=False):
1717         """Add a command to run in a given directory."""
1718         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1719                       command, always_run)
1720         self.cmdlist.append(cmd)
1721
1722     def add_command(self, desc, command, always_run=False):
1723         """Add a command to run in the default directory."""
1724         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1725                       self.path, command, always_run)
1726         self.cmdlist.append(cmd)
1727
1728     def cleanup_dir(self, desc='cleanup', dir=None):
1729         """Clean up a build directory.  If no directory is specified, the
1730         default directory is cleaned up and ceases to be the default
1731         directory."""
1732         if dir is None:
1733             dir = self.dir
1734             self.use_dir(None)
1735         if self.keep != 'all':
1736             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1737                                  always_run=(self.keep == 'none'))
1738
1739     def makefile_commands(self, wrapper, logsdir):
1740         """Return the sequence of commands in the form of text for a Makefile.
1741         The given wrapper script takes arguments: base of logs for
1742         previous command, or empty; base of logs for this command;
1743         description; directory; PATH addition; the command itself."""
1744         # prev_base is the base of the name for logs of the previous
1745         # command that is not always-run (that is, a build command,
1746         # whose failure should stop subsequent build commands from
1747         # being run, as opposed to a cleanup command, which is run
1748         # even if previous commands failed).
1749         prev_base = ''
1750         cmds = []
1751         for c in self.cmdlist:
1752             ctxt = c.shell_make_quote()
1753             if prev_base and not c.always_run:
1754                 prev_log = os.path.join(logsdir, prev_base)
1755             else:
1756                 prev_log = ''
1757             this_log = os.path.join(logsdir, c.logbase)
1758             if not c.always_run:
1759                 prev_base = c.logbase
1760             if c.dir is None:
1761                 dir = ''
1762             else:
1763                 dir = c.dir
1764             if c.path is None:
1765                 path = ''
1766             else:
1767                 path = c.path
1768             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1769             prelim_txt = Command.shell_make_quote_list(prelims, False)
1770             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1771         return '\n'.join(cmds)
1772
1773     def status_logs(self, logsdir):
1774         """Return the list of log files with command status."""
1775         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1776                 for c in self.cmdlist]
1777
1778
1779 def get_parser():
1780     """Return an argument parser for this module."""
1781     parser = argparse.ArgumentParser(description=__doc__)
1782     parser.add_argument('-j', dest='parallelism',
1783                         help='Run this number of jobs in parallel',
1784                         type=int, default=os.cpu_count())
1785     parser.add_argument('--keep', dest='keep',
1786                         help='Whether to keep all build directories, '
1787                         'none or only those from failed builds',
1788                         default='none', choices=('none', 'all', 'failed'))
1789     parser.add_argument('--replace-sources', action='store_true',
1790                         help='Remove and replace source directories '
1791                         'with the wrong version of a component')
1792     parser.add_argument('--strip', action='store_true',
1793                         help='Strip installed glibc libraries')
1794     parser.add_argument('--full-gcc', action='store_true',
1795                         help='Build GCC with all languages and libsanitizer')
1796     parser.add_argument('--shallow', action='store_true',
1797                         help='Do not download Git history during checkout')
1798     parser.add_argument('topdir',
1799                         help='Toplevel working directory')
1800     parser.add_argument('action',
1801                         help='What to do',
1802                         choices=('checkout', 'bot-cycle', 'bot',
1803                                  'host-libraries', 'compilers', 'glibcs',
1804                                  'update-syscalls', 'list-compilers',
1805                                  'list-glibcs'))
1806     parser.add_argument('configs',
1807                         help='Versions to check out or configurations to build',
1808                         nargs='*')
1809     return parser
1810
1811
1812 def main(argv):
1813     """The main entry point."""
1814     parser = get_parser()
1815     opts = parser.parse_args(argv)
1816     topdir = os.path.abspath(opts.topdir)
1817     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1818                   opts.strip, opts.full_gcc, opts.action,
1819                   shallow=opts.shallow)
1820     ctx.run_builds(opts.action, opts.configs)
1821
1822
1823 if __name__ == '__main__':
1824     main(sys.argv[1:])