2 # Build many configurations of glibc.
3 # Copyright (C) 2016-2024 Free Software Foundation, Inc.
4 # Copyright The GNU Toolchain Authors.
5 # This file is part of the GNU C Library.
7 # The GNU C Library is free software; you can redistribute it and/or
8 # modify it under the terms of the GNU Lesser General Public
9 # License as published by the Free Software Foundation; either
10 # version 2.1 of the License, or (at your option) any later version.
12 # The GNU C Library is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 # Lesser General Public License for more details.
17 # You should have received a copy of the GNU Lesser General Public
18 # License along with the GNU C Library; if not, see
19 # <https://www.gnu.org/licenses/>.
21 """Build many configurations of glibc.
23 This script takes as arguments a directory name (containing a src
24 subdirectory with sources of the relevant toolchain components) and a
25 description of what to do: 'checkout', to check out sources into that
26 directory, 'bot-cycle', to run a series of checkout and build steps,
27 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
28 libraries required by the toolchain, 'compilers', to build
29 cross-compilers for various configurations, or 'glibcs', to build
30 glibc for various configurations and run the compilation parts of the
31 testsuite. Subsequent arguments name the versions of components to
32 check out (<component>-<version), for 'checkout', or, for actions
33 other than 'checkout' and 'bot-cycle', name configurations for which
34 compilers or glibc are to be built.
36 The 'list-compilers' command prints the name of each available
37 compiler configuration, without building anything. The 'list-glibcs'
38 command prints the name of each glibc compiler configuration, followed
39 by the space, followed by the name of the compiler configuration used
40 for building this glibc variant.
46 import email
.mime
.text
59 # This is a list of system utilities that are expected to be available
60 # to this script, and, if a non-zero version is included, the minimum
61 # version required to work with this sccript.
62 def get_list_of_required_tools():
65 'awk' : (get_version_awk
, (0,0,0)),
66 'bison' : (get_version
, (0,0)),
67 'flex' : (get_version
, (0,0,0)),
68 'git' : (get_version
, (1,8,3)),
69 'make' : (get_version
, (4,0)),
70 'makeinfo' : (get_version
, (0,0)),
71 'patch' : (get_version
, (0,0,0)),
72 'sed' : (get_version
, (0,0)),
73 'tar' : (get_version
, (0,0,0)),
74 'gzip' : (get_version
, (0,0)),
75 'bzip2' : (get_version_bzip2
, (0,0,0)),
76 'xz' : (get_version
, (0,0,0)),
82 class _CompletedProcess
:
83 def __init__(self
, args
, returncode
, stdout
=None, stderr
=None):
85 self
.returncode
= returncode
89 def _run(*popenargs
, input=None, timeout
=None, check
=False, **kwargs
):
90 assert(timeout
is None)
91 with subprocess
.Popen(*popenargs
, **kwargs
) as process
:
93 stdout
, stderr
= process
.communicate(input)
98 returncode
= process
.poll()
99 if check
and returncode
:
100 raise subprocess
.CalledProcessError(returncode
, popenargs
)
101 return _CompletedProcess(popenargs
, returncode
, stdout
, stderr
)
103 subprocess
.run
= _run
106 class Context(object):
107 """The global state associated with builds in a given directory."""
109 def __init__(self
, topdir
, parallelism
, keep
, replace_sources
, strip
,
110 full_gcc
, action
, shallow
=False):
111 """Initialize the context."""
113 self
.parallelism
= parallelism
115 self
.replace_sources
= replace_sources
117 self
.full_gcc
= full_gcc
118 self
.shallow
= shallow
119 self
.srcdir
= os
.path
.join(topdir
, 'src')
120 self
.versions_json
= os
.path
.join(self
.srcdir
, 'versions.json')
121 self
.build_state_json
= os
.path
.join(topdir
, 'build-state.json')
122 self
.bot_config_json
= os
.path
.join(topdir
, 'bot-config.json')
123 self
.installdir
= os
.path
.join(topdir
, 'install')
124 self
.host_libraries_installdir
= os
.path
.join(self
.installdir
,
126 self
.builddir
= os
.path
.join(topdir
, 'build')
127 self
.logsdir
= os
.path
.join(topdir
, 'logs')
128 self
.logsdir_old
= os
.path
.join(topdir
, 'logs-old')
129 self
.makefile
= os
.path
.join(self
.builddir
, 'Makefile')
130 self
.wrapper
= os
.path
.join(self
.builddir
, 'wrapper')
131 self
.save_logs
= os
.path
.join(self
.builddir
, 'save-logs')
132 self
.script_text
= self
.get_script_text()
133 if action
not in ('checkout', 'list-compilers', 'list-glibcs'):
134 self
.build_triplet
= self
.get_build_triplet()
135 self
.glibc_version
= self
.get_glibc_version()
137 self
.glibc_configs
= {}
138 self
.makefile_pieces
= ['.PHONY: all\n']
139 self
.add_all_configs()
140 self
.load_versions_json()
141 self
.load_build_state_json()
142 self
.status_log_list
= []
143 self
.email_warning
= False
145 def get_script_text(self
):
146 """Return the text of this script."""
147 with
open(sys
.argv
[0], 'r') as f
:
151 """Re-execute this script with the same arguments."""
153 os
.execv(sys
.executable
, [sys
.executable
] + sys
.argv
)
155 def get_build_triplet(self
):
156 """Determine the build triplet with config.guess."""
157 config_guess
= os
.path
.join(self
.component_srcdir('gcc'),
159 cg_out
= subprocess
.run([config_guess
], stdout
=subprocess
.PIPE
,
160 check
=True, universal_newlines
=True).stdout
161 return cg_out
.rstrip()
163 def get_glibc_version(self
):
164 """Determine the glibc version number (major.minor)."""
165 version_h
= os
.path
.join(self
.component_srcdir('glibc'), 'version.h')
166 with
open(version_h
, 'r') as f
:
167 lines
= f
.readlines()
168 starttext
= '#define VERSION "'
170 if l
.startswith(starttext
):
171 l
= l
[len(starttext
):]
173 m
= re
.fullmatch(r
'([0-9]+)\.([0-9]+)[.0-9]*', l
)
174 return '%s.%s' % m
.group(1, 2)
175 print('error: could not determine glibc version')
178 def add_all_configs(self
):
179 """Add all known glibc build configurations."""
180 self
.add_config(arch
='aarch64',
182 extra_glibcs
=[{'variant': 'disable-multi-arch',
183 'cfg': ['--disable-multi-arch']}])
184 self
.add_config(arch
='aarch64_be',
186 self
.add_config(arch
='arc',
188 gcc_cfg
=['--disable-multilib', '--with-cpu=hs38'])
189 self
.add_config(arch
='arc',
190 os_name
='linux-gnuhf',
191 gcc_cfg
=['--disable-multilib', '--with-cpu=hs38_linux'])
192 self
.add_config(arch
='arceb',
194 gcc_cfg
=['--disable-multilib', '--with-cpu=hs38'])
195 self
.add_config(arch
='alpha',
197 self
.add_config(arch
='arm',
198 os_name
='linux-gnueabi',
199 extra_glibcs
=[{'variant': 'v4t',
200 'ccopts': '-march=armv4t'}])
201 self
.add_config(arch
='armeb',
202 os_name
='linux-gnueabi')
203 self
.add_config(arch
='armeb',
204 os_name
='linux-gnueabi',
206 gcc_cfg
=['--with-arch=armv7-a'])
207 self
.add_config(arch
='arm',
208 os_name
='linux-gnueabihf',
209 gcc_cfg
=['--with-float=hard', '--with-cpu=arm926ej-s'],
210 extra_glibcs
=[{'variant': 'v7a',
211 'ccopts': '-march=armv7-a -mfpu=vfpv3'},
214 '-mthumb -march=armv7-a -mfpu=vfpv3'},
215 {'variant': 'v7a-disable-multi-arch',
216 'ccopts': '-march=armv7-a -mfpu=vfpv3',
217 'cfg': ['--disable-multi-arch']}])
218 self
.add_config(arch
='armeb',
219 os_name
='linux-gnueabihf',
220 gcc_cfg
=['--with-float=hard', '--with-cpu=arm926ej-s'])
221 self
.add_config(arch
='armeb',
222 os_name
='linux-gnueabihf',
224 gcc_cfg
=['--with-float=hard', '--with-arch=armv7-a',
226 self
.add_config(arch
='csky',
227 os_name
='linux-gnuabiv2',
229 gcc_cfg
=['--disable-multilib'])
230 self
.add_config(arch
='csky',
231 os_name
='linux-gnuabiv2',
232 gcc_cfg
=['--with-float=hard', '--disable-multilib'])
233 self
.add_config(arch
='hppa',
235 self
.add_config(arch
='i686',
237 self
.add_config(arch
='loongarch64',
240 gcc_cfg
=['--with-abi=lp64d','--disable-multilib'])
241 self
.add_config(arch
='loongarch64',
244 gcc_cfg
=['--with-abi=lp64s','--disable-multilib'])
245 self
.add_config(arch
='m68k',
247 gcc_cfg
=['--disable-multilib'])
248 self
.add_config(arch
='m68k',
251 gcc_cfg
=['--with-arch=cf', '--disable-multilib'])
252 self
.add_config(arch
='m68k',
254 variant
='coldfire-soft',
255 gcc_cfg
=['--with-arch=cf', '--with-cpu=54455',
256 '--disable-multilib'])
257 self
.add_config(arch
='microblaze',
259 gcc_cfg
=['--disable-multilib'])
260 self
.add_config(arch
='microblazeel',
262 gcc_cfg
=['--disable-multilib'])
263 self
.add_config(arch
='mips64',
265 gcc_cfg
=['--with-mips-plt'],
266 glibcs
=[{'variant': 'n32'},
268 'ccopts': '-mabi=32'},
270 'ccopts': '-mabi=64'}])
271 self
.add_config(arch
='mips64',
274 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
275 glibcs
=[{'variant': 'n32-soft'},
278 'ccopts': '-mabi=32'},
279 {'variant': 'n64-soft',
280 'ccopts': '-mabi=64'}])
281 self
.add_config(arch
='mips64',
284 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
285 '--with-arch-64=mips64r2',
286 '--with-arch-32=mips32r2'],
287 glibcs
=[{'variant': 'n32-nan2008'},
288 {'variant': 'nan2008',
290 'ccopts': '-mabi=32'},
291 {'variant': 'n64-nan2008',
292 'ccopts': '-mabi=64'}])
293 self
.add_config(arch
='mips64',
295 variant
='nan2008-soft',
296 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
297 '--with-arch-64=mips64r2',
298 '--with-arch-32=mips32r2',
299 '--with-float=soft'],
300 glibcs
=[{'variant': 'n32-nan2008-soft'},
301 {'variant': 'nan2008-soft',
303 'ccopts': '-mabi=32'},
304 {'variant': 'n64-nan2008-soft',
305 'ccopts': '-mabi=64'}])
306 self
.add_config(arch
='mips64el',
308 gcc_cfg
=['--with-mips-plt'],
309 glibcs
=[{'variant': 'n32'},
311 'ccopts': '-mabi=32'},
313 'ccopts': '-mabi=64'}])
314 self
.add_config(arch
='mips64el',
317 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
318 glibcs
=[{'variant': 'n32-soft'},
321 'ccopts': '-mabi=32'},
322 {'variant': 'n64-soft',
323 'ccopts': '-mabi=64'}])
324 self
.add_config(arch
='mips64el',
327 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
328 '--with-arch-64=mips64r2',
329 '--with-arch-32=mips32r2'],
330 glibcs
=[{'variant': 'n32-nan2008'},
331 {'variant': 'nan2008',
333 'ccopts': '-mabi=32'},
334 {'variant': 'n64-nan2008',
335 'ccopts': '-mabi=64'}])
336 self
.add_config(arch
='mips64el',
338 variant
='nan2008-soft',
339 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
340 '--with-arch-64=mips64r2',
341 '--with-arch-32=mips32r2',
342 '--with-float=soft'],
343 glibcs
=[{'variant': 'n32-nan2008-soft'},
344 {'variant': 'nan2008-soft',
346 'ccopts': '-mabi=32'},
347 {'variant': 'n64-nan2008-soft',
348 'ccopts': '-mabi=64'}])
349 self
.add_config(arch
='mipsisa64r6el',
351 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
352 '--with-arch-64=mips64r6',
353 '--with-arch-32=mips32r6',
354 '--with-float=hard'],
355 glibcs
=[{'variant': 'n32'},
356 {'arch': 'mipsisa32r6el',
357 'ccopts': '-mabi=32'},
359 'ccopts': '-mabi=64'}])
360 self
.add_config(arch
='nios2',
362 self
.add_config(arch
='or1k',
365 gcc_cfg
=['--with-multilib-list=mcmov'])
366 self
.add_config(arch
='powerpc',
368 gcc_cfg
=['--disable-multilib', '--enable-secureplt'],
369 extra_glibcs
=[{'variant': 'power4',
370 'ccopts': '-mcpu=power4',
371 'cfg': ['--with-cpu=power4']}])
372 self
.add_config(arch
='powerpc',
375 gcc_cfg
=['--disable-multilib', '--with-float=soft',
376 '--enable-secureplt'])
377 self
.add_config(arch
='powerpc64',
379 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
380 self
.add_config(arch
='powerpc64le',
382 gcc_cfg
=['--disable-multilib', '--enable-secureplt'],
383 extra_glibcs
=[{'variant': 'disable-multi-arch',
384 'cfg': ['--disable-multi-arch']}])
385 self
.add_config(arch
='riscv32',
387 variant
='rv32imac-ilp32',
388 gcc_cfg
=['--with-arch=rv32imac', '--with-abi=ilp32',
389 '--disable-multilib'])
390 self
.add_config(arch
='riscv32',
392 variant
='rv32imafdc-ilp32',
393 gcc_cfg
=['--with-arch=rv32imafdc', '--with-abi=ilp32',
394 '--disable-multilib'])
395 self
.add_config(arch
='riscv32',
397 variant
='rv32imafdc-ilp32d',
398 gcc_cfg
=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
399 '--disable-multilib'])
400 self
.add_config(arch
='riscv64',
402 variant
='rv64imac-lp64',
403 gcc_cfg
=['--with-arch=rv64imac', '--with-abi=lp64',
404 '--disable-multilib'])
405 self
.add_config(arch
='riscv64',
407 variant
='rv64imafdc-lp64',
408 gcc_cfg
=['--with-arch=rv64imafdc', '--with-abi=lp64',
409 '--disable-multilib'])
410 self
.add_config(arch
='riscv64',
412 variant
='rv64imafdc-lp64d',
413 gcc_cfg
=['--with-arch=rv64imafdc', '--with-abi=lp64d',
414 '--disable-multilib'])
415 self
.add_config(arch
='s390x',
418 {'arch': 's390', 'ccopts': '-m31'}],
419 extra_glibcs
=[{'variant': 'O3',
421 self
.add_config(arch
='sh3',
423 self
.add_config(arch
='sh3eb',
425 self
.add_config(arch
='sh4',
427 self
.add_config(arch
='sh4eb',
429 self
.add_config(arch
='sh4',
432 gcc_cfg
=['--without-fp'])
433 self
.add_config(arch
='sh4eb',
436 gcc_cfg
=['--without-fp'])
437 self
.add_config(arch
='sparc64',
441 'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
442 extra_glibcs
=[{'variant': 'leon3',
444 'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
445 {'variant': 'disable-multi-arch',
446 'cfg': ['--disable-multi-arch']},
447 {'variant': 'disable-multi-arch',
449 'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
450 'cfg': ['--disable-multi-arch']}])
451 self
.add_config(arch
='x86_64',
453 gcc_cfg
=['--with-multilib-list=m64,m32,mx32'],
455 {'variant': 'x32', 'ccopts': '-mx32'},
456 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
457 extra_glibcs
=[{'variant': 'disable-multi-arch',
458 'cfg': ['--disable-multi-arch']},
459 {'variant': 'minimal',
460 'cfg': ['--disable-multi-arch',
462 '--disable-timezone-tools',
464 '--disable-build-nscd',
466 {'variant': 'no-pie',
467 'cfg': ['--disable-default-pie']},
468 {'variant': 'x32-no-pie',
470 'cfg': ['--disable-default-pie']},
471 {'variant': 'no-pie',
473 'ccopts': '-m32 -march=i686',
474 'cfg': ['--disable-default-pie']},
475 {'variant': 'disable-multi-arch',
477 'ccopts': '-m32 -march=i686',
478 'cfg': ['--disable-multi-arch']},
480 'ccopts': '-m32 -march=i486'},
482 'ccopts': '-m32 -march=i586'},
483 {'variant': 'enable-fortify-source',
484 'cfg': ['--enable-fortify-source']}])
485 self
.add_config(arch
='x86_64',
487 gcc_cfg
=['--disable-multilib'])
489 def add_config(self
, **args
):
490 """Add an individual build configuration."""
491 cfg
= Config(self
, **args
)
492 if cfg
.name
in self
.configs
:
493 print('error: duplicate config %s' % cfg
.name
)
495 self
.configs
[cfg
.name
] = cfg
496 for c
in cfg
.all_glibcs
:
497 if c
.name
in self
.glibc_configs
:
498 print('error: duplicate glibc config %s' % c
.name
)
500 self
.glibc_configs
[c
.name
] = c
502 def component_srcdir(self
, component
):
503 """Return the source directory for a given component, e.g. gcc."""
504 return os
.path
.join(self
.srcdir
, component
)
506 def component_builddir(self
, action
, config
, component
, subconfig
=None):
507 """Return the directory to use for a build."""
510 assert subconfig
is None
511 return os
.path
.join(self
.builddir
, action
, component
)
512 if subconfig
is None:
513 return os
.path
.join(self
.builddir
, action
, config
, component
)
515 # glibc build as part of compiler build.
516 return os
.path
.join(self
.builddir
, action
, config
, component
,
519 def compiler_installdir(self
, config
):
520 """Return the directory in which to install a compiler."""
521 return os
.path
.join(self
.installdir
, 'compilers', config
)
523 def compiler_bindir(self
, config
):
524 """Return the directory in which to find compiler binaries."""
525 return os
.path
.join(self
.compiler_installdir(config
), 'bin')
527 def compiler_sysroot(self
, config
):
528 """Return the sysroot directory for a compiler."""
529 return os
.path
.join(self
.compiler_installdir(config
), 'sysroot')
531 def glibc_installdir(self
, config
):
532 """Return the directory in which to install glibc."""
533 return os
.path
.join(self
.installdir
, 'glibcs', config
)
535 def run_builds(self
, action
, configs
):
536 """Run the requested builds."""
537 if action
== 'checkout':
538 self
.checkout(configs
)
540 if action
== 'bot-cycle':
542 print('error: configurations specified for bot-cycle')
548 print('error: configurations specified for bot')
552 if action
in ('host-libraries', 'list-compilers',
553 'list-glibcs') and configs
:
554 print('error: configurations specified for ' + action
)
556 if action
== 'list-compilers':
557 for name
in sorted(self
.configs
.keys()):
560 if action
== 'list-glibcs':
561 for config
in sorted(self
.glibc_configs
.values(),
562 key
=lambda c
: c
.name
):
563 print(config
.name
, config
.compiler
.name
)
565 self
.clear_last_build_state(action
)
566 build_time
= datetime
.datetime
.utcnow()
567 if action
== 'host-libraries':
568 build_components
= ('gmp', 'mpfr', 'mpc')
571 self
.build_host_libraries()
572 elif action
== 'compilers':
573 build_components
= ('binutils', 'gcc', 'glibc', 'linux', 'mig',
575 old_components
= ('gmp', 'mpfr', 'mpc')
576 old_versions
= self
.build_state
['host-libraries']['build-versions']
577 self
.build_compilers(configs
)
579 build_components
= ('glibc',)
580 old_components
= ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
581 'mig', 'gnumach', 'hurd')
582 old_versions
= self
.build_state
['compilers']['build-versions']
583 if action
== 'update-syscalls':
584 self
.update_syscalls(configs
)
586 self
.build_glibcs(configs
)
590 # Partial build, do not update stored state.
593 for k
in build_components
:
594 if k
in self
.versions
:
595 build_versions
[k
] = {'version': self
.versions
[k
]['version'],
596 'revision': self
.versions
[k
]['revision']}
597 for k
in old_components
:
598 if k
in old_versions
:
599 build_versions
[k
] = {'version': old_versions
[k
]['version'],
600 'revision': old_versions
[k
]['revision']}
601 self
.update_build_state(action
, build_time
, build_versions
)
604 def remove_dirs(*args
):
605 """Remove directories and their contents if they exist."""
607 shutil
.rmtree(dir, ignore_errors
=True)
610 def remove_recreate_dirs(*args
):
611 """Remove directories if they exist, and create them as empty."""
612 Context
.remove_dirs(*args
)
614 os
.makedirs(dir, exist_ok
=True)
616 def add_makefile_cmdlist(self
, target
, cmdlist
, logsdir
):
617 """Add makefile text for a list of commands."""
618 commands
= cmdlist
.makefile_commands(self
.wrapper
, logsdir
)
619 self
.makefile_pieces
.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
620 (target
, target
, target
, commands
))
621 self
.status_log_list
.extend(cmdlist
.status_logs(logsdir
))
623 def write_files(self
):
624 """Write out the Makefile and wrapper script."""
625 mftext
= ''.join(self
.makefile_pieces
)
626 with
open(self
.makefile
, 'w') as f
:
636 'prev_status=$prev_base-status.txt\n'
637 'this_status=$this_base-status.txt\n'
638 'this_log=$this_base-log.txt\n'
639 'date > "$this_log"\n'
640 'echo >> "$this_log"\n'
641 'echo "Description: $desc" >> "$this_log"\n'
642 'printf "%s" "Command:" >> "$this_log"\n'
643 'for word in "$@"; do\n'
644 ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
645 ' printf " %s" "$word"\n'
648 ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
651 'done >> "$this_log"\n'
652 'echo >> "$this_log"\n'
653 'echo "Directory: $dir" >> "$this_log"\n'
654 'echo "Path addition: $path" >> "$this_log"\n'
655 'echo >> "$this_log"\n'
658 ' echo >> "$this_log"\n'
659 ' echo "$1: $desc" > "$this_status"\n'
660 ' echo "$1: $desc" >> "$this_log"\n'
661 ' echo >> "$this_log"\n'
662 ' date >> "$this_log"\n'
663 ' echo "$1: $desc"\n'
668 ' if [ "$1" != "0" ]; then\n'
669 ' record_status FAIL\n'
672 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
673 ' record_status UNRESOLVED\n'
675 'if [ "$dir" ]; then\n'
677 ' check_error "$?"\n'
679 'if [ "$path" ]; then\n'
680 ' PATH=$path:$PATH\n'
682 '"$@" < /dev/null >> "$this_log" 2>&1\n'
684 'record_status PASS\n')
685 with
open(self
.wrapper
, 'w') as f
:
686 f
.write(wrapper_text
)
688 mode_exec
= (stat
.S_IRWXU|stat
.S_IRGRP|stat
.S_IXGRP|
689 stat
.S_IROTH|stat
.S_IXOTH
)
690 os
.chmod(self
.wrapper
, mode_exec
)
693 'if ! [ -f tests.sum ]; then\n'
694 ' echo "No test summary available."\n'
699 ' echo "Contents of $1:"\n'
703 ' echo "End of contents of $1."\n'
706 'save_file tests.sum\n'
707 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
708 'for t in $non_pass_tests; do\n'
709 ' if [ -f "$t.out" ]; then\n'
710 ' save_file "$t.out"\n'
713 with
open(self
.save_logs
, 'w') as f
:
714 f
.write(save_logs_text
)
715 os
.chmod(self
.save_logs
, mode_exec
)
718 """Do the actual build."""
719 cmd
= ['make', '-O', '-j%d' % self
.parallelism
]
720 subprocess
.run(cmd
, cwd
=self
.builddir
, check
=True)
722 def build_host_libraries(self
):
723 """Build the host libraries."""
724 installdir
= self
.host_libraries_installdir
725 builddir
= os
.path
.join(self
.builddir
, 'host-libraries')
726 logsdir
= os
.path
.join(self
.logsdir
, 'host-libraries')
727 self
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
728 cmdlist
= CommandList('host-libraries', self
.keep
)
729 self
.build_host_library(cmdlist
, 'gmp')
730 self
.build_host_library(cmdlist
, 'mpfr',
731 ['--with-gmp=%s' % installdir
])
732 self
.build_host_library(cmdlist
, 'mpc',
733 ['--with-gmp=%s' % installdir
,
734 '--with-mpfr=%s' % installdir
])
735 cmdlist
.add_command('done', ['touch', os
.path
.join(installdir
, 'ok')])
736 self
.add_makefile_cmdlist('host-libraries', cmdlist
, logsdir
)
738 def build_host_library(self
, cmdlist
, lib
, extra_opts
=None):
739 """Build one host library."""
740 srcdir
= self
.component_srcdir(lib
)
741 builddir
= self
.component_builddir('host-libraries', None, lib
)
742 installdir
= self
.host_libraries_installdir
743 cmdlist
.push_subdesc(lib
)
744 cmdlist
.create_use_dir(builddir
)
745 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
746 '--prefix=%s' % installdir
,
749 cfg_cmd
.extend (extra_opts
)
750 cmdlist
.add_command('configure', cfg_cmd
)
751 cmdlist
.add_command('build', ['make'])
752 cmdlist
.add_command('check', ['make', 'check'])
753 cmdlist
.add_command('install', ['make', 'install'])
754 cmdlist
.cleanup_dir()
755 cmdlist
.pop_subdesc()
757 def build_compilers(self
, configs
):
758 """Build the compilers."""
760 self
.remove_dirs(os
.path
.join(self
.builddir
, 'compilers'))
761 self
.remove_dirs(os
.path
.join(self
.installdir
, 'compilers'))
762 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'compilers'))
763 configs
= sorted(self
.configs
.keys())
765 self
.configs
[c
].build()
767 def build_glibcs(self
, configs
):
768 """Build the glibcs."""
770 self
.remove_dirs(os
.path
.join(self
.builddir
, 'glibcs'))
771 self
.remove_dirs(os
.path
.join(self
.installdir
, 'glibcs'))
772 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'glibcs'))
773 configs
= sorted(self
.glibc_configs
.keys())
775 self
.glibc_configs
[c
].build()
777 def update_syscalls(self
, configs
):
778 """Update the glibc syscall lists."""
780 self
.remove_dirs(os
.path
.join(self
.builddir
, 'update-syscalls'))
781 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'update-syscalls'))
782 configs
= sorted(self
.glibc_configs
.keys())
784 self
.glibc_configs
[c
].update_syscalls()
786 def load_versions_json(self
):
787 """Load information about source directory versions."""
788 if not os
.access(self
.versions_json
, os
.F_OK
):
791 with
open(self
.versions_json
, 'r') as f
:
792 self
.versions
= json
.load(f
)
794 def store_json(self
, data
, filename
):
795 """Store information in a JSON file."""
796 filename_tmp
= filename
+ '.tmp'
797 with
open(filename_tmp
, 'w') as f
:
798 json
.dump(data
, f
, indent
=2, sort_keys
=True)
799 os
.rename(filename_tmp
, filename
)
801 def store_versions_json(self
):
802 """Store information about source directory versions."""
803 self
.store_json(self
.versions
, self
.versions_json
)
805 def set_component_version(self
, component
, version
, explicit
, revision
):
806 """Set the version information for a component."""
807 self
.versions
[component
] = {'version': version
,
808 'explicit': explicit
,
809 'revision': revision
}
810 self
.store_versions_json()
812 def checkout(self
, versions
):
813 """Check out the desired component versions."""
814 default_versions
= {'binutils': 'vcs-2.41',
816 'glibc': 'vcs-mainline',
821 'mig': 'vcs-mainline',
822 'gnumach': 'vcs-mainline',
823 'hurd': 'vcs-mainline'}
825 explicit_versions
= {}
828 for k
in default_versions
.keys():
832 if k
in use_versions
:
833 print('error: multiple versions for %s' % k
)
836 explicit_versions
[k
] = True
840 print('error: unknown component in %s' % v
)
842 for k
in default_versions
.keys():
843 if k
not in use_versions
:
844 if k
in self
.versions
and self
.versions
[k
]['explicit']:
845 use_versions
[k
] = self
.versions
[k
]['version']
846 explicit_versions
[k
] = True
848 use_versions
[k
] = default_versions
[k
]
849 explicit_versions
[k
] = False
850 os
.makedirs(self
.srcdir
, exist_ok
=True)
851 for k
in sorted(default_versions
.keys()):
852 update
= os
.access(self
.component_srcdir(k
), os
.F_OK
)
855 k
in self
.versions
and
856 v
!= self
.versions
[k
]['version']):
857 if not self
.replace_sources
:
858 print('error: version of %s has changed from %s to %s, '
859 'use --replace-sources to check out again' %
860 (k
, self
.versions
[k
]['version'], v
))
862 shutil
.rmtree(self
.component_srcdir(k
))
864 if v
.startswith('vcs-'):
865 revision
= self
.checkout_vcs(k
, v
[4:], update
)
867 self
.checkout_tar(k
, v
, update
)
869 self
.set_component_version(k
, v
, explicit_versions
[k
], revision
)
870 if self
.get_script_text() != self
.script_text
:
871 # Rerun the checkout process in case the updated script
872 # uses different default versions or new components.
875 def checkout_vcs(self
, component
, version
, update
):
876 """Check out the given version of the given component from version
877 control. Return a revision identifier."""
878 if component
== 'binutils':
879 git_url
= 'https://sourceware.org/git/binutils-gdb.git'
880 if version
== 'mainline':
881 git_branch
= 'master'
883 trans
= str.maketrans({'.': '_'})
884 git_branch
= 'binutils-%s-branch' % version
.translate(trans
)
885 return self
.git_checkout(component
, git_url
, git_branch
, update
)
886 elif component
== 'gcc':
887 if version
== 'mainline':
890 branch
= 'releases/gcc-%s' % version
891 return self
.gcc_checkout(branch
, update
)
892 elif component
== 'glibc':
893 git_url
= 'https://sourceware.org/git/glibc.git'
894 if version
== 'mainline':
895 git_branch
= 'master'
897 git_branch
= 'release/%s/master' % version
898 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
899 self
.fix_glibc_timestamps()
901 elif component
== 'gnumach':
902 git_url
= 'git://git.savannah.gnu.org/hurd/gnumach.git'
903 git_branch
= 'master'
904 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
905 subprocess
.run(['autoreconf', '-i'],
906 cwd
=self
.component_srcdir(component
), check
=True)
908 elif component
== 'mig':
909 git_url
= 'git://git.savannah.gnu.org/hurd/mig.git'
910 git_branch
= 'master'
911 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
912 subprocess
.run(['autoreconf', '-i'],
913 cwd
=self
.component_srcdir(component
), check
=True)
915 elif component
== 'hurd':
916 git_url
= 'git://git.savannah.gnu.org/hurd/hurd.git'
917 git_branch
= 'master'
918 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
919 subprocess
.run(['autoconf'],
920 cwd
=self
.component_srcdir(component
), check
=True)
923 print('error: component %s coming from VCS' % component
)
926 def git_checkout(self
, component
, git_url
, git_branch
, update
):
927 """Check out a component from git. Return a commit identifier."""
929 subprocess
.run(['git', 'remote', 'prune', 'origin'],
930 cwd
=self
.component_srcdir(component
), check
=True)
931 if self
.replace_sources
:
932 subprocess
.run(['git', 'clean', '-dxfq'],
933 cwd
=self
.component_srcdir(component
), check
=True)
934 subprocess
.run(['git', 'pull', '-q'],
935 cwd
=self
.component_srcdir(component
), check
=True)
938 depth_arg
= ('--depth', '1')
941 subprocess
.run(['git', 'clone', '-q', '-b', git_branch
,
943 self
.component_srcdir(component
)], check
=True)
944 r
= subprocess
.run(['git', 'rev-parse', 'HEAD'],
945 cwd
=self
.component_srcdir(component
),
946 stdout
=subprocess
.PIPE
,
947 check
=True, universal_newlines
=True).stdout
950 def fix_glibc_timestamps(self
):
951 """Fix timestamps in a glibc checkout."""
952 # Ensure that builds do not try to regenerate generated files
953 # in the source tree.
954 srcdir
= self
.component_srcdir('glibc')
955 # These files have Makefile dependencies to regenerate them in
956 # the source tree that may be active during a normal build.
957 # Some other files have such dependencies but do not need to
958 # be touched because nothing in a build depends on the files
960 for f
in ('sysdeps/mach/hurd/bits/errno.h',):
961 to_touch
= os
.path
.join(srcdir
, f
)
962 subprocess
.run(['touch', '-c', to_touch
], check
=True)
963 for dirpath
, dirnames
, filenames
in os
.walk(srcdir
):
965 if (f
== 'configure' or
966 f
== 'preconfigure' or
967 f
.endswith('-kw.h')):
968 to_touch
= os
.path
.join(dirpath
, f
)
969 subprocess
.run(['touch', to_touch
], check
=True)
971 def gcc_checkout(self
, branch
, update
):
972 """Check out GCC from git. Return the commit identifier."""
973 if os
.access(os
.path
.join(self
.component_srcdir('gcc'), '.svn'),
975 if not self
.replace_sources
:
976 print('error: GCC has moved from SVN to git, use '
977 '--replace-sources to check out again')
979 shutil
.rmtree(self
.component_srcdir('gcc'))
982 self
.git_checkout('gcc', 'https://gcc.gnu.org/git/gcc.git',
984 subprocess
.run(['contrib/gcc_update', '--silent'],
985 cwd
=self
.component_srcdir('gcc'), check
=True)
986 r
= subprocess
.run(['git', 'rev-parse', 'HEAD'],
987 cwd
=self
.component_srcdir('gcc'),
988 stdout
=subprocess
.PIPE
,
989 check
=True, universal_newlines
=True).stdout
992 def checkout_tar(self
, component
, version
, update
):
993 """Check out the given version of the given component from a
997 url_map
= {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
998 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
999 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
1000 'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
1001 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
1002 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
1003 'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
1004 'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
1005 'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
1006 if component
not in url_map
:
1007 print('error: component %s coming from tarball' % component
)
1009 version_major
= version
.split('.')[0]
1010 url
= url_map
[component
] % {'version': version
, 'major': version_major
}
1011 filename
= os
.path
.join(self
.srcdir
, url
.split('/')[-1])
1012 response
= urllib
.request
.urlopen(url
)
1013 data
= response
.read()
1014 with
open(filename
, 'wb') as f
:
1016 subprocess
.run(['tar', '-C', self
.srcdir
, '-x', '-f', filename
],
1018 os
.rename(os
.path
.join(self
.srcdir
, '%s-%s' % (component
, version
)),
1019 self
.component_srcdir(component
))
1022 def load_build_state_json(self
):
1023 """Load information about the state of previous builds."""
1024 if os
.access(self
.build_state_json
, os
.F_OK
):
1025 with
open(self
.build_state_json
, 'r') as f
:
1026 self
.build_state
= json
.load(f
)
1028 self
.build_state
= {}
1029 for k
in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
1030 if k
not in self
.build_state
:
1031 self
.build_state
[k
] = {}
1032 if 'build-time' not in self
.build_state
[k
]:
1033 self
.build_state
[k
]['build-time'] = ''
1034 if 'build-versions' not in self
.build_state
[k
]:
1035 self
.build_state
[k
]['build-versions'] = {}
1036 if 'build-results' not in self
.build_state
[k
]:
1037 self
.build_state
[k
]['build-results'] = {}
1038 if 'result-changes' not in self
.build_state
[k
]:
1039 self
.build_state
[k
]['result-changes'] = {}
1040 if 'ever-passed' not in self
.build_state
[k
]:
1041 self
.build_state
[k
]['ever-passed'] = []
1043 def store_build_state_json(self
):
1044 """Store information about the state of previous builds."""
1045 self
.store_json(self
.build_state
, self
.build_state_json
)
1047 def clear_last_build_state(self
, action
):
1048 """Clear information about the state of part of the build."""
1049 # We clear the last build time and versions when starting a
1050 # new build. The results of the last build are kept around,
1051 # as comparison is still meaningful if this build is aborted
1052 # and a new one started.
1053 self
.build_state
[action
]['build-time'] = ''
1054 self
.build_state
[action
]['build-versions'] = {}
1055 self
.store_build_state_json()
1057 def update_build_state(self
, action
, build_time
, build_versions
):
1058 """Update the build state after a build."""
1059 build_time
= build_time
.replace(microsecond
=0)
1060 self
.build_state
[action
]['build-time'] = str(build_time
)
1061 self
.build_state
[action
]['build-versions'] = build_versions
1063 for log
in self
.status_log_list
:
1064 with
open(log
, 'r') as f
:
1066 log_text
= log_text
.rstrip()
1067 m
= re
.fullmatch('([A-Z]+): (.*)', log_text
)
1069 test_name
= m
.group(2)
1070 assert test_name
not in build_results
1071 build_results
[test_name
] = result
1072 old_build_results
= self
.build_state
[action
]['build-results']
1073 self
.build_state
[action
]['build-results'] = build_results
1075 all_tests
= set(old_build_results
.keys()) |
set(build_results
.keys())
1077 if t
in old_build_results
:
1078 old_res
= old_build_results
[t
]
1080 old_res
= '(New test)'
1081 if t
in build_results
:
1082 new_res
= build_results
[t
]
1084 new_res
= '(Test removed)'
1085 if old_res
!= new_res
:
1086 result_changes
[t
] = '%s -> %s' % (old_res
, new_res
)
1087 self
.build_state
[action
]['result-changes'] = result_changes
1088 old_ever_passed
= {t
for t
in self
.build_state
[action
]['ever-passed']
1089 if t
in build_results
}
1090 new_passes
= {t
for t
in build_results
if build_results
[t
] == 'PASS'}
1091 self
.build_state
[action
]['ever-passed'] = sorted(old_ever_passed |
1093 self
.store_build_state_json()
1095 def load_bot_config_json(self
):
1096 """Load bot configuration."""
1097 with
open(self
.bot_config_json
, 'r') as f
:
1098 self
.bot_config
= json
.load(f
)
1100 def part_build_old(self
, action
, delay
):
1101 """Return whether the last build for a given action was at least a
1102 given number of seconds ago, or does not have a time recorded."""
1103 old_time_str
= self
.build_state
[action
]['build-time']
1104 if not old_time_str
:
1106 old_time
= datetime
.datetime
.strptime(old_time_str
,
1107 '%Y-%m-%d %H:%M:%S')
1108 new_time
= datetime
.datetime
.utcnow()
1109 delta
= new_time
- old_time
1110 return delta
.total_seconds() >= delay
1112 def bot_cycle(self
):
1113 """Run a single round of checkout and builds."""
1114 print('Bot cycle starting %s.' % str(datetime
.datetime
.utcnow()))
1115 self
.load_bot_config_json()
1116 actions
= ('host-libraries', 'compilers', 'glibcs')
1117 self
.bot_run_self(['--replace-sources'], 'checkout')
1118 self
.load_versions_json()
1119 if self
.get_script_text() != self
.script_text
:
1120 print('Script changed, re-execing.')
1121 # On script change, all parts of the build should be rerun.
1123 self
.clear_last_build_state(a
)
1125 check_components
= {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1126 'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1127 'mig', 'gnumach', 'hurd'),
1128 'glibcs': ('glibc',)}
1131 build_vers
= self
.build_state
[a
]['build-versions']
1132 must_build
[a
] = False
1133 if not self
.build_state
[a
]['build-time']:
1134 must_build
[a
] = True
1137 for c
in check_components
[a
]:
1139 old_vers
[c
] = build_vers
[c
]
1140 new_vers
[c
] = {'version': self
.versions
[c
]['version'],
1141 'revision': self
.versions
[c
]['revision']}
1142 if new_vers
== old_vers
:
1143 print('Versions for %s unchanged.' % a
)
1145 print('Versions changed or rebuild forced for %s.' % a
)
1146 if a
== 'compilers' and not self
.part_build_old(
1147 a
, self
.bot_config
['compilers-rebuild-delay']):
1148 print('Not requiring rebuild of compilers this soon.')
1150 must_build
[a
] = True
1151 if must_build
['host-libraries']:
1152 must_build
['compilers'] = True
1153 if must_build
['compilers']:
1154 must_build
['glibcs'] = True
1157 print('Must rebuild %s.' % a
)
1158 self
.clear_last_build_state(a
)
1160 print('No need to rebuild %s.' % a
)
1161 if os
.access(self
.logsdir
, os
.F_OK
):
1162 shutil
.rmtree(self
.logsdir_old
, ignore_errors
=True)
1163 shutil
.copytree(self
.logsdir
, self
.logsdir_old
)
1166 build_time
= datetime
.datetime
.utcnow()
1167 print('Rebuilding %s at %s.' % (a
, str(build_time
)))
1168 self
.bot_run_self([], a
)
1169 self
.load_build_state_json()
1170 self
.bot_build_mail(a
, build_time
)
1171 print('Bot cycle done at %s.' % str(datetime
.datetime
.utcnow()))
1173 def bot_build_mail(self
, action
, build_time
):
1174 """Send email with the results of a build."""
1175 if not ('email-from' in self
.bot_config
and
1176 'email-server' in self
.bot_config
and
1177 'email-subject' in self
.bot_config
and
1178 'email-to' in self
.bot_config
):
1179 if not self
.email_warning
:
1180 print("Email not configured, not sending.")
1181 self
.email_warning
= True
1184 build_time
= build_time
.replace(microsecond
=0)
1185 subject
= (self
.bot_config
['email-subject'] %
1187 'build-time': str(build_time
)})
1188 results
= self
.build_state
[action
]['build-results']
1189 changes
= self
.build_state
[action
]['result-changes']
1190 ever_passed
= set(self
.build_state
[action
]['ever-passed'])
1191 versions
= self
.build_state
[action
]['build-versions']
1192 new_regressions
= {k
for k
in changes
if changes
[k
] == 'PASS -> FAIL'}
1193 all_regressions
= {k
for k
in ever_passed
if results
[k
] == 'FAIL'}
1194 all_fails
= {k
for k
in results
if results
[k
] == 'FAIL'}
1196 new_reg_list
= sorted(['FAIL: %s' % k
for k
in new_regressions
])
1197 new_reg_text
= ('New regressions:\n\n%s\n\n' %
1198 '\n'.join(new_reg_list
))
1202 all_reg_list
= sorted(['FAIL: %s' % k
for k
in all_regressions
])
1203 all_reg_text
= ('All regressions:\n\n%s\n\n' %
1204 '\n'.join(all_reg_list
))
1208 all_fail_list
= sorted(['FAIL: %s' % k
for k
in all_fails
])
1209 all_fail_text
= ('All failures:\n\n%s\n\n' %
1210 '\n'.join(all_fail_list
))
1214 changes_list
= sorted(changes
.keys())
1215 changes_list
= ['%s: %s' % (changes
[k
], k
) for k
in changes_list
]
1216 changes_text
= ('All changed results:\n\n%s\n\n' %
1217 '\n'.join(changes_list
))
1220 results_text
= (new_reg_text
+ all_reg_text
+ all_fail_text
+
1222 if not results_text
:
1223 results_text
= 'Clean build with unchanged results.\n\n'
1224 versions_list
= sorted(versions
.keys())
1225 versions_list
= ['%s: %s (%s)' % (k
, versions
[k
]['version'],
1226 versions
[k
]['revision'])
1227 for k
in versions_list
]
1228 versions_text
= ('Component versions for this build:\n\n%s\n' %
1229 '\n'.join(versions_list
))
1230 body_text
= results_text
+ versions_text
1231 msg
= email
.mime
.text
.MIMEText(body_text
)
1232 msg
['Subject'] = subject
1233 msg
['From'] = self
.bot_config
['email-from']
1234 msg
['To'] = self
.bot_config
['email-to']
1235 msg
['Message-ID'] = email
.utils
.make_msgid()
1236 msg
['Date'] = email
.utils
.format_datetime(datetime
.datetime
.utcnow())
1237 with smtplib
.SMTP(self
.bot_config
['email-server']) as s
:
1240 def bot_run_self(self
, opts
, action
, check
=True):
1241 """Run a copy of this script with given options."""
1242 cmd
= [sys
.executable
, sys
.argv
[0], '--keep=none',
1243 '-j%d' % self
.parallelism
]
1245 cmd
.append('--full-gcc')
1247 cmd
.extend([self
.topdir
, action
])
1249 subprocess
.run(cmd
, check
=check
)
1252 """Run repeated rounds of checkout and builds."""
1254 self
.load_bot_config_json()
1255 if not self
.bot_config
['run']:
1256 print('Bot exiting by request.')
1258 self
.bot_run_self([], 'bot-cycle', check
=False)
1259 self
.load_bot_config_json()
1260 if not self
.bot_config
['run']:
1261 print('Bot exiting by request.')
1263 time
.sleep(self
.bot_config
['delay'])
1264 if self
.get_script_text() != self
.script_text
:
1265 print('Script changed, bot re-execing.')
1268 class LinuxHeadersPolicyForBuild(object):
1269 """Names and directories for installing Linux headers. Build variant."""
1271 def __init__(self
, config
):
1272 self
.arch
= config
.arch
1273 self
.srcdir
= config
.ctx
.component_srcdir('linux')
1274 self
.builddir
= config
.component_builddir('linux')
1275 self
.headers_dir
= os
.path
.join(config
.sysroot
, 'usr')
1277 class LinuxHeadersPolicyForUpdateSyscalls(object):
1278 """Names and directories for Linux headers. update-syscalls variant."""
1280 def __init__(self
, glibc
, headers_dir
):
1281 self
.arch
= glibc
.compiler
.arch
1282 self
.srcdir
= glibc
.compiler
.ctx
.component_srcdir('linux')
1283 self
.builddir
= glibc
.ctx
.component_builddir(
1284 'update-syscalls', glibc
.name
, 'build-linux')
1285 self
.headers_dir
= headers_dir
1287 def install_linux_headers(policy
, cmdlist
):
1288 """Install Linux kernel headers."""
1289 arch_map
= {'aarch64': 'arm64',
1299 'loongarch64': 'loongarch',
1301 'microblaze': 'microblaze',
1305 'powerpc': 'powerpc',
1314 if policy
.arch
.startswith(k
):
1315 linux_arch
= arch_map
[k
]
1317 assert linux_arch
is not None
1318 cmdlist
.push_subdesc('linux')
1319 cmdlist
.create_use_dir(policy
.builddir
)
1320 cmdlist
.add_command('install-headers',
1321 ['make', '-C', policy
.srcdir
, 'O=%s' % policy
.builddir
,
1322 'ARCH=%s' % linux_arch
,
1323 'INSTALL_HDR_PATH=%s' % policy
.headers_dir
,
1325 cmdlist
.cleanup_dir()
1326 cmdlist
.pop_subdesc()
1328 class Config(object):
1329 """A configuration for building a compiler and associated libraries."""
1331 def __init__(self
, ctx
, arch
, os_name
, variant
=None, gcc_cfg
=None,
1332 first_gcc_cfg
=None, binutils_cfg
=None, glibcs
=None,
1334 """Initialize a Config object."""
1338 self
.variant
= variant
1340 self
.name
= '%s-%s' % (arch
, os_name
)
1342 self
.name
= '%s-%s-%s' % (arch
, os_name
, variant
)
1343 self
.triplet
= '%s-glibc-%s' % (arch
, os_name
)
1347 self
.gcc_cfg
= gcc_cfg
1348 if first_gcc_cfg
is None:
1349 self
.first_gcc_cfg
= []
1351 self
.first_gcc_cfg
= first_gcc_cfg
1352 if binutils_cfg
is None:
1353 self
.binutils_cfg
= []
1355 self
.binutils_cfg
= binutils_cfg
1357 glibcs
= [{'variant': variant
}]
1358 if extra_glibcs
is None:
1360 glibcs
= [Glibc(self
, **g
) for g
in glibcs
]
1361 extra_glibcs
= [Glibc(self
, **g
) for g
in extra_glibcs
]
1362 self
.all_glibcs
= glibcs
+ extra_glibcs
1363 self
.compiler_glibcs
= glibcs
1364 self
.installdir
= ctx
.compiler_installdir(self
.name
)
1365 self
.bindir
= ctx
.compiler_bindir(self
.name
)
1366 self
.sysroot
= ctx
.compiler_sysroot(self
.name
)
1367 self
.builddir
= os
.path
.join(ctx
.builddir
, 'compilers', self
.name
)
1368 self
.logsdir
= os
.path
.join(ctx
.logsdir
, 'compilers', self
.name
)
1370 def component_builddir(self
, component
):
1371 """Return the directory to use for a (non-glibc) build."""
1372 return self
.ctx
.component_builddir('compilers', self
.name
, component
)
1375 """Generate commands to build this compiler."""
1376 self
.ctx
.remove_recreate_dirs(self
.installdir
, self
.builddir
,
1378 cmdlist
= CommandList('compilers-%s' % self
.name
, self
.ctx
.keep
)
1379 cmdlist
.add_command('check-host-libraries',
1381 os
.path
.join(self
.ctx
.host_libraries_installdir
,
1383 cmdlist
.use_path(self
.bindir
)
1384 self
.build_cross_tool(cmdlist
, 'binutils', 'binutils',
1386 '--disable-gdbserver',
1387 '--disable-libdecnumber',
1388 '--disable-readline',
1389 '--disable-sim'] + self
.binutils_cfg
)
1390 if self
.os
.startswith('linux'):
1391 install_linux_headers(LinuxHeadersPolicyForBuild(self
), cmdlist
)
1392 self
.build_gcc(cmdlist
, True)
1393 if self
.os
== 'gnu':
1394 self
.install_gnumach_headers(cmdlist
)
1395 self
.build_cross_tool(cmdlist
, 'mig', 'mig')
1396 self
.install_hurd_headers(cmdlist
)
1397 for g
in self
.compiler_glibcs
:
1398 cmdlist
.push_subdesc('glibc')
1399 cmdlist
.push_subdesc(g
.name
)
1400 g
.build_glibc(cmdlist
, GlibcPolicyForCompiler(g
))
1401 cmdlist
.pop_subdesc()
1402 cmdlist
.pop_subdesc()
1403 self
.build_gcc(cmdlist
, False)
1404 cmdlist
.add_command('done', ['touch',
1405 os
.path
.join(self
.installdir
, 'ok')])
1406 self
.ctx
.add_makefile_cmdlist('compilers-%s' % self
.name
, cmdlist
,
1409 def build_cross_tool(self
, cmdlist
, tool_src
, tool_build
, extra_opts
=None):
1410 """Build one cross tool."""
1411 srcdir
= self
.ctx
.component_srcdir(tool_src
)
1412 builddir
= self
.component_builddir(tool_build
)
1413 cmdlist
.push_subdesc(tool_build
)
1414 cmdlist
.create_use_dir(builddir
)
1415 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
1416 '--prefix=%s' % self
.installdir
,
1417 '--build=%s' % self
.ctx
.build_triplet
,
1418 '--host=%s' % self
.ctx
.build_triplet
,
1419 '--target=%s' % self
.triplet
,
1420 '--with-sysroot=%s' % self
.sysroot
]
1422 cfg_cmd
.extend(extra_opts
)
1423 cmdlist
.add_command('configure', cfg_cmd
)
1424 cmdlist
.add_command('build', ['make'])
1425 # Parallel "make install" for GCC has race conditions that can
1426 # cause it to fail; see
1427 # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>. Such
1428 # problems are not known for binutils, but doing the
1429 # installation in parallel within a particular toolchain build
1430 # (as opposed to installation of one toolchain from
1431 # build-many-glibcs.py running in parallel to the installation
1432 # of other toolchains being built) is not known to be
1433 # significantly beneficial, so it is simplest just to disable
1434 # parallel install for cross tools here.
1435 cmdlist
.add_command('install', ['make', '-j1', 'install'])
1436 cmdlist
.cleanup_dir()
1437 cmdlist
.pop_subdesc()
1439 def install_gnumach_headers(self
, cmdlist
):
1440 """Install GNU Mach headers."""
1441 srcdir
= self
.ctx
.component_srcdir('gnumach')
1442 builddir
= self
.component_builddir('gnumach')
1443 cmdlist
.push_subdesc('gnumach')
1444 cmdlist
.create_use_dir(builddir
)
1445 cmdlist
.add_command('configure',
1446 [os
.path
.join(srcdir
, 'configure'),
1447 '--build=%s' % self
.ctx
.build_triplet
,
1448 '--host=%s' % self
.triplet
,
1451 'CC=%s-gcc -nostdlib' % self
.triplet
])
1452 cmdlist
.add_command('install', ['make', 'DESTDIR=%s' % self
.sysroot
,
1454 cmdlist
.cleanup_dir()
1455 cmdlist
.pop_subdesc()
1457 def install_hurd_headers(self
, cmdlist
):
1458 """Install Hurd headers."""
1459 srcdir
= self
.ctx
.component_srcdir('hurd')
1460 builddir
= self
.component_builddir('hurd')
1461 cmdlist
.push_subdesc('hurd')
1462 cmdlist
.create_use_dir(builddir
)
1463 cmdlist
.add_command('configure',
1464 [os
.path
.join(srcdir
, 'configure'),
1465 '--build=%s' % self
.ctx
.build_triplet
,
1466 '--host=%s' % self
.triplet
,
1468 '--disable-profile', '--without-parted',
1469 'CC=%s-gcc -nostdlib' % self
.triplet
])
1470 cmdlist
.add_command('install', ['make', 'prefix=%s' % self
.sysroot
,
1471 'no_deps=t', 'install-headers'])
1472 cmdlist
.cleanup_dir()
1473 cmdlist
.pop_subdesc()
1475 def build_gcc(self
, cmdlist
, bootstrap
):
1477 # libssp is of little relevance with glibc's own stack
1478 # checking support. libcilkrts does not support GNU/Hurd (and
1479 # has been removed in GCC 8, so --disable-libcilkrts can be
1480 # removed once glibc no longer supports building with older
1481 # GCC versions). --enable-initfini-array is enabled by default
1482 # in GCC 12, which can be removed when GCC 12 becomes the
1483 # minimum requirement.
1484 cfg_opts
= list(self
.gcc_cfg
)
1485 cfg_opts
+= ['--enable-initfini-array']
1486 cfg_opts
+= ['--disable-libssp', '--disable-libcilkrts']
1487 host_libs
= self
.ctx
.host_libraries_installdir
1488 cfg_opts
+= ['--with-gmp=%s' % host_libs
,
1489 '--with-mpfr=%s' % host_libs
,
1490 '--with-mpc=%s' % host_libs
]
1492 tool_build
= 'gcc-first'
1493 # Building a static-only, C-only compiler that is
1494 # sufficient to build glibc. Various libraries and
1495 # features that may require libc headers must be disabled.
1496 # When configuring with a sysroot, --with-newlib is
1497 # required to define inhibit_libc (to stop some parts of
1498 # libgcc including libc headers); --without-headers is not
1500 cfg_opts
+= ['--enable-languages=c', '--disable-shared',
1501 '--disable-threads',
1502 '--disable-libatomic',
1503 '--disable-decimal-float',
1506 '--disable-libgomp',
1509 '--disable-libquadmath',
1510 '--disable-libsanitizer',
1511 '--without-headers', '--with-newlib',
1512 '--with-glibc-version=%s' % self
.ctx
.glibc_version
1514 cfg_opts
+= self
.first_gcc_cfg
1517 # libsanitizer commonly breaks because of glibc header
1518 # changes, or on unusual targets. C++ pre-compiled
1519 # headers are not used during the glibc build and are
1520 # expensive to create.
1521 if not self
.ctx
.full_gcc
:
1522 cfg_opts
+= ['--disable-libsanitizer',
1523 '--disable-libstdcxx-pch']
1524 langs
= 'all' if self
.ctx
.full_gcc
else 'c,c++'
1525 cfg_opts
+= ['--enable-languages=%s' % langs
,
1526 '--enable-shared', '--enable-threads']
1527 self
.build_cross_tool(cmdlist
, 'gcc', tool_build
, cfg_opts
)
1529 class GlibcPolicyDefault(object):
1530 """Build policy for glibc: common defaults."""
1532 def __init__(self
, glibc
):
1533 self
.srcdir
= glibc
.ctx
.component_srcdir('glibc')
1534 self
.use_usr
= glibc
.os
!= 'gnu'
1535 self
.prefix
= '/usr' if self
.use_usr
else ''
1536 self
.configure_args
= [
1537 '--prefix=%s' % self
.prefix
,
1539 '--build=%s' % glibc
.ctx
.build_triplet
,
1540 '--host=%s' % glibc
.triplet
,
1541 'CC=%s' % glibc
.tool_name('gcc'),
1542 'CXX=%s' % glibc
.tool_name('g++'),
1544 if glibc
.os
== 'gnu':
1545 self
.configure_args
.append('MIG=%s' % glibc
.tool_name('mig'))
1547 self
.configure_args
.append('CFLAGS=%s' % glibc
.cflags
)
1548 self
.configure_args
.append('CXXFLAGS=%s' % glibc
.cflags
)
1549 self
.configure_args
+= glibc
.cfg
1551 def configure(self
, cmdlist
):
1552 """Invoked to add the configure command to the command list."""
1553 cmdlist
.add_command('configure',
1554 [os
.path
.join(self
.srcdir
, 'configure'),
1555 *self
.configure_args
])
1557 def extra_commands(self
, cmdlist
):
1558 """Invoked to inject additional commands (make check) after build."""
1561 class GlibcPolicyForCompiler(GlibcPolicyDefault
):
1562 """Build policy for glibc during the compilers stage."""
1564 def __init__(self
, glibc
):
1565 super().__init
__(glibc
)
1566 self
.builddir
= glibc
.ctx
.component_builddir(
1567 'compilers', glibc
.compiler
.name
, 'glibc', glibc
.name
)
1568 self
.installdir
= glibc
.compiler
.sysroot
1570 class GlibcPolicyForBuild(GlibcPolicyDefault
):
1571 """Build policy for glibc during the glibcs stage."""
1573 def __init__(self
, glibc
):
1574 super().__init
__(glibc
)
1575 self
.builddir
= glibc
.ctx
.component_builddir(
1576 'glibcs', glibc
.name
, 'glibc')
1577 self
.installdir
= glibc
.ctx
.glibc_installdir(glibc
.name
)
1579 self
.strip
= glibc
.tool_name('strip')
1582 self
.save_logs
= glibc
.ctx
.save_logs
1584 def extra_commands(self
, cmdlist
):
1586 # Avoid stripping libc.so and libpthread.so, which are
1587 # linker scripts stored in /lib on Hurd.
1588 find_command
= 'find %s/lib* -name "*.so*"' % self
.installdir
1589 cmdlist
.add_command('strip', ['sh', '-c', (
1590 'set -e; for f in $(%s); do '
1591 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1592 'done' % (find_command
, self
.strip
))])
1593 cmdlist
.add_command('check', ['make', 'check'])
1594 cmdlist
.add_command('save-logs', [self
.save_logs
], always_run
=True)
1596 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault
):
1597 """Build policy for glibc during update-syscalls."""
1599 def __init__(self
, glibc
):
1600 super().__init
__(glibc
)
1601 self
.builddir
= glibc
.ctx
.component_builddir(
1602 'update-syscalls', glibc
.name
, 'glibc')
1603 self
.linuxdir
= glibc
.ctx
.component_builddir(
1604 'update-syscalls', glibc
.name
, 'linux')
1605 self
.linux_policy
= LinuxHeadersPolicyForUpdateSyscalls(
1606 glibc
, self
.linuxdir
)
1607 self
.configure_args
.insert(
1608 0, '--with-headers=%s' % os
.path
.join(self
.linuxdir
, 'include'))
1609 # self.installdir not set because installation is not supported
1611 class Glibc(object):
1612 """A configuration for building glibc."""
1614 def __init__(self
, compiler
, arch
=None, os_name
=None, variant
=None,
1615 cfg
=None, ccopts
=None, cflags
=None):
1616 """Initialize a Glibc object."""
1617 self
.ctx
= compiler
.ctx
1618 self
.compiler
= compiler
1620 self
.arch
= compiler
.arch
1624 self
.os
= compiler
.os
1627 self
.variant
= variant
1629 self
.name
= '%s-%s' % (self
.arch
, self
.os
)
1631 self
.name
= '%s-%s-%s' % (self
.arch
, self
.os
, variant
)
1632 self
.triplet
= '%s-glibc-%s' % (self
.arch
, self
.os
)
1637 # ccopts contain ABI options and are passed to configure as CC / CXX.
1638 self
.ccopts
= ccopts
1639 # cflags contain non-ABI options like -g or -O and are passed to
1640 # configure as CFLAGS / CXXFLAGS.
1641 self
.cflags
= cflags
1643 def tool_name(self
, tool
):
1644 """Return the name of a cross-compilation tool."""
1645 ctool
= '%s-%s' % (self
.compiler
.triplet
, tool
)
1646 if self
.ccopts
and (tool
== 'gcc' or tool
== 'g++'):
1647 ctool
= '%s %s' % (ctool
, self
.ccopts
)
1651 """Generate commands to build this glibc."""
1652 builddir
= self
.ctx
.component_builddir('glibcs', self
.name
, 'glibc')
1653 installdir
= self
.ctx
.glibc_installdir(self
.name
)
1654 logsdir
= os
.path
.join(self
.ctx
.logsdir
, 'glibcs', self
.name
)
1655 self
.ctx
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
1656 cmdlist
= CommandList('glibcs-%s' % self
.name
, self
.ctx
.keep
)
1657 cmdlist
.add_command('check-compilers',
1659 os
.path
.join(self
.compiler
.installdir
, 'ok')])
1660 cmdlist
.use_path(self
.compiler
.bindir
)
1661 self
.build_glibc(cmdlist
, GlibcPolicyForBuild(self
))
1662 self
.ctx
.add_makefile_cmdlist('glibcs-%s' % self
.name
, cmdlist
,
1665 def build_glibc(self
, cmdlist
, policy
):
1666 """Generate commands to build this glibc, either as part of a compiler
1667 build or with the bootstrapped compiler (and in the latter case, run
1669 cmdlist
.create_use_dir(policy
.builddir
)
1670 policy
.configure(cmdlist
)
1671 cmdlist
.add_command('build', ['make'])
1672 cmdlist
.add_command('install', ['make', 'install',
1673 'install_root=%s' % policy
.installdir
])
1674 # GCC uses paths such as lib/../lib64, so make sure lib
1675 # directories always exist.
1676 mkdir_cmd
= ['mkdir', '-p',
1677 os
.path
.join(policy
.installdir
, 'lib')]
1679 mkdir_cmd
+= [os
.path
.join(policy
.installdir
, 'usr', 'lib')]
1680 cmdlist
.add_command('mkdir-lib', mkdir_cmd
)
1681 policy
.extra_commands(cmdlist
)
1682 cmdlist
.cleanup_dir()
1684 def update_syscalls(self
):
1685 if self
.os
== 'gnu':
1686 # Hurd does not have system call tables that need updating.
1689 policy
= GlibcPolicyForUpdateSyscalls(self
)
1690 logsdir
= os
.path
.join(self
.ctx
.logsdir
, 'update-syscalls', self
.name
)
1691 self
.ctx
.remove_recreate_dirs(policy
.builddir
, logsdir
)
1692 cmdlist
= CommandList('update-syscalls-%s' % self
.name
, self
.ctx
.keep
)
1693 cmdlist
.add_command('check-compilers',
1695 os
.path
.join(self
.compiler
.installdir
, 'ok')])
1696 cmdlist
.use_path(self
.compiler
.bindir
)
1698 install_linux_headers(policy
.linux_policy
, cmdlist
)
1700 cmdlist
.create_use_dir(policy
.builddir
)
1701 policy
.configure(cmdlist
)
1702 cmdlist
.add_command('build', ['make', 'update-syscall-lists'])
1703 cmdlist
.cleanup_dir()
1704 self
.ctx
.add_makefile_cmdlist('update-syscalls-%s' % self
.name
,
1707 class Command(object):
1708 """A command run in the build process."""
1710 def __init__(self
, desc
, num
, dir, path
, command
, always_run
=False):
1711 """Initialize a Command object."""
1715 trans
= str.maketrans({' ': '-'})
1716 self
.logbase
= '%03d-%s' % (num
, desc
.translate(trans
))
1717 self
.command
= command
1718 self
.always_run
= always_run
1721 def shell_make_quote_string(s
):
1722 """Given a string not containing a newline, quote it for use by the
1724 assert '\n' not in s
1725 if re
.fullmatch('[]+,./0-9@A-Z_a-z-]+', s
):
1727 strans
= str.maketrans({"'": "'\\''"})
1728 s
= "'%s'" % s
.translate(strans
)
1729 mtrans
= str.maketrans({'$': '$$'})
1730 return s
.translate(mtrans
)
1733 def shell_make_quote_list(l
, translate_make
):
1734 """Given a list of strings not containing newlines, quote them for use
1735 by the shell and make, returning a single string. If translate_make
1736 is true and the first string is 'make', change it to $(MAKE)."""
1737 l
= [Command
.shell_make_quote_string(s
) for s
in l
]
1738 if translate_make
and l
[0] == 'make':
1742 def shell_make_quote(self
):
1743 """Return this command quoted for the shell and make."""
1744 return self
.shell_make_quote_list(self
.command
, True)
1747 class CommandList(object):
1748 """A list of commands run in the build process."""
1750 def __init__(self
, desc
, keep
):
1751 """Initialize a CommandList object."""
1758 def desc_txt(self
, desc
):
1759 """Return the description to use for a command."""
1760 return '%s %s' % (' '.join(self
.desc
), desc
)
1762 def use_dir(self
, dir):
1763 """Set the default directory for subsequent commands."""
1766 def use_path(self
, path
):
1767 """Set a directory to be prepended to the PATH for subsequent
1771 def push_subdesc(self
, subdesc
):
1772 """Set the default subdescription for subsequent commands (e.g., the
1773 name of a component being built, within the series of commands
1775 self
.desc
.append(subdesc
)
1777 def pop_subdesc(self
):
1778 """Pop a subdescription from the list of descriptions."""
1781 def create_use_dir(self
, dir):
1782 """Remove and recreate a directory and use it for subsequent
1784 self
.add_command_dir('rm', None, ['rm', '-rf', dir])
1785 self
.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1788 def add_command_dir(self
, desc
, dir, command
, always_run
=False):
1789 """Add a command to run in a given directory."""
1790 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), dir, self
.path
,
1791 command
, always_run
)
1792 self
.cmdlist
.append(cmd
)
1794 def add_command(self
, desc
, command
, always_run
=False):
1795 """Add a command to run in the default directory."""
1796 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), self
.dir,
1797 self
.path
, command
, always_run
)
1798 self
.cmdlist
.append(cmd
)
1800 def cleanup_dir(self
, desc
='cleanup', dir=None):
1801 """Clean up a build directory. If no directory is specified, the
1802 default directory is cleaned up and ceases to be the default
1807 if self
.keep
!= 'all':
1808 self
.add_command_dir(desc
, None, ['rm', '-rf', dir],
1809 always_run
=(self
.keep
== 'none'))
1811 def makefile_commands(self
, wrapper
, logsdir
):
1812 """Return the sequence of commands in the form of text for a Makefile.
1813 The given wrapper script takes arguments: base of logs for
1814 previous command, or empty; base of logs for this command;
1815 description; directory; PATH addition; the command itself."""
1816 # prev_base is the base of the name for logs of the previous
1817 # command that is not always-run (that is, a build command,
1818 # whose failure should stop subsequent build commands from
1819 # being run, as opposed to a cleanup command, which is run
1820 # even if previous commands failed).
1823 for c
in self
.cmdlist
:
1824 ctxt
= c
.shell_make_quote()
1825 if prev_base
and not c
.always_run
:
1826 prev_log
= os
.path
.join(logsdir
, prev_base
)
1829 this_log
= os
.path
.join(logsdir
, c
.logbase
)
1830 if not c
.always_run
:
1831 prev_base
= c
.logbase
1840 prelims
= [wrapper
, prev_log
, this_log
, c
.desc
, dir, path
]
1841 prelim_txt
= Command
.shell_make_quote_list(prelims
, False)
1842 cmds
.append('\t@%s %s' % (prelim_txt
, ctxt
))
1843 return '\n'.join(cmds
)
1845 def status_logs(self
, logsdir
):
1846 """Return the list of log files with command status."""
1847 return [os
.path
.join(logsdir
, '%s-status.txt' % c
.logbase
)
1848 for c
in self
.cmdlist
]
1852 """Return an argument parser for this module."""
1853 parser
= argparse
.ArgumentParser(description
=__doc__
)
1854 parser
.add_argument('-j', dest
='parallelism',
1855 help='Run this number of jobs in parallel',
1856 type=int, default
=os
.cpu_count())
1857 parser
.add_argument('--keep', dest
='keep',
1858 help='Whether to keep all build directories, '
1859 'none or only those from failed builds',
1860 default
='none', choices
=('none', 'all', 'failed'))
1861 parser
.add_argument('--replace-sources', action
='store_true',
1862 help='Remove and replace source directories '
1863 'with the wrong version of a component')
1864 parser
.add_argument('--strip', action
='store_true',
1865 help='Strip installed glibc libraries')
1866 parser
.add_argument('--full-gcc', action
='store_true',
1867 help='Build GCC with all languages and libsanitizer')
1868 parser
.add_argument('--shallow', action
='store_true',
1869 help='Do not download Git history during checkout')
1870 parser
.add_argument('topdir',
1871 help='Toplevel working directory')
1872 parser
.add_argument('action',
1874 choices
=('checkout', 'bot-cycle', 'bot',
1875 'host-libraries', 'compilers', 'glibcs',
1876 'update-syscalls', 'list-compilers',
1878 parser
.add_argument('configs',
1879 help='Versions to check out or configurations to build',
1884 def get_version_common(progname
,line
,word
,delchars
,arg1
):
1886 out
= subprocess
.run([progname
, arg1
],
1887 stdout
=subprocess
.PIPE
,
1888 stderr
=subprocess
.DEVNULL
,
1889 stdin
=subprocess
.DEVNULL
,
1890 check
=True, universal_newlines
=True)
1891 v
= out
.stdout
.splitlines()[line
].split()[word
]
1893 v
= v
.replace(delchars
,'')
1894 return [int(x
) for x
in v
.split('.')]
1898 def get_version_common_stderr(progname
,line
,word
,delchars
,arg1
):
1900 out
= subprocess
.run([progname
, arg1
],
1901 stdout
=subprocess
.DEVNULL
,
1902 stderr
=subprocess
.PIPE
,
1903 stdin
=subprocess
.DEVNULL
,
1904 check
=True, universal_newlines
=True)
1905 v
= out
.stderr
.splitlines()[line
].split()[word
]
1907 v
= v
.replace(delchars
,'')
1908 return [int(x
) for x
in v
.split('.')]
1912 def get_version(progname
):
1913 return get_version_common (progname
, 0, -1, None, '--version');
1915 def get_version_awk(progname
):
1916 return get_version_common (progname
, 0, 2, ',', '--version');
1918 def get_version_bzip2(progname
):
1919 return get_version_common_stderr (progname
, 0, 6, ',', '-h');
1921 def check_version(ver
, req
):
1922 for v
, r
in zip(ver
, req
):
1929 def version_str(ver
):
1930 return '.'.join([str (x
) for x
in ver
])
1932 def check_for_required_tools():
1933 get_list_of_required_tools()
1935 count_missing_tools
= 0
1937 for k
, v
in REQUIRED_TOOLS
.items():
1939 if version
== 'missing':
1942 ok
= 'ok' if check_version (version
, v
[1]) else 'old'
1944 if count_old_tools
== 0:
1945 print("One or more required tools are too old:")
1946 count_old_tools
= count_old_tools
+ 1
1947 print('{:9}: {:3} (obtained=\"{}\" required=\"{}\")'.format(k
, ok
,
1948 version_str(version
), version_str(v
[1])))
1950 if count_missing_tools
== 0:
1951 print("One or more required tools are missing:")
1952 count_missing_tools
= count_missing_tools
+ 1
1953 print('{:9}: {:3} (required=\"{}\")'.format(k
, ok
,
1956 if count_old_tools
> 0 or count_missing_tools
> 0:
1960 """The main entry point."""
1961 check_for_required_tools();
1962 parser
= get_parser()
1963 opts
= parser
.parse_args(argv
)
1964 topdir
= os
.path
.abspath(opts
.topdir
)
1965 ctx
= Context(topdir
, opts
.parallelism
, opts
.keep
, opts
.replace_sources
,
1966 opts
.strip
, opts
.full_gcc
, opts
.action
,
1967 shallow
=opts
.shallow
)
1968 ctx
.run_builds(opts
.action
, opts
.configs
)
1971 if __name__
== '__main__':