2 # Build many configurations of glibc.
3 # Copyright (C) 2016-2023 Free Software Foundation, Inc.
4 # Copyright The GNU Toolchain Authors.
5 # This file is part of the GNU C Library.
7 # The GNU C Library is free software; you can redistribute it and/or
8 # modify it under the terms of the GNU Lesser General Public
9 # License as published by the Free Software Foundation; either
10 # version 2.1 of the License, or (at your option) any later version.
12 # The GNU C Library is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 # Lesser General Public License for more details.
17 # You should have received a copy of the GNU Lesser General Public
18 # License along with the GNU C Library; if not, see
19 # <https://www.gnu.org/licenses/>.
21 """Build many configurations of glibc.
23 This script takes as arguments a directory name (containing a src
24 subdirectory with sources of the relevant toolchain components) and a
25 description of what to do: 'checkout', to check out sources into that
26 directory, 'bot-cycle', to run a series of checkout and build steps,
27 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
28 libraries required by the toolchain, 'compilers', to build
29 cross-compilers for various configurations, or 'glibcs', to build
30 glibc for various configurations and run the compilation parts of the
31 testsuite. Subsequent arguments name the versions of components to
32 check out (<component>-<version), for 'checkout', or, for actions
33 other than 'checkout' and 'bot-cycle', name configurations for which
34 compilers or glibc are to be built.
36 The 'list-compilers' command prints the name of each available
37 compiler configuration, without building anything. The 'list-glibcs'
38 command prints the name of each glibc compiler configuration, followed
39 by the space, followed by the name of the compiler configuration used
40 for building this glibc variant.
46 import email
.mime
.text
59 # This is a list of system utilities that are expected to be available
60 # to this script, and, if a non-zero version is included, the minimum
61 # version required to work with this sccript.
62 def get_list_of_required_tools():
65 'awk' : (get_version_awk
, (0,0,0)),
66 'bison' : (get_version
, (0,0)),
67 'flex' : (get_version
, (0,0,0)),
68 'git' : (get_version
, (1,8,3)),
69 'make' : (get_version
, (4,0)),
70 'makeinfo' : (get_version
, (0,0)),
71 'patch' : (get_version
, (0,0,0)),
72 'sed' : (get_version
, (0,0)),
73 'tar' : (get_version
, (0,0,0)),
74 'gzip' : (get_version
, (0,0)),
75 'bzip2' : (get_version_bzip2
, (0,0,0)),
76 'xz' : (get_version
, (0,0,0)),
82 class _CompletedProcess
:
83 def __init__(self
, args
, returncode
, stdout
=None, stderr
=None):
85 self
.returncode
= returncode
89 def _run(*popenargs
, input=None, timeout
=None, check
=False, **kwargs
):
90 assert(timeout
is None)
91 with subprocess
.Popen(*popenargs
, **kwargs
) as process
:
93 stdout
, stderr
= process
.communicate(input)
98 returncode
= process
.poll()
99 if check
and returncode
:
100 raise subprocess
.CalledProcessError(returncode
, popenargs
)
101 return _CompletedProcess(popenargs
, returncode
, stdout
, stderr
)
103 subprocess
.run
= _run
106 class Context(object):
107 """The global state associated with builds in a given directory."""
109 def __init__(self
, topdir
, parallelism
, keep
, replace_sources
, strip
,
110 full_gcc
, action
, shallow
=False):
111 """Initialize the context."""
113 self
.parallelism
= parallelism
115 self
.replace_sources
= replace_sources
117 self
.full_gcc
= full_gcc
118 self
.shallow
= shallow
119 self
.srcdir
= os
.path
.join(topdir
, 'src')
120 self
.versions_json
= os
.path
.join(self
.srcdir
, 'versions.json')
121 self
.build_state_json
= os
.path
.join(topdir
, 'build-state.json')
122 self
.bot_config_json
= os
.path
.join(topdir
, 'bot-config.json')
123 self
.installdir
= os
.path
.join(topdir
, 'install')
124 self
.host_libraries_installdir
= os
.path
.join(self
.installdir
,
126 self
.builddir
= os
.path
.join(topdir
, 'build')
127 self
.logsdir
= os
.path
.join(topdir
, 'logs')
128 self
.logsdir_old
= os
.path
.join(topdir
, 'logs-old')
129 self
.makefile
= os
.path
.join(self
.builddir
, 'Makefile')
130 self
.wrapper
= os
.path
.join(self
.builddir
, 'wrapper')
131 self
.save_logs
= os
.path
.join(self
.builddir
, 'save-logs')
132 self
.script_text
= self
.get_script_text()
133 if action
not in ('checkout', 'list-compilers', 'list-glibcs'):
134 self
.build_triplet
= self
.get_build_triplet()
135 self
.glibc_version
= self
.get_glibc_version()
137 self
.glibc_configs
= {}
138 self
.makefile_pieces
= ['.PHONY: all\n']
139 self
.add_all_configs()
140 self
.load_versions_json()
141 self
.load_build_state_json()
142 self
.status_log_list
= []
143 self
.email_warning
= False
145 def get_script_text(self
):
146 """Return the text of this script."""
147 with
open(sys
.argv
[0], 'r') as f
:
151 """Re-execute this script with the same arguments."""
153 os
.execv(sys
.executable
, [sys
.executable
] + sys
.argv
)
155 def get_build_triplet(self
):
156 """Determine the build triplet with config.guess."""
157 config_guess
= os
.path
.join(self
.component_srcdir('gcc'),
159 cg_out
= subprocess
.run([config_guess
], stdout
=subprocess
.PIPE
,
160 check
=True, universal_newlines
=True).stdout
161 return cg_out
.rstrip()
163 def get_glibc_version(self
):
164 """Determine the glibc version number (major.minor)."""
165 version_h
= os
.path
.join(self
.component_srcdir('glibc'), 'version.h')
166 with
open(version_h
, 'r') as f
:
167 lines
= f
.readlines()
168 starttext
= '#define VERSION "'
170 if l
.startswith(starttext
):
171 l
= l
[len(starttext
):]
173 m
= re
.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l
)
174 return '%s.%s' % m
.group(1, 2)
175 print('error: could not determine glibc version')
178 def add_all_configs(self
):
179 """Add all known glibc build configurations."""
180 self
.add_config(arch
='aarch64',
182 extra_glibcs
=[{'variant': 'disable-multi-arch',
183 'cfg': ['--disable-multi-arch']}])
184 self
.add_config(arch
='aarch64_be',
186 self
.add_config(arch
='arc',
188 gcc_cfg
=['--disable-multilib', '--with-cpu=hs38'])
189 self
.add_config(arch
='arc',
190 os_name
='linux-gnuhf',
191 gcc_cfg
=['--disable-multilib', '--with-cpu=hs38_linux'])
192 self
.add_config(arch
='arceb',
194 gcc_cfg
=['--disable-multilib', '--with-cpu=hs38'])
195 self
.add_config(arch
='alpha',
197 self
.add_config(arch
='arm',
198 os_name
='linux-gnueabi',
199 extra_glibcs
=[{'variant': 'v4t',
200 'ccopts': '-march=armv4t'}])
201 self
.add_config(arch
='armeb',
202 os_name
='linux-gnueabi')
203 self
.add_config(arch
='armeb',
204 os_name
='linux-gnueabi',
206 gcc_cfg
=['--with-arch=armv7-a'])
207 self
.add_config(arch
='arm',
208 os_name
='linux-gnueabihf',
209 gcc_cfg
=['--with-float=hard', '--with-cpu=arm926ej-s'],
210 extra_glibcs
=[{'variant': 'v7a',
211 'ccopts': '-march=armv7-a -mfpu=vfpv3'},
214 '-mthumb -march=armv7-a -mfpu=vfpv3'},
215 {'variant': 'v7a-disable-multi-arch',
216 'ccopts': '-march=armv7-a -mfpu=vfpv3',
217 'cfg': ['--disable-multi-arch']}])
218 self
.add_config(arch
='armeb',
219 os_name
='linux-gnueabihf',
220 gcc_cfg
=['--with-float=hard', '--with-cpu=arm926ej-s'])
221 self
.add_config(arch
='armeb',
222 os_name
='linux-gnueabihf',
224 gcc_cfg
=['--with-float=hard', '--with-arch=armv7-a',
226 self
.add_config(arch
='csky',
227 os_name
='linux-gnuabiv2',
229 gcc_cfg
=['--disable-multilib'])
230 self
.add_config(arch
='csky',
231 os_name
='linux-gnuabiv2',
232 gcc_cfg
=['--with-float=hard', '--disable-multilib'])
233 self
.add_config(arch
='hppa',
235 self
.add_config(arch
='i686',
237 self
.add_config(arch
='ia64',
239 first_gcc_cfg
=['--with-system-libunwind'],
240 binutils_cfg
=['--enable-obsolete'])
241 self
.add_config(arch
='loongarch64',
244 gcc_cfg
=['--with-abi=lp64d','--disable-multilib'])
245 self
.add_config(arch
='loongarch64',
248 gcc_cfg
=['--with-abi=lp64s','--disable-multilib'])
249 self
.add_config(arch
='m68k',
251 gcc_cfg
=['--disable-multilib'])
252 self
.add_config(arch
='m68k',
255 gcc_cfg
=['--with-arch=cf', '--disable-multilib'])
256 self
.add_config(arch
='m68k',
258 variant
='coldfire-soft',
259 gcc_cfg
=['--with-arch=cf', '--with-cpu=54455',
260 '--disable-multilib'])
261 self
.add_config(arch
='microblaze',
263 gcc_cfg
=['--disable-multilib'])
264 self
.add_config(arch
='microblazeel',
266 gcc_cfg
=['--disable-multilib'])
267 self
.add_config(arch
='mips64',
269 gcc_cfg
=['--with-mips-plt'],
270 glibcs
=[{'variant': 'n32'},
272 'ccopts': '-mabi=32'},
274 'ccopts': '-mabi=64'}])
275 self
.add_config(arch
='mips64',
278 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
279 glibcs
=[{'variant': 'n32-soft'},
282 'ccopts': '-mabi=32'},
283 {'variant': 'n64-soft',
284 'ccopts': '-mabi=64'}])
285 self
.add_config(arch
='mips64',
288 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
289 '--with-arch-64=mips64r2',
290 '--with-arch-32=mips32r2'],
291 glibcs
=[{'variant': 'n32-nan2008'},
292 {'variant': 'nan2008',
294 'ccopts': '-mabi=32'},
295 {'variant': 'n64-nan2008',
296 'ccopts': '-mabi=64'}])
297 self
.add_config(arch
='mips64',
299 variant
='nan2008-soft',
300 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
301 '--with-arch-64=mips64r2',
302 '--with-arch-32=mips32r2',
303 '--with-float=soft'],
304 glibcs
=[{'variant': 'n32-nan2008-soft'},
305 {'variant': 'nan2008-soft',
307 'ccopts': '-mabi=32'},
308 {'variant': 'n64-nan2008-soft',
309 'ccopts': '-mabi=64'}])
310 self
.add_config(arch
='mips64el',
312 gcc_cfg
=['--with-mips-plt'],
313 glibcs
=[{'variant': 'n32'},
315 'ccopts': '-mabi=32'},
317 'ccopts': '-mabi=64'}])
318 self
.add_config(arch
='mips64el',
321 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
322 glibcs
=[{'variant': 'n32-soft'},
325 'ccopts': '-mabi=32'},
326 {'variant': 'n64-soft',
327 'ccopts': '-mabi=64'}])
328 self
.add_config(arch
='mips64el',
331 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
332 '--with-arch-64=mips64r2',
333 '--with-arch-32=mips32r2'],
334 glibcs
=[{'variant': 'n32-nan2008'},
335 {'variant': 'nan2008',
337 'ccopts': '-mabi=32'},
338 {'variant': 'n64-nan2008',
339 'ccopts': '-mabi=64'}])
340 self
.add_config(arch
='mips64el',
342 variant
='nan2008-soft',
343 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
344 '--with-arch-64=mips64r2',
345 '--with-arch-32=mips32r2',
346 '--with-float=soft'],
347 glibcs
=[{'variant': 'n32-nan2008-soft'},
348 {'variant': 'nan2008-soft',
350 'ccopts': '-mabi=32'},
351 {'variant': 'n64-nan2008-soft',
352 'ccopts': '-mabi=64'}])
353 self
.add_config(arch
='mipsisa64r6el',
355 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
356 '--with-arch-64=mips64r6',
357 '--with-arch-32=mips32r6',
358 '--with-float=hard'],
359 glibcs
=[{'variant': 'n32'},
360 {'arch': 'mipsisa32r6el',
361 'ccopts': '-mabi=32'},
363 'ccopts': '-mabi=64'}])
364 self
.add_config(arch
='nios2',
366 self
.add_config(arch
='or1k',
369 gcc_cfg
=['--with-multilib-list=mcmov'])
370 self
.add_config(arch
='powerpc',
372 gcc_cfg
=['--disable-multilib', '--enable-secureplt'],
373 extra_glibcs
=[{'variant': 'power4',
374 'ccopts': '-mcpu=power4',
375 'cfg': ['--with-cpu=power4']}])
376 self
.add_config(arch
='powerpc',
379 gcc_cfg
=['--disable-multilib', '--with-float=soft',
380 '--enable-secureplt'])
381 self
.add_config(arch
='powerpc64',
383 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
384 self
.add_config(arch
='powerpc64le',
386 gcc_cfg
=['--disable-multilib', '--enable-secureplt'],
387 extra_glibcs
=[{'variant': 'disable-multi-arch',
388 'cfg': ['--disable-multi-arch']}])
389 self
.add_config(arch
='riscv32',
391 variant
='rv32imac-ilp32',
392 gcc_cfg
=['--with-arch=rv32imac', '--with-abi=ilp32',
393 '--disable-multilib'])
394 self
.add_config(arch
='riscv32',
396 variant
='rv32imafdc-ilp32',
397 gcc_cfg
=['--with-arch=rv32imafdc', '--with-abi=ilp32',
398 '--disable-multilib'])
399 self
.add_config(arch
='riscv32',
401 variant
='rv32imafdc-ilp32d',
402 gcc_cfg
=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
403 '--disable-multilib'])
404 self
.add_config(arch
='riscv64',
406 variant
='rv64imac-lp64',
407 gcc_cfg
=['--with-arch=rv64imac', '--with-abi=lp64',
408 '--disable-multilib'])
409 self
.add_config(arch
='riscv64',
411 variant
='rv64imafdc-lp64',
412 gcc_cfg
=['--with-arch=rv64imafdc', '--with-abi=lp64',
413 '--disable-multilib'])
414 self
.add_config(arch
='riscv64',
416 variant
='rv64imafdc-lp64d',
417 gcc_cfg
=['--with-arch=rv64imafdc', '--with-abi=lp64d',
418 '--disable-multilib'])
419 self
.add_config(arch
='s390x',
422 {'arch': 's390', 'ccopts': '-m31'}],
423 extra_glibcs
=[{'variant': 'O3',
425 self
.add_config(arch
='sh3',
427 self
.add_config(arch
='sh3eb',
429 self
.add_config(arch
='sh4',
431 self
.add_config(arch
='sh4eb',
433 self
.add_config(arch
='sh4',
436 gcc_cfg
=['--without-fp'])
437 self
.add_config(arch
='sh4eb',
440 gcc_cfg
=['--without-fp'])
441 self
.add_config(arch
='sparc64',
445 'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
446 extra_glibcs
=[{'variant': 'leon3',
448 'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
449 {'variant': 'disable-multi-arch',
450 'cfg': ['--disable-multi-arch']},
451 {'variant': 'disable-multi-arch',
453 'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
454 'cfg': ['--disable-multi-arch']},
455 {'variant': 'enable-crypt',
456 'cfg': ['--enable-crypt']}])
457 self
.add_config(arch
='x86_64',
459 gcc_cfg
=['--with-multilib-list=m64,m32,mx32'],
461 {'variant': 'x32', 'ccopts': '-mx32'},
462 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
463 extra_glibcs
=[{'variant': 'disable-multi-arch',
464 'cfg': ['--disable-multi-arch']},
465 {'variant': 'minimal',
466 'cfg': ['--disable-multi-arch',
468 '--disable-timezone-tools',
471 '--disable-build-nscd',
473 {'variant': 'no-pie',
474 'cfg': ['--disable-default-pie']},
475 {'variant': 'x32-no-pie',
477 'cfg': ['--disable-default-pie']},
478 {'variant': 'no-pie',
480 'ccopts': '-m32 -march=i686',
481 'cfg': ['--disable-default-pie']},
482 {'variant': 'disable-multi-arch',
484 'ccopts': '-m32 -march=i686',
485 'cfg': ['--disable-multi-arch']},
487 'ccopts': '-m32 -march=i486'},
489 'ccopts': '-m32 -march=i586'},
490 {'variant': 'enable-fortify-source',
491 'cfg': ['--enable-fortify-source']},
492 {'variant': 'enable-crypt',
493 'cfg': ['--enable-crypt']}])
494 self
.add_config(arch
='x86_64',
496 gcc_cfg
=['--disable-multilib'])
498 def add_config(self
, **args
):
499 """Add an individual build configuration."""
500 cfg
= Config(self
, **args
)
501 if cfg
.name
in self
.configs
:
502 print('error: duplicate config %s' % cfg
.name
)
504 self
.configs
[cfg
.name
] = cfg
505 for c
in cfg
.all_glibcs
:
506 if c
.name
in self
.glibc_configs
:
507 print('error: duplicate glibc config %s' % c
.name
)
509 self
.glibc_configs
[c
.name
] = c
511 def component_srcdir(self
, component
):
512 """Return the source directory for a given component, e.g. gcc."""
513 return os
.path
.join(self
.srcdir
, component
)
515 def component_builddir(self
, action
, config
, component
, subconfig
=None):
516 """Return the directory to use for a build."""
519 assert subconfig
is None
520 return os
.path
.join(self
.builddir
, action
, component
)
521 if subconfig
is None:
522 return os
.path
.join(self
.builddir
, action
, config
, component
)
524 # glibc build as part of compiler build.
525 return os
.path
.join(self
.builddir
, action
, config
, component
,
528 def compiler_installdir(self
, config
):
529 """Return the directory in which to install a compiler."""
530 return os
.path
.join(self
.installdir
, 'compilers', config
)
532 def compiler_bindir(self
, config
):
533 """Return the directory in which to find compiler binaries."""
534 return os
.path
.join(self
.compiler_installdir(config
), 'bin')
536 def compiler_sysroot(self
, config
):
537 """Return the sysroot directory for a compiler."""
538 return os
.path
.join(self
.compiler_installdir(config
), 'sysroot')
540 def glibc_installdir(self
, config
):
541 """Return the directory in which to install glibc."""
542 return os
.path
.join(self
.installdir
, 'glibcs', config
)
544 def run_builds(self
, action
, configs
):
545 """Run the requested builds."""
546 if action
== 'checkout':
547 self
.checkout(configs
)
549 if action
== 'bot-cycle':
551 print('error: configurations specified for bot-cycle')
557 print('error: configurations specified for bot')
561 if action
in ('host-libraries', 'list-compilers',
562 'list-glibcs') and configs
:
563 print('error: configurations specified for ' + action
)
565 if action
== 'list-compilers':
566 for name
in sorted(self
.configs
.keys()):
569 if action
== 'list-glibcs':
570 for config
in sorted(self
.glibc_configs
.values(),
571 key
=lambda c
: c
.name
):
572 print(config
.name
, config
.compiler
.name
)
574 self
.clear_last_build_state(action
)
575 build_time
= datetime
.datetime
.utcnow()
576 if action
== 'host-libraries':
577 build_components
= ('gmp', 'mpfr', 'mpc')
580 self
.build_host_libraries()
581 elif action
== 'compilers':
582 build_components
= ('binutils', 'gcc', 'glibc', 'linux', 'mig',
584 old_components
= ('gmp', 'mpfr', 'mpc')
585 old_versions
= self
.build_state
['host-libraries']['build-versions']
586 self
.build_compilers(configs
)
588 build_components
= ('glibc',)
589 old_components
= ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
590 'mig', 'gnumach', 'hurd')
591 old_versions
= self
.build_state
['compilers']['build-versions']
592 if action
== 'update-syscalls':
593 self
.update_syscalls(configs
)
595 self
.build_glibcs(configs
)
599 # Partial build, do not update stored state.
602 for k
in build_components
:
603 if k
in self
.versions
:
604 build_versions
[k
] = {'version': self
.versions
[k
]['version'],
605 'revision': self
.versions
[k
]['revision']}
606 for k
in old_components
:
607 if k
in old_versions
:
608 build_versions
[k
] = {'version': old_versions
[k
]['version'],
609 'revision': old_versions
[k
]['revision']}
610 self
.update_build_state(action
, build_time
, build_versions
)
613 def remove_dirs(*args
):
614 """Remove directories and their contents if they exist."""
616 shutil
.rmtree(dir, ignore_errors
=True)
619 def remove_recreate_dirs(*args
):
620 """Remove directories if they exist, and create them as empty."""
621 Context
.remove_dirs(*args
)
623 os
.makedirs(dir, exist_ok
=True)
625 def add_makefile_cmdlist(self
, target
, cmdlist
, logsdir
):
626 """Add makefile text for a list of commands."""
627 commands
= cmdlist
.makefile_commands(self
.wrapper
, logsdir
)
628 self
.makefile_pieces
.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
629 (target
, target
, target
, commands
))
630 self
.status_log_list
.extend(cmdlist
.status_logs(logsdir
))
632 def write_files(self
):
633 """Write out the Makefile and wrapper script."""
634 mftext
= ''.join(self
.makefile_pieces
)
635 with
open(self
.makefile
, 'w') as f
:
645 'prev_status=$prev_base-status.txt\n'
646 'this_status=$this_base-status.txt\n'
647 'this_log=$this_base-log.txt\n'
648 'date > "$this_log"\n'
649 'echo >> "$this_log"\n'
650 'echo "Description: $desc" >> "$this_log"\n'
651 'printf "%s" "Command:" >> "$this_log"\n'
652 'for word in "$@"; do\n'
653 ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
654 ' printf " %s" "$word"\n'
657 ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
660 'done >> "$this_log"\n'
661 'echo >> "$this_log"\n'
662 'echo "Directory: $dir" >> "$this_log"\n'
663 'echo "Path addition: $path" >> "$this_log"\n'
664 'echo >> "$this_log"\n'
667 ' echo >> "$this_log"\n'
668 ' echo "$1: $desc" > "$this_status"\n'
669 ' echo "$1: $desc" >> "$this_log"\n'
670 ' echo >> "$this_log"\n'
671 ' date >> "$this_log"\n'
672 ' echo "$1: $desc"\n'
677 ' if [ "$1" != "0" ]; then\n'
678 ' record_status FAIL\n'
681 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
682 ' record_status UNRESOLVED\n'
684 'if [ "$dir" ]; then\n'
686 ' check_error "$?"\n'
688 'if [ "$path" ]; then\n'
689 ' PATH=$path:$PATH\n'
691 '"$@" < /dev/null >> "$this_log" 2>&1\n'
693 'record_status PASS\n')
694 with
open(self
.wrapper
, 'w') as f
:
695 f
.write(wrapper_text
)
697 mode_exec
= (stat
.S_IRWXU|stat
.S_IRGRP|stat
.S_IXGRP|
698 stat
.S_IROTH|stat
.S_IXOTH
)
699 os
.chmod(self
.wrapper
, mode_exec
)
702 'if ! [ -f tests.sum ]; then\n'
703 ' echo "No test summary available."\n'
708 ' echo "Contents of $1:"\n'
712 ' echo "End of contents of $1."\n'
715 'save_file tests.sum\n'
716 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
717 'for t in $non_pass_tests; do\n'
718 ' if [ -f "$t.out" ]; then\n'
719 ' save_file "$t.out"\n'
722 with
open(self
.save_logs
, 'w') as f
:
723 f
.write(save_logs_text
)
724 os
.chmod(self
.save_logs
, mode_exec
)
727 """Do the actual build."""
728 cmd
= ['make', '-O', '-j%d' % self
.parallelism
]
729 subprocess
.run(cmd
, cwd
=self
.builddir
, check
=True)
731 def build_host_libraries(self
):
732 """Build the host libraries."""
733 installdir
= self
.host_libraries_installdir
734 builddir
= os
.path
.join(self
.builddir
, 'host-libraries')
735 logsdir
= os
.path
.join(self
.logsdir
, 'host-libraries')
736 self
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
737 cmdlist
= CommandList('host-libraries', self
.keep
)
738 self
.build_host_library(cmdlist
, 'gmp')
739 self
.build_host_library(cmdlist
, 'mpfr',
740 ['--with-gmp=%s' % installdir
])
741 self
.build_host_library(cmdlist
, 'mpc',
742 ['--with-gmp=%s' % installdir
,
743 '--with-mpfr=%s' % installdir
])
744 cmdlist
.add_command('done', ['touch', os
.path
.join(installdir
, 'ok')])
745 self
.add_makefile_cmdlist('host-libraries', cmdlist
, logsdir
)
747 def build_host_library(self
, cmdlist
, lib
, extra_opts
=None):
748 """Build one host library."""
749 srcdir
= self
.component_srcdir(lib
)
750 builddir
= self
.component_builddir('host-libraries', None, lib
)
751 installdir
= self
.host_libraries_installdir
752 cmdlist
.push_subdesc(lib
)
753 cmdlist
.create_use_dir(builddir
)
754 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
755 '--prefix=%s' % installdir
,
758 cfg_cmd
.extend (extra_opts
)
759 cmdlist
.add_command('configure', cfg_cmd
)
760 cmdlist
.add_command('build', ['make'])
761 cmdlist
.add_command('check', ['make', 'check'])
762 cmdlist
.add_command('install', ['make', 'install'])
763 cmdlist
.cleanup_dir()
764 cmdlist
.pop_subdesc()
766 def build_compilers(self
, configs
):
767 """Build the compilers."""
769 self
.remove_dirs(os
.path
.join(self
.builddir
, 'compilers'))
770 self
.remove_dirs(os
.path
.join(self
.installdir
, 'compilers'))
771 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'compilers'))
772 configs
= sorted(self
.configs
.keys())
774 self
.configs
[c
].build()
776 def build_glibcs(self
, configs
):
777 """Build the glibcs."""
779 self
.remove_dirs(os
.path
.join(self
.builddir
, 'glibcs'))
780 self
.remove_dirs(os
.path
.join(self
.installdir
, 'glibcs'))
781 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'glibcs'))
782 configs
= sorted(self
.glibc_configs
.keys())
784 self
.glibc_configs
[c
].build()
786 def update_syscalls(self
, configs
):
787 """Update the glibc syscall lists."""
789 self
.remove_dirs(os
.path
.join(self
.builddir
, 'update-syscalls'))
790 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'update-syscalls'))
791 configs
= sorted(self
.glibc_configs
.keys())
793 self
.glibc_configs
[c
].update_syscalls()
795 def load_versions_json(self
):
796 """Load information about source directory versions."""
797 if not os
.access(self
.versions_json
, os
.F_OK
):
800 with
open(self
.versions_json
, 'r') as f
:
801 self
.versions
= json
.load(f
)
803 def store_json(self
, data
, filename
):
804 """Store information in a JSON file."""
805 filename_tmp
= filename
+ '.tmp'
806 with
open(filename_tmp
, 'w') as f
:
807 json
.dump(data
, f
, indent
=2, sort_keys
=True)
808 os
.rename(filename_tmp
, filename
)
810 def store_versions_json(self
):
811 """Store information about source directory versions."""
812 self
.store_json(self
.versions
, self
.versions_json
)
814 def set_component_version(self
, component
, version
, explicit
, revision
):
815 """Set the version information for a component."""
816 self
.versions
[component
] = {'version': version
,
817 'explicit': explicit
,
818 'revision': revision
}
819 self
.store_versions_json()
821 def checkout(self
, versions
):
822 """Check out the desired component versions."""
823 default_versions
= {'binutils': 'vcs-2.41',
825 'glibc': 'vcs-mainline',
830 'mig': 'vcs-mainline',
831 'gnumach': 'vcs-mainline',
832 'hurd': 'vcs-mainline'}
834 explicit_versions
= {}
837 for k
in default_versions
.keys():
841 if k
in use_versions
:
842 print('error: multiple versions for %s' % k
)
845 explicit_versions
[k
] = True
849 print('error: unknown component in %s' % v
)
851 for k
in default_versions
.keys():
852 if k
not in use_versions
:
853 if k
in self
.versions
and self
.versions
[k
]['explicit']:
854 use_versions
[k
] = self
.versions
[k
]['version']
855 explicit_versions
[k
] = True
857 use_versions
[k
] = default_versions
[k
]
858 explicit_versions
[k
] = False
859 os
.makedirs(self
.srcdir
, exist_ok
=True)
860 for k
in sorted(default_versions
.keys()):
861 update
= os
.access(self
.component_srcdir(k
), os
.F_OK
)
864 k
in self
.versions
and
865 v
!= self
.versions
[k
]['version']):
866 if not self
.replace_sources
:
867 print('error: version of %s has changed from %s to %s, '
868 'use --replace-sources to check out again' %
869 (k
, self
.versions
[k
]['version'], v
))
871 shutil
.rmtree(self
.component_srcdir(k
))
873 if v
.startswith('vcs-'):
874 revision
= self
.checkout_vcs(k
, v
[4:], update
)
876 self
.checkout_tar(k
, v
, update
)
878 self
.set_component_version(k
, v
, explicit_versions
[k
], revision
)
879 if self
.get_script_text() != self
.script_text
:
880 # Rerun the checkout process in case the updated script
881 # uses different default versions or new components.
884 def checkout_vcs(self
, component
, version
, update
):
885 """Check out the given version of the given component from version
886 control. Return a revision identifier."""
887 if component
== 'binutils':
888 git_url
= 'https://sourceware.org/git/binutils-gdb.git'
889 if version
== 'mainline':
890 git_branch
= 'master'
892 trans
= str.maketrans({'.': '_'})
893 git_branch
= 'binutils-%s-branch' % version
.translate(trans
)
894 return self
.git_checkout(component
, git_url
, git_branch
, update
)
895 elif component
== 'gcc':
896 if version
== 'mainline':
899 branch
= 'releases/gcc-%s' % version
900 return self
.gcc_checkout(branch
, update
)
901 elif component
== 'glibc':
902 git_url
= 'https://sourceware.org/git/glibc.git'
903 if version
== 'mainline':
904 git_branch
= 'master'
906 git_branch
= 'release/%s/master' % version
907 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
908 self
.fix_glibc_timestamps()
910 elif component
== 'gnumach':
911 git_url
= 'git://git.savannah.gnu.org/hurd/gnumach.git'
912 git_branch
= 'master'
913 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
914 subprocess
.run(['autoreconf', '-i'],
915 cwd
=self
.component_srcdir(component
), check
=True)
917 elif component
== 'mig':
918 git_url
= 'git://git.savannah.gnu.org/hurd/mig.git'
919 git_branch
= 'master'
920 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
921 subprocess
.run(['autoreconf', '-i'],
922 cwd
=self
.component_srcdir(component
), check
=True)
924 elif component
== 'hurd':
925 git_url
= 'git://git.savannah.gnu.org/hurd/hurd.git'
926 git_branch
= 'master'
927 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
928 subprocess
.run(['autoconf'],
929 cwd
=self
.component_srcdir(component
), check
=True)
932 print('error: component %s coming from VCS' % component
)
935 def git_checkout(self
, component
, git_url
, git_branch
, update
):
936 """Check out a component from git. Return a commit identifier."""
938 subprocess
.run(['git', 'remote', 'prune', 'origin'],
939 cwd
=self
.component_srcdir(component
), check
=True)
940 if self
.replace_sources
:
941 subprocess
.run(['git', 'clean', '-dxfq'],
942 cwd
=self
.component_srcdir(component
), check
=True)
943 subprocess
.run(['git', 'pull', '-q'],
944 cwd
=self
.component_srcdir(component
), check
=True)
947 depth_arg
= ('--depth', '1')
950 subprocess
.run(['git', 'clone', '-q', '-b', git_branch
,
952 self
.component_srcdir(component
)], check
=True)
953 r
= subprocess
.run(['git', 'rev-parse', 'HEAD'],
954 cwd
=self
.component_srcdir(component
),
955 stdout
=subprocess
.PIPE
,
956 check
=True, universal_newlines
=True).stdout
959 def fix_glibc_timestamps(self
):
960 """Fix timestamps in a glibc checkout."""
961 # Ensure that builds do not try to regenerate generated files
962 # in the source tree.
963 srcdir
= self
.component_srcdir('glibc')
964 # These files have Makefile dependencies to regenerate them in
965 # the source tree that may be active during a normal build.
966 # Some other files have such dependencies but do not need to
967 # be touched because nothing in a build depends on the files
969 for f
in ('sysdeps/mach/hurd/bits/errno.h',):
970 to_touch
= os
.path
.join(srcdir
, f
)
971 subprocess
.run(['touch', '-c', to_touch
], check
=True)
972 for dirpath
, dirnames
, filenames
in os
.walk(srcdir
):
974 if (f
== 'configure' or
975 f
== 'preconfigure' or
976 f
.endswith('-kw.h')):
977 to_touch
= os
.path
.join(dirpath
, f
)
978 subprocess
.run(['touch', to_touch
], check
=True)
980 def gcc_checkout(self
, branch
, update
):
981 """Check out GCC from git. Return the commit identifier."""
982 if os
.access(os
.path
.join(self
.component_srcdir('gcc'), '.svn'),
984 if not self
.replace_sources
:
985 print('error: GCC has moved from SVN to git, use '
986 '--replace-sources to check out again')
988 shutil
.rmtree(self
.component_srcdir('gcc'))
991 self
.git_checkout('gcc', 'https://gcc.gnu.org/git/gcc.git',
993 subprocess
.run(['contrib/gcc_update', '--silent'],
994 cwd
=self
.component_srcdir('gcc'), check
=True)
995 r
= subprocess
.run(['git', 'rev-parse', 'HEAD'],
996 cwd
=self
.component_srcdir('gcc'),
997 stdout
=subprocess
.PIPE
,
998 check
=True, universal_newlines
=True).stdout
1001 def checkout_tar(self
, component
, version
, update
):
1002 """Check out the given version of the given component from a
1006 url_map
= {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
1007 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
1008 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
1009 'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
1010 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
1011 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
1012 'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
1013 'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
1014 'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
1015 if component
not in url_map
:
1016 print('error: component %s coming from tarball' % component
)
1018 version_major
= version
.split('.')[0]
1019 url
= url_map
[component
] % {'version': version
, 'major': version_major
}
1020 filename
= os
.path
.join(self
.srcdir
, url
.split('/')[-1])
1021 response
= urllib
.request
.urlopen(url
)
1022 data
= response
.read()
1023 with
open(filename
, 'wb') as f
:
1025 subprocess
.run(['tar', '-C', self
.srcdir
, '-x', '-f', filename
],
1027 os
.rename(os
.path
.join(self
.srcdir
, '%s-%s' % (component
, version
)),
1028 self
.component_srcdir(component
))
1031 def load_build_state_json(self
):
1032 """Load information about the state of previous builds."""
1033 if os
.access(self
.build_state_json
, os
.F_OK
):
1034 with
open(self
.build_state_json
, 'r') as f
:
1035 self
.build_state
= json
.load(f
)
1037 self
.build_state
= {}
1038 for k
in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
1039 if k
not in self
.build_state
:
1040 self
.build_state
[k
] = {}
1041 if 'build-time' not in self
.build_state
[k
]:
1042 self
.build_state
[k
]['build-time'] = ''
1043 if 'build-versions' not in self
.build_state
[k
]:
1044 self
.build_state
[k
]['build-versions'] = {}
1045 if 'build-results' not in self
.build_state
[k
]:
1046 self
.build_state
[k
]['build-results'] = {}
1047 if 'result-changes' not in self
.build_state
[k
]:
1048 self
.build_state
[k
]['result-changes'] = {}
1049 if 'ever-passed' not in self
.build_state
[k
]:
1050 self
.build_state
[k
]['ever-passed'] = []
1052 def store_build_state_json(self
):
1053 """Store information about the state of previous builds."""
1054 self
.store_json(self
.build_state
, self
.build_state_json
)
1056 def clear_last_build_state(self
, action
):
1057 """Clear information about the state of part of the build."""
1058 # We clear the last build time and versions when starting a
1059 # new build. The results of the last build are kept around,
1060 # as comparison is still meaningful if this build is aborted
1061 # and a new one started.
1062 self
.build_state
[action
]['build-time'] = ''
1063 self
.build_state
[action
]['build-versions'] = {}
1064 self
.store_build_state_json()
1066 def update_build_state(self
, action
, build_time
, build_versions
):
1067 """Update the build state after a build."""
1068 build_time
= build_time
.replace(microsecond
=0)
1069 self
.build_state
[action
]['build-time'] = str(build_time
)
1070 self
.build_state
[action
]['build-versions'] = build_versions
1072 for log
in self
.status_log_list
:
1073 with
open(log
, 'r') as f
:
1075 log_text
= log_text
.rstrip()
1076 m
= re
.fullmatch('([A-Z]+): (.*)', log_text
)
1078 test_name
= m
.group(2)
1079 assert test_name
not in build_results
1080 build_results
[test_name
] = result
1081 old_build_results
= self
.build_state
[action
]['build-results']
1082 self
.build_state
[action
]['build-results'] = build_results
1084 all_tests
= set(old_build_results
.keys()) |
set(build_results
.keys())
1086 if t
in old_build_results
:
1087 old_res
= old_build_results
[t
]
1089 old_res
= '(New test)'
1090 if t
in build_results
:
1091 new_res
= build_results
[t
]
1093 new_res
= '(Test removed)'
1094 if old_res
!= new_res
:
1095 result_changes
[t
] = '%s -> %s' % (old_res
, new_res
)
1096 self
.build_state
[action
]['result-changes'] = result_changes
1097 old_ever_passed
= {t
for t
in self
.build_state
[action
]['ever-passed']
1098 if t
in build_results
}
1099 new_passes
= {t
for t
in build_results
if build_results
[t
] == 'PASS'}
1100 self
.build_state
[action
]['ever-passed'] = sorted(old_ever_passed |
1102 self
.store_build_state_json()
1104 def load_bot_config_json(self
):
1105 """Load bot configuration."""
1106 with
open(self
.bot_config_json
, 'r') as f
:
1107 self
.bot_config
= json
.load(f
)
1109 def part_build_old(self
, action
, delay
):
1110 """Return whether the last build for a given action was at least a
1111 given number of seconds ago, or does not have a time recorded."""
1112 old_time_str
= self
.build_state
[action
]['build-time']
1113 if not old_time_str
:
1115 old_time
= datetime
.datetime
.strptime(old_time_str
,
1116 '%Y-%m-%d %H:%M:%S')
1117 new_time
= datetime
.datetime
.utcnow()
1118 delta
= new_time
- old_time
1119 return delta
.total_seconds() >= delay
1121 def bot_cycle(self
):
1122 """Run a single round of checkout and builds."""
1123 print('Bot cycle starting %s.' % str(datetime
.datetime
.utcnow()))
1124 self
.load_bot_config_json()
1125 actions
= ('host-libraries', 'compilers', 'glibcs')
1126 self
.bot_run_self(['--replace-sources'], 'checkout')
1127 self
.load_versions_json()
1128 if self
.get_script_text() != self
.script_text
:
1129 print('Script changed, re-execing.')
1130 # On script change, all parts of the build should be rerun.
1132 self
.clear_last_build_state(a
)
1134 check_components
= {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1135 'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1136 'mig', 'gnumach', 'hurd'),
1137 'glibcs': ('glibc',)}
1140 build_vers
= self
.build_state
[a
]['build-versions']
1141 must_build
[a
] = False
1142 if not self
.build_state
[a
]['build-time']:
1143 must_build
[a
] = True
1146 for c
in check_components
[a
]:
1148 old_vers
[c
] = build_vers
[c
]
1149 new_vers
[c
] = {'version': self
.versions
[c
]['version'],
1150 'revision': self
.versions
[c
]['revision']}
1151 if new_vers
== old_vers
:
1152 print('Versions for %s unchanged.' % a
)
1154 print('Versions changed or rebuild forced for %s.' % a
)
1155 if a
== 'compilers' and not self
.part_build_old(
1156 a
, self
.bot_config
['compilers-rebuild-delay']):
1157 print('Not requiring rebuild of compilers this soon.')
1159 must_build
[a
] = True
1160 if must_build
['host-libraries']:
1161 must_build
['compilers'] = True
1162 if must_build
['compilers']:
1163 must_build
['glibcs'] = True
1166 print('Must rebuild %s.' % a
)
1167 self
.clear_last_build_state(a
)
1169 print('No need to rebuild %s.' % a
)
1170 if os
.access(self
.logsdir
, os
.F_OK
):
1171 shutil
.rmtree(self
.logsdir_old
, ignore_errors
=True)
1172 shutil
.copytree(self
.logsdir
, self
.logsdir_old
)
1175 build_time
= datetime
.datetime
.utcnow()
1176 print('Rebuilding %s at %s.' % (a
, str(build_time
)))
1177 self
.bot_run_self([], a
)
1178 self
.load_build_state_json()
1179 self
.bot_build_mail(a
, build_time
)
1180 print('Bot cycle done at %s.' % str(datetime
.datetime
.utcnow()))
1182 def bot_build_mail(self
, action
, build_time
):
1183 """Send email with the results of a build."""
1184 if not ('email-from' in self
.bot_config
and
1185 'email-server' in self
.bot_config
and
1186 'email-subject' in self
.bot_config
and
1187 'email-to' in self
.bot_config
):
1188 if not self
.email_warning
:
1189 print("Email not configured, not sending.")
1190 self
.email_warning
= True
1193 build_time
= build_time
.replace(microsecond
=0)
1194 subject
= (self
.bot_config
['email-subject'] %
1196 'build-time': str(build_time
)})
1197 results
= self
.build_state
[action
]['build-results']
1198 changes
= self
.build_state
[action
]['result-changes']
1199 ever_passed
= set(self
.build_state
[action
]['ever-passed'])
1200 versions
= self
.build_state
[action
]['build-versions']
1201 new_regressions
= {k
for k
in changes
if changes
[k
] == 'PASS -> FAIL'}
1202 all_regressions
= {k
for k
in ever_passed
if results
[k
] == 'FAIL'}
1203 all_fails
= {k
for k
in results
if results
[k
] == 'FAIL'}
1205 new_reg_list
= sorted(['FAIL: %s' % k
for k
in new_regressions
])
1206 new_reg_text
= ('New regressions:\n\n%s\n\n' %
1207 '\n'.join(new_reg_list
))
1211 all_reg_list
= sorted(['FAIL: %s' % k
for k
in all_regressions
])
1212 all_reg_text
= ('All regressions:\n\n%s\n\n' %
1213 '\n'.join(all_reg_list
))
1217 all_fail_list
= sorted(['FAIL: %s' % k
for k
in all_fails
])
1218 all_fail_text
= ('All failures:\n\n%s\n\n' %
1219 '\n'.join(all_fail_list
))
1223 changes_list
= sorted(changes
.keys())
1224 changes_list
= ['%s: %s' % (changes
[k
], k
) for k
in changes_list
]
1225 changes_text
= ('All changed results:\n\n%s\n\n' %
1226 '\n'.join(changes_list
))
1229 results_text
= (new_reg_text
+ all_reg_text
+ all_fail_text
+
1231 if not results_text
:
1232 results_text
= 'Clean build with unchanged results.\n\n'
1233 versions_list
= sorted(versions
.keys())
1234 versions_list
= ['%s: %s (%s)' % (k
, versions
[k
]['version'],
1235 versions
[k
]['revision'])
1236 for k
in versions_list
]
1237 versions_text
= ('Component versions for this build:\n\n%s\n' %
1238 '\n'.join(versions_list
))
1239 body_text
= results_text
+ versions_text
1240 msg
= email
.mime
.text
.MIMEText(body_text
)
1241 msg
['Subject'] = subject
1242 msg
['From'] = self
.bot_config
['email-from']
1243 msg
['To'] = self
.bot_config
['email-to']
1244 msg
['Message-ID'] = email
.utils
.make_msgid()
1245 msg
['Date'] = email
.utils
.format_datetime(datetime
.datetime
.utcnow())
1246 with smtplib
.SMTP(self
.bot_config
['email-server']) as s
:
1249 def bot_run_self(self
, opts
, action
, check
=True):
1250 """Run a copy of this script with given options."""
1251 cmd
= [sys
.executable
, sys
.argv
[0], '--keep=none',
1252 '-j%d' % self
.parallelism
]
1254 cmd
.append('--full-gcc')
1256 cmd
.extend([self
.topdir
, action
])
1258 subprocess
.run(cmd
, check
=check
)
1261 """Run repeated rounds of checkout and builds."""
1263 self
.load_bot_config_json()
1264 if not self
.bot_config
['run']:
1265 print('Bot exiting by request.')
1267 self
.bot_run_self([], 'bot-cycle', check
=False)
1268 self
.load_bot_config_json()
1269 if not self
.bot_config
['run']:
1270 print('Bot exiting by request.')
1272 time
.sleep(self
.bot_config
['delay'])
1273 if self
.get_script_text() != self
.script_text
:
1274 print('Script changed, bot re-execing.')
1277 class LinuxHeadersPolicyForBuild(object):
1278 """Names and directories for installing Linux headers. Build variant."""
1280 def __init__(self
, config
):
1281 self
.arch
= config
.arch
1282 self
.srcdir
= config
.ctx
.component_srcdir('linux')
1283 self
.builddir
= config
.component_builddir('linux')
1284 self
.headers_dir
= os
.path
.join(config
.sysroot
, 'usr')
1286 class LinuxHeadersPolicyForUpdateSyscalls(object):
1287 """Names and directories for Linux headers. update-syscalls variant."""
1289 def __init__(self
, glibc
, headers_dir
):
1290 self
.arch
= glibc
.compiler
.arch
1291 self
.srcdir
= glibc
.compiler
.ctx
.component_srcdir('linux')
1292 self
.builddir
= glibc
.ctx
.component_builddir(
1293 'update-syscalls', glibc
.name
, 'build-linux')
1294 self
.headers_dir
= headers_dir
1296 def install_linux_headers(policy
, cmdlist
):
1297 """Install Linux kernel headers."""
1298 arch_map
= {'aarch64': 'arm64',
1309 'loongarch64': 'loongarch',
1311 'microblaze': 'microblaze',
1315 'powerpc': 'powerpc',
1324 if policy
.arch
.startswith(k
):
1325 linux_arch
= arch_map
[k
]
1327 assert linux_arch
is not None
1328 cmdlist
.push_subdesc('linux')
1329 cmdlist
.create_use_dir(policy
.builddir
)
1330 cmdlist
.add_command('install-headers',
1331 ['make', '-C', policy
.srcdir
, 'O=%s' % policy
.builddir
,
1332 'ARCH=%s' % linux_arch
,
1333 'INSTALL_HDR_PATH=%s' % policy
.headers_dir
,
1335 cmdlist
.cleanup_dir()
1336 cmdlist
.pop_subdesc()
1338 class Config(object):
1339 """A configuration for building a compiler and associated libraries."""
1341 def __init__(self
, ctx
, arch
, os_name
, variant
=None, gcc_cfg
=None,
1342 first_gcc_cfg
=None, binutils_cfg
=None, glibcs
=None,
1344 """Initialize a Config object."""
1348 self
.variant
= variant
1350 self
.name
= '%s-%s' % (arch
, os_name
)
1352 self
.name
= '%s-%s-%s' % (arch
, os_name
, variant
)
1353 self
.triplet
= '%s-glibc-%s' % (arch
, os_name
)
1357 self
.gcc_cfg
= gcc_cfg
1358 if first_gcc_cfg
is None:
1359 self
.first_gcc_cfg
= []
1361 self
.first_gcc_cfg
= first_gcc_cfg
1362 if binutils_cfg
is None:
1363 self
.binutils_cfg
= []
1365 self
.binutils_cfg
= binutils_cfg
1367 glibcs
= [{'variant': variant
}]
1368 if extra_glibcs
is None:
1370 glibcs
= [Glibc(self
, **g
) for g
in glibcs
]
1371 extra_glibcs
= [Glibc(self
, **g
) for g
in extra_glibcs
]
1372 self
.all_glibcs
= glibcs
+ extra_glibcs
1373 self
.compiler_glibcs
= glibcs
1374 self
.installdir
= ctx
.compiler_installdir(self
.name
)
1375 self
.bindir
= ctx
.compiler_bindir(self
.name
)
1376 self
.sysroot
= ctx
.compiler_sysroot(self
.name
)
1377 self
.builddir
= os
.path
.join(ctx
.builddir
, 'compilers', self
.name
)
1378 self
.logsdir
= os
.path
.join(ctx
.logsdir
, 'compilers', self
.name
)
1380 def component_builddir(self
, component
):
1381 """Return the directory to use for a (non-glibc) build."""
1382 return self
.ctx
.component_builddir('compilers', self
.name
, component
)
1385 """Generate commands to build this compiler."""
1386 self
.ctx
.remove_recreate_dirs(self
.installdir
, self
.builddir
,
1388 cmdlist
= CommandList('compilers-%s' % self
.name
, self
.ctx
.keep
)
1389 cmdlist
.add_command('check-host-libraries',
1391 os
.path
.join(self
.ctx
.host_libraries_installdir
,
1393 cmdlist
.use_path(self
.bindir
)
1394 self
.build_cross_tool(cmdlist
, 'binutils', 'binutils',
1396 '--disable-gdbserver',
1397 '--disable-libdecnumber',
1398 '--disable-readline',
1399 '--disable-sim'] + self
.binutils_cfg
)
1400 if self
.os
.startswith('linux'):
1401 install_linux_headers(LinuxHeadersPolicyForBuild(self
), cmdlist
)
1402 self
.build_gcc(cmdlist
, True)
1403 if self
.os
== 'gnu':
1404 self
.install_gnumach_headers(cmdlist
)
1405 self
.build_cross_tool(cmdlist
, 'mig', 'mig')
1406 self
.install_hurd_headers(cmdlist
)
1407 for g
in self
.compiler_glibcs
:
1408 cmdlist
.push_subdesc('glibc')
1409 cmdlist
.push_subdesc(g
.name
)
1410 g
.build_glibc(cmdlist
, GlibcPolicyForCompiler(g
))
1411 cmdlist
.pop_subdesc()
1412 cmdlist
.pop_subdesc()
1413 self
.build_gcc(cmdlist
, False)
1414 cmdlist
.add_command('done', ['touch',
1415 os
.path
.join(self
.installdir
, 'ok')])
1416 self
.ctx
.add_makefile_cmdlist('compilers-%s' % self
.name
, cmdlist
,
1419 def build_cross_tool(self
, cmdlist
, tool_src
, tool_build
, extra_opts
=None):
1420 """Build one cross tool."""
1421 srcdir
= self
.ctx
.component_srcdir(tool_src
)
1422 builddir
= self
.component_builddir(tool_build
)
1423 cmdlist
.push_subdesc(tool_build
)
1424 cmdlist
.create_use_dir(builddir
)
1425 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
1426 '--prefix=%s' % self
.installdir
,
1427 '--build=%s' % self
.ctx
.build_triplet
,
1428 '--host=%s' % self
.ctx
.build_triplet
,
1429 '--target=%s' % self
.triplet
,
1430 '--with-sysroot=%s' % self
.sysroot
]
1432 cfg_cmd
.extend(extra_opts
)
1433 cmdlist
.add_command('configure', cfg_cmd
)
1434 cmdlist
.add_command('build', ['make'])
1435 # Parallel "make install" for GCC has race conditions that can
1436 # cause it to fail; see
1437 # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>. Such
1438 # problems are not known for binutils, but doing the
1439 # installation in parallel within a particular toolchain build
1440 # (as opposed to installation of one toolchain from
1441 # build-many-glibcs.py running in parallel to the installation
1442 # of other toolchains being built) is not known to be
1443 # significantly beneficial, so it is simplest just to disable
1444 # parallel install for cross tools here.
1445 cmdlist
.add_command('install', ['make', '-j1', 'install'])
1446 cmdlist
.cleanup_dir()
1447 cmdlist
.pop_subdesc()
1449 def install_gnumach_headers(self
, cmdlist
):
1450 """Install GNU Mach headers."""
1451 srcdir
= self
.ctx
.component_srcdir('gnumach')
1452 builddir
= self
.component_builddir('gnumach')
1453 cmdlist
.push_subdesc('gnumach')
1454 cmdlist
.create_use_dir(builddir
)
1455 cmdlist
.add_command('configure',
1456 [os
.path
.join(srcdir
, 'configure'),
1457 '--build=%s' % self
.ctx
.build_triplet
,
1458 '--host=%s' % self
.triplet
,
1461 'CC=%s-gcc -nostdlib' % self
.triplet
])
1462 cmdlist
.add_command('install', ['make', 'DESTDIR=%s' % self
.sysroot
,
1464 cmdlist
.cleanup_dir()
1465 cmdlist
.pop_subdesc()
1467 def install_hurd_headers(self
, cmdlist
):
1468 """Install Hurd headers."""
1469 srcdir
= self
.ctx
.component_srcdir('hurd')
1470 builddir
= self
.component_builddir('hurd')
1471 cmdlist
.push_subdesc('hurd')
1472 cmdlist
.create_use_dir(builddir
)
1473 cmdlist
.add_command('configure',
1474 [os
.path
.join(srcdir
, 'configure'),
1475 '--build=%s' % self
.ctx
.build_triplet
,
1476 '--host=%s' % self
.triplet
,
1478 '--disable-profile', '--without-parted',
1479 'CC=%s-gcc -nostdlib' % self
.triplet
])
1480 cmdlist
.add_command('install', ['make', 'prefix=%s' % self
.sysroot
,
1481 'no_deps=t', 'install-headers'])
1482 cmdlist
.cleanup_dir()
1483 cmdlist
.pop_subdesc()
1485 def build_gcc(self
, cmdlist
, bootstrap
):
1487 # libssp is of little relevance with glibc's own stack
1488 # checking support. libcilkrts does not support GNU/Hurd (and
1489 # has been removed in GCC 8, so --disable-libcilkrts can be
1490 # removed once glibc no longer supports building with older
1491 # GCC versions). --enable-initfini-array is enabled by default
1492 # in GCC 12, which can be removed when GCC 12 becomes the
1493 # minimum requirement.
1494 cfg_opts
= list(self
.gcc_cfg
)
1495 cfg_opts
+= ['--enable-initfini-array']
1496 cfg_opts
+= ['--disable-libssp', '--disable-libcilkrts']
1497 host_libs
= self
.ctx
.host_libraries_installdir
1498 cfg_opts
+= ['--with-gmp=%s' % host_libs
,
1499 '--with-mpfr=%s' % host_libs
,
1500 '--with-mpc=%s' % host_libs
]
1502 tool_build
= 'gcc-first'
1503 # Building a static-only, C-only compiler that is
1504 # sufficient to build glibc. Various libraries and
1505 # features that may require libc headers must be disabled.
1506 # When configuring with a sysroot, --with-newlib is
1507 # required to define inhibit_libc (to stop some parts of
1508 # libgcc including libc headers); --without-headers is not
1510 cfg_opts
+= ['--enable-languages=c', '--disable-shared',
1511 '--disable-threads',
1512 '--disable-libatomic',
1513 '--disable-decimal-float',
1516 '--disable-libgomp',
1519 '--disable-libquadmath',
1520 '--disable-libsanitizer',
1521 '--without-headers', '--with-newlib',
1522 '--with-glibc-version=%s' % self
.ctx
.glibc_version
1524 cfg_opts
+= self
.first_gcc_cfg
1527 # libsanitizer commonly breaks because of glibc header
1528 # changes, or on unusual targets. C++ pre-compiled
1529 # headers are not used during the glibc build and are
1530 # expensive to create.
1531 if not self
.ctx
.full_gcc
:
1532 cfg_opts
+= ['--disable-libsanitizer',
1533 '--disable-libstdcxx-pch']
1534 langs
= 'all' if self
.ctx
.full_gcc
else 'c,c++'
1535 cfg_opts
+= ['--enable-languages=%s' % langs
,
1536 '--enable-shared', '--enable-threads']
1537 self
.build_cross_tool(cmdlist
, 'gcc', tool_build
, cfg_opts
)
1539 class GlibcPolicyDefault(object):
1540 """Build policy for glibc: common defaults."""
1542 def __init__(self
, glibc
):
1543 self
.srcdir
= glibc
.ctx
.component_srcdir('glibc')
1544 self
.use_usr
= glibc
.os
!= 'gnu'
1545 self
.prefix
= '/usr' if self
.use_usr
else ''
1546 self
.configure_args
= [
1547 '--prefix=%s' % self
.prefix
,
1549 '--build=%s' % glibc
.ctx
.build_triplet
,
1550 '--host=%s' % glibc
.triplet
,
1551 'CC=%s' % glibc
.tool_name('gcc'),
1552 'CXX=%s' % glibc
.tool_name('g++'),
1554 if glibc
.os
== 'gnu':
1555 self
.configure_args
.append('MIG=%s' % glibc
.tool_name('mig'))
1557 self
.configure_args
.append('CFLAGS=%s' % glibc
.cflags
)
1558 self
.configure_args
.append('CXXFLAGS=%s' % glibc
.cflags
)
1559 self
.configure_args
+= glibc
.cfg
1561 def configure(self
, cmdlist
):
1562 """Invoked to add the configure command to the command list."""
1563 cmdlist
.add_command('configure',
1564 [os
.path
.join(self
.srcdir
, 'configure'),
1565 *self
.configure_args
])
1567 def extra_commands(self
, cmdlist
):
1568 """Invoked to inject additional commands (make check) after build."""
1571 class GlibcPolicyForCompiler(GlibcPolicyDefault
):
1572 """Build policy for glibc during the compilers stage."""
1574 def __init__(self
, glibc
):
1575 super().__init
__(glibc
)
1576 self
.builddir
= glibc
.ctx
.component_builddir(
1577 'compilers', glibc
.compiler
.name
, 'glibc', glibc
.name
)
1578 self
.installdir
= glibc
.compiler
.sysroot
1580 class GlibcPolicyForBuild(GlibcPolicyDefault
):
1581 """Build policy for glibc during the glibcs stage."""
1583 def __init__(self
, glibc
):
1584 super().__init
__(glibc
)
1585 self
.builddir
= glibc
.ctx
.component_builddir(
1586 'glibcs', glibc
.name
, 'glibc')
1587 self
.installdir
= glibc
.ctx
.glibc_installdir(glibc
.name
)
1589 self
.strip
= glibc
.tool_name('strip')
1592 self
.save_logs
= glibc
.ctx
.save_logs
1594 def extra_commands(self
, cmdlist
):
1596 # Avoid stripping libc.so and libpthread.so, which are
1597 # linker scripts stored in /lib on Hurd.
1598 find_command
= 'find %s/lib* -name "*.so*"' % self
.installdir
1599 cmdlist
.add_command('strip', ['sh', '-c', (
1600 'set -e; for f in $(%s); do '
1601 'if ! head -c16 $f | grep -q "GNU ld script"; then %s $f; fi; '
1602 'done' % (find_command
, self
.strip
))])
1603 cmdlist
.add_command('check', ['make', 'check'])
1604 cmdlist
.add_command('save-logs', [self
.save_logs
], always_run
=True)
1606 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault
):
1607 """Build policy for glibc during update-syscalls."""
1609 def __init__(self
, glibc
):
1610 super().__init
__(glibc
)
1611 self
.builddir
= glibc
.ctx
.component_builddir(
1612 'update-syscalls', glibc
.name
, 'glibc')
1613 self
.linuxdir
= glibc
.ctx
.component_builddir(
1614 'update-syscalls', glibc
.name
, 'linux')
1615 self
.linux_policy
= LinuxHeadersPolicyForUpdateSyscalls(
1616 glibc
, self
.linuxdir
)
1617 self
.configure_args
.insert(
1618 0, '--with-headers=%s' % os
.path
.join(self
.linuxdir
, 'include'))
1619 # self.installdir not set because installation is not supported
1621 class Glibc(object):
1622 """A configuration for building glibc."""
1624 def __init__(self
, compiler
, arch
=None, os_name
=None, variant
=None,
1625 cfg
=None, ccopts
=None, cflags
=None):
1626 """Initialize a Glibc object."""
1627 self
.ctx
= compiler
.ctx
1628 self
.compiler
= compiler
1630 self
.arch
= compiler
.arch
1634 self
.os
= compiler
.os
1637 self
.variant
= variant
1639 self
.name
= '%s-%s' % (self
.arch
, self
.os
)
1641 self
.name
= '%s-%s-%s' % (self
.arch
, self
.os
, variant
)
1642 self
.triplet
= '%s-glibc-%s' % (self
.arch
, self
.os
)
1647 # ccopts contain ABI options and are passed to configure as CC / CXX.
1648 self
.ccopts
= ccopts
1649 # cflags contain non-ABI options like -g or -O and are passed to
1650 # configure as CFLAGS / CXXFLAGS.
1651 self
.cflags
= cflags
1653 def tool_name(self
, tool
):
1654 """Return the name of a cross-compilation tool."""
1655 ctool
= '%s-%s' % (self
.compiler
.triplet
, tool
)
1656 if self
.ccopts
and (tool
== 'gcc' or tool
== 'g++'):
1657 ctool
= '%s %s' % (ctool
, self
.ccopts
)
1661 """Generate commands to build this glibc."""
1662 builddir
= self
.ctx
.component_builddir('glibcs', self
.name
, 'glibc')
1663 installdir
= self
.ctx
.glibc_installdir(self
.name
)
1664 logsdir
= os
.path
.join(self
.ctx
.logsdir
, 'glibcs', self
.name
)
1665 self
.ctx
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
1666 cmdlist
= CommandList('glibcs-%s' % self
.name
, self
.ctx
.keep
)
1667 cmdlist
.add_command('check-compilers',
1669 os
.path
.join(self
.compiler
.installdir
, 'ok')])
1670 cmdlist
.use_path(self
.compiler
.bindir
)
1671 self
.build_glibc(cmdlist
, GlibcPolicyForBuild(self
))
1672 self
.ctx
.add_makefile_cmdlist('glibcs-%s' % self
.name
, cmdlist
,
1675 def build_glibc(self
, cmdlist
, policy
):
1676 """Generate commands to build this glibc, either as part of a compiler
1677 build or with the bootstrapped compiler (and in the latter case, run
1679 cmdlist
.create_use_dir(policy
.builddir
)
1680 policy
.configure(cmdlist
)
1681 cmdlist
.add_command('build', ['make'])
1682 cmdlist
.add_command('install', ['make', 'install',
1683 'install_root=%s' % policy
.installdir
])
1684 # GCC uses paths such as lib/../lib64, so make sure lib
1685 # directories always exist.
1686 mkdir_cmd
= ['mkdir', '-p',
1687 os
.path
.join(policy
.installdir
, 'lib')]
1689 mkdir_cmd
+= [os
.path
.join(policy
.installdir
, 'usr', 'lib')]
1690 cmdlist
.add_command('mkdir-lib', mkdir_cmd
)
1691 policy
.extra_commands(cmdlist
)
1692 cmdlist
.cleanup_dir()
1694 def update_syscalls(self
):
1695 if self
.os
== 'gnu':
1696 # Hurd does not have system call tables that need updating.
1699 policy
= GlibcPolicyForUpdateSyscalls(self
)
1700 logsdir
= os
.path
.join(self
.ctx
.logsdir
, 'update-syscalls', self
.name
)
1701 self
.ctx
.remove_recreate_dirs(policy
.builddir
, logsdir
)
1702 cmdlist
= CommandList('update-syscalls-%s' % self
.name
, self
.ctx
.keep
)
1703 cmdlist
.add_command('check-compilers',
1705 os
.path
.join(self
.compiler
.installdir
, 'ok')])
1706 cmdlist
.use_path(self
.compiler
.bindir
)
1708 install_linux_headers(policy
.linux_policy
, cmdlist
)
1710 cmdlist
.create_use_dir(policy
.builddir
)
1711 policy
.configure(cmdlist
)
1712 cmdlist
.add_command('build', ['make', 'update-syscall-lists'])
1713 cmdlist
.cleanup_dir()
1714 self
.ctx
.add_makefile_cmdlist('update-syscalls-%s' % self
.name
,
1717 class Command(object):
1718 """A command run in the build process."""
1720 def __init__(self
, desc
, num
, dir, path
, command
, always_run
=False):
1721 """Initialize a Command object."""
1725 trans
= str.maketrans({' ': '-'})
1726 self
.logbase
= '%03d-%s' % (num
, desc
.translate(trans
))
1727 self
.command
= command
1728 self
.always_run
= always_run
1731 def shell_make_quote_string(s
):
1732 """Given a string not containing a newline, quote it for use by the
1734 assert '\n' not in s
1735 if re
.fullmatch('[]+,./0-9@A-Z_a-z-]+', s
):
1737 strans
= str.maketrans({"'": "'\\''"})
1738 s
= "'%s'" % s
.translate(strans
)
1739 mtrans
= str.maketrans({'$': '$$'})
1740 return s
.translate(mtrans
)
1743 def shell_make_quote_list(l
, translate_make
):
1744 """Given a list of strings not containing newlines, quote them for use
1745 by the shell and make, returning a single string. If translate_make
1746 is true and the first string is 'make', change it to $(MAKE)."""
1747 l
= [Command
.shell_make_quote_string(s
) for s
in l
]
1748 if translate_make
and l
[0] == 'make':
1752 def shell_make_quote(self
):
1753 """Return this command quoted for the shell and make."""
1754 return self
.shell_make_quote_list(self
.command
, True)
1757 class CommandList(object):
1758 """A list of commands run in the build process."""
1760 def __init__(self
, desc
, keep
):
1761 """Initialize a CommandList object."""
1768 def desc_txt(self
, desc
):
1769 """Return the description to use for a command."""
1770 return '%s %s' % (' '.join(self
.desc
), desc
)
1772 def use_dir(self
, dir):
1773 """Set the default directory for subsequent commands."""
1776 def use_path(self
, path
):
1777 """Set a directory to be prepended to the PATH for subsequent
1781 def push_subdesc(self
, subdesc
):
1782 """Set the default subdescription for subsequent commands (e.g., the
1783 name of a component being built, within the series of commands
1785 self
.desc
.append(subdesc
)
1787 def pop_subdesc(self
):
1788 """Pop a subdescription from the list of descriptions."""
1791 def create_use_dir(self
, dir):
1792 """Remove and recreate a directory and use it for subsequent
1794 self
.add_command_dir('rm', None, ['rm', '-rf', dir])
1795 self
.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1798 def add_command_dir(self
, desc
, dir, command
, always_run
=False):
1799 """Add a command to run in a given directory."""
1800 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), dir, self
.path
,
1801 command
, always_run
)
1802 self
.cmdlist
.append(cmd
)
1804 def add_command(self
, desc
, command
, always_run
=False):
1805 """Add a command to run in the default directory."""
1806 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), self
.dir,
1807 self
.path
, command
, always_run
)
1808 self
.cmdlist
.append(cmd
)
1810 def cleanup_dir(self
, desc
='cleanup', dir=None):
1811 """Clean up a build directory. If no directory is specified, the
1812 default directory is cleaned up and ceases to be the default
1817 if self
.keep
!= 'all':
1818 self
.add_command_dir(desc
, None, ['rm', '-rf', dir],
1819 always_run
=(self
.keep
== 'none'))
1821 def makefile_commands(self
, wrapper
, logsdir
):
1822 """Return the sequence of commands in the form of text for a Makefile.
1823 The given wrapper script takes arguments: base of logs for
1824 previous command, or empty; base of logs for this command;
1825 description; directory; PATH addition; the command itself."""
1826 # prev_base is the base of the name for logs of the previous
1827 # command that is not always-run (that is, a build command,
1828 # whose failure should stop subsequent build commands from
1829 # being run, as opposed to a cleanup command, which is run
1830 # even if previous commands failed).
1833 for c
in self
.cmdlist
:
1834 ctxt
= c
.shell_make_quote()
1835 if prev_base
and not c
.always_run
:
1836 prev_log
= os
.path
.join(logsdir
, prev_base
)
1839 this_log
= os
.path
.join(logsdir
, c
.logbase
)
1840 if not c
.always_run
:
1841 prev_base
= c
.logbase
1850 prelims
= [wrapper
, prev_log
, this_log
, c
.desc
, dir, path
]
1851 prelim_txt
= Command
.shell_make_quote_list(prelims
, False)
1852 cmds
.append('\t@%s %s' % (prelim_txt
, ctxt
))
1853 return '\n'.join(cmds
)
1855 def status_logs(self
, logsdir
):
1856 """Return the list of log files with command status."""
1857 return [os
.path
.join(logsdir
, '%s-status.txt' % c
.logbase
)
1858 for c
in self
.cmdlist
]
1862 """Return an argument parser for this module."""
1863 parser
= argparse
.ArgumentParser(description
=__doc__
)
1864 parser
.add_argument('-j', dest
='parallelism',
1865 help='Run this number of jobs in parallel',
1866 type=int, default
=os
.cpu_count())
1867 parser
.add_argument('--keep', dest
='keep',
1868 help='Whether to keep all build directories, '
1869 'none or only those from failed builds',
1870 default
='none', choices
=('none', 'all', 'failed'))
1871 parser
.add_argument('--replace-sources', action
='store_true',
1872 help='Remove and replace source directories '
1873 'with the wrong version of a component')
1874 parser
.add_argument('--strip', action
='store_true',
1875 help='Strip installed glibc libraries')
1876 parser
.add_argument('--full-gcc', action
='store_true',
1877 help='Build GCC with all languages and libsanitizer')
1878 parser
.add_argument('--shallow', action
='store_true',
1879 help='Do not download Git history during checkout')
1880 parser
.add_argument('topdir',
1881 help='Toplevel working directory')
1882 parser
.add_argument('action',
1884 choices
=('checkout', 'bot-cycle', 'bot',
1885 'host-libraries', 'compilers', 'glibcs',
1886 'update-syscalls', 'list-compilers',
1888 parser
.add_argument('configs',
1889 help='Versions to check out or configurations to build',
1894 def get_version_common(progname
,line
,word
,delchars
,arg1
):
1896 out
= subprocess
.run([progname
, arg1
],
1897 stdout
=subprocess
.PIPE
,
1898 stderr
=subprocess
.DEVNULL
,
1899 stdin
=subprocess
.DEVNULL
,
1900 check
=True, universal_newlines
=True)
1901 v
= out
.stdout
.splitlines()[line
].split()[word
]
1903 v
= v
.replace(delchars
,'')
1904 return [int(x
) for x
in v
.split('.')]
1908 def get_version_common_stderr(progname
,line
,word
,delchars
,arg1
):
1910 out
= subprocess
.run([progname
, arg1
],
1911 stdout
=subprocess
.DEVNULL
,
1912 stderr
=subprocess
.PIPE
,
1913 stdin
=subprocess
.DEVNULL
,
1914 check
=True, universal_newlines
=True)
1915 v
= out
.stderr
.splitlines()[line
].split()[word
]
1917 v
= v
.replace(delchars
,'')
1918 return [int(x
) for x
in v
.split('.')]
1922 def get_version(progname
):
1923 return get_version_common (progname
, 0, -1, None, '--version');
1925 def get_version_awk(progname
):
1926 return get_version_common (progname
, 0, 2, ',', '--version');
1928 def get_version_bzip2(progname
):
1929 return get_version_common_stderr (progname
, 0, 6, ',', '-h');
1931 def check_version(ver
, req
):
1932 for v
, r
in zip(ver
, req
):
1939 def version_str(ver
):
1940 return '.'.join([str (x
) for x
in ver
])
1942 def check_for_required_tools():
1943 get_list_of_required_tools()
1945 count_missing_tools
= 0
1947 for k
, v
in REQUIRED_TOOLS
.items():
1949 if version
== 'missing':
1952 ok
= 'ok' if check_version (version
, v
[1]) else 'old'
1954 if count_old_tools
== 0:
1955 print("One or more required tools are too old:")
1956 count_old_tools
= count_old_tools
+ 1
1957 print('{:9}: {:3} (obtained=\"{}\" required=\"{}\")'.format(k
, ok
,
1958 version_str(version
), version_str(v
[1])))
1960 if count_missing_tools
== 0:
1961 print("One or more required tools are missing:")
1962 count_missing_tools
= count_missing_tools
+ 1
1963 print('{:9}: {:3} (required=\"{}\")'.format(k
, ok
,
1966 if count_old_tools
> 0 or count_missing_tools
> 0:
1970 """The main entry point."""
1971 check_for_required_tools();
1972 parser
= get_parser()
1973 opts
= parser
.parse_args(argv
)
1974 topdir
= os
.path
.abspath(opts
.topdir
)
1975 ctx
= Context(topdir
, opts
.parallelism
, opts
.keep
, opts
.replace_sources
,
1976 opts
.strip
, opts
.full_gcc
, opts
.action
,
1977 shallow
=opts
.shallow
)
1978 ctx
.run_builds(opts
.action
, opts
.configs
)
1981 if __name__
== '__main__':