2 # Build many configurations of glibc.
3 # Copyright (C) 2016-2021 Free Software Foundation, Inc.
4 # This file is part of the GNU C Library.
6 # The GNU C Library is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU Lesser General Public
8 # License as published by the Free Software Foundation; either
9 # version 2.1 of the License, or (at your option) any later version.
11 # The GNU C Library is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # Lesser General Public License for more details.
16 # You should have received a copy of the GNU Lesser General Public
17 # License along with the GNU C Library; if not, see
18 # <https://www.gnu.org/licenses/>.
20 """Build many configurations of glibc.
22 This script takes as arguments a directory name (containing a src
23 subdirectory with sources of the relevant toolchain components) and a
24 description of what to do: 'checkout', to check out sources into that
25 directory, 'bot-cycle', to run a series of checkout and build steps,
26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
27 libraries required by the toolchain, 'compilers', to build
28 cross-compilers for various configurations, or 'glibcs', to build
29 glibc for various configurations and run the compilation parts of the
30 testsuite. Subsequent arguments name the versions of components to
31 check out (<component>-<version), for 'checkout', or, for actions
32 other than 'checkout' and 'bot-cycle', name configurations for which
33 compilers or glibc are to be built.
35 The 'list-compilers' command prints the name of each available
36 compiler configuration, without building anything. The 'list-glibcs'
37 command prints the name of each glibc compiler configuration, followed
38 by the space, followed by the name of the compiler configuration used
39 for building this glibc variant.
45 import email
.mime
.text
61 class _CompletedProcess
:
62 def __init__(self
, args
, returncode
, stdout
=None, stderr
=None):
64 self
.returncode
= returncode
68 def _run(*popenargs
, input=None, timeout
=None, check
=False, **kwargs
):
69 assert(timeout
is None)
70 with subprocess
.Popen(*popenargs
, **kwargs
) as process
:
72 stdout
, stderr
= process
.communicate(input)
77 returncode
= process
.poll()
78 if check
and returncode
:
79 raise subprocess
.CalledProcessError(returncode
, popenargs
)
80 return _CompletedProcess(popenargs
, returncode
, stdout
, stderr
)
85 class Context(object):
86 """The global state associated with builds in a given directory."""
88 def __init__(self
, topdir
, parallelism
, keep
, replace_sources
, strip
,
89 full_gcc
, action
, shallow
=False):
90 """Initialize the context."""
92 self
.parallelism
= parallelism
94 self
.replace_sources
= replace_sources
96 self
.full_gcc
= full_gcc
97 self
.shallow
= shallow
98 self
.srcdir
= os
.path
.join(topdir
, 'src')
99 self
.versions_json
= os
.path
.join(self
.srcdir
, 'versions.json')
100 self
.build_state_json
= os
.path
.join(topdir
, 'build-state.json')
101 self
.bot_config_json
= os
.path
.join(topdir
, 'bot-config.json')
102 self
.installdir
= os
.path
.join(topdir
, 'install')
103 self
.host_libraries_installdir
= os
.path
.join(self
.installdir
,
105 self
.builddir
= os
.path
.join(topdir
, 'build')
106 self
.logsdir
= os
.path
.join(topdir
, 'logs')
107 self
.logsdir_old
= os
.path
.join(topdir
, 'logs-old')
108 self
.makefile
= os
.path
.join(self
.builddir
, 'Makefile')
109 self
.wrapper
= os
.path
.join(self
.builddir
, 'wrapper')
110 self
.save_logs
= os
.path
.join(self
.builddir
, 'save-logs')
111 self
.script_text
= self
.get_script_text()
112 if action
not in ('checkout', 'list-compilers', 'list-glibcs'):
113 self
.build_triplet
= self
.get_build_triplet()
114 self
.glibc_version
= self
.get_glibc_version()
116 self
.glibc_configs
= {}
117 self
.makefile_pieces
= ['.PHONY: all\n']
118 self
.add_all_configs()
119 self
.load_versions_json()
120 self
.load_build_state_json()
121 self
.status_log_list
= []
122 self
.email_warning
= False
124 def get_script_text(self
):
125 """Return the text of this script."""
126 with
open(sys
.argv
[0], 'r') as f
:
130 """Re-execute this script with the same arguments."""
132 os
.execv(sys
.executable
, [sys
.executable
] + sys
.argv
)
134 def get_build_triplet(self
):
135 """Determine the build triplet with config.guess."""
136 config_guess
= os
.path
.join(self
.component_srcdir('gcc'),
138 cg_out
= subprocess
.run([config_guess
], stdout
=subprocess
.PIPE
,
139 check
=True, universal_newlines
=True).stdout
140 return cg_out
.rstrip()
142 def get_glibc_version(self
):
143 """Determine the glibc version number (major.minor)."""
144 version_h
= os
.path
.join(self
.component_srcdir('glibc'), 'version.h')
145 with
open(version_h
, 'r') as f
:
146 lines
= f
.readlines()
147 starttext
= '#define VERSION "'
149 if l
.startswith(starttext
):
150 l
= l
[len(starttext
):]
152 m
= re
.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l
)
153 return '%s.%s' % m
.group(1, 2)
154 print('error: could not determine glibc version')
157 def add_all_configs(self
):
158 """Add all known glibc build configurations."""
159 self
.add_config(arch
='aarch64',
161 extra_glibcs
=[{'variant': 'disable-multi-arch',
162 'cfg': ['--disable-multi-arch']}])
163 self
.add_config(arch
='aarch64_be',
165 self
.add_config(arch
='arc',
167 gcc_cfg
=['--disable-multilib', '--with-cpu=hs38'])
168 self
.add_config(arch
='arc',
169 os_name
='linux-gnuhf',
170 gcc_cfg
=['--disable-multilib', '--with-cpu=hs38_linux'])
171 self
.add_config(arch
='arceb',
173 gcc_cfg
=['--disable-multilib', '--with-cpu=hs38'])
174 self
.add_config(arch
='alpha',
176 self
.add_config(arch
='arm',
177 os_name
='linux-gnueabi',
178 extra_glibcs
=[{'variant': 'v4t',
179 'ccopts': '-march=armv4t'}])
180 self
.add_config(arch
='armeb',
181 os_name
='linux-gnueabi')
182 self
.add_config(arch
='armeb',
183 os_name
='linux-gnueabi',
185 gcc_cfg
=['--with-arch=armv7-a'])
186 self
.add_config(arch
='arm',
187 os_name
='linux-gnueabihf',
188 gcc_cfg
=['--with-float=hard', '--with-cpu=arm926ej-s'],
189 extra_glibcs
=[{'variant': 'v7a',
190 'ccopts': '-march=armv7-a -mfpu=vfpv3'},
191 {'variant': 'v7a-disable-multi-arch',
192 'ccopts': '-march=armv7-a -mfpu=vfpv3',
193 'cfg': ['--disable-multi-arch']}])
194 self
.add_config(arch
='armeb',
195 os_name
='linux-gnueabihf',
196 gcc_cfg
=['--with-float=hard', '--with-cpu=arm926ej-s'])
197 self
.add_config(arch
='armeb',
198 os_name
='linux-gnueabihf',
200 gcc_cfg
=['--with-float=hard', '--with-arch=armv7-a',
202 self
.add_config(arch
='csky',
203 os_name
='linux-gnuabiv2',
205 gcc_cfg
=['--disable-multilib'])
206 self
.add_config(arch
='csky',
207 os_name
='linux-gnuabiv2',
208 gcc_cfg
=['--with-float=hard', '--disable-multilib'])
209 self
.add_config(arch
='hppa',
211 self
.add_config(arch
='i686',
213 self
.add_config(arch
='ia64',
215 first_gcc_cfg
=['--with-system-libunwind'],
216 binutils_cfg
=['--enable-obsolete'])
217 self
.add_config(arch
='m68k',
219 gcc_cfg
=['--disable-multilib'])
220 self
.add_config(arch
='m68k',
223 gcc_cfg
=['--with-arch=cf', '--disable-multilib'])
224 self
.add_config(arch
='m68k',
226 variant
='coldfire-soft',
227 gcc_cfg
=['--with-arch=cf', '--with-cpu=54455',
228 '--disable-multilib'])
229 self
.add_config(arch
='microblaze',
231 gcc_cfg
=['--disable-multilib'])
232 self
.add_config(arch
='microblazeel',
234 gcc_cfg
=['--disable-multilib'])
235 self
.add_config(arch
='mips64',
237 gcc_cfg
=['--with-mips-plt'],
238 glibcs
=[{'variant': 'n32'},
240 'ccopts': '-mabi=32'},
242 'ccopts': '-mabi=64'}])
243 self
.add_config(arch
='mips64',
246 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
247 glibcs
=[{'variant': 'n32-soft'},
250 'ccopts': '-mabi=32'},
251 {'variant': 'n64-soft',
252 'ccopts': '-mabi=64'}])
253 self
.add_config(arch
='mips64',
256 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
257 '--with-arch-64=mips64r2',
258 '--with-arch-32=mips32r2'],
259 glibcs
=[{'variant': 'n32-nan2008'},
260 {'variant': 'nan2008',
262 'ccopts': '-mabi=32'},
263 {'variant': 'n64-nan2008',
264 'ccopts': '-mabi=64'}])
265 self
.add_config(arch
='mips64',
267 variant
='nan2008-soft',
268 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
269 '--with-arch-64=mips64r2',
270 '--with-arch-32=mips32r2',
271 '--with-float=soft'],
272 glibcs
=[{'variant': 'n32-nan2008-soft'},
273 {'variant': 'nan2008-soft',
275 'ccopts': '-mabi=32'},
276 {'variant': 'n64-nan2008-soft',
277 'ccopts': '-mabi=64'}])
278 self
.add_config(arch
='mips64el',
280 gcc_cfg
=['--with-mips-plt'],
281 glibcs
=[{'variant': 'n32'},
283 'ccopts': '-mabi=32'},
285 'ccopts': '-mabi=64'}])
286 self
.add_config(arch
='mips64el',
289 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
290 glibcs
=[{'variant': 'n32-soft'},
293 'ccopts': '-mabi=32'},
294 {'variant': 'n64-soft',
295 'ccopts': '-mabi=64'}])
296 self
.add_config(arch
='mips64el',
299 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
300 '--with-arch-64=mips64r2',
301 '--with-arch-32=mips32r2'],
302 glibcs
=[{'variant': 'n32-nan2008'},
303 {'variant': 'nan2008',
305 'ccopts': '-mabi=32'},
306 {'variant': 'n64-nan2008',
307 'ccopts': '-mabi=64'}])
308 self
.add_config(arch
='mips64el',
310 variant
='nan2008-soft',
311 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
312 '--with-arch-64=mips64r2',
313 '--with-arch-32=mips32r2',
314 '--with-float=soft'],
315 glibcs
=[{'variant': 'n32-nan2008-soft'},
316 {'variant': 'nan2008-soft',
318 'ccopts': '-mabi=32'},
319 {'variant': 'n64-nan2008-soft',
320 'ccopts': '-mabi=64'}])
321 self
.add_config(arch
='mipsisa64r6el',
323 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
324 '--with-arch-64=mips64r6',
325 '--with-arch-32=mips32r6',
326 '--with-float=hard'],
327 glibcs
=[{'variant': 'n32'},
328 {'arch': 'mipsisa32r6el',
329 'ccopts': '-mabi=32'},
331 'ccopts': '-mabi=64'}])
332 self
.add_config(arch
='nios2',
334 self
.add_config(arch
='powerpc',
336 gcc_cfg
=['--disable-multilib', '--enable-secureplt'],
337 extra_glibcs
=[{'variant': 'power4',
338 'ccopts': '-mcpu=power4',
339 'cfg': ['--with-cpu=power4']}])
340 self
.add_config(arch
='powerpc',
343 gcc_cfg
=['--disable-multilib', '--with-float=soft',
344 '--enable-secureplt'])
345 self
.add_config(arch
='powerpc64',
347 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
348 self
.add_config(arch
='powerpc64le',
350 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
351 self
.add_config(arch
='riscv32',
353 variant
='rv32imac-ilp32',
354 gcc_cfg
=['--with-arch=rv32imac', '--with-abi=ilp32',
355 '--disable-multilib'])
356 self
.add_config(arch
='riscv32',
358 variant
='rv32imafdc-ilp32',
359 gcc_cfg
=['--with-arch=rv32imafdc', '--with-abi=ilp32',
360 '--disable-multilib'])
361 self
.add_config(arch
='riscv32',
363 variant
='rv32imafdc-ilp32d',
364 gcc_cfg
=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
365 '--disable-multilib'])
366 self
.add_config(arch
='riscv64',
368 variant
='rv64imac-lp64',
369 gcc_cfg
=['--with-arch=rv64imac', '--with-abi=lp64',
370 '--disable-multilib'])
371 self
.add_config(arch
='riscv64',
373 variant
='rv64imafdc-lp64',
374 gcc_cfg
=['--with-arch=rv64imafdc', '--with-abi=lp64',
375 '--disable-multilib'])
376 self
.add_config(arch
='riscv64',
378 variant
='rv64imafdc-lp64d',
379 gcc_cfg
=['--with-arch=rv64imafdc', '--with-abi=lp64d',
380 '--disable-multilib'])
381 self
.add_config(arch
='s390x',
384 {'arch': 's390', 'ccopts': '-m31'}],
385 extra_glibcs
=[{'variant': 'O3',
387 self
.add_config(arch
='sh3',
389 self
.add_config(arch
='sh3eb',
391 self
.add_config(arch
='sh4',
393 self
.add_config(arch
='sh4eb',
395 self
.add_config(arch
='sh4',
398 gcc_cfg
=['--without-fp'])
399 self
.add_config(arch
='sh4eb',
402 gcc_cfg
=['--without-fp'])
403 self
.add_config(arch
='sparc64',
407 'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
408 extra_glibcs
=[{'variant': 'leon3',
410 'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
411 {'variant': 'disable-multi-arch',
412 'cfg': ['--disable-multi-arch']},
413 {'variant': 'disable-multi-arch',
415 'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
416 'cfg': ['--disable-multi-arch']}])
417 self
.add_config(arch
='x86_64',
419 gcc_cfg
=['--with-multilib-list=m64,m32,mx32'],
421 {'variant': 'x32', 'ccopts': '-mx32'},
422 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
423 extra_glibcs
=[{'variant': 'disable-multi-arch',
424 'cfg': ['--disable-multi-arch']},
425 {'variant': 'static-pie',
426 'cfg': ['--enable-static-pie']},
427 {'variant': 'x32-static-pie',
429 'cfg': ['--enable-static-pie']},
430 {'variant': 'static-pie',
432 'ccopts': '-m32 -march=i686',
433 'cfg': ['--enable-static-pie']},
434 {'variant': 'disable-multi-arch',
436 'ccopts': '-m32 -march=i686',
437 'cfg': ['--disable-multi-arch']},
439 'ccopts': '-m32 -march=i486'},
441 'ccopts': '-m32 -march=i586'}])
443 def add_config(self
, **args
):
444 """Add an individual build configuration."""
445 cfg
= Config(self
, **args
)
446 if cfg
.name
in self
.configs
:
447 print('error: duplicate config %s' % cfg
.name
)
449 self
.configs
[cfg
.name
] = cfg
450 for c
in cfg
.all_glibcs
:
451 if c
.name
in self
.glibc_configs
:
452 print('error: duplicate glibc config %s' % c
.name
)
454 self
.glibc_configs
[c
.name
] = c
456 def component_srcdir(self
, component
):
457 """Return the source directory for a given component, e.g. gcc."""
458 return os
.path
.join(self
.srcdir
, component
)
460 def component_builddir(self
, action
, config
, component
, subconfig
=None):
461 """Return the directory to use for a build."""
464 assert subconfig
is None
465 return os
.path
.join(self
.builddir
, action
, component
)
466 if subconfig
is None:
467 return os
.path
.join(self
.builddir
, action
, config
, component
)
469 # glibc build as part of compiler build.
470 return os
.path
.join(self
.builddir
, action
, config
, component
,
473 def compiler_installdir(self
, config
):
474 """Return the directory in which to install a compiler."""
475 return os
.path
.join(self
.installdir
, 'compilers', config
)
477 def compiler_bindir(self
, config
):
478 """Return the directory in which to find compiler binaries."""
479 return os
.path
.join(self
.compiler_installdir(config
), 'bin')
481 def compiler_sysroot(self
, config
):
482 """Return the sysroot directory for a compiler."""
483 return os
.path
.join(self
.compiler_installdir(config
), 'sysroot')
485 def glibc_installdir(self
, config
):
486 """Return the directory in which to install glibc."""
487 return os
.path
.join(self
.installdir
, 'glibcs', config
)
489 def run_builds(self
, action
, configs
):
490 """Run the requested builds."""
491 if action
== 'checkout':
492 self
.checkout(configs
)
494 if action
== 'bot-cycle':
496 print('error: configurations specified for bot-cycle')
502 print('error: configurations specified for bot')
506 if action
in ('host-libraries', 'list-compilers',
507 'list-glibcs') and configs
:
508 print('error: configurations specified for ' + action
)
510 if action
== 'list-compilers':
511 for name
in sorted(self
.configs
.keys()):
514 if action
== 'list-glibcs':
515 for config
in sorted(self
.glibc_configs
.values(),
516 key
=lambda c
: c
.name
):
517 print(config
.name
, config
.compiler
.name
)
519 self
.clear_last_build_state(action
)
520 build_time
= datetime
.datetime
.utcnow()
521 if action
== 'host-libraries':
522 build_components
= ('gmp', 'mpfr', 'mpc')
525 self
.build_host_libraries()
526 elif action
== 'compilers':
527 build_components
= ('binutils', 'gcc', 'glibc', 'linux', 'mig',
529 old_components
= ('gmp', 'mpfr', 'mpc')
530 old_versions
= self
.build_state
['host-libraries']['build-versions']
531 self
.build_compilers(configs
)
533 build_components
= ('glibc',)
534 old_components
= ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
535 'mig', 'gnumach', 'hurd')
536 old_versions
= self
.build_state
['compilers']['build-versions']
537 if action
== 'update-syscalls':
538 self
.update_syscalls(configs
)
540 self
.build_glibcs(configs
)
544 # Partial build, do not update stored state.
547 for k
in build_components
:
548 if k
in self
.versions
:
549 build_versions
[k
] = {'version': self
.versions
[k
]['version'],
550 'revision': self
.versions
[k
]['revision']}
551 for k
in old_components
:
552 if k
in old_versions
:
553 build_versions
[k
] = {'version': old_versions
[k
]['version'],
554 'revision': old_versions
[k
]['revision']}
555 self
.update_build_state(action
, build_time
, build_versions
)
558 def remove_dirs(*args
):
559 """Remove directories and their contents if they exist."""
561 shutil
.rmtree(dir, ignore_errors
=True)
564 def remove_recreate_dirs(*args
):
565 """Remove directories if they exist, and create them as empty."""
566 Context
.remove_dirs(*args
)
568 os
.makedirs(dir, exist_ok
=True)
570 def add_makefile_cmdlist(self
, target
, cmdlist
, logsdir
):
571 """Add makefile text for a list of commands."""
572 commands
= cmdlist
.makefile_commands(self
.wrapper
, logsdir
)
573 self
.makefile_pieces
.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
574 (target
, target
, target
, commands
))
575 self
.status_log_list
.extend(cmdlist
.status_logs(logsdir
))
577 def write_files(self
):
578 """Write out the Makefile and wrapper script."""
579 mftext
= ''.join(self
.makefile_pieces
)
580 with
open(self
.makefile
, 'w') as f
:
590 'prev_status=$prev_base-status.txt\n'
591 'this_status=$this_base-status.txt\n'
592 'this_log=$this_base-log.txt\n'
593 'date > "$this_log"\n'
594 'echo >> "$this_log"\n'
595 'echo "Description: $desc" >> "$this_log"\n'
596 'printf "%s" "Command:" >> "$this_log"\n'
597 'for word in "$@"; do\n'
598 ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
599 ' printf " %s" "$word"\n'
602 ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
605 'done >> "$this_log"\n'
606 'echo >> "$this_log"\n'
607 'echo "Directory: $dir" >> "$this_log"\n'
608 'echo "Path addition: $path" >> "$this_log"\n'
609 'echo >> "$this_log"\n'
612 ' echo >> "$this_log"\n'
613 ' echo "$1: $desc" > "$this_status"\n'
614 ' echo "$1: $desc" >> "$this_log"\n'
615 ' echo >> "$this_log"\n'
616 ' date >> "$this_log"\n'
617 ' echo "$1: $desc"\n'
622 ' if [ "$1" != "0" ]; then\n'
623 ' record_status FAIL\n'
626 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
627 ' record_status UNRESOLVED\n'
629 'if [ "$dir" ]; then\n'
631 ' check_error "$?"\n'
633 'if [ "$path" ]; then\n'
634 ' PATH=$path:$PATH\n'
636 '"$@" < /dev/null >> "$this_log" 2>&1\n'
638 'record_status PASS\n')
639 with
open(self
.wrapper
, 'w') as f
:
640 f
.write(wrapper_text
)
642 mode_exec
= (stat
.S_IRWXU|stat
.S_IRGRP|stat
.S_IXGRP|
643 stat
.S_IROTH|stat
.S_IXOTH
)
644 os
.chmod(self
.wrapper
, mode_exec
)
647 'if ! [ -f tests.sum ]; then\n'
648 ' echo "No test summary available."\n'
653 ' echo "Contents of $1:"\n'
657 ' echo "End of contents of $1."\n'
660 'save_file tests.sum\n'
661 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
662 'for t in $non_pass_tests; do\n'
663 ' if [ -f "$t.out" ]; then\n'
664 ' save_file "$t.out"\n'
667 with
open(self
.save_logs
, 'w') as f
:
668 f
.write(save_logs_text
)
669 os
.chmod(self
.save_logs
, mode_exec
)
672 """Do the actual build."""
673 cmd
= ['make', '-O', '-j%d' % self
.parallelism
]
674 subprocess
.run(cmd
, cwd
=self
.builddir
, check
=True)
676 def build_host_libraries(self
):
677 """Build the host libraries."""
678 installdir
= self
.host_libraries_installdir
679 builddir
= os
.path
.join(self
.builddir
, 'host-libraries')
680 logsdir
= os
.path
.join(self
.logsdir
, 'host-libraries')
681 self
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
682 cmdlist
= CommandList('host-libraries', self
.keep
)
683 self
.build_host_library(cmdlist
, 'gmp')
684 self
.build_host_library(cmdlist
, 'mpfr',
685 ['--with-gmp=%s' % installdir
])
686 self
.build_host_library(cmdlist
, 'mpc',
687 ['--with-gmp=%s' % installdir
,
688 '--with-mpfr=%s' % installdir
])
689 cmdlist
.add_command('done', ['touch', os
.path
.join(installdir
, 'ok')])
690 self
.add_makefile_cmdlist('host-libraries', cmdlist
, logsdir
)
692 def build_host_library(self
, cmdlist
, lib
, extra_opts
=None):
693 """Build one host library."""
694 srcdir
= self
.component_srcdir(lib
)
695 builddir
= self
.component_builddir('host-libraries', None, lib
)
696 installdir
= self
.host_libraries_installdir
697 cmdlist
.push_subdesc(lib
)
698 cmdlist
.create_use_dir(builddir
)
699 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
700 '--prefix=%s' % installdir
,
703 cfg_cmd
.extend (extra_opts
)
704 cmdlist
.add_command('configure', cfg_cmd
)
705 cmdlist
.add_command('build', ['make'])
706 cmdlist
.add_command('check', ['make', 'check'])
707 cmdlist
.add_command('install', ['make', 'install'])
708 cmdlist
.cleanup_dir()
709 cmdlist
.pop_subdesc()
711 def build_compilers(self
, configs
):
712 """Build the compilers."""
714 self
.remove_dirs(os
.path
.join(self
.builddir
, 'compilers'))
715 self
.remove_dirs(os
.path
.join(self
.installdir
, 'compilers'))
716 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'compilers'))
717 configs
= sorted(self
.configs
.keys())
719 self
.configs
[c
].build()
721 def build_glibcs(self
, configs
):
722 """Build the glibcs."""
724 self
.remove_dirs(os
.path
.join(self
.builddir
, 'glibcs'))
725 self
.remove_dirs(os
.path
.join(self
.installdir
, 'glibcs'))
726 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'glibcs'))
727 configs
= sorted(self
.glibc_configs
.keys())
729 self
.glibc_configs
[c
].build()
731 def update_syscalls(self
, configs
):
732 """Update the glibc syscall lists."""
734 self
.remove_dirs(os
.path
.join(self
.builddir
, 'update-syscalls'))
735 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'update-syscalls'))
736 configs
= sorted(self
.glibc_configs
.keys())
738 self
.glibc_configs
[c
].update_syscalls()
740 def load_versions_json(self
):
741 """Load information about source directory versions."""
742 if not os
.access(self
.versions_json
, os
.F_OK
):
745 with
open(self
.versions_json
, 'r') as f
:
746 self
.versions
= json
.load(f
)
748 def store_json(self
, data
, filename
):
749 """Store information in a JSON file."""
750 filename_tmp
= filename
+ '.tmp'
751 with
open(filename_tmp
, 'w') as f
:
752 json
.dump(data
, f
, indent
=2, sort_keys
=True)
753 os
.rename(filename_tmp
, filename
)
755 def store_versions_json(self
):
756 """Store information about source directory versions."""
757 self
.store_json(self
.versions
, self
.versions_json
)
759 def set_component_version(self
, component
, version
, explicit
, revision
):
760 """Set the version information for a component."""
761 self
.versions
[component
] = {'version': version
,
762 'explicit': explicit
,
763 'revision': revision
}
764 self
.store_versions_json()
766 def checkout(self
, versions
):
767 """Check out the desired component versions."""
768 default_versions
= {'binutils': 'vcs-2.36',
770 'glibc': 'vcs-mainline',
775 'mig': 'vcs-mainline',
776 'gnumach': 'vcs-mainline',
777 'hurd': 'vcs-mainline'}
779 explicit_versions
= {}
782 for k
in default_versions
.keys():
786 if k
in use_versions
:
787 print('error: multiple versions for %s' % k
)
790 explicit_versions
[k
] = True
794 print('error: unknown component in %s' % v
)
796 for k
in default_versions
.keys():
797 if k
not in use_versions
:
798 if k
in self
.versions
and self
.versions
[k
]['explicit']:
799 use_versions
[k
] = self
.versions
[k
]['version']
800 explicit_versions
[k
] = True
802 use_versions
[k
] = default_versions
[k
]
803 explicit_versions
[k
] = False
804 os
.makedirs(self
.srcdir
, exist_ok
=True)
805 for k
in sorted(default_versions
.keys()):
806 update
= os
.access(self
.component_srcdir(k
), os
.F_OK
)
809 k
in self
.versions
and
810 v
!= self
.versions
[k
]['version']):
811 if not self
.replace_sources
:
812 print('error: version of %s has changed from %s to %s, '
813 'use --replace-sources to check out again' %
814 (k
, self
.versions
[k
]['version'], v
))
816 shutil
.rmtree(self
.component_srcdir(k
))
818 if v
.startswith('vcs-'):
819 revision
= self
.checkout_vcs(k
, v
[4:], update
)
821 self
.checkout_tar(k
, v
, update
)
823 self
.set_component_version(k
, v
, explicit_versions
[k
], revision
)
824 if self
.get_script_text() != self
.script_text
:
825 # Rerun the checkout process in case the updated script
826 # uses different default versions or new components.
829 def checkout_vcs(self
, component
, version
, update
):
830 """Check out the given version of the given component from version
831 control. Return a revision identifier."""
832 if component
== 'binutils':
833 git_url
= 'git://sourceware.org/git/binutils-gdb.git'
834 if version
== 'mainline':
835 git_branch
= 'master'
837 trans
= str.maketrans({'.': '_'})
838 git_branch
= 'binutils-%s-branch' % version
.translate(trans
)
839 return self
.git_checkout(component
, git_url
, git_branch
, update
)
840 elif component
== 'gcc':
841 if version
== 'mainline':
844 branch
= 'releases/gcc-%s' % version
845 return self
.gcc_checkout(branch
, update
)
846 elif component
== 'glibc':
847 git_url
= 'git://sourceware.org/git/glibc.git'
848 if version
== 'mainline':
849 git_branch
= 'master'
851 git_branch
= 'release/%s/master' % version
852 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
853 self
.fix_glibc_timestamps()
855 elif component
== 'gnumach':
856 git_url
= 'git://git.savannah.gnu.org/hurd/gnumach.git'
857 git_branch
= 'master'
858 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
859 subprocess
.run(['autoreconf', '-i'],
860 cwd
=self
.component_srcdir(component
), check
=True)
862 elif component
== 'mig':
863 git_url
= 'git://git.savannah.gnu.org/hurd/mig.git'
864 git_branch
= 'master'
865 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
866 subprocess
.run(['autoreconf', '-i'],
867 cwd
=self
.component_srcdir(component
), check
=True)
869 elif component
== 'hurd':
870 git_url
= 'git://git.savannah.gnu.org/hurd/hurd.git'
871 git_branch
= 'master'
872 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
873 subprocess
.run(['autoconf'],
874 cwd
=self
.component_srcdir(component
), check
=True)
877 print('error: component %s coming from VCS' % component
)
880 def git_checkout(self
, component
, git_url
, git_branch
, update
):
881 """Check out a component from git. Return a commit identifier."""
883 subprocess
.run(['git', 'remote', 'prune', 'origin'],
884 cwd
=self
.component_srcdir(component
), check
=True)
885 if self
.replace_sources
:
886 subprocess
.run(['git', 'clean', '-dxfq'],
887 cwd
=self
.component_srcdir(component
), check
=True)
888 subprocess
.run(['git', 'pull', '-q'],
889 cwd
=self
.component_srcdir(component
), check
=True)
892 depth_arg
= ('--depth', '1')
895 subprocess
.run(['git', 'clone', '-q', '-b', git_branch
,
897 self
.component_srcdir(component
)], check
=True)
898 r
= subprocess
.run(['git', 'rev-parse', 'HEAD'],
899 cwd
=self
.component_srcdir(component
),
900 stdout
=subprocess
.PIPE
,
901 check
=True, universal_newlines
=True).stdout
904 def fix_glibc_timestamps(self
):
905 """Fix timestamps in a glibc checkout."""
906 # Ensure that builds do not try to regenerate generated files
907 # in the source tree.
908 srcdir
= self
.component_srcdir('glibc')
909 # These files have Makefile dependencies to regenerate them in
910 # the source tree that may be active during a normal build.
911 # Some other files have such dependencies but do not need to
912 # be touched because nothing in a build depends on the files
914 for f
in ('sysdeps/mach/hurd/bits/errno.h',):
915 to_touch
= os
.path
.join(srcdir
, f
)
916 subprocess
.run(['touch', '-c', to_touch
], check
=True)
917 for dirpath
, dirnames
, filenames
in os
.walk(srcdir
):
919 if (f
== 'configure' or
920 f
== 'preconfigure' or
921 f
.endswith('-kw.h')):
922 to_touch
= os
.path
.join(dirpath
, f
)
923 subprocess
.run(['touch', to_touch
], check
=True)
925 def gcc_checkout(self
, branch
, update
):
926 """Check out GCC from git. Return the commit identifier."""
927 if os
.access(os
.path
.join(self
.component_srcdir('gcc'), '.svn'),
929 if not self
.replace_sources
:
930 print('error: GCC has moved from SVN to git, use '
931 '--replace-sources to check out again')
933 shutil
.rmtree(self
.component_srcdir('gcc'))
936 self
.git_checkout('gcc', 'git://gcc.gnu.org/git/gcc.git',
938 subprocess
.run(['contrib/gcc_update', '--silent'],
939 cwd
=self
.component_srcdir('gcc'), check
=True)
940 r
= subprocess
.run(['git', 'rev-parse', 'HEAD'],
941 cwd
=self
.component_srcdir('gcc'),
942 stdout
=subprocess
.PIPE
,
943 check
=True, universal_newlines
=True).stdout
946 def checkout_tar(self
, component
, version
, update
):
947 """Check out the given version of the given component from a
951 url_map
= {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
952 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
953 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
954 'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
955 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
956 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
957 'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
958 'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
959 'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
960 if component
not in url_map
:
961 print('error: component %s coming from tarball' % component
)
963 version_major
= version
.split('.')[0]
964 url
= url_map
[component
] % {'version': version
, 'major': version_major
}
965 filename
= os
.path
.join(self
.srcdir
, url
.split('/')[-1])
966 response
= urllib
.request
.urlopen(url
)
967 data
= response
.read()
968 with
open(filename
, 'wb') as f
:
970 subprocess
.run(['tar', '-C', self
.srcdir
, '-x', '-f', filename
],
972 os
.rename(os
.path
.join(self
.srcdir
, '%s-%s' % (component
, version
)),
973 self
.component_srcdir(component
))
976 def load_build_state_json(self
):
977 """Load information about the state of previous builds."""
978 if os
.access(self
.build_state_json
, os
.F_OK
):
979 with
open(self
.build_state_json
, 'r') as f
:
980 self
.build_state
= json
.load(f
)
982 self
.build_state
= {}
983 for k
in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
984 if k
not in self
.build_state
:
985 self
.build_state
[k
] = {}
986 if 'build-time' not in self
.build_state
[k
]:
987 self
.build_state
[k
]['build-time'] = ''
988 if 'build-versions' not in self
.build_state
[k
]:
989 self
.build_state
[k
]['build-versions'] = {}
990 if 'build-results' not in self
.build_state
[k
]:
991 self
.build_state
[k
]['build-results'] = {}
992 if 'result-changes' not in self
.build_state
[k
]:
993 self
.build_state
[k
]['result-changes'] = {}
994 if 'ever-passed' not in self
.build_state
[k
]:
995 self
.build_state
[k
]['ever-passed'] = []
997 def store_build_state_json(self
):
998 """Store information about the state of previous builds."""
999 self
.store_json(self
.build_state
, self
.build_state_json
)
1001 def clear_last_build_state(self
, action
):
1002 """Clear information about the state of part of the build."""
1003 # We clear the last build time and versions when starting a
1004 # new build. The results of the last build are kept around,
1005 # as comparison is still meaningful if this build is aborted
1006 # and a new one started.
1007 self
.build_state
[action
]['build-time'] = ''
1008 self
.build_state
[action
]['build-versions'] = {}
1009 self
.store_build_state_json()
1011 def update_build_state(self
, action
, build_time
, build_versions
):
1012 """Update the build state after a build."""
1013 build_time
= build_time
.replace(microsecond
=0)
1014 self
.build_state
[action
]['build-time'] = str(build_time
)
1015 self
.build_state
[action
]['build-versions'] = build_versions
1017 for log
in self
.status_log_list
:
1018 with
open(log
, 'r') as f
:
1020 log_text
= log_text
.rstrip()
1021 m
= re
.fullmatch('([A-Z]+): (.*)', log_text
)
1023 test_name
= m
.group(2)
1024 assert test_name
not in build_results
1025 build_results
[test_name
] = result
1026 old_build_results
= self
.build_state
[action
]['build-results']
1027 self
.build_state
[action
]['build-results'] = build_results
1029 all_tests
= set(old_build_results
.keys()) |
set(build_results
.keys())
1031 if t
in old_build_results
:
1032 old_res
= old_build_results
[t
]
1034 old_res
= '(New test)'
1035 if t
in build_results
:
1036 new_res
= build_results
[t
]
1038 new_res
= '(Test removed)'
1039 if old_res
!= new_res
:
1040 result_changes
[t
] = '%s -> %s' % (old_res
, new_res
)
1041 self
.build_state
[action
]['result-changes'] = result_changes
1042 old_ever_passed
= {t
for t
in self
.build_state
[action
]['ever-passed']
1043 if t
in build_results
}
1044 new_passes
= {t
for t
in build_results
if build_results
[t
] == 'PASS'}
1045 self
.build_state
[action
]['ever-passed'] = sorted(old_ever_passed |
1047 self
.store_build_state_json()
1049 def load_bot_config_json(self
):
1050 """Load bot configuration."""
1051 with
open(self
.bot_config_json
, 'r') as f
:
1052 self
.bot_config
= json
.load(f
)
1054 def part_build_old(self
, action
, delay
):
1055 """Return whether the last build for a given action was at least a
1056 given number of seconds ago, or does not have a time recorded."""
1057 old_time_str
= self
.build_state
[action
]['build-time']
1058 if not old_time_str
:
1060 old_time
= datetime
.datetime
.strptime(old_time_str
,
1061 '%Y-%m-%d %H:%M:%S')
1062 new_time
= datetime
.datetime
.utcnow()
1063 delta
= new_time
- old_time
1064 return delta
.total_seconds() >= delay
1066 def bot_cycle(self
):
1067 """Run a single round of checkout and builds."""
1068 print('Bot cycle starting %s.' % str(datetime
.datetime
.utcnow()))
1069 self
.load_bot_config_json()
1070 actions
= ('host-libraries', 'compilers', 'glibcs')
1071 self
.bot_run_self(['--replace-sources'], 'checkout')
1072 self
.load_versions_json()
1073 if self
.get_script_text() != self
.script_text
:
1074 print('Script changed, re-execing.')
1075 # On script change, all parts of the build should be rerun.
1077 self
.clear_last_build_state(a
)
1079 check_components
= {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1080 'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1081 'mig', 'gnumach', 'hurd'),
1082 'glibcs': ('glibc',)}
1085 build_vers
= self
.build_state
[a
]['build-versions']
1086 must_build
[a
] = False
1087 if not self
.build_state
[a
]['build-time']:
1088 must_build
[a
] = True
1091 for c
in check_components
[a
]:
1093 old_vers
[c
] = build_vers
[c
]
1094 new_vers
[c
] = {'version': self
.versions
[c
]['version'],
1095 'revision': self
.versions
[c
]['revision']}
1096 if new_vers
== old_vers
:
1097 print('Versions for %s unchanged.' % a
)
1099 print('Versions changed or rebuild forced for %s.' % a
)
1100 if a
== 'compilers' and not self
.part_build_old(
1101 a
, self
.bot_config
['compilers-rebuild-delay']):
1102 print('Not requiring rebuild of compilers this soon.')
1104 must_build
[a
] = True
1105 if must_build
['host-libraries']:
1106 must_build
['compilers'] = True
1107 if must_build
['compilers']:
1108 must_build
['glibcs'] = True
1111 print('Must rebuild %s.' % a
)
1112 self
.clear_last_build_state(a
)
1114 print('No need to rebuild %s.' % a
)
1115 if os
.access(self
.logsdir
, os
.F_OK
):
1116 shutil
.rmtree(self
.logsdir_old
, ignore_errors
=True)
1117 shutil
.copytree(self
.logsdir
, self
.logsdir_old
)
1120 build_time
= datetime
.datetime
.utcnow()
1121 print('Rebuilding %s at %s.' % (a
, str(build_time
)))
1122 self
.bot_run_self([], a
)
1123 self
.load_build_state_json()
1124 self
.bot_build_mail(a
, build_time
)
1125 print('Bot cycle done at %s.' % str(datetime
.datetime
.utcnow()))
1127 def bot_build_mail(self
, action
, build_time
):
1128 """Send email with the results of a build."""
1129 if not ('email-from' in self
.bot_config
and
1130 'email-server' in self
.bot_config
and
1131 'email-subject' in self
.bot_config
and
1132 'email-to' in self
.bot_config
):
1133 if not self
.email_warning
:
1134 print("Email not configured, not sending.")
1135 self
.email_warning
= True
1138 build_time
= build_time
.replace(microsecond
=0)
1139 subject
= (self
.bot_config
['email-subject'] %
1141 'build-time': str(build_time
)})
1142 results
= self
.build_state
[action
]['build-results']
1143 changes
= self
.build_state
[action
]['result-changes']
1144 ever_passed
= set(self
.build_state
[action
]['ever-passed'])
1145 versions
= self
.build_state
[action
]['build-versions']
1146 new_regressions
= {k
for k
in changes
if changes
[k
] == 'PASS -> FAIL'}
1147 all_regressions
= {k
for k
in ever_passed
if results
[k
] == 'FAIL'}
1148 all_fails
= {k
for k
in results
if results
[k
] == 'FAIL'}
1150 new_reg_list
= sorted(['FAIL: %s' % k
for k
in new_regressions
])
1151 new_reg_text
= ('New regressions:\n\n%s\n\n' %
1152 '\n'.join(new_reg_list
))
1156 all_reg_list
= sorted(['FAIL: %s' % k
for k
in all_regressions
])
1157 all_reg_text
= ('All regressions:\n\n%s\n\n' %
1158 '\n'.join(all_reg_list
))
1162 all_fail_list
= sorted(['FAIL: %s' % k
for k
in all_fails
])
1163 all_fail_text
= ('All failures:\n\n%s\n\n' %
1164 '\n'.join(all_fail_list
))
1168 changes_list
= sorted(changes
.keys())
1169 changes_list
= ['%s: %s' % (changes
[k
], k
) for k
in changes_list
]
1170 changes_text
= ('All changed results:\n\n%s\n\n' %
1171 '\n'.join(changes_list
))
1174 results_text
= (new_reg_text
+ all_reg_text
+ all_fail_text
+
1176 if not results_text
:
1177 results_text
= 'Clean build with unchanged results.\n\n'
1178 versions_list
= sorted(versions
.keys())
1179 versions_list
= ['%s: %s (%s)' % (k
, versions
[k
]['version'],
1180 versions
[k
]['revision'])
1181 for k
in versions_list
]
1182 versions_text
= ('Component versions for this build:\n\n%s\n' %
1183 '\n'.join(versions_list
))
1184 body_text
= results_text
+ versions_text
1185 msg
= email
.mime
.text
.MIMEText(body_text
)
1186 msg
['Subject'] = subject
1187 msg
['From'] = self
.bot_config
['email-from']
1188 msg
['To'] = self
.bot_config
['email-to']
1189 msg
['Message-ID'] = email
.utils
.make_msgid()
1190 msg
['Date'] = email
.utils
.format_datetime(datetime
.datetime
.utcnow())
1191 with smtplib
.SMTP(self
.bot_config
['email-server']) as s
:
1194 def bot_run_self(self
, opts
, action
, check
=True):
1195 """Run a copy of this script with given options."""
1196 cmd
= [sys
.executable
, sys
.argv
[0], '--keep=none',
1197 '-j%d' % self
.parallelism
]
1199 cmd
.append('--full-gcc')
1201 cmd
.extend([self
.topdir
, action
])
1203 subprocess
.run(cmd
, check
=check
)
1206 """Run repeated rounds of checkout and builds."""
1208 self
.load_bot_config_json()
1209 if not self
.bot_config
['run']:
1210 print('Bot exiting by request.')
1212 self
.bot_run_self([], 'bot-cycle', check
=False)
1213 self
.load_bot_config_json()
1214 if not self
.bot_config
['run']:
1215 print('Bot exiting by request.')
1217 time
.sleep(self
.bot_config
['delay'])
1218 if self
.get_script_text() != self
.script_text
:
1219 print('Script changed, bot re-execing.')
1222 class LinuxHeadersPolicyForBuild(object):
1223 """Names and directories for installing Linux headers. Build variant."""
1225 def __init__(self
, config
):
1226 self
.arch
= config
.arch
1227 self
.srcdir
= config
.ctx
.component_srcdir('linux')
1228 self
.builddir
= config
.component_builddir('linux')
1229 self
.headers_dir
= os
.path
.join(config
.sysroot
, 'usr')
1231 class LinuxHeadersPolicyForUpdateSyscalls(object):
1232 """Names and directories for Linux headers. update-syscalls variant."""
1234 def __init__(self
, glibc
, headers_dir
):
1235 self
.arch
= glibc
.compiler
.arch
1236 self
.srcdir
= glibc
.compiler
.ctx
.component_srcdir('linux')
1237 self
.builddir
= glibc
.ctx
.component_builddir(
1238 'update-syscalls', glibc
.name
, 'build-linux')
1239 self
.headers_dir
= headers_dir
1241 def install_linux_headers(policy
, cmdlist
):
1242 """Install Linux kernel headers."""
1243 arch_map
= {'aarch64': 'arm64',
1255 'microblaze': 'microblaze',
1258 'powerpc': 'powerpc',
1267 if policy
.arch
.startswith(k
):
1268 linux_arch
= arch_map
[k
]
1270 assert linux_arch
is not None
1271 cmdlist
.push_subdesc('linux')
1272 cmdlist
.create_use_dir(policy
.builddir
)
1273 cmdlist
.add_command('install-headers',
1274 ['make', '-C', policy
.srcdir
, 'O=%s' % policy
.builddir
,
1275 'ARCH=%s' % linux_arch
,
1276 'INSTALL_HDR_PATH=%s' % policy
.headers_dir
,
1278 cmdlist
.cleanup_dir()
1279 cmdlist
.pop_subdesc()
1281 class Config(object):
1282 """A configuration for building a compiler and associated libraries."""
1284 def __init__(self
, ctx
, arch
, os_name
, variant
=None, gcc_cfg
=None,
1285 first_gcc_cfg
=None, binutils_cfg
=None, glibcs
=None,
1287 """Initialize a Config object."""
1291 self
.variant
= variant
1293 self
.name
= '%s-%s' % (arch
, os_name
)
1295 self
.name
= '%s-%s-%s' % (arch
, os_name
, variant
)
1296 self
.triplet
= '%s-glibc-%s' % (arch
, os_name
)
1300 self
.gcc_cfg
= gcc_cfg
1301 if first_gcc_cfg
is None:
1302 self
.first_gcc_cfg
= []
1304 self
.first_gcc_cfg
= first_gcc_cfg
1305 if binutils_cfg
is None:
1306 self
.binutils_cfg
= []
1308 self
.binutils_cfg
= binutils_cfg
1310 glibcs
= [{'variant': variant
}]
1311 if extra_glibcs
is None:
1313 glibcs
= [Glibc(self
, **g
) for g
in glibcs
]
1314 extra_glibcs
= [Glibc(self
, **g
) for g
in extra_glibcs
]
1315 self
.all_glibcs
= glibcs
+ extra_glibcs
1316 self
.compiler_glibcs
= glibcs
1317 self
.installdir
= ctx
.compiler_installdir(self
.name
)
1318 self
.bindir
= ctx
.compiler_bindir(self
.name
)
1319 self
.sysroot
= ctx
.compiler_sysroot(self
.name
)
1320 self
.builddir
= os
.path
.join(ctx
.builddir
, 'compilers', self
.name
)
1321 self
.logsdir
= os
.path
.join(ctx
.logsdir
, 'compilers', self
.name
)
1323 def component_builddir(self
, component
):
1324 """Return the directory to use for a (non-glibc) build."""
1325 return self
.ctx
.component_builddir('compilers', self
.name
, component
)
1328 """Generate commands to build this compiler."""
1329 self
.ctx
.remove_recreate_dirs(self
.installdir
, self
.builddir
,
1331 cmdlist
= CommandList('compilers-%s' % self
.name
, self
.ctx
.keep
)
1332 cmdlist
.add_command('check-host-libraries',
1334 os
.path
.join(self
.ctx
.host_libraries_installdir
,
1336 cmdlist
.use_path(self
.bindir
)
1337 self
.build_cross_tool(cmdlist
, 'binutils', 'binutils',
1339 '--disable-gdbserver',
1340 '--disable-libdecnumber',
1341 '--disable-readline',
1342 '--disable-sim'] + self
.binutils_cfg
)
1343 if self
.os
.startswith('linux'):
1344 install_linux_headers(LinuxHeadersPolicyForBuild(self
), cmdlist
)
1345 self
.build_gcc(cmdlist
, True)
1346 if self
.os
== 'gnu':
1347 self
.install_gnumach_headers(cmdlist
)
1348 self
.build_cross_tool(cmdlist
, 'mig', 'mig')
1349 self
.install_hurd_headers(cmdlist
)
1350 for g
in self
.compiler_glibcs
:
1351 cmdlist
.push_subdesc('glibc')
1352 cmdlist
.push_subdesc(g
.name
)
1353 g
.build_glibc(cmdlist
, GlibcPolicyForCompiler(g
))
1354 cmdlist
.pop_subdesc()
1355 cmdlist
.pop_subdesc()
1356 self
.build_gcc(cmdlist
, False)
1357 cmdlist
.add_command('done', ['touch',
1358 os
.path
.join(self
.installdir
, 'ok')])
1359 self
.ctx
.add_makefile_cmdlist('compilers-%s' % self
.name
, cmdlist
,
1362 def build_cross_tool(self
, cmdlist
, tool_src
, tool_build
, extra_opts
=None):
1363 """Build one cross tool."""
1364 srcdir
= self
.ctx
.component_srcdir(tool_src
)
1365 builddir
= self
.component_builddir(tool_build
)
1366 cmdlist
.push_subdesc(tool_build
)
1367 cmdlist
.create_use_dir(builddir
)
1368 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
1369 '--prefix=%s' % self
.installdir
,
1370 '--build=%s' % self
.ctx
.build_triplet
,
1371 '--host=%s' % self
.ctx
.build_triplet
,
1372 '--target=%s' % self
.triplet
,
1373 '--with-sysroot=%s' % self
.sysroot
]
1375 cfg_cmd
.extend(extra_opts
)
1376 cmdlist
.add_command('configure', cfg_cmd
)
1377 cmdlist
.add_command('build', ['make'])
1378 # Parallel "make install" for GCC has race conditions that can
1379 # cause it to fail; see
1380 # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>. Such
1381 # problems are not known for binutils, but doing the
1382 # installation in parallel within a particular toolchain build
1383 # (as opposed to installation of one toolchain from
1384 # build-many-glibcs.py running in parallel to the installation
1385 # of other toolchains being built) is not known to be
1386 # significantly beneficial, so it is simplest just to disable
1387 # parallel install for cross tools here.
1388 cmdlist
.add_command('install', ['make', '-j1', 'install'])
1389 cmdlist
.cleanup_dir()
1390 cmdlist
.pop_subdesc()
1392 def install_gnumach_headers(self
, cmdlist
):
1393 """Install GNU Mach headers."""
1394 srcdir
= self
.ctx
.component_srcdir('gnumach')
1395 builddir
= self
.component_builddir('gnumach')
1396 cmdlist
.push_subdesc('gnumach')
1397 cmdlist
.create_use_dir(builddir
)
1398 cmdlist
.add_command('configure',
1399 [os
.path
.join(srcdir
, 'configure'),
1400 '--build=%s' % self
.ctx
.build_triplet
,
1401 '--host=%s' % self
.triplet
,
1403 'CC=%s-gcc -nostdlib' % self
.triplet
])
1404 cmdlist
.add_command('install', ['make', 'DESTDIR=%s' % self
.sysroot
,
1406 cmdlist
.cleanup_dir()
1407 cmdlist
.pop_subdesc()
1409 def install_hurd_headers(self
, cmdlist
):
1410 """Install Hurd headers."""
1411 srcdir
= self
.ctx
.component_srcdir('hurd')
1412 builddir
= self
.component_builddir('hurd')
1413 cmdlist
.push_subdesc('hurd')
1414 cmdlist
.create_use_dir(builddir
)
1415 cmdlist
.add_command('configure',
1416 [os
.path
.join(srcdir
, 'configure'),
1417 '--build=%s' % self
.ctx
.build_triplet
,
1418 '--host=%s' % self
.triplet
,
1420 '--disable-profile', '--without-parted',
1421 'CC=%s-gcc -nostdlib' % self
.triplet
])
1422 cmdlist
.add_command('install', ['make', 'prefix=%s' % self
.sysroot
,
1423 'no_deps=t', 'install-headers'])
1424 cmdlist
.cleanup_dir()
1425 cmdlist
.pop_subdesc()
1427 def build_gcc(self
, cmdlist
, bootstrap
):
1429 # libssp is of little relevance with glibc's own stack
1430 # checking support. libcilkrts does not support GNU/Hurd (and
1431 # has been removed in GCC 8, so --disable-libcilkrts can be
1432 # removed once glibc no longer supports building with older
1434 cfg_opts
= list(self
.gcc_cfg
)
1435 cfg_opts
+= ['--disable-libssp', '--disable-libcilkrts']
1436 host_libs
= self
.ctx
.host_libraries_installdir
1437 cfg_opts
+= ['--with-gmp=%s' % host_libs
,
1438 '--with-mpfr=%s' % host_libs
,
1439 '--with-mpc=%s' % host_libs
]
1441 tool_build
= 'gcc-first'
1442 # Building a static-only, C-only compiler that is
1443 # sufficient to build glibc. Various libraries and
1444 # features that may require libc headers must be disabled.
1445 # When configuring with a sysroot, --with-newlib is
1446 # required to define inhibit_libc (to stop some parts of
1447 # libgcc including libc headers); --without-headers is not
1449 cfg_opts
+= ['--enable-languages=c', '--disable-shared',
1450 '--disable-threads',
1451 '--disable-libatomic',
1452 '--disable-decimal-float',
1454 '--disable-libgomp',
1457 '--disable-libquadmath',
1458 '--disable-libsanitizer',
1459 '--without-headers', '--with-newlib',
1460 '--with-glibc-version=%s' % self
.ctx
.glibc_version
1462 cfg_opts
+= self
.first_gcc_cfg
1465 # libsanitizer commonly breaks because of glibc header
1466 # changes, or on unusual targets. C++ pre-compiled
1467 # headers are not used during the glibc build and are
1468 # expensive to create.
1469 if not self
.ctx
.full_gcc
:
1470 cfg_opts
+= ['--disable-libsanitizer',
1471 '--disable-libstdcxx-pch']
1472 langs
= 'all' if self
.ctx
.full_gcc
else 'c,c++'
1473 cfg_opts
+= ['--enable-languages=%s' % langs
,
1474 '--enable-shared', '--enable-threads']
1475 self
.build_cross_tool(cmdlist
, 'gcc', tool_build
, cfg_opts
)
1477 class GlibcPolicyDefault(object):
1478 """Build policy for glibc: common defaults."""
1480 def __init__(self
, glibc
):
1481 self
.srcdir
= glibc
.ctx
.component_srcdir('glibc')
1482 self
.use_usr
= glibc
.os
!= 'gnu'
1483 self
.prefix
= '/usr' if self
.use_usr
else ''
1484 self
.configure_args
= [
1485 '--prefix=%s' % self
.prefix
,
1487 '--build=%s' % glibc
.ctx
.build_triplet
,
1488 '--host=%s' % glibc
.triplet
,
1489 'CC=%s' % glibc
.tool_name('gcc'),
1490 'CXX=%s' % glibc
.tool_name('g++'),
1491 'AR=%s' % glibc
.tool_name('ar'),
1492 'AS=%s' % glibc
.tool_name('as'),
1493 'LD=%s' % glibc
.tool_name('ld'),
1494 'NM=%s' % glibc
.tool_name('nm'),
1495 'OBJCOPY=%s' % glibc
.tool_name('objcopy'),
1496 'OBJDUMP=%s' % glibc
.tool_name('objdump'),
1497 'RANLIB=%s' % glibc
.tool_name('ranlib'),
1498 'READELF=%s' % glibc
.tool_name('readelf'),
1499 'STRIP=%s' % glibc
.tool_name('strip'),
1501 if glibc
.os
== 'gnu':
1502 self
.configure_args
.append('MIG=%s' % glibc
.tool_name('mig'))
1504 self
.configure_args
.append('CFLAGS=%s' % glibc
.cflags
)
1505 self
.configure_args
.append('CXXFLAGS=%s' % glibc
.cflags
)
1506 self
.configure_args
+= glibc
.cfg
1508 def configure(self
, cmdlist
):
1509 """Invoked to add the configure command to the command list."""
1510 cmdlist
.add_command('configure',
1511 [os
.path
.join(self
.srcdir
, 'configure'),
1512 *self
.configure_args
])
1514 def extra_commands(self
, cmdlist
):
1515 """Invoked to inject additional commands (make check) after build."""
1518 class GlibcPolicyForCompiler(GlibcPolicyDefault
):
1519 """Build policy for glibc during the compilers stage."""
1521 def __init__(self
, glibc
):
1522 super().__init
__(glibc
)
1523 self
.builddir
= glibc
.ctx
.component_builddir(
1524 'compilers', glibc
.compiler
.name
, 'glibc', glibc
.name
)
1525 self
.installdir
= glibc
.compiler
.sysroot
1527 class GlibcPolicyForBuild(GlibcPolicyDefault
):
1528 """Build policy for glibc during the glibcs stage."""
1530 def __init__(self
, glibc
):
1531 super().__init
__(glibc
)
1532 self
.builddir
= glibc
.ctx
.component_builddir(
1533 'glibcs', glibc
.name
, 'glibc')
1534 self
.installdir
= glibc
.ctx
.glibc_installdir(glibc
.name
)
1536 self
.strip
= glibc
.tool_name('strip')
1539 self
.save_logs
= glibc
.ctx
.save_logs
1541 def extra_commands(self
, cmdlist
):
1543 # Avoid picking up libc.so and libpthread.so, which are
1544 # linker scripts stored in /lib on Hurd. libc and
1545 # libpthread are still stripped via their libc-X.YY.so
1546 # implementation files.
1547 find_command
= (('find %s/lib* -name "*.so"'
1548 + r
' \! -name libc.so \! -name libpthread.so')
1550 cmdlist
.add_command('strip', ['sh', '-c', ('%s $(%s)' %
1551 (self
.strip
, find_command
))])
1552 cmdlist
.add_command('check', ['make', 'check'])
1553 cmdlist
.add_command('save-logs', [self
.save_logs
], always_run
=True)
1555 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault
):
1556 """Build policy for glibc during update-syscalls."""
1558 def __init__(self
, glibc
):
1559 super().__init
__(glibc
)
1560 self
.builddir
= glibc
.ctx
.component_builddir(
1561 'update-syscalls', glibc
.name
, 'glibc')
1562 self
.linuxdir
= glibc
.ctx
.component_builddir(
1563 'update-syscalls', glibc
.name
, 'linux')
1564 self
.linux_policy
= LinuxHeadersPolicyForUpdateSyscalls(
1565 glibc
, self
.linuxdir
)
1566 self
.configure_args
.insert(
1567 0, '--with-headers=%s' % os
.path
.join(self
.linuxdir
, 'include'))
1568 # self.installdir not set because installation is not supported
1570 class Glibc(object):
1571 """A configuration for building glibc."""
1573 def __init__(self
, compiler
, arch
=None, os_name
=None, variant
=None,
1574 cfg
=None, ccopts
=None, cflags
=None):
1575 """Initialize a Glibc object."""
1576 self
.ctx
= compiler
.ctx
1577 self
.compiler
= compiler
1579 self
.arch
= compiler
.arch
1583 self
.os
= compiler
.os
1586 self
.variant
= variant
1588 self
.name
= '%s-%s' % (self
.arch
, self
.os
)
1590 self
.name
= '%s-%s-%s' % (self
.arch
, self
.os
, variant
)
1591 self
.triplet
= '%s-glibc-%s' % (self
.arch
, self
.os
)
1596 # ccopts contain ABI options and are passed to configure as CC / CXX.
1597 self
.ccopts
= ccopts
1598 # cflags contain non-ABI options like -g or -O and are passed to
1599 # configure as CFLAGS / CXXFLAGS.
1600 self
.cflags
= cflags
1602 def tool_name(self
, tool
):
1603 """Return the name of a cross-compilation tool."""
1604 ctool
= '%s-%s' % (self
.compiler
.triplet
, tool
)
1605 if self
.ccopts
and (tool
== 'gcc' or tool
== 'g++'):
1606 ctool
= '%s %s' % (ctool
, self
.ccopts
)
1610 """Generate commands to build this glibc."""
1611 builddir
= self
.ctx
.component_builddir('glibcs', self
.name
, 'glibc')
1612 installdir
= self
.ctx
.glibc_installdir(self
.name
)
1613 logsdir
= os
.path
.join(self
.ctx
.logsdir
, 'glibcs', self
.name
)
1614 self
.ctx
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
1615 cmdlist
= CommandList('glibcs-%s' % self
.name
, self
.ctx
.keep
)
1616 cmdlist
.add_command('check-compilers',
1618 os
.path
.join(self
.compiler
.installdir
, 'ok')])
1619 cmdlist
.use_path(self
.compiler
.bindir
)
1620 self
.build_glibc(cmdlist
, GlibcPolicyForBuild(self
))
1621 self
.ctx
.add_makefile_cmdlist('glibcs-%s' % self
.name
, cmdlist
,
1624 def build_glibc(self
, cmdlist
, policy
):
1625 """Generate commands to build this glibc, either as part of a compiler
1626 build or with the bootstrapped compiler (and in the latter case, run
1628 cmdlist
.create_use_dir(policy
.builddir
)
1629 policy
.configure(cmdlist
)
1630 cmdlist
.add_command('build', ['make'])
1631 cmdlist
.add_command('install', ['make', 'install',
1632 'install_root=%s' % policy
.installdir
])
1633 # GCC uses paths such as lib/../lib64, so make sure lib
1634 # directories always exist.
1635 mkdir_cmd
= ['mkdir', '-p',
1636 os
.path
.join(policy
.installdir
, 'lib')]
1638 mkdir_cmd
+= [os
.path
.join(policy
.installdir
, 'usr', 'lib')]
1639 cmdlist
.add_command('mkdir-lib', mkdir_cmd
)
1640 policy
.extra_commands(cmdlist
)
1641 cmdlist
.cleanup_dir()
1643 def update_syscalls(self
):
1644 if self
.os
== 'gnu':
1645 # Hurd does not have system call tables that need updating.
1648 policy
= GlibcPolicyForUpdateSyscalls(self
)
1649 logsdir
= os
.path
.join(self
.ctx
.logsdir
, 'update-syscalls', self
.name
)
1650 self
.ctx
.remove_recreate_dirs(policy
.builddir
, logsdir
)
1651 cmdlist
= CommandList('update-syscalls-%s' % self
.name
, self
.ctx
.keep
)
1652 cmdlist
.add_command('check-compilers',
1654 os
.path
.join(self
.compiler
.installdir
, 'ok')])
1655 cmdlist
.use_path(self
.compiler
.bindir
)
1657 install_linux_headers(policy
.linux_policy
, cmdlist
)
1659 cmdlist
.create_use_dir(policy
.builddir
)
1660 policy
.configure(cmdlist
)
1661 cmdlist
.add_command('build', ['make', 'update-syscall-lists'])
1662 cmdlist
.cleanup_dir()
1663 self
.ctx
.add_makefile_cmdlist('update-syscalls-%s' % self
.name
,
1666 class Command(object):
1667 """A command run in the build process."""
1669 def __init__(self
, desc
, num
, dir, path
, command
, always_run
=False):
1670 """Initialize a Command object."""
1674 trans
= str.maketrans({' ': '-'})
1675 self
.logbase
= '%03d-%s' % (num
, desc
.translate(trans
))
1676 self
.command
= command
1677 self
.always_run
= always_run
1680 def shell_make_quote_string(s
):
1681 """Given a string not containing a newline, quote it for use by the
1683 assert '\n' not in s
1684 if re
.fullmatch('[]+,./0-9@A-Z_a-z-]+', s
):
1686 strans
= str.maketrans({"'": "'\\''"})
1687 s
= "'%s'" % s
.translate(strans
)
1688 mtrans
= str.maketrans({'$': '$$'})
1689 return s
.translate(mtrans
)
1692 def shell_make_quote_list(l
, translate_make
):
1693 """Given a list of strings not containing newlines, quote them for use
1694 by the shell and make, returning a single string. If translate_make
1695 is true and the first string is 'make', change it to $(MAKE)."""
1696 l
= [Command
.shell_make_quote_string(s
) for s
in l
]
1697 if translate_make
and l
[0] == 'make':
1701 def shell_make_quote(self
):
1702 """Return this command quoted for the shell and make."""
1703 return self
.shell_make_quote_list(self
.command
, True)
1706 class CommandList(object):
1707 """A list of commands run in the build process."""
1709 def __init__(self
, desc
, keep
):
1710 """Initialize a CommandList object."""
1717 def desc_txt(self
, desc
):
1718 """Return the description to use for a command."""
1719 return '%s %s' % (' '.join(self
.desc
), desc
)
1721 def use_dir(self
, dir):
1722 """Set the default directory for subsequent commands."""
1725 def use_path(self
, path
):
1726 """Set a directory to be prepended to the PATH for subsequent
1730 def push_subdesc(self
, subdesc
):
1731 """Set the default subdescription for subsequent commands (e.g., the
1732 name of a component being built, within the series of commands
1734 self
.desc
.append(subdesc
)
1736 def pop_subdesc(self
):
1737 """Pop a subdescription from the list of descriptions."""
1740 def create_use_dir(self
, dir):
1741 """Remove and recreate a directory and use it for subsequent
1743 self
.add_command_dir('rm', None, ['rm', '-rf', dir])
1744 self
.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1747 def add_command_dir(self
, desc
, dir, command
, always_run
=False):
1748 """Add a command to run in a given directory."""
1749 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), dir, self
.path
,
1750 command
, always_run
)
1751 self
.cmdlist
.append(cmd
)
1753 def add_command(self
, desc
, command
, always_run
=False):
1754 """Add a command to run in the default directory."""
1755 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), self
.dir,
1756 self
.path
, command
, always_run
)
1757 self
.cmdlist
.append(cmd
)
1759 def cleanup_dir(self
, desc
='cleanup', dir=None):
1760 """Clean up a build directory. If no directory is specified, the
1761 default directory is cleaned up and ceases to be the default
1766 if self
.keep
!= 'all':
1767 self
.add_command_dir(desc
, None, ['rm', '-rf', dir],
1768 always_run
=(self
.keep
== 'none'))
1770 def makefile_commands(self
, wrapper
, logsdir
):
1771 """Return the sequence of commands in the form of text for a Makefile.
1772 The given wrapper script takes arguments: base of logs for
1773 previous command, or empty; base of logs for this command;
1774 description; directory; PATH addition; the command itself."""
1775 # prev_base is the base of the name for logs of the previous
1776 # command that is not always-run (that is, a build command,
1777 # whose failure should stop subsequent build commands from
1778 # being run, as opposed to a cleanup command, which is run
1779 # even if previous commands failed).
1782 for c
in self
.cmdlist
:
1783 ctxt
= c
.shell_make_quote()
1784 if prev_base
and not c
.always_run
:
1785 prev_log
= os
.path
.join(logsdir
, prev_base
)
1788 this_log
= os
.path
.join(logsdir
, c
.logbase
)
1789 if not c
.always_run
:
1790 prev_base
= c
.logbase
1799 prelims
= [wrapper
, prev_log
, this_log
, c
.desc
, dir, path
]
1800 prelim_txt
= Command
.shell_make_quote_list(prelims
, False)
1801 cmds
.append('\t@%s %s' % (prelim_txt
, ctxt
))
1802 return '\n'.join(cmds
)
1804 def status_logs(self
, logsdir
):
1805 """Return the list of log files with command status."""
1806 return [os
.path
.join(logsdir
, '%s-status.txt' % c
.logbase
)
1807 for c
in self
.cmdlist
]
1811 """Return an argument parser for this module."""
1812 parser
= argparse
.ArgumentParser(description
=__doc__
)
1813 parser
.add_argument('-j', dest
='parallelism',
1814 help='Run this number of jobs in parallel',
1815 type=int, default
=os
.cpu_count())
1816 parser
.add_argument('--keep', dest
='keep',
1817 help='Whether to keep all build directories, '
1818 'none or only those from failed builds',
1819 default
='none', choices
=('none', 'all', 'failed'))
1820 parser
.add_argument('--replace-sources', action
='store_true',
1821 help='Remove and replace source directories '
1822 'with the wrong version of a component')
1823 parser
.add_argument('--strip', action
='store_true',
1824 help='Strip installed glibc libraries')
1825 parser
.add_argument('--full-gcc', action
='store_true',
1826 help='Build GCC with all languages and libsanitizer')
1827 parser
.add_argument('--shallow', action
='store_true',
1828 help='Do not download Git history during checkout')
1829 parser
.add_argument('topdir',
1830 help='Toplevel working directory')
1831 parser
.add_argument('action',
1833 choices
=('checkout', 'bot-cycle', 'bot',
1834 'host-libraries', 'compilers', 'glibcs',
1835 'update-syscalls', 'list-compilers',
1837 parser
.add_argument('configs',
1838 help='Versions to check out or configurations to build',
1844 """The main entry point."""
1845 parser
= get_parser()
1846 opts
= parser
.parse_args(argv
)
1847 topdir
= os
.path
.abspath(opts
.topdir
)
1848 ctx
= Context(topdir
, opts
.parallelism
, opts
.keep
, opts
.replace_sources
,
1849 opts
.strip
, opts
.full_gcc
, opts
.action
,
1850 shallow
=opts
.shallow
)
1851 ctx
.run_builds(opts
.action
, opts
.configs
)
1854 if __name__
== '__main__':