2 # Build many configurations of glibc.
3 # Copyright (C) 2016-2017 Free Software Foundation, Inc.
4 # This file is part of the GNU C Library.
6 # The GNU C Library is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU Lesser General Public
8 # License as published by the Free Software Foundation; either
9 # version 2.1 of the License, or (at your option) any later version.
11 # The GNU C Library is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # Lesser General Public License for more details.
16 # You should have received a copy of the GNU Lesser General Public
17 # License along with the GNU C Library; if not, see
18 # <http://www.gnu.org/licenses/>.
20 """Build many configurations of glibc.
22 This script takes as arguments a directory name (containing a src
23 subdirectory with sources of the relevant toolchain components) and a
24 description of what to do: 'checkout', to check out sources into that
25 directory, 'bot-cycle', to run a series of checkout and build steps,
26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
27 libraries required by the toolchain, 'compilers', to build
28 cross-compilers for various configurations, or 'glibcs', to build
29 glibc for various configurations and run the compilation parts of the
30 testsuite. Subsequent arguments name the versions of components to
31 check out (<component>-<version), for 'checkout', or, for actions
32 other than 'checkout' and 'bot-cycle', name configurations for which
33 compilers or glibc are to be built.
39 import email
.mime
.text
55 import multiprocessing
56 os
.cpu_count
= lambda: multiprocessing
.cpu_count()
61 re
.fullmatch
= lambda p
,s
,f
=0: re
.match(p
+"\\Z",s
,f
)
66 class _CompletedProcess
:
67 def __init__(self
, args
, returncode
, stdout
=None, stderr
=None):
69 self
.returncode
= returncode
73 def _run(*popenargs
, input=None, timeout
=None, check
=False, **kwargs
):
74 assert(timeout
is None)
75 with subprocess
.Popen(*popenargs
, **kwargs
) as process
:
77 stdout
, stderr
= process
.communicate(input)
82 returncode
= process
.poll()
83 if check
and returncode
:
84 raise subprocess
.CalledProcessError(returncode
, popenargs
)
85 return _CompletedProcess(popenargs
, returncode
, stdout
, stderr
)
90 class Context(object):
91 """The global state associated with builds in a given directory."""
93 def __init__(self
, topdir
, parallelism
, keep
, replace_sources
, strip
,
95 """Initialize the context."""
97 self
.parallelism
= parallelism
99 self
.replace_sources
= replace_sources
101 self
.srcdir
= os
.path
.join(topdir
, 'src')
102 self
.versions_json
= os
.path
.join(self
.srcdir
, 'versions.json')
103 self
.build_state_json
= os
.path
.join(topdir
, 'build-state.json')
104 self
.bot_config_json
= os
.path
.join(topdir
, 'bot-config.json')
105 self
.installdir
= os
.path
.join(topdir
, 'install')
106 self
.host_libraries_installdir
= os
.path
.join(self
.installdir
,
108 self
.builddir
= os
.path
.join(topdir
, 'build')
109 self
.logsdir
= os
.path
.join(topdir
, 'logs')
110 self
.logsdir_old
= os
.path
.join(topdir
, 'logs-old')
111 self
.makefile
= os
.path
.join(self
.builddir
, 'Makefile')
112 self
.wrapper
= os
.path
.join(self
.builddir
, 'wrapper')
113 self
.save_logs
= os
.path
.join(self
.builddir
, 'save-logs')
114 self
.script_text
= self
.get_script_text()
115 if action
!= 'checkout':
116 self
.build_triplet
= self
.get_build_triplet()
117 self
.glibc_version
= self
.get_glibc_version()
119 self
.glibc_configs
= {}
120 self
.makefile_pieces
= ['.PHONY: all\n']
121 self
.add_all_configs()
122 self
.load_versions_json()
123 self
.load_build_state_json()
124 self
.status_log_list
= []
126 def get_script_text(self
):
127 """Return the text of this script."""
128 with
open(sys
.argv
[0], 'r') as f
:
132 """Re-execute this script with the same arguments."""
134 os
.execv(sys
.executable
, [sys
.executable
] + sys
.argv
)
136 def get_build_triplet(self
):
137 """Determine the build triplet with config.guess."""
138 config_guess
= os
.path
.join(self
.component_srcdir('gcc'),
140 cg_out
= subprocess
.run([config_guess
], stdout
=subprocess
.PIPE
,
141 check
=True, universal_newlines
=True).stdout
142 return cg_out
.rstrip()
144 def get_glibc_version(self
):
145 """Determine the glibc version number (major.minor)."""
146 version_h
= os
.path
.join(self
.component_srcdir('glibc'), 'version.h')
147 with
open(version_h
, 'r') as f
:
148 lines
= f
.readlines()
149 starttext
= '#define VERSION "'
151 if l
.startswith(starttext
):
152 l
= l
[len(starttext
):]
154 m
= re
.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l
)
155 return '%s.%s' % m
.group(1, 2)
156 print('error: could not determine glibc version')
159 def add_all_configs(self
):
160 """Add all known glibc build configurations."""
161 # On architectures missing __builtin_trap support, these
162 # options may be needed as a workaround; see
163 # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70216> for SH.
164 no_isolate
= ('-fno-isolate-erroneous-paths-dereference'
165 ' -fno-isolate-erroneous-paths-attribute')
166 self
.add_config(arch
='aarch64',
168 self
.add_config(arch
='aarch64_be',
170 self
.add_config(arch
='alpha',
172 self
.add_config(arch
='arm',
173 os_name
='linux-gnueabi')
174 self
.add_config(arch
='armeb',
175 os_name
='linux-gnueabi')
176 self
.add_config(arch
='armeb',
177 os_name
='linux-gnueabi',
179 gcc_cfg
=['--with-arch=armv7-a'])
180 self
.add_config(arch
='arm',
181 os_name
='linux-gnueabihf')
182 self
.add_config(arch
='armeb',
183 os_name
='linux-gnueabihf')
184 self
.add_config(arch
='armeb',
185 os_name
='linux-gnueabihf',
187 gcc_cfg
=['--with-arch=armv7-a'])
188 self
.add_config(arch
='hppa',
190 self
.add_config(arch
='ia64',
192 first_gcc_cfg
=['--with-system-libunwind'])
193 self
.add_config(arch
='m68k',
195 gcc_cfg
=['--disable-multilib'])
196 self
.add_config(arch
='m68k',
199 gcc_cfg
=['--with-arch=cf', '--disable-multilib'])
200 self
.add_config(arch
='microblaze',
202 gcc_cfg
=['--disable-multilib'])
203 self
.add_config(arch
='microblazeel',
205 gcc_cfg
=['--disable-multilib'])
206 self
.add_config(arch
='mips64',
208 gcc_cfg
=['--with-mips-plt'],
209 glibcs
=[{'variant': 'n32'},
211 'ccopts': '-mabi=32'},
213 'ccopts': '-mabi=64'}])
214 self
.add_config(arch
='mips64',
217 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
218 glibcs
=[{'variant': 'n32-soft',
219 'cfg': ['--without-fp']},
222 'ccopts': '-mabi=32',
223 'cfg': ['--without-fp']},
224 {'variant': 'n64-soft',
225 'ccopts': '-mabi=64',
226 'cfg': ['--without-fp']}])
227 self
.add_config(arch
='mips64',
230 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
231 '--with-arch-64=mips64r2',
232 '--with-arch-32=mips32r2'],
233 glibcs
=[{'variant': 'n32-nan2008'},
234 {'variant': 'nan2008',
236 'ccopts': '-mabi=32'},
237 {'variant': 'n64-nan2008',
238 'ccopts': '-mabi=64'}])
239 self
.add_config(arch
='mips64',
241 variant
='nan2008-soft',
242 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
243 '--with-arch-64=mips64r2',
244 '--with-arch-32=mips32r2',
245 '--with-float=soft'],
246 glibcs
=[{'variant': 'n32-nan2008-soft',
247 'cfg': ['--without-fp']},
248 {'variant': 'nan2008-soft',
250 'ccopts': '-mabi=32',
251 'cfg': ['--without-fp']},
252 {'variant': 'n64-nan2008-soft',
253 'ccopts': '-mabi=64',
254 'cfg': ['--without-fp']}])
255 self
.add_config(arch
='mips64el',
257 gcc_cfg
=['--with-mips-plt'],
258 glibcs
=[{'variant': 'n32'},
260 'ccopts': '-mabi=32'},
262 'ccopts': '-mabi=64'}])
263 self
.add_config(arch
='mips64el',
266 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
267 glibcs
=[{'variant': 'n32-soft',
268 'cfg': ['--without-fp']},
271 'ccopts': '-mabi=32',
272 'cfg': ['--without-fp']},
273 {'variant': 'n64-soft',
274 'ccopts': '-mabi=64',
275 'cfg': ['--without-fp']}])
276 self
.add_config(arch
='mips64el',
279 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
280 '--with-arch-64=mips64r2',
281 '--with-arch-32=mips32r2'],
282 glibcs
=[{'variant': 'n32-nan2008'},
283 {'variant': 'nan2008',
285 'ccopts': '-mabi=32'},
286 {'variant': 'n64-nan2008',
287 'ccopts': '-mabi=64'}])
288 self
.add_config(arch
='mips64el',
290 variant
='nan2008-soft',
291 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
292 '--with-arch-64=mips64r2',
293 '--with-arch-32=mips32r2',
294 '--with-float=soft'],
295 glibcs
=[{'variant': 'n32-nan2008-soft',
296 'cfg': ['--without-fp']},
297 {'variant': 'nan2008-soft',
299 'ccopts': '-mabi=32',
300 'cfg': ['--without-fp']},
301 {'variant': 'n64-nan2008-soft',
302 'ccopts': '-mabi=64',
303 'cfg': ['--without-fp']}])
304 self
.add_config(arch
='nios2',
306 self
.add_config(arch
='powerpc',
308 gcc_cfg
=['--disable-multilib', '--enable-secureplt'],
309 extra_glibcs
=[{'variant': 'power4',
310 'ccopts': '-mcpu=power4',
311 'cfg': ['--with-cpu=power4']}])
312 self
.add_config(arch
='powerpc',
315 gcc_cfg
=['--disable-multilib', '--with-float=soft',
316 '--enable-secureplt'],
317 glibcs
=[{'variant': 'soft', 'cfg': ['--without-fp']}])
318 self
.add_config(arch
='powerpc64',
320 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
321 self
.add_config(arch
='powerpc64le',
323 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
324 self
.add_config(arch
='powerpc',
325 os_name
='linux-gnuspe',
326 gcc_cfg
=['--disable-multilib', '--enable-secureplt',
327 '--enable-e500-double'],
328 glibcs
=[{'cfg': ['--without-fp']}])
329 self
.add_config(arch
='powerpc',
330 os_name
='linux-gnuspe',
332 gcc_cfg
=['--disable-multilib', '--enable-secureplt'],
333 glibcs
=[{'variant': 'e500v1', 'cfg': ['--without-fp']}])
334 self
.add_config(arch
='s390x',
337 {'arch': 's390', 'ccopts': '-m31'}])
338 self
.add_config(arch
='sh3',
340 glibcs
=[{'ccopts': no_isolate
}])
341 self
.add_config(arch
='sh3eb',
343 glibcs
=[{'ccopts': no_isolate
}])
344 self
.add_config(arch
='sh4',
346 glibcs
=[{'ccopts': no_isolate
}])
347 self
.add_config(arch
='sh4eb',
349 glibcs
=[{'ccopts': no_isolate
}])
350 self
.add_config(arch
='sh4',
353 gcc_cfg
=['--without-fp'],
354 glibcs
=[{'variant': 'soft',
355 'cfg': ['--without-fp'],
356 'ccopts': no_isolate
}])
357 self
.add_config(arch
='sh4eb',
360 gcc_cfg
=['--without-fp'],
361 glibcs
=[{'variant': 'soft',
362 'cfg': ['--without-fp'],
363 'ccopts': no_isolate
}])
364 self
.add_config(arch
='sparc64',
368 'ccopts': '-m32 -mlong-double-128'}])
369 self
.add_config(arch
='tilegx',
372 {'variant': '32', 'ccopts': '-m32'}])
373 self
.add_config(arch
='tilegxbe',
376 {'variant': '32', 'ccopts': '-m32'}])
377 self
.add_config(arch
='tilepro',
379 self
.add_config(arch
='x86_64',
381 gcc_cfg
=['--with-multilib-list=m64,m32,mx32'],
383 {'variant': 'x32', 'ccopts': '-mx32'},
384 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
385 extra_glibcs
=[{'variant': 'disable-multi-arch',
386 'cfg': ['--disable-multi-arch']},
387 {'variant': 'disable-multi-arch',
389 'ccopts': '-m32 -march=i686',
390 'cfg': ['--disable-multi-arch']},
392 'ccopts': '-m32 -march=i486'},
394 'ccopts': '-m32 -march=i586'}])
396 def add_config(self
, **args
):
397 """Add an individual build configuration."""
398 cfg
= Config(self
, **args
)
399 if cfg
.name
in self
.configs
:
400 print('error: duplicate config %s' % cfg
.name
)
402 self
.configs
[cfg
.name
] = cfg
403 for c
in cfg
.all_glibcs
:
404 if c
.name
in self
.glibc_configs
:
405 print('error: duplicate glibc config %s' % c
.name
)
407 self
.glibc_configs
[c
.name
] = c
409 def component_srcdir(self
, component
):
410 """Return the source directory for a given component, e.g. gcc."""
411 return os
.path
.join(self
.srcdir
, component
)
413 def component_builddir(self
, action
, config
, component
, subconfig
=None):
414 """Return the directory to use for a build."""
417 assert subconfig
is None
418 return os
.path
.join(self
.builddir
, action
, component
)
419 if subconfig
is None:
420 return os
.path
.join(self
.builddir
, action
, config
, component
)
422 # glibc build as part of compiler build.
423 return os
.path
.join(self
.builddir
, action
, config
, component
,
426 def compiler_installdir(self
, config
):
427 """Return the directory in which to install a compiler."""
428 return os
.path
.join(self
.installdir
, 'compilers', config
)
430 def compiler_bindir(self
, config
):
431 """Return the directory in which to find compiler binaries."""
432 return os
.path
.join(self
.compiler_installdir(config
), 'bin')
434 def compiler_sysroot(self
, config
):
435 """Return the sysroot directory for a compiler."""
436 return os
.path
.join(self
.compiler_installdir(config
), 'sysroot')
438 def glibc_installdir(self
, config
):
439 """Return the directory in which to install glibc."""
440 return os
.path
.join(self
.installdir
, 'glibcs', config
)
442 def run_builds(self
, action
, configs
):
443 """Run the requested builds."""
444 if action
== 'checkout':
445 self
.checkout(configs
)
447 if action
== 'bot-cycle':
449 print('error: configurations specified for bot-cycle')
455 print('error: configurations specified for bot')
459 if action
== 'host-libraries' and configs
:
460 print('error: configurations specified for host-libraries')
462 self
.clear_last_build_state(action
)
463 build_time
= datetime
.datetime
.utcnow()
464 if action
== 'host-libraries':
465 build_components
= ('gmp', 'mpfr', 'mpc')
468 self
.build_host_libraries()
469 elif action
== 'compilers':
470 build_components
= ('binutils', 'gcc', 'glibc', 'linux')
471 old_components
= ('gmp', 'mpfr', 'mpc')
472 old_versions
= self
.build_state
['host-libraries']['build-versions']
473 self
.build_compilers(configs
)
475 build_components
= ('glibc',)
476 old_components
= ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
477 old_versions
= self
.build_state
['compilers']['build-versions']
478 self
.build_glibcs(configs
)
482 # Partial build, do not update stored state.
485 for k
in build_components
:
486 if k
in self
.versions
:
487 build_versions
[k
] = {'version': self
.versions
[k
]['version'],
488 'revision': self
.versions
[k
]['revision']}
489 for k
in old_components
:
490 if k
in old_versions
:
491 build_versions
[k
] = {'version': old_versions
[k
]['version'],
492 'revision': old_versions
[k
]['revision']}
493 self
.update_build_state(action
, build_time
, build_versions
)
496 def remove_dirs(*args
):
497 """Remove directories and their contents if they exist."""
499 shutil
.rmtree(dir, ignore_errors
=True)
502 def remove_recreate_dirs(*args
):
503 """Remove directories if they exist, and create them as empty."""
504 Context
.remove_dirs(*args
)
506 os
.makedirs(dir, exist_ok
=True)
508 def add_makefile_cmdlist(self
, target
, cmdlist
, logsdir
):
509 """Add makefile text for a list of commands."""
510 commands
= cmdlist
.makefile_commands(self
.wrapper
, logsdir
)
511 self
.makefile_pieces
.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
512 (target
, target
, target
, commands
))
513 self
.status_log_list
.extend(cmdlist
.status_logs(logsdir
))
515 def write_files(self
):
516 """Write out the Makefile and wrapper script."""
517 mftext
= ''.join(self
.makefile_pieces
)
518 with
open(self
.makefile
, 'w') as f
:
528 'prev_status=$prev_base-status.txt\n'
529 'this_status=$this_base-status.txt\n'
530 'this_log=$this_base-log.txt\n'
531 'date > "$this_log"\n'
532 'echo >> "$this_log"\n'
533 'echo "Description: $desc" >> "$this_log"\n'
534 'printf "%s" "Command:" >> "$this_log"\n'
535 'for word in "$@"; do\n'
536 ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
537 ' printf " %s" "$word"\n'
540 ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
543 'done >> "$this_log"\n'
544 'echo >> "$this_log"\n'
545 'echo "Directory: $dir" >> "$this_log"\n'
546 'echo "Path addition: $path" >> "$this_log"\n'
547 'echo >> "$this_log"\n'
550 ' echo >> "$this_log"\n'
551 ' echo "$1: $desc" > "$this_status"\n'
552 ' echo "$1: $desc" >> "$this_log"\n'
553 ' echo >> "$this_log"\n'
554 ' date >> "$this_log"\n'
555 ' echo "$1: $desc"\n'
560 ' if [ "$1" != "0" ]; then\n'
561 ' record_status FAIL\n'
564 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
565 ' record_status UNRESOLVED\n'
567 'if [ "$dir" ]; then\n'
569 ' check_error "$?"\n'
571 'if [ "$path" ]; then\n'
572 ' PATH=$path:$PATH\n'
574 '"$@" < /dev/null >> "$this_log" 2>&1\n'
576 'record_status PASS\n')
577 with
open(self
.wrapper
, 'w') as f
:
578 f
.write(wrapper_text
)
580 mode_exec
= (stat
.S_IRWXU|stat
.S_IRGRP|stat
.S_IXGRP|
581 stat
.S_IROTH|stat
.S_IXOTH
)
582 os
.chmod(self
.wrapper
, mode_exec
)
585 'if ! [ -f tests.sum ]; then\n'
586 ' echo "No test summary available."\n'
591 ' echo "Contents of $1:"\n'
595 ' echo "End of contents of $1."\n'
598 'save_file tests.sum\n'
599 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
600 'for t in $non_pass_tests; do\n'
601 ' if [ -f "$t.out" ]; then\n'
602 ' save_file "$t.out"\n'
605 with
open(self
.save_logs
, 'w') as f
:
606 f
.write(save_logs_text
)
607 os
.chmod(self
.save_logs
, mode_exec
)
610 """Do the actual build."""
611 cmd
= ['make', '-j%d' % self
.parallelism
]
612 subprocess
.run(cmd
, cwd
=self
.builddir
, check
=True)
614 def build_host_libraries(self
):
615 """Build the host libraries."""
616 installdir
= self
.host_libraries_installdir
617 builddir
= os
.path
.join(self
.builddir
, 'host-libraries')
618 logsdir
= os
.path
.join(self
.logsdir
, 'host-libraries')
619 self
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
620 cmdlist
= CommandList('host-libraries', self
.keep
)
621 self
.build_host_library(cmdlist
, 'gmp')
622 self
.build_host_library(cmdlist
, 'mpfr',
623 ['--with-gmp=%s' % installdir
])
624 self
.build_host_library(cmdlist
, 'mpc',
625 ['--with-gmp=%s' % installdir
,
626 '--with-mpfr=%s' % installdir
])
627 cmdlist
.add_command('done', ['touch', os
.path
.join(installdir
, 'ok')])
628 self
.add_makefile_cmdlist('host-libraries', cmdlist
, logsdir
)
630 def build_host_library(self
, cmdlist
, lib
, extra_opts
=None):
631 """Build one host library."""
632 srcdir
= self
.component_srcdir(lib
)
633 builddir
= self
.component_builddir('host-libraries', None, lib
)
634 installdir
= self
.host_libraries_installdir
635 cmdlist
.push_subdesc(lib
)
636 cmdlist
.create_use_dir(builddir
)
637 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
638 '--prefix=%s' % installdir
,
641 cfg_cmd
.extend (extra_opts
)
642 cmdlist
.add_command('configure', cfg_cmd
)
643 cmdlist
.add_command('build', ['make'])
644 cmdlist
.add_command('check', ['make', 'check'])
645 cmdlist
.add_command('install', ['make', 'install'])
646 cmdlist
.cleanup_dir()
647 cmdlist
.pop_subdesc()
649 def build_compilers(self
, configs
):
650 """Build the compilers."""
652 self
.remove_dirs(os
.path
.join(self
.builddir
, 'compilers'))
653 self
.remove_dirs(os
.path
.join(self
.installdir
, 'compilers'))
654 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'compilers'))
655 configs
= sorted(self
.configs
.keys())
657 self
.configs
[c
].build()
659 def build_glibcs(self
, configs
):
660 """Build the glibcs."""
662 self
.remove_dirs(os
.path
.join(self
.builddir
, 'glibcs'))
663 self
.remove_dirs(os
.path
.join(self
.installdir
, 'glibcs'))
664 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'glibcs'))
665 configs
= sorted(self
.glibc_configs
.keys())
667 self
.glibc_configs
[c
].build()
669 def load_versions_json(self
):
670 """Load information about source directory versions."""
671 if not os
.access(self
.versions_json
, os
.F_OK
):
674 with
open(self
.versions_json
, 'r') as f
:
675 self
.versions
= json
.load(f
)
677 def store_json(self
, data
, filename
):
678 """Store information in a JSON file."""
679 filename_tmp
= filename
+ '.tmp'
680 with
open(filename_tmp
, 'w') as f
:
681 json
.dump(data
, f
, indent
=2, sort_keys
=True)
682 os
.rename(filename_tmp
, filename
)
684 def store_versions_json(self
):
685 """Store information about source directory versions."""
686 self
.store_json(self
.versions
, self
.versions_json
)
688 def set_component_version(self
, component
, version
, explicit
, revision
):
689 """Set the version information for a component."""
690 self
.versions
[component
] = {'version': version
,
691 'explicit': explicit
,
692 'revision': revision
}
693 self
.store_versions_json()
695 def checkout(self
, versions
):
696 """Check out the desired component versions."""
697 default_versions
= {'binutils': 'vcs-2.28',
699 'glibc': 'vcs-mainline',
705 explicit_versions
= {}
708 for k
in default_versions
.keys():
712 if k
in use_versions
:
713 print('error: multiple versions for %s' % k
)
716 explicit_versions
[k
] = True
720 print('error: unknown component in %s' % v
)
722 for k
in default_versions
.keys():
723 if k
not in use_versions
:
724 if k
in self
.versions
and self
.versions
[k
]['explicit']:
725 use_versions
[k
] = self
.versions
[k
]['version']
726 explicit_versions
[k
] = True
728 use_versions
[k
] = default_versions
[k
]
729 explicit_versions
[k
] = False
730 os
.makedirs(self
.srcdir
, exist_ok
=True)
731 for k
in sorted(default_versions
.keys()):
732 update
= os
.access(self
.component_srcdir(k
), os
.F_OK
)
735 k
in self
.versions
and
736 v
!= self
.versions
[k
]['version']):
737 if not self
.replace_sources
:
738 print('error: version of %s has changed from %s to %s, '
739 'use --replace-sources to check out again' %
740 (k
, self
.versions
[k
]['version'], v
))
742 shutil
.rmtree(self
.component_srcdir(k
))
744 if v
.startswith('vcs-'):
745 revision
= self
.checkout_vcs(k
, v
[4:], update
)
747 self
.checkout_tar(k
, v
, update
)
749 self
.set_component_version(k
, v
, explicit_versions
[k
], revision
)
750 if self
.get_script_text() != self
.script_text
:
751 # Rerun the checkout process in case the updated script
752 # uses different default versions or new components.
755 def checkout_vcs(self
, component
, version
, update
):
756 """Check out the given version of the given component from version
757 control. Return a revision identifier."""
758 if component
== 'binutils':
759 git_url
= 'git://sourceware.org/git/binutils-gdb.git'
760 if version
== 'mainline':
761 git_branch
= 'master'
763 trans
= str.maketrans({'.': '_'})
764 git_branch
= 'binutils-%s-branch' % version
.translate(trans
)
765 return self
.git_checkout(component
, git_url
, git_branch
, update
)
766 elif component
== 'gcc':
767 if version
== 'mainline':
770 trans
= str.maketrans({'.': '_'})
771 branch
= 'branches/gcc-%s-branch' % version
.translate(trans
)
772 svn_url
= 'svn://gcc.gnu.org/svn/gcc/%s' % branch
773 return self
.gcc_checkout(svn_url
, update
)
774 elif component
== 'glibc':
775 git_url
= 'git://sourceware.org/git/glibc.git'
776 if version
== 'mainline':
777 git_branch
= 'master'
779 git_branch
= 'release/%s/master' % version
780 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
781 self
.fix_glibc_timestamps()
784 print('error: component %s coming from VCS' % component
)
787 def git_checkout(self
, component
, git_url
, git_branch
, update
):
788 """Check out a component from git. Return a commit identifier."""
790 subprocess
.run(['git', 'remote', 'prune', 'origin'],
791 cwd
=self
.component_srcdir(component
), check
=True)
792 subprocess
.run(['git', 'pull', '-q'],
793 cwd
=self
.component_srcdir(component
), check
=True)
795 subprocess
.run(['git', 'clone', '-q', '-b', git_branch
, git_url
,
796 self
.component_srcdir(component
)], check
=True)
797 r
= subprocess
.run(['git', 'rev-parse', 'HEAD'],
798 cwd
=self
.component_srcdir(component
),
799 stdout
=subprocess
.PIPE
,
800 check
=True, universal_newlines
=True).stdout
803 def fix_glibc_timestamps(self
):
804 """Fix timestamps in a glibc checkout."""
805 # Ensure that builds do not try to regenerate generated files
806 # in the source tree.
807 srcdir
= self
.component_srcdir('glibc')
808 for dirpath
, dirnames
, filenames
in os
.walk(srcdir
):
810 if (f
== 'configure' or
811 f
== 'preconfigure' or
812 f
.endswith('-kw.h')):
813 to_touch
= os
.path
.join(dirpath
, f
)
814 subprocess
.run(['touch', to_touch
], check
=True)
816 def gcc_checkout(self
, svn_url
, update
):
817 """Check out GCC from SVN. Return the revision number."""
819 subprocess
.run(['svn', 'co', '-q', svn_url
,
820 self
.component_srcdir('gcc')], check
=True)
821 subprocess
.run(['contrib/gcc_update', '--silent'],
822 cwd
=self
.component_srcdir('gcc'), check
=True)
823 r
= subprocess
.run(['svnversion', self
.component_srcdir('gcc')],
824 stdout
=subprocess
.PIPE
,
825 check
=True, universal_newlines
=True).stdout
828 def checkout_tar(self
, component
, version
, update
):
829 """Check out the given version of the given component from a
833 url_map
= {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
834 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
835 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
836 'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
837 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
838 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
839 if component
not in url_map
:
840 print('error: component %s coming from tarball' % component
)
842 url
= url_map
[component
] % {'version': version
}
843 filename
= os
.path
.join(self
.srcdir
, url
.split('/')[-1])
844 response
= urllib
.request
.urlopen(url
)
845 data
= response
.read()
846 with
open(filename
, 'wb') as f
:
848 subprocess
.run(['tar', '-C', self
.srcdir
, '-x', '-f', filename
],
850 os
.rename(os
.path
.join(self
.srcdir
, '%s-%s' % (component
, version
)),
851 self
.component_srcdir(component
))
854 def load_build_state_json(self
):
855 """Load information about the state of previous builds."""
856 if os
.access(self
.build_state_json
, os
.F_OK
):
857 with
open(self
.build_state_json
, 'r') as f
:
858 self
.build_state
= json
.load(f
)
860 self
.build_state
= {}
861 for k
in ('host-libraries', 'compilers', 'glibcs'):
862 if k
not in self
.build_state
:
863 self
.build_state
[k
] = {}
864 if 'build-time' not in self
.build_state
[k
]:
865 self
.build_state
[k
]['build-time'] = ''
866 if 'build-versions' not in self
.build_state
[k
]:
867 self
.build_state
[k
]['build-versions'] = {}
868 if 'build-results' not in self
.build_state
[k
]:
869 self
.build_state
[k
]['build-results'] = {}
870 if 'result-changes' not in self
.build_state
[k
]:
871 self
.build_state
[k
]['result-changes'] = {}
872 if 'ever-passed' not in self
.build_state
[k
]:
873 self
.build_state
[k
]['ever-passed'] = []
875 def store_build_state_json(self
):
876 """Store information about the state of previous builds."""
877 self
.store_json(self
.build_state
, self
.build_state_json
)
879 def clear_last_build_state(self
, action
):
880 """Clear information about the state of part of the build."""
881 # We clear the last build time and versions when starting a
882 # new build. The results of the last build are kept around,
883 # as comparison is still meaningful if this build is aborted
884 # and a new one started.
885 self
.build_state
[action
]['build-time'] = ''
886 self
.build_state
[action
]['build-versions'] = {}
887 self
.store_build_state_json()
889 def update_build_state(self
, action
, build_time
, build_versions
):
890 """Update the build state after a build."""
891 build_time
= build_time
.replace(microsecond
=0)
892 self
.build_state
[action
]['build-time'] = str(build_time
)
893 self
.build_state
[action
]['build-versions'] = build_versions
895 for log
in self
.status_log_list
:
896 with
open(log
, 'r') as f
:
898 log_text
= log_text
.rstrip()
899 m
= re
.fullmatch('([A-Z]+): (.*)', log_text
)
901 test_name
= m
.group(2)
902 assert test_name
not in build_results
903 build_results
[test_name
] = result
904 old_build_results
= self
.build_state
[action
]['build-results']
905 self
.build_state
[action
]['build-results'] = build_results
907 all_tests
= set(old_build_results
.keys()) |
set(build_results
.keys())
909 if t
in old_build_results
:
910 old_res
= old_build_results
[t
]
912 old_res
= '(New test)'
913 if t
in build_results
:
914 new_res
= build_results
[t
]
916 new_res
= '(Test removed)'
917 if old_res
!= new_res
:
918 result_changes
[t
] = '%s -> %s' % (old_res
, new_res
)
919 self
.build_state
[action
]['result-changes'] = result_changes
920 old_ever_passed
= {t
for t
in self
.build_state
[action
]['ever-passed']
921 if t
in build_results
}
922 new_passes
= {t
for t
in build_results
if build_results
[t
] == 'PASS'}
923 self
.build_state
[action
]['ever-passed'] = sorted(old_ever_passed |
925 self
.store_build_state_json()
927 def load_bot_config_json(self
):
928 """Load bot configuration."""
929 with
open(self
.bot_config_json
, 'r') as f
:
930 self
.bot_config
= json
.load(f
)
932 def part_build_old(self
, action
, delay
):
933 """Return whether the last build for a given action was at least a
934 given number of seconds ago, or does not have a time recorded."""
935 old_time_str
= self
.build_state
[action
]['build-time']
938 old_time
= datetime
.datetime
.strptime(old_time_str
,
940 new_time
= datetime
.datetime
.utcnow()
941 delta
= new_time
- old_time
942 return delta
.total_seconds() >= delay
945 """Run a single round of checkout and builds."""
946 print('Bot cycle starting %s.' % str(datetime
.datetime
.utcnow()))
947 self
.load_bot_config_json()
948 actions
= ('host-libraries', 'compilers', 'glibcs')
949 self
.bot_run_self(['--replace-sources'], 'checkout')
950 self
.load_versions_json()
951 if self
.get_script_text() != self
.script_text
:
952 print('Script changed, re-execing.')
953 # On script change, all parts of the build should be rerun.
955 self
.clear_last_build_state(a
)
957 check_components
= {'host-libraries': ('gmp', 'mpfr', 'mpc'),
958 'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
959 'glibcs': ('glibc',)}
962 build_vers
= self
.build_state
[a
]['build-versions']
963 must_build
[a
] = False
964 if not self
.build_state
[a
]['build-time']:
968 for c
in check_components
[a
]:
970 old_vers
[c
] = build_vers
[c
]
971 new_vers
[c
] = {'version': self
.versions
[c
]['version'],
972 'revision': self
.versions
[c
]['revision']}
973 if new_vers
== old_vers
:
974 print('Versions for %s unchanged.' % a
)
976 print('Versions changed or rebuild forced for %s.' % a
)
977 if a
== 'compilers' and not self
.part_build_old(
978 a
, self
.bot_config
['compilers-rebuild-delay']):
979 print('Not requiring rebuild of compilers this soon.')
982 if must_build
['host-libraries']:
983 must_build
['compilers'] = True
984 if must_build
['compilers']:
985 must_build
['glibcs'] = True
988 print('Must rebuild %s.' % a
)
989 self
.clear_last_build_state(a
)
991 print('No need to rebuild %s.' % a
)
992 if os
.access(self
.logsdir
, os
.F_OK
):
993 shutil
.rmtree(self
.logsdir_old
, ignore_errors
=True)
994 shutil
.copytree(self
.logsdir
, self
.logsdir_old
)
997 build_time
= datetime
.datetime
.utcnow()
998 print('Rebuilding %s at %s.' % (a
, str(build_time
)))
999 self
.bot_run_self([], a
)
1000 self
.load_build_state_json()
1001 self
.bot_build_mail(a
, build_time
)
1002 print('Bot cycle done at %s.' % str(datetime
.datetime
.utcnow()))
1004 def bot_build_mail(self
, action
, build_time
):
1005 """Send email with the results of a build."""
1006 build_time
= build_time
.replace(microsecond
=0)
1007 subject
= (self
.bot_config
['email-subject'] %
1009 'build-time': str(build_time
)})
1010 results
= self
.build_state
[action
]['build-results']
1011 changes
= self
.build_state
[action
]['result-changes']
1012 ever_passed
= set(self
.build_state
[action
]['ever-passed'])
1013 versions
= self
.build_state
[action
]['build-versions']
1014 new_regressions
= {k
for k
in changes
if changes
[k
] == 'PASS -> FAIL'}
1015 all_regressions
= {k
for k
in ever_passed
if results
[k
] == 'FAIL'}
1016 all_fails
= {k
for k
in results
if results
[k
] == 'FAIL'}
1018 new_reg_list
= sorted(['FAIL: %s' % k
for k
in new_regressions
])
1019 new_reg_text
= ('New regressions:\n\n%s\n\n' %
1020 '\n'.join(new_reg_list
))
1024 all_reg_list
= sorted(['FAIL: %s' % k
for k
in all_regressions
])
1025 all_reg_text
= ('All regressions:\n\n%s\n\n' %
1026 '\n'.join(all_reg_list
))
1030 all_fail_list
= sorted(['FAIL: %s' % k
for k
in all_fails
])
1031 all_fail_text
= ('All failures:\n\n%s\n\n' %
1032 '\n'.join(all_fail_list
))
1036 changes_list
= sorted(changes
.keys())
1037 changes_list
= ['%s: %s' % (changes
[k
], k
) for k
in changes_list
]
1038 changes_text
= ('All changed results:\n\n%s\n\n' %
1039 '\n'.join(changes_list
))
1042 results_text
= (new_reg_text
+ all_reg_text
+ all_fail_text
+
1044 if not results_text
:
1045 results_text
= 'Clean build with unchanged results.\n\n'
1046 versions_list
= sorted(versions
.keys())
1047 versions_list
= ['%s: %s (%s)' % (k
, versions
[k
]['version'],
1048 versions
[k
]['revision'])
1049 for k
in versions_list
]
1050 versions_text
= ('Component versions for this build:\n\n%s\n' %
1051 '\n'.join(versions_list
))
1052 body_text
= results_text
+ versions_text
1053 msg
= email
.mime
.text
.MIMEText(body_text
)
1054 msg
['Subject'] = subject
1055 msg
['From'] = self
.bot_config
['email-from']
1056 msg
['To'] = self
.bot_config
['email-to']
1057 msg
['Message-ID'] = email
.utils
.make_msgid()
1058 msg
['Date'] = email
.utils
.format_datetime(datetime
.datetime
.utcnow())
1059 with smtplib
.SMTP(self
.bot_config
['email-server']) as s
:
1062 def bot_run_self(self
, opts
, action
, check
=True):
1063 """Run a copy of this script with given options."""
1064 cmd
= [sys
.executable
, sys
.argv
[0], '--keep=none',
1065 '-j%d' % self
.parallelism
]
1067 cmd
.extend([self
.topdir
, action
])
1069 subprocess
.run(cmd
, check
=check
)
1072 """Run repeated rounds of checkout and builds."""
1074 self
.load_bot_config_json()
1075 if not self
.bot_config
['run']:
1076 print('Bot exiting by request.')
1078 self
.bot_run_self([], 'bot-cycle', check
=False)
1079 self
.load_bot_config_json()
1080 if not self
.bot_config
['run']:
1081 print('Bot exiting by request.')
1083 time
.sleep(self
.bot_config
['delay'])
1084 if self
.get_script_text() != self
.script_text
:
1085 print('Script changed, bot re-execing.')
1089 class Config(object):
1090 """A configuration for building a compiler and associated libraries."""
1092 def __init__(self
, ctx
, arch
, os_name
, variant
=None, gcc_cfg
=None,
1093 first_gcc_cfg
=None, glibcs
=None, extra_glibcs
=None):
1094 """Initialize a Config object."""
1098 self
.variant
= variant
1100 self
.name
= '%s-%s' % (arch
, os_name
)
1102 self
.name
= '%s-%s-%s' % (arch
, os_name
, variant
)
1103 self
.triplet
= '%s-glibc-%s' % (arch
, os_name
)
1107 self
.gcc_cfg
= gcc_cfg
1108 if first_gcc_cfg
is None:
1109 self
.first_gcc_cfg
= []
1111 self
.first_gcc_cfg
= first_gcc_cfg
1113 glibcs
= [{'variant': variant
}]
1114 if extra_glibcs
is None:
1116 glibcs
= [Glibc(self
, **g
) for g
in glibcs
]
1117 extra_glibcs
= [Glibc(self
, **g
) for g
in extra_glibcs
]
1118 self
.all_glibcs
= glibcs
+ extra_glibcs
1119 self
.compiler_glibcs
= glibcs
1120 self
.installdir
= ctx
.compiler_installdir(self
.name
)
1121 self
.bindir
= ctx
.compiler_bindir(self
.name
)
1122 self
.sysroot
= ctx
.compiler_sysroot(self
.name
)
1123 self
.builddir
= os
.path
.join(ctx
.builddir
, 'compilers', self
.name
)
1124 self
.logsdir
= os
.path
.join(ctx
.logsdir
, 'compilers', self
.name
)
1126 def component_builddir(self
, component
):
1127 """Return the directory to use for a (non-glibc) build."""
1128 return self
.ctx
.component_builddir('compilers', self
.name
, component
)
1131 """Generate commands to build this compiler."""
1132 self
.ctx
.remove_recreate_dirs(self
.installdir
, self
.builddir
,
1134 cmdlist
= CommandList('compilers-%s' % self
.name
, self
.ctx
.keep
)
1135 cmdlist
.add_command('check-host-libraries',
1137 os
.path
.join(self
.ctx
.host_libraries_installdir
,
1139 cmdlist
.use_path(self
.bindir
)
1140 self
.build_cross_tool(cmdlist
, 'binutils', 'binutils',
1142 '--disable-libdecnumber',
1143 '--disable-readline',
1145 if self
.os
.startswith('linux'):
1146 self
.install_linux_headers(cmdlist
)
1147 self
.build_gcc(cmdlist
, True)
1148 for g
in self
.compiler_glibcs
:
1149 cmdlist
.push_subdesc('glibc')
1150 cmdlist
.push_subdesc(g
.name
)
1151 g
.build_glibc(cmdlist
, True)
1152 cmdlist
.pop_subdesc()
1153 cmdlist
.pop_subdesc()
1154 self
.build_gcc(cmdlist
, False)
1155 cmdlist
.add_command('done', ['touch',
1156 os
.path
.join(self
.installdir
, 'ok')])
1157 self
.ctx
.add_makefile_cmdlist('compilers-%s' % self
.name
, cmdlist
,
1160 def build_cross_tool(self
, cmdlist
, tool_src
, tool_build
, extra_opts
=None):
1161 """Build one cross tool."""
1162 srcdir
= self
.ctx
.component_srcdir(tool_src
)
1163 builddir
= self
.component_builddir(tool_build
)
1164 cmdlist
.push_subdesc(tool_build
)
1165 cmdlist
.create_use_dir(builddir
)
1166 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
1167 '--prefix=%s' % self
.installdir
,
1168 '--build=%s' % self
.ctx
.build_triplet
,
1169 '--host=%s' % self
.ctx
.build_triplet
,
1170 '--target=%s' % self
.triplet
,
1171 '--with-sysroot=%s' % self
.sysroot
]
1173 cfg_cmd
.extend(extra_opts
)
1174 cmdlist
.add_command('configure', cfg_cmd
)
1175 cmdlist
.add_command('build', ['make'])
1176 cmdlist
.add_command('install', ['make', 'install'])
1177 cmdlist
.cleanup_dir()
1178 cmdlist
.pop_subdesc()
1180 def install_linux_headers(self
, cmdlist
):
1181 """Install Linux kernel headers."""
1182 arch_map
= {'aarch64': 'arm64',
1192 'microblaze': 'microblaze',
1195 'powerpc': 'powerpc',
1203 if self
.arch
.startswith(k
):
1204 linux_arch
= arch_map
[k
]
1206 assert linux_arch
is not None
1207 srcdir
= self
.ctx
.component_srcdir('linux')
1208 builddir
= self
.component_builddir('linux')
1209 headers_dir
= os
.path
.join(self
.sysroot
, 'usr')
1210 cmdlist
.push_subdesc('linux')
1211 cmdlist
.create_use_dir(builddir
)
1212 cmdlist
.add_command('install-headers',
1213 ['make', '-C', srcdir
, 'O=%s' % builddir
,
1214 'ARCH=%s' % linux_arch
,
1215 'INSTALL_HDR_PATH=%s' % headers_dir
,
1217 cmdlist
.cleanup_dir()
1218 cmdlist
.pop_subdesc()
1220 def build_gcc(self
, cmdlist
, bootstrap
):
1222 # libsanitizer commonly breaks because of glibc header
1223 # changes, or on unusual targets. libssp is of little
1224 # relevance with glibc's own stack checking support.
1225 cfg_opts
= list(self
.gcc_cfg
)
1226 cfg_opts
+= ['--disable-libsanitizer', '--disable-libssp']
1227 host_libs
= self
.ctx
.host_libraries_installdir
1228 cfg_opts
+= ['--with-gmp=%s' % host_libs
,
1229 '--with-mpfr=%s' % host_libs
,
1230 '--with-mpc=%s' % host_libs
]
1232 tool_build
= 'gcc-first'
1233 # Building a static-only, C-only compiler that is
1234 # sufficient to build glibc. Various libraries and
1235 # features that may require libc headers must be disabled.
1236 # When configuring with a sysroot, --with-newlib is
1237 # required to define inhibit_libc (to stop some parts of
1238 # libgcc including libc headers); --without-headers is not
1240 cfg_opts
+= ['--enable-languages=c', '--disable-shared',
1241 '--disable-threads',
1242 '--disable-libatomic',
1243 '--disable-decimal-float',
1245 '--disable-libgomp',
1248 '--disable-libquadmath',
1249 '--without-headers', '--with-newlib',
1250 '--with-glibc-version=%s' % self
.ctx
.glibc_version
1252 cfg_opts
+= self
.first_gcc_cfg
1255 cfg_opts
+= ['--enable-languages=c,c++', '--enable-shared',
1257 self
.build_cross_tool(cmdlist
, 'gcc', tool_build
, cfg_opts
)
1260 class Glibc(object):
1261 """A configuration for building glibc."""
1263 def __init__(self
, compiler
, arch
=None, os_name
=None, variant
=None,
1264 cfg
=None, ccopts
=None):
1265 """Initialize a Glibc object."""
1266 self
.ctx
= compiler
.ctx
1267 self
.compiler
= compiler
1269 self
.arch
= compiler
.arch
1273 self
.os
= compiler
.os
1276 self
.variant
= variant
1278 self
.name
= '%s-%s' % (self
.arch
, self
.os
)
1280 self
.name
= '%s-%s-%s' % (self
.arch
, self
.os
, variant
)
1281 self
.triplet
= '%s-glibc-%s' % (self
.arch
, self
.os
)
1286 self
.ccopts
= ccopts
1288 def tool_name(self
, tool
):
1289 """Return the name of a cross-compilation tool."""
1290 ctool
= '%s-%s' % (self
.compiler
.triplet
, tool
)
1291 if self
.ccopts
and (tool
== 'gcc' or tool
== 'g++'):
1292 ctool
= '%s %s' % (ctool
, self
.ccopts
)
1296 """Generate commands to build this glibc."""
1297 builddir
= self
.ctx
.component_builddir('glibcs', self
.name
, 'glibc')
1298 installdir
= self
.ctx
.glibc_installdir(self
.name
)
1299 logsdir
= os
.path
.join(self
.ctx
.logsdir
, 'glibcs', self
.name
)
1300 self
.ctx
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
1301 cmdlist
= CommandList('glibcs-%s' % self
.name
, self
.ctx
.keep
)
1302 cmdlist
.add_command('check-compilers',
1304 os
.path
.join(self
.compiler
.installdir
, 'ok')])
1305 cmdlist
.use_path(self
.compiler
.bindir
)
1306 self
.build_glibc(cmdlist
, False)
1307 self
.ctx
.add_makefile_cmdlist('glibcs-%s' % self
.name
, cmdlist
,
1310 def build_glibc(self
, cmdlist
, for_compiler
):
1311 """Generate commands to build this glibc, either as part of a compiler
1312 build or with the bootstrapped compiler (and in the latter case, run
1314 srcdir
= self
.ctx
.component_srcdir('glibc')
1316 builddir
= self
.ctx
.component_builddir('compilers',
1317 self
.compiler
.name
, 'glibc',
1319 installdir
= self
.compiler
.sysroot
1320 srcdir_copy
= self
.ctx
.component_builddir('compilers',
1325 builddir
= self
.ctx
.component_builddir('glibcs', self
.name
,
1327 installdir
= self
.ctx
.glibc_installdir(self
.name
)
1328 srcdir_copy
= self
.ctx
.component_builddir('glibcs', self
.name
,
1330 cmdlist
.create_use_dir(builddir
)
1331 # glibc builds write into the source directory, and even if
1332 # not intentionally there is a risk of bugs that involve
1333 # writing into the working directory. To avoid possible
1334 # concurrency issues, copy the source directory.
1335 cmdlist
.create_copy_dir(srcdir
, srcdir_copy
)
1336 cfg_cmd
= [os
.path
.join(srcdir_copy
, 'configure'),
1339 '--build=%s' % self
.ctx
.build_triplet
,
1340 '--host=%s' % self
.triplet
,
1341 'CC=%s' % self
.tool_name('gcc'),
1342 'CXX=%s' % self
.tool_name('g++'),
1343 'AR=%s' % self
.tool_name('ar'),
1344 'AS=%s' % self
.tool_name('as'),
1345 'LD=%s' % self
.tool_name('ld'),
1346 'NM=%s' % self
.tool_name('nm'),
1347 'OBJCOPY=%s' % self
.tool_name('objcopy'),
1348 'OBJDUMP=%s' % self
.tool_name('objdump'),
1349 'RANLIB=%s' % self
.tool_name('ranlib'),
1350 'READELF=%s' % self
.tool_name('readelf'),
1351 'STRIP=%s' % self
.tool_name('strip')]
1353 cmdlist
.add_command('configure', cfg_cmd
)
1354 cmdlist
.add_command('build', ['make'])
1355 cmdlist
.add_command('install', ['make', 'install',
1356 'install_root=%s' % installdir
])
1357 # GCC uses paths such as lib/../lib64, so make sure lib
1358 # directories always exist.
1359 cmdlist
.add_command('mkdir-lib', ['mkdir', '-p',
1360 os
.path
.join(installdir
, 'lib'),
1361 os
.path
.join(installdir
,
1363 if not for_compiler
:
1365 cmdlist
.add_command('strip',
1367 ('%s %s/lib*/*.so' %
1368 (self
.tool_name('strip'), installdir
))])
1369 cmdlist
.add_command('check', ['make', 'check'])
1370 cmdlist
.add_command('save-logs', [self
.ctx
.save_logs
],
1372 cmdlist
.cleanup_dir('cleanup-src', srcdir_copy
)
1373 cmdlist
.cleanup_dir()
1376 class Command(object):
1377 """A command run in the build process."""
1379 def __init__(self
, desc
, num
, dir, path
, command
, always_run
=False):
1380 """Initialize a Command object."""
1384 trans
= str.maketrans({' ': '-'})
1385 self
.logbase
= '%03d-%s' % (num
, desc
.translate(trans
))
1386 self
.command
= command
1387 self
.always_run
= always_run
1390 def shell_make_quote_string(s
):
1391 """Given a string not containing a newline, quote it for use by the
1393 assert '\n' not in s
1394 if re
.fullmatch('[]+,./0-9@A-Z_a-z-]+', s
):
1396 strans
= str.maketrans({"'": "'\\''"})
1397 s
= "'%s'" % s
.translate(strans
)
1398 mtrans
= str.maketrans({'$': '$$'})
1399 return s
.translate(mtrans
)
1402 def shell_make_quote_list(l
, translate_make
):
1403 """Given a list of strings not containing newlines, quote them for use
1404 by the shell and make, returning a single string. If translate_make
1405 is true and the first string is 'make', change it to $(MAKE)."""
1406 l
= [Command
.shell_make_quote_string(s
) for s
in l
]
1407 if translate_make
and l
[0] == 'make':
1411 def shell_make_quote(self
):
1412 """Return this command quoted for the shell and make."""
1413 return self
.shell_make_quote_list(self
.command
, True)
1416 class CommandList(object):
1417 """A list of commands run in the build process."""
1419 def __init__(self
, desc
, keep
):
1420 """Initialize a CommandList object."""
1427 def desc_txt(self
, desc
):
1428 """Return the description to use for a command."""
1429 return '%s %s' % (' '.join(self
.desc
), desc
)
1431 def use_dir(self
, dir):
1432 """Set the default directory for subsequent commands."""
1435 def use_path(self
, path
):
1436 """Set a directory to be prepended to the PATH for subsequent
1440 def push_subdesc(self
, subdesc
):
1441 """Set the default subdescription for subsequent commands (e.g., the
1442 name of a component being built, within the series of commands
1444 self
.desc
.append(subdesc
)
1446 def pop_subdesc(self
):
1447 """Pop a subdescription from the list of descriptions."""
1450 def create_use_dir(self
, dir):
1451 """Remove and recreate a directory and use it for subsequent
1453 self
.add_command_dir('rm', None, ['rm', '-rf', dir])
1454 self
.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1457 def create_copy_dir(self
, src
, dest
):
1458 """Remove a directory and recreate it as a copy from the given
1460 self
.add_command_dir('copy-rm', None, ['rm', '-rf', dest
])
1461 parent
= os
.path
.dirname(dest
)
1462 self
.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent
])
1463 self
.add_command_dir('copy', None, ['cp', '-a', src
, dest
])
1465 def add_command_dir(self
, desc
, dir, command
, always_run
=False):
1466 """Add a command to run in a given directory."""
1467 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), dir, self
.path
,
1468 command
, always_run
)
1469 self
.cmdlist
.append(cmd
)
1471 def add_command(self
, desc
, command
, always_run
=False):
1472 """Add a command to run in the default directory."""
1473 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), self
.dir,
1474 self
.path
, command
, always_run
)
1475 self
.cmdlist
.append(cmd
)
1477 def cleanup_dir(self
, desc
='cleanup', dir=None):
1478 """Clean up a build directory. If no directory is specified, the
1479 default directory is cleaned up and ceases to be the default
1484 if self
.keep
!= 'all':
1485 self
.add_command_dir(desc
, None, ['rm', '-rf', dir],
1486 always_run
=(self
.keep
== 'none'))
1488 def makefile_commands(self
, wrapper
, logsdir
):
1489 """Return the sequence of commands in the form of text for a Makefile.
1490 The given wrapper script takes arguments: base of logs for
1491 previous command, or empty; base of logs for this command;
1492 description; directory; PATH addition; the command itself."""
1493 # prev_base is the base of the name for logs of the previous
1494 # command that is not always-run (that is, a build command,
1495 # whose failure should stop subsequent build commands from
1496 # being run, as opposed to a cleanup command, which is run
1497 # even if previous commands failed).
1500 for c
in self
.cmdlist
:
1501 ctxt
= c
.shell_make_quote()
1502 if prev_base
and not c
.always_run
:
1503 prev_log
= os
.path
.join(logsdir
, prev_base
)
1506 this_log
= os
.path
.join(logsdir
, c
.logbase
)
1507 if not c
.always_run
:
1508 prev_base
= c
.logbase
1517 prelims
= [wrapper
, prev_log
, this_log
, c
.desc
, dir, path
]
1518 prelim_txt
= Command
.shell_make_quote_list(prelims
, False)
1519 cmds
.append('\t@%s %s' % (prelim_txt
, ctxt
))
1520 return '\n'.join(cmds
)
1522 def status_logs(self
, logsdir
):
1523 """Return the list of log files with command status."""
1524 return [os
.path
.join(logsdir
, '%s-status.txt' % c
.logbase
)
1525 for c
in self
.cmdlist
]
1529 """Return an argument parser for this module."""
1530 parser
= argparse
.ArgumentParser(description
=__doc__
)
1531 parser
.add_argument('-j', dest
='parallelism',
1532 help='Run this number of jobs in parallel',
1533 type=int, default
=os
.cpu_count())
1534 parser
.add_argument('--keep', dest
='keep',
1535 help='Whether to keep all build directories, '
1536 'none or only those from failed builds',
1537 default
='none', choices
=('none', 'all', 'failed'))
1538 parser
.add_argument('--replace-sources', action
='store_true',
1539 help='Remove and replace source directories '
1540 'with the wrong version of a component')
1541 parser
.add_argument('--strip', action
='store_true',
1542 help='Strip installed glibc libraries')
1543 parser
.add_argument('topdir',
1544 help='Toplevel working directory')
1545 parser
.add_argument('action',
1547 choices
=('checkout', 'bot-cycle', 'bot',
1548 'host-libraries', 'compilers', 'glibcs'))
1549 parser
.add_argument('configs',
1550 help='Versions to check out or configurations to build',
1556 """The main entry point."""
1557 parser
= get_parser()
1558 opts
= parser
.parse_args(argv
)
1559 topdir
= os
.path
.abspath(opts
.topdir
)
1560 ctx
= Context(topdir
, opts
.parallelism
, opts
.keep
, opts
.replace_sources
,
1561 opts
.strip
, opts
.action
)
1562 ctx
.run_builds(opts
.action
, opts
.configs
)
1565 if __name__
== '__main__':