2 # Build many configurations of glibc.
3 # Copyright (C) 2016 Free Software Foundation, Inc.
4 # This file is part of the GNU C Library.
6 # The GNU C Library is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU Lesser General Public
8 # License as published by the Free Software Foundation; either
9 # version 2.1 of the License, or (at your option) any later version.
11 # The GNU C Library is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # Lesser General Public License for more details.
16 # You should have received a copy of the GNU Lesser General Public
17 # License along with the GNU C Library; if not, see
18 # <http://www.gnu.org/licenses/>.
20 """Build many configurations of glibc.
22 This script takes as arguments a directory name (containing a src
23 subdirectory with sources of the relevant toolchain components) and a
24 description of what to do: 'checkout', to check out sources into that
25 directory, 'bot-cycle', to run a series of checkout and build steps,
26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
27 libraries required by the toolchain, 'compilers', to build
28 cross-compilers for various configurations, or 'glibcs', to build
29 glibc for various configurations and run the compilation parts of the
30 testsuite. Subsequent arguments name the versions of components to
31 check out (<component>-<version), for 'checkout', or, for actions
32 other than 'checkout' and 'bot-cycle', name configurations for which
33 compilers or glibc are to be built.
39 import email
.mime
.text
53 class Context(object):
54 """The global state associated with builds in a given directory."""
56 def __init__(self
, topdir
, parallelism
, keep
, replace_sources
, strip
,
58 """Initialize the context."""
60 self
.parallelism
= parallelism
62 self
.replace_sources
= replace_sources
64 self
.srcdir
= os
.path
.join(topdir
, 'src')
65 self
.versions_json
= os
.path
.join(self
.srcdir
, 'versions.json')
66 self
.build_state_json
= os
.path
.join(topdir
, 'build-state.json')
67 self
.bot_config_json
= os
.path
.join(topdir
, 'bot-config.json')
68 self
.installdir
= os
.path
.join(topdir
, 'install')
69 self
.host_libraries_installdir
= os
.path
.join(self
.installdir
,
71 self
.builddir
= os
.path
.join(topdir
, 'build')
72 self
.logsdir
= os
.path
.join(topdir
, 'logs')
73 self
.logsdir_old
= os
.path
.join(topdir
, 'logs-old')
74 self
.makefile
= os
.path
.join(self
.builddir
, 'Makefile')
75 self
.wrapper
= os
.path
.join(self
.builddir
, 'wrapper')
76 self
.save_logs
= os
.path
.join(self
.builddir
, 'save-logs')
77 self
.script_text
= self
.get_script_text()
78 if action
!= 'checkout':
79 self
.build_triplet
= self
.get_build_triplet()
80 self
.glibc_version
= self
.get_glibc_version()
82 self
.glibc_configs
= {}
83 self
.makefile_pieces
= ['.PHONY: all\n']
84 self
.add_all_configs()
85 self
.load_versions_json()
86 self
.load_build_state_json()
87 self
.status_log_list
= []
89 def get_script_text(self
):
90 """Return the text of this script."""
91 with
open(sys
.argv
[0], 'r') as f
:
95 """Re-execute this script with the same arguments."""
96 os
.execv(sys
.executable
, [sys
.executable
] + sys
.argv
)
98 def get_build_triplet(self
):
99 """Determine the build triplet with config.guess."""
100 config_guess
= os
.path
.join(self
.component_srcdir('gcc'),
102 cg_out
= subprocess
.run([config_guess
], stdout
=subprocess
.PIPE
,
103 check
=True, universal_newlines
=True).stdout
104 return cg_out
.rstrip()
106 def get_glibc_version(self
):
107 """Determine the glibc version number (major.minor)."""
108 version_h
= os
.path
.join(self
.component_srcdir('glibc'), 'version.h')
109 with
open(version_h
, 'r') as f
:
110 lines
= f
.readlines()
111 starttext
= '#define VERSION "'
113 if l
.startswith(starttext
):
114 l
= l
[len(starttext
):]
116 m
= re
.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l
)
117 return '%s.%s' % m
.group(1, 2)
118 print('error: could not determine glibc version')
121 def add_all_configs(self
):
122 """Add all known glibc build configurations."""
123 # On architectures missing __builtin_trap support, these
124 # options may be needed as a workaround; see
125 # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70216> for SH.
126 no_isolate
= ('-fno-isolate-erroneous-paths-dereference'
127 ' -fno-isolate-erroneous-paths-attribute')
128 self
.add_config(arch
='aarch64',
130 self
.add_config(arch
='aarch64_be',
132 self
.add_config(arch
='alpha',
134 self
.add_config(arch
='arm',
135 os_name
='linux-gnueabi')
136 self
.add_config(arch
='armeb',
137 os_name
='linux-gnueabi')
138 self
.add_config(arch
='armeb',
139 os_name
='linux-gnueabi',
141 gcc_cfg
=['--with-arch=armv7-a'])
142 self
.add_config(arch
='arm',
143 os_name
='linux-gnueabihf')
144 self
.add_config(arch
='armeb',
145 os_name
='linux-gnueabihf')
146 self
.add_config(arch
='armeb',
147 os_name
='linux-gnueabihf',
149 gcc_cfg
=['--with-arch=armv7-a'])
150 self
.add_config(arch
='hppa',
152 self
.add_config(arch
='ia64',
154 first_gcc_cfg
=['--with-system-libunwind'])
155 self
.add_config(arch
='m68k',
157 gcc_cfg
=['--disable-multilib'])
158 self
.add_config(arch
='m68k',
161 gcc_cfg
=['--with-arch=cf', '--disable-multilib'])
162 self
.add_config(arch
='microblaze',
164 gcc_cfg
=['--disable-multilib'])
165 self
.add_config(arch
='microblazeel',
167 gcc_cfg
=['--disable-multilib'])
168 self
.add_config(arch
='mips64',
170 gcc_cfg
=['--with-mips-plt'],
171 glibcs
=[{'variant': 'n32'},
173 'ccopts': '-mabi=32'},
175 'ccopts': '-mabi=64'}])
176 self
.add_config(arch
='mips64',
179 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
180 glibcs
=[{'variant': 'n32-soft',
181 'cfg': ['--without-fp']},
184 'ccopts': '-mabi=32',
185 'cfg': ['--without-fp']},
186 {'variant': 'n64-soft',
187 'ccopts': '-mabi=64',
188 'cfg': ['--without-fp']}])
189 self
.add_config(arch
='mips64',
192 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
193 '--with-arch-64=mips64r2',
194 '--with-arch-32=mips32r2'],
195 glibcs
=[{'variant': 'n32-nan2008'},
196 {'variant': 'nan2008',
198 'ccopts': '-mabi=32'},
199 {'variant': 'n64-nan2008',
200 'ccopts': '-mabi=64'}])
201 self
.add_config(arch
='mips64',
203 variant
='nan2008-soft',
204 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
205 '--with-arch-64=mips64r2',
206 '--with-arch-32=mips32r2',
207 '--with-float=soft'],
208 glibcs
=[{'variant': 'n32-nan2008-soft',
209 'cfg': ['--without-fp']},
210 {'variant': 'nan2008-soft',
212 'ccopts': '-mabi=32',
213 'cfg': ['--without-fp']},
214 {'variant': 'n64-nan2008-soft',
215 'ccopts': '-mabi=64',
216 'cfg': ['--without-fp']}])
217 self
.add_config(arch
='mips64el',
219 gcc_cfg
=['--with-mips-plt'],
220 glibcs
=[{'variant': 'n32'},
222 'ccopts': '-mabi=32'},
224 'ccopts': '-mabi=64'}])
225 self
.add_config(arch
='mips64el',
228 gcc_cfg
=['--with-mips-plt', '--with-float=soft'],
229 glibcs
=[{'variant': 'n32-soft',
230 'cfg': ['--without-fp']},
233 'ccopts': '-mabi=32',
234 'cfg': ['--without-fp']},
235 {'variant': 'n64-soft',
236 'ccopts': '-mabi=64',
237 'cfg': ['--without-fp']}])
238 self
.add_config(arch
='mips64el',
241 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
242 '--with-arch-64=mips64r2',
243 '--with-arch-32=mips32r2'],
244 glibcs
=[{'variant': 'n32-nan2008'},
245 {'variant': 'nan2008',
247 'ccopts': '-mabi=32'},
248 {'variant': 'n64-nan2008',
249 'ccopts': '-mabi=64'}])
250 self
.add_config(arch
='mips64el',
252 variant
='nan2008-soft',
253 gcc_cfg
=['--with-mips-plt', '--with-nan=2008',
254 '--with-arch-64=mips64r2',
255 '--with-arch-32=mips32r2',
256 '--with-float=soft'],
257 glibcs
=[{'variant': 'n32-nan2008-soft',
258 'cfg': ['--without-fp']},
259 {'variant': 'nan2008-soft',
261 'ccopts': '-mabi=32',
262 'cfg': ['--without-fp']},
263 {'variant': 'n64-nan2008-soft',
264 'ccopts': '-mabi=64',
265 'cfg': ['--without-fp']}])
266 self
.add_config(arch
='nios2',
268 self
.add_config(arch
='powerpc',
270 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
271 self
.add_config(arch
='powerpc',
274 gcc_cfg
=['--disable-multilib', '--with-float=soft',
275 '--enable-secureplt'],
276 glibcs
=[{'variant': 'soft', 'cfg': ['--without-fp']}])
277 self
.add_config(arch
='powerpc64',
279 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
280 self
.add_config(arch
='powerpc64le',
282 gcc_cfg
=['--disable-multilib', '--enable-secureplt'])
283 self
.add_config(arch
='powerpc',
284 os_name
='linux-gnuspe',
285 gcc_cfg
=['--disable-multilib', '--enable-secureplt',
286 '--enable-e500-double'],
287 glibcs
=[{'cfg': ['--without-fp']}])
288 self
.add_config(arch
='powerpc',
289 os_name
='linux-gnuspe',
291 gcc_cfg
=['--disable-multilib', '--enable-secureplt'],
292 glibcs
=[{'variant': 'e500v1', 'cfg': ['--without-fp']}])
293 self
.add_config(arch
='s390x',
296 {'arch': 's390', 'ccopts': '-m31'}])
297 self
.add_config(arch
='sh3',
299 glibcs
=[{'ccopts': no_isolate
}])
300 self
.add_config(arch
='sh3eb',
302 glibcs
=[{'ccopts': no_isolate
}])
303 self
.add_config(arch
='sh4',
305 glibcs
=[{'ccopts': no_isolate
}])
306 self
.add_config(arch
='sh4eb',
308 glibcs
=[{'ccopts': no_isolate
}])
309 self
.add_config(arch
='sh4',
312 gcc_cfg
=['--without-fp'],
313 glibcs
=[{'variant': 'soft',
314 'cfg': ['--without-fp'],
315 'ccopts': no_isolate
}])
316 self
.add_config(arch
='sh4eb',
319 gcc_cfg
=['--without-fp'],
320 glibcs
=[{'variant': 'soft',
321 'cfg': ['--without-fp'],
322 'ccopts': no_isolate
}])
323 self
.add_config(arch
='sparc64',
327 'ccopts': '-m32 -mlong-double-128'}])
328 self
.add_config(arch
='tilegx',
331 {'variant': '32', 'ccopts': '-m32'}])
332 self
.add_config(arch
='tilegxbe',
335 {'variant': '32', 'ccopts': '-m32'}])
336 self
.add_config(arch
='tilepro',
338 self
.add_config(arch
='x86_64',
340 gcc_cfg
=['--with-multilib-list=m64,m32,mx32'],
342 {'variant': 'x32', 'ccopts': '-mx32'},
343 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
344 extra_glibcs
=[{'variant': 'disable-multi-arch',
345 'cfg': ['--disable-multi-arch']},
346 {'variant': 'disable-multi-arch',
348 'ccopts': '-m32 -march=i686',
349 'cfg': ['--disable-multi-arch']},
351 'ccopts': '-m32 -march=i486'},
353 'ccopts': '-m32 -march=i586'}])
355 def add_config(self
, **args
):
356 """Add an individual build configuration."""
357 cfg
= Config(self
, **args
)
358 if cfg
.name
in self
.configs
:
359 print('error: duplicate config %s' % cfg
.name
)
361 self
.configs
[cfg
.name
] = cfg
362 for c
in cfg
.all_glibcs
:
363 if c
.name
in self
.glibc_configs
:
364 print('error: duplicate glibc config %s' % c
.name
)
366 self
.glibc_configs
[c
.name
] = c
368 def component_srcdir(self
, component
):
369 """Return the source directory for a given component, e.g. gcc."""
370 return os
.path
.join(self
.srcdir
, component
)
372 def component_builddir(self
, action
, config
, component
, subconfig
=None):
373 """Return the directory to use for a build."""
376 assert subconfig
is None
377 return os
.path
.join(self
.builddir
, action
, component
)
378 if subconfig
is None:
379 return os
.path
.join(self
.builddir
, action
, config
, component
)
381 # glibc build as part of compiler build.
382 return os
.path
.join(self
.builddir
, action
, config
, component
,
385 def compiler_installdir(self
, config
):
386 """Return the directory in which to install a compiler."""
387 return os
.path
.join(self
.installdir
, 'compilers', config
)
389 def compiler_bindir(self
, config
):
390 """Return the directory in which to find compiler binaries."""
391 return os
.path
.join(self
.compiler_installdir(config
), 'bin')
393 def compiler_sysroot(self
, config
):
394 """Return the sysroot directory for a compiler."""
395 return os
.path
.join(self
.compiler_installdir(config
), 'sysroot')
397 def glibc_installdir(self
, config
):
398 """Return the directory in which to install glibc."""
399 return os
.path
.join(self
.installdir
, 'glibcs', config
)
401 def run_builds(self
, action
, configs
):
402 """Run the requested builds."""
403 if action
== 'checkout':
404 self
.checkout(configs
)
406 if action
== 'bot-cycle':
408 print('error: configurations specified for bot-cycle')
414 print('error: configurations specified for bot')
418 if action
== 'host-libraries' and configs
:
419 print('error: configurations specified for host-libraries')
421 self
.clear_last_build_state(action
)
422 build_time
= datetime
.datetime
.utcnow()
423 if action
== 'host-libraries':
424 build_components
= ('gmp', 'mpfr', 'mpc')
427 self
.build_host_libraries()
428 elif action
== 'compilers':
429 build_components
= ('binutils', 'gcc', 'glibc', 'linux')
430 old_components
= ('gmp', 'mpfr', 'mpc')
431 old_versions
= self
.build_state
['host-libraries']['build-versions']
432 self
.build_compilers(configs
)
434 build_components
= ('glibc',)
435 old_components
= ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
436 old_versions
= self
.build_state
['compilers']['build-versions']
437 self
.build_glibcs(configs
)
441 # Partial build, do not update stored state.
444 for k
in build_components
:
445 if k
in self
.versions
:
446 build_versions
[k
] = {'version': self
.versions
[k
]['version'],
447 'revision': self
.versions
[k
]['revision']}
448 for k
in old_components
:
449 if k
in old_versions
:
450 build_versions
[k
] = {'version': old_versions
[k
]['version'],
451 'revision': old_versions
[k
]['revision']}
452 self
.update_build_state(action
, build_time
, build_versions
)
455 def remove_dirs(*args
):
456 """Remove directories and their contents if they exist."""
458 shutil
.rmtree(dir, ignore_errors
=True)
461 def remove_recreate_dirs(*args
):
462 """Remove directories if they exist, and create them as empty."""
463 Context
.remove_dirs(*args
)
465 os
.makedirs(dir, exist_ok
=True)
467 def add_makefile_cmdlist(self
, target
, cmdlist
, logsdir
):
468 """Add makefile text for a list of commands."""
469 commands
= cmdlist
.makefile_commands(self
.wrapper
, logsdir
)
470 self
.makefile_pieces
.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
471 (target
, target
, target
, commands
))
472 self
.status_log_list
.extend(cmdlist
.status_logs(logsdir
))
474 def write_files(self
):
475 """Write out the Makefile and wrapper script."""
476 mftext
= ''.join(self
.makefile_pieces
)
477 with
open(self
.makefile
, 'w') as f
:
487 'prev_status=$prev_base-status.txt\n'
488 'this_status=$this_base-status.txt\n'
489 'this_log=$this_base-log.txt\n'
490 'date > "$this_log"\n'
491 'echo >> "$this_log"\n'
492 'echo "Description: $desc" >> "$this_log"\n'
493 'printf "%s" "Command:" >> "$this_log"\n'
494 'for word in "$@"; do\n'
495 ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
496 ' printf " %s" "$word"\n'
499 ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
502 'done >> "$this_log"\n'
503 'echo >> "$this_log"\n'
504 'echo "Directory: $dir" >> "$this_log"\n'
505 'echo "Path addition: $path" >> "$this_log"\n'
506 'echo >> "$this_log"\n'
509 ' echo >> "$this_log"\n'
510 ' echo "$1: $desc" > "$this_status"\n'
511 ' echo "$1: $desc" >> "$this_log"\n'
512 ' echo >> "$this_log"\n'
513 ' date >> "$this_log"\n'
514 ' echo "$1: $desc"\n'
519 ' if [ "$1" != "0" ]; then\n'
520 ' record_status FAIL\n'
523 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
524 ' record_status UNRESOLVED\n'
526 'if [ "$dir" ]; then\n'
528 ' check_error "$?"\n'
530 'if [ "$path" ]; then\n'
531 ' PATH=$path:$PATH\n'
533 '"$@" < /dev/null >> "$this_log" 2>&1\n'
535 'record_status PASS\n')
536 with
open(self
.wrapper
, 'w') as f
:
537 f
.write(wrapper_text
)
539 mode_exec
= (stat
.S_IRWXU|stat
.S_IRGRP|stat
.S_IXGRP|
540 stat
.S_IROTH|stat
.S_IXOTH
)
541 os
.chmod(self
.wrapper
, mode_exec
)
544 'if ! [ -f tests.sum ]; then\n'
545 ' echo "No test summary available."\n'
550 ' echo "Contents of $1:"\n'
554 ' echo "End of contents of $1."\n'
557 'save_file tests.sum\n'
558 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
559 'for t in $non_pass_tests; do\n'
560 ' if [ -f "$t.out" ]; then\n'
561 ' save_file "$t.out"\n'
564 with
open(self
.save_logs
, 'w') as f
:
565 f
.write(save_logs_text
)
566 os
.chmod(self
.save_logs
, mode_exec
)
569 """Do the actual build."""
570 cmd
= ['make', '-j%d' % self
.parallelism
]
571 subprocess
.run(cmd
, cwd
=self
.builddir
, check
=True)
573 def build_host_libraries(self
):
574 """Build the host libraries."""
575 installdir
= self
.host_libraries_installdir
576 builddir
= os
.path
.join(self
.builddir
, 'host-libraries')
577 logsdir
= os
.path
.join(self
.logsdir
, 'host-libraries')
578 self
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
579 cmdlist
= CommandList('host-libraries', self
.keep
)
580 self
.build_host_library(cmdlist
, 'gmp')
581 self
.build_host_library(cmdlist
, 'mpfr',
582 ['--with-gmp=%s' % installdir
])
583 self
.build_host_library(cmdlist
, 'mpc',
584 ['--with-gmp=%s' % installdir
,
585 '--with-mpfr=%s' % installdir
])
586 cmdlist
.add_command('done', ['touch', os
.path
.join(installdir
, 'ok')])
587 self
.add_makefile_cmdlist('host-libraries', cmdlist
, logsdir
)
589 def build_host_library(self
, cmdlist
, lib
, extra_opts
=None):
590 """Build one host library."""
591 srcdir
= self
.component_srcdir(lib
)
592 builddir
= self
.component_builddir('host-libraries', None, lib
)
593 installdir
= self
.host_libraries_installdir
594 cmdlist
.push_subdesc(lib
)
595 cmdlist
.create_use_dir(builddir
)
596 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
597 '--prefix=%s' % installdir
,
600 cfg_cmd
.extend (extra_opts
)
601 cmdlist
.add_command('configure', cfg_cmd
)
602 cmdlist
.add_command('build', ['make'])
603 cmdlist
.add_command('check', ['make', 'check'])
604 cmdlist
.add_command('install', ['make', 'install'])
605 cmdlist
.cleanup_dir()
606 cmdlist
.pop_subdesc()
608 def build_compilers(self
, configs
):
609 """Build the compilers."""
611 self
.remove_dirs(os
.path
.join(self
.builddir
, 'compilers'))
612 self
.remove_dirs(os
.path
.join(self
.installdir
, 'compilers'))
613 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'compilers'))
614 configs
= sorted(self
.configs
.keys())
616 self
.configs
[c
].build()
618 def build_glibcs(self
, configs
):
619 """Build the glibcs."""
621 self
.remove_dirs(os
.path
.join(self
.builddir
, 'glibcs'))
622 self
.remove_dirs(os
.path
.join(self
.installdir
, 'glibcs'))
623 self
.remove_dirs(os
.path
.join(self
.logsdir
, 'glibcs'))
624 configs
= sorted(self
.glibc_configs
.keys())
626 self
.glibc_configs
[c
].build()
628 def load_versions_json(self
):
629 """Load information about source directory versions."""
630 if not os
.access(self
.versions_json
, os
.F_OK
):
633 with
open(self
.versions_json
, 'r') as f
:
634 self
.versions
= json
.load(f
)
636 def store_json(self
, data
, filename
):
637 """Store information in a JSON file."""
638 filename_tmp
= filename
+ '.tmp'
639 with
open(filename_tmp
, 'w') as f
:
640 json
.dump(data
, f
, indent
=2, sort_keys
=True)
641 os
.rename(filename_tmp
, filename
)
643 def store_versions_json(self
):
644 """Store information about source directory versions."""
645 self
.store_json(self
.versions
, self
.versions_json
)
647 def set_component_version(self
, component
, version
, explicit
, revision
):
648 """Set the version information for a component."""
649 self
.versions
[component
] = {'version': version
,
650 'explicit': explicit
,
651 'revision': revision
}
652 self
.store_versions_json()
654 def checkout(self
, versions
):
655 """Check out the desired component versions."""
656 default_versions
= {'binutils': 'vcs-2.27',
658 'glibc': 'vcs-mainline',
664 explicit_versions
= {}
667 for k
in default_versions
.keys():
671 if k
in use_versions
:
672 print('error: multiple versions for %s' % k
)
675 explicit_versions
[k
] = True
679 print('error: unknown component in %s' % v
)
681 for k
in default_versions
.keys():
682 if k
not in use_versions
:
683 if k
in self
.versions
and self
.versions
[k
]['explicit']:
684 use_versions
[k
] = self
.versions
[k
]['version']
685 explicit_versions
[k
] = True
687 use_versions
[k
] = default_versions
[k
]
688 explicit_versions
[k
] = False
689 os
.makedirs(self
.srcdir
, exist_ok
=True)
690 for k
in sorted(default_versions
.keys()):
691 update
= os
.access(self
.component_srcdir(k
), os
.F_OK
)
694 k
in self
.versions
and
695 v
!= self
.versions
[k
]['version']):
696 if not self
.replace_sources
:
697 print('error: version of %s has changed from %s to %s, '
698 'use --replace-sources to check out again' %
699 (k
, self
.versions
[k
]['version'], v
))
701 shutil
.rmtree(self
.component_srcdir(k
))
703 if v
.startswith('vcs-'):
704 revision
= self
.checkout_vcs(k
, v
[4:], update
)
706 self
.checkout_tar(k
, v
, update
)
708 self
.set_component_version(k
, v
, explicit_versions
[k
], revision
)
709 if self
.get_script_text() != self
.script_text
:
710 # Rerun the checkout process in case the updated script
711 # uses different default versions or new components.
714 def checkout_vcs(self
, component
, version
, update
):
715 """Check out the given version of the given component from version
716 control. Return a revision identifier."""
717 if component
== 'binutils':
718 git_url
= 'git://sourceware.org/git/binutils-gdb.git'
719 if version
== 'mainline':
720 git_branch
= 'master'
722 trans
= str.maketrans({'.': '_'})
723 git_branch
= 'binutils-%s-branch' % version
.translate(trans
)
724 return self
.git_checkout(component
, git_url
, git_branch
, update
)
725 elif component
== 'gcc':
726 if version
== 'mainline':
729 trans
= str.maketrans({'.': '_'})
730 branch
= 'branches/gcc-%s-branch' % version
.translate(trans
)
731 svn_url
= 'svn://gcc.gnu.org/svn/gcc/%s' % branch
732 return self
.gcc_checkout(svn_url
, update
)
733 elif component
== 'glibc':
734 git_url
= 'git://sourceware.org/git/glibc.git'
735 if version
== 'mainline':
736 git_branch
= 'master'
738 git_branch
= 'release/%s/master' % version
739 r
= self
.git_checkout(component
, git_url
, git_branch
, update
)
740 self
.fix_glibc_timestamps()
743 print('error: component %s coming from VCS' % component
)
746 def git_checkout(self
, component
, git_url
, git_branch
, update
):
747 """Check out a component from git. Return a commit identifier."""
749 subprocess
.run(['git', 'remote', 'prune', 'origin'],
750 cwd
=self
.component_srcdir(component
), check
=True)
751 subprocess
.run(['git', 'pull', '-q'],
752 cwd
=self
.component_srcdir(component
), check
=True)
754 subprocess
.run(['git', 'clone', '-q', '-b', git_branch
, git_url
,
755 self
.component_srcdir(component
)], check
=True)
756 r
= subprocess
.run(['git', 'rev-parse', 'HEAD'],
757 cwd
=self
.component_srcdir(component
),
758 stdout
=subprocess
.PIPE
,
759 check
=True, universal_newlines
=True).stdout
762 def fix_glibc_timestamps(self
):
763 """Fix timestamps in a glibc checkout."""
764 # Ensure that builds do not try to regenerate generated files
765 # in the source tree.
766 srcdir
= self
.component_srcdir('glibc')
767 for dirpath
, dirnames
, filenames
in os
.walk(srcdir
):
769 if (f
== 'configure' or
770 f
== 'preconfigure' or
771 f
.endswith('-kw.h')):
772 to_touch
= os
.path
.join(dirpath
, f
)
773 subprocess
.run(['touch', to_touch
], check
=True)
775 def gcc_checkout(self
, svn_url
, update
):
776 """Check out GCC from SVN. Return the revision number."""
778 subprocess
.run(['svn', 'co', '-q', svn_url
,
779 self
.component_srcdir('gcc')], check
=True)
780 subprocess
.run(['contrib/gcc_update', '--silent'],
781 cwd
=self
.component_srcdir('gcc'), check
=True)
782 r
= subprocess
.run(['svnversion', self
.component_srcdir('gcc')],
783 stdout
=subprocess
.PIPE
,
784 check
=True, universal_newlines
=True).stdout
787 def checkout_tar(self
, component
, version
, update
):
788 """Check out the given version of the given component from a
792 url_map
= {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
793 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
794 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
795 'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
796 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
797 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
798 if component
not in url_map
:
799 print('error: component %s coming from tarball' % component
)
801 url
= url_map
[component
] % {'version': version
}
802 filename
= os
.path
.join(self
.srcdir
, url
.split('/')[-1])
803 response
= urllib
.request
.urlopen(url
)
804 data
= response
.read()
805 with
open(filename
, 'wb') as f
:
807 subprocess
.run(['tar', '-C', self
.srcdir
, '-x', '-f', filename
],
809 os
.rename(os
.path
.join(self
.srcdir
, '%s-%s' % (component
, version
)),
810 self
.component_srcdir(component
))
813 def load_build_state_json(self
):
814 """Load information about the state of previous builds."""
815 if os
.access(self
.build_state_json
, os
.F_OK
):
816 with
open(self
.build_state_json
, 'r') as f
:
817 self
.build_state
= json
.load(f
)
819 self
.build_state
= {}
820 for k
in ('host-libraries', 'compilers', 'glibcs'):
821 if k
not in self
.build_state
:
822 self
.build_state
[k
] = {}
823 if 'build-time' not in self
.build_state
[k
]:
824 self
.build_state
[k
]['build-time'] = ''
825 if 'build-versions' not in self
.build_state
[k
]:
826 self
.build_state
[k
]['build-versions'] = {}
827 if 'build-results' not in self
.build_state
[k
]:
828 self
.build_state
[k
]['build-results'] = {}
829 if 'result-changes' not in self
.build_state
[k
]:
830 self
.build_state
[k
]['result-changes'] = {}
831 if 'ever-passed' not in self
.build_state
[k
]:
832 self
.build_state
[k
]['ever-passed'] = []
834 def store_build_state_json(self
):
835 """Store information about the state of previous builds."""
836 self
.store_json(self
.build_state
, self
.build_state_json
)
838 def clear_last_build_state(self
, action
):
839 """Clear information about the state of part of the build."""
840 # We clear the last build time and versions when starting a
841 # new build. The results of the last build are kept around,
842 # as comparison is still meaningful if this build is aborted
843 # and a new one started.
844 self
.build_state
[action
]['build-time'] = ''
845 self
.build_state
[action
]['build-versions'] = {}
846 self
.store_build_state_json()
848 def update_build_state(self
, action
, build_time
, build_versions
):
849 """Update the build state after a build."""
850 build_time
= build_time
.replace(microsecond
=0)
851 self
.build_state
[action
]['build-time'] = str(build_time
)
852 self
.build_state
[action
]['build-versions'] = build_versions
854 for log
in self
.status_log_list
:
855 with
open(log
, 'r') as f
:
857 log_text
= log_text
.rstrip()
858 m
= re
.fullmatch('([A-Z]+): (.*)', log_text
)
860 test_name
= m
.group(2)
861 assert test_name
not in build_results
862 build_results
[test_name
] = result
863 old_build_results
= self
.build_state
[action
]['build-results']
864 self
.build_state
[action
]['build-results'] = build_results
866 all_tests
= set(old_build_results
.keys()) |
set(build_results
.keys())
868 if t
in old_build_results
:
869 old_res
= old_build_results
[t
]
871 old_res
= '(New test)'
872 if t
in build_results
:
873 new_res
= build_results
[t
]
875 new_res
= '(Test removed)'
876 if old_res
!= new_res
:
877 result_changes
[t
] = '%s -> %s' % (old_res
, new_res
)
878 self
.build_state
[action
]['result-changes'] = result_changes
879 old_ever_passed
= {t
for t
in self
.build_state
[action
]['ever-passed']
880 if t
in build_results
}
881 new_passes
= {t
for t
in build_results
if build_results
[t
] == 'PASS'}
882 self
.build_state
[action
]['ever-passed'] = sorted(old_ever_passed |
884 self
.store_build_state_json()
886 def load_bot_config_json(self
):
887 """Load bot configuration."""
888 with
open(self
.bot_config_json
, 'r') as f
:
889 self
.bot_config
= json
.load(f
)
891 def part_build_old(self
, action
, delay
):
892 """Return whether the last build for a given action was at least a
893 given number of seconds ago, or does not have a time recorded."""
894 old_time_str
= self
.build_state
[action
]['build-time']
897 old_time
= datetime
.datetime
.strptime(old_time_str
,
899 new_time
= datetime
.datetime
.utcnow()
900 delta
= new_time
- old_time
901 return delta
.total_seconds() >= delay
904 """Run a single round of checkout and builds."""
905 print('Bot cycle starting %s.' % str(datetime
.datetime
.utcnow()))
906 self
.load_bot_config_json()
907 actions
= ('host-libraries', 'compilers', 'glibcs')
908 self
.bot_run_self(['--replace-sources'], 'checkout')
909 self
.load_versions_json()
910 if self
.get_script_text() != self
.script_text
:
911 print('Script changed, re-execing.')
912 # On script change, all parts of the build should be rerun.
914 self
.clear_last_build_state(a
)
916 check_components
= {'host-libraries': ('gmp', 'mpfr', 'mpc'),
917 'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
918 'glibcs': ('glibc',)}
921 build_vers
= self
.build_state
[a
]['build-versions']
922 must_build
[a
] = False
923 if not self
.build_state
[a
]['build-time']:
927 for c
in check_components
[a
]:
929 old_vers
[c
] = build_vers
[c
]
930 new_vers
[c
] = {'version': self
.versions
[c
]['version'],
931 'revision': self
.versions
[c
]['revision']}
932 if new_vers
== old_vers
:
933 print('Versions for %s unchanged.' % a
)
935 print('Versions changed or rebuild forced for %s.' % a
)
936 if a
== 'compilers' and not self
.part_build_old(
937 a
, self
.bot_config
['compilers-rebuild-delay']):
938 print('Not requiring rebuild of compilers this soon.')
941 if must_build
['host-libraries']:
942 must_build
['compilers'] = True
943 if must_build
['compilers']:
944 must_build
['glibcs'] = True
947 print('Must rebuild %s.' % a
)
948 self
.clear_last_build_state(a
)
950 print('No need to rebuild %s.' % a
)
951 if os
.access(self
.logsdir
, os
.F_OK
):
952 shutil
.rmtree(self
.logsdir_old
, ignore_errors
=True)
953 shutil
.copytree(self
.logsdir
, self
.logsdir_old
)
956 build_time
= datetime
.datetime
.utcnow()
957 print('Rebuilding %s at %s.' % (a
, str(build_time
)))
958 self
.bot_run_self([], a
)
959 self
.load_build_state_json()
960 self
.bot_build_mail(a
, build_time
)
961 print('Bot cycle done at %s.' % str(datetime
.datetime
.utcnow()))
963 def bot_build_mail(self
, action
, build_time
):
964 """Send email with the results of a build."""
965 build_time
= build_time
.replace(microsecond
=0)
966 subject
= (self
.bot_config
['email-subject'] %
968 'build-time': str(build_time
)})
969 results
= self
.build_state
[action
]['build-results']
970 changes
= self
.build_state
[action
]['result-changes']
971 ever_passed
= set(self
.build_state
[action
]['ever-passed'])
972 versions
= self
.build_state
[action
]['build-versions']
973 new_regressions
= {k
for k
in changes
if changes
[k
] == 'PASS -> FAIL'}
974 all_regressions
= {k
for k
in ever_passed
if results
[k
] == 'FAIL'}
975 all_fails
= {k
for k
in results
if results
[k
] == 'FAIL'}
977 new_reg_list
= sorted(['FAIL: %s' % k
for k
in new_regressions
])
978 new_reg_text
= ('New regressions:\n\n%s\n\n' %
979 '\n'.join(new_reg_list
))
983 all_reg_list
= sorted(['FAIL: %s' % k
for k
in all_regressions
])
984 all_reg_text
= ('All regressions:\n\n%s\n\n' %
985 '\n'.join(all_reg_list
))
989 all_fail_list
= sorted(['FAIL: %s' % k
for k
in all_fails
])
990 all_fail_text
= ('All failures:\n\n%s\n\n' %
991 '\n'.join(all_fail_list
))
995 changes_list
= sorted(changes
.keys())
996 changes_list
= ['%s: %s' % (changes
[k
], k
) for k
in changes_list
]
997 changes_text
= ('All changed results:\n\n%s\n\n' %
998 '\n'.join(changes_list
))
1001 results_text
= (new_reg_text
+ all_reg_text
+ all_fail_text
+
1003 if not results_text
:
1004 results_text
= 'Clean build with unchanged results.\n\n'
1005 versions_list
= sorted(versions
.keys())
1006 versions_list
= ['%s: %s (%s)' % (k
, versions
[k
]['version'],
1007 versions
[k
]['revision'])
1008 for k
in versions_list
]
1009 versions_text
= ('Component versions for this build:\n\n%s\n' %
1010 '\n'.join(versions_list
))
1011 body_text
= results_text
+ versions_text
1012 msg
= email
.mime
.text
.MIMEText(body_text
)
1013 msg
['Subject'] = subject
1014 msg
['From'] = self
.bot_config
['email-from']
1015 msg
['To'] = self
.bot_config
['email-to']
1016 msg
['Message-ID'] = email
.utils
.make_msgid()
1017 msg
['Date'] = email
.utils
.format_datetime(datetime
.datetime
.utcnow())
1018 with smtplib
.SMTP(self
.bot_config
['email-server']) as s
:
1021 def bot_run_self(self
, opts
, action
, check
=True):
1022 """Run a copy of this script with given options."""
1023 cmd
= [sys
.executable
, sys
.argv
[0], '--keep=none',
1024 '-j%d' % self
.parallelism
]
1026 cmd
.extend([self
.topdir
, action
])
1028 subprocess
.run(cmd
, check
=check
)
1031 """Run repeated rounds of checkout and builds."""
1033 self
.load_bot_config_json()
1034 if not self
.bot_config
['run']:
1035 print('Bot exiting by request.')
1037 self
.bot_run_self([], 'bot-cycle', check
=False)
1038 self
.load_bot_config_json()
1039 if not self
.bot_config
['run']:
1040 print('Bot exiting by request.')
1042 time
.sleep(self
.bot_config
['delay'])
1043 if self
.get_script_text() != self
.script_text
:
1044 print('Script changed, bot re-execing.')
1048 class Config(object):
1049 """A configuration for building a compiler and associated libraries."""
1051 def __init__(self
, ctx
, arch
, os_name
, variant
=None, gcc_cfg
=None,
1052 first_gcc_cfg
=None, glibcs
=None, extra_glibcs
=None):
1053 """Initialize a Config object."""
1057 self
.variant
= variant
1059 self
.name
= '%s-%s' % (arch
, os_name
)
1061 self
.name
= '%s-%s-%s' % (arch
, os_name
, variant
)
1062 self
.triplet
= '%s-glibc-%s' % (arch
, os_name
)
1066 self
.gcc_cfg
= gcc_cfg
1067 if first_gcc_cfg
is None:
1068 self
.first_gcc_cfg
= []
1070 self
.first_gcc_cfg
= first_gcc_cfg
1072 glibcs
= [{'variant': variant
}]
1073 if extra_glibcs
is None:
1075 glibcs
= [Glibc(self
, **g
) for g
in glibcs
]
1076 extra_glibcs
= [Glibc(self
, **g
) for g
in extra_glibcs
]
1077 self
.all_glibcs
= glibcs
+ extra_glibcs
1078 self
.compiler_glibcs
= glibcs
1079 self
.installdir
= ctx
.compiler_installdir(self
.name
)
1080 self
.bindir
= ctx
.compiler_bindir(self
.name
)
1081 self
.sysroot
= ctx
.compiler_sysroot(self
.name
)
1082 self
.builddir
= os
.path
.join(ctx
.builddir
, 'compilers', self
.name
)
1083 self
.logsdir
= os
.path
.join(ctx
.logsdir
, 'compilers', self
.name
)
1085 def component_builddir(self
, component
):
1086 """Return the directory to use for a (non-glibc) build."""
1087 return self
.ctx
.component_builddir('compilers', self
.name
, component
)
1090 """Generate commands to build this compiler."""
1091 self
.ctx
.remove_recreate_dirs(self
.installdir
, self
.builddir
,
1093 cmdlist
= CommandList('compilers-%s' % self
.name
, self
.ctx
.keep
)
1094 cmdlist
.add_command('check-host-libraries',
1096 os
.path
.join(self
.ctx
.host_libraries_installdir
,
1098 cmdlist
.use_path(self
.bindir
)
1099 self
.build_cross_tool(cmdlist
, 'binutils', 'binutils',
1101 '--disable-libdecnumber',
1102 '--disable-readline',
1104 if self
.os
.startswith('linux'):
1105 self
.install_linux_headers(cmdlist
)
1106 self
.build_gcc(cmdlist
, True)
1107 for g
in self
.compiler_glibcs
:
1108 cmdlist
.push_subdesc('glibc')
1109 cmdlist
.push_subdesc(g
.name
)
1110 g
.build_glibc(cmdlist
, True)
1111 cmdlist
.pop_subdesc()
1112 cmdlist
.pop_subdesc()
1113 self
.build_gcc(cmdlist
, False)
1114 cmdlist
.add_command('done', ['touch',
1115 os
.path
.join(self
.installdir
, 'ok')])
1116 self
.ctx
.add_makefile_cmdlist('compilers-%s' % self
.name
, cmdlist
,
1119 def build_cross_tool(self
, cmdlist
, tool_src
, tool_build
, extra_opts
=None):
1120 """Build one cross tool."""
1121 srcdir
= self
.ctx
.component_srcdir(tool_src
)
1122 builddir
= self
.component_builddir(tool_build
)
1123 cmdlist
.push_subdesc(tool_build
)
1124 cmdlist
.create_use_dir(builddir
)
1125 cfg_cmd
= [os
.path
.join(srcdir
, 'configure'),
1126 '--prefix=%s' % self
.installdir
,
1127 '--build=%s' % self
.ctx
.build_triplet
,
1128 '--host=%s' % self
.ctx
.build_triplet
,
1129 '--target=%s' % self
.triplet
,
1130 '--with-sysroot=%s' % self
.sysroot
]
1132 cfg_cmd
.extend(extra_opts
)
1133 cmdlist
.add_command('configure', cfg_cmd
)
1134 cmdlist
.add_command('build', ['make'])
1135 cmdlist
.add_command('install', ['make', 'install'])
1136 cmdlist
.cleanup_dir()
1137 cmdlist
.pop_subdesc()
1139 def install_linux_headers(self
, cmdlist
):
1140 """Install Linux kernel headers."""
1141 arch_map
= {'aarch64': 'arm64',
1151 'microblaze': 'microblaze',
1154 'powerpc': 'powerpc',
1162 if self
.arch
.startswith(k
):
1163 linux_arch
= arch_map
[k
]
1165 assert linux_arch
is not None
1166 srcdir
= self
.ctx
.component_srcdir('linux')
1167 builddir
= self
.component_builddir('linux')
1168 headers_dir
= os
.path
.join(self
.sysroot
, 'usr')
1169 cmdlist
.push_subdesc('linux')
1170 cmdlist
.create_use_dir(builddir
)
1171 cmdlist
.add_command('install-headers',
1172 ['make', '-C', srcdir
, 'O=%s' % builddir
,
1173 'ARCH=%s' % linux_arch
,
1174 'INSTALL_HDR_PATH=%s' % headers_dir
,
1176 cmdlist
.cleanup_dir()
1177 cmdlist
.pop_subdesc()
1179 def build_gcc(self
, cmdlist
, bootstrap
):
1181 # libsanitizer commonly breaks because of glibc header
1182 # changes, or on unusual targets. libssp is of little
1183 # relevance with glibc's own stack checking support.
1184 cfg_opts
= list(self
.gcc_cfg
)
1185 cfg_opts
+= ['--disable-libsanitizer', '--disable-libssp']
1186 host_libs
= self
.ctx
.host_libraries_installdir
1187 cfg_opts
+= ['--with-gmp=%s' % host_libs
,
1188 '--with-mpfr=%s' % host_libs
,
1189 '--with-mpc=%s' % host_libs
]
1191 tool_build
= 'gcc-first'
1192 # Building a static-only, C-only compiler that is
1193 # sufficient to build glibc. Various libraries and
1194 # features that may require libc headers must be disabled.
1195 # When configuring with a sysroot, --with-newlib is
1196 # required to define inhibit_libc (to stop some parts of
1197 # libgcc including libc headers); --without-headers is not
1199 cfg_opts
+= ['--enable-languages=c', '--disable-shared',
1200 '--disable-threads',
1201 '--disable-libatomic',
1202 '--disable-decimal-float',
1204 '--disable-libgomp',
1207 '--disable-libquadmath',
1208 '--without-headers', '--with-newlib',
1209 '--with-glibc-version=%s' % self
.ctx
.glibc_version
1211 cfg_opts
+= self
.first_gcc_cfg
1214 cfg_opts
+= ['--enable-languages=c,c++', '--enable-shared',
1216 self
.build_cross_tool(cmdlist
, 'gcc', tool_build
, cfg_opts
)
1219 class Glibc(object):
1220 """A configuration for building glibc."""
1222 def __init__(self
, compiler
, arch
=None, os_name
=None, variant
=None,
1223 cfg
=None, ccopts
=None):
1224 """Initialize a Glibc object."""
1225 self
.ctx
= compiler
.ctx
1226 self
.compiler
= compiler
1228 self
.arch
= compiler
.arch
1232 self
.os
= compiler
.os
1235 self
.variant
= variant
1237 self
.name
= '%s-%s' % (self
.arch
, self
.os
)
1239 self
.name
= '%s-%s-%s' % (self
.arch
, self
.os
, variant
)
1240 self
.triplet
= '%s-glibc-%s' % (self
.arch
, self
.os
)
1245 self
.ccopts
= ccopts
1247 def tool_name(self
, tool
):
1248 """Return the name of a cross-compilation tool."""
1249 ctool
= '%s-%s' % (self
.compiler
.triplet
, tool
)
1250 if self
.ccopts
and (tool
== 'gcc' or tool
== 'g++'):
1251 ctool
= '%s %s' % (ctool
, self
.ccopts
)
1255 """Generate commands to build this glibc."""
1256 builddir
= self
.ctx
.component_builddir('glibcs', self
.name
, 'glibc')
1257 installdir
= self
.ctx
.glibc_installdir(self
.name
)
1258 logsdir
= os
.path
.join(self
.ctx
.logsdir
, 'glibcs', self
.name
)
1259 self
.ctx
.remove_recreate_dirs(installdir
, builddir
, logsdir
)
1260 cmdlist
= CommandList('glibcs-%s' % self
.name
, self
.ctx
.keep
)
1261 cmdlist
.add_command('check-compilers',
1263 os
.path
.join(self
.compiler
.installdir
, 'ok')])
1264 cmdlist
.use_path(self
.compiler
.bindir
)
1265 self
.build_glibc(cmdlist
, False)
1266 self
.ctx
.add_makefile_cmdlist('glibcs-%s' % self
.name
, cmdlist
,
1269 def build_glibc(self
, cmdlist
, for_compiler
):
1270 """Generate commands to build this glibc, either as part of a compiler
1271 build or with the bootstrapped compiler (and in the latter case, run
1273 srcdir
= self
.ctx
.component_srcdir('glibc')
1275 builddir
= self
.ctx
.component_builddir('compilers',
1276 self
.compiler
.name
, 'glibc',
1278 installdir
= self
.compiler
.sysroot
1279 srcdir_copy
= self
.ctx
.component_builddir('compilers',
1284 builddir
= self
.ctx
.component_builddir('glibcs', self
.name
,
1286 installdir
= self
.ctx
.glibc_installdir(self
.name
)
1287 srcdir_copy
= self
.ctx
.component_builddir('glibcs', self
.name
,
1289 cmdlist
.create_use_dir(builddir
)
1290 # glibc builds write into the source directory, and even if
1291 # not intentionally there is a risk of bugs that involve
1292 # writing into the working directory. To avoid possible
1293 # concurrency issues, copy the source directory.
1294 cmdlist
.create_copy_dir(srcdir
, srcdir_copy
)
1295 cfg_cmd
= [os
.path
.join(srcdir_copy
, 'configure'),
1298 '--build=%s' % self
.ctx
.build_triplet
,
1299 '--host=%s' % self
.triplet
,
1300 'CC=%s' % self
.tool_name('gcc'),
1301 'CXX=%s' % self
.tool_name('g++'),
1302 'AR=%s' % self
.tool_name('ar'),
1303 'AS=%s' % self
.tool_name('as'),
1304 'LD=%s' % self
.tool_name('ld'),
1305 'NM=%s' % self
.tool_name('nm'),
1306 'OBJCOPY=%s' % self
.tool_name('objcopy'),
1307 'OBJDUMP=%s' % self
.tool_name('objdump'),
1308 'RANLIB=%s' % self
.tool_name('ranlib'),
1309 'READELF=%s' % self
.tool_name('readelf'),
1310 'STRIP=%s' % self
.tool_name('strip')]
1312 cmdlist
.add_command('configure', cfg_cmd
)
1313 cmdlist
.add_command('build', ['make'])
1314 cmdlist
.add_command('install', ['make', 'install',
1315 'install_root=%s' % installdir
])
1316 # GCC uses paths such as lib/../lib64, so make sure lib
1317 # directories always exist.
1318 cmdlist
.add_command('mkdir-lib', ['mkdir', '-p',
1319 os
.path
.join(installdir
, 'lib'),
1320 os
.path
.join(installdir
,
1322 if not for_compiler
:
1324 cmdlist
.add_command('strip',
1326 ('%s %s/lib*/*.so' %
1327 (self
.tool_name('strip'), installdir
))])
1328 cmdlist
.add_command('check', ['make', 'check'])
1329 cmdlist
.add_command('save-logs', [self
.ctx
.save_logs
],
1331 cmdlist
.cleanup_dir('cleanup-src', srcdir_copy
)
1332 cmdlist
.cleanup_dir()
1335 class Command(object):
1336 """A command run in the build process."""
1338 def __init__(self
, desc
, num
, dir, path
, command
, always_run
=False):
1339 """Initialize a Command object."""
1343 trans
= str.maketrans({' ': '-'})
1344 self
.logbase
= '%03d-%s' % (num
, desc
.translate(trans
))
1345 self
.command
= command
1346 self
.always_run
= always_run
1349 def shell_make_quote_string(s
):
1350 """Given a string not containing a newline, quote it for use by the
1352 assert '\n' not in s
1353 if re
.fullmatch('[]+,./0-9@A-Z_a-z-]+', s
):
1355 strans
= str.maketrans({"'": "'\\''"})
1356 s
= "'%s'" % s
.translate(strans
)
1357 mtrans
= str.maketrans({'$': '$$'})
1358 return s
.translate(mtrans
)
1361 def shell_make_quote_list(l
, translate_make
):
1362 """Given a list of strings not containing newlines, quote them for use
1363 by the shell and make, returning a single string. If translate_make
1364 is true and the first string is 'make', change it to $(MAKE)."""
1365 l
= [Command
.shell_make_quote_string(s
) for s
in l
]
1366 if translate_make
and l
[0] == 'make':
1370 def shell_make_quote(self
):
1371 """Return this command quoted for the shell and make."""
1372 return self
.shell_make_quote_list(self
.command
, True)
1375 class CommandList(object):
1376 """A list of commands run in the build process."""
1378 def __init__(self
, desc
, keep
):
1379 """Initialize a CommandList object."""
1386 def desc_txt(self
, desc
):
1387 """Return the description to use for a command."""
1388 return '%s %s' % (' '.join(self
.desc
), desc
)
1390 def use_dir(self
, dir):
1391 """Set the default directory for subsequent commands."""
1394 def use_path(self
, path
):
1395 """Set a directory to be prepended to the PATH for subsequent
1399 def push_subdesc(self
, subdesc
):
1400 """Set the default subdescription for subsequent commands (e.g., the
1401 name of a component being built, within the series of commands
1403 self
.desc
.append(subdesc
)
1405 def pop_subdesc(self
):
1406 """Pop a subdescription from the list of descriptions."""
1409 def create_use_dir(self
, dir):
1410 """Remove and recreate a directory and use it for subsequent
1412 self
.add_command_dir('rm', None, ['rm', '-rf', dir])
1413 self
.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1416 def create_copy_dir(self
, src
, dest
):
1417 """Remove a directory and recreate it as a copy from the given
1419 self
.add_command_dir('copy-rm', None, ['rm', '-rf', dest
])
1420 parent
= os
.path
.dirname(dest
)
1421 self
.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent
])
1422 self
.add_command_dir('copy', None, ['cp', '-a', src
, dest
])
1424 def add_command_dir(self
, desc
, dir, command
, always_run
=False):
1425 """Add a command to run in a given directory."""
1426 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), dir, self
.path
,
1427 command
, always_run
)
1428 self
.cmdlist
.append(cmd
)
1430 def add_command(self
, desc
, command
, always_run
=False):
1431 """Add a command to run in the default directory."""
1432 cmd
= Command(self
.desc_txt(desc
), len(self
.cmdlist
), self
.dir,
1433 self
.path
, command
, always_run
)
1434 self
.cmdlist
.append(cmd
)
1436 def cleanup_dir(self
, desc
='cleanup', dir=None):
1437 """Clean up a build directory. If no directory is specified, the
1438 default directory is cleaned up and ceases to be the default
1443 if self
.keep
!= 'all':
1444 self
.add_command_dir(desc
, None, ['rm', '-rf', dir],
1445 always_run
=(self
.keep
== 'none'))
1447 def makefile_commands(self
, wrapper
, logsdir
):
1448 """Return the sequence of commands in the form of text for a Makefile.
1449 The given wrapper script takes arguments: base of logs for
1450 previous command, or empty; base of logs for this command;
1451 description; directory; PATH addition; the command itself."""
1452 # prev_base is the base of the name for logs of the previous
1453 # command that is not always-run (that is, a build command,
1454 # whose failure should stop subsequent build commands from
1455 # being run, as opposed to a cleanup command, which is run
1456 # even if previous commands failed).
1459 for c
in self
.cmdlist
:
1460 ctxt
= c
.shell_make_quote()
1461 if prev_base
and not c
.always_run
:
1462 prev_log
= os
.path
.join(logsdir
, prev_base
)
1465 this_log
= os
.path
.join(logsdir
, c
.logbase
)
1466 if not c
.always_run
:
1467 prev_base
= c
.logbase
1476 prelims
= [wrapper
, prev_log
, this_log
, c
.desc
, dir, path
]
1477 prelim_txt
= Command
.shell_make_quote_list(prelims
, False)
1478 cmds
.append('\t@%s %s' % (prelim_txt
, ctxt
))
1479 return '\n'.join(cmds
)
1481 def status_logs(self
, logsdir
):
1482 """Return the list of log files with command status."""
1483 return [os
.path
.join(logsdir
, '%s-status.txt' % c
.logbase
)
1484 for c
in self
.cmdlist
]
1488 """Return an argument parser for this module."""
1489 parser
= argparse
.ArgumentParser(description
=__doc__
)
1490 parser
.add_argument('-j', dest
='parallelism',
1491 help='Run this number of jobs in parallel',
1492 type=int, default
=os
.cpu_count())
1493 parser
.add_argument('--keep', dest
='keep',
1494 help='Whether to keep all build directories, '
1495 'none or only those from failed builds',
1496 default
='none', choices
=('none', 'all', 'failed'))
1497 parser
.add_argument('--replace-sources', action
='store_true',
1498 help='Remove and replace source directories '
1499 'with the wrong version of a component')
1500 parser
.add_argument('--strip', action
='store_true',
1501 help='Strip installed glibc libraries')
1502 parser
.add_argument('topdir',
1503 help='Toplevel working directory')
1504 parser
.add_argument('action',
1506 choices
=('checkout', 'bot-cycle', 'bot',
1507 'host-libraries', 'compilers', 'glibcs'))
1508 parser
.add_argument('configs',
1509 help='Versions to check out or configurations to build',
1515 """The main entry point."""
1516 parser
= get_parser()
1517 opts
= parser
.parse_args(argv
)
1518 topdir
= os
.path
.abspath(opts
.topdir
)
1519 ctx
= Context(topdir
, opts
.parallelism
, opts
.keep
, opts
.replace_sources
,
1520 opts
.strip
, opts
.action
)
1521 ctx
.run_builds(opts
.action
, opts
.configs
)
1524 if __name__
== '__main__':