Fix elf/sotruss-lib format-truncation error.
[glibc.git] / scripts / build-many-glibcs.py
blobd27e70b8bdcf06c0ac6eb79a8acabdafc0d3dd5d
1 #!/usr/bin/python3
2 # Build many configurations of glibc.
3 # Copyright (C) 2016-2017 Free Software Foundation, Inc.
4 # This file is part of the GNU C Library.
6 # The GNU C Library is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU Lesser General Public
8 # License as published by the Free Software Foundation; either
9 # version 2.1 of the License, or (at your option) any later version.
11 # The GNU C Library is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # Lesser General Public License for more details.
16 # You should have received a copy of the GNU Lesser General Public
17 # License along with the GNU C Library; if not, see
18 # <http://www.gnu.org/licenses/>.
20 """Build many configurations of glibc.
22 This script takes as arguments a directory name (containing a src
23 subdirectory with sources of the relevant toolchain components) and a
24 description of what to do: 'checkout', to check out sources into that
25 directory, 'bot-cycle', to run a series of checkout and build steps,
26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
27 libraries required by the toolchain, 'compilers', to build
28 cross-compilers for various configurations, or 'glibcs', to build
29 glibc for various configurations and run the compilation parts of the
30 testsuite. Subsequent arguments name the versions of components to
31 check out (<component>-<version), for 'checkout', or, for actions
32 other than 'checkout' and 'bot-cycle', name configurations for which
33 compilers or glibc are to be built.
35 """
37 import argparse
38 import datetime
39 import email.mime.text
40 import email.utils
41 import json
42 import os
43 import re
44 import shutil
45 import smtplib
46 import stat
47 import subprocess
48 import sys
49 import time
50 import urllib.request
52 try:
53 os.cpu_count
54 except:
55 import multiprocessing
56 os.cpu_count = lambda: multiprocessing.cpu_count()
58 try:
59 re.fullmatch
60 except:
61 re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)
63 try:
64 subprocess.run
65 except:
66 class _CompletedProcess:
67 def __init__(self, args, returncode, stdout=None, stderr=None):
68 self.args = args
69 self.returncode = returncode
70 self.stdout = stdout
71 self.stderr = stderr
73 def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
74 assert(timeout is None)
75 with subprocess.Popen(*popenargs, **kwargs) as process:
76 try:
77 stdout, stderr = process.communicate(input)
78 except:
79 process.kill()
80 process.wait()
81 raise
82 returncode = process.poll()
83 if check and returncode:
84 raise subprocess.CalledProcessError(returncode, popenargs)
85 return _CompletedProcess(popenargs, returncode, stdout, stderr)
87 subprocess.run = _run
90 class Context(object):
91 """The global state associated with builds in a given directory."""
93 def __init__(self, topdir, parallelism, keep, replace_sources, strip,
94 action):
95 """Initialize the context."""
96 self.topdir = topdir
97 self.parallelism = parallelism
98 self.keep = keep
99 self.replace_sources = replace_sources
100 self.strip = strip
101 self.srcdir = os.path.join(topdir, 'src')
102 self.versions_json = os.path.join(self.srcdir, 'versions.json')
103 self.build_state_json = os.path.join(topdir, 'build-state.json')
104 self.bot_config_json = os.path.join(topdir, 'bot-config.json')
105 self.installdir = os.path.join(topdir, 'install')
106 self.host_libraries_installdir = os.path.join(self.installdir,
107 'host-libraries')
108 self.builddir = os.path.join(topdir, 'build')
109 self.logsdir = os.path.join(topdir, 'logs')
110 self.logsdir_old = os.path.join(topdir, 'logs-old')
111 self.makefile = os.path.join(self.builddir, 'Makefile')
112 self.wrapper = os.path.join(self.builddir, 'wrapper')
113 self.save_logs = os.path.join(self.builddir, 'save-logs')
114 self.script_text = self.get_script_text()
115 if action != 'checkout':
116 self.build_triplet = self.get_build_triplet()
117 self.glibc_version = self.get_glibc_version()
118 self.configs = {}
119 self.glibc_configs = {}
120 self.makefile_pieces = ['.PHONY: all\n']
121 self.add_all_configs()
122 self.load_versions_json()
123 self.load_build_state_json()
124 self.status_log_list = []
126 def get_script_text(self):
127 """Return the text of this script."""
128 with open(sys.argv[0], 'r') as f:
129 return f.read()
131 def exec_self(self):
132 """Re-execute this script with the same arguments."""
133 sys.stdout.flush()
134 os.execv(sys.executable, [sys.executable] + sys.argv)
136 def get_build_triplet(self):
137 """Determine the build triplet with config.guess."""
138 config_guess = os.path.join(self.component_srcdir('gcc'),
139 'config.guess')
140 cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
141 check=True, universal_newlines=True).stdout
142 return cg_out.rstrip()
144 def get_glibc_version(self):
145 """Determine the glibc version number (major.minor)."""
146 version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
147 with open(version_h, 'r') as f:
148 lines = f.readlines()
149 starttext = '#define VERSION "'
150 for l in lines:
151 if l.startswith(starttext):
152 l = l[len(starttext):]
153 l = l.rstrip('"\n')
154 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
155 return '%s.%s' % m.group(1, 2)
156 print('error: could not determine glibc version')
157 exit(1)
159 def add_all_configs(self):
160 """Add all known glibc build configurations."""
161 # On architectures missing __builtin_trap support, these
162 # options may be needed as a workaround; see
163 # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70216> for SH.
164 no_isolate = ('-fno-isolate-erroneous-paths-dereference'
165 ' -fno-isolate-erroneous-paths-attribute')
166 self.add_config(arch='aarch64',
167 os_name='linux-gnu')
168 self.add_config(arch='aarch64_be',
169 os_name='linux-gnu')
170 self.add_config(arch='alpha',
171 os_name='linux-gnu')
172 self.add_config(arch='arm',
173 os_name='linux-gnueabi')
174 self.add_config(arch='armeb',
175 os_name='linux-gnueabi')
176 self.add_config(arch='armeb',
177 os_name='linux-gnueabi',
178 variant='be8',
179 gcc_cfg=['--with-arch=armv7-a'])
180 self.add_config(arch='arm',
181 os_name='linux-gnueabihf')
182 self.add_config(arch='armeb',
183 os_name='linux-gnueabihf')
184 self.add_config(arch='armeb',
185 os_name='linux-gnueabihf',
186 variant='be8',
187 gcc_cfg=['--with-arch=armv7-a'])
188 self.add_config(arch='hppa',
189 os_name='linux-gnu')
190 self.add_config(arch='ia64',
191 os_name='linux-gnu',
192 first_gcc_cfg=['--with-system-libunwind'])
193 self.add_config(arch='m68k',
194 os_name='linux-gnu',
195 gcc_cfg=['--disable-multilib'])
196 self.add_config(arch='m68k',
197 os_name='linux-gnu',
198 variant='coldfire',
199 gcc_cfg=['--with-arch=cf', '--disable-multilib'])
200 self.add_config(arch='microblaze',
201 os_name='linux-gnu',
202 gcc_cfg=['--disable-multilib'])
203 self.add_config(arch='microblazeel',
204 os_name='linux-gnu',
205 gcc_cfg=['--disable-multilib'])
206 self.add_config(arch='mips64',
207 os_name='linux-gnu',
208 gcc_cfg=['--with-mips-plt'],
209 glibcs=[{'variant': 'n32'},
210 {'arch': 'mips',
211 'ccopts': '-mabi=32'},
212 {'variant': 'n64',
213 'ccopts': '-mabi=64'}])
214 self.add_config(arch='mips64',
215 os_name='linux-gnu',
216 variant='soft',
217 gcc_cfg=['--with-mips-plt', '--with-float=soft'],
218 glibcs=[{'variant': 'n32-soft',
219 'cfg': ['--without-fp']},
220 {'variant': 'soft',
221 'arch': 'mips',
222 'ccopts': '-mabi=32',
223 'cfg': ['--without-fp']},
224 {'variant': 'n64-soft',
225 'ccopts': '-mabi=64',
226 'cfg': ['--without-fp']}])
227 self.add_config(arch='mips64',
228 os_name='linux-gnu',
229 variant='nan2008',
230 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
231 '--with-arch-64=mips64r2',
232 '--with-arch-32=mips32r2'],
233 glibcs=[{'variant': 'n32-nan2008'},
234 {'variant': 'nan2008',
235 'arch': 'mips',
236 'ccopts': '-mabi=32'},
237 {'variant': 'n64-nan2008',
238 'ccopts': '-mabi=64'}])
239 self.add_config(arch='mips64',
240 os_name='linux-gnu',
241 variant='nan2008-soft',
242 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
243 '--with-arch-64=mips64r2',
244 '--with-arch-32=mips32r2',
245 '--with-float=soft'],
246 glibcs=[{'variant': 'n32-nan2008-soft',
247 'cfg': ['--without-fp']},
248 {'variant': 'nan2008-soft',
249 'arch': 'mips',
250 'ccopts': '-mabi=32',
251 'cfg': ['--without-fp']},
252 {'variant': 'n64-nan2008-soft',
253 'ccopts': '-mabi=64',
254 'cfg': ['--without-fp']}])
255 self.add_config(arch='mips64el',
256 os_name='linux-gnu',
257 gcc_cfg=['--with-mips-plt'],
258 glibcs=[{'variant': 'n32'},
259 {'arch': 'mipsel',
260 'ccopts': '-mabi=32'},
261 {'variant': 'n64',
262 'ccopts': '-mabi=64'}])
263 self.add_config(arch='mips64el',
264 os_name='linux-gnu',
265 variant='soft',
266 gcc_cfg=['--with-mips-plt', '--with-float=soft'],
267 glibcs=[{'variant': 'n32-soft',
268 'cfg': ['--without-fp']},
269 {'variant': 'soft',
270 'arch': 'mipsel',
271 'ccopts': '-mabi=32',
272 'cfg': ['--without-fp']},
273 {'variant': 'n64-soft',
274 'ccopts': '-mabi=64',
275 'cfg': ['--without-fp']}])
276 self.add_config(arch='mips64el',
277 os_name='linux-gnu',
278 variant='nan2008',
279 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
280 '--with-arch-64=mips64r2',
281 '--with-arch-32=mips32r2'],
282 glibcs=[{'variant': 'n32-nan2008'},
283 {'variant': 'nan2008',
284 'arch': 'mipsel',
285 'ccopts': '-mabi=32'},
286 {'variant': 'n64-nan2008',
287 'ccopts': '-mabi=64'}])
288 self.add_config(arch='mips64el',
289 os_name='linux-gnu',
290 variant='nan2008-soft',
291 gcc_cfg=['--with-mips-plt', '--with-nan=2008',
292 '--with-arch-64=mips64r2',
293 '--with-arch-32=mips32r2',
294 '--with-float=soft'],
295 glibcs=[{'variant': 'n32-nan2008-soft',
296 'cfg': ['--without-fp']},
297 {'variant': 'nan2008-soft',
298 'arch': 'mipsel',
299 'ccopts': '-mabi=32',
300 'cfg': ['--without-fp']},
301 {'variant': 'n64-nan2008-soft',
302 'ccopts': '-mabi=64',
303 'cfg': ['--without-fp']}])
304 self.add_config(arch='nios2',
305 os_name='linux-gnu')
306 self.add_config(arch='powerpc',
307 os_name='linux-gnu',
308 gcc_cfg=['--disable-multilib', '--enable-secureplt'],
309 extra_glibcs=[{'variant': 'power4',
310 'ccopts': '-mcpu=power4',
311 'cfg': ['--with-cpu=power4']}])
312 self.add_config(arch='powerpc',
313 os_name='linux-gnu',
314 variant='soft',
315 gcc_cfg=['--disable-multilib', '--with-float=soft',
316 '--enable-secureplt'],
317 glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
318 self.add_config(arch='powerpc64',
319 os_name='linux-gnu',
320 gcc_cfg=['--disable-multilib', '--enable-secureplt'])
321 self.add_config(arch='powerpc64le',
322 os_name='linux-gnu',
323 gcc_cfg=['--disable-multilib', '--enable-secureplt'])
324 self.add_config(arch='powerpc',
325 os_name='linux-gnuspe',
326 gcc_cfg=['--disable-multilib', '--enable-secureplt',
327 '--enable-e500-double'],
328 glibcs=[{'cfg': ['--without-fp']}])
329 self.add_config(arch='powerpc',
330 os_name='linux-gnuspe',
331 variant='e500v1',
332 gcc_cfg=['--disable-multilib', '--enable-secureplt'],
333 glibcs=[{'variant': 'e500v1', 'cfg': ['--without-fp']}])
334 self.add_config(arch='s390x',
335 os_name='linux-gnu',
336 glibcs=[{},
337 {'arch': 's390', 'ccopts': '-m31'}])
338 self.add_config(arch='sh3',
339 os_name='linux-gnu',
340 glibcs=[{'ccopts': no_isolate}])
341 self.add_config(arch='sh3eb',
342 os_name='linux-gnu',
343 glibcs=[{'ccopts': no_isolate}])
344 self.add_config(arch='sh4',
345 os_name='linux-gnu',
346 glibcs=[{'ccopts': no_isolate}])
347 self.add_config(arch='sh4eb',
348 os_name='linux-gnu',
349 glibcs=[{'ccopts': no_isolate}])
350 self.add_config(arch='sh4',
351 os_name='linux-gnu',
352 variant='soft',
353 gcc_cfg=['--without-fp'],
354 glibcs=[{'variant': 'soft',
355 'cfg': ['--without-fp'],
356 'ccopts': no_isolate}])
357 self.add_config(arch='sh4eb',
358 os_name='linux-gnu',
359 variant='soft',
360 gcc_cfg=['--without-fp'],
361 glibcs=[{'variant': 'soft',
362 'cfg': ['--without-fp'],
363 'ccopts': no_isolate}])
364 self.add_config(arch='sparc64',
365 os_name='linux-gnu',
366 glibcs=[{},
367 {'arch': 'sparcv9',
368 'ccopts': '-m32 -mlong-double-128'}])
369 self.add_config(arch='tilegx',
370 os_name='linux-gnu',
371 glibcs=[{},
372 {'variant': '32', 'ccopts': '-m32'}])
373 self.add_config(arch='tilegxbe',
374 os_name='linux-gnu',
375 glibcs=[{},
376 {'variant': '32', 'ccopts': '-m32'}])
377 self.add_config(arch='tilepro',
378 os_name='linux-gnu')
379 self.add_config(arch='x86_64',
380 os_name='linux-gnu',
381 gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
382 glibcs=[{},
383 {'variant': 'x32', 'ccopts': '-mx32'},
384 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
385 extra_glibcs=[{'variant': 'disable-multi-arch',
386 'cfg': ['--disable-multi-arch']},
387 {'variant': 'disable-multi-arch',
388 'arch': 'i686',
389 'ccopts': '-m32 -march=i686',
390 'cfg': ['--disable-multi-arch']},
391 {'arch': 'i486',
392 'ccopts': '-m32 -march=i486'},
393 {'arch': 'i586',
394 'ccopts': '-m32 -march=i586'}])
396 def add_config(self, **args):
397 """Add an individual build configuration."""
398 cfg = Config(self, **args)
399 if cfg.name in self.configs:
400 print('error: duplicate config %s' % cfg.name)
401 exit(1)
402 self.configs[cfg.name] = cfg
403 for c in cfg.all_glibcs:
404 if c.name in self.glibc_configs:
405 print('error: duplicate glibc config %s' % c.name)
406 exit(1)
407 self.glibc_configs[c.name] = c
409 def component_srcdir(self, component):
410 """Return the source directory for a given component, e.g. gcc."""
411 return os.path.join(self.srcdir, component)
413 def component_builddir(self, action, config, component, subconfig=None):
414 """Return the directory to use for a build."""
415 if config is None:
416 # Host libraries.
417 assert subconfig is None
418 return os.path.join(self.builddir, action, component)
419 if subconfig is None:
420 return os.path.join(self.builddir, action, config, component)
421 else:
422 # glibc build as part of compiler build.
423 return os.path.join(self.builddir, action, config, component,
424 subconfig)
426 def compiler_installdir(self, config):
427 """Return the directory in which to install a compiler."""
428 return os.path.join(self.installdir, 'compilers', config)
430 def compiler_bindir(self, config):
431 """Return the directory in which to find compiler binaries."""
432 return os.path.join(self.compiler_installdir(config), 'bin')
434 def compiler_sysroot(self, config):
435 """Return the sysroot directory for a compiler."""
436 return os.path.join(self.compiler_installdir(config), 'sysroot')
438 def glibc_installdir(self, config):
439 """Return the directory in which to install glibc."""
440 return os.path.join(self.installdir, 'glibcs', config)
442 def run_builds(self, action, configs):
443 """Run the requested builds."""
444 if action == 'checkout':
445 self.checkout(configs)
446 return
447 if action == 'bot-cycle':
448 if configs:
449 print('error: configurations specified for bot-cycle')
450 exit(1)
451 self.bot_cycle()
452 return
453 if action == 'bot':
454 if configs:
455 print('error: configurations specified for bot')
456 exit(1)
457 self.bot()
458 return
459 if action == 'host-libraries' and configs:
460 print('error: configurations specified for host-libraries')
461 exit(1)
462 self.clear_last_build_state(action)
463 build_time = datetime.datetime.utcnow()
464 if action == 'host-libraries':
465 build_components = ('gmp', 'mpfr', 'mpc')
466 old_components = ()
467 old_versions = {}
468 self.build_host_libraries()
469 elif action == 'compilers':
470 build_components = ('binutils', 'gcc', 'glibc', 'linux')
471 old_components = ('gmp', 'mpfr', 'mpc')
472 old_versions = self.build_state['host-libraries']['build-versions']
473 self.build_compilers(configs)
474 else:
475 build_components = ('glibc',)
476 old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
477 old_versions = self.build_state['compilers']['build-versions']
478 self.build_glibcs(configs)
479 self.write_files()
480 self.do_build()
481 if configs:
482 # Partial build, do not update stored state.
483 return
484 build_versions = {}
485 for k in build_components:
486 if k in self.versions:
487 build_versions[k] = {'version': self.versions[k]['version'],
488 'revision': self.versions[k]['revision']}
489 for k in old_components:
490 if k in old_versions:
491 build_versions[k] = {'version': old_versions[k]['version'],
492 'revision': old_versions[k]['revision']}
493 self.update_build_state(action, build_time, build_versions)
495 @staticmethod
496 def remove_dirs(*args):
497 """Remove directories and their contents if they exist."""
498 for dir in args:
499 shutil.rmtree(dir, ignore_errors=True)
501 @staticmethod
502 def remove_recreate_dirs(*args):
503 """Remove directories if they exist, and create them as empty."""
504 Context.remove_dirs(*args)
505 for dir in args:
506 os.makedirs(dir, exist_ok=True)
508 def add_makefile_cmdlist(self, target, cmdlist, logsdir):
509 """Add makefile text for a list of commands."""
510 commands = cmdlist.makefile_commands(self.wrapper, logsdir)
511 self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
512 (target, target, target, commands))
513 self.status_log_list.extend(cmdlist.status_logs(logsdir))
515 def write_files(self):
516 """Write out the Makefile and wrapper script."""
517 mftext = ''.join(self.makefile_pieces)
518 with open(self.makefile, 'w') as f:
519 f.write(mftext)
520 wrapper_text = (
521 '#!/bin/sh\n'
522 'prev_base=$1\n'
523 'this_base=$2\n'
524 'desc=$3\n'
525 'dir=$4\n'
526 'path=$5\n'
527 'shift 5\n'
528 'prev_status=$prev_base-status.txt\n'
529 'this_status=$this_base-status.txt\n'
530 'this_log=$this_base-log.txt\n'
531 'date > "$this_log"\n'
532 'echo >> "$this_log"\n'
533 'echo "Description: $desc" >> "$this_log"\n'
534 'printf "%s" "Command:" >> "$this_log"\n'
535 'for word in "$@"; do\n'
536 ' if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
537 ' printf " %s" "$word"\n'
538 ' else\n'
539 ' printf " \'"\n'
540 ' printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
541 ' printf "\'"\n'
542 ' fi\n'
543 'done >> "$this_log"\n'
544 'echo >> "$this_log"\n'
545 'echo "Directory: $dir" >> "$this_log"\n'
546 'echo "Path addition: $path" >> "$this_log"\n'
547 'echo >> "$this_log"\n'
548 'record_status ()\n'
549 '{\n'
550 ' echo >> "$this_log"\n'
551 ' echo "$1: $desc" > "$this_status"\n'
552 ' echo "$1: $desc" >> "$this_log"\n'
553 ' echo >> "$this_log"\n'
554 ' date >> "$this_log"\n'
555 ' echo "$1: $desc"\n'
556 ' exit 0\n'
557 '}\n'
558 'check_error ()\n'
559 '{\n'
560 ' if [ "$1" != "0" ]; then\n'
561 ' record_status FAIL\n'
562 ' fi\n'
563 '}\n'
564 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
565 ' record_status UNRESOLVED\n'
566 'fi\n'
567 'if [ "$dir" ]; then\n'
568 ' cd "$dir"\n'
569 ' check_error "$?"\n'
570 'fi\n'
571 'if [ "$path" ]; then\n'
572 ' PATH=$path:$PATH\n'
573 'fi\n'
574 '"$@" < /dev/null >> "$this_log" 2>&1\n'
575 'check_error "$?"\n'
576 'record_status PASS\n')
577 with open(self.wrapper, 'w') as f:
578 f.write(wrapper_text)
579 # Mode 0o755.
580 mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
581 stat.S_IROTH|stat.S_IXOTH)
582 os.chmod(self.wrapper, mode_exec)
583 save_logs_text = (
584 '#!/bin/sh\n'
585 'if ! [ -f tests.sum ]; then\n'
586 ' echo "No test summary available."\n'
587 ' exit 0\n'
588 'fi\n'
589 'save_file ()\n'
590 '{\n'
591 ' echo "Contents of $1:"\n'
592 ' echo\n'
593 ' cat "$1"\n'
594 ' echo\n'
595 ' echo "End of contents of $1."\n'
596 ' echo\n'
597 '}\n'
598 'save_file tests.sum\n'
599 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
600 'for t in $non_pass_tests; do\n'
601 ' if [ -f "$t.out" ]; then\n'
602 ' save_file "$t.out"\n'
603 ' fi\n'
604 'done\n')
605 with open(self.save_logs, 'w') as f:
606 f.write(save_logs_text)
607 os.chmod(self.save_logs, mode_exec)
609 def do_build(self):
610 """Do the actual build."""
611 cmd = ['make', '-j%d' % self.parallelism]
612 subprocess.run(cmd, cwd=self.builddir, check=True)
614 def build_host_libraries(self):
615 """Build the host libraries."""
616 installdir = self.host_libraries_installdir
617 builddir = os.path.join(self.builddir, 'host-libraries')
618 logsdir = os.path.join(self.logsdir, 'host-libraries')
619 self.remove_recreate_dirs(installdir, builddir, logsdir)
620 cmdlist = CommandList('host-libraries', self.keep)
621 self.build_host_library(cmdlist, 'gmp')
622 self.build_host_library(cmdlist, 'mpfr',
623 ['--with-gmp=%s' % installdir])
624 self.build_host_library(cmdlist, 'mpc',
625 ['--with-gmp=%s' % installdir,
626 '--with-mpfr=%s' % installdir])
627 cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
628 self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
630 def build_host_library(self, cmdlist, lib, extra_opts=None):
631 """Build one host library."""
632 srcdir = self.component_srcdir(lib)
633 builddir = self.component_builddir('host-libraries', None, lib)
634 installdir = self.host_libraries_installdir
635 cmdlist.push_subdesc(lib)
636 cmdlist.create_use_dir(builddir)
637 cfg_cmd = [os.path.join(srcdir, 'configure'),
638 '--prefix=%s' % installdir,
639 '--disable-shared']
640 if extra_opts:
641 cfg_cmd.extend (extra_opts)
642 cmdlist.add_command('configure', cfg_cmd)
643 cmdlist.add_command('build', ['make'])
644 cmdlist.add_command('check', ['make', 'check'])
645 cmdlist.add_command('install', ['make', 'install'])
646 cmdlist.cleanup_dir()
647 cmdlist.pop_subdesc()
649 def build_compilers(self, configs):
650 """Build the compilers."""
651 if not configs:
652 self.remove_dirs(os.path.join(self.builddir, 'compilers'))
653 self.remove_dirs(os.path.join(self.installdir, 'compilers'))
654 self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
655 configs = sorted(self.configs.keys())
656 for c in configs:
657 self.configs[c].build()
659 def build_glibcs(self, configs):
660 """Build the glibcs."""
661 if not configs:
662 self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
663 self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
664 self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
665 configs = sorted(self.glibc_configs.keys())
666 for c in configs:
667 self.glibc_configs[c].build()
669 def load_versions_json(self):
670 """Load information about source directory versions."""
671 if not os.access(self.versions_json, os.F_OK):
672 self.versions = {}
673 return
674 with open(self.versions_json, 'r') as f:
675 self.versions = json.load(f)
677 def store_json(self, data, filename):
678 """Store information in a JSON file."""
679 filename_tmp = filename + '.tmp'
680 with open(filename_tmp, 'w') as f:
681 json.dump(data, f, indent=2, sort_keys=True)
682 os.rename(filename_tmp, filename)
684 def store_versions_json(self):
685 """Store information about source directory versions."""
686 self.store_json(self.versions, self.versions_json)
688 def set_component_version(self, component, version, explicit, revision):
689 """Set the version information for a component."""
690 self.versions[component] = {'version': version,
691 'explicit': explicit,
692 'revision': revision}
693 self.store_versions_json()
695 def checkout(self, versions):
696 """Check out the desired component versions."""
697 default_versions = {'binutils': 'vcs-2.28',
698 'gcc': 'vcs-6',
699 'glibc': 'vcs-mainline',
700 'gmp': '6.1.1',
701 'linux': '4.9',
702 'mpc': '1.0.3',
703 'mpfr': '3.1.5'}
704 use_versions = {}
705 explicit_versions = {}
706 for v in versions:
707 found_v = False
708 for k in default_versions.keys():
709 kx = k + '-'
710 if v.startswith(kx):
711 vx = v[len(kx):]
712 if k in use_versions:
713 print('error: multiple versions for %s' % k)
714 exit(1)
715 use_versions[k] = vx
716 explicit_versions[k] = True
717 found_v = True
718 break
719 if not found_v:
720 print('error: unknown component in %s' % v)
721 exit(1)
722 for k in default_versions.keys():
723 if k not in use_versions:
724 if k in self.versions and self.versions[k]['explicit']:
725 use_versions[k] = self.versions[k]['version']
726 explicit_versions[k] = True
727 else:
728 use_versions[k] = default_versions[k]
729 explicit_versions[k] = False
730 os.makedirs(self.srcdir, exist_ok=True)
731 for k in sorted(default_versions.keys()):
732 update = os.access(self.component_srcdir(k), os.F_OK)
733 v = use_versions[k]
734 if (update and
735 k in self.versions and
736 v != self.versions[k]['version']):
737 if not self.replace_sources:
738 print('error: version of %s has changed from %s to %s, '
739 'use --replace-sources to check out again' %
740 (k, self.versions[k]['version'], v))
741 exit(1)
742 shutil.rmtree(self.component_srcdir(k))
743 update = False
744 if v.startswith('vcs-'):
745 revision = self.checkout_vcs(k, v[4:], update)
746 else:
747 self.checkout_tar(k, v, update)
748 revision = v
749 self.set_component_version(k, v, explicit_versions[k], revision)
750 if self.get_script_text() != self.script_text:
751 # Rerun the checkout process in case the updated script
752 # uses different default versions or new components.
753 self.exec_self()
755 def checkout_vcs(self, component, version, update):
756 """Check out the given version of the given component from version
757 control. Return a revision identifier."""
758 if component == 'binutils':
759 git_url = 'git://sourceware.org/git/binutils-gdb.git'
760 if version == 'mainline':
761 git_branch = 'master'
762 else:
763 trans = str.maketrans({'.': '_'})
764 git_branch = 'binutils-%s-branch' % version.translate(trans)
765 return self.git_checkout(component, git_url, git_branch, update)
766 elif component == 'gcc':
767 if version == 'mainline':
768 branch = 'trunk'
769 else:
770 trans = str.maketrans({'.': '_'})
771 branch = 'branches/gcc-%s-branch' % version.translate(trans)
772 svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
773 return self.gcc_checkout(svn_url, update)
774 elif component == 'glibc':
775 git_url = 'git://sourceware.org/git/glibc.git'
776 if version == 'mainline':
777 git_branch = 'master'
778 else:
779 git_branch = 'release/%s/master' % version
780 r = self.git_checkout(component, git_url, git_branch, update)
781 self.fix_glibc_timestamps()
782 return r
783 else:
784 print('error: component %s coming from VCS' % component)
785 exit(1)
787 def git_checkout(self, component, git_url, git_branch, update):
788 """Check out a component from git. Return a commit identifier."""
789 if update:
790 subprocess.run(['git', 'remote', 'prune', 'origin'],
791 cwd=self.component_srcdir(component), check=True)
792 subprocess.run(['git', 'pull', '-q'],
793 cwd=self.component_srcdir(component), check=True)
794 else:
795 subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
796 self.component_srcdir(component)], check=True)
797 r = subprocess.run(['git', 'rev-parse', 'HEAD'],
798 cwd=self.component_srcdir(component),
799 stdout=subprocess.PIPE,
800 check=True, universal_newlines=True).stdout
801 return r.rstrip()
803 def fix_glibc_timestamps(self):
804 """Fix timestamps in a glibc checkout."""
805 # Ensure that builds do not try to regenerate generated files
806 # in the source tree.
807 srcdir = self.component_srcdir('glibc')
808 for dirpath, dirnames, filenames in os.walk(srcdir):
809 for f in filenames:
810 if (f == 'configure' or
811 f == 'preconfigure' or
812 f.endswith('-kw.h')):
813 to_touch = os.path.join(dirpath, f)
814 subprocess.run(['touch', to_touch], check=True)
816 def gcc_checkout(self, svn_url, update):
817 """Check out GCC from SVN. Return the revision number."""
818 if not update:
819 subprocess.run(['svn', 'co', '-q', svn_url,
820 self.component_srcdir('gcc')], check=True)
821 subprocess.run(['contrib/gcc_update', '--silent'],
822 cwd=self.component_srcdir('gcc'), check=True)
823 r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
824 stdout=subprocess.PIPE,
825 check=True, universal_newlines=True).stdout
826 return r.rstrip()
828 def checkout_tar(self, component, version, update):
829 """Check out the given version of the given component from a
830 tarball."""
831 if update:
832 return
833 url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
834 'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
835 'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
836 'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
837 'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
838 'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
839 if component not in url_map:
840 print('error: component %s coming from tarball' % component)
841 exit(1)
842 url = url_map[component] % {'version': version}
843 filename = os.path.join(self.srcdir, url.split('/')[-1])
844 response = urllib.request.urlopen(url)
845 data = response.read()
846 with open(filename, 'wb') as f:
847 f.write(data)
848 subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
849 check=True)
850 os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
851 self.component_srcdir(component))
852 os.remove(filename)
854 def load_build_state_json(self):
855 """Load information about the state of previous builds."""
856 if os.access(self.build_state_json, os.F_OK):
857 with open(self.build_state_json, 'r') as f:
858 self.build_state = json.load(f)
859 else:
860 self.build_state = {}
861 for k in ('host-libraries', 'compilers', 'glibcs'):
862 if k not in self.build_state:
863 self.build_state[k] = {}
864 if 'build-time' not in self.build_state[k]:
865 self.build_state[k]['build-time'] = ''
866 if 'build-versions' not in self.build_state[k]:
867 self.build_state[k]['build-versions'] = {}
868 if 'build-results' not in self.build_state[k]:
869 self.build_state[k]['build-results'] = {}
870 if 'result-changes' not in self.build_state[k]:
871 self.build_state[k]['result-changes'] = {}
872 if 'ever-passed' not in self.build_state[k]:
873 self.build_state[k]['ever-passed'] = []
875 def store_build_state_json(self):
876 """Store information about the state of previous builds."""
877 self.store_json(self.build_state, self.build_state_json)
879 def clear_last_build_state(self, action):
880 """Clear information about the state of part of the build."""
881 # We clear the last build time and versions when starting a
882 # new build. The results of the last build are kept around,
883 # as comparison is still meaningful if this build is aborted
884 # and a new one started.
885 self.build_state[action]['build-time'] = ''
886 self.build_state[action]['build-versions'] = {}
887 self.store_build_state_json()
889 def update_build_state(self, action, build_time, build_versions):
890 """Update the build state after a build."""
891 build_time = build_time.replace(microsecond=0)
892 self.build_state[action]['build-time'] = str(build_time)
893 self.build_state[action]['build-versions'] = build_versions
894 build_results = {}
895 for log in self.status_log_list:
896 with open(log, 'r') as f:
897 log_text = f.read()
898 log_text = log_text.rstrip()
899 m = re.fullmatch('([A-Z]+): (.*)', log_text)
900 result = m.group(1)
901 test_name = m.group(2)
902 assert test_name not in build_results
903 build_results[test_name] = result
904 old_build_results = self.build_state[action]['build-results']
905 self.build_state[action]['build-results'] = build_results
906 result_changes = {}
907 all_tests = set(old_build_results.keys()) | set(build_results.keys())
908 for t in all_tests:
909 if t in old_build_results:
910 old_res = old_build_results[t]
911 else:
912 old_res = '(New test)'
913 if t in build_results:
914 new_res = build_results[t]
915 else:
916 new_res = '(Test removed)'
917 if old_res != new_res:
918 result_changes[t] = '%s -> %s' % (old_res, new_res)
919 self.build_state[action]['result-changes'] = result_changes
920 old_ever_passed = {t for t in self.build_state[action]['ever-passed']
921 if t in build_results}
922 new_passes = {t for t in build_results if build_results[t] == 'PASS'}
923 self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
924 new_passes)
925 self.store_build_state_json()
927 def load_bot_config_json(self):
928 """Load bot configuration."""
929 with open(self.bot_config_json, 'r') as f:
930 self.bot_config = json.load(f)
932 def part_build_old(self, action, delay):
933 """Return whether the last build for a given action was at least a
934 given number of seconds ago, or does not have a time recorded."""
935 old_time_str = self.build_state[action]['build-time']
936 if not old_time_str:
937 return True
938 old_time = datetime.datetime.strptime(old_time_str,
939 '%Y-%m-%d %H:%M:%S')
940 new_time = datetime.datetime.utcnow()
941 delta = new_time - old_time
942 return delta.total_seconds() >= delay
944 def bot_cycle(self):
945 """Run a single round of checkout and builds."""
946 print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
947 self.load_bot_config_json()
948 actions = ('host-libraries', 'compilers', 'glibcs')
949 self.bot_run_self(['--replace-sources'], 'checkout')
950 self.load_versions_json()
951 if self.get_script_text() != self.script_text:
952 print('Script changed, re-execing.')
953 # On script change, all parts of the build should be rerun.
954 for a in actions:
955 self.clear_last_build_state(a)
956 self.exec_self()
957 check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
958 'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
959 'glibcs': ('glibc',)}
960 must_build = {}
961 for a in actions:
962 build_vers = self.build_state[a]['build-versions']
963 must_build[a] = False
964 if not self.build_state[a]['build-time']:
965 must_build[a] = True
966 old_vers = {}
967 new_vers = {}
968 for c in check_components[a]:
969 if c in build_vers:
970 old_vers[c] = build_vers[c]
971 new_vers[c] = {'version': self.versions[c]['version'],
972 'revision': self.versions[c]['revision']}
973 if new_vers == old_vers:
974 print('Versions for %s unchanged.' % a)
975 else:
976 print('Versions changed or rebuild forced for %s.' % a)
977 if a == 'compilers' and not self.part_build_old(
978 a, self.bot_config['compilers-rebuild-delay']):
979 print('Not requiring rebuild of compilers this soon.')
980 else:
981 must_build[a] = True
982 if must_build['host-libraries']:
983 must_build['compilers'] = True
984 if must_build['compilers']:
985 must_build['glibcs'] = True
986 for a in actions:
987 if must_build[a]:
988 print('Must rebuild %s.' % a)
989 self.clear_last_build_state(a)
990 else:
991 print('No need to rebuild %s.' % a)
992 if os.access(self.logsdir, os.F_OK):
993 shutil.rmtree(self.logsdir_old, ignore_errors=True)
994 shutil.copytree(self.logsdir, self.logsdir_old)
995 for a in actions:
996 if must_build[a]:
997 build_time = datetime.datetime.utcnow()
998 print('Rebuilding %s at %s.' % (a, str(build_time)))
999 self.bot_run_self([], a)
1000 self.load_build_state_json()
1001 self.bot_build_mail(a, build_time)
1002 print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1004 def bot_build_mail(self, action, build_time):
1005 """Send email with the results of a build."""
1006 build_time = build_time.replace(microsecond=0)
1007 subject = (self.bot_config['email-subject'] %
1008 {'action': action,
1009 'build-time': str(build_time)})
1010 results = self.build_state[action]['build-results']
1011 changes = self.build_state[action]['result-changes']
1012 ever_passed = set(self.build_state[action]['ever-passed'])
1013 versions = self.build_state[action]['build-versions']
1014 new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1015 all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1016 all_fails = {k for k in results if results[k] == 'FAIL'}
1017 if new_regressions:
1018 new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1019 new_reg_text = ('New regressions:\n\n%s\n\n' %
1020 '\n'.join(new_reg_list))
1021 else:
1022 new_reg_text = ''
1023 if all_regressions:
1024 all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1025 all_reg_text = ('All regressions:\n\n%s\n\n' %
1026 '\n'.join(all_reg_list))
1027 else:
1028 all_reg_text = ''
1029 if all_fails:
1030 all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1031 all_fail_text = ('All failures:\n\n%s\n\n' %
1032 '\n'.join(all_fail_list))
1033 else:
1034 all_fail_text = ''
1035 if changes:
1036 changes_list = sorted(changes.keys())
1037 changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1038 changes_text = ('All changed results:\n\n%s\n\n' %
1039 '\n'.join(changes_list))
1040 else:
1041 changes_text = ''
1042 results_text = (new_reg_text + all_reg_text + all_fail_text +
1043 changes_text)
1044 if not results_text:
1045 results_text = 'Clean build with unchanged results.\n\n'
1046 versions_list = sorted(versions.keys())
1047 versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1048 versions[k]['revision'])
1049 for k in versions_list]
1050 versions_text = ('Component versions for this build:\n\n%s\n' %
1051 '\n'.join(versions_list))
1052 body_text = results_text + versions_text
1053 msg = email.mime.text.MIMEText(body_text)
1054 msg['Subject'] = subject
1055 msg['From'] = self.bot_config['email-from']
1056 msg['To'] = self.bot_config['email-to']
1057 msg['Message-ID'] = email.utils.make_msgid()
1058 msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1059 with smtplib.SMTP(self.bot_config['email-server']) as s:
1060 s.send_message(msg)
1062 def bot_run_self(self, opts, action, check=True):
1063 """Run a copy of this script with given options."""
1064 cmd = [sys.executable, sys.argv[0], '--keep=none',
1065 '-j%d' % self.parallelism]
1066 cmd.extend(opts)
1067 cmd.extend([self.topdir, action])
1068 sys.stdout.flush()
1069 subprocess.run(cmd, check=check)
1071 def bot(self):
1072 """Run repeated rounds of checkout and builds."""
1073 while True:
1074 self.load_bot_config_json()
1075 if not self.bot_config['run']:
1076 print('Bot exiting by request.')
1077 exit(0)
1078 self.bot_run_self([], 'bot-cycle', check=False)
1079 self.load_bot_config_json()
1080 if not self.bot_config['run']:
1081 print('Bot exiting by request.')
1082 exit(0)
1083 time.sleep(self.bot_config['delay'])
1084 if self.get_script_text() != self.script_text:
1085 print('Script changed, bot re-execing.')
1086 self.exec_self()
1089 class Config(object):
1090 """A configuration for building a compiler and associated libraries."""
1092 def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1093 first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1094 """Initialize a Config object."""
1095 self.ctx = ctx
1096 self.arch = arch
1097 self.os = os_name
1098 self.variant = variant
1099 if variant is None:
1100 self.name = '%s-%s' % (arch, os_name)
1101 else:
1102 self.name = '%s-%s-%s' % (arch, os_name, variant)
1103 self.triplet = '%s-glibc-%s' % (arch, os_name)
1104 if gcc_cfg is None:
1105 self.gcc_cfg = []
1106 else:
1107 self.gcc_cfg = gcc_cfg
1108 if first_gcc_cfg is None:
1109 self.first_gcc_cfg = []
1110 else:
1111 self.first_gcc_cfg = first_gcc_cfg
1112 if glibcs is None:
1113 glibcs = [{'variant': variant}]
1114 if extra_glibcs is None:
1115 extra_glibcs = []
1116 glibcs = [Glibc(self, **g) for g in glibcs]
1117 extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1118 self.all_glibcs = glibcs + extra_glibcs
1119 self.compiler_glibcs = glibcs
1120 self.installdir = ctx.compiler_installdir(self.name)
1121 self.bindir = ctx.compiler_bindir(self.name)
1122 self.sysroot = ctx.compiler_sysroot(self.name)
1123 self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1124 self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1126 def component_builddir(self, component):
1127 """Return the directory to use for a (non-glibc) build."""
1128 return self.ctx.component_builddir('compilers', self.name, component)
1130 def build(self):
1131 """Generate commands to build this compiler."""
1132 self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1133 self.logsdir)
1134 cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1135 cmdlist.add_command('check-host-libraries',
1136 ['test', '-f',
1137 os.path.join(self.ctx.host_libraries_installdir,
1138 'ok')])
1139 cmdlist.use_path(self.bindir)
1140 self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1141 ['--disable-gdb',
1142 '--disable-libdecnumber',
1143 '--disable-readline',
1144 '--disable-sim'])
1145 if self.os.startswith('linux'):
1146 self.install_linux_headers(cmdlist)
1147 self.build_gcc(cmdlist, True)
1148 for g in self.compiler_glibcs:
1149 cmdlist.push_subdesc('glibc')
1150 cmdlist.push_subdesc(g.name)
1151 g.build_glibc(cmdlist, True)
1152 cmdlist.pop_subdesc()
1153 cmdlist.pop_subdesc()
1154 self.build_gcc(cmdlist, False)
1155 cmdlist.add_command('done', ['touch',
1156 os.path.join(self.installdir, 'ok')])
1157 self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1158 self.logsdir)
1160 def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1161 """Build one cross tool."""
1162 srcdir = self.ctx.component_srcdir(tool_src)
1163 builddir = self.component_builddir(tool_build)
1164 cmdlist.push_subdesc(tool_build)
1165 cmdlist.create_use_dir(builddir)
1166 cfg_cmd = [os.path.join(srcdir, 'configure'),
1167 '--prefix=%s' % self.installdir,
1168 '--build=%s' % self.ctx.build_triplet,
1169 '--host=%s' % self.ctx.build_triplet,
1170 '--target=%s' % self.triplet,
1171 '--with-sysroot=%s' % self.sysroot]
1172 if extra_opts:
1173 cfg_cmd.extend(extra_opts)
1174 cmdlist.add_command('configure', cfg_cmd)
1175 cmdlist.add_command('build', ['make'])
1176 cmdlist.add_command('install', ['make', 'install'])
1177 cmdlist.cleanup_dir()
1178 cmdlist.pop_subdesc()
1180 def install_linux_headers(self, cmdlist):
1181 """Install Linux kernel headers."""
1182 arch_map = {'aarch64': 'arm64',
1183 'alpha': 'alpha',
1184 'arm': 'arm',
1185 'hppa': 'parisc',
1186 'i486': 'x86',
1187 'i586': 'x86',
1188 'i686': 'x86',
1189 'i786': 'x86',
1190 'ia64': 'ia64',
1191 'm68k': 'm68k',
1192 'microblaze': 'microblaze',
1193 'mips': 'mips',
1194 'nios2': 'nios2',
1195 'powerpc': 'powerpc',
1196 's390': 's390',
1197 'sh': 'sh',
1198 'sparc': 'sparc',
1199 'tile': 'tile',
1200 'x86_64': 'x86'}
1201 linux_arch = None
1202 for k in arch_map:
1203 if self.arch.startswith(k):
1204 linux_arch = arch_map[k]
1205 break
1206 assert linux_arch is not None
1207 srcdir = self.ctx.component_srcdir('linux')
1208 builddir = self.component_builddir('linux')
1209 headers_dir = os.path.join(self.sysroot, 'usr')
1210 cmdlist.push_subdesc('linux')
1211 cmdlist.create_use_dir(builddir)
1212 cmdlist.add_command('install-headers',
1213 ['make', '-C', srcdir, 'O=%s' % builddir,
1214 'ARCH=%s' % linux_arch,
1215 'INSTALL_HDR_PATH=%s' % headers_dir,
1216 'headers_install'])
1217 cmdlist.cleanup_dir()
1218 cmdlist.pop_subdesc()
1220 def build_gcc(self, cmdlist, bootstrap):
1221 """Build GCC."""
1222 # libsanitizer commonly breaks because of glibc header
1223 # changes, or on unusual targets. libssp is of little
1224 # relevance with glibc's own stack checking support.
1225 cfg_opts = list(self.gcc_cfg)
1226 cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1227 host_libs = self.ctx.host_libraries_installdir
1228 cfg_opts += ['--with-gmp=%s' % host_libs,
1229 '--with-mpfr=%s' % host_libs,
1230 '--with-mpc=%s' % host_libs]
1231 if bootstrap:
1232 tool_build = 'gcc-first'
1233 # Building a static-only, C-only compiler that is
1234 # sufficient to build glibc. Various libraries and
1235 # features that may require libc headers must be disabled.
1236 # When configuring with a sysroot, --with-newlib is
1237 # required to define inhibit_libc (to stop some parts of
1238 # libgcc including libc headers); --without-headers is not
1239 # sufficient.
1240 cfg_opts += ['--enable-languages=c', '--disable-shared',
1241 '--disable-threads',
1242 '--disable-libatomic',
1243 '--disable-decimal-float',
1244 '--disable-libffi',
1245 '--disable-libgomp',
1246 '--disable-libitm',
1247 '--disable-libmpx',
1248 '--disable-libquadmath',
1249 '--without-headers', '--with-newlib',
1250 '--with-glibc-version=%s' % self.ctx.glibc_version
1252 cfg_opts += self.first_gcc_cfg
1253 else:
1254 tool_build = 'gcc'
1255 cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1256 '--enable-threads']
1257 self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1260 class Glibc(object):
1261 """A configuration for building glibc."""
1263 def __init__(self, compiler, arch=None, os_name=None, variant=None,
1264 cfg=None, ccopts=None):
1265 """Initialize a Glibc object."""
1266 self.ctx = compiler.ctx
1267 self.compiler = compiler
1268 if arch is None:
1269 self.arch = compiler.arch
1270 else:
1271 self.arch = arch
1272 if os_name is None:
1273 self.os = compiler.os
1274 else:
1275 self.os = os_name
1276 self.variant = variant
1277 if variant is None:
1278 self.name = '%s-%s' % (self.arch, self.os)
1279 else:
1280 self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1281 self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1282 if cfg is None:
1283 self.cfg = []
1284 else:
1285 self.cfg = cfg
1286 self.ccopts = ccopts
1288 def tool_name(self, tool):
1289 """Return the name of a cross-compilation tool."""
1290 ctool = '%s-%s' % (self.compiler.triplet, tool)
1291 if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1292 ctool = '%s %s' % (ctool, self.ccopts)
1293 return ctool
1295 def build(self):
1296 """Generate commands to build this glibc."""
1297 builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1298 installdir = self.ctx.glibc_installdir(self.name)
1299 logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1300 self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1301 cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1302 cmdlist.add_command('check-compilers',
1303 ['test', '-f',
1304 os.path.join(self.compiler.installdir, 'ok')])
1305 cmdlist.use_path(self.compiler.bindir)
1306 self.build_glibc(cmdlist, False)
1307 self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1308 logsdir)
1310 def build_glibc(self, cmdlist, for_compiler):
1311 """Generate commands to build this glibc, either as part of a compiler
1312 build or with the bootstrapped compiler (and in the latter case, run
1313 tests as well)."""
1314 srcdir = self.ctx.component_srcdir('glibc')
1315 if for_compiler:
1316 builddir = self.ctx.component_builddir('compilers',
1317 self.compiler.name, 'glibc',
1318 self.name)
1319 installdir = self.compiler.sysroot
1320 srcdir_copy = self.ctx.component_builddir('compilers',
1321 self.compiler.name,
1322 'glibc-src',
1323 self.name)
1324 else:
1325 builddir = self.ctx.component_builddir('glibcs', self.name,
1326 'glibc')
1327 installdir = self.ctx.glibc_installdir(self.name)
1328 srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1329 'glibc-src')
1330 cmdlist.create_use_dir(builddir)
1331 # glibc builds write into the source directory, and even if
1332 # not intentionally there is a risk of bugs that involve
1333 # writing into the working directory. To avoid possible
1334 # concurrency issues, copy the source directory.
1335 cmdlist.create_copy_dir(srcdir, srcdir_copy)
1336 cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1337 '--prefix=/usr',
1338 '--enable-add-ons',
1339 '--build=%s' % self.ctx.build_triplet,
1340 '--host=%s' % self.triplet,
1341 'CC=%s' % self.tool_name('gcc'),
1342 'CXX=%s' % self.tool_name('g++'),
1343 'AR=%s' % self.tool_name('ar'),
1344 'AS=%s' % self.tool_name('as'),
1345 'LD=%s' % self.tool_name('ld'),
1346 'NM=%s' % self.tool_name('nm'),
1347 'OBJCOPY=%s' % self.tool_name('objcopy'),
1348 'OBJDUMP=%s' % self.tool_name('objdump'),
1349 'RANLIB=%s' % self.tool_name('ranlib'),
1350 'READELF=%s' % self.tool_name('readelf'),
1351 'STRIP=%s' % self.tool_name('strip')]
1352 cfg_cmd += self.cfg
1353 cmdlist.add_command('configure', cfg_cmd)
1354 cmdlist.add_command('build', ['make'])
1355 cmdlist.add_command('install', ['make', 'install',
1356 'install_root=%s' % installdir])
1357 # GCC uses paths such as lib/../lib64, so make sure lib
1358 # directories always exist.
1359 cmdlist.add_command('mkdir-lib', ['mkdir', '-p',
1360 os.path.join(installdir, 'lib'),
1361 os.path.join(installdir,
1362 'usr', 'lib')])
1363 if not for_compiler:
1364 if self.ctx.strip:
1365 cmdlist.add_command('strip',
1366 ['sh', '-c',
1367 ('%s %s/lib*/*.so' %
1368 (self.tool_name('strip'), installdir))])
1369 cmdlist.add_command('check', ['make', 'check'])
1370 cmdlist.add_command('save-logs', [self.ctx.save_logs],
1371 always_run=True)
1372 cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1373 cmdlist.cleanup_dir()
1376 class Command(object):
1377 """A command run in the build process."""
1379 def __init__(self, desc, num, dir, path, command, always_run=False):
1380 """Initialize a Command object."""
1381 self.dir = dir
1382 self.path = path
1383 self.desc = desc
1384 trans = str.maketrans({' ': '-'})
1385 self.logbase = '%03d-%s' % (num, desc.translate(trans))
1386 self.command = command
1387 self.always_run = always_run
1389 @staticmethod
1390 def shell_make_quote_string(s):
1391 """Given a string not containing a newline, quote it for use by the
1392 shell and make."""
1393 assert '\n' not in s
1394 if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1395 return s
1396 strans = str.maketrans({"'": "'\\''"})
1397 s = "'%s'" % s.translate(strans)
1398 mtrans = str.maketrans({'$': '$$'})
1399 return s.translate(mtrans)
1401 @staticmethod
1402 def shell_make_quote_list(l, translate_make):
1403 """Given a list of strings not containing newlines, quote them for use
1404 by the shell and make, returning a single string. If translate_make
1405 is true and the first string is 'make', change it to $(MAKE)."""
1406 l = [Command.shell_make_quote_string(s) for s in l]
1407 if translate_make and l[0] == 'make':
1408 l[0] = '$(MAKE)'
1409 return ' '.join(l)
1411 def shell_make_quote(self):
1412 """Return this command quoted for the shell and make."""
1413 return self.shell_make_quote_list(self.command, True)
1416 class CommandList(object):
1417 """A list of commands run in the build process."""
1419 def __init__(self, desc, keep):
1420 """Initialize a CommandList object."""
1421 self.cmdlist = []
1422 self.dir = None
1423 self.path = None
1424 self.desc = [desc]
1425 self.keep = keep
1427 def desc_txt(self, desc):
1428 """Return the description to use for a command."""
1429 return '%s %s' % (' '.join(self.desc), desc)
1431 def use_dir(self, dir):
1432 """Set the default directory for subsequent commands."""
1433 self.dir = dir
1435 def use_path(self, path):
1436 """Set a directory to be prepended to the PATH for subsequent
1437 commands."""
1438 self.path = path
1440 def push_subdesc(self, subdesc):
1441 """Set the default subdescription for subsequent commands (e.g., the
1442 name of a component being built, within the series of commands
1443 building it)."""
1444 self.desc.append(subdesc)
1446 def pop_subdesc(self):
1447 """Pop a subdescription from the list of descriptions."""
1448 self.desc.pop()
1450 def create_use_dir(self, dir):
1451 """Remove and recreate a directory and use it for subsequent
1452 commands."""
1453 self.add_command_dir('rm', None, ['rm', '-rf', dir])
1454 self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1455 self.use_dir(dir)
1457 def create_copy_dir(self, src, dest):
1458 """Remove a directory and recreate it as a copy from the given
1459 source."""
1460 self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1461 parent = os.path.dirname(dest)
1462 self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1463 self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1465 def add_command_dir(self, desc, dir, command, always_run=False):
1466 """Add a command to run in a given directory."""
1467 cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1468 command, always_run)
1469 self.cmdlist.append(cmd)
1471 def add_command(self, desc, command, always_run=False):
1472 """Add a command to run in the default directory."""
1473 cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1474 self.path, command, always_run)
1475 self.cmdlist.append(cmd)
1477 def cleanup_dir(self, desc='cleanup', dir=None):
1478 """Clean up a build directory. If no directory is specified, the
1479 default directory is cleaned up and ceases to be the default
1480 directory."""
1481 if dir is None:
1482 dir = self.dir
1483 self.use_dir(None)
1484 if self.keep != 'all':
1485 self.add_command_dir(desc, None, ['rm', '-rf', dir],
1486 always_run=(self.keep == 'none'))
1488 def makefile_commands(self, wrapper, logsdir):
1489 """Return the sequence of commands in the form of text for a Makefile.
1490 The given wrapper script takes arguments: base of logs for
1491 previous command, or empty; base of logs for this command;
1492 description; directory; PATH addition; the command itself."""
1493 # prev_base is the base of the name for logs of the previous
1494 # command that is not always-run (that is, a build command,
1495 # whose failure should stop subsequent build commands from
1496 # being run, as opposed to a cleanup command, which is run
1497 # even if previous commands failed).
1498 prev_base = ''
1499 cmds = []
1500 for c in self.cmdlist:
1501 ctxt = c.shell_make_quote()
1502 if prev_base and not c.always_run:
1503 prev_log = os.path.join(logsdir, prev_base)
1504 else:
1505 prev_log = ''
1506 this_log = os.path.join(logsdir, c.logbase)
1507 if not c.always_run:
1508 prev_base = c.logbase
1509 if c.dir is None:
1510 dir = ''
1511 else:
1512 dir = c.dir
1513 if c.path is None:
1514 path = ''
1515 else:
1516 path = c.path
1517 prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1518 prelim_txt = Command.shell_make_quote_list(prelims, False)
1519 cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1520 return '\n'.join(cmds)
1522 def status_logs(self, logsdir):
1523 """Return the list of log files with command status."""
1524 return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1525 for c in self.cmdlist]
1528 def get_parser():
1529 """Return an argument parser for this module."""
1530 parser = argparse.ArgumentParser(description=__doc__)
1531 parser.add_argument('-j', dest='parallelism',
1532 help='Run this number of jobs in parallel',
1533 type=int, default=os.cpu_count())
1534 parser.add_argument('--keep', dest='keep',
1535 help='Whether to keep all build directories, '
1536 'none or only those from failed builds',
1537 default='none', choices=('none', 'all', 'failed'))
1538 parser.add_argument('--replace-sources', action='store_true',
1539 help='Remove and replace source directories '
1540 'with the wrong version of a component')
1541 parser.add_argument('--strip', action='store_true',
1542 help='Strip installed glibc libraries')
1543 parser.add_argument('topdir',
1544 help='Toplevel working directory')
1545 parser.add_argument('action',
1546 help='What to do',
1547 choices=('checkout', 'bot-cycle', 'bot',
1548 'host-libraries', 'compilers', 'glibcs'))
1549 parser.add_argument('configs',
1550 help='Versions to check out or configurations to build',
1551 nargs='*')
1552 return parser
1555 def main(argv):
1556 """The main entry point."""
1557 parser = get_parser()
1558 opts = parser.parse_args(argv)
1559 topdir = os.path.abspath(opts.topdir)
1560 ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1561 opts.strip, opts.action)
1562 ctx.run_builds(opts.action, opts.configs)
1565 if __name__ == '__main__':
1566 main(sys.argv[1:])