2 # This Source Code Form is subject to the terms of the Mozilla Public
3 # License, v. 2.0. If a copy of the MPL was not distributed with this
4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
17 from contextlib
import contextmanager
20 from distutils
.dir_util
import copy_tree
23 def symlink(source
, link_name
):
24 os_symlink
= getattr(os
, "symlink", None)
25 if callable(os_symlink
):
26 os_symlink(source
, link_name
)
28 if os
.path
.isdir(source
):
29 # Fall back to copying the directory :(
30 copy_tree(source
, link_name
)
34 print >> sys
.stderr
, ' '.join(args
)
35 p
= subprocess
.Popen(args
, stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
36 # CMake `message(STATUS)` messages, as appearing in failed source code
37 # compiles, appear on stdout, so we only capture that.
38 (out
, _
) = p
.communicate()
41 cmake_output_re
= re
.compile("See also \"(.*/CMakeOutput.log)\"")
42 cmake_error_re
= re
.compile("See also \"(.*/CMakeError.log)\"")
43 output_match
= cmake_output_re
.search(out
)
44 error_match
= cmake_error_re
.search(out
)
46 print >> sys
.stderr
, out
49 with
open(log
, 'rb') as f
:
50 print >> sys
.stderr
, "\nContents of", log
, "follow\n"
51 print >> sys
.stderr
, f
.read()
53 dump_file(output_match
.group(1))
55 dump_file(error_match
.group(1))
59 def run_in(path
, args
):
61 print >> sys
.stderr
, 'cd "%s"' % path
64 print >> sys
.stderr
, 'cd "%s"' % d
68 def patch(patch
, srcdir
):
69 patch
= os
.path
.realpath(patch
)
70 check_run(['patch', '-d', srcdir
, '-p1', '-i', patch
, '--fuzz=0',
74 def import_clang_tidy(source_dir
):
75 clang_plugin_path
= os
.path
.join(os
.path
.dirname(sys
.argv
[0]),
77 clang_tidy_path
= os
.path
.join(source_dir
,
78 'tools/clang/tools/extra/clang-tidy')
79 sys
.path
.append(clang_plugin_path
)
80 from import_mozilla_checks
import do_import
81 do_import(clang_plugin_path
, clang_tidy_path
)
84 def build_package(package_build_dir
, cmake_args
):
85 if not os
.path
.exists(package_build_dir
):
86 os
.mkdir(package_build_dir
)
87 # If CMake has already been run, it may have been run with different
88 # arguments, so we need to re-run it. Make sure the cached copy of the
89 # previous CMake run is cleared before running it again.
90 if os
.path
.exists(package_build_dir
+ "/CMakeCache.txt"):
91 os
.remove(package_build_dir
+ "/CMakeCache.txt")
92 if os
.path
.exists(package_build_dir
+ "/CMakeFiles"):
93 shutil
.rmtree(package_build_dir
+ "/CMakeFiles")
95 run_in(package_build_dir
, ["cmake"] + cmake_args
)
96 run_in(package_build_dir
, ["ninja", "install"])
100 def updated_env(env
):
101 old_env
= os
.environ
.copy()
102 os
.environ
.update(env
)
105 os
.environ
.update(old_env
)
108 def build_tar_package(tar
, name
, base
, directory
):
109 name
= os
.path
.realpath(name
)
110 # On Windows, we have to convert this into an msys path so that tar can
113 name
= name
.replace('\\', '/')
116 return '/' + match
.group(1).lower()
117 name
= re
.sub(r
'^([A-Za-z]):', f
, name
)
120 "-%s" % ("J" if ".xz" in name
else "j"),
129 if e
.errno
!= errno
.EEXIST
or not os
.path
.isdir(path
):
134 if os
.path
.isdir(path
):
143 def install_libgcc(gcc_dir
, clang_dir
, is_final_stage
):
144 gcc_bin_dir
= os
.path
.join(gcc_dir
, 'bin')
146 # Copy over gcc toolchain bits that clang looks for, to ensure that
147 # clang is using a consistent version of ld, since the system ld may
148 # be incompatible with the output clang produces. But copy it to a
149 # target-specific directory so a cross-compiler to Mac doesn't pick
150 # up the (Linux-specific) ld with disastrous results.
152 # Only install this for the bootstrap process; we expect any consumers of
153 # the newly-built toolchain to provide an appropriate ld themselves.
154 if not is_final_stage
:
155 x64_bin_dir
= os
.path
.join(clang_dir
, 'x86_64-unknown-linux-gnu', 'bin')
157 shutil
.copy2(os
.path
.join(gcc_bin_dir
, 'ld'), x64_bin_dir
)
159 out
= subprocess
.check_output([os
.path
.join(gcc_bin_dir
, "gcc"),
160 '-print-libgcc-file-name'])
162 libgcc_dir
= os
.path
.dirname(out
.rstrip())
163 clang_lib_dir
= os
.path
.join(clang_dir
, "lib", "gcc",
164 "x86_64-unknown-linux-gnu",
165 os
.path
.basename(libgcc_dir
))
166 mkdir_p(clang_lib_dir
)
167 copy_tree(libgcc_dir
, clang_lib_dir
)
168 libgcc_dir
= os
.path
.join(gcc_dir
, "lib64")
169 clang_lib_dir
= os
.path
.join(clang_dir
, "lib")
170 copy_tree(libgcc_dir
, clang_lib_dir
)
171 libgcc_dir
= os
.path
.join(gcc_dir
, "lib32")
172 clang_lib_dir
= os
.path
.join(clang_dir
, "lib32")
173 copy_tree(libgcc_dir
, clang_lib_dir
)
174 include_dir
= os
.path
.join(gcc_dir
, "include")
175 clang_include_dir
= os
.path
.join(clang_dir
, "include")
176 copy_tree(include_dir
, clang_include_dir
)
179 def install_import_library(build_dir
, clang_dir
):
180 shutil
.copy2(os
.path
.join(build_dir
, "lib", "clang.lib"),
181 os
.path
.join(clang_dir
, "lib"))
184 def install_asan_symbols(build_dir
, clang_dir
):
185 lib_path_pattern
= os
.path
.join("lib", "clang", "*.*.*", "lib", "windows")
186 src_path
= glob
.glob(os
.path
.join(build_dir
, lib_path_pattern
,
187 "clang_rt.asan_dynamic-*.pdb"))
188 dst_path
= glob
.glob(os
.path
.join(clang_dir
, lib_path_pattern
))
190 if len(src_path
) != 1:
191 raise Exception("Source path pattern did not resolve uniquely")
193 if len(src_path
) != 1:
194 raise Exception("Destination path pattern did not resolve uniquely")
196 shutil
.copy2(src_path
[0], dst_path
[0])
199 def svn_co(source_dir
, url
, directory
, revision
):
200 run_in(source_dir
, ["svn", "co", "-q", "-r", revision
, url
, directory
])
203 def svn_update(directory
, revision
):
204 run_in(directory
, ["svn", "revert", "-q", "-R", "."])
205 run_in(directory
, ["svn", "update", "-q", "-r", revision
])
209 return platform
.system() == "Darwin"
213 return platform
.system() == "Linux"
217 return platform
.system() == "Windows"
220 def build_one_stage(cc
, cxx
, asm
, ld
, ar
, ranlib
, libtool
,
221 src_dir
, stage_dir
, package_name
, build_libcxx
,
222 osx_cross_compile
, build_type
, assertions
,
223 python_path
, gcc_dir
, libcxx_include_dir
,
224 compiler_rt_source_dir
=None, runtimes_source_link
=None,
225 compiler_rt_source_link
=None,
226 is_final_stage
=False, android_targets
=None,
228 if is_final_stage
and (android_targets
or extra_targets
):
229 # Linking compiler-rt under "runtimes" activates LLVM_RUNTIME_TARGETS
230 # and related arguments.
231 symlink(compiler_rt_source_dir
, runtimes_source_link
)
233 os
.unlink(compiler_rt_source_link
)
237 if not os
.path
.exists(stage_dir
):
240 build_dir
= stage_dir
+ "/build"
241 inst_dir
= stage_dir
+ "/" + package_name
243 # cmake doesn't deal well with backslashes in paths.
244 def slashify_path(path
):
245 return path
.replace('\\', '/')
247 def cmake_base_args(cc
, cxx
, asm
, ld
, ar
, ranlib
, libtool
, inst_dir
):
250 "-DCMAKE_C_COMPILER=%s" % slashify_path(cc
[0]),
251 "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx
[0]),
252 "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm
[0]),
253 "-DCMAKE_LINKER=%s" % slashify_path(ld
[0]),
254 "-DCMAKE_AR=%s" % slashify_path(ar
),
255 "-DCMAKE_C_FLAGS=%s" % ' '.join(cc
[1:]),
256 "-DCMAKE_CXX_FLAGS=%s" % ' '.join(cxx
[1:]),
257 "-DCMAKE_ASM_FLAGS=%s" % ' '.join(asm
[1:]),
258 "-DCMAKE_EXE_LINKER_FLAGS=%s" % ' '.join(ld
[1:]),
259 "-DCMAKE_SHARED_LINKER_FLAGS=%s" % ' '.join(ld
[1:]),
260 "-DCMAKE_BUILD_TYPE=%s" % build_type
,
261 "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir
,
262 "-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64",
263 "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions
else "OFF"),
264 "-DPYTHON_EXECUTABLE=%s" % slashify_path(python_path
),
265 "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx
else "OFF"),
266 "-DLIBCXX_LIBCPPABI_VERSION=\"\"",
269 cmake_args
+= ["-DLLVM_BINUTILS_INCDIR=%s/include" % gcc_dir
]
271 cmake_args
.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
272 cmake_args
.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
274 # libllvm as a shared library is not supported on Windows
275 cmake_args
+= ["-DLLVM_LINK_LLVM_DYLIB=ON"]
276 if ranlib
is not None:
277 cmake_args
+= ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib
)]
278 if libtool
is not None:
279 cmake_args
+= ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool
)]
280 if osx_cross_compile
:
282 "-DCMAKE_SYSTEM_NAME=Darwin",
283 "-DCMAKE_SYSTEM_VERSION=10.10",
284 "-DLLVM_ENABLE_THREADS=OFF",
285 # Xray requires a OSX 10.12 SDK (https://bugs.llvm.org/show_bug.cgi?id=38959)
286 "-DCOMPILER_RT_BUILD_XRAY=OFF",
287 "-DLIBCXXABI_LIBCXX_INCLUDES=%s" % libcxx_include_dir
,
288 "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os
.getenv("CROSS_SYSROOT")),
289 "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os
.getenv("CROSS_CCTOOLS_PATH")), # noqa
290 "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
291 "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
292 "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
293 "-DCMAKE_MACOSX_RPATH=ON",
294 "-DCMAKE_OSX_ARCHITECTURES=x86_64",
295 "-DDARWIN_osx_ARCHS=x86_64",
296 "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os
.getenv("CROSS_SYSROOT")),
297 "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin"
306 runtime_targets
= list(sorted(android_targets
.keys()))
308 runtime_targets
.extend(sorted(extra_targets
))
312 "-DLLVM_BUILTIN_TARGETS=%s" % ";".join(runtime_targets
),
313 "-DLLVM_RUNTIME_TARGETS=%s" % ";".join(runtime_targets
),
316 for target
in runtime_targets
:
318 "-DRUNTIMES_%s_COMPILER_RT_BUILD_PROFILE=ON" % target
,
319 "-DRUNTIMES_%s_COMPILER_RT_BUILD_SANITIZERS=ON" % target
,
320 "-DRUNTIMES_%s_COMPILER_RT_BUILD_XRAY=OFF" % target
,
321 "-DRUNTIMES_%s_SANITIZER_ALLOW_CXXABI=OFF" % target
,
322 "-DRUNTIMES_%s_COMPILER_RT_BUILD_LIBFUZZER=OFF" % target
,
323 "-DRUNTIMES_%s_COMPILER_RT_INCLUDE_TESTS=OFF" % target
,
324 "-DRUNTIMES_%s_LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" % target
,
325 "-DRUNTIMES_%s_LLVM_INCLUDE_TESTS=OFF" % target
,
328 # The above code flipped switches to build various runtime libraries on
329 # Android; we now have to provide all the necessary compiler switches to
331 if is_final_stage
and android_targets
:
333 "-DLLVM_LIBDIR_SUFFIX=64",
336 android_link_flags
= "-fuse-ld=lld"
338 for target
, cfg
in android_targets
.iteritems():
339 sysroot_dir
= cfg
["ndk_sysroot"]
340 android_gcc_dir
= cfg
["ndk_toolchain"]
341 android_include_dirs
= cfg
["ndk_includes"]
342 api_level
= cfg
["api_level"]
344 android_flags
= ["-isystem %s" % d
for d
in android_include_dirs
]
345 android_flags
+= ["--gcc-toolchain=%s" % android_gcc_dir
]
346 android_flags
+= ["-D__ANDROID_API__=%s" % api_level
]
348 # Our flags go last to override any --gcc-toolchain that may have
350 rt_c_flags
= " ".join(cc
[1:] + android_flags
)
351 rt_cxx_flags
= " ".join(cxx
[1:] + android_flags
)
352 rt_asm_flags
= " ".join(asm
[1:] + android_flags
)
354 for kind
in ('BUILTINS', 'RUNTIMES'):
357 ('CMAKE_ASM_FLAGS', rt_asm_flags
),
358 ('CMAKE_CXX_FLAGS', rt_cxx_flags
),
359 ('CMAKE_C_FLAGS', rt_c_flags
),
360 ('CMAKE_EXE_LINKER_FLAGS', android_link_flags
),
361 ('CMAKE_SHARED_LINKER_FLAGS', android_link_flags
),
362 ('CMAKE_SYSROOT', sysroot_dir
),
363 ('ANDROID_NATIVE_API_LEVEL', api_level
),
365 cmake_args
+= ['-D%s_%s_%s=%s' % (kind
, target
, var
, arg
)]
367 cmake_args
+= cmake_base_args(
368 cc
, cxx
, asm
, ld
, ar
, ranlib
, libtool
, inst_dir
)
372 build_package(build_dir
, cmake_args
)
375 install_libgcc(gcc_dir
, inst_dir
, is_final_stage
)
376 # For some reasons the import library clang.lib of clang.exe is not
377 # installed, so we copy it by ourselves.
379 # The compiler-rt cmake scripts don't allow to build it for multiple
380 # targets at once on Windows, so manually build the 32-bits compiler-rt
381 # during the final stage.
384 # Only build the 32-bits compiler-rt when we originally built for
385 # 64-bits, which we detect through the contents of the LIB
386 # environment variable, which we also adjust for a 32-bits build
388 old_lib
= os
.environ
['LIB']
390 for l
in old_lib
.split(os
.pathsep
):
391 if l
.endswith('x64'):
394 elif l
.endswith('amd64'):
399 os
.environ
['LIB'] = os
.pathsep
.join(new_lib
)
400 compiler_rt_build_dir
= stage_dir
+ '/compiler-rt'
401 compiler_rt_inst_dir
= inst_dir
+ '/lib/clang/'
402 subdirs
= os
.listdir(compiler_rt_inst_dir
)
403 assert len(subdirs
) == 1
404 compiler_rt_inst_dir
+= subdirs
[0]
405 cmake_args
= cmake_base_args(
406 [os
.path
.join(inst_dir
, 'bin', 'clang-cl.exe'), '-m32'] + cc
[1:],
407 [os
.path
.join(inst_dir
, 'bin', 'clang-cl.exe'), '-m32'] + cxx
[1:],
408 [os
.path
.join(inst_dir
, 'bin', 'clang-cl.exe'), '-m32'] + asm
[1:],
409 ld
, ar
, ranlib
, libtool
, compiler_rt_inst_dir
)
411 '-DLLVM_CONFIG_PATH=%s' % slashify_path(
412 os
.path
.join(inst_dir
, 'bin', 'llvm-config')),
413 os
.path
.join(src_dir
, 'projects', 'compiler-rt'),
415 build_package(compiler_rt_build_dir
, cmake_args
)
416 os
.environ
['LIB'] = old_lib
417 install_import_library(build_dir
, inst_dir
)
418 install_asan_symbols(build_dir
, inst_dir
)
421 # Return the absolute path of a build tool. We first look to see if the
422 # variable is defined in the config file, and if so we make sure it's an
423 # absolute path to an existing tool, otherwise we look for a program in
426 # This expects the name of the key in the config file to match the name of
427 # the tool in the default toolchain on the system (for example, "ld" on Unix
428 # and "link" on Windows).
429 def get_tool(config
, key
):
434 if not os
.path
.exists(f
):
435 raise ValueError("%s must point to an existing path" % key
)
438 # Assume that we have the name of some program that should be on PATH.
440 return which
.which(f
) if f
else which
.which(key
)
441 except which
.WhichError
:
442 raise ValueError("%s not found on PATH" % f
)
445 # This function is intended to be called on the final build directory when
446 # building clang-tidy. Also clang-format binaries are included that can be used
447 # in conjunction with clang-tidy.
448 # Its job is to remove all of the files which won't be used for clang-tidy or
449 # clang-format to reduce the download size. Currently when this function
450 # finishes its job, it will leave final_dir with a layout like this:
454 # clang-apply-replacements
458 # * (nothing will be deleted here)
463 # * (nothing will be deleted here)
466 # clang-format-diff.py
469 def prune_final_dir_for_clang_tidy(final_dir
, osx_cross_compile
):
470 # Make sure we only have what we expect.
471 dirs
= ["bin", "include", "lib", "lib32", "libexec", "msbuild-bin", "share", "tools"]
473 dirs
.append("x86_64-unknown-linux-gnu")
474 for f
in glob
.glob("%s/*" % final_dir
):
475 if os
.path
.basename(f
) not in dirs
:
476 raise Exception("Found unknown file %s in the final directory" % f
)
477 if not os
.path
.isdir(f
):
478 raise Exception("Expected %s to be a directory" % f
)
480 # In bin/, only keep clang-tidy and clang-apply-replacements. The last one
481 # is used to auto-fix some of the issues detected by clang-tidy.
482 re_clang_tidy
= re
.compile(
483 r
"^clang-(apply-replacements|format|tidy)(\.exe)?$", re
.I
)
484 for f
in glob
.glob("%s/bin/*" % final_dir
):
485 if re_clang_tidy
.search(os
.path
.basename(f
)) is None:
488 # Keep include/ intact.
490 # Remove the target-specific files.
492 shutil
.rmtree(os
.path
.join(final_dir
, "x86_64-unknown-linux-gnu"))
494 # In lib/, only keep lib/clang/N.M.O/include and the LLVM shared library.
495 re_ver_num
= re
.compile(r
"^\d+\.\d+\.\d+$", re
.I
)
496 for f
in glob
.glob("%s/lib/*" % final_dir
):
497 name
= os
.path
.basename(f
)
500 if osx_cross_compile
and name
== 'libLLVM.dylib':
502 if is_linux() and fnmatch
.fnmatch(name
, 'libLLVM*.so'):
505 for f
in glob
.glob("%s/lib/clang/*" % final_dir
):
506 if re_ver_num
.search(os
.path
.basename(f
)) is None:
508 for f
in glob
.glob("%s/lib/clang/*/*" % final_dir
):
509 if os
.path
.basename(f
) != "include":
512 # Completely remove libexec/, msbuilld-bin and tools, if it exists.
513 shutil
.rmtree(os
.path
.join(final_dir
, "libexec"))
514 for d
in ("msbuild-bin", "tools"):
515 d
= os
.path
.join(final_dir
, d
)
516 if os
.path
.exists(d
):
519 # In share/, only keep share/clang/*tidy*
520 re_clang_tidy
= re
.compile(r
"format|tidy", re
.I
)
521 for f
in glob
.glob("%s/share/*" % final_dir
):
522 if os
.path
.basename(f
) != "clang":
524 for f
in glob
.glob("%s/share/clang/*" % final_dir
):
525 if re_clang_tidy
.search(os
.path
.basename(f
)) is None:
529 if __name__
== "__main__":
530 parser
= argparse
.ArgumentParser()
531 parser
.add_argument('-c', '--config', required
=True,
532 type=argparse
.FileType('r'),
533 help="Clang configuration file")
534 parser
.add_argument('-b', '--base-dir', required
=False,
535 help="Base directory for code and build artifacts")
536 parser
.add_argument('--clean', required
=False,
538 help="Clean the build directory")
539 parser
.add_argument('--skip-tar', required
=False,
541 help="Skip tar packaging stage")
542 parser
.add_argument('--skip-checkout', required
=False,
544 help="Do not checkout/revert source")
546 args
= parser
.parse_args()
548 # The directories end up in the debug info, so the easy way of getting
549 # a reproducible build is to run it in a know absolute directory.
550 # We use a directory that is registered as a volume in the Docker image.
553 base_dir
= args
.base_dir
554 elif os
.environ
.get('MOZ_AUTOMATION') and not is_windows():
555 base_dir
= "/builds/worker/workspace/moz-toolchain"
557 # Handles both the Windows automation case and the local build case
558 # TODO: Because Windows taskcluster builds are run with distinct
559 # user IDs for each job, we can't store things in some globally
560 # accessible directory: one job will run, checkout LLVM to that
561 # directory, and then if another job runs, the new user won't be
562 # able to access the previously-checked out code--or be able to
563 # delete it. So on Windows, we build in the task-specific home
564 # directory; we will eventually add -fdebug-prefix-map options
565 # to the LLVM build to bring back reproducibility.
566 base_dir
= os
.path
.join(os
.getcwd(), 'build-clang')
568 source_dir
= base_dir
+ "/src"
569 build_dir
= base_dir
+ "/build"
571 if not os
.path
.exists(base_dir
):
572 os
.makedirs(base_dir
)
573 elif os
.listdir(base_dir
) and not os
.path
.exists(os
.path
.join(base_dir
, '.build-clang')):
574 raise ValueError("Base directory %s exists and is not a build-clang directory. "
575 "Supply a non-existent or empty directory with --base-dir" % base_dir
)
576 open(os
.path
.join(base_dir
, '.build-clang'), 'a').close()
579 shutil
.rmtree(build_dir
)
582 llvm_source_dir
= source_dir
+ "/llvm"
583 clang_source_dir
= source_dir
+ "/clang"
584 extra_source_dir
= source_dir
+ "/extra"
585 lld_source_dir
= source_dir
+ "/lld"
586 compiler_rt_source_dir
= source_dir
+ "/compiler-rt"
587 libcxx_source_dir
= source_dir
+ "/libcxx"
588 libcxxabi_source_dir
= source_dir
+ "/libcxxabi"
591 os
.environ
['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
601 cxx_name
= "clang-cl"
603 config
= json
.load(args
.config
)
605 llvm_revision
= config
["llvm_revision"]
606 llvm_repo
= config
["llvm_repo"]
607 clang_repo
= config
["clang_repo"]
608 extra_repo
= config
.get("extra_repo")
609 lld_repo
= config
.get("lld_repo")
610 # On some packages we don't use compiler_repo
611 compiler_repo
= config
.get("compiler_repo")
612 libcxx_repo
= config
["libcxx_repo"]
613 libcxxabi_repo
= config
.get("libcxxabi_repo")
615 if "stages" in config
:
616 stages
= int(config
["stages"])
617 if stages
not in (1, 2, 3):
618 raise ValueError("We only know how to build 1, 2, or 3 stages")
619 build_type
= "Release"
620 if "build_type" in config
:
621 build_type
= config
["build_type"]
622 if build_type
not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"):
623 raise ValueError("We only know how to do Release, Debug, RelWithDebInfo or "
626 if "build_libcxx" in config
:
627 build_libcxx
= config
["build_libcxx"]
628 if build_libcxx
not in (True, False):
629 raise ValueError("Only boolean values are accepted for build_libcxx.")
630 build_clang_tidy
= False
631 if "build_clang_tidy" in config
:
632 build_clang_tidy
= config
["build_clang_tidy"]
633 if build_clang_tidy
not in (True, False):
634 raise ValueError("Only boolean values are accepted for build_clang_tidy.")
635 osx_cross_compile
= False
636 if "osx_cross_compile" in config
:
637 osx_cross_compile
= config
["osx_cross_compile"]
638 if osx_cross_compile
not in (True, False):
639 raise ValueError("Only boolean values are accepted for osx_cross_compile.")
640 if osx_cross_compile
and not is_linux():
641 raise ValueError("osx_cross_compile can only be used on Linux.")
643 if "assertions" in config
:
644 assertions
= config
["assertions"]
645 if assertions
not in (True, False):
646 raise ValueError("Only boolean values are accepted for assertions.")
648 if "python_path" not in config
:
649 raise ValueError("Config file needs to set python_path")
650 python_path
= config
["python_path"]
652 if "gcc_dir" in config
:
653 gcc_dir
= config
["gcc_dir"]
654 if not os
.path
.exists(gcc_dir
):
655 raise ValueError("gcc_dir must point to an existing path")
657 android_targets
= None
658 if "android_targets" in config
:
659 android_targets
= config
["android_targets"]
660 for attr
in ("ndk_toolchain", "ndk_sysroot", "ndk_includes", "api_level"):
661 for target
, cfg
in android_targets
.iteritems():
663 raise ValueError("must specify '%s' as a key for android target: %s" %
666 if "extra_targets" in config
:
667 extra_targets
= config
["extra_targets"]
668 if not isinstance(extra_targets
, list):
669 raise ValueError("extra_targets must be a list")
670 if not all(isinstance(t
, (str, unicode)) for t
in extra_targets
):
671 raise ValueError("members of extra_targets should be strings")
672 if is_linux() and gcc_dir
is None:
673 raise ValueError("Config file needs to set gcc_dir")
674 cc
= get_tool(config
, "cc")
675 cxx
= get_tool(config
, "cxx")
676 asm
= get_tool(config
, "ml" if is_windows() else "as")
677 ld
= get_tool(config
, "link" if is_windows() else "ld")
678 ar
= get_tool(config
, "lib" if is_windows() else "ar")
679 ranlib
= None if is_windows() else get_tool(config
, "ranlib")
681 if "libtool" in config
:
682 libtool
= get_tool(config
, "libtool")
684 if not os
.path
.exists(source_dir
):
685 os
.makedirs(source_dir
)
687 def checkout_or_update(repo
, checkout_dir
):
688 if os
.path
.exists(checkout_dir
):
689 svn_update(checkout_dir
, llvm_revision
)
691 svn_co(source_dir
, repo
, checkout_dir
, llvm_revision
)
693 if not args
.skip_checkout
:
694 checkout_or_update(llvm_repo
, llvm_source_dir
)
695 checkout_or_update(clang_repo
, clang_source_dir
)
696 if compiler_repo
is not None:
697 checkout_or_update(compiler_repo
, compiler_rt_source_dir
)
698 checkout_or_update(libcxx_repo
, libcxx_source_dir
)
700 checkout_or_update(lld_repo
, lld_source_dir
)
702 checkout_or_update(libcxxabi_repo
, libcxxabi_source_dir
)
704 checkout_or_update(extra_repo
, extra_source_dir
)
705 for p
in config
.get("patches", []):
708 compiler_rt_source_link
= llvm_source_dir
+ "/projects/compiler-rt"
710 symlinks
= [(clang_source_dir
,
711 llvm_source_dir
+ "/tools/clang"),
713 llvm_source_dir
+ "/tools/clang/tools/extra"),
715 llvm_source_dir
+ "/tools/lld"),
716 (compiler_rt_source_dir
, compiler_rt_source_link
),
718 llvm_source_dir
+ "/projects/libcxx"),
719 (libcxxabi_source_dir
,
720 llvm_source_dir
+ "/projects/libcxxabi")]
722 # On Windows, we have to re-copy the whole directory every time.
723 if not is_windows() and os
.path
.islink(l
[1]):
726 if os
.path
.exists(l
[0]):
729 package_name
= "clang"
731 package_name
= "clang-tidy"
732 import_clang_tidy(llvm_source_dir
)
734 if not os
.path
.exists(build_dir
):
735 os
.makedirs(build_dir
)
737 libcxx_include_dir
= os
.path
.join(llvm_source_dir
, "projects",
740 stage1_dir
= build_dir
+ '/stage1'
741 stage1_inst_dir
= stage1_dir
+ '/' + package_name
743 final_stage_dir
= stage1_dir
747 extra_cxxflags
= ["-stdlib=libc++"]
749 extra_cxxflags2
= ["-stdlib=libc++"]
755 # When building stage2 and stage3, we want the newly-built clang to pick
756 # up whatever headers were installed from the gcc we used to build stage1,
757 # always, rather than the system headers. Providing -gcc-toolchain
758 # encourages clang to do that.
759 extra_cflags2
= ["-fPIC", '-gcc-toolchain', stage1_inst_dir
]
760 # Silence clang's warnings about arguments not being used in compilation.
761 extra_cxxflags2
= ["-fPIC", '-Qunused-arguments', '-gcc-toolchain', stage1_inst_dir
]
763 # Avoid libLLVM internal function calls going through the PLT.
764 extra_ldflags
= ['-Wl,-Bsymbolic-functions']
766 if 'LD_LIBRARY_PATH' in os
.environ
:
767 os
.environ
['LD_LIBRARY_PATH'] = ('%s/lib64/:%s' %
768 (gcc_dir
, os
.environ
['LD_LIBRARY_PATH']))
770 os
.environ
['LD_LIBRARY_PATH'] = '%s/lib64/' % gcc_dir
774 # clang-cl would like to figure out what it's supposed to be emulating
775 # by looking at an MSVC install, but we don't really have that here.
778 extra_cxxflags2
= ['-fms-compatibility-version=19.13.26128', '-Xclang', '-std=c++14']
782 if osx_cross_compile
:
783 # undo the damage done in the is_linux() block above, and also simulate
784 # the is_darwin() block above.
786 extra_cxxflags
= ["-stdlib=libc++"]
787 extra_cxxflags2
= ["-stdlib=libc++"]
789 extra_flags
= ["-target", "x86_64-apple-darwin", "-mlinker-version=137",
790 "-B", "%s/bin" % os
.getenv("CROSS_CCTOOLS_PATH"),
791 "-isysroot", os
.getenv("CROSS_SYSROOT"),
792 # technically the sysroot flag there should be enough to deduce this,
793 # but clang needs some help to figure this out.
794 "-I%s/usr/include" % os
.getenv("CROSS_SYSROOT"),
795 "-iframework", "%s/System/Library/Frameworks" % os
.getenv("CROSS_SYSROOT")]
796 extra_cflags
+= extra_flags
797 extra_cxxflags
+= extra_flags
798 extra_cflags2
+= extra_flags
799 extra_cxxflags2
+= extra_flags
800 extra_asmflags
+= extra_flags
801 extra_ldflags
= ["-Wl,-syslibroot,%s" % os
.getenv("CROSS_SYSROOT"),
806 [cxx
] + extra_cxxflags
,
807 [asm
] + extra_asmflags
,
808 [ld
] + extra_ldflags
,
810 llvm_source_dir
, stage1_dir
, package_name
, build_libcxx
, osx_cross_compile
,
811 build_type
, assertions
, python_path
, gcc_dir
, libcxx_include_dir
)
813 runtimes_source_link
= llvm_source_dir
+ "/runtimes/compiler-rt"
816 stage2_dir
= build_dir
+ '/stage2'
817 stage2_inst_dir
= stage2_dir
+ '/' + package_name
818 final_stage_dir
= stage2_dir
820 [stage1_inst_dir
+ "/bin/%s%s" %
821 (cc_name
, exe_ext
)] + extra_cflags2
,
822 [stage1_inst_dir
+ "/bin/%s%s" %
823 (cxx_name
, exe_ext
)] + extra_cxxflags2
,
824 [stage1_inst_dir
+ "/bin/%s%s" %
825 (cc_name
, exe_ext
)] + extra_asmflags
,
826 [ld
] + extra_ldflags
,
828 llvm_source_dir
, stage2_dir
, package_name
, build_libcxx
, osx_cross_compile
,
829 build_type
, assertions
, python_path
, gcc_dir
, libcxx_include_dir
,
830 compiler_rt_source_dir
, runtimes_source_link
, compiler_rt_source_link
,
831 is_final_stage
=(stages
== 2), android_targets
=android_targets
,
832 extra_targets
=extra_targets
)
835 stage3_dir
= build_dir
+ '/stage3'
836 final_stage_dir
= stage3_dir
838 [stage2_inst_dir
+ "/bin/%s%s" %
839 (cc_name
, exe_ext
)] + extra_cflags2
,
840 [stage2_inst_dir
+ "/bin/%s%s" %
841 (cxx_name
, exe_ext
)] + extra_cxxflags2
,
842 [stage2_inst_dir
+ "/bin/%s%s" %
843 (cc_name
, exe_ext
)] + extra_asmflags
,
844 [ld
] + extra_ldflags
,
846 llvm_source_dir
, stage3_dir
, package_name
, build_libcxx
, osx_cross_compile
,
847 build_type
, assertions
, python_path
, gcc_dir
, libcxx_include_dir
,
848 compiler_rt_source_dir
, runtimes_source_link
, compiler_rt_source_link
,
849 (stages
== 3), extra_targets
=extra_targets
)
852 prune_final_dir_for_clang_tidy(os
.path
.join(final_stage_dir
, package_name
),
855 if not args
.skip_tar
:
856 ext
= "bz2" if is_darwin() or is_windows() else "xz"
857 build_tar_package("tar", "%s.tar.%s" % (package_name
, ext
), final_stage_dir
, package_name
)