build/build-clang/build-clang.py

   1 #!/usr/bin/python3
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 # Only necessary for flake8 to be happy...
   7 from __future__ import print_function
   8
   9 import os
  10 import os.path
  11 import shutil
  12 import subprocess
  13 import platform
  14 import json
  15 import argparse
  16 import fnmatch
  17 import glob
  18 import errno
  19 import re
  20 import sys
  21 import tarfile
  22 from contextlib import contextmanager
  23 from distutils.dir_util import copy_tree
  24
  25 from shutil import which
  26
  27 import zstandard
  28
  29
  30 def symlink(source, link_name):
  31     os_symlink = getattr(os, "symlink", None)
  32     if callable(os_symlink):
  33         os_symlink(source, link_name)
  34     else:
  35         if os.path.isdir(source):
  36             # Fall back to copying the directory :(
  37             copy_tree(source, link_name)
  38
  39
  40 def check_run(args):
  41     print(" ".join(args), file=sys.stderr, flush=True)
  42     if args[0] == "cmake":
  43         # CMake `message(STATUS)` messages, as appearing in failed source code
  44         # compiles, appear on stdout, so we only capture that.
  45         p = subprocess.Popen(args, stdout=subprocess.PIPE)
  46         lines = []
  47         for line in p.stdout:
  48             lines.append(line)
  49             sys.stdout.write(line.decode())
  50             sys.stdout.flush()
  51         r = p.wait()
  52         if r != 0:
  53             cmake_output_re = re.compile(b'See also "(.*/CMakeOutput.log)"')
  54             cmake_error_re = re.compile(b'See also "(.*/CMakeError.log)"')
  55
  56             def find_first_match(re):
  57                 for l in lines:
  58                     match = re.search(l)
  59                     if match:
  60                         return match
  61
  62             output_match = find_first_match(cmake_output_re)
  63             error_match = find_first_match(cmake_error_re)
  64
  65             def dump_file(log):
  66                 with open(log, "r", errors="replace") as f:
  67                     print("\nContents of", log, "follow\n", file=sys.stderr)
  68                     for line in f:
  69                         print(line, file=sys.stderr)
  70
  71             if output_match:
  72                 dump_file(output_match.group(1))
  73             if error_match:
  74                 dump_file(error_match.group(1))
  75     else:
  76         r = subprocess.call(args)
  77     assert r == 0
  78
  79
  80 def run_in(path, args):
  81     with chdir(path):
  82         check_run(args)
  83
  84
  85 @contextmanager
  86 def chdir(path):
  87     d = os.getcwd()
  88     print('cd "%s"' % path, file=sys.stderr)
  89     os.chdir(path)
  90     try:
  91         yield
  92     finally:
  93         print('cd "%s"' % d, file=sys.stderr)
  94         os.chdir(d)
  95
  96
  97 def patch(patch, srcdir):
  98     patch = os.path.realpath(patch)
  99     check_run(["patch", "-d", srcdir, "-p1", "-i", patch, "--fuzz=0", "-s"])
 100
 101
 102 def import_clang_tidy(source_dir, build_clang_tidy_alpha, build_clang_tidy_external):
 103     clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]), "..", "clang-plugin")
 104     clang_tidy_path = os.path.join(source_dir, "clang-tools-extra/clang-tidy")
 105     sys.path.append(clang_plugin_path)
 106     from import_mozilla_checks import do_import
 107
 108     import_options = {
 109         "alpha": build_clang_tidy_alpha,
 110         "external": build_clang_tidy_external,
 111     }
 112     do_import(clang_plugin_path, clang_tidy_path, import_options)
 113
 114
 115 def build_package(package_build_dir, cmake_args):
 116     if not os.path.exists(package_build_dir):
 117         os.mkdir(package_build_dir)
 118     # If CMake has already been run, it may have been run with different
 119     # arguments, so we need to re-run it.  Make sure the cached copy of the
 120     # previous CMake run is cleared before running it again.
 121     if os.path.exists(package_build_dir + "/CMakeCache.txt"):
 122         os.remove(package_build_dir + "/CMakeCache.txt")
 123     if os.path.exists(package_build_dir + "/CMakeFiles"):
 124         shutil.rmtree(package_build_dir + "/CMakeFiles")
 125
 126     run_in(package_build_dir, ["cmake"] + cmake_args)
 127     run_in(package_build_dir, ["ninja", "install", "-v"])
 128
 129
 130 @contextmanager
 131 def updated_env(env):
 132     old_env = os.environ.copy()
 133     os.environ.update(env)
 134     yield
 135     os.environ.clear()
 136     os.environ.update(old_env)
 137
 138
 139 def build_tar_package(name, base, directory):
 140     name = os.path.realpath(name)
 141     print("tarring {} from {}/{}".format(name, base, directory), file=sys.stderr)
 142     assert name.endswith(".tar.zst")
 143
 144     cctx = zstandard.ZstdCompressor()
 145     with open(name, "wb") as f, cctx.stream_writer(f) as z:
 146         with tarfile.open(mode="w|", fileobj=z) as tf:
 147             with chdir(base):
 148                 tf.add(directory)
 149
 150
 151 def mkdir_p(path):
 152     try:
 153         os.makedirs(path)
 154     except OSError as e:
 155         if e.errno != errno.EEXIST or not os.path.isdir(path):
 156             raise
 157
 158
 159 def delete(path):
 160     if os.path.isdir(path):
 161         shutil.rmtree(path)
 162     else:
 163         try:
 164             os.unlink(path)
 165         except Exception:
 166             pass
 167
 168
 169 def install_import_library(build_dir, clang_dir):
 170     shutil.copy2(
 171         os.path.join(build_dir, "lib", "clang.lib"), os.path.join(clang_dir, "lib")
 172     )
 173
 174
 175 def install_asan_symbols(build_dir, clang_dir):
 176     lib_path_pattern = os.path.join("lib", "clang", "*.*.*", "lib", "windows")
 177     src_path = glob.glob(
 178         os.path.join(build_dir, lib_path_pattern, "clang_rt.asan_dynamic-*.pdb")
 179     )
 180     dst_path = glob.glob(os.path.join(clang_dir, lib_path_pattern))
 181
 182     if len(src_path) != 1:
 183         raise Exception("Source path pattern did not resolve uniquely")
 184
 185     if len(src_path) != 1:
 186         raise Exception("Destination path pattern did not resolve uniquely")
 187
 188     shutil.copy2(src_path[0], dst_path[0])
 189
 190
 191 def is_darwin():
 192     return platform.system() == "Darwin"
 193
 194
 195 def is_linux():
 196     return platform.system() == "Linux"
 197
 198
 199 def is_windows():
 200     return platform.system() == "Windows"
 201
 202
 203 def build_one_stage(
 204     cc,
 205     cxx,
 206     asm,
 207     ld,
 208     ar,
 209     ranlib,
 210     libtool,
 211     src_dir,
 212     stage_dir,
 213     package_name,
 214     build_libcxx,
 215     osx_cross_compile,
 216     build_type,
 217     assertions,
 218     libcxx_include_dir,
 219     build_wasm,
 220     compiler_rt_source_dir=None,
 221     runtimes_source_link=None,
 222     compiler_rt_source_link=None,
 223     is_final_stage=False,
 224     android_targets=None,
 225     extra_targets=None,
 226     pgo_phase=None,
 227 ):
 228     if is_final_stage and (android_targets or extra_targets):
 229         # Linking compiler-rt under "runtimes" activates LLVM_RUNTIME_TARGETS
 230         # and related arguments.
 231         symlink(compiler_rt_source_dir, runtimes_source_link)
 232         try:
 233             os.unlink(compiler_rt_source_link)
 234         except Exception:
 235             pass
 236
 237     if not os.path.exists(stage_dir):
 238         os.mkdir(stage_dir)
 239
 240     build_dir = stage_dir + "/build"
 241     inst_dir = stage_dir + "/" + package_name
 242
 243     # cmake doesn't deal well with backslashes in paths.
 244     def slashify_path(path):
 245         return path.replace("\\", "/")
 246
 247     def cmake_base_args(cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir):
 248         machine_targets = "X86;ARM;AArch64" if is_final_stage else "X86"
 249         cmake_args = [
 250             "-GNinja",
 251             "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]),
 252             "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]),
 253             "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]),
 254             "-DCMAKE_LINKER=%s" % slashify_path(ld[0]),
 255             "-DCMAKE_AR=%s" % slashify_path(ar),
 256             "-DCMAKE_C_FLAGS=%s" % " ".join(cc[1:]),
 257             "-DCMAKE_CXX_FLAGS=%s" % " ".join(cxx[1:]),
 258             "-DCMAKE_ASM_FLAGS=%s" % " ".join(asm[1:]),
 259             "-DCMAKE_EXE_LINKER_FLAGS=%s" % " ".join(ld[1:]),
 260             "-DCMAKE_SHARED_LINKER_FLAGS=%s" % " ".join(ld[1:]),
 261             "-DCMAKE_BUILD_TYPE=%s" % build_type,
 262             "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir,
 263             "-DLLVM_TARGETS_TO_BUILD=%s" % machine_targets,
 264             "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"),
 265             "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"),
 266             "-DLLVM_ENABLE_BINDINGS=OFF",
 267         ]
 268         if "TASK_ID" in os.environ:
 269             cmake_args += [
 270                 "-DCLANG_REPOSITORY_STRING=taskcluster-%s" % os.environ["TASK_ID"],
 271             ]
 272         if not is_final_stage:
 273             cmake_args += [
 274                 "-DLLVM_ENABLE_PROJECTS=clang;compiler-rt",
 275                 "-DLLVM_INCLUDE_TESTS=OFF",
 276                 "-DLLVM_TOOL_LLI_BUILD=OFF",
 277                 "-DCOMPILER_RT_BUILD_SANITIZERS=OFF",
 278                 "-DCOMPILER_RT_BUILD_XRAY=OFF",
 279                 "-DCOMPILER_RT_BUILD_MEMPROF=OFF",
 280                 "-DCOMPILER_RT_BUILD_LIBFUZZER=OFF",
 281             ]
 282         if build_wasm:
 283             cmake_args += ["-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly"]
 284         if is_linux() and not osx_cross_compile and is_final_stage:
 285             cmake_args += ["-DLLVM_BINUTILS_INCDIR=/usr/include"]
 286             cmake_args += ["-DLLVM_ENABLE_LIBXML2=FORCE_ON"]
 287             sysroot = os.path.join(os.environ.get("MOZ_FETCHES_DIR", ""), "sysroot")
 288             if os.path.exists(sysroot):
 289                 cmake_args += ["-DCMAKE_SYSROOT=%s" % sysroot]
 290                 # Work around the LLVM build system not building the i386 compiler-rt
 291                 # because it doesn't allow to use a sysroot for that during the cmake
 292                 # checks.
 293                 cmake_args += ["-DCAN_TARGET_i386=1"]
 294         if is_windows():
 295             cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
 296             cmake_args.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
 297         else:
 298             # libllvm as a shared library is not supported on Windows
 299             cmake_args += ["-DLLVM_LINK_LLVM_DYLIB=ON"]
 300         if ranlib is not None:
 301             cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
 302         if libtool is not None:
 303             cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
 304         if osx_cross_compile:
 305             arch = "arm64" if os.environ.get("OSX_ARCH") == "arm64" else "x86_64"
 306             target_cpu = (
 307                 "aarch64" if os.environ.get("OSX_ARCH") == "arm64" else "x86_64"
 308             )
 309             cmake_args += [
 310                 "-DCMAKE_SYSTEM_NAME=Darwin",
 311                 "-DCMAKE_SYSTEM_VERSION=%s" % os.environ["MACOSX_DEPLOYMENT_TARGET"],
 312                 "-DLIBCXXABI_LIBCXX_INCLUDES=%s" % libcxx_include_dir,
 313                 "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 314                 "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 315                 "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
 316                 "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
 317                 "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
 318                 "-DCMAKE_MACOSX_RPATH=ON",
 319                 "-DCMAKE_OSX_ARCHITECTURES=%s" % arch,
 320                 "-DDARWIN_osx_ARCHS=%s" % arch,
 321                 "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 322                 "-DLLVM_DEFAULT_TARGET_TRIPLE=%s-apple-darwin" % target_cpu,
 323             ]
 324             if os.environ.get("OSX_ARCH") == "arm64":
 325                 cmake_args += [
 326                     "-DDARWIN_osx_BUILTIN_ARCHS=arm64",
 327                 ]
 328             # Starting in LLVM 11 (which requires SDK 10.12) the build tries to
 329             # detect the SDK version by calling xcrun. Cross-compiles don't have
 330             # an xcrun, so we have to set the version explicitly.
 331             cmake_args += [
 332                 "-DDARWIN_macosx_OVERRIDE_SDK_VERSION=%s"
 333                 % os.environ["MACOSX_DEPLOYMENT_TARGET"],
 334             ]
 335         if pgo_phase == "gen":
 336             # Per https://releases.llvm.org/10.0.0/docs/HowToBuildWithPGO.html
 337             cmake_args += [
 338                 "-DLLVM_BUILD_INSTRUMENTED=IR",
 339                 "-DLLVM_BUILD_RUNTIME=No",
 340             ]
 341         if pgo_phase == "use":
 342             cmake_args += [
 343                 "-DLLVM_PROFDATA_FILE=%s/merged.profdata" % stage_dir,
 344             ]
 345         return cmake_args
 346
 347     cmake_args = []
 348
 349     runtime_targets = []
 350     if is_final_stage:
 351         if android_targets:
 352             runtime_targets = list(sorted(android_targets.keys()))
 353         if extra_targets:
 354             runtime_targets.extend(sorted(extra_targets))
 355
 356     if runtime_targets:
 357         cmake_args += [
 358             "-DLLVM_BUILTIN_TARGETS=%s" % ";".join(runtime_targets),
 359             "-DLLVM_RUNTIME_TARGETS=%s" % ";".join(runtime_targets),
 360         ]
 361
 362         for target in runtime_targets:
 363             cmake_args += [
 364                 "-DRUNTIMES_%s_COMPILER_RT_BUILD_PROFILE=ON" % target,
 365                 "-DRUNTIMES_%s_COMPILER_RT_BUILD_SANITIZERS=ON" % target,
 366                 "-DRUNTIMES_%s_COMPILER_RT_BUILD_XRAY=OFF" % target,
 367                 "-DRUNTIMES_%s_SANITIZER_ALLOW_CXXABI=OFF" % target,
 368                 "-DRUNTIMES_%s_COMPILER_RT_BUILD_LIBFUZZER=OFF" % target,
 369                 "-DRUNTIMES_%s_COMPILER_RT_INCLUDE_TESTS=OFF" % target,
 370                 "-DRUNTIMES_%s_LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" % target,
 371                 "-DRUNTIMES_%s_LLVM_INCLUDE_TESTS=OFF" % target,
 372             ]
 373
 374     # The above code flipped switches to build various runtime libraries on
 375     # Android; we now have to provide all the necessary compiler switches to
 376     # make that work.
 377     if is_final_stage and android_targets:
 378         android_link_flags = "-fuse-ld=lld"
 379
 380         for target, cfg in android_targets.items():
 381             sysroot_dir = cfg["ndk_sysroot"].format(**os.environ)
 382             android_gcc_dir = cfg["ndk_toolchain"].format(**os.environ)
 383             android_include_dirs = cfg["ndk_includes"]
 384             api_level = cfg["api_level"]
 385
 386             android_flags = [
 387                 "-isystem %s" % d.format(**os.environ) for d in android_include_dirs
 388             ]
 389             android_flags += ["--gcc-toolchain=%s" % android_gcc_dir]
 390             android_flags += ["-D__ANDROID_API__=%s" % api_level]
 391
 392             # Our flags go last to override any --gcc-toolchain that may have
 393             # been set earlier.
 394             rt_c_flags = " ".join(cc[1:] + android_flags)
 395             rt_cxx_flags = " ".join(cxx[1:] + android_flags)
 396             rt_asm_flags = " ".join(asm[1:] + android_flags)
 397
 398             for kind in ("BUILTINS", "RUNTIMES"):
 399                 for var, arg in (
 400                     ("ANDROID", "1"),
 401                     ("CMAKE_ASM_FLAGS", rt_asm_flags),
 402                     ("CMAKE_CXX_FLAGS", rt_cxx_flags),
 403                     ("CMAKE_C_FLAGS", rt_c_flags),
 404                     ("CMAKE_EXE_LINKER_FLAGS", android_link_flags),
 405                     ("CMAKE_SHARED_LINKER_FLAGS", android_link_flags),
 406                     ("CMAKE_SYSROOT", sysroot_dir),
 407                     ("ANDROID_NATIVE_API_LEVEL", api_level),
 408                 ):
 409                     cmake_args += ["-D%s_%s_%s=%s" % (kind, target, var, arg)]
 410
 411     cmake_args += cmake_base_args(cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir)
 412     cmake_args += [src_dir]
 413     build_package(build_dir, cmake_args)
 414
 415     # For some reasons the import library clang.lib of clang.exe is not
 416     # installed, so we copy it by ourselves.
 417     if is_windows():
 418         # The compiler-rt cmake scripts don't allow to build it for multiple
 419         # targets at once on Windows, so manually build the 32-bits compiler-rt
 420         # during the final stage.
 421         build_32_bit = False
 422         if is_final_stage:
 423             # Only build the 32-bits compiler-rt when we originally built for
 424             # 64-bits, which we detect through the contents of the LIB
 425             # environment variable, which we also adjust for a 32-bits build
 426             # at the same time.
 427             old_lib = os.environ["LIB"]
 428             new_lib = []
 429             for l in old_lib.split(os.pathsep):
 430                 if l.endswith("x64"):
 431                     l = l[:-3] + "x86"
 432                     build_32_bit = True
 433                 elif l.endswith("amd64"):
 434                     l = l[:-5]
 435                     build_32_bit = True
 436                 new_lib.append(l)
 437         if build_32_bit:
 438             os.environ["LIB"] = os.pathsep.join(new_lib)
 439             compiler_rt_build_dir = stage_dir + "/compiler-rt"
 440             compiler_rt_inst_dir = inst_dir + "/lib/clang/"
 441             subdirs = os.listdir(compiler_rt_inst_dir)
 442             assert len(subdirs) == 1
 443             compiler_rt_inst_dir += subdirs[0]
 444             cmake_args = cmake_base_args(
 445                 [os.path.join(inst_dir, "bin", "clang-cl.exe"), "-m32"] + cc[1:],
 446                 [os.path.join(inst_dir, "bin", "clang-cl.exe"), "-m32"] + cxx[1:],
 447                 [os.path.join(inst_dir, "bin", "clang-cl.exe"), "-m32"] + asm[1:],
 448                 ld,
 449                 ar,
 450                 ranlib,
 451                 libtool,
 452                 compiler_rt_inst_dir,
 453             )
 454             cmake_args += [
 455                 "-DLLVM_CONFIG_PATH=%s"
 456                 % slashify_path(os.path.join(inst_dir, "bin", "llvm-config")),
 457                 os.path.join(src_dir, "projects", "compiler-rt"),
 458             ]
 459             build_package(compiler_rt_build_dir, cmake_args)
 460             os.environ["LIB"] = old_lib
 461         if is_final_stage:
 462             install_import_library(build_dir, inst_dir)
 463             install_asan_symbols(build_dir, inst_dir)
 464
 465
 466 # Return the absolute path of a build tool.  We first look to see if the
 467 # variable is defined in the config file, and if so we make sure it's an
 468 # absolute path to an existing tool, otherwise we look for a program in
 469 # $PATH named "key".
 470 #
 471 # This expects the name of the key in the config file to match the name of
 472 # the tool in the default toolchain on the system (for example, "ld" on Unix
 473 # and "link" on Windows).
 474 def get_tool(config, key):
 475     f = None
 476     if key in config:
 477         f = config[key].format(**os.environ)
 478         if os.path.isabs(f):
 479             if not os.path.exists(f):
 480                 raise ValueError("%s must point to an existing path" % key)
 481             return f
 482
 483     # Assume that we have the name of some program that should be on PATH.
 484     tool = which(f) if f else which(key)
 485     if not tool:
 486         raise ValueError("%s not found on PATH" % (f or key))
 487     return tool
 488
 489
 490 # This function is intended to be called on the final build directory when
 491 # building clang-tidy. Also clang-format binaries are included that can be used
 492 # in conjunction with clang-tidy.
 493 # As a separate binary we also ship clangd for the language server protocol that
 494 # can be used as a plugin in `vscode`.
 495 # Its job is to remove all of the files which won't be used for clang-tidy or
 496 # clang-format to reduce the download size.  Currently when this function
 497 # finishes its job, it will leave final_dir with a layout like this:
 498 #
 499 # clang/
 500 #   bin/
 501 #     clang-apply-replacements
 502 #     clang-format
 503 #     clang-tidy
 504 #     clangd
 505 #   include/
 506 #     * (nothing will be deleted here)
 507 #   lib/
 508 #     clang/
 509 #       4.0.0/
 510 #         include/
 511 #           * (nothing will be deleted here)
 512 #   share/
 513 #     clang/
 514 #       clang-format-diff.py
 515 #       clang-tidy-diff.py
 516 #       run-clang-tidy.py
 517 def prune_final_dir_for_clang_tidy(final_dir, osx_cross_compile):
 518     # Make sure we only have what we expect.
 519     dirs = [
 520         "bin",
 521         "include",
 522         "lib",
 523         "lib32",
 524         "libexec",
 525         "msbuild-bin",
 526         "share",
 527         "tools",
 528     ]
 529     if is_linux():
 530         dirs.append("x86_64-unknown-linux-gnu")
 531     for f in glob.glob("%s/*" % final_dir):
 532         if os.path.basename(f) not in dirs:
 533             raise Exception("Found unknown file %s in the final directory" % f)
 534         if not os.path.isdir(f):
 535             raise Exception("Expected %s to be a directory" % f)
 536
 537     kept_binaries = [
 538         "clang-apply-replacements",
 539         "clang-format",
 540         "clang-tidy",
 541         "clangd",
 542         "clang-query",
 543     ]
 544     re_clang_tidy = re.compile(r"^(" + "|".join(kept_binaries) + r")(\.exe)?$", re.I)
 545     for f in glob.glob("%s/bin/*" % final_dir):
 546         if re_clang_tidy.search(os.path.basename(f)) is None:
 547             delete(f)
 548
 549     # Keep include/ intact.
 550
 551     # Remove the target-specific files.
 552     if is_linux():
 553         if os.path.exists(os.path.join(final_dir, "x86_64-unknown-linux-gnu")):
 554             shutil.rmtree(os.path.join(final_dir, "x86_64-unknown-linux-gnu"))
 555
 556     # In lib/, only keep lib/clang/N.M.O/include and the LLVM shared library.
 557     re_ver_num = re.compile(r"^\d+\.\d+\.\d+$", re.I)
 558     for f in glob.glob("%s/lib/*" % final_dir):
 559         name = os.path.basename(f)
 560         if name == "clang":
 561             continue
 562         if osx_cross_compile and name in ["libLLVM.dylib", "libclang-cpp.dylib"]:
 563             continue
 564         if is_linux() and (
 565             fnmatch.fnmatch(name, "libLLVM*.so")
 566             or fnmatch.fnmatch(name, "libclang-cpp.so*")
 567         ):
 568             continue
 569         delete(f)
 570     for f in glob.glob("%s/lib/clang/*" % final_dir):
 571         if re_ver_num.search(os.path.basename(f)) is None:
 572             delete(f)
 573     for f in glob.glob("%s/lib/clang/*/*" % final_dir):
 574         if os.path.basename(f) != "include":
 575             delete(f)
 576
 577     # Completely remove libexec/, msbuild-bin and tools, if it exists.
 578     shutil.rmtree(os.path.join(final_dir, "libexec"))
 579     for d in ("msbuild-bin", "tools"):
 580         d = os.path.join(final_dir, d)
 581         if os.path.exists(d):
 582             shutil.rmtree(d)
 583
 584     # In share/, only keep share/clang/*tidy*
 585     re_clang_tidy = re.compile(r"format|tidy", re.I)
 586     for f in glob.glob("%s/share/*" % final_dir):
 587         if os.path.basename(f) != "clang":
 588             delete(f)
 589     for f in glob.glob("%s/share/clang/*" % final_dir):
 590         if re_clang_tidy.search(os.path.basename(f)) is None:
 591             delete(f)
 592
 593
 594 if __name__ == "__main__":
 595     parser = argparse.ArgumentParser()
 596     parser.add_argument(
 597         "-c",
 598         "--config",
 599         required=True,
 600         type=argparse.FileType("r"),
 601         help="Clang configuration file",
 602     )
 603     parser.add_argument(
 604         "--clean", required=False, action="store_true", help="Clean the build directory"
 605     )
 606     parser.add_argument(
 607         "--skip-tar",
 608         required=False,
 609         action="store_true",
 610         help="Skip tar packaging stage",
 611     )
 612     parser.add_argument(
 613         "--skip-patch",
 614         required=False,
 615         action="store_true",
 616         help="Do not patch source",
 617     )
 618
 619     args = parser.parse_args()
 620
 621     if not os.path.exists("llvm/README.txt"):
 622         raise Exception(
 623             "The script must be run from the root directory of the llvm-project tree"
 624         )
 625     source_dir = os.getcwd()
 626     build_dir = source_dir + "/build"
 627
 628     if args.clean:
 629         shutil.rmtree(build_dir)
 630         os.sys.exit(0)
 631
 632     llvm_source_dir = source_dir + "/llvm"
 633     extra_source_dir = source_dir + "/clang-tools-extra"
 634     clang_source_dir = source_dir + "/clang"
 635     lld_source_dir = source_dir + "/lld"
 636     compiler_rt_source_dir = source_dir + "/compiler-rt"
 637     libcxx_source_dir = source_dir + "/libcxx"
 638     libcxxabi_source_dir = source_dir + "/libcxxabi"
 639
 640     exe_ext = ""
 641     if is_windows():
 642         exe_ext = ".exe"
 643
 644     cc_name = "clang"
 645     cxx_name = "clang++"
 646     if is_windows():
 647         cc_name = "clang-cl"
 648         cxx_name = "clang-cl"
 649
 650     config_dir = os.path.dirname(args.config.name)
 651     config = json.load(args.config)
 652
 653     stages = 3
 654     if "stages" in config:
 655         stages = int(config["stages"])
 656         if stages not in (1, 2, 3, 4):
 657             raise ValueError("We only know how to build 1, 2, 3, or 4 stages.")
 658     pgo = False
 659     if "pgo" in config:
 660         pgo = config["pgo"]
 661         if pgo not in (True, False):
 662             raise ValueError("Only boolean values are accepted for pgo.")
 663         if pgo and stages != 4:
 664             raise ValueError("PGO is only supported in 4-stage builds.")
 665     build_type = "Release"
 666     if "build_type" in config:
 667         build_type = config["build_type"]
 668         if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"):
 669             raise ValueError(
 670                 "We only know how to do Release, Debug, RelWithDebInfo or "
 671                 "MinSizeRel builds"
 672             )
 673     build_libcxx = False
 674     if "build_libcxx" in config:
 675         build_libcxx = config["build_libcxx"]
 676         if build_libcxx not in (True, False):
 677             raise ValueError("Only boolean values are accepted for build_libcxx.")
 678     build_wasm = False
 679     if "build_wasm" in config:
 680         build_wasm = config["build_wasm"]
 681         if build_wasm not in (True, False):
 682             raise ValueError("Only boolean values are accepted for build_wasm.")
 683     build_clang_tidy = False
 684     if "build_clang_tidy" in config:
 685         build_clang_tidy = config["build_clang_tidy"]
 686         if build_clang_tidy not in (True, False):
 687             raise ValueError("Only boolean values are accepted for build_clang_tidy.")
 688     build_clang_tidy_alpha = False
 689     # check for build_clang_tidy_alpha only if build_clang_tidy is true
 690     if build_clang_tidy and "build_clang_tidy_alpha" in config:
 691         build_clang_tidy_alpha = config["build_clang_tidy_alpha"]
 692         if build_clang_tidy_alpha not in (True, False):
 693             raise ValueError(
 694                 "Only boolean values are accepted for build_clang_tidy_alpha."
 695             )
 696     build_clang_tidy_external = False
 697     # check for build_clang_tidy_external only if build_clang_tidy is true
 698     if build_clang_tidy and "build_clang_tidy_external" in config:
 699         build_clang_tidy_external = config["build_clang_tidy_external"]
 700         if build_clang_tidy_external not in (True, False):
 701             raise ValueError(
 702                 "Only boolean values are accepted for build_clang_tidy_external."
 703             )
 704     osx_cross_compile = False
 705     if "osx_cross_compile" in config:
 706         osx_cross_compile = config["osx_cross_compile"]
 707         if osx_cross_compile not in (True, False):
 708             raise ValueError("Only boolean values are accepted for osx_cross_compile.")
 709         if osx_cross_compile and not is_linux():
 710             raise ValueError("osx_cross_compile can only be used on Linux.")
 711     assertions = False
 712     if "assertions" in config:
 713         assertions = config["assertions"]
 714         if assertions not in (True, False):
 715             raise ValueError("Only boolean values are accepted for assertions.")
 716     ndk_dir = None
 717     android_targets = None
 718     if "android_targets" in config:
 719         android_targets = config["android_targets"]
 720         for attr in ("ndk_toolchain", "ndk_sysroot", "ndk_includes", "api_level"):
 721             for target, cfg in android_targets.items():
 722                 if attr not in cfg:
 723                     raise ValueError(
 724                         "must specify '%s' as a key for android target: %s"
 725                         % (attr, target)
 726                     )
 727     extra_targets = None
 728     if "extra_targets" in config:
 729         extra_targets = config["extra_targets"]
 730         if not isinstance(extra_targets, list):
 731             raise ValueError("extra_targets must be a list")
 732         if not all(isinstance(t, str) for t in extra_targets):
 733             raise ValueError("members of extra_targets should be strings")
 734
 735     if is_darwin() or osx_cross_compile:
 736         os.environ["MACOSX_DEPLOYMENT_TARGET"] = (
 737             "11.0" if os.environ.get("OSX_ARCH") == "arm64" else "10.12"
 738         )
 739
 740     cc = get_tool(config, "cc")
 741     cxx = get_tool(config, "cxx")
 742     asm = get_tool(config, "ml" if is_windows() else "as")
 743     ld = get_tool(config, "link" if is_windows() else "ld")
 744     ar = get_tool(config, "lib" if is_windows() else "ar")
 745     ranlib = None if is_windows() else get_tool(config, "ranlib")
 746     libtool = None
 747     if "libtool" in config:
 748         libtool = get_tool(config, "libtool")
 749
 750     if not os.path.exists(source_dir):
 751         os.makedirs(source_dir)
 752
 753     if not args.skip_patch:
 754         for p in config.get("patches", []):
 755             patch(os.path.join(config_dir, p), source_dir)
 756
 757     compiler_rt_source_link = llvm_source_dir + "/projects/compiler-rt"
 758
 759     symlinks = [
 760         (clang_source_dir, llvm_source_dir + "/tools/clang"),
 761         (extra_source_dir, llvm_source_dir + "/tools/clang/tools/extra"),
 762         (lld_source_dir, llvm_source_dir + "/tools/lld"),
 763         (compiler_rt_source_dir, compiler_rt_source_link),
 764         (libcxx_source_dir, llvm_source_dir + "/projects/libcxx"),
 765         (libcxxabi_source_dir, llvm_source_dir + "/projects/libcxxabi"),
 766     ]
 767     for l in symlinks:
 768         # On Windows, we have to re-copy the whole directory every time.
 769         if not is_windows() and os.path.islink(l[1]):
 770             continue
 771         delete(l[1])
 772         if os.path.exists(l[0]):
 773             symlink(l[0], l[1])
 774
 775     package_name = "clang"
 776     if build_clang_tidy:
 777         package_name = "clang-tidy"
 778         if not args.skip_patch:
 779             import_clang_tidy(
 780                 source_dir, build_clang_tidy_alpha, build_clang_tidy_external
 781             )
 782
 783     if not os.path.exists(build_dir):
 784         os.makedirs(build_dir)
 785
 786     libcxx_include_dir = os.path.join(llvm_source_dir, "projects", "libcxx", "include")
 787
 788     stage1_dir = build_dir + "/stage1"
 789     stage1_inst_dir = stage1_dir + "/" + package_name
 790
 791     final_stage_dir = stage1_dir
 792     final_inst_dir = stage1_inst_dir
 793
 794     if is_darwin():
 795         extra_cflags = []
 796         extra_cxxflags = ["-stdlib=libc++"]
 797         extra_cflags2 = []
 798         extra_cxxflags2 = ["-stdlib=libc++"]
 799         extra_asmflags = []
 800         extra_ldflags = []
 801     elif is_linux():
 802         extra_cflags = []
 803         extra_cxxflags = []
 804         extra_cflags2 = ["-fPIC"]
 805         # Silence clang's warnings about arguments not being used in compilation.
 806         extra_cxxflags2 = [
 807             "-fPIC",
 808             "-Qunused-arguments",
 809         ]
 810         extra_asmflags = []
 811         # Avoid libLLVM internal function calls going through the PLT.
 812         extra_ldflags = ["-Wl,-Bsymbolic-functions"]
 813         # For whatever reason, LLVM's build system will set things up to turn
 814         # on -ffunction-sections and -fdata-sections, but won't turn on the
 815         # corresponding option to strip unused sections.  We do it explicitly
 816         # here.  LLVM's build system is also picky about turning on ICF, so
 817         # we do that explicitly here, too.
 818         extra_ldflags += ["-fuse-ld=gold", "-Wl,--gc-sections", "-Wl,--icf=safe"]
 819     elif is_windows():
 820         extra_cflags = []
 821         extra_cxxflags = []
 822         # clang-cl would like to figure out what it's supposed to be emulating
 823         # by looking at an MSVC install, but we don't really have that here.
 824         # Force things on.
 825         extra_cflags2 = []
 826         extra_cxxflags2 = [
 827             "-fms-compatibility-version=19.15.26726",
 828             "-Xclang",
 829             "-std=c++14",
 830         ]
 831         extra_asmflags = []
 832         extra_ldflags = []
 833
 834     if osx_cross_compile:
 835         # undo the damage done in the is_linux() block above, and also simulate
 836         # the is_darwin() block above.
 837         extra_cflags = []
 838         extra_cxxflags = ["-stdlib=libc++"]
 839         extra_cxxflags2 = ["-stdlib=libc++"]
 840
 841         extra_flags = [
 842             "-target",
 843             "x86_64-apple-darwin",
 844             "-mlinker-version=137",
 845             "-B",
 846             "%s/bin" % os.getenv("CROSS_CCTOOLS_PATH"),
 847             "-isysroot",
 848             os.getenv("CROSS_SYSROOT"),
 849             # technically the sysroot flag there should be enough to deduce this,
 850             # but clang needs some help to figure this out.
 851             "-I%s/usr/include" % os.getenv("CROSS_SYSROOT"),
 852             "-iframework",
 853             "%s/System/Library/Frameworks" % os.getenv("CROSS_SYSROOT"),
 854         ]
 855         extra_cflags += extra_flags
 856         extra_cxxflags += extra_flags
 857         extra_cflags2 += extra_flags
 858         extra_cxxflags2 += extra_flags
 859         extra_asmflags += extra_flags
 860         extra_ldflags = [
 861             "-Wl,-syslibroot,%s" % os.getenv("CROSS_SYSROOT"),
 862             "-Wl,-dead_strip",
 863         ]
 864
 865     upload_dir = os.getenv("UPLOAD_DIR")
 866     if assertions and upload_dir:
 867         extra_cflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir]
 868         extra_cxxflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir]
 869
 870     build_one_stage(
 871         [cc] + extra_cflags,
 872         [cxx] + extra_cxxflags,
 873         [asm] + extra_asmflags,
 874         [ld] + extra_ldflags,
 875         ar,
 876         ranlib,
 877         libtool,
 878         llvm_source_dir,
 879         stage1_dir,
 880         package_name,
 881         build_libcxx,
 882         osx_cross_compile,
 883         build_type,
 884         assertions,
 885         libcxx_include_dir,
 886         build_wasm,
 887         is_final_stage=(stages == 1),
 888     )
 889
 890     runtimes_source_link = llvm_source_dir + "/runtimes/compiler-rt"
 891
 892     if stages >= 2:
 893         stage2_dir = build_dir + "/stage2"
 894         stage2_inst_dir = stage2_dir + "/" + package_name
 895         final_stage_dir = stage2_dir
 896         final_inst_dir = stage2_inst_dir
 897         pgo_phase = "gen" if pgo else None
 898         build_one_stage(
 899             [stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)] + extra_cflags2,
 900             [stage1_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)] + extra_cxxflags2,
 901             [stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)] + extra_asmflags,
 902             [ld] + extra_ldflags,
 903             ar,
 904             ranlib,
 905             libtool,
 906             llvm_source_dir,
 907             stage2_dir,
 908             package_name,
 909             build_libcxx,
 910             osx_cross_compile,
 911             build_type,
 912             assertions,
 913             libcxx_include_dir,
 914             build_wasm,
 915             compiler_rt_source_dir,
 916             runtimes_source_link,
 917             compiler_rt_source_link,
 918             is_final_stage=(stages == 2),
 919             android_targets=android_targets,
 920             extra_targets=extra_targets,
 921             pgo_phase=pgo_phase,
 922         )
 923
 924     if stages >= 3:
 925         stage3_dir = build_dir + "/stage3"
 926         stage3_inst_dir = stage3_dir + "/" + package_name
 927         final_stage_dir = stage3_dir
 928         final_inst_dir = stage3_inst_dir
 929         build_one_stage(
 930             [stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)] + extra_cflags2,
 931             [stage2_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)] + extra_cxxflags2,
 932             [stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)] + extra_asmflags,
 933             [ld] + extra_ldflags,
 934             ar,
 935             ranlib,
 936             libtool,
 937             llvm_source_dir,
 938             stage3_dir,
 939             package_name,
 940             build_libcxx,
 941             osx_cross_compile,
 942             build_type,
 943             assertions,
 944             libcxx_include_dir,
 945             build_wasm,
 946             compiler_rt_source_dir,
 947             runtimes_source_link,
 948             compiler_rt_source_link,
 949             (stages == 3),
 950             extra_targets=extra_targets,
 951         )
 952
 953     if stages >= 4:
 954         stage4_dir = build_dir + "/stage4"
 955         stage4_inst_dir = stage4_dir + "/" + package_name
 956         final_stage_dir = stage4_dir
 957         final_inst_dir = stage4_inst_dir
 958         pgo_phase = None
 959         if pgo:
 960             pgo_phase = "use"
 961             llvm_profdata = stage3_inst_dir + "/bin/llvm-profdata%s" % exe_ext
 962             merge_cmd = [llvm_profdata, "merge", "-o", "merged.profdata"]
 963             profraw_files = glob.glob(
 964                 os.path.join(stage2_dir, "build", "profiles", "*.profraw")
 965             )
 966             if not os.path.exists(stage4_dir):
 967                 os.mkdir(stage4_dir)
 968             run_in(stage4_dir, merge_cmd + profraw_files)
 969         build_one_stage(
 970             [stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)] + extra_cflags2,
 971             [stage3_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)] + extra_cxxflags2,
 972             [stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)] + extra_asmflags,
 973             [ld] + extra_ldflags,
 974             ar,
 975             ranlib,
 976             libtool,
 977             llvm_source_dir,
 978             stage4_dir,
 979             package_name,
 980             build_libcxx,
 981             osx_cross_compile,
 982             build_type,
 983             assertions,
 984             libcxx_include_dir,
 985             build_wasm,
 986             compiler_rt_source_dir,
 987             runtimes_source_link,
 988             compiler_rt_source_link,
 989             (stages == 4),
 990             extra_targets=extra_targets,
 991             pgo_phase=pgo_phase,
 992         )
 993
 994     if build_clang_tidy:
 995         prune_final_dir_for_clang_tidy(
 996             os.path.join(final_stage_dir, package_name), osx_cross_compile
 997         )
 998
 999     # Copy the wasm32 builtins to the final_inst_dir if the archive is present.
1000     if "wasi-compiler-rt" in config:
1001         compiler_rt = config["wasi-compiler-rt"].format(**os.environ)
1002         if os.path.isdir(compiler_rt):
1003             for libdir in glob.glob(
1004                 os.path.join(final_inst_dir, "lib", "clang", "*", "lib")
1005             ):
1006                 srcdir = os.path.join(compiler_rt, "lib", "wasi")
1007                 print("Copying from wasi-compiler-rt srcdir %s" % srcdir)
1008                 # Copy the contents of the "lib/wasi" subdirectory to the
1009                 # appropriate location in final_inst_dir.
1010                 destdir = os.path.join(libdir, "wasi")
1011                 mkdir_p(destdir)
1012                 copy_tree(srcdir, destdir)
1013
1014     if not args.skip_tar:
1015         build_tar_package("%s.tar.zst" % package_name, final_stage_dir, package_name)