build/build-clang/build-clang.py

   1 #!/usr/bin/python3
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 # Only necessary for flake8 to be happy...
   7 import argparse
   8 import errno
   9 import fnmatch
  10 import glob
  11 import json
  12 import os
  13 import os.path
  14 import platform
  15 import re
  16 import shutil
  17 import subprocess
  18 import sys
  19 import tarfile
  20 from contextlib import contextmanager
  21 from shutil import which
  22
  23 import zstandard
  24
  25 SUPPORTED_TARGETS = {
  26     "x86_64-unknown-linux-gnu": ("Linux", "x86_64"),
  27     "x86_64-pc-windows-msvc": ("Windows", "AMD64"),
  28     "x86_64-apple-darwin": ("Darwin", "x86_64"),
  29     "aarch64-apple-darwin": ("Darwin", "arm64"),
  30 }
  31
  32
  33 def is_llvm_toolchain(cc, cxx):
  34     return "clang" in cc and "clang" in cxx
  35
  36
  37 def check_run(args):
  38     print(" ".join(args), file=sys.stderr, flush=True)
  39     if args[0] == "cmake":
  40         # CMake `message(STATUS)` messages, as appearing in failed source code
  41         # compiles, appear on stdout, so we only capture that.
  42         p = subprocess.Popen(args, stdout=subprocess.PIPE)
  43         lines = []
  44         for line in p.stdout:
  45             lines.append(line)
  46             sys.stdout.write(line.decode())
  47             sys.stdout.flush()
  48         r = p.wait()
  49         if r != 0 and os.environ.get("UPLOAD_DIR"):
  50             cmake_output_re = re.compile(b'See also "(.*/CMakeOutput.log)"')
  51             cmake_error_re = re.compile(b'See also "(.*/CMakeError.log)"')
  52
  53             def find_first_match(re):
  54                 for l in lines:
  55                     match = re.search(l)
  56                     if match:
  57                         return match
  58
  59             output_match = find_first_match(cmake_output_re)
  60             error_match = find_first_match(cmake_error_re)
  61
  62             upload_dir = os.environ["UPLOAD_DIR"].encode("utf-8")
  63             if output_match or error_match:
  64                 mkdir_p(upload_dir)
  65             if output_match:
  66                 shutil.copy2(output_match.group(1), upload_dir)
  67             if error_match:
  68                 shutil.copy2(error_match.group(1), upload_dir)
  69     else:
  70         r = subprocess.call(args)
  71     assert r == 0
  72
  73
  74 def run_in(path, args):
  75     with chdir(path):
  76         check_run(args)
  77
  78
  79 @contextmanager
  80 def chdir(path):
  81     d = os.getcwd()
  82     print('cd "%s"' % path, file=sys.stderr)
  83     os.chdir(path)
  84     try:
  85         yield
  86     finally:
  87         print('cd "%s"' % d, file=sys.stderr)
  88         os.chdir(d)
  89
  90
  91 def patch(patch, srcdir):
  92     patch = os.path.realpath(patch)
  93     check_run(["patch", "-d", srcdir, "-p1", "-i", patch, "--fuzz=0", "-s"])
  94
  95
  96 def import_clang_tidy(source_dir, build_clang_tidy_alpha, build_clang_tidy_external):
  97     clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]), "..", "clang-plugin")
  98     clang_tidy_path = os.path.join(source_dir, "clang-tools-extra/clang-tidy")
  99     sys.path.append(clang_plugin_path)
 100     from import_mozilla_checks import do_import
 101
 102     import_options = {
 103         "alpha": build_clang_tidy_alpha,
 104         "external": build_clang_tidy_external,
 105     }
 106     do_import(clang_plugin_path, clang_tidy_path, import_options)
 107
 108
 109 def build_package(package_build_dir, cmake_args):
 110     if not os.path.exists(package_build_dir):
 111         os.mkdir(package_build_dir)
 112     # If CMake has already been run, it may have been run with different
 113     # arguments, so we need to re-run it.  Make sure the cached copy of the
 114     # previous CMake run is cleared before running it again.
 115     if os.path.exists(package_build_dir + "/CMakeCache.txt"):
 116         os.remove(package_build_dir + "/CMakeCache.txt")
 117     if os.path.exists(package_build_dir + "/CMakeFiles"):
 118         shutil.rmtree(package_build_dir + "/CMakeFiles")
 119
 120     run_in(package_build_dir, ["cmake"] + cmake_args)
 121     run_in(package_build_dir, ["ninja", "install", "-v"])
 122
 123
 124 @contextmanager
 125 def updated_env(env):
 126     old_env = os.environ.copy()
 127     os.environ.update(env)
 128     yield
 129     os.environ.clear()
 130     os.environ.update(old_env)
 131
 132
 133 def build_tar_package(name, base, directory):
 134     name = os.path.realpath(name)
 135     print("tarring {} from {}/{}".format(name, base, directory), file=sys.stderr)
 136     assert name.endswith(".tar.zst")
 137
 138     cctx = zstandard.ZstdCompressor()
 139     with open(name, "wb") as f, cctx.stream_writer(f) as z:
 140         with tarfile.open(mode="w|", fileobj=z) as tf:
 141             with chdir(base):
 142                 tf.add(directory)
 143
 144
 145 def mkdir_p(path):
 146     try:
 147         os.makedirs(path)
 148     except OSError as e:
 149         if e.errno != errno.EEXIST or not os.path.isdir(path):
 150             raise
 151
 152
 153 def delete(path):
 154     if os.path.isdir(path):
 155         shutil.rmtree(path)
 156     else:
 157         try:
 158             os.unlink(path)
 159         except Exception:
 160             pass
 161
 162
 163 def install_import_library(build_dir, clang_dir):
 164     shutil.copy2(
 165         os.path.join(build_dir, "lib", "clang.lib"), os.path.join(clang_dir, "lib")
 166     )
 167
 168
 169 def is_darwin(target):
 170     return "-apple-darwin" in target
 171
 172
 173 def is_linux(target):
 174     return "-linux-gnu" in target
 175
 176
 177 def is_windows(target):
 178     return "-windows-msvc" in target
 179
 180
 181 def is_cross_compile(target):
 182     return SUPPORTED_TARGETS[target] != (platform.system(), platform.machine())
 183
 184
 185 def build_one_stage(
 186     cc,
 187     cxx,
 188     asm,
 189     ar,
 190     ranlib,
 191     ldflags,
 192     src_dir,
 193     stage_dir,
 194     package_name,
 195     build_type,
 196     assertions,
 197     target,
 198     targets,
 199     is_final_stage=False,
 200     profile=None,
 201 ):
 202     if not os.path.exists(stage_dir):
 203         os.mkdir(stage_dir)
 204
 205     build_dir = stage_dir + "/build"
 206     inst_dir = stage_dir + "/" + package_name
 207
 208     # cmake doesn't deal well with backslashes in paths.
 209     def slashify_path(path):
 210         return path.replace("\\", "/")
 211
 212     def cmake_base_args(cc, cxx, asm, ar, ranlib, ldflags, inst_dir):
 213         machine_targets = targets if is_final_stage and targets else "X86"
 214
 215         cmake_args = [
 216             "-GNinja",
 217             "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]),
 218             "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]),
 219             "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]),
 220             "-DCMAKE_AR=%s" % slashify_path(ar),
 221             "-DCMAKE_C_FLAGS=%s" % " ".join(cc[1:]),
 222             "-DCMAKE_CXX_FLAGS=%s" % " ".join(cxx[1:]),
 223             "-DCMAKE_ASM_FLAGS=%s" % " ".join(asm[1:]),
 224             "-DCMAKE_EXE_LINKER_FLAGS=%s" % " ".join(ldflags),
 225             "-DCMAKE_SHARED_LINKER_FLAGS=%s" % " ".join(ldflags),
 226             "-DCMAKE_BUILD_TYPE=%s" % build_type,
 227             "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir,
 228             "-DLLVM_TARGETS_TO_BUILD=%s" % machine_targets,
 229             "-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF",
 230             "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"),
 231             "-DLLVM_ENABLE_BINDINGS=OFF",
 232             "-DLLVM_ENABLE_CURL=OFF",
 233             "-DLLVM_INCLUDE_TESTS=OFF",
 234         ]
 235         if is_llvm_toolchain(cc[0], cxx[0]):
 236             cmake_args += ["-DLLVM_ENABLE_LLD=ON"]
 237         elif is_windows(target) and is_cross_compile(target):
 238             raise Exception(
 239                 "Cannot cross-compile for Windows with a compiler that is not clang"
 240             )
 241
 242         if "TASK_ID" in os.environ:
 243             cmake_args += [
 244                 "-DCLANG_REPOSITORY_STRING=taskcluster-%s" % os.environ["TASK_ID"],
 245             ]
 246         projects = ["clang", "lld"]
 247         if is_final_stage:
 248             projects.append("clang-tools-extra")
 249         else:
 250             cmake_args.append("-DLLVM_TOOL_LLI_BUILD=OFF")
 251
 252         cmake_args.append("-DLLVM_ENABLE_PROJECTS=%s" % ";".join(projects))
 253
 254         # There is no libxml2 on Windows except if we build one ourselves.
 255         # libxml2 is only necessary for llvm-mt, but Windows can just use the
 256         # native MT tool.
 257         if not is_windows(target) and is_final_stage:
 258             cmake_args += ["-DLLVM_ENABLE_LIBXML2=FORCE_ON"]
 259         if is_linux(target) and is_final_stage:
 260             sysroot = os.path.join(os.environ.get("MOZ_FETCHES_DIR", ""), "sysroot")
 261             if os.path.exists(sysroot):
 262                 cmake_args += ["-DLLVM_BINUTILS_INCDIR=/usr/include"]
 263                 cmake_args += ["-DCMAKE_SYSROOT=%s" % sysroot]
 264                 # Work around the LLVM build system not building the i386 compiler-rt
 265                 # because it doesn't allow to use a sysroot for that during the cmake
 266                 # checks.
 267                 cmake_args += ["-DCAN_TARGET_i386=1"]
 268             cmake_args += ["-DLLVM_ENABLE_TERMINFO=OFF"]
 269         if is_windows(target):
 270             cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
 271             cmake_args.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
 272             if is_cross_compile(target):
 273                 cmake_args += [
 274                     f"-DCMAKE_TOOLCHAIN_FILE={src_dir}/cmake/platforms/WinMsvc.cmake",
 275                     f"-DLLVM_NATIVE_TOOLCHAIN={os.path.dirname(os.path.dirname(cc[0]))}",
 276                     f"-DHOST_ARCH={target[: -len('-pc-windows-msvc')]}",
 277                     f"-DLLVM_WINSYSROOT={os.environ['VSINSTALLDIR']}",
 278                     "-DLLVM_DISABLE_ASSEMBLY_FILES=ON",
 279                 ]
 280         else:
 281             # libllvm as a shared library is not supported on Windows
 282             cmake_args += ["-DLLVM_LINK_LLVM_DYLIB=ON"]
 283         if ranlib is not None:
 284             cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
 285         if is_darwin(target) and is_cross_compile(target):
 286             arch = "arm64" if target.startswith("aarch64") else "x86_64"
 287             cmake_args += [
 288                 "-DCMAKE_SYSTEM_NAME=Darwin",
 289                 "-DCMAKE_SYSTEM_VERSION=%s" % os.environ["MACOSX_DEPLOYMENT_TARGET"],
 290                 "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 291                 "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 292                 "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
 293                 "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
 294                 "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
 295                 "-DCMAKE_MACOSX_RPATH=ON",
 296                 "-DCMAKE_OSX_ARCHITECTURES=%s" % arch,
 297                 "-DDARWIN_osx_ARCHS=%s" % arch,
 298                 "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 299                 "-DLLVM_DEFAULT_TARGET_TRIPLE=%s" % target,
 300                 "-DCMAKE_C_COMPILER_TARGET=%s" % target,
 301                 "-DCMAKE_CXX_COMPILER_TARGET=%s" % target,
 302                 "-DCMAKE_ASM_COMPILER_TARGET=%s" % target,
 303             ]
 304             if arch == "arm64":
 305                 cmake_args += [
 306                     "-DDARWIN_osx_BUILTIN_ARCHS=arm64",
 307                 ]
 308             # Starting in LLVM 11 (which requires SDK 10.12) the build tries to
 309             # detect the SDK version by calling xcrun. Cross-compiles don't have
 310             # an xcrun, so we have to set the version explicitly.
 311             cmake_args += [
 312                 "-DDARWIN_macosx_OVERRIDE_SDK_VERSION=%s"
 313                 % os.environ["MACOSX_DEPLOYMENT_TARGET"],
 314             ]
 315
 316         if profile == "gen":
 317             # Per https://releases.llvm.org/10.0.0/docs/HowToBuildWithPGO.html
 318             cmake_args += [
 319                 "-DLLVM_BUILD_INSTRUMENTED=IR",
 320                 "-DLLVM_BUILD_RUNTIME=No",
 321             ]
 322         elif profile:
 323             cmake_args += [
 324                 "-DLLVM_PROFDATA_FILE=%s" % profile,
 325             ]
 326
 327         # Using LTO for both profile generation and usage to avoid most
 328         # "function control flow change detected (hash mismatch)" error.
 329         if profile and not is_windows(target):
 330             cmake_args.append("-DLLVM_ENABLE_LTO=Thin")
 331         return cmake_args
 332
 333     cmake_args = []
 334     cmake_args += cmake_base_args(cc, cxx, asm, ar, ranlib, ldflags, inst_dir)
 335     cmake_args += [src_dir]
 336     build_package(build_dir, cmake_args)
 337
 338     # For some reasons the import library clang.lib of clang.exe is not
 339     # installed, so we copy it by ourselves.
 340     if is_windows(target) and is_final_stage:
 341         install_import_library(build_dir, inst_dir)
 342
 343
 344 # Return the absolute path of a build tool.  We first look to see if the
 345 # variable is defined in the config file, and if so we make sure it's an
 346 # absolute path to an existing tool, otherwise we look for a program in
 347 # $PATH named "key".
 348 #
 349 # This expects the name of the key in the config file to match the name of
 350 # the tool in the default toolchain on the system (for example, "ld" on Unix
 351 # and "link" on Windows).
 352 def get_tool(config, key):
 353     f = None
 354     if key in config:
 355         f = config[key].format(**os.environ)
 356         if os.path.isabs(f):
 357             if not os.path.exists(f):
 358                 raise ValueError("%s must point to an existing path" % key)
 359             return f
 360
 361     # Assume that we have the name of some program that should be on PATH.
 362     tool = which(f) if f else which(key)
 363     if not tool:
 364         raise ValueError("%s not found on PATH" % (f or key))
 365     return tool
 366
 367
 368 # This function is intended to be called on the final build directory when
 369 # building clang-tidy. Also clang-format binaries are included that can be used
 370 # in conjunction with clang-tidy.
 371 # As a separate binary we also ship clangd for the language server protocol that
 372 # can be used as a plugin in `vscode`.
 373 # Its job is to remove all of the files which won't be used for clang-tidy or
 374 # clang-format to reduce the download size.  Currently when this function
 375 # finishes its job, it will leave final_dir with a layout like this:
 376 #
 377 # clang/
 378 #   bin/
 379 #     clang-apply-replacements
 380 #     clang-format
 381 #     clang-tidy
 382 #     clangd
 383 #     run-clang-tidy
 384 #   include/
 385 #     * (nothing will be deleted here)
 386 #   lib/
 387 #     clang/
 388 #       4.0.0/
 389 #         include/
 390 #           * (nothing will be deleted here)
 391 #   share/
 392 #     clang/
 393 #       clang-format-diff.py
 394 #       clang-tidy-diff.py
 395 #       run-clang-tidy.py
 396 def prune_final_dir_for_clang_tidy(final_dir, target):
 397     # Make sure we only have what we expect.
 398     dirs = [
 399         "bin",
 400         "include",
 401         "lib",
 402         "lib32",
 403         "libexec",
 404         "msbuild-bin",
 405         "share",
 406         "tools",
 407     ]
 408     if is_linux(target):
 409         dirs.append("x86_64-unknown-linux-gnu")
 410     for f in glob.glob("%s/*" % final_dir):
 411         if os.path.basename(f) not in dirs:
 412             raise Exception("Found unknown file %s in the final directory" % f)
 413         if not os.path.isdir(f):
 414             raise Exception("Expected %s to be a directory" % f)
 415
 416     kept_binaries = [
 417         "clang-apply-replacements",
 418         "clang-format",
 419         "clang-tidy",
 420         "clangd",
 421         "clang-query",
 422         "run-clang-tidy",
 423     ]
 424     re_clang_tidy = re.compile(r"^(" + "|".join(kept_binaries) + r")(\.exe)?$", re.I)
 425     for f in glob.glob("%s/bin/*" % final_dir):
 426         if re_clang_tidy.search(os.path.basename(f)) is None:
 427             delete(f)
 428
 429     # Keep include/ intact.
 430
 431     # Remove the target-specific files.
 432     if is_linux(target):
 433         if os.path.exists(os.path.join(final_dir, "x86_64-unknown-linux-gnu")):
 434             shutil.rmtree(os.path.join(final_dir, "x86_64-unknown-linux-gnu"))
 435
 436     # In lib/, only keep lib/clang/N.M.O/include and the LLVM shared library.
 437     re_ver_num = re.compile(r"^\d+(?:\.\d+\.\d+)?$", re.I)
 438     for f in glob.glob("%s/lib/*" % final_dir):
 439         name = os.path.basename(f)
 440         if name == "clang":
 441             continue
 442         if is_darwin(target) and name in ["libLLVM.dylib", "libclang-cpp.dylib"]:
 443             continue
 444         if is_linux(target) and (
 445             fnmatch.fnmatch(name, "libLLVM*.so")
 446             or fnmatch.fnmatch(name, "libclang-cpp.so*")
 447         ):
 448             continue
 449         delete(f)
 450     for f in glob.glob("%s/lib/clang/*" % final_dir):
 451         if re_ver_num.search(os.path.basename(f)) is None:
 452             delete(f)
 453     for f in glob.glob("%s/lib/clang/*/*" % final_dir):
 454         if os.path.basename(f) != "include":
 455             delete(f)
 456
 457     # Completely remove libexec/, msbuild-bin and tools, if it exists.
 458     shutil.rmtree(os.path.join(final_dir, "libexec"))
 459     for d in ("msbuild-bin", "tools"):
 460         d = os.path.join(final_dir, d)
 461         if os.path.exists(d):
 462             shutil.rmtree(d)
 463
 464     # In share/, only keep share/clang/*tidy*
 465     re_clang_tidy = re.compile(r"format|tidy", re.I)
 466     for f in glob.glob("%s/share/*" % final_dir):
 467         if os.path.basename(f) != "clang":
 468             delete(f)
 469     for f in glob.glob("%s/share/clang/*" % final_dir):
 470         if re_clang_tidy.search(os.path.basename(f)) is None:
 471             delete(f)
 472
 473
 474 def main():
 475     parser = argparse.ArgumentParser()
 476     parser.add_argument(
 477         "-c",
 478         "--config",
 479         action="append",
 480         required=True,
 481         type=argparse.FileType("r"),
 482         help="Clang configuration file",
 483     )
 484     parser.add_argument(
 485         "--clean", required=False, action="store_true", help="Clean the build directory"
 486     )
 487     parser.add_argument(
 488         "--skip-tar",
 489         required=False,
 490         action="store_true",
 491         help="Skip tar packaging stage",
 492     )
 493     parser.add_argument(
 494         "--skip-patch",
 495         required=False,
 496         action="store_true",
 497         help="Do not patch source",
 498     )
 499
 500     args = parser.parse_args()
 501
 502     if not os.path.exists("llvm/README.txt"):
 503         raise Exception(
 504             "The script must be run from the root directory of the llvm-project tree"
 505         )
 506     source_dir = os.getcwd()
 507     build_dir = source_dir + "/build"
 508
 509     if args.clean:
 510         shutil.rmtree(build_dir)
 511         os.sys.exit(0)
 512
 513     llvm_source_dir = source_dir + "/llvm"
 514
 515     config = {}
 516     # Merge all the configs we got from the command line.
 517     for c in args.config:
 518         this_config_dir = os.path.dirname(c.name)
 519         this_config = json.load(c)
 520         patches = this_config.get("patches")
 521         if patches:
 522             this_config["patches"] = [os.path.join(this_config_dir, p) for p in patches]
 523         for key, value in this_config.items():
 524             old_value = config.get(key)
 525             if old_value is None:
 526                 config[key] = value
 527             elif value is None:
 528                 if key in config:
 529                     del config[key]
 530             elif type(old_value) != type(value):
 531                 raise Exception(
 532                     "{} is overriding `{}` with a value of the wrong type".format(
 533                         c.name, key
 534                     )
 535                 )
 536             elif isinstance(old_value, list):
 537                 for v in value:
 538                     if v not in old_value:
 539                         old_value.append(v)
 540             elif isinstance(old_value, dict):
 541                 raise Exception("{} is setting `{}` to a dict?".format(c.name, key))
 542             else:
 543                 config[key] = value
 544
 545     stages = 2
 546     if "stages" in config:
 547         stages = int(config["stages"])
 548         if stages not in (1, 2, 3, 4):
 549             raise ValueError("We only know how to build 1, 2, 3, or 4 stages.")
 550     skip_stages = 0
 551     if "skip_stages" in config:
 552         # The assumption here is that the compiler given in `cc` and other configs
 553         # is the result of the last skip stage, built somewhere else.
 554         skip_stages = int(config["skip_stages"])
 555         if skip_stages >= stages:
 556             raise ValueError("Cannot skip more stages than are built.")
 557     pgo = False
 558     if "pgo" in config:
 559         pgo = config["pgo"]
 560         if pgo not in (True, False):
 561             raise ValueError("Only boolean values are accepted for pgo.")
 562     build_type = "Release"
 563     if "build_type" in config:
 564         build_type = config["build_type"]
 565         if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"):
 566             raise ValueError(
 567                 "We only know how to do Release, Debug, RelWithDebInfo or "
 568                 "MinSizeRel builds"
 569             )
 570     targets = config.get("targets")
 571     build_clang_tidy = False
 572     if "build_clang_tidy" in config:
 573         build_clang_tidy = config["build_clang_tidy"]
 574         if build_clang_tidy not in (True, False):
 575             raise ValueError("Only boolean values are accepted for build_clang_tidy.")
 576     build_clang_tidy_alpha = False
 577     # check for build_clang_tidy_alpha only if build_clang_tidy is true
 578     if build_clang_tidy and "build_clang_tidy_alpha" in config:
 579         build_clang_tidy_alpha = config["build_clang_tidy_alpha"]
 580         if build_clang_tidy_alpha not in (True, False):
 581             raise ValueError(
 582                 "Only boolean values are accepted for build_clang_tidy_alpha."
 583             )
 584     build_clang_tidy_external = False
 585     # check for build_clang_tidy_external only if build_clang_tidy is true
 586     if build_clang_tidy and "build_clang_tidy_external" in config:
 587         build_clang_tidy_external = config["build_clang_tidy_external"]
 588         if build_clang_tidy_external not in (True, False):
 589             raise ValueError(
 590                 "Only boolean values are accepted for build_clang_tidy_external."
 591             )
 592     assertions = False
 593     if "assertions" in config:
 594         assertions = config["assertions"]
 595         if assertions not in (True, False):
 596             raise ValueError("Only boolean values are accepted for assertions.")
 597
 598     for t in SUPPORTED_TARGETS:
 599         if not is_cross_compile(t):
 600             host = t
 601             break
 602     else:
 603         raise Exception(
 604             f"Cannot use this script on {platform.system()} {platform.machine()}"
 605         )
 606
 607     target = config.get("target", host)
 608     if target not in SUPPORTED_TARGETS:
 609         raise ValueError(f"{target} is not a supported target.")
 610
 611     if is_cross_compile(target) and not is_linux(host):
 612         raise Exception("Cross-compilation is only supported on Linux")
 613
 614     if is_darwin(target):
 615         os.environ["MACOSX_DEPLOYMENT_TARGET"] = (
 616             "11.0" if target.startswith("aarch64") else "10.12"
 617         )
 618
 619     if is_windows(target):
 620         exe_ext = ".exe"
 621         cc_name = "clang-cl"
 622         cxx_name = "clang-cl"
 623     else:
 624         exe_ext = ""
 625         cc_name = "clang"
 626         cxx_name = "clang++"
 627
 628     cc = get_tool(config, "cc")
 629     cxx = get_tool(config, "cxx")
 630     asm = get_tool(config, "ml" if is_windows(target) else "as")
 631     # Not using lld here as default here because it's not in PATH. But clang
 632     # knows how to find it when they are installed alongside each others.
 633     ar = get_tool(config, "lib" if is_windows(target) else "ar")
 634     ranlib = None if is_windows(target) else get_tool(config, "ranlib")
 635
 636     if not os.path.exists(source_dir):
 637         os.makedirs(source_dir)
 638
 639     if not args.skip_patch:
 640         for p in config.get("patches", []):
 641             patch(p, source_dir)
 642
 643     package_name = "clang"
 644     if build_clang_tidy:
 645         package_name = "clang-tidy"
 646         if not args.skip_patch:
 647             import_clang_tidy(
 648                 source_dir, build_clang_tidy_alpha, build_clang_tidy_external
 649             )
 650
 651     if not os.path.exists(build_dir):
 652         os.makedirs(build_dir)
 653
 654     stage1_dir = build_dir + "/stage1"
 655     stage1_inst_dir = stage1_dir + "/" + package_name
 656
 657     final_stage_dir = stage1_dir
 658
 659     if is_darwin(target):
 660         extra_cflags = []
 661         extra_cxxflags = []
 662         extra_cflags2 = []
 663         extra_cxxflags2 = []
 664         extra_asmflags = []
 665         # It's unfortunately required to specify the linker used here because
 666         # the linker flags are used in LLVM's configure step before
 667         # -DLLVM_ENABLE_LLD is actually processed.
 668         extra_ldflags = [
 669             "-fuse-ld=lld",
 670             "-Wl,-dead_strip",
 671         ]
 672     elif is_linux(target):
 673         extra_cflags = []
 674         extra_cxxflags = []
 675         extra_cflags2 = ["-fPIC"]
 676         # Silence clang's warnings about arguments not being used in compilation.
 677         extra_cxxflags2 = [
 678             "-fPIC",
 679             "-Qunused-arguments",
 680         ]
 681         extra_asmflags = []
 682         # Avoid libLLVM internal function calls going through the PLT.
 683         extra_ldflags = ["-Wl,-Bsymbolic-functions"]
 684         # For whatever reason, LLVM's build system will set things up to turn
 685         # on -ffunction-sections and -fdata-sections, but won't turn on the
 686         # corresponding option to strip unused sections.  We do it explicitly
 687         # here.  LLVM's build system is also picky about turning on ICF, so
 688         # we do that explicitly here, too.
 689
 690         # It's unfortunately required to specify the linker used here because
 691         # the linker flags are used in LLVM's configure step before
 692         # -DLLVM_ENABLE_LLD is actually processed.
 693         if is_llvm_toolchain(cc, cxx):
 694             extra_ldflags += ["-fuse-ld=lld", "-Wl,--icf=safe"]
 695         extra_ldflags += ["-Wl,--gc-sections"]
 696     elif is_windows(target):
 697         extra_cflags = []
 698         extra_cxxflags = []
 699         # clang-cl would like to figure out what it's supposed to be emulating
 700         # by looking at an MSVC install, but we don't really have that here.
 701         # Force things on based on WinMsvc.cmake.
 702         # Ideally, we'd just use WinMsvc.cmake as a toolchain file, but it only
 703         # really works for cross-compiles, which this is not.
 704         with open(os.path.join(llvm_source_dir, "cmake/platforms/WinMsvc.cmake")) as f:
 705             compat = [
 706                 item
 707                 for line in f
 708                 for item in line.split()
 709                 if "-fms-compatibility-version=" in item
 710             ][0]
 711         extra_cflags2 = [compat]
 712         extra_cxxflags2 = [compat]
 713         extra_asmflags = []
 714         extra_ldflags = []
 715
 716     upload_dir = os.getenv("UPLOAD_DIR")
 717     if assertions and upload_dir:
 718         extra_cflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir]
 719         extra_cxxflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir]
 720
 721     if skip_stages < 1:
 722         build_one_stage(
 723             [cc] + extra_cflags,
 724             [cxx] + extra_cxxflags,
 725             [asm] + extra_asmflags,
 726             ar,
 727             ranlib,
 728             extra_ldflags,
 729             llvm_source_dir,
 730             stage1_dir,
 731             package_name,
 732             build_type,
 733             assertions,
 734             target,
 735             targets,
 736             is_final_stage=(stages == 1),
 737         )
 738
 739     if stages >= 2 and skip_stages < 2:
 740         stage2_dir = build_dir + "/stage2"
 741         stage2_inst_dir = stage2_dir + "/" + package_name
 742         final_stage_dir = stage2_dir
 743         if skip_stages < 1:
 744             cc = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 745             cxx = stage1_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
 746             asm = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 747         build_one_stage(
 748             [cc] + extra_cflags2,
 749             [cxx] + extra_cxxflags2,
 750             [asm] + extra_asmflags,
 751             ar,
 752             ranlib,
 753             extra_ldflags,
 754             llvm_source_dir,
 755             stage2_dir,
 756             package_name,
 757             build_type,
 758             assertions,
 759             target,
 760             targets,
 761             is_final_stage=(stages == 2),
 762             profile="gen" if pgo else None,
 763         )
 764
 765     if stages >= 3 and skip_stages < 3:
 766         stage3_dir = build_dir + "/stage3"
 767         stage3_inst_dir = stage3_dir + "/" + package_name
 768         final_stage_dir = stage3_dir
 769         if skip_stages < 2:
 770             cc = stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 771             cxx = stage2_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
 772             asm = stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 773         build_one_stage(
 774             [cc] + extra_cflags2,
 775             [cxx] + extra_cxxflags2,
 776             [asm] + extra_asmflags,
 777             ar,
 778             ranlib,
 779             extra_ldflags,
 780             llvm_source_dir,
 781             stage3_dir,
 782             package_name,
 783             build_type,
 784             assertions,
 785             target,
 786             targets,
 787             (stages == 3),
 788         )
 789         if pgo:
 790             llvm_profdata = stage2_inst_dir + "/bin/llvm-profdata%s" % exe_ext
 791             merge_cmd = [llvm_profdata, "merge", "-o", "merged.profdata"]
 792             profraw_files = glob.glob(
 793                 os.path.join(stage2_dir, "build", "profiles", "*.profraw")
 794             )
 795             run_in(stage3_dir, merge_cmd + profraw_files)
 796             if stages == 3:
 797                 mkdir_p(upload_dir)
 798                 shutil.copy2(os.path.join(stage3_dir, "merged.profdata"), upload_dir)
 799                 return
 800
 801     if stages >= 4 and skip_stages < 4:
 802         stage4_dir = build_dir + "/stage4"
 803         final_stage_dir = stage4_dir
 804         profile = None
 805         if pgo:
 806             if skip_stages == 3:
 807                 profile_dir = os.environ.get("MOZ_FETCHES_DIR", "")
 808             else:
 809                 profile_dir = stage3_dir
 810             profile = os.path.join(profile_dir, "merged.profdata")
 811         if skip_stages < 3:
 812             cc = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 813             cxx = stage3_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
 814             asm = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 815         build_one_stage(
 816             [cc] + extra_cflags2,
 817             [cxx] + extra_cxxflags2,
 818             [asm] + extra_asmflags,
 819             ar,
 820             ranlib,
 821             extra_ldflags,
 822             llvm_source_dir,
 823             stage4_dir,
 824             package_name,
 825             build_type,
 826             assertions,
 827             target,
 828             targets,
 829             (stages == 4),
 830             profile=profile,
 831         )
 832
 833     if build_clang_tidy:
 834         prune_final_dir_for_clang_tidy(
 835             os.path.join(final_stage_dir, package_name), target
 836         )
 837
 838     if not args.skip_tar:
 839         build_tar_package("%s.tar.zst" % package_name, final_stage_dir, package_name)
 840
 841
 842 if __name__ == "__main__":
 843     main()