build/build-clang/build-clang.py

   1 #!/usr/bin/python3
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 # Only necessary for flake8 to be happy...
   7 import argparse
   8 import errno
   9 import fnmatch
  10 import glob
  11 import json
  12 import os
  13 import os.path
  14 import platform
  15 import re
  16 import shutil
  17 import subprocess
  18 import sys
  19 import tarfile
  20 from contextlib import contextmanager
  21 from shutil import which
  22
  23 import zstandard
  24
  25 SUPPORTED_TARGETS = {
  26     "x86_64-unknown-linux-gnu": ("Linux", "x86_64"),
  27     "x86_64-pc-windows-msvc": ("Windows", "AMD64"),
  28     "x86_64-apple-darwin": ("Darwin", "x86_64"),
  29     "aarch64-apple-darwin": ("Darwin", "arm64"),
  30 }
  31
  32
  33 def is_llvm_toolchain(cc, cxx):
  34     return "clang" in cc and "clang" in cxx
  35
  36
  37 def check_run(args):
  38     print(" ".join(args), file=sys.stderr, flush=True)
  39     if args[0] == "cmake":
  40         # CMake `message(STATUS)` messages, as appearing in failed source code
  41         # compiles, appear on stdout, so we only capture that.
  42         p = subprocess.Popen(args, stdout=subprocess.PIPE)
  43         lines = []
  44         for line in p.stdout:
  45             lines.append(line)
  46             sys.stdout.write(line.decode())
  47             sys.stdout.flush()
  48         r = p.wait()
  49         if r != 0 and os.environ.get("UPLOAD_DIR"):
  50             cmake_output_re = re.compile(b'See also "(.*/CMakeOutput.log)"')
  51             cmake_error_re = re.compile(b'See also "(.*/CMakeError.log)"')
  52
  53             def find_first_match(re):
  54                 for l in lines:
  55                     match = re.search(l)
  56                     if match:
  57                         return match
  58
  59             output_match = find_first_match(cmake_output_re)
  60             error_match = find_first_match(cmake_error_re)
  61
  62             upload_dir = os.environ["UPLOAD_DIR"].encode("utf-8")
  63             if output_match or error_match:
  64                 mkdir_p(upload_dir)
  65             if output_match:
  66                 shutil.copy2(output_match.group(1), upload_dir)
  67             if error_match:
  68                 shutil.copy2(error_match.group(1), upload_dir)
  69     else:
  70         r = subprocess.call(args)
  71     assert r == 0
  72
  73
  74 def run_in(path, args):
  75     with chdir(path):
  76         check_run(args)
  77
  78
  79 @contextmanager
  80 def chdir(path):
  81     d = os.getcwd()
  82     print('cd "%s"' % path, file=sys.stderr)
  83     os.chdir(path)
  84     try:
  85         yield
  86     finally:
  87         print('cd "%s"' % d, file=sys.stderr)
  88         os.chdir(d)
  89
  90
  91 def patch(patch, srcdir):
  92     patch = os.path.realpath(patch)
  93     check_run(["patch", "-d", srcdir, "-p1", "-i", patch, "--fuzz=0", "-s"])
  94
  95
  96 def import_clang_tidy(source_dir, build_clang_tidy_alpha, build_clang_tidy_external):
  97     clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]), "..", "clang-plugin")
  98     clang_tidy_path = os.path.join(source_dir, "clang-tools-extra/clang-tidy")
  99     sys.path.append(clang_plugin_path)
 100     from import_mozilla_checks import do_import
 101
 102     import_options = {
 103         "alpha": build_clang_tidy_alpha,
 104         "external": build_clang_tidy_external,
 105     }
 106     do_import(clang_plugin_path, clang_tidy_path, import_options)
 107
 108
 109 def build_package(package_build_dir, cmake_args):
 110     if not os.path.exists(package_build_dir):
 111         os.mkdir(package_build_dir)
 112     # If CMake has already been run, it may have been run with different
 113     # arguments, so we need to re-run it.  Make sure the cached copy of the
 114     # previous CMake run is cleared before running it again.
 115     if os.path.exists(package_build_dir + "/CMakeCache.txt"):
 116         os.remove(package_build_dir + "/CMakeCache.txt")
 117     if os.path.exists(package_build_dir + "/CMakeFiles"):
 118         shutil.rmtree(package_build_dir + "/CMakeFiles")
 119
 120     run_in(package_build_dir, ["cmake"] + cmake_args)
 121     run_in(package_build_dir, ["ninja", "install", "-v"])
 122
 123
 124 @contextmanager
 125 def updated_env(env):
 126     old_env = os.environ.copy()
 127     os.environ.update(env)
 128     yield
 129     os.environ.clear()
 130     os.environ.update(old_env)
 131
 132
 133 def build_tar_package(name, base, directory):
 134     name = os.path.realpath(name)
 135     print("tarring {} from {}/{}".format(name, base, directory), file=sys.stderr)
 136     assert name.endswith(".tar.zst")
 137
 138     cctx = zstandard.ZstdCompressor()
 139     with open(name, "wb") as f, cctx.stream_writer(f) as z:
 140         with tarfile.open(mode="w|", fileobj=z) as tf:
 141             with chdir(base):
 142                 tf.add(directory)
 143
 144
 145 def mkdir_p(path):
 146     try:
 147         os.makedirs(path)
 148     except OSError as e:
 149         if e.errno != errno.EEXIST or not os.path.isdir(path):
 150             raise
 151
 152
 153 def delete(path):
 154     if os.path.isdir(path):
 155         shutil.rmtree(path)
 156     else:
 157         try:
 158             os.unlink(path)
 159         except Exception:
 160             pass
 161
 162
 163 def install_import_library(build_dir, clang_dir):
 164     shutil.copy2(
 165         os.path.join(build_dir, "lib", "clang.lib"), os.path.join(clang_dir, "lib")
 166     )
 167
 168
 169 def is_darwin(target):
 170     return "-apple-darwin" in target
 171
 172
 173 def is_linux(target):
 174     return "-linux-gnu" in target
 175
 176
 177 def is_windows(target):
 178     return "-windows-msvc" in target
 179
 180
 181 def is_cross_compile(target):
 182     return SUPPORTED_TARGETS[target] != (platform.system(), platform.machine())
 183
 184
 185 def build_one_stage(
 186     cc,
 187     cxx,
 188     asm,
 189     ar,
 190     ranlib,
 191     ldflags,
 192     src_dir,
 193     stage_dir,
 194     package_name,
 195     build_type,
 196     assertions,
 197     target,
 198     targets,
 199     is_final_stage=False,
 200     profile=None,
 201 ):
 202     if not os.path.exists(stage_dir):
 203         os.mkdir(stage_dir)
 204
 205     build_dir = stage_dir + "/build"
 206     inst_dir = stage_dir + "/" + package_name
 207
 208     # cmake doesn't deal well with backslashes in paths.
 209     def slashify_path(path):
 210         return path.replace("\\", "/")
 211
 212     def cmake_base_args(cc, cxx, asm, ar, ranlib, ldflags, inst_dir):
 213         machine_targets = targets if is_final_stage and targets else "X86"
 214
 215         cmake_args = [
 216             "-GNinja",
 217             "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]),
 218             "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]),
 219             "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]),
 220             "-DCMAKE_AR=%s" % slashify_path(ar),
 221             "-DCMAKE_C_FLAGS_INIT=%s" % " ".join(cc[1:]),
 222             "-DCMAKE_CXX_FLAGS_INIT=%s" % " ".join(cxx[1:]),
 223             "-DCMAKE_ASM_FLAGS_INIT=%s" % " ".join(asm[1:]),
 224             "-DCMAKE_EXE_LINKER_FLAGS_INIT=%s" % " ".join(ldflags),
 225             "-DCMAKE_SHARED_LINKER_FLAGS_INIT=%s" % " ".join(ldflags),
 226             "-DCMAKE_BUILD_TYPE=%s" % build_type,
 227             "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir,
 228             "-DLLVM_TARGETS_TO_BUILD=%s" % machine_targets,
 229             "-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF",
 230             "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"),
 231             "-DLLVM_ENABLE_BINDINGS=OFF",
 232             "-DLLVM_ENABLE_CURL=OFF",
 233             "-DLLVM_INCLUDE_TESTS=OFF",
 234         ]
 235         if is_llvm_toolchain(cc[0], cxx[0]):
 236             cmake_args += ["-DLLVM_ENABLE_LLD=ON"]
 237         elif is_windows(target) and is_cross_compile(target):
 238             raise Exception(
 239                 "Cannot cross-compile for Windows with a compiler that is not clang"
 240             )
 241
 242         if "TASK_ID" in os.environ:
 243             cmake_args += [
 244                 "-DCLANG_REPOSITORY_STRING=taskcluster-%s" % os.environ["TASK_ID"],
 245             ]
 246         projects = ["clang", "lld"]
 247         if is_final_stage:
 248             projects.append("clang-tools-extra")
 249         else:
 250             cmake_args.append("-DLLVM_TOOL_LLI_BUILD=OFF")
 251
 252         cmake_args.append("-DLLVM_ENABLE_PROJECTS=%s" % ";".join(projects))
 253
 254         if is_final_stage:
 255             cmake_args += ["-DLLVM_ENABLE_LIBXML2=FORCE_ON"]
 256         if is_linux(target) and is_final_stage:
 257             sysroot = os.path.join(os.environ.get("MOZ_FETCHES_DIR", ""), "sysroot")
 258             if os.path.exists(sysroot):
 259                 cmake_args += ["-DLLVM_BINUTILS_INCDIR=/usr/include"]
 260                 cmake_args += ["-DCMAKE_SYSROOT=%s" % sysroot]
 261                 # Work around the LLVM build system not building the i386 compiler-rt
 262                 # because it doesn't allow to use a sysroot for that during the cmake
 263                 # checks.
 264                 cmake_args += ["-DCAN_TARGET_i386=1"]
 265             cmake_args += ["-DLLVM_ENABLE_TERMINFO=OFF"]
 266         if is_windows(target):
 267             cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
 268             cmake_args.insert(-1, "-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded")
 269             if is_cross_compile(target):
 270                 cmake_args += [
 271                     f"-DCMAKE_TOOLCHAIN_FILE={src_dir}/cmake/platforms/WinMsvc.cmake",
 272                     f"-DLLVM_NATIVE_TOOLCHAIN={os.path.dirname(os.path.dirname(cc[0]))}",
 273                     f"-DHOST_ARCH={target[: -len('-pc-windows-msvc')]}",
 274                     f"-DLLVM_WINSYSROOT={os.environ['VSINSTALLDIR']}",
 275                     "-DLLVM_DISABLE_ASSEMBLY_FILES=ON",
 276                 ]
 277             if is_final_stage:
 278                 fetches = os.environ["MOZ_FETCHES_DIR"]
 279                 cmake_args += [
 280                     "-DLIBXML2_DEFINITIONS=-DLIBXML_STATIC",
 281                     f"-DLIBXML2_INCLUDE_DIR={fetches}/libxml2/include/libxml2",
 282                     f"-DLIBXML2_LIBRARIES={fetches}/libxml2/lib/libxml2s.lib",
 283                 ]
 284         else:
 285             # libllvm as a shared library is not supported on Windows
 286             cmake_args += ["-DLLVM_LINK_LLVM_DYLIB=ON"]
 287         if ranlib is not None:
 288             cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
 289         if is_darwin(target) and is_cross_compile(target):
 290             arch = "arm64" if target.startswith("aarch64") else "x86_64"
 291             cmake_args += [
 292                 "-DCMAKE_SYSTEM_NAME=Darwin",
 293                 "-DCMAKE_SYSTEM_VERSION=%s" % os.environ["MACOSX_DEPLOYMENT_TARGET"],
 294                 "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 295                 "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 296                 "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
 297                 "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
 298                 "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
 299                 "-DCMAKE_MACOSX_RPATH=ON",
 300                 "-DCMAKE_OSX_ARCHITECTURES=%s" % arch,
 301                 "-DDARWIN_osx_ARCHS=%s" % arch,
 302                 "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 303                 "-DLLVM_DEFAULT_TARGET_TRIPLE=%s" % target,
 304                 "-DCMAKE_C_COMPILER_TARGET=%s" % target,
 305                 "-DCMAKE_CXX_COMPILER_TARGET=%s" % target,
 306                 "-DCMAKE_ASM_COMPILER_TARGET=%s" % target,
 307             ]
 308             if arch == "arm64":
 309                 cmake_args += [
 310                     "-DDARWIN_osx_BUILTIN_ARCHS=arm64",
 311                 ]
 312             # Starting in LLVM 11 (which requires SDK 10.12) the build tries to
 313             # detect the SDK version by calling xcrun. Cross-compiles don't have
 314             # an xcrun, so we have to set the version explicitly.
 315             cmake_args += [
 316                 "-DDARWIN_macosx_OVERRIDE_SDK_VERSION=%s"
 317                 % os.environ["MACOSX_DEPLOYMENT_TARGET"],
 318             ]
 319
 320         if profile == "gen":
 321             # Per https://releases.llvm.org/10.0.0/docs/HowToBuildWithPGO.html
 322             cmake_args += [
 323                 "-DLLVM_BUILD_INSTRUMENTED=IR",
 324                 "-DLLVM_BUILD_RUNTIME=No",
 325             ]
 326         elif profile:
 327             cmake_args += [
 328                 "-DLLVM_PROFDATA_FILE=%s" % profile,
 329             ]
 330
 331         # Using LTO for both profile generation and usage to avoid most
 332         # "function control flow change detected (hash mismatch)" error.
 333         if profile and not is_windows(target):
 334             cmake_args.append("-DLLVM_ENABLE_LTO=Thin")
 335         return cmake_args
 336
 337     cmake_args = []
 338     cmake_args += cmake_base_args(cc, cxx, asm, ar, ranlib, ldflags, inst_dir)
 339     cmake_args += [src_dir]
 340     build_package(build_dir, cmake_args)
 341
 342     # For some reasons the import library clang.lib of clang.exe is not
 343     # installed, so we copy it by ourselves.
 344     if is_windows(target) and is_final_stage:
 345         install_import_library(build_dir, inst_dir)
 346
 347
 348 # Return the absolute path of a build tool.  We first look to see if the
 349 # variable is defined in the config file, and if so we make sure it's an
 350 # absolute path to an existing tool, otherwise we look for a program in
 351 # $PATH named "key".
 352 #
 353 # This expects the name of the key in the config file to match the name of
 354 # the tool in the default toolchain on the system (for example, "ld" on Unix
 355 # and "link" on Windows).
 356 def get_tool(config, key):
 357     f = None
 358     if key in config:
 359         f = config[key].format(**os.environ)
 360         if os.path.isabs(f):
 361             if not os.path.exists(f):
 362                 raise ValueError("%s must point to an existing path" % key)
 363             return f
 364
 365     # Assume that we have the name of some program that should be on PATH.
 366     tool = which(f) if f else which(key)
 367     if not tool:
 368         raise ValueError("%s not found on PATH" % (f or key))
 369     return tool
 370
 371
 372 # This function is intended to be called on the final build directory when
 373 # building clang-tidy. Also clang-format binaries are included that can be used
 374 # in conjunction with clang-tidy.
 375 # As a separate binary we also ship clangd for the language server protocol that
 376 # can be used as a plugin in `vscode`.
 377 # Its job is to remove all of the files which won't be used for clang-tidy or
 378 # clang-format to reduce the download size.  Currently when this function
 379 # finishes its job, it will leave final_dir with a layout like this:
 380 #
 381 # clang/
 382 #   bin/
 383 #     clang-apply-replacements
 384 #     clang-format
 385 #     clang-tidy
 386 #     clangd
 387 #     run-clang-tidy
 388 #   include/
 389 #     * (nothing will be deleted here)
 390 #   lib/
 391 #     clang/
 392 #       4.0.0/
 393 #         include/
 394 #           * (nothing will be deleted here)
 395 #   share/
 396 #     clang/
 397 #       clang-format-diff.py
 398 #       clang-tidy-diff.py
 399 #       run-clang-tidy.py
 400 def prune_final_dir_for_clang_tidy(final_dir, target):
 401     # Make sure we only have what we expect.
 402     dirs = [
 403         "bin",
 404         "include",
 405         "lib",
 406         "lib32",
 407         "libexec",
 408         "msbuild-bin",
 409         "share",
 410         "tools",
 411     ]
 412     if is_linux(target):
 413         dirs.append("x86_64-unknown-linux-gnu")
 414     for f in glob.glob("%s/*" % final_dir):
 415         if os.path.basename(f) not in dirs:
 416             raise Exception("Found unknown file %s in the final directory" % f)
 417         if not os.path.isdir(f):
 418             raise Exception("Expected %s to be a directory" % f)
 419
 420     kept_binaries = [
 421         "clang-apply-replacements",
 422         "clang-format",
 423         "clang-tidy",
 424         "clangd",
 425         "clang-query",
 426         "run-clang-tidy",
 427     ]
 428     re_clang_tidy = re.compile(r"^(" + "|".join(kept_binaries) + r")(\.exe)?$", re.I)
 429     for f in glob.glob("%s/bin/*" % final_dir):
 430         if re_clang_tidy.search(os.path.basename(f)) is None:
 431             delete(f)
 432
 433     # Keep include/ intact.
 434
 435     # Remove the target-specific files.
 436     if is_linux(target):
 437         if os.path.exists(os.path.join(final_dir, "x86_64-unknown-linux-gnu")):
 438             shutil.rmtree(os.path.join(final_dir, "x86_64-unknown-linux-gnu"))
 439
 440     # In lib/, only keep lib/clang/N.M.O/include and the LLVM shared library.
 441     re_ver_num = re.compile(r"^\d+(?:\.\d+\.\d+)?$", re.I)
 442     for f in glob.glob("%s/lib/*" % final_dir):
 443         name = os.path.basename(f)
 444         if name == "clang":
 445             continue
 446         if is_darwin(target) and name in ["libLLVM.dylib", "libclang-cpp.dylib"]:
 447             continue
 448         if is_linux(target) and (
 449             fnmatch.fnmatch(name, "libLLVM*.so")
 450             or fnmatch.fnmatch(name, "libclang-cpp.so*")
 451         ):
 452             continue
 453         delete(f)
 454     for f in glob.glob("%s/lib/clang/*" % final_dir):
 455         if re_ver_num.search(os.path.basename(f)) is None:
 456             delete(f)
 457     for f in glob.glob("%s/lib/clang/*/*" % final_dir):
 458         if os.path.basename(f) != "include":
 459             delete(f)
 460
 461     # Completely remove libexec/, msbuild-bin and tools, if it exists.
 462     shutil.rmtree(os.path.join(final_dir, "libexec"))
 463     for d in ("msbuild-bin", "tools"):
 464         d = os.path.join(final_dir, d)
 465         if os.path.exists(d):
 466             shutil.rmtree(d)
 467
 468     # In share/, only keep share/clang/*tidy*
 469     re_clang_tidy = re.compile(r"format|tidy", re.I)
 470     for f in glob.glob("%s/share/*" % final_dir):
 471         if os.path.basename(f) != "clang":
 472             delete(f)
 473     for f in glob.glob("%s/share/clang/*" % final_dir):
 474         if re_clang_tidy.search(os.path.basename(f)) is None:
 475             delete(f)
 476
 477
 478 def main():
 479     parser = argparse.ArgumentParser()
 480     parser.add_argument(
 481         "-c",
 482         "--config",
 483         action="append",
 484         required=True,
 485         type=argparse.FileType("r"),
 486         help="Clang configuration file",
 487     )
 488     parser.add_argument(
 489         "--clean", required=False, action="store_true", help="Clean the build directory"
 490     )
 491     parser.add_argument(
 492         "--skip-tar",
 493         required=False,
 494         action="store_true",
 495         help="Skip tar packaging stage",
 496     )
 497     parser.add_argument(
 498         "--skip-patch",
 499         required=False,
 500         action="store_true",
 501         help="Do not patch source",
 502     )
 503
 504     args = parser.parse_args()
 505
 506     if not os.path.exists("llvm/README.txt"):
 507         raise Exception(
 508             "The script must be run from the root directory of the llvm-project tree"
 509         )
 510     source_dir = os.getcwd()
 511     build_dir = source_dir + "/build"
 512
 513     if args.clean:
 514         shutil.rmtree(build_dir)
 515         os.sys.exit(0)
 516
 517     llvm_source_dir = source_dir + "/llvm"
 518
 519     config = {}
 520     # Merge all the configs we got from the command line.
 521     for c in args.config:
 522         this_config_dir = os.path.dirname(c.name)
 523         this_config = json.load(c)
 524         patches = this_config.get("patches")
 525         if patches:
 526             this_config["patches"] = [os.path.join(this_config_dir, p) for p in patches]
 527         for key, value in this_config.items():
 528             old_value = config.get(key)
 529             if old_value is None:
 530                 config[key] = value
 531             elif value is None:
 532                 if key in config:
 533                     del config[key]
 534             elif type(old_value) != type(value):
 535                 raise Exception(
 536                     "{} is overriding `{}` with a value of the wrong type".format(
 537                         c.name, key
 538                     )
 539                 )
 540             elif isinstance(old_value, list):
 541                 for v in value:
 542                     if v not in old_value:
 543                         old_value.append(v)
 544             elif isinstance(old_value, dict):
 545                 raise Exception("{} is setting `{}` to a dict?".format(c.name, key))
 546             else:
 547                 config[key] = value
 548
 549     stages = 2
 550     if "stages" in config:
 551         stages = int(config["stages"])
 552         if stages not in (1, 2, 3, 4):
 553             raise ValueError("We only know how to build 1, 2, 3, or 4 stages.")
 554     skip_stages = 0
 555     if "skip_stages" in config:
 556         # The assumption here is that the compiler given in `cc` and other configs
 557         # is the result of the last skip stage, built somewhere else.
 558         skip_stages = int(config["skip_stages"])
 559         if skip_stages >= stages:
 560             raise ValueError("Cannot skip more stages than are built.")
 561     pgo = False
 562     if "pgo" in config:
 563         pgo = config["pgo"]
 564         if pgo not in (True, False):
 565             raise ValueError("Only boolean values are accepted for pgo.")
 566     build_type = "Release"
 567     if "build_type" in config:
 568         build_type = config["build_type"]
 569         if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"):
 570             raise ValueError(
 571                 "We only know how to do Release, Debug, RelWithDebInfo or "
 572                 "MinSizeRel builds"
 573             )
 574     targets = config.get("targets")
 575     build_clang_tidy = False
 576     if "build_clang_tidy" in config:
 577         build_clang_tidy = config["build_clang_tidy"]
 578         if build_clang_tidy not in (True, False):
 579             raise ValueError("Only boolean values are accepted for build_clang_tidy.")
 580     build_clang_tidy_alpha = False
 581     # check for build_clang_tidy_alpha only if build_clang_tidy is true
 582     if build_clang_tidy and "build_clang_tidy_alpha" in config:
 583         build_clang_tidy_alpha = config["build_clang_tidy_alpha"]
 584         if build_clang_tidy_alpha not in (True, False):
 585             raise ValueError(
 586                 "Only boolean values are accepted for build_clang_tidy_alpha."
 587             )
 588     build_clang_tidy_external = False
 589     # check for build_clang_tidy_external only if build_clang_tidy is true
 590     if build_clang_tidy and "build_clang_tidy_external" in config:
 591         build_clang_tidy_external = config["build_clang_tidy_external"]
 592         if build_clang_tidy_external not in (True, False):
 593             raise ValueError(
 594                 "Only boolean values are accepted for build_clang_tidy_external."
 595             )
 596     assertions = False
 597     if "assertions" in config:
 598         assertions = config["assertions"]
 599         if assertions not in (True, False):
 600             raise ValueError("Only boolean values are accepted for assertions.")
 601
 602     for t in SUPPORTED_TARGETS:
 603         if not is_cross_compile(t):
 604             host = t
 605             break
 606     else:
 607         raise Exception(
 608             f"Cannot use this script on {platform.system()} {platform.machine()}"
 609         )
 610
 611     target = config.get("target", host)
 612     if target not in SUPPORTED_TARGETS:
 613         raise ValueError(f"{target} is not a supported target.")
 614
 615     if is_cross_compile(target) and not is_linux(host):
 616         raise Exception("Cross-compilation is only supported on Linux")
 617
 618     if is_darwin(target):
 619         os.environ["MACOSX_DEPLOYMENT_TARGET"] = (
 620             "11.0" if target.startswith("aarch64") else "10.12"
 621         )
 622
 623     if is_windows(target):
 624         exe_ext = ".exe"
 625         cc_name = "clang-cl"
 626         cxx_name = "clang-cl"
 627     else:
 628         exe_ext = ""
 629         cc_name = "clang"
 630         cxx_name = "clang++"
 631
 632     cc = get_tool(config, "cc")
 633     cxx = get_tool(config, "cxx")
 634     asm = get_tool(config, "ml" if is_windows(target) else "as")
 635     # Not using lld here as default here because it's not in PATH. But clang
 636     # knows how to find it when they are installed alongside each others.
 637     ar = get_tool(config, "lib" if is_windows(target) else "ar")
 638     ranlib = None if is_windows(target) else get_tool(config, "ranlib")
 639
 640     if not os.path.exists(source_dir):
 641         os.makedirs(source_dir)
 642
 643     if not args.skip_patch:
 644         for p in config.get("patches", []):
 645             patch(p, source_dir)
 646
 647     package_name = "clang"
 648     if build_clang_tidy:
 649         package_name = "clang-tidy"
 650         if not args.skip_patch:
 651             import_clang_tidy(
 652                 source_dir, build_clang_tidy_alpha, build_clang_tidy_external
 653             )
 654
 655     if not os.path.exists(build_dir):
 656         os.makedirs(build_dir)
 657
 658     stage1_dir = build_dir + "/stage1"
 659     stage1_inst_dir = stage1_dir + "/" + package_name
 660
 661     final_stage_dir = stage1_dir
 662
 663     if is_darwin(target):
 664         extra_cflags = []
 665         extra_cxxflags = []
 666         extra_cflags2 = []
 667         extra_cxxflags2 = []
 668         extra_asmflags = []
 669         # It's unfortunately required to specify the linker used here because
 670         # the linker flags are used in LLVM's configure step before
 671         # -DLLVM_ENABLE_LLD is actually processed.
 672         extra_ldflags = [
 673             "-fuse-ld=lld",
 674             "-Wl,-dead_strip",
 675         ]
 676     elif is_linux(target):
 677         extra_cflags = []
 678         extra_cxxflags = []
 679         extra_cflags2 = ["-fPIC"]
 680         # Silence clang's warnings about arguments not being used in compilation.
 681         extra_cxxflags2 = [
 682             "-fPIC",
 683             "-Qunused-arguments",
 684         ]
 685         extra_asmflags = []
 686         # Avoid libLLVM internal function calls going through the PLT.
 687         extra_ldflags = ["-Wl,-Bsymbolic-functions"]
 688         # For whatever reason, LLVM's build system will set things up to turn
 689         # on -ffunction-sections and -fdata-sections, but won't turn on the
 690         # corresponding option to strip unused sections.  We do it explicitly
 691         # here.  LLVM's build system is also picky about turning on ICF, so
 692         # we do that explicitly here, too.
 693
 694         # It's unfortunately required to specify the linker used here because
 695         # the linker flags are used in LLVM's configure step before
 696         # -DLLVM_ENABLE_LLD is actually processed.
 697         if is_llvm_toolchain(cc, cxx):
 698             extra_ldflags += ["-fuse-ld=lld", "-Wl,--icf=safe"]
 699         extra_ldflags += ["-Wl,--gc-sections"]
 700     elif is_windows(target):
 701         extra_cflags = []
 702         extra_cxxflags = []
 703         # clang-cl would like to figure out what it's supposed to be emulating
 704         # by looking at an MSVC install, but we don't really have that here.
 705         # Force things on based on WinMsvc.cmake.
 706         # Ideally, we'd just use WinMsvc.cmake as a toolchain file, but it only
 707         # really works for cross-compiles, which this is not.
 708         with open(os.path.join(llvm_source_dir, "cmake/platforms/WinMsvc.cmake")) as f:
 709             compat = [
 710                 item
 711                 for line in f
 712                 for item in line.split()
 713                 if "-fms-compatibility-version=" in item
 714             ][0]
 715         extra_cflags2 = [compat]
 716         extra_cxxflags2 = [compat]
 717         extra_asmflags = []
 718         extra_ldflags = []
 719
 720     upload_dir = os.getenv("UPLOAD_DIR")
 721     if assertions and upload_dir:
 722         extra_cflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir]
 723         extra_cxxflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir]
 724
 725     if skip_stages < 1:
 726         build_one_stage(
 727             [cc] + extra_cflags,
 728             [cxx] + extra_cxxflags,
 729             [asm] + extra_asmflags,
 730             ar,
 731             ranlib,
 732             extra_ldflags,
 733             llvm_source_dir,
 734             stage1_dir,
 735             package_name,
 736             build_type,
 737             assertions,
 738             target,
 739             targets,
 740             is_final_stage=(stages == 1),
 741         )
 742
 743     if stages >= 2 and skip_stages < 2:
 744         stage2_dir = build_dir + "/stage2"
 745         stage2_inst_dir = stage2_dir + "/" + package_name
 746         final_stage_dir = stage2_dir
 747         if skip_stages < 1:
 748             cc = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 749             cxx = stage1_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
 750             asm = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 751         build_one_stage(
 752             [cc] + extra_cflags2,
 753             [cxx] + extra_cxxflags2,
 754             [asm] + extra_asmflags,
 755             ar,
 756             ranlib,
 757             extra_ldflags,
 758             llvm_source_dir,
 759             stage2_dir,
 760             package_name,
 761             build_type,
 762             assertions,
 763             target,
 764             targets,
 765             is_final_stage=(stages == 2),
 766             profile="gen" if pgo else None,
 767         )
 768
 769     if stages >= 3 and skip_stages < 3:
 770         stage3_dir = build_dir + "/stage3"
 771         stage3_inst_dir = stage3_dir + "/" + package_name
 772         final_stage_dir = stage3_dir
 773         if skip_stages < 2:
 774             cc = stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 775             cxx = stage2_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
 776             asm = stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 777         build_one_stage(
 778             [cc] + extra_cflags2,
 779             [cxx] + extra_cxxflags2,
 780             [asm] + extra_asmflags,
 781             ar,
 782             ranlib,
 783             extra_ldflags,
 784             llvm_source_dir,
 785             stage3_dir,
 786             package_name,
 787             build_type,
 788             assertions,
 789             target,
 790             targets,
 791             (stages == 3),
 792         )
 793         if pgo:
 794             llvm_profdata = stage2_inst_dir + "/bin/llvm-profdata%s" % exe_ext
 795             merge_cmd = [llvm_profdata, "merge", "-o", "merged.profdata"]
 796             profraw_files = glob.glob(
 797                 os.path.join(stage2_dir, "build", "profiles", "*.profraw")
 798             )
 799             run_in(stage3_dir, merge_cmd + profraw_files)
 800             if stages == 3:
 801                 mkdir_p(upload_dir)
 802                 shutil.copy2(os.path.join(stage3_dir, "merged.profdata"), upload_dir)
 803                 return
 804
 805     if stages >= 4 and skip_stages < 4:
 806         stage4_dir = build_dir + "/stage4"
 807         final_stage_dir = stage4_dir
 808         profile = None
 809         if pgo:
 810             if skip_stages == 3:
 811                 profile_dir = os.environ.get("MOZ_FETCHES_DIR", "")
 812             else:
 813                 profile_dir = stage3_dir
 814             profile = os.path.join(profile_dir, "merged.profdata")
 815         if skip_stages < 3:
 816             cc = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 817             cxx = stage3_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
 818             asm = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 819         build_one_stage(
 820             [cc] + extra_cflags2,
 821             [cxx] + extra_cxxflags2,
 822             [asm] + extra_asmflags,
 823             ar,
 824             ranlib,
 825             extra_ldflags,
 826             llvm_source_dir,
 827             stage4_dir,
 828             package_name,
 829             build_type,
 830             assertions,
 831             target,
 832             targets,
 833             (stages == 4),
 834             profile=profile,
 835         )
 836
 837     if build_clang_tidy:
 838         prune_final_dir_for_clang_tidy(
 839             os.path.join(final_stage_dir, package_name), target
 840         )
 841
 842     if not args.skip_tar:
 843         build_tar_package("%s.tar.zst" % package_name, final_stage_dir, package_name)
 844
 845
 846 if __name__ == "__main__":
 847     main()