build/build-clang/build-clang.py

   1 #!/usr/bin/python3
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 # Only necessary for flake8 to be happy...
   7 from __future__ import print_function
   8
   9 import os
  10 import os.path
  11 import shutil
  12 import subprocess
  13 import platform
  14 import json
  15 import argparse
  16 import fnmatch
  17 import glob
  18 import errno
  19 import re
  20 import sys
  21 import tarfile
  22 from contextlib import contextmanager
  23 from distutils.dir_util import copy_tree
  24
  25 from shutil import which
  26
  27 import zstandard
  28
  29
  30 def symlink(source, link_name):
  31     os_symlink = getattr(os, "symlink", None)
  32     if callable(os_symlink):
  33         os_symlink(source, link_name)
  34     else:
  35         if os.path.isdir(source):
  36             # Fall back to copying the directory :(
  37             copy_tree(source, link_name)
  38
  39
  40 def check_run(args):
  41     print(" ".join(args), file=sys.stderr, flush=True)
  42     if args[0] == "cmake":
  43         # CMake `message(STATUS)` messages, as appearing in failed source code
  44         # compiles, appear on stdout, so we only capture that.
  45         p = subprocess.Popen(args, stdout=subprocess.PIPE)
  46         lines = []
  47         for line in p.stdout:
  48             lines.append(line)
  49             sys.stdout.write(line.decode())
  50             sys.stdout.flush()
  51         r = p.wait()
  52         if r != 0 and os.environ.get("UPLOAD_DIR"):
  53             cmake_output_re = re.compile(b'See also "(.*/CMakeOutput.log)"')
  54             cmake_error_re = re.compile(b'See also "(.*/CMakeError.log)"')
  55
  56             def find_first_match(re):
  57                 for l in lines:
  58                     match = re.search(l)
  59                     if match:
  60                         return match
  61
  62             output_match = find_first_match(cmake_output_re)
  63             error_match = find_first_match(cmake_error_re)
  64
  65             upload_dir = os.environ["UPLOAD_DIR"].encode("utf-8")
  66             if output_match or error_match:
  67                 mkdir_p(upload_dir)
  68             if output_match:
  69                 shutil.copy2(output_match.group(1), upload_dir)
  70             if error_match:
  71                 shutil.copy2(error_match.group(1), upload_dir)
  72     else:
  73         r = subprocess.call(args)
  74     assert r == 0
  75
  76
  77 def run_in(path, args):
  78     with chdir(path):
  79         check_run(args)
  80
  81
  82 @contextmanager
  83 def chdir(path):
  84     d = os.getcwd()
  85     print('cd "%s"' % path, file=sys.stderr)
  86     os.chdir(path)
  87     try:
  88         yield
  89     finally:
  90         print('cd "%s"' % d, file=sys.stderr)
  91         os.chdir(d)
  92
  93
  94 def patch(patch, srcdir):
  95     patch = os.path.realpath(patch)
  96     check_run(["patch", "-d", srcdir, "-p1", "-i", patch, "--fuzz=0", "-s"])
  97
  98
  99 def import_clang_tidy(source_dir, build_clang_tidy_alpha, build_clang_tidy_external):
 100     clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]), "..", "clang-plugin")
 101     clang_tidy_path = os.path.join(source_dir, "clang-tools-extra/clang-tidy")
 102     sys.path.append(clang_plugin_path)
 103     from import_mozilla_checks import do_import
 104
 105     import_options = {
 106         "alpha": build_clang_tidy_alpha,
 107         "external": build_clang_tidy_external,
 108     }
 109     do_import(clang_plugin_path, clang_tidy_path, import_options)
 110
 111
 112 def build_package(package_build_dir, cmake_args):
 113     if not os.path.exists(package_build_dir):
 114         os.mkdir(package_build_dir)
 115     # If CMake has already been run, it may have been run with different
 116     # arguments, so we need to re-run it.  Make sure the cached copy of the
 117     # previous CMake run is cleared before running it again.
 118     if os.path.exists(package_build_dir + "/CMakeCache.txt"):
 119         os.remove(package_build_dir + "/CMakeCache.txt")
 120     if os.path.exists(package_build_dir + "/CMakeFiles"):
 121         shutil.rmtree(package_build_dir + "/CMakeFiles")
 122
 123     run_in(package_build_dir, ["cmake"] + cmake_args)
 124     run_in(package_build_dir, ["ninja", "install", "-v"])
 125
 126
 127 @contextmanager
 128 def updated_env(env):
 129     old_env = os.environ.copy()
 130     os.environ.update(env)
 131     yield
 132     os.environ.clear()
 133     os.environ.update(old_env)
 134
 135
 136 def build_tar_package(name, base, directory):
 137     name = os.path.realpath(name)
 138     print("tarring {} from {}/{}".format(name, base, directory), file=sys.stderr)
 139     assert name.endswith(".tar.zst")
 140
 141     cctx = zstandard.ZstdCompressor()
 142     with open(name, "wb") as f, cctx.stream_writer(f) as z:
 143         with tarfile.open(mode="w|", fileobj=z) as tf:
 144             with chdir(base):
 145                 tf.add(directory)
 146
 147
 148 def mkdir_p(path):
 149     try:
 150         os.makedirs(path)
 151     except OSError as e:
 152         if e.errno != errno.EEXIST or not os.path.isdir(path):
 153             raise
 154
 155
 156 def delete(path):
 157     if os.path.isdir(path):
 158         shutil.rmtree(path)
 159     else:
 160         try:
 161             os.unlink(path)
 162         except Exception:
 163             pass
 164
 165
 166 def install_import_library(build_dir, clang_dir):
 167     shutil.copy2(
 168         os.path.join(build_dir, "lib", "clang.lib"), os.path.join(clang_dir, "lib")
 169     )
 170
 171
 172 def is_darwin():
 173     return platform.system() == "Darwin"
 174
 175
 176 def is_linux():
 177     return platform.system() == "Linux"
 178
 179
 180 def is_windows():
 181     return platform.system() == "Windows"
 182
 183
 184 def build_one_stage(
 185     cc,
 186     cxx,
 187     asm,
 188     ld,
 189     ar,
 190     ranlib,
 191     libtool,
 192     src_dir,
 193     stage_dir,
 194     package_name,
 195     build_libcxx,
 196     osx_cross_compile,
 197     build_type,
 198     assertions,
 199     libcxx_include_dir,
 200     build_wasm,
 201     is_final_stage=False,
 202     profile=None,
 203 ):
 204     if not os.path.exists(stage_dir):
 205         os.mkdir(stage_dir)
 206
 207     build_dir = stage_dir + "/build"
 208     inst_dir = stage_dir + "/" + package_name
 209
 210     # cmake doesn't deal well with backslashes in paths.
 211     def slashify_path(path):
 212         return path.replace("\\", "/")
 213
 214     def cmake_base_args(cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir):
 215         machine_targets = "X86;ARM;AArch64" if is_final_stage else "X86"
 216         if build_wasm and is_final_stage:
 217             machine_targets += ";WebAssembly"
 218
 219         cmake_args = [
 220             "-GNinja",
 221             "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]),
 222             "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]),
 223             "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]),
 224             "-DCMAKE_LINKER=%s" % slashify_path(ld[0]),
 225             "-DCMAKE_AR=%s" % slashify_path(ar),
 226             "-DCMAKE_C_FLAGS=%s" % " ".join(cc[1:]),
 227             "-DCMAKE_CXX_FLAGS=%s" % " ".join(cxx[1:]),
 228             "-DCMAKE_ASM_FLAGS=%s" % " ".join(asm[1:]),
 229             "-DCMAKE_EXE_LINKER_FLAGS=%s" % " ".join(ld[1:]),
 230             "-DCMAKE_SHARED_LINKER_FLAGS=%s" % " ".join(ld[1:]),
 231             "-DCMAKE_BUILD_TYPE=%s" % build_type,
 232             "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir,
 233             "-DLLVM_TARGETS_TO_BUILD=%s" % machine_targets,
 234             "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"),
 235             "-DLLVM_ENABLE_BINDINGS=OFF",
 236             "-DLLVM_ENABLE_CURL=OFF",
 237             "-DLLVM_INCLUDE_TESTS=OFF",
 238         ]
 239         if "TASK_ID" in os.environ:
 240             cmake_args += [
 241                 "-DCLANG_REPOSITORY_STRING=taskcluster-%s" % os.environ["TASK_ID"],
 242             ]
 243         # libc++ doesn't build with MSVC because of the use of #include_next.
 244         if is_final_stage and os.path.basename(cc[0]).lower() != "cl.exe":
 245             cmake_args += [
 246                 "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"),
 247                 # libc++abi has conflicting definitions between the shared and static
 248                 # library on Windows because of the import library for the dll having
 249                 # the same name as the static library. libc++abi is not necessary on
 250                 # Windows anyways.
 251                 "-DLLVM_TOOL_LIBCXXABI_BUILD=%s" % ("OFF" if is_windows() else "ON"),
 252             ]
 253         if not is_final_stage:
 254             cmake_args += [
 255                 "-DLLVM_ENABLE_PROJECTS=clang",
 256                 "-DLLVM_TOOL_LLI_BUILD=OFF",
 257             ]
 258
 259         # There is no libxml2 on Windows except if we build one ourselves.
 260         # libxml2 is only necessary for llvm-mt, but Windows can just use the
 261         # native MT tool.
 262         if not is_windows() and is_final_stage:
 263             cmake_args += ["-DLLVM_ENABLE_LIBXML2=FORCE_ON"]
 264         if is_linux() and not osx_cross_compile and is_final_stage:
 265             sysroot = os.path.join(os.environ.get("MOZ_FETCHES_DIR", ""), "sysroot")
 266             if os.path.exists(sysroot):
 267                 cmake_args += ["-DLLVM_BINUTILS_INCDIR=/usr/include"]
 268                 cmake_args += ["-DCMAKE_SYSROOT=%s" % sysroot]
 269                 # Work around the LLVM build system not building the i386 compiler-rt
 270                 # because it doesn't allow to use a sysroot for that during the cmake
 271                 # checks.
 272                 cmake_args += ["-DCAN_TARGET_i386=1"]
 273         if is_windows():
 274             cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
 275             cmake_args.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
 276         else:
 277             # libllvm as a shared library is not supported on Windows
 278             cmake_args += ["-DLLVM_LINK_LLVM_DYLIB=ON"]
 279         if ranlib is not None:
 280             cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
 281         if libtool is not None:
 282             cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
 283         if osx_cross_compile:
 284             arch = "arm64" if os.environ.get("OSX_ARCH") == "arm64" else "x86_64"
 285             target_cpu = (
 286                 "aarch64" if os.environ.get("OSX_ARCH") == "arm64" else "x86_64"
 287             )
 288             cmake_args += [
 289                 "-DCMAKE_SYSTEM_NAME=Darwin",
 290                 "-DCMAKE_SYSTEM_VERSION=%s" % os.environ["MACOSX_DEPLOYMENT_TARGET"],
 291                 "-DLIBCXXABI_LIBCXX_INCLUDES=%s" % libcxx_include_dir,
 292                 "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 293                 "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 294                 "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
 295                 "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
 296                 "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
 297                 "-DCMAKE_MACOSX_RPATH=ON",
 298                 "-DCMAKE_OSX_ARCHITECTURES=%s" % arch,
 299                 "-DDARWIN_osx_ARCHS=%s" % arch,
 300                 "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 301                 "-DLLVM_DEFAULT_TARGET_TRIPLE=%s-apple-darwin" % target_cpu,
 302             ]
 303             if os.environ.get("OSX_ARCH") == "arm64":
 304                 cmake_args += [
 305                     "-DDARWIN_osx_BUILTIN_ARCHS=arm64",
 306                 ]
 307             # Starting in LLVM 11 (which requires SDK 10.12) the build tries to
 308             # detect the SDK version by calling xcrun. Cross-compiles don't have
 309             # an xcrun, so we have to set the version explicitly.
 310             cmake_args += [
 311                 "-DDARWIN_macosx_OVERRIDE_SDK_VERSION=%s"
 312                 % os.environ["MACOSX_DEPLOYMENT_TARGET"],
 313             ]
 314         if profile == "gen":
 315             # Per https://releases.llvm.org/10.0.0/docs/HowToBuildWithPGO.html
 316             cmake_args += [
 317                 "-DLLVM_BUILD_INSTRUMENTED=IR",
 318                 "-DLLVM_BUILD_RUNTIME=No",
 319             ]
 320         elif profile:
 321             cmake_args += [
 322                 "-DLLVM_PROFDATA_FILE=%s" % profile,
 323             ]
 324         return cmake_args
 325
 326     cmake_args = []
 327     cmake_args += cmake_base_args(cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir)
 328     cmake_args += [src_dir]
 329     build_package(build_dir, cmake_args)
 330
 331     # For some reasons the import library clang.lib of clang.exe is not
 332     # installed, so we copy it by ourselves.
 333     if is_windows() and is_final_stage:
 334         install_import_library(build_dir, inst_dir)
 335
 336
 337 # Return the absolute path of a build tool.  We first look to see if the
 338 # variable is defined in the config file, and if so we make sure it's an
 339 # absolute path to an existing tool, otherwise we look for a program in
 340 # $PATH named "key".
 341 #
 342 # This expects the name of the key in the config file to match the name of
 343 # the tool in the default toolchain on the system (for example, "ld" on Unix
 344 # and "link" on Windows).
 345 def get_tool(config, key):
 346     f = None
 347     if key in config:
 348         f = config[key].format(**os.environ)
 349         if os.path.isabs(f):
 350             if not os.path.exists(f):
 351                 raise ValueError("%s must point to an existing path" % key)
 352             return f
 353
 354     # Assume that we have the name of some program that should be on PATH.
 355     tool = which(f) if f else which(key)
 356     if not tool:
 357         raise ValueError("%s not found on PATH" % (f or key))
 358     return tool
 359
 360
 361 # This function is intended to be called on the final build directory when
 362 # building clang-tidy. Also clang-format binaries are included that can be used
 363 # in conjunction with clang-tidy.
 364 # As a separate binary we also ship clangd for the language server protocol that
 365 # can be used as a plugin in `vscode`.
 366 # Its job is to remove all of the files which won't be used for clang-tidy or
 367 # clang-format to reduce the download size.  Currently when this function
 368 # finishes its job, it will leave final_dir with a layout like this:
 369 #
 370 # clang/
 371 #   bin/
 372 #     clang-apply-replacements
 373 #     clang-format
 374 #     clang-tidy
 375 #     clangd
 376 #     run-clang-tidy
 377 #   include/
 378 #     * (nothing will be deleted here)
 379 #   lib/
 380 #     clang/
 381 #       4.0.0/
 382 #         include/
 383 #           * (nothing will be deleted here)
 384 #   share/
 385 #     clang/
 386 #       clang-format-diff.py
 387 #       clang-tidy-diff.py
 388 #       run-clang-tidy.py
 389 def prune_final_dir_for_clang_tidy(final_dir, osx_cross_compile):
 390     # Make sure we only have what we expect.
 391     dirs = [
 392         "bin",
 393         "include",
 394         "lib",
 395         "lib32",
 396         "libexec",
 397         "msbuild-bin",
 398         "share",
 399         "tools",
 400     ]
 401     if is_linux():
 402         dirs.append("x86_64-unknown-linux-gnu")
 403     for f in glob.glob("%s/*" % final_dir):
 404         if os.path.basename(f) not in dirs:
 405             raise Exception("Found unknown file %s in the final directory" % f)
 406         if not os.path.isdir(f):
 407             raise Exception("Expected %s to be a directory" % f)
 408
 409     kept_binaries = [
 410         "clang-apply-replacements",
 411         "clang-format",
 412         "clang-tidy",
 413         "clangd",
 414         "clang-query",
 415         "run-clang-tidy",
 416     ]
 417     re_clang_tidy = re.compile(r"^(" + "|".join(kept_binaries) + r")(\.exe)?$", re.I)
 418     for f in glob.glob("%s/bin/*" % final_dir):
 419         if re_clang_tidy.search(os.path.basename(f)) is None:
 420             delete(f)
 421
 422     # Keep include/ intact.
 423
 424     # Remove the target-specific files.
 425     if is_linux():
 426         if os.path.exists(os.path.join(final_dir, "x86_64-unknown-linux-gnu")):
 427             shutil.rmtree(os.path.join(final_dir, "x86_64-unknown-linux-gnu"))
 428
 429     # In lib/, only keep lib/clang/N.M.O/include and the LLVM shared library.
 430     re_ver_num = re.compile(r"^\d+\.\d+\.\d+$", re.I)
 431     for f in glob.glob("%s/lib/*" % final_dir):
 432         name = os.path.basename(f)
 433         if name == "clang":
 434             continue
 435         if osx_cross_compile and name in ["libLLVM.dylib", "libclang-cpp.dylib"]:
 436             continue
 437         if is_linux() and (
 438             fnmatch.fnmatch(name, "libLLVM*.so")
 439             or fnmatch.fnmatch(name, "libclang-cpp.so*")
 440         ):
 441             continue
 442         delete(f)
 443     for f in glob.glob("%s/lib/clang/*" % final_dir):
 444         if re_ver_num.search(os.path.basename(f)) is None:
 445             delete(f)
 446     for f in glob.glob("%s/lib/clang/*/*" % final_dir):
 447         if os.path.basename(f) != "include":
 448             delete(f)
 449
 450     # Completely remove libexec/, msbuild-bin and tools, if it exists.
 451     shutil.rmtree(os.path.join(final_dir, "libexec"))
 452     for d in ("msbuild-bin", "tools"):
 453         d = os.path.join(final_dir, d)
 454         if os.path.exists(d):
 455             shutil.rmtree(d)
 456
 457     # In share/, only keep share/clang/*tidy*
 458     re_clang_tidy = re.compile(r"format|tidy", re.I)
 459     for f in glob.glob("%s/share/*" % final_dir):
 460         if os.path.basename(f) != "clang":
 461             delete(f)
 462     for f in glob.glob("%s/share/clang/*" % final_dir):
 463         if re_clang_tidy.search(os.path.basename(f)) is None:
 464             delete(f)
 465
 466
 467 def main():
 468     parser = argparse.ArgumentParser()
 469     parser.add_argument(
 470         "-c",
 471         "--config",
 472         action="append",
 473         required=True,
 474         type=argparse.FileType("r"),
 475         help="Clang configuration file",
 476     )
 477     parser.add_argument(
 478         "--clean", required=False, action="store_true", help="Clean the build directory"
 479     )
 480     parser.add_argument(
 481         "--skip-tar",
 482         required=False,
 483         action="store_true",
 484         help="Skip tar packaging stage",
 485     )
 486     parser.add_argument(
 487         "--skip-patch",
 488         required=False,
 489         action="store_true",
 490         help="Do not patch source",
 491     )
 492
 493     args = parser.parse_args()
 494
 495     if not os.path.exists("llvm/README.txt"):
 496         raise Exception(
 497             "The script must be run from the root directory of the llvm-project tree"
 498         )
 499     source_dir = os.getcwd()
 500     build_dir = source_dir + "/build"
 501
 502     if args.clean:
 503         shutil.rmtree(build_dir)
 504         os.sys.exit(0)
 505
 506     llvm_source_dir = source_dir + "/llvm"
 507     extra_source_dir = source_dir + "/clang-tools-extra"
 508     clang_source_dir = source_dir + "/clang"
 509     lld_source_dir = source_dir + "/lld"
 510     libcxx_source_dir = source_dir + "/libcxx"
 511     libcxxabi_source_dir = source_dir + "/libcxxabi"
 512
 513     exe_ext = ""
 514     if is_windows():
 515         exe_ext = ".exe"
 516
 517     cc_name = "clang"
 518     cxx_name = "clang++"
 519     if is_windows():
 520         cc_name = "clang-cl"
 521         cxx_name = "clang-cl"
 522
 523     config = {}
 524     # Merge all the configs we got from the command line.
 525     for c in args.config:
 526         this_config_dir = os.path.dirname(c.name)
 527         this_config = json.load(c)
 528         patches = this_config.get("patches")
 529         if patches:
 530             this_config["patches"] = [os.path.join(this_config_dir, p) for p in patches]
 531         for key, value in this_config.items():
 532             old_value = config.get(key)
 533             if old_value is None:
 534                 config[key] = value
 535             elif value is None:
 536                 if key in config:
 537                     del config[key]
 538             elif type(old_value) != type(value):
 539                 raise Exception(
 540                     "{} is overriding `{}` with a value of the wrong type".format(
 541                         c.name, key
 542                     )
 543                 )
 544             elif isinstance(old_value, list):
 545                 for v in value:
 546                     if v not in old_value:
 547                         old_value.append(v)
 548             elif isinstance(old_value, dict):
 549                 raise Exception("{} is setting `{}` to a dict?".format(c.name, key))
 550             else:
 551                 config[key] = value
 552
 553     stages = 2
 554     if "stages" in config:
 555         stages = int(config["stages"])
 556         if stages not in (1, 2, 3, 4):
 557             raise ValueError("We only know how to build 1, 2, 3, or 4 stages.")
 558     skip_stages = 0
 559     if "skip_stages" in config:
 560         # The assumption here is that the compiler given in `cc` and other configs
 561         # is the result of the last skip stage, built somewhere else.
 562         skip_stages = int(config["skip_stages"])
 563         if skip_stages >= stages:
 564             raise ValueError("Cannot skip more stages than are built.")
 565     pgo = False
 566     if "pgo" in config:
 567         pgo = config["pgo"]
 568         if pgo not in (True, False):
 569             raise ValueError("Only boolean values are accepted for pgo.")
 570     build_type = "Release"
 571     if "build_type" in config:
 572         build_type = config["build_type"]
 573         if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"):
 574             raise ValueError(
 575                 "We only know how to do Release, Debug, RelWithDebInfo or "
 576                 "MinSizeRel builds"
 577             )
 578     build_libcxx = True
 579     if "build_libcxx" in config:
 580         build_libcxx = config["build_libcxx"]
 581         if build_libcxx not in (True, False):
 582             raise ValueError("Only boolean values are accepted for build_libcxx.")
 583     build_wasm = False
 584     if "build_wasm" in config:
 585         build_wasm = config["build_wasm"]
 586         if build_wasm not in (True, False):
 587             raise ValueError("Only boolean values are accepted for build_wasm.")
 588     build_clang_tidy = False
 589     if "build_clang_tidy" in config:
 590         build_clang_tidy = config["build_clang_tidy"]
 591         if build_clang_tidy not in (True, False):
 592             raise ValueError("Only boolean values are accepted for build_clang_tidy.")
 593     build_clang_tidy_alpha = False
 594     # check for build_clang_tidy_alpha only if build_clang_tidy is true
 595     if build_clang_tidy and "build_clang_tidy_alpha" in config:
 596         build_clang_tidy_alpha = config["build_clang_tidy_alpha"]
 597         if build_clang_tidy_alpha not in (True, False):
 598             raise ValueError(
 599                 "Only boolean values are accepted for build_clang_tidy_alpha."
 600             )
 601     build_clang_tidy_external = False
 602     # check for build_clang_tidy_external only if build_clang_tidy is true
 603     if build_clang_tidy and "build_clang_tidy_external" in config:
 604         build_clang_tidy_external = config["build_clang_tidy_external"]
 605         if build_clang_tidy_external not in (True, False):
 606             raise ValueError(
 607                 "Only boolean values are accepted for build_clang_tidy_external."
 608             )
 609     osx_cross_compile = False
 610     if "osx_cross_compile" in config:
 611         osx_cross_compile = config["osx_cross_compile"]
 612         if osx_cross_compile not in (True, False):
 613             raise ValueError("Only boolean values are accepted for osx_cross_compile.")
 614         if osx_cross_compile and not is_linux():
 615             raise ValueError("osx_cross_compile can only be used on Linux.")
 616     assertions = False
 617     if "assertions" in config:
 618         assertions = config["assertions"]
 619         if assertions not in (True, False):
 620             raise ValueError("Only boolean values are accepted for assertions.")
 621
 622     if is_darwin() or osx_cross_compile:
 623         os.environ["MACOSX_DEPLOYMENT_TARGET"] = (
 624             "11.0" if os.environ.get("OSX_ARCH") == "arm64" else "10.12"
 625         )
 626
 627     cc = get_tool(config, "cc")
 628     cxx = get_tool(config, "cxx")
 629     asm = get_tool(config, "ml" if is_windows() else "as")
 630     ld = get_tool(config, "link" if is_windows() else "ld")
 631     ar = get_tool(config, "lib" if is_windows() else "ar")
 632     ranlib = None if is_windows() else get_tool(config, "ranlib")
 633     libtool = None
 634     if "libtool" in config:
 635         libtool = get_tool(config, "libtool")
 636
 637     if not os.path.exists(source_dir):
 638         os.makedirs(source_dir)
 639
 640     if not args.skip_patch:
 641         for p in config.get("patches", []):
 642             patch(p, source_dir)
 643
 644     symlinks = [
 645         (clang_source_dir, llvm_source_dir + "/tools/clang"),
 646         (extra_source_dir, llvm_source_dir + "/tools/clang/tools/extra"),
 647         (lld_source_dir, llvm_source_dir + "/tools/lld"),
 648         (libcxx_source_dir, llvm_source_dir + "/projects/libcxx"),
 649         (libcxxabi_source_dir, llvm_source_dir + "/projects/libcxxabi"),
 650         (source_dir + "/cmake", llvm_source_dir + "/projects/cmake"),
 651     ]
 652     for l in symlinks:
 653         # On Windows, we have to re-copy the whole directory every time.
 654         if not is_windows() and os.path.islink(l[1]):
 655             continue
 656         delete(l[1])
 657         if os.path.exists(l[0]):
 658             symlink(l[0], l[1])
 659
 660     package_name = "clang"
 661     if build_clang_tidy:
 662         package_name = "clang-tidy"
 663         if not args.skip_patch:
 664             import_clang_tidy(
 665                 source_dir, build_clang_tidy_alpha, build_clang_tidy_external
 666             )
 667
 668     if not os.path.exists(build_dir):
 669         os.makedirs(build_dir)
 670
 671     libcxx_include_dir = os.path.join(llvm_source_dir, "projects", "libcxx", "include")
 672
 673     stage1_dir = build_dir + "/stage1"
 674     stage1_inst_dir = stage1_dir + "/" + package_name
 675
 676     final_stage_dir = stage1_dir
 677
 678     if is_darwin():
 679         extra_cflags = []
 680         extra_cxxflags = ["-stdlib=libc++"]
 681         extra_cflags2 = []
 682         extra_cxxflags2 = ["-stdlib=libc++"]
 683         extra_asmflags = []
 684         extra_ldflags = []
 685     elif is_linux():
 686         extra_cflags = []
 687         extra_cxxflags = []
 688         extra_cflags2 = ["-fPIC"]
 689         # Silence clang's warnings about arguments not being used in compilation.
 690         extra_cxxflags2 = [
 691             "-fPIC",
 692             "-Qunused-arguments",
 693         ]
 694         extra_asmflags = []
 695         # Avoid libLLVM internal function calls going through the PLT.
 696         extra_ldflags = ["-Wl,-Bsymbolic-functions"]
 697         # For whatever reason, LLVM's build system will set things up to turn
 698         # on -ffunction-sections and -fdata-sections, but won't turn on the
 699         # corresponding option to strip unused sections.  We do it explicitly
 700         # here.  LLVM's build system is also picky about turning on ICF, so
 701         # we do that explicitly here, too.
 702         extra_ldflags += ["-fuse-ld=gold", "-Wl,--gc-sections", "-Wl,--icf=safe"]
 703     elif is_windows():
 704         extra_cflags = []
 705         extra_cxxflags = []
 706         # clang-cl would like to figure out what it's supposed to be emulating
 707         # by looking at an MSVC install, but we don't really have that here.
 708         # Force things on based on WinMsvc.cmake.
 709         # Ideally, we'd just use WinMsvc.cmake as a toolchain file, but it only
 710         # really works for cross-compiles, which this is not.
 711         with open(os.path.join(llvm_source_dir, "cmake/platforms/WinMsvc.cmake")) as f:
 712             compat = [
 713                 item
 714                 for line in f
 715                 for item in line.split()
 716                 if "-fms-compatibility-version=" in item
 717             ][0]
 718         extra_cflags2 = [compat]
 719         extra_cxxflags2 = [compat]
 720         extra_asmflags = []
 721         extra_ldflags = []
 722
 723     if osx_cross_compile:
 724         # undo the damage done in the is_linux() block above, and also simulate
 725         # the is_darwin() block above.
 726         extra_cflags = []
 727         extra_cxxflags = ["-stdlib=libc++"]
 728         extra_cxxflags2 = ["-stdlib=libc++"]
 729
 730         extra_flags = [
 731             "-target",
 732             "x86_64-apple-darwin",
 733             "-mlinker-version=137",
 734             "-B",
 735             "%s/bin" % os.getenv("CROSS_CCTOOLS_PATH"),
 736             "-isysroot",
 737             os.getenv("CROSS_SYSROOT"),
 738             # technically the sysroot flag there should be enough to deduce this,
 739             # but clang needs some help to figure this out.
 740             "-I%s/usr/include" % os.getenv("CROSS_SYSROOT"),
 741             "-iframework",
 742             "%s/System/Library/Frameworks" % os.getenv("CROSS_SYSROOT"),
 743         ]
 744         extra_cflags += extra_flags
 745         extra_cxxflags += extra_flags
 746         extra_cflags2 += extra_flags
 747         extra_cxxflags2 += extra_flags
 748         extra_asmflags += extra_flags
 749         extra_ldflags = [
 750             "-Wl,-syslibroot,%s" % os.getenv("CROSS_SYSROOT"),
 751             "-Wl,-dead_strip",
 752         ]
 753
 754     upload_dir = os.getenv("UPLOAD_DIR")
 755     if assertions and upload_dir:
 756         extra_cflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir]
 757         extra_cxxflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir]
 758
 759     if skip_stages < 1:
 760         build_one_stage(
 761             [cc] + extra_cflags,
 762             [cxx] + extra_cxxflags,
 763             [asm] + extra_asmflags,
 764             [ld] + extra_ldflags,
 765             ar,
 766             ranlib,
 767             libtool,
 768             llvm_source_dir,
 769             stage1_dir,
 770             package_name,
 771             build_libcxx,
 772             osx_cross_compile,
 773             build_type,
 774             assertions,
 775             libcxx_include_dir,
 776             build_wasm,
 777             is_final_stage=(stages == 1),
 778         )
 779
 780     if stages >= 2 and skip_stages < 2:
 781         stage2_dir = build_dir + "/stage2"
 782         stage2_inst_dir = stage2_dir + "/" + package_name
 783         final_stage_dir = stage2_dir
 784         if skip_stages < 1:
 785             cc = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 786             cxx = stage1_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
 787             asm = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 788         build_one_stage(
 789             [cc] + extra_cflags2,
 790             [cxx] + extra_cxxflags2,
 791             [asm] + extra_asmflags,
 792             [ld] + extra_ldflags,
 793             ar,
 794             ranlib,
 795             libtool,
 796             llvm_source_dir,
 797             stage2_dir,
 798             package_name,
 799             build_libcxx,
 800             osx_cross_compile,
 801             build_type,
 802             assertions,
 803             libcxx_include_dir,
 804             build_wasm,
 805             is_final_stage=(stages == 2),
 806             profile="gen" if pgo else None,
 807         )
 808
 809     if stages >= 3 and skip_stages < 3:
 810         stage3_dir = build_dir + "/stage3"
 811         stage3_inst_dir = stage3_dir + "/" + package_name
 812         final_stage_dir = stage3_dir
 813         if skip_stages < 2:
 814             cc = stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 815             cxx = stage2_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
 816             asm = stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 817         build_one_stage(
 818             [cc] + extra_cflags2,
 819             [cxx] + extra_cxxflags2,
 820             [asm] + extra_asmflags,
 821             [ld] + extra_ldflags,
 822             ar,
 823             ranlib,
 824             libtool,
 825             llvm_source_dir,
 826             stage3_dir,
 827             package_name,
 828             build_libcxx,
 829             osx_cross_compile,
 830             build_type,
 831             assertions,
 832             libcxx_include_dir,
 833             build_wasm,
 834             (stages == 3),
 835         )
 836         if pgo:
 837             llvm_profdata = stage2_inst_dir + "/bin/llvm-profdata%s" % exe_ext
 838             merge_cmd = [llvm_profdata, "merge", "-o", "merged.profdata"]
 839             profraw_files = glob.glob(
 840                 os.path.join(stage2_dir, "build", "profiles", "*.profraw")
 841             )
 842             run_in(stage3_dir, merge_cmd + profraw_files)
 843             if stages == 3:
 844                 mkdir_p(upload_dir)
 845                 shutil.copy2(os.path.join(stage3_dir, "merged.profdata"), upload_dir)
 846                 return
 847
 848     if stages >= 4 and skip_stages < 4:
 849         stage4_dir = build_dir + "/stage4"
 850         final_stage_dir = stage4_dir
 851         profile = None
 852         if pgo:
 853             if skip_stages == 3:
 854                 profile_dir = os.environ.get("MOZ_FETCHES_DIR", "")
 855             else:
 856                 profile_dir = stage3_dir
 857             profile = os.path.join(profile_dir, "merged.profdata")
 858         if skip_stages < 3:
 859             cc = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 860             cxx = stage3_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
 861             asm = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
 862         build_one_stage(
 863             [cc] + extra_cflags2,
 864             [cxx] + extra_cxxflags2,
 865             [asm] + extra_asmflags,
 866             [ld] + extra_ldflags,
 867             ar,
 868             ranlib,
 869             libtool,
 870             llvm_source_dir,
 871             stage4_dir,
 872             package_name,
 873             build_libcxx,
 874             osx_cross_compile,
 875             build_type,
 876             assertions,
 877             libcxx_include_dir,
 878             build_wasm,
 879             (stages == 4),
 880             profile=profile,
 881         )
 882
 883     if build_clang_tidy:
 884         prune_final_dir_for_clang_tidy(
 885             os.path.join(final_stage_dir, package_name), osx_cross_compile
 886         )
 887
 888     if not args.skip_tar:
 889         build_tar_package("%s.tar.zst" % package_name, final_stage_dir, package_name)
 890
 891
 892 if __name__ == "__main__":
 893     main()