build/build-clang/build-clang.py

   1 #!/usr/bin/python2.7
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 import os
   7 import os.path
   8 import shutil
   9 import subprocess
  10 import platform
  11 import json
  12 import argparse
  13 import fnmatch
  14 import glob
  15 import errno
  16 import re
  17 from contextlib import contextmanager
  18 import sys
  19 import which
  20 from distutils.dir_util import copy_tree
  21
  22
  23 def symlink(source, link_name):
  24     os_symlink = getattr(os, "symlink", None)
  25     if callable(os_symlink):
  26         os_symlink(source, link_name)
  27     else:
  28         if os.path.isdir(source):
  29             # Fall back to copying the directory :(
  30             copy_tree(source, link_name)
  31
  32
  33 def check_run(args):
  34     print >> sys.stderr, ' '.join(args)
  35     p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  36     # CMake `message(STATUS)` messages, as appearing in failed source code
  37     # compiles, appear on stdout, so we only capture that.
  38     (out, _) = p.communicate()
  39     r = p.returncode
  40     if r != 0:
  41         cmake_output_re = re.compile("See also \"(.*/CMakeOutput.log)\"")
  42         cmake_error_re = re.compile("See also \"(.*/CMakeError.log)\"")
  43         output_match = cmake_output_re.search(out)
  44         error_match = cmake_error_re.search(out)
  45
  46         print >> sys.stderr, out
  47
  48         def dump_file(log):
  49             with open(log, 'rb') as f:
  50                 print >> sys.stderr, "\nContents of", log, "follow\n"
  51                 print >> sys.stderr, f.read()
  52         if output_match:
  53             dump_file(output_match.group(1))
  54         if error_match:
  55             dump_file(error_match.group(1))
  56     assert r == 0
  57
  58
  59 def run_in(path, args):
  60     d = os.getcwd()
  61     print >> sys.stderr, 'cd "%s"' % path
  62     os.chdir(path)
  63     check_run(args)
  64     print >> sys.stderr, 'cd "%s"' % d
  65     os.chdir(d)
  66
  67
  68 def patch(patch, srcdir):
  69     patch = os.path.realpath(patch)
  70     check_run(['patch', '-d', srcdir, '-p1', '-i', patch, '--fuzz=0',
  71                '-s'])
  72
  73
  74 def import_clang_tidy(source_dir):
  75     clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]),
  76                                      '..', 'clang-plugin')
  77     clang_tidy_path = os.path.join(source_dir,
  78                                    'tools/clang/tools/extra/clang-tidy')
  79     sys.path.append(clang_plugin_path)
  80     from import_mozilla_checks import do_import
  81     do_import(clang_plugin_path, clang_tidy_path)
  82
  83
  84 def build_package(package_build_dir, cmake_args):
  85     if not os.path.exists(package_build_dir):
  86         os.mkdir(package_build_dir)
  87     # If CMake has already been run, it may have been run with different
  88     # arguments, so we need to re-run it.  Make sure the cached copy of the
  89     # previous CMake run is cleared before running it again.
  90     if os.path.exists(package_build_dir + "/CMakeCache.txt"):
  91         os.remove(package_build_dir + "/CMakeCache.txt")
  92     if os.path.exists(package_build_dir + "/CMakeFiles"):
  93         shutil.rmtree(package_build_dir + "/CMakeFiles")
  94
  95     run_in(package_build_dir, ["cmake"] + cmake_args)
  96     run_in(package_build_dir, ["ninja", "install"])
  97
  98
  99 @contextmanager
 100 def updated_env(env):
 101     old_env = os.environ.copy()
 102     os.environ.update(env)
 103     yield
 104     os.environ.clear()
 105     os.environ.update(old_env)
 106
 107
 108 def build_tar_package(tar, name, base, directory):
 109     name = os.path.realpath(name)
 110     # On Windows, we have to convert this into an msys path so that tar can
 111     # understand it.
 112     if is_windows():
 113         name = name.replace('\\', '/')
 114
 115         def f(match):
 116             return '/' + match.group(1).lower()
 117         name = re.sub(r'^([A-Za-z]):', f, name)
 118     run_in(base, [tar,
 119                   "-c",
 120                   "-%s" % ("J" if ".xz" in name else "j"),
 121                   "-f",
 122                   name, directory])
 123
 124
 125 def mkdir_p(path):
 126     try:
 127         os.makedirs(path)
 128     except OSError as e:
 129         if e.errno != errno.EEXIST or not os.path.isdir(path):
 130             raise
 131
 132
 133 def delete(path):
 134     if os.path.isdir(path):
 135         shutil.rmtree(path)
 136     else:
 137         try:
 138             os.unlink(path)
 139         except Exception:
 140             pass
 141
 142
 143 def install_libgcc(gcc_dir, clang_dir, is_final_stage):
 144     gcc_bin_dir = os.path.join(gcc_dir, 'bin')
 145
 146     # Copy over gcc toolchain bits that clang looks for, to ensure that
 147     # clang is using a consistent version of ld, since the system ld may
 148     # be incompatible with the output clang produces.  But copy it to a
 149     # target-specific directory so a cross-compiler to Mac doesn't pick
 150     # up the (Linux-specific) ld with disastrous results.
 151     #
 152     # Only install this for the bootstrap process; we expect any consumers of
 153     # the newly-built toolchain to provide an appropriate ld themselves.
 154     if not is_final_stage:
 155         x64_bin_dir = os.path.join(clang_dir, 'x86_64-unknown-linux-gnu', 'bin')
 156         mkdir_p(x64_bin_dir)
 157         shutil.copy2(os.path.join(gcc_bin_dir, 'ld'), x64_bin_dir)
 158
 159     out = subprocess.check_output([os.path.join(gcc_bin_dir, "gcc"),
 160                                    '-print-libgcc-file-name'])
 161
 162     libgcc_dir = os.path.dirname(out.rstrip())
 163     clang_lib_dir = os.path.join(clang_dir, "lib", "gcc",
 164                                  "x86_64-unknown-linux-gnu",
 165                                  os.path.basename(libgcc_dir))
 166     mkdir_p(clang_lib_dir)
 167     copy_tree(libgcc_dir, clang_lib_dir)
 168     libgcc_dir = os.path.join(gcc_dir, "lib64")
 169     clang_lib_dir = os.path.join(clang_dir, "lib")
 170     copy_tree(libgcc_dir, clang_lib_dir)
 171     libgcc_dir = os.path.join(gcc_dir, "lib32")
 172     clang_lib_dir = os.path.join(clang_dir, "lib32")
 173     copy_tree(libgcc_dir, clang_lib_dir)
 174     include_dir = os.path.join(gcc_dir, "include")
 175     clang_include_dir = os.path.join(clang_dir, "include")
 176     copy_tree(include_dir, clang_include_dir)
 177
 178
 179 def install_import_library(build_dir, clang_dir):
 180     shutil.copy2(os.path.join(build_dir, "lib", "clang.lib"),
 181                  os.path.join(clang_dir, "lib"))
 182
 183
 184 def install_asan_symbols(build_dir, clang_dir):
 185     lib_path_pattern = os.path.join("lib", "clang", "*.*.*", "lib", "windows")
 186     src_path = glob.glob(os.path.join(build_dir, lib_path_pattern,
 187                                       "clang_rt.asan_dynamic-*.pdb"))
 188     dst_path = glob.glob(os.path.join(clang_dir, lib_path_pattern))
 189
 190     if len(src_path) != 1:
 191         raise Exception("Source path pattern did not resolve uniquely")
 192
 193     if len(src_path) != 1:
 194         raise Exception("Destination path pattern did not resolve uniquely")
 195
 196     shutil.copy2(src_path[0], dst_path[0])
 197
 198
 199 def svn_co(source_dir, url, directory, revision):
 200     run_in(source_dir, ["svn", "co", "-q", "-r", revision, url, directory])
 201
 202
 203 def svn_update(directory, revision):
 204     run_in(directory, ["svn", "revert", "-q", "-R", "."])
 205     run_in(directory, ["svn", "update", "-q", "-r", revision])
 206
 207
 208 def is_darwin():
 209     return platform.system() == "Darwin"
 210
 211
 212 def is_linux():
 213     return platform.system() == "Linux"
 214
 215
 216 def is_windows():
 217     return platform.system() == "Windows"
 218
 219
 220 def build_one_stage(cc, cxx, asm, ld, ar, ranlib, libtool,
 221                     src_dir, stage_dir, package_name, build_libcxx,
 222                     osx_cross_compile, build_type, assertions,
 223                     python_path, gcc_dir, libcxx_include_dir,
 224                     compiler_rt_source_dir=None, runtimes_source_link=None,
 225                     compiler_rt_source_link=None,
 226                     is_final_stage=False, android_targets=None,
 227                     extra_targets=None):
 228     if is_final_stage and (android_targets or extra_targets):
 229         # Linking compiler-rt under "runtimes" activates LLVM_RUNTIME_TARGETS
 230         # and related arguments.
 231         symlink(compiler_rt_source_dir, runtimes_source_link)
 232         try:
 233             os.unlink(compiler_rt_source_link)
 234         except Exception:
 235             pass
 236
 237     if not os.path.exists(stage_dir):
 238         os.mkdir(stage_dir)
 239
 240     build_dir = stage_dir + "/build"
 241     inst_dir = stage_dir + "/" + package_name
 242
 243     # cmake doesn't deal well with backslashes in paths.
 244     def slashify_path(path):
 245         return path.replace('\\', '/')
 246
 247     def cmake_base_args(cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir):
 248         cmake_args = [
 249             "-GNinja",
 250             "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]),
 251             "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]),
 252             "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]),
 253             "-DCMAKE_LINKER=%s" % slashify_path(ld[0]),
 254             "-DCMAKE_AR=%s" % slashify_path(ar),
 255             "-DCMAKE_C_FLAGS=%s" % ' '.join(cc[1:]),
 256             "-DCMAKE_CXX_FLAGS=%s" % ' '.join(cxx[1:]),
 257             "-DCMAKE_ASM_FLAGS=%s" % ' '.join(asm[1:]),
 258             "-DCMAKE_EXE_LINKER_FLAGS=%s" % ' '.join(ld[1:]),
 259             "-DCMAKE_SHARED_LINKER_FLAGS=%s" % ' '.join(ld[1:]),
 260             "-DCMAKE_BUILD_TYPE=%s" % build_type,
 261             "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir,
 262             "-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64",
 263             "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"),
 264             "-DPYTHON_EXECUTABLE=%s" % slashify_path(python_path),
 265             "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"),
 266             "-DLIBCXX_LIBCPPABI_VERSION=\"\"",
 267         ]
 268         if is_linux():
 269             cmake_args += ["-DLLVM_BINUTILS_INCDIR=%s/include" % gcc_dir]
 270         if is_windows():
 271             cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
 272             cmake_args.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
 273         else:
 274             # libllvm as a shared library is not supported on Windows
 275             cmake_args += ["-DLLVM_LINK_LLVM_DYLIB=ON"]
 276         if ranlib is not None:
 277             cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
 278         if libtool is not None:
 279             cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
 280         if osx_cross_compile:
 281             cmake_args += [
 282                 "-DCMAKE_SYSTEM_NAME=Darwin",
 283                 "-DCMAKE_SYSTEM_VERSION=10.10",
 284                 "-DLLVM_ENABLE_THREADS=OFF",
 285                 # Xray requires a OSX 10.12 SDK (https://bugs.llvm.org/show_bug.cgi?id=38959)
 286                 "-DCOMPILER_RT_BUILD_XRAY=OFF",
 287                 "-DLIBCXXABI_LIBCXX_INCLUDES=%s" % libcxx_include_dir,
 288                 "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 289                 "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_CCTOOLS_PATH")), # noqa
 290                 "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
 291                 "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
 292                 "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
 293                 "-DCMAKE_MACOSX_RPATH=ON",
 294                 "-DCMAKE_OSX_ARCHITECTURES=x86_64",
 295                 "-DDARWIN_osx_ARCHS=x86_64",
 296                 "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
 297                 "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin"
 298             ]
 299         return cmake_args
 300
 301     cmake_args = []
 302
 303     runtime_targets = []
 304     if is_final_stage:
 305         if android_targets:
 306             runtime_targets = list(sorted(android_targets.keys()))
 307         if extra_targets:
 308             runtime_targets.extend(sorted(extra_targets))
 309
 310     if runtime_targets:
 311         cmake_args += [
 312             "-DLLVM_BUILTIN_TARGETS=%s" % ";".join(runtime_targets),
 313             "-DLLVM_RUNTIME_TARGETS=%s" % ";".join(runtime_targets),
 314         ]
 315
 316         for target in runtime_targets:
 317             cmake_args += [
 318                 "-DRUNTIMES_%s_COMPILER_RT_BUILD_PROFILE=ON" % target,
 319                 "-DRUNTIMES_%s_COMPILER_RT_BUILD_SANITIZERS=ON" % target,
 320                 "-DRUNTIMES_%s_COMPILER_RT_BUILD_XRAY=OFF" % target,
 321                 "-DRUNTIMES_%s_SANITIZER_ALLOW_CXXABI=OFF" % target,
 322                 "-DRUNTIMES_%s_COMPILER_RT_BUILD_LIBFUZZER=OFF" % target,
 323                 "-DRUNTIMES_%s_COMPILER_RT_INCLUDE_TESTS=OFF" % target,
 324                 "-DRUNTIMES_%s_LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" % target,
 325                 "-DRUNTIMES_%s_LLVM_INCLUDE_TESTS=OFF" % target,
 326             ]
 327
 328     # The above code flipped switches to build various runtime libraries on
 329     # Android; we now have to provide all the necessary compiler switches to
 330     # make that work.
 331     if is_final_stage and android_targets:
 332         cmake_args += [
 333             "-DLLVM_LIBDIR_SUFFIX=64",
 334         ]
 335
 336         android_link_flags = "-fuse-ld=lld"
 337
 338         for target, cfg in android_targets.iteritems():
 339             sysroot_dir = cfg["ndk_sysroot"]
 340             android_gcc_dir = cfg["ndk_toolchain"]
 341             android_include_dirs = cfg["ndk_includes"]
 342             api_level = cfg["api_level"]
 343
 344             android_flags = ["-isystem %s" % d for d in android_include_dirs]
 345             android_flags += ["--gcc-toolchain=%s" % android_gcc_dir]
 346             android_flags += ["-D__ANDROID_API__=%s" % api_level]
 347
 348             # Our flags go last to override any --gcc-toolchain that may have
 349             # been set earlier.
 350             rt_c_flags = " ".join(cc[1:] + android_flags)
 351             rt_cxx_flags = " ".join(cxx[1:] + android_flags)
 352             rt_asm_flags = " ".join(asm[1:] + android_flags)
 353
 354             for kind in ('BUILTINS', 'RUNTIMES'):
 355                 for var, arg in (
 356                         ('ANDROID', '1'),
 357                         ('CMAKE_ASM_FLAGS', rt_asm_flags),
 358                         ('CMAKE_CXX_FLAGS', rt_cxx_flags),
 359                         ('CMAKE_C_FLAGS', rt_c_flags),
 360                         ('CMAKE_EXE_LINKER_FLAGS', android_link_flags),
 361                         ('CMAKE_SHARED_LINKER_FLAGS', android_link_flags),
 362                         ('CMAKE_SYSROOT', sysroot_dir),
 363                         ('ANDROID_NATIVE_API_LEVEL', api_level),
 364                 ):
 365                     cmake_args += ['-D%s_%s_%s=%s' % (kind, target, var, arg)]
 366
 367     cmake_args += cmake_base_args(
 368         cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir)
 369     cmake_args += [
 370         src_dir
 371     ]
 372     build_package(build_dir, cmake_args)
 373
 374     if is_linux():
 375         install_libgcc(gcc_dir, inst_dir, is_final_stage)
 376     # For some reasons the import library clang.lib of clang.exe is not
 377     # installed, so we copy it by ourselves.
 378     if is_windows():
 379         # The compiler-rt cmake scripts don't allow to build it for multiple
 380         # targets at once on Windows, so manually build the 32-bits compiler-rt
 381         # during the final stage.
 382         build_32_bit = False
 383         if is_final_stage:
 384             # Only build the 32-bits compiler-rt when we originally built for
 385             # 64-bits, which we detect through the contents of the LIB
 386             # environment variable, which we also adjust for a 32-bits build
 387             # at the same time.
 388             old_lib = os.environ['LIB']
 389             new_lib = []
 390             for l in old_lib.split(os.pathsep):
 391                 if l.endswith('x64'):
 392                     l = l[:-3] + 'x86'
 393                     build_32_bit = True
 394                 elif l.endswith('amd64'):
 395                     l = l[:-5]
 396                     build_32_bit = True
 397                 new_lib.append(l)
 398         if build_32_bit:
 399             os.environ['LIB'] = os.pathsep.join(new_lib)
 400             compiler_rt_build_dir = stage_dir + '/compiler-rt'
 401             compiler_rt_inst_dir = inst_dir + '/lib/clang/'
 402             subdirs = os.listdir(compiler_rt_inst_dir)
 403             assert len(subdirs) == 1
 404             compiler_rt_inst_dir += subdirs[0]
 405             cmake_args = cmake_base_args(
 406                 [os.path.join(inst_dir, 'bin', 'clang-cl.exe'), '-m32'] + cc[1:],
 407                 [os.path.join(inst_dir, 'bin', 'clang-cl.exe'), '-m32'] + cxx[1:],
 408                 [os.path.join(inst_dir, 'bin', 'clang-cl.exe'), '-m32'] + asm[1:],
 409                 ld, ar, ranlib, libtool, compiler_rt_inst_dir)
 410             cmake_args += [
 411                 '-DLLVM_CONFIG_PATH=%s' % slashify_path(
 412                     os.path.join(inst_dir, 'bin', 'llvm-config')),
 413                 os.path.join(src_dir, 'projects', 'compiler-rt'),
 414             ]
 415             build_package(compiler_rt_build_dir, cmake_args)
 416             os.environ['LIB'] = old_lib
 417         install_import_library(build_dir, inst_dir)
 418         install_asan_symbols(build_dir, inst_dir)
 419
 420
 421 # Return the absolute path of a build tool.  We first look to see if the
 422 # variable is defined in the config file, and if so we make sure it's an
 423 # absolute path to an existing tool, otherwise we look for a program in
 424 # $PATH named "key".
 425 #
 426 # This expects the name of the key in the config file to match the name of
 427 # the tool in the default toolchain on the system (for example, "ld" on Unix
 428 # and "link" on Windows).
 429 def get_tool(config, key):
 430     f = None
 431     if key in config:
 432         f = config[key]
 433         if os.path.isabs(f):
 434             if not os.path.exists(f):
 435                 raise ValueError("%s must point to an existing path" % key)
 436             return f
 437
 438     # Assume that we have the name of some program that should be on PATH.
 439     try:
 440         return which.which(f) if f else which.which(key)
 441     except which.WhichError:
 442         raise ValueError("%s not found on PATH" % f)
 443
 444
 445 # This function is intended to be called on the final build directory when
 446 # building clang-tidy. Also clang-format binaries are included that can be used
 447 # in conjunction with clang-tidy.
 448 # Its job is to remove all of the files which won't be used for clang-tidy or
 449 # clang-format to reduce the download size.  Currently when this function
 450 # finishes its job, it will leave final_dir with a layout like this:
 451 #
 452 # clang/
 453 #   bin/
 454 #     clang-apply-replacements
 455 #     clang-format
 456 #     clang-tidy
 457 #   include/
 458 #     * (nothing will be deleted here)
 459 #   lib/
 460 #     clang/
 461 #       4.0.0/
 462 #         include/
 463 #           * (nothing will be deleted here)
 464 #   share/
 465 #     clang/
 466 #       clang-format-diff.py
 467 #       clang-tidy-diff.py
 468 #       run-clang-tidy.py
 469 def prune_final_dir_for_clang_tidy(final_dir, osx_cross_compile):
 470     # Make sure we only have what we expect.
 471     dirs = ["bin", "include", "lib", "lib32", "libexec", "msbuild-bin", "share", "tools"]
 472     if is_linux():
 473         dirs.append("x86_64-unknown-linux-gnu")
 474     for f in glob.glob("%s/*" % final_dir):
 475         if os.path.basename(f) not in dirs:
 476             raise Exception("Found unknown file %s in the final directory" % f)
 477         if not os.path.isdir(f):
 478             raise Exception("Expected %s to be a directory" % f)
 479
 480     # In bin/, only keep clang-tidy and clang-apply-replacements. The last one
 481     # is used to auto-fix some of the issues detected by clang-tidy.
 482     re_clang_tidy = re.compile(
 483         r"^clang-(apply-replacements|format|tidy)(\.exe)?$", re.I)
 484     for f in glob.glob("%s/bin/*" % final_dir):
 485         if re_clang_tidy.search(os.path.basename(f)) is None:
 486             delete(f)
 487
 488     # Keep include/ intact.
 489
 490     # Remove the target-specific files.
 491     if is_linux():
 492         shutil.rmtree(os.path.join(final_dir, "x86_64-unknown-linux-gnu"))
 493
 494     # In lib/, only keep lib/clang/N.M.O/include and the LLVM shared library.
 495     re_ver_num = re.compile(r"^\d+\.\d+\.\d+$", re.I)
 496     for f in glob.glob("%s/lib/*" % final_dir):
 497         name = os.path.basename(f)
 498         if name == "clang":
 499             continue
 500         if osx_cross_compile and name == 'libLLVM.dylib':
 501             continue
 502         if is_linux() and fnmatch.fnmatch(name, 'libLLVM*.so'):
 503             continue
 504         delete(f)
 505     for f in glob.glob("%s/lib/clang/*" % final_dir):
 506         if re_ver_num.search(os.path.basename(f)) is None:
 507             delete(f)
 508     for f in glob.glob("%s/lib/clang/*/*" % final_dir):
 509         if os.path.basename(f) != "include":
 510             delete(f)
 511
 512     # Completely remove libexec/, msbuilld-bin and tools, if it exists.
 513     shutil.rmtree(os.path.join(final_dir, "libexec"))
 514     for d in ("msbuild-bin", "tools"):
 515         d = os.path.join(final_dir, d)
 516         if os.path.exists(d):
 517             shutil.rmtree(d)
 518
 519     # In share/, only keep share/clang/*tidy*
 520     re_clang_tidy = re.compile(r"format|tidy", re.I)
 521     for f in glob.glob("%s/share/*" % final_dir):
 522         if os.path.basename(f) != "clang":
 523             delete(f)
 524     for f in glob.glob("%s/share/clang/*" % final_dir):
 525         if re_clang_tidy.search(os.path.basename(f)) is None:
 526             delete(f)
 527
 528
 529 if __name__ == "__main__":
 530     parser = argparse.ArgumentParser()
 531     parser.add_argument('-c', '--config', required=True,
 532                         type=argparse.FileType('r'),
 533                         help="Clang configuration file")
 534     parser.add_argument('-b', '--base-dir', required=False,
 535                         help="Base directory for code and build artifacts")
 536     parser.add_argument('--clean', required=False,
 537                         action='store_true',
 538                         help="Clean the build directory")
 539     parser.add_argument('--skip-tar', required=False,
 540                         action='store_true',
 541                         help="Skip tar packaging stage")
 542     parser.add_argument('--skip-checkout', required=False,
 543                         action='store_true',
 544                         help="Do not checkout/revert source")
 545
 546     args = parser.parse_args()
 547
 548     # The directories end up in the debug info, so the easy way of getting
 549     # a reproducible build is to run it in a know absolute directory.
 550     # We use a directory that is registered as a volume in the Docker image.
 551
 552     if args.base_dir:
 553         base_dir = args.base_dir
 554     elif os.environ.get('MOZ_AUTOMATION') and not is_windows():
 555         base_dir = "/builds/worker/workspace/moz-toolchain"
 556     else:
 557         # Handles both the Windows automation case and the local build case
 558         # TODO: Because Windows taskcluster builds are run with distinct
 559         # user IDs for each job, we can't store things in some globally
 560         # accessible directory: one job will run, checkout LLVM to that
 561         # directory, and then if another job runs, the new user won't be
 562         # able to access the previously-checked out code--or be able to
 563         # delete it.  So on Windows, we build in the task-specific home
 564         # directory; we will eventually add -fdebug-prefix-map options
 565         # to the LLVM build to bring back reproducibility.
 566         base_dir = os.path.join(os.getcwd(), 'build-clang')
 567
 568     source_dir = base_dir + "/src"
 569     build_dir = base_dir + "/build"
 570
 571     if not os.path.exists(base_dir):
 572         os.makedirs(base_dir)
 573     elif os.listdir(base_dir) and not os.path.exists(os.path.join(base_dir, '.build-clang')):
 574         raise ValueError("Base directory %s exists and is not a build-clang directory. "
 575                          "Supply a non-existent or empty directory with --base-dir" % base_dir)
 576     open(os.path.join(base_dir, '.build-clang'), 'a').close()
 577
 578     if args.clean:
 579         shutil.rmtree(build_dir)
 580         os.sys.exit(0)
 581
 582     llvm_source_dir = source_dir + "/llvm"
 583     clang_source_dir = source_dir + "/clang"
 584     extra_source_dir = source_dir + "/extra"
 585     lld_source_dir = source_dir + "/lld"
 586     compiler_rt_source_dir = source_dir + "/compiler-rt"
 587     libcxx_source_dir = source_dir + "/libcxx"
 588     libcxxabi_source_dir = source_dir + "/libcxxabi"
 589
 590     if is_darwin():
 591         os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
 592
 593     exe_ext = ""
 594     if is_windows():
 595         exe_ext = ".exe"
 596
 597     cc_name = "clang"
 598     cxx_name = "clang++"
 599     if is_windows():
 600         cc_name = "clang-cl"
 601         cxx_name = "clang-cl"
 602
 603     config = json.load(args.config)
 604
 605     llvm_revision = config["llvm_revision"]
 606     llvm_repo = config["llvm_repo"]
 607     clang_repo = config["clang_repo"]
 608     extra_repo = config.get("extra_repo")
 609     lld_repo = config.get("lld_repo")
 610     # On some packages we don't use compiler_repo
 611     compiler_repo = config.get("compiler_repo")
 612     libcxx_repo = config["libcxx_repo"]
 613     libcxxabi_repo = config.get("libcxxabi_repo")
 614     stages = 3
 615     if "stages" in config:
 616         stages = int(config["stages"])
 617         if stages not in (1, 2, 3):
 618             raise ValueError("We only know how to build 1, 2, or 3 stages")
 619     build_type = "Release"
 620     if "build_type" in config:
 621         build_type = config["build_type"]
 622         if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"):
 623             raise ValueError("We only know how to do Release, Debug, RelWithDebInfo or "
 624                              "MinSizeRel builds")
 625     build_libcxx = False
 626     if "build_libcxx" in config:
 627         build_libcxx = config["build_libcxx"]
 628         if build_libcxx not in (True, False):
 629             raise ValueError("Only boolean values are accepted for build_libcxx.")
 630     build_clang_tidy = False
 631     if "build_clang_tidy" in config:
 632         build_clang_tidy = config["build_clang_tidy"]
 633         if build_clang_tidy not in (True, False):
 634             raise ValueError("Only boolean values are accepted for build_clang_tidy.")
 635     osx_cross_compile = False
 636     if "osx_cross_compile" in config:
 637         osx_cross_compile = config["osx_cross_compile"]
 638         if osx_cross_compile not in (True, False):
 639             raise ValueError("Only boolean values are accepted for osx_cross_compile.")
 640         if osx_cross_compile and not is_linux():
 641             raise ValueError("osx_cross_compile can only be used on Linux.")
 642     assertions = False
 643     if "assertions" in config:
 644         assertions = config["assertions"]
 645         if assertions not in (True, False):
 646             raise ValueError("Only boolean values are accepted for assertions.")
 647     python_path = None
 648     if "python_path" not in config:
 649         raise ValueError("Config file needs to set python_path")
 650     python_path = config["python_path"]
 651     gcc_dir = None
 652     if "gcc_dir" in config:
 653         gcc_dir = config["gcc_dir"]
 654         if not os.path.exists(gcc_dir):
 655             raise ValueError("gcc_dir must point to an existing path")
 656     ndk_dir = None
 657     android_targets = None
 658     if "android_targets" in config:
 659         android_targets = config["android_targets"]
 660         for attr in ("ndk_toolchain", "ndk_sysroot", "ndk_includes", "api_level"):
 661             for target, cfg in android_targets.iteritems():
 662                 if attr not in cfg:
 663                     raise ValueError("must specify '%s' as a key for android target: %s" %
 664                                      (attr, target))
 665     extra_targets = None
 666     if "extra_targets" in config:
 667         extra_targets = config["extra_targets"]
 668         if not isinstance(extra_targets, list):
 669             raise ValueError("extra_targets must be a list")
 670         if not all(isinstance(t, (str, unicode)) for t in extra_targets):
 671             raise ValueError("members of extra_targets should be strings")
 672     if is_linux() and gcc_dir is None:
 673         raise ValueError("Config file needs to set gcc_dir")
 674     cc = get_tool(config, "cc")
 675     cxx = get_tool(config, "cxx")
 676     asm = get_tool(config, "ml" if is_windows() else "as")
 677     ld = get_tool(config, "link" if is_windows() else "ld")
 678     ar = get_tool(config, "lib" if is_windows() else "ar")
 679     ranlib = None if is_windows() else get_tool(config, "ranlib")
 680     libtool = None
 681     if "libtool" in config:
 682         libtool = get_tool(config, "libtool")
 683
 684     if not os.path.exists(source_dir):
 685         os.makedirs(source_dir)
 686
 687     def checkout_or_update(repo, checkout_dir):
 688         if os.path.exists(checkout_dir):
 689             svn_update(checkout_dir, llvm_revision)
 690         else:
 691             svn_co(source_dir, repo, checkout_dir, llvm_revision)
 692
 693     if not args.skip_checkout:
 694         checkout_or_update(llvm_repo, llvm_source_dir)
 695         checkout_or_update(clang_repo, clang_source_dir)
 696         if compiler_repo is not None:
 697             checkout_or_update(compiler_repo, compiler_rt_source_dir)
 698         checkout_or_update(libcxx_repo, libcxx_source_dir)
 699         if lld_repo:
 700             checkout_or_update(lld_repo, lld_source_dir)
 701         if libcxxabi_repo:
 702             checkout_or_update(libcxxabi_repo, libcxxabi_source_dir)
 703         if extra_repo:
 704             checkout_or_update(extra_repo, extra_source_dir)
 705         for p in config.get("patches", []):
 706             patch(p, source_dir)
 707
 708     compiler_rt_source_link = llvm_source_dir + "/projects/compiler-rt"
 709
 710     symlinks = [(clang_source_dir,
 711                  llvm_source_dir + "/tools/clang"),
 712                 (extra_source_dir,
 713                  llvm_source_dir + "/tools/clang/tools/extra"),
 714                 (lld_source_dir,
 715                  llvm_source_dir + "/tools/lld"),
 716                 (compiler_rt_source_dir, compiler_rt_source_link),
 717                 (libcxx_source_dir,
 718                  llvm_source_dir + "/projects/libcxx"),
 719                 (libcxxabi_source_dir,
 720                  llvm_source_dir + "/projects/libcxxabi")]
 721     for l in symlinks:
 722         # On Windows, we have to re-copy the whole directory every time.
 723         if not is_windows() and os.path.islink(l[1]):
 724             continue
 725         delete(l[1])
 726         if os.path.exists(l[0]):
 727             symlink(l[0], l[1])
 728
 729     package_name = "clang"
 730     if build_clang_tidy:
 731         package_name = "clang-tidy"
 732         import_clang_tidy(llvm_source_dir)
 733
 734     if not os.path.exists(build_dir):
 735         os.makedirs(build_dir)
 736
 737     libcxx_include_dir = os.path.join(llvm_source_dir, "projects",
 738                                       "libcxx", "include")
 739
 740     stage1_dir = build_dir + '/stage1'
 741     stage1_inst_dir = stage1_dir + '/' + package_name
 742
 743     final_stage_dir = stage1_dir
 744
 745     if is_darwin():
 746         extra_cflags = []
 747         extra_cxxflags = ["-stdlib=libc++"]
 748         extra_cflags2 = []
 749         extra_cxxflags2 = ["-stdlib=libc++"]
 750         extra_asmflags = []
 751         extra_ldflags = []
 752     elif is_linux():
 753         extra_cflags = []
 754         extra_cxxflags = []
 755         # When building stage2 and stage3, we want the newly-built clang to pick
 756         # up whatever headers were installed from the gcc we used to build stage1,
 757         # always, rather than the system headers.  Providing -gcc-toolchain
 758         # encourages clang to do that.
 759         extra_cflags2 = ["-fPIC", '-gcc-toolchain', stage1_inst_dir]
 760         # Silence clang's warnings about arguments not being used in compilation.
 761         extra_cxxflags2 = ["-fPIC", '-Qunused-arguments', '-gcc-toolchain', stage1_inst_dir]
 762         extra_asmflags = []
 763         # Avoid libLLVM internal function calls going through the PLT.
 764         extra_ldflags = ['-Wl,-Bsymbolic-functions']
 765
 766         if 'LD_LIBRARY_PATH' in os.environ:
 767             os.environ['LD_LIBRARY_PATH'] = ('%s/lib64/:%s' %
 768                                              (gcc_dir, os.environ['LD_LIBRARY_PATH']))
 769         else:
 770             os.environ['LD_LIBRARY_PATH'] = '%s/lib64/' % gcc_dir
 771     elif is_windows():
 772         extra_cflags = []
 773         extra_cxxflags = []
 774         # clang-cl would like to figure out what it's supposed to be emulating
 775         # by looking at an MSVC install, but we don't really have that here.
 776         # Force things on.
 777         extra_cflags2 = []
 778         extra_cxxflags2 = ['-fms-compatibility-version=19.13.26128', '-Xclang', '-std=c++14']
 779         extra_asmflags = []
 780         extra_ldflags = []
 781
 782     if osx_cross_compile:
 783         # undo the damage done in the is_linux() block above, and also simulate
 784         # the is_darwin() block above.
 785         extra_cflags = []
 786         extra_cxxflags = ["-stdlib=libc++"]
 787         extra_cxxflags2 = ["-stdlib=libc++"]
 788
 789         extra_flags = ["-target", "x86_64-apple-darwin", "-mlinker-version=137",
 790                        "-B", "%s/bin" % os.getenv("CROSS_CCTOOLS_PATH"),
 791                        "-isysroot", os.getenv("CROSS_SYSROOT"),
 792                        # technically the sysroot flag there should be enough to deduce this,
 793                        # but clang needs some help to figure this out.
 794                        "-I%s/usr/include" % os.getenv("CROSS_SYSROOT"),
 795                        "-iframework", "%s/System/Library/Frameworks" % os.getenv("CROSS_SYSROOT")]
 796         extra_cflags += extra_flags
 797         extra_cxxflags += extra_flags
 798         extra_cflags2 += extra_flags
 799         extra_cxxflags2 += extra_flags
 800         extra_asmflags += extra_flags
 801         extra_ldflags = ["-Wl,-syslibroot,%s" % os.getenv("CROSS_SYSROOT"),
 802                          "-Wl,-dead_strip"]
 803
 804     build_one_stage(
 805         [cc] + extra_cflags,
 806         [cxx] + extra_cxxflags,
 807         [asm] + extra_asmflags,
 808         [ld] + extra_ldflags,
 809         ar, ranlib, libtool,
 810         llvm_source_dir, stage1_dir, package_name, build_libcxx, osx_cross_compile,
 811         build_type, assertions, python_path, gcc_dir, libcxx_include_dir)
 812
 813     runtimes_source_link = llvm_source_dir + "/runtimes/compiler-rt"
 814
 815     if stages > 1:
 816         stage2_dir = build_dir + '/stage2'
 817         stage2_inst_dir = stage2_dir + '/' + package_name
 818         final_stage_dir = stage2_dir
 819         build_one_stage(
 820             [stage1_inst_dir + "/bin/%s%s" %
 821                 (cc_name, exe_ext)] + extra_cflags2,
 822             [stage1_inst_dir + "/bin/%s%s" %
 823                 (cxx_name, exe_ext)] + extra_cxxflags2,
 824             [stage1_inst_dir + "/bin/%s%s" %
 825                 (cc_name, exe_ext)] + extra_asmflags,
 826             [ld] + extra_ldflags,
 827             ar, ranlib, libtool,
 828             llvm_source_dir, stage2_dir, package_name, build_libcxx, osx_cross_compile,
 829             build_type, assertions, python_path, gcc_dir, libcxx_include_dir,
 830             compiler_rt_source_dir, runtimes_source_link, compiler_rt_source_link,
 831             is_final_stage=(stages == 2), android_targets=android_targets,
 832             extra_targets=extra_targets)
 833
 834     if stages > 2:
 835         stage3_dir = build_dir + '/stage3'
 836         final_stage_dir = stage3_dir
 837         build_one_stage(
 838             [stage2_inst_dir + "/bin/%s%s" %
 839                 (cc_name, exe_ext)] + extra_cflags2,
 840             [stage2_inst_dir + "/bin/%s%s" %
 841                 (cxx_name, exe_ext)] + extra_cxxflags2,
 842             [stage2_inst_dir + "/bin/%s%s" %
 843                 (cc_name, exe_ext)] + extra_asmflags,
 844             [ld] + extra_ldflags,
 845             ar, ranlib, libtool,
 846             llvm_source_dir, stage3_dir, package_name, build_libcxx, osx_cross_compile,
 847             build_type, assertions, python_path, gcc_dir, libcxx_include_dir,
 848             compiler_rt_source_dir, runtimes_source_link, compiler_rt_source_link,
 849             (stages == 3), extra_targets=extra_targets)
 850
 851     if build_clang_tidy:
 852         prune_final_dir_for_clang_tidy(os.path.join(final_stage_dir, package_name),
 853                                        osx_cross_compile)
 854
 855     if not args.skip_tar:
 856         ext = "bz2" if is_darwin() or is_windows() else "xz"
 857         build_tar_package("tar", "%s.tar.%s" % (package_name, ext), final_stage_dir, package_name)