tools/clang/scripts/update.sh

   1 #!/usr/bin/env bash
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 # This script will check out llvm and clang into third_party/llvm and build it.
   7
   8 # Do NOT CHANGE this if you don't know what you're doing -- see
   9 # https://code.google.com/p/chromium/wiki/UpdatingClang
  10 # Reverting problematic clang rolls is safe, though.
  11 CLANG_REVISION=198389
  12
  13 THIS_DIR="$(dirname "${0}")"
  14 LLVM_DIR="${THIS_DIR}/../../../third_party/llvm"
  15 LLVM_BUILD_DIR="${LLVM_DIR}/../llvm-build"
  16 LLVM_BOOTSTRAP_DIR="${LLVM_DIR}/../llvm-bootstrap"
  17 CLANG_DIR="${LLVM_DIR}/tools/clang"
  18 CLANG_TOOLS_EXTRA_DIR="${CLANG_DIR}/tools/extra"
  19 COMPILER_RT_DIR="${LLVM_DIR}/projects/compiler-rt"
  20 ANDROID_NDK_DIR="${LLVM_DIR}/../android_tools/ndk"
  21 STAMP_FILE="${LLVM_BUILD_DIR}/cr_build_revision"
  22
  23 # ${A:-a} returns $A if it's set, a else.
  24 LLVM_REPO_URL=${LLVM_URL:-https://llvm.org/svn/llvm-project}
  25
  26 if [[ -z "$GYP_DEFINES" ]]; then
  27   GYP_DEFINES=
  28 fi
  29 if [[ -z "$GYP_GENERATORS" ]]; then
  30   GYP_GENERATORS=
  31 fi
  32
  33
  34 # Die if any command dies, error on undefined variable expansions.
  35 set -eu
  36
  37 OS="$(uname -s)"
  38
  39 # Parse command line options.
  40 force_local_build=
  41 mac_only=
  42 run_tests=
  43 bootstrap=
  44 with_android=yes
  45 chrome_tools="plugins"
  46
  47 if [[ "${OS}" = "Darwin" ]]; then
  48   with_android=
  49 fi
  50
  51 while [[ $# > 0 ]]; do
  52   case $1 in
  53     --bootstrap)
  54       bootstrap=yes
  55       ;;
  56     --force-local-build)
  57       force_local_build=yes
  58       ;;
  59     --mac-only)
  60       mac_only=yes
  61       ;;
  62     --run-tests)
  63       run_tests=yes
  64       ;;
  65     --without-android)
  66       with_android=
  67       ;;
  68     --with-chrome-tools)
  69       shift
  70       if [[ $# == 0 ]]; then
  71         echo "--with-chrome-tools requires an argument."
  72         exit 1
  73       fi
  74       chrome_tools=$1
  75       ;;
  76     --gcc-toolchain)
  77       shift
  78       if [[ $# == 0 ]]; then
  79         echo "--gcc-toolchain requires an argument."
  80         exit 1
  81       fi
  82       if [[ -x "$1/bin/gcc" ]]; then
  83         gcc_toolchain=$1
  84       else
  85         echo "Invalid --gcc-toolchain: '$1'."
  86         echo "'$1/bin/gcc' does not appear to be valid."
  87         exit 1
  88       fi
  89       ;;
  90
  91     --help)
  92       echo "usage: $0 [--force-local-build] [--mac-only] [--run-tests] "
  93       echo "--bootstrap: First build clang with CC, then with itself."
  94       echo "--force-local-build: Don't try to download prebuilt binaries."
  95       echo "--mac-only: Do initial download only on Mac systems."
  96       echo "--run-tests: Run tests after building. Only for local builds."
  97       echo "--without-android: Don't build ASan Android runtime library."
  98       echo "--with-chrome-tools: Select which chrome tools to build." \
  99            "Defaults to plugins."
 100       echo "    Example: --with-chrome-tools 'plugins empty-string'"
 101       echo "--gcc-toolchain: Set the prefix for which GCC version should"
 102       echo "    be used for building. For example, to use gcc in"
 103       echo "    /opt/foo/bin/gcc, use '--gcc-toolchain '/opt/foo"
 104       echo
 105       exit 1
 106       ;;
 107     *)
 108       echo "Unknown argument: '$1'."
 109       echo "Use --help for help."
 110       exit 1
 111       ;;
 112   esac
 113   shift
 114 done
 115
 116 # --mac-only prevents the initial download on non-mac systems, but if clang has
 117 # already been downloaded in the past, this script keeps it up to date even if
 118 # --mac-only is passed in and the system isn't a mac. People who don't like this
 119 # can just delete their third_party/llvm-build directory.
 120 if [[ -n "$mac_only" ]] && [[ "${OS}" != "Darwin" ]] &&
 121     [[ ! ( "$GYP_DEFINES" =~ .*(clang|tsan|asan|lsan|msan)=1.* ) ]] &&
 122     ! [[ -d "${LLVM_BUILD_DIR}" ]]; then
 123   exit 0
 124 fi
 125
 126 # Xcode and clang don't get along when predictive compilation is enabled.
 127 # http://crbug.com/96315
 128 if [[ "${OS}" = "Darwin" ]] && xcodebuild -version | grep -q 'Xcode 3.2' ; then
 129   XCONF=com.apple.Xcode
 130   if [[ "${GYP_GENERATORS}" != "make" ]] && \
 131      [ "$(defaults read "${XCONF}" EnablePredictiveCompilation)" != "0" ]; then
 132     echo
 133     echo "          HEARKEN!"
 134     echo "You're using Xcode3 and you have 'Predictive Compilation' enabled."
 135     echo "This does not work well with clang (http://crbug.com/96315)."
 136     echo "Disable it in Preferences->Building (lower right), or run"
 137     echo "    defaults write ${XCONF} EnablePredictiveCompilation -boolean NO"
 138     echo "while Xcode is not running."
 139     echo
 140   fi
 141
 142   SUB_VERSION=$(xcodebuild -version | sed -Ene 's/Xcode 3\.2\.([0-9]+)/\1/p')
 143   if [[ "${SUB_VERSION}" < 6 ]]; then
 144     echo
 145     echo "          YOUR LD IS BUGGY!"
 146     echo "Please upgrade Xcode to at least 3.2.6."
 147     echo
 148   fi
 149 fi
 150
 151
 152 # Check if there's anything to be done, exit early if not.
 153 if [[ -f "${STAMP_FILE}" ]]; then
 154   PREVIOUSLY_BUILT_REVISON=$(cat "${STAMP_FILE}")
 155   if [[ -z "$force_local_build" ]] && \
 156        [[ "${PREVIOUSLY_BUILT_REVISON}" = "${CLANG_REVISION}" ]]; then
 157     echo "Clang already at ${CLANG_REVISION}"
 158     exit 0
 159   fi
 160 fi
 161 # To always force a new build if someone interrupts their build half way.
 162 rm -f "${STAMP_FILE}"
 163
 164
 165 # Clobber build files. PCH files only work with the compiler that created them.
 166 # We delete .o files to make sure all files are built with the new compiler.
 167 echo "Clobbering build files"
 168 MAKE_DIR="${THIS_DIR}/../../../out"
 169 XCODEBUILD_DIR="${THIS_DIR}/../../../xcodebuild"
 170 for DIR in "${XCODEBUILD_DIR}" "${MAKE_DIR}/Debug" "${MAKE_DIR}/Release"; do
 171   if [[ -d "${DIR}" ]]; then
 172     find "${DIR}" -name '*.o' -exec rm {} +
 173     find "${DIR}" -name '*.o.d' -exec rm {} +
 174     find "${DIR}" -name '*.gch' -exec rm {} +
 175     find "${DIR}" -name '*.dylib' -exec rm -rf {} +
 176     find "${DIR}" -name 'SharedPrecompiledHeaders' -exec rm -rf {} +
 177   fi
 178 done
 179
 180 # Clobber NaCl toolchain stamp files, see http://crbug.com/159793
 181 if [[ -d "${MAKE_DIR}" ]]; then
 182   find "${MAKE_DIR}" -name 'stamp.untar' -exec rm {} +
 183 fi
 184 if [[ "${OS}" = "Darwin" ]]; then
 185   if [[ -d "${XCODEBUILD_DIR}" ]]; then
 186     find "${XCODEBUILD_DIR}" -name 'stamp.untar' -exec rm {} +
 187   fi
 188 fi
 189
 190 if [[ -z "$force_local_build" ]]; then
 191   # Check if there's a prebuilt binary and if so just fetch that. That's faster,
 192   # and goma relies on having matching binary hashes on client and server too.
 193   CDS_URL=https://commondatastorage.googleapis.com/chromium-browser-clang
 194   CDS_FILE="clang-${CLANG_REVISION}.tgz"
 195   CDS_OUT_DIR=$(mktemp -d -t clang_download.XXXXXX)
 196   CDS_OUTPUT="${CDS_OUT_DIR}/${CDS_FILE}"
 197   if [ "${OS}" = "Linux" ]; then
 198     CDS_FULL_URL="${CDS_URL}/Linux_x64/${CDS_FILE}"
 199   elif [ "${OS}" = "Darwin" ]; then
 200     CDS_FULL_URL="${CDS_URL}/Mac/${CDS_FILE}"
 201   fi
 202   echo Trying to download prebuilt clang
 203   if which curl > /dev/null; then
 204     curl -L --fail "${CDS_FULL_URL}" -o "${CDS_OUTPUT}" || \
 205         rm -rf "${CDS_OUT_DIR}"
 206   elif which wget > /dev/null; then
 207     wget "${CDS_FULL_URL}" -O "${CDS_OUTPUT}" || rm -rf "${CDS_OUT_DIR}"
 208   else
 209     echo "Neither curl nor wget found. Please install one of these."
 210     exit 1
 211   fi
 212   if [ -f "${CDS_OUTPUT}" ]; then
 213     rm -rf "${LLVM_BUILD_DIR}/Release+Asserts"
 214     mkdir -p "${LLVM_BUILD_DIR}/Release+Asserts"
 215     tar -xzf "${CDS_OUTPUT}" -C "${LLVM_BUILD_DIR}/Release+Asserts"
 216     echo clang "${CLANG_REVISION}" unpacked
 217     echo "${CLANG_REVISION}" > "${STAMP_FILE}"
 218     rm -rf "${CDS_OUT_DIR}"
 219     exit 0
 220   else
 221     echo Did not find prebuilt clang at r"${CLANG_REVISION}", building
 222   fi
 223 fi
 224
 225 if [[ -n "${with_android}" ]] && ! [[ -d "${ANDROID_NDK_DIR}" ]]; then
 226   echo "Android NDK not found at ${ANDROID_NDK_DIR}"
 227   echo "The Android NDK is needed to build a Clang whose -fsanitize=address"
 228   echo "works on Android. See "
 229   echo "http://code.google.com/p/chromium/wiki/AndroidBuildInstructions for how"
 230   echo "to install the NDK, or pass --without-android."
 231   exit 1
 232 fi
 233
 234 echo Getting LLVM r"${CLANG_REVISION}" in "${LLVM_DIR}"
 235 if ! svn co --force "${LLVM_REPO_URL}/llvm/trunk@${CLANG_REVISION}" \
 236                     "${LLVM_DIR}"; then
 237   echo Checkout failed, retrying
 238   rm -rf "${LLVM_DIR}"
 239   svn co --force "${LLVM_REPO_URL}/llvm/trunk@${CLANG_REVISION}" "${LLVM_DIR}"
 240 fi
 241
 242 echo Getting clang r"${CLANG_REVISION}" in "${CLANG_DIR}"
 243 svn co --force "${LLVM_REPO_URL}/cfe/trunk@${CLANG_REVISION}" "${CLANG_DIR}"
 244
 245 echo Getting compiler-rt r"${CLANG_REVISION}" in "${COMPILER_RT_DIR}"
 246 svn co --force "${LLVM_REPO_URL}/compiler-rt/trunk@${CLANG_REVISION}" \
 247                "${COMPILER_RT_DIR}"
 248
 249 # Echo all commands.
 250 set -x
 251
 252 NUM_JOBS=3
 253 if [[ "${OS}" = "Linux" ]]; then
 254   NUM_JOBS="$(grep -c "^processor" /proc/cpuinfo)"
 255 elif [ "${OS}" = "Darwin" ]; then
 256   NUM_JOBS="$(sysctl -n hw.ncpu)"
 257 fi
 258
 259 if [[ -n "${gcc_toolchain}" ]]; then
 260   # Use the specified gcc installation for building.
 261   export CC="$gcc_toolchain/bin/gcc"
 262   export CXX="$gcc_toolchain/bin/g++"
 263 fi
 264
 265 export CFLAGS=""
 266 export CXXFLAGS=""
 267
 268 # Build bootstrap clang if requested.
 269 if [[ -n "${bootstrap}" ]]; then
 270   echo "Building bootstrap compiler"
 271   mkdir -p "${LLVM_BOOTSTRAP_DIR}"
 272   cd "${LLVM_BOOTSTRAP_DIR}"
 273   if [[ ! -f ./config.status ]]; then
 274     # The bootstrap compiler only needs to be able to build the real compiler,
 275     # so it needs no cross-compiler output support. In general, the host
 276     # compiler should be as similar to the final compiler as possible, so do
 277     # keep --disable-threads & co.
 278     ../llvm/configure \
 279         --enable-optimized \
 280         --enable-targets=host-only \
 281         --disable-threads \
 282         --disable-pthreads \
 283         --without-llvmgcc \
 284         --without-llvmgxx
 285   fi
 286
 287   if [[ -n "${gcc_toolchain}" ]]; then
 288     # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
 289     # compiler can start.
 290     mkdir -p Release+Asserts/lib
 291     cp -v "$(${CXX} -print-file-name=libstdc++.so.6)" \
 292       "Release+Asserts/lib/"
 293   fi
 294
 295
 296   MACOSX_DEPLOYMENT_TARGET=10.5 make -j"${NUM_JOBS}"
 297   if [[ -n "${run_tests}" ]]; then
 298     make check-all
 299   fi
 300   cd -
 301   export CC="${PWD}/${LLVM_BOOTSTRAP_DIR}/Release+Asserts/bin/clang"
 302   export CXX="${PWD}/${LLVM_BOOTSTRAP_DIR}/Release+Asserts/bin/clang++"
 303
 304   if [[ -n "${gcc_toolchain}" ]]; then
 305     # Tell the bootstrap compiler to use a specific gcc prefix to search
 306     # for standard library headers and shared object file.
 307     export CFLAGS="--gcc-toolchain=${gcc_toolchain}"
 308     export CXXFLAGS="--gcc-toolchain=${gcc_toolchain}"
 309   fi
 310
 311   echo "Building final compiler"
 312 fi
 313
 314 # Build clang (in a separate directory).
 315 # The clang bots have this path hardcoded in built/scripts/slave/compile.py,
 316 # so if you change it you also need to change these links.
 317 mkdir -p "${LLVM_BUILD_DIR}"
 318 cd "${LLVM_BUILD_DIR}"
 319 if [[ ! -f ./config.status ]]; then
 320   ../llvm/configure \
 321       --enable-optimized \
 322       --disable-threads \
 323       --disable-pthreads \
 324       --without-llvmgcc \
 325       --without-llvmgxx
 326 fi
 327
 328 if [[ -n "${gcc_toolchain}" ]]; then
 329   # Copy in the right stdlibc++.so.6 so clang can start.
 330   mkdir -p Release+Asserts/lib
 331   cp -v "$(${CXX} ${CXXFLAGS} -print-file-name=libstdc++.so.6)" \
 332     Release+Asserts/lib/
 333 fi
 334 MACOSX_DEPLOYMENT_TARGET=10.5 make -j"${NUM_JOBS}"
 335 STRIP_FLAGS=
 336 if [ "${OS}" = "Darwin" ]; then
 337   # See http://crbug.com/256342
 338   STRIP_FLAGS=-x
 339 fi
 340 strip ${STRIP_FLAGS} Release+Asserts/bin/clang
 341 cd -
 342
 343 if [[ -n "${with_android}" ]]; then
 344   # Make a standalone Android toolchain.
 345   ${ANDROID_NDK_DIR}/build/tools/make-standalone-toolchain.sh \
 346       --platform=android-14 \
 347       --install-dir="${LLVM_BUILD_DIR}/android-toolchain" \
 348       --system=linux-x86_64 \
 349       --stl=stlport
 350
 351   # Build ASan runtime for Android.
 352   # Note: LLVM_ANDROID_TOOLCHAIN_DIR is not relative to PWD, but to where we
 353   # build the runtime, i.e. third_party/llvm/projects/compiler-rt.
 354   cd "${LLVM_BUILD_DIR}"
 355   make -C tools/clang/runtime/ \
 356     LLVM_ANDROID_TOOLCHAIN_DIR="../../../llvm-build/android-toolchain"
 357   cd -
 358 fi
 359
 360 # Build Chrome-specific clang tools. Paths in this list should be relative to
 361 # tools/clang.
 362 # For each tool directory, copy it into the clang tree and use clang's build
 363 # system to compile it.
 364 for CHROME_TOOL_DIR in ${chrome_tools}; do
 365   TOOL_SRC_DIR="${THIS_DIR}/../${CHROME_TOOL_DIR}"
 366   TOOL_DST_DIR="${LLVM_DIR}/tools/clang/tools/chrome-${CHROME_TOOL_DIR}"
 367   TOOL_BUILD_DIR="${LLVM_BUILD_DIR}/tools/clang/tools/chrome-${CHROME_TOOL_DIR}"
 368   rm -rf "${TOOL_DST_DIR}"
 369   cp -R "${TOOL_SRC_DIR}" "${TOOL_DST_DIR}"
 370   rm -rf "${TOOL_BUILD_DIR}"
 371   mkdir -p "${TOOL_BUILD_DIR}"
 372   cp "${TOOL_SRC_DIR}/Makefile" "${TOOL_BUILD_DIR}"
 373   MACOSX_DEPLOYMENT_TARGET=10.5 make -j"${NUM_JOBS}" -C "${TOOL_BUILD_DIR}"
 374 done
 375
 376 if [[ -n "$run_tests" ]]; then
 377   # Run a few tests.
 378   for CHROME_TOOL_DIR in ${chrome_tools}; do
 379     TOOL_SRC_DIR="${THIS_DIR}/../${CHROME_TOOL_DIR}"
 380     if [[ -f "${TOOL_SRC_DIR}/tests/test.sh" ]]; then
 381       "${TOOL_SRC_DIR}/tests/test.sh" "${LLVM_BUILD_DIR}/Release+Asserts"
 382     fi
 383   done
 384   cd "${LLVM_BUILD_DIR}"
 385   make check-all
 386   cd -
 387 fi
 388
 389 # After everything is done, log success for this revision.
 390 echo "${CLANG_REVISION}" > "${STAMP_FILE}"