taskcluster/scripts/misc/build-custom-car.sh

   1 #!/bin/bash
   2 set -x -e -v
   3
   4 # This script is for building a custom version of chromium-as-release
   5
   6 # First argument must be the artifact name
   7 ARTIFACT_NAME=$(basename $TOOLCHAIN_ARTIFACT)
   8 shift
   9
  10 # Use the rest of the arguments as the build config for gn
  11 CONFIG=$(echo $* | tr -d "'")
  12
  13 # Android build flag
  14 if [[ "$ARTIFACT_NAME" == *"android"* ]]; then
  15   IS_ANDROID=true
  16 fi
  17
  18 mkdir custom_car
  19 cd custom_car
  20 CUSTOM_CAR_DIR=$PWD
  21
  22 # Setup depot_tools
  23 git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
  24 export PATH="$PATH:$CUSTOM_CAR_DIR/depot_tools"
  25
  26 # Log the current revision of depot tools for easier tracking in the future
  27 DEPOT_TOOLS_REV=$(cd depot_tools && git rev-parse HEAD && cd ..)
  28 echo "Current depot_tools revision: $DEPOT_TOOLS_REV"
  29
  30 # Set up some env variables depending on the target OS
  31 # Linux is the default case, with minor adjustments for
  32 # android since it is built with a linux host
  33
  34 # Final folder structure before compressing is
  35 # the same for linux and windows
  36 FINAL_BIN_PATH="src/out/Default"
  37
  38 # Final binary name for autoninja build sequence
  39 FINAL_BIN=chrome
  40
  41 # Unique substring for PGO data for Linux
  42 PGO_SUBSTR="chrome-linux-main"
  43
  44 # Default (non android) fetch name for upstream src
  45 FETCH_NAME=chromium
  46
  47 # Android specific vars
  48 if [ "$IS_ANDROID" = true ]; then
  49   FETCH_NAME=android
  50   PGO_SUBSTR="android64"
  51   FINAL_BIN_PATH="src/out/Default/apks"
  52   FINAL_BIN=chrome_public_apk
  53 fi
  54
  55 # Logic for macosx64
  56 if [[ $(uname -s) == "Darwin" ]]; then
  57   # Modify the config with fetched sdk path
  58   export MACOS_SYSROOT="$MOZ_FETCHES_DIR/MacOSX14.2.sdk"
  59
  60   # Avoid mixing up the system python and toolchain python in the
  61   # python path configuration
  62   # https://bugs.python.org/issue22490
  63   unset __PYVENV_LAUNCHER__
  64
  65   # Set the SDK path for build, which is technically a higher version
  66   # than what is associated with the current OS version (10.15).
  67   # This should work as long as MACOSX_DEPLOYMENT_TARGET is set correctly
  68   CONFIG=$(echo $CONFIG mac_sdk_path='"'$MACOS_SYSROOT'"')
  69
  70   # Ensure we don't use ARM64 profdata with this unique sub string
  71   PGO_SUBSTR="chrome-mac-main"
  72
  73   # Temporary hacky way for now while we build this on intel workers.
  74   # Afterwards we can replace it with a $(uname -m) == "arm64" check.
  75   # Bug 1858740
  76   if [[ "$ARTIFACT_NAME" == *"macosx_arm"* ]]; then
  77     PGO_SUBSTR="chrome-mac-arm-main"
  78   fi
  79
  80   # macOS final build folder is different than linux/win
  81   FINAL_BIN_PATH="src/out/Default/Chromium.app"
  82 fi
  83
  84 # Logic for win64 using the mingw environment
  85 if [[ $(uname -o) == "Msys" ]]; then
  86   # Setup VS 2022
  87   . $GECKO_PATH/taskcluster/scripts/misc/vs-setup.sh
  88
  89   # Setup some environment variables for chromium build scripts
  90   export DEPOT_TOOLS_WIN_TOOLCHAIN=0
  91   export GYP_MSVS_OVERRIDE_PATH="$MOZ_FETCHES_DIR/VS"
  92   export GYP_MSVS_VERSION=2022
  93   export vs2022_install="$MOZ_FETCHES_DIR/VS"
  94   export WINDOWSSDKDIR="$MOZ_FETCHES_DIR/VS/Windows Kits/10"
  95   export DEPOT_TOOLS_UPDATE=1
  96   export GCLIENT_PY3=1
  97   # Fool GYP
  98   touch "$MOZ_FETCHES_DIR/VS/VC/vcvarsall.bat"
  99
 100   # Construct some of our own dirs and move VS dlls + other files
 101   # to a path that chromium build files & scripts are expecting
 102   mkdir chrome_dll
 103   cd chrome_dll
 104   mkdir system32
 105   cd ../
 106   pushd "$WINDOWSSDKDIR"
 107   mkdir -p Debuggers/x64/
 108   popd
 109   mv $MOZ_FETCHES_DIR/VS/VC/Redist/MSVC/14.38.33135/x64/Microsoft.VC143.CRT/* chrome_dll/system32/
 110   mv "$WINDOWSSDKDIR/App Certification Kit/"* "$WINDOWSSDKDIR"/Debuggers/x64/
 111   export WINDIR="$PWD/chrome_dll"
 112
 113   # Run glcient once first to get some windows deps
 114   gclient
 115
 116   # Ensure we don't use WIN32 profdata with this unique sub string
 117   PGO_SUBSTR="chrome-win64-main"
 118 fi
 119
 120 # Get chromium source code and dependencies
 121 mkdir chromium
 122 cd chromium
 123
 124 fetch --no-history --nohooks $FETCH_NAME
 125
 126 # Setup the .gclient file to ensure pgo profiles are downloaded.
 127 # For some reason we need to set --name flag even though it already exists.
 128 # Currently the gclient.py file does NOT recognize --custom-var as it's own argument
 129 gclient config --name src "https://chromium.googlesource.com/chromium/src.git" --custom-var="checkout_pgo_profiles=True" --unmanaged
 130
 131 cd src
 132
 133 # Log the current revision of the chromium src for easier tracking in the future
 134 CHROMIUM_REV=$(git rev-parse HEAD)
 135 echo "Current chromium revision: $CHROMIUM_REV"
 136
 137 # Amend gclient file
 138 if [ "$IS_ANDROID" = true ]; then
 139   echo "target_os = [ 'android' ]" >> ../.gclient
 140 fi
 141
 142 if [[ $(uname -o) == "Msys" ]]; then
 143   # For fast fetches it seems we will be missing some dummy files in windows.
 144   # We can create a dummy this way to satisfy the rest of the build sequence.
 145   # This is ok because we are not doing any development here and don't need
 146   # the development history, but this file is still needed to proceed.
 147   python3 build/util/lastchange.py -o build/util/LASTCHANGE
 148 fi
 149
 150 if [[ $(uname -s) == "Linux" ]] || [[ $(uname -s) == "Darwin" ]]; then
 151   # Bug 1847210
 152   # Modifications to how the dirname and depot_tools and other env variables
 153   # change how cipd is setup for Mac and Linux.
 154   # Easily resolved by just running the setup script.
 155   source ./third_party/depot_tools/cipd_bin_setup.sh
 156   cipd_bin_setup
 157 fi
 158
 159 # Sync again for android, after cipd bin setup
 160 if [ "$IS_ANDROID" = true ]; then
 161   gclient sync
 162 fi
 163
 164 # Now we can run hooks and fetch PGO + everything else
 165 gclient runhooks
 166
 167 # PGO data should be in src/chrome/build/pgo_profiles/
 168 # with a name like "chrome-{OS}-<some unique identifier>"
 169 export PGO_DATA_DIR="$CUSTOM_CAR_DIR/chromium/src/chrome/build/pgo_profiles"
 170 for entry in "$PGO_DATA_DIR"/*
 171 do
 172   if [ -f "$entry" ]; then
 173     if [[ "$entry" == *"$PGO_SUBSTR"* ]]; then
 174         echo "Found the correct profdata"
 175         export PGO_DATA_PATH="$entry"
 176         break
 177     fi
 178   fi
 179 done
 180
 181 PGO_FILE=$PGO_DATA_PATH
 182 if [[ $(uname -o) == "Msys" ]]; then
 183   # Compute a relative path that the build scripts looks for.
 184   # This odd pathing seems to only happen on windows
 185   PGO_FILE=${PGO_DATA_PATH#*/*/*/*/*/*/*/*/*/}
 186   mv $PGO_DATA_PATH build/config/compiler/pgo/
 187 fi
 188
 189 CONFIG=$(echo $CONFIG pgo_data_path='"'$PGO_FILE'"')
 190
 191 # Set up then build chrome
 192 gn gen out/Default --args="$CONFIG"
 193 autoninja -C out/Default $FINAL_BIN
 194
 195 # Gather binary and related files into a zip, and upload it
 196 cd ..
 197 mkdir chromium
 198
 199 mv "$FINAL_BIN_PATH" chromium
 200 chmod -R +x chromium
 201
 202 tar -c chromium | python3 $GECKO_PATH/taskcluster/scripts/misc/zstdpy > $ARTIFACT_NAME
 203
 204 mkdir -p $UPLOAD_DIR
 205 mv "$ARTIFACT_NAME" "$UPLOAD_DIR"