scripts/ci/run-jenkins.sh

   1 #!/bin/bash -e
   2
   3 export MONO_REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../" && pwd )"
   4 export TESTCMD=${MONO_REPO_ROOT}/scripts/ci/run-step.sh
   5 export CI=1
   6 export CI_PR=$([[ ${CI_TAGS} == *'pull-request'* ]] && echo 1 || true)
   7 export CI_CPU_COUNT=$(getconf _NPROCESSORS_ONLN || echo 4)
   8 export TEST_HARNESS_VERBOSE=1
   9
  10 # workaround for acceptance-tests submodules leaving files behind since Jenkins only does "git clean -xdf" (no second 'f')
  11 # which won't clean untracked .git repos (remove once https://github.com/jenkinsci/git-plugin/pull/449 is available)
  12 for dir in acceptance-tests/external/*; do [ -d "$dir" ] && (cd "$dir" && echo "Cleaning $dir" && git clean -xdff); done
  13
  14 source ${MONO_REPO_ROOT}/scripts/ci/util.sh
  15
  16 helix_set_env_vars
  17 helix_send_build_start_event "build/source/$MONO_HELIX_TYPE/"
  18
  19 make_timeout=300m
  20 gnumake=$(which gmake || which gnumake || which make)
  21
  22 if [[ ${CI_TAGS} == *'clang-sanitizer'* ]]; then
  23         export CC="clang"
  24         export CXX="clang++"
  25         export CFLAGS="$CFLAGS -g -O1 -fsanitize=thread -fsanitize-blacklist=${MONO_REPO_ROOT}/scripts/ci/clang-thread-sanitizer-blacklist -mllvm -tsan-instrument-atomics=false"
  26         export LDFLAGS="-fsanitize=thread"
  27         # TSAN_OPTIONS are used by programs that were compiled with Clang's ThreadSanitizer
  28         # see https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags for more details
  29         export TSAN_OPTIONS="history_size=7:exitcode=0:force_seq_cst_atomics=1"
  30         make_timeout=30m
  31 fi
  32
  33 if [[ ${CI_TAGS} == *'win-'* ]]; then
  34     # Passing -ggdb3 on Cygwin breaks linking against libmonosgen-x.y.dll
  35     export CFLAGS="$CFLAGS -g -O2"
  36 else
  37     export EXTRA_CFLAGS="$EXTRA_CFLAGS -ggdb3 -O2"
  38 fi
  39
  40 if [[ $CI_TAGS == *'collect-coverage'* ]]; then
  41     # Collect coverage for further use by lcov and similar tools.
  42     # Coverage must be collected with -O0 and debug information.
  43     export CFLAGS="$CFLAGS -ggdb3 --coverage -O0"
  44 fi
  45
  46 if [[ $CI_TAGS == *'retry-flaky-tests'* ]]; then
  47     export MONO_FLAKY_TEST_RETRIES=5
  48 fi
  49
  50 # We don't want to have to maintain symbolification blobs for CI
  51 export EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-crash-privacy=no "
  52
  53 if [[ ${CI_TAGS} == *'osx-i386'* ]]; then EXTRA_CFLAGS="$EXTRA_CFLAGS -m32 -arch i386 -mmacosx-version-min=10.9"; EXTRA_LDFLAGS="$EXTRA_LDFLAGS -m32 -arch i386"; EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-libgdiplus=/Library/Frameworks/Mono.framework/Versions/Current/lib/libgdiplus.dylib --host=i386-apple-darwin13.0.0 --build=i386-apple-darwin13.0.0"; fi
  54 if [[ ${CI_TAGS} == *'osx-amd64'* ]]; then EXTRA_CFLAGS="$EXTRA_CFLAGS -m64 -arch x86_64 -mmacosx-version-min=10.9"; EXTRA_LDFLAGS="$EXTRA_LDFLAGS -m64 -arch x86_64" EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-libgdiplus=/Library/Frameworks/Mono.framework/Versions/Current/lib/libgdiplus.dylib"; fi
  55 if [[ ${CI_TAGS} == *'win-i386'* ]]; then PLATFORM=Win32; EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --host=i686-w64-mingw32 --enable-btls"; export MONO_EXECUTABLE="${MONO_REPO_ROOT}/msvc/build/sgen/Win32/bin/Release/mono-sgen.exe"; fi
  56 if [[ ${CI_TAGS} == *'win-amd64'* && ${CI_TAGS} != *'sdks-android'* ]]; then PLATFORM=x64; EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --host=x86_64-w64-mingw32 --disable-boehm --enable-btls"; export MONO_EXECUTABLE="${MONO_REPO_ROOT}/msvc/build/sgen/x64/bin/Release/mono-sgen.exe"; fi
  57 if [[ ${CI_TAGS} == *'make-install'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-loadedllvm --prefix=${MONO_REPO_ROOT}/tmp/monoprefix"; fi
  58
  59 if   [[ ${CI_TAGS} == *'coop-suspend'* ]];   then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --disable-hybrid-suspend --enable-cooperative-suspend";
  60 elif [[ ${CI_TAGS} == *'hybrid-suspend'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-hybrid-suspend";
  61 elif [[ ${CI_TAGS} == *'preemptive-suspend'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --disable-hybrid-suspend";
  62 fi
  63
  64 if [[ ${CI_TAGS} == *'checked-coop'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-checked-build=gc,thread"; fi
  65 if [[ ${CI_TAGS} == *'checked-all'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-checked-build=all"; fi
  66
  67 if [[ ${CI_TAGS} == *'mcs-compiler'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-csc=mcs"; fi
  68 if [[ ${CI_TAGS} == *'disable-mcs-build'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --disable-mcs-build"; fi
  69
  70 if   [[ ${CI_TAGS} == *'fullaot_llvm'* ]];       then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=fullaot_llvm ";
  71 elif [[ ${CI_TAGS} == *'hybridaot_llvm'* ]];     then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=hybridaot_llvm";
  72 elif [[ ${CI_TAGS} == *'winaot_llvm'* ]];        then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=winaot_llvm";
  73 elif [[ ${CI_TAGS} == *'aot_llvm'* ]];           then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=aot_llvm ";
  74 elif [[ ${CI_TAGS} == *'jit_llvm'* ]];           then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes"; export MONO_ENV_OPTIONS="$MONO_ENV_OPTIONS --llvm";
  75 elif [[ ${CI_TAGS} == *'fullaotinterp_llvm'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=fullaotinterp_llvm";
  76 elif [[ ${CI_TAGS} == *'fullaotinterp'* ]];      then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=fullaotinterp";
  77 elif [[ ${CI_TAGS} == *'winaotinterp'* ]];       then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=winaotinterp";
  78 elif [[ ${CI_TAGS} == *'winaotinterp_llvm'* ]];  then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=winaotinterp_llvm";
  79 elif [[ ${CI_TAGS} == *'fullaot'* ]];            then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=fullaot";
  80 elif [[ ${CI_TAGS} == *'hybridaot'* ]];          then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=hybridaot";
  81 elif [[ ${CI_TAGS} == *'winaot'* ]];             then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=winaot";
  82 elif [[ ${CI_TAGS} == *'aot'* ]];                then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=aot";
  83 elif [[ ${CI_TAGS} == *'bitcodeinterp'* ]];      then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=bitcodeinterp"; export PATH="$PATH:${MONO_REPO_ROOT}/llvm/usr/bin";
  84 elif [[ ${CI_TAGS} == *'bitcode'* ]];            then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=bitcode"; export PATH="$PATH:${MONO_REPO_ROOT}/llvm/usr/bin";
  85 elif [[ ${CI_TAGS} == *'acceptance-tests'* ]];   then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --prefix=${MONO_REPO_ROOT}/tmp/mono-acceptance-tests --with-sgen-default-concurrent=yes";
  86 elif [[ ${CI_TAGS} == *'all-profiles'* ]];       then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=all";
  87 elif [[ ${CI_TAGS} == *'compile-msbuild-source'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --prefix=/tmp/mono-from-source";
  88 fi
  89
  90 if [ -x "/usr/bin/dpkg-architecture" ];
  91         then
  92         EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS --host=`/usr/bin/dpkg-architecture -qDEB_HOST_GNU_TYPE`"
  93         #force build arch = dpkg arch, sometimes misdetected
  94         mkdir -p ~/.config/.mono/
  95         wget -O- https://download.mono-project.com/test/new-certs.tgz | tar zx -C ~/.config/.mono/
  96 fi
  97
  98 if [[ ${CI_TAGS} == *'cxx'* ]]; then
  99         EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS -enable-cxx"
 100         MSBUILD_CXX="/p:MONO_COMPILE_AS_CPP=true"
 101 fi
 102
 103 if [[ ${CI_TAGS} == *'microbench'* ]]; then
 104         EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS --with-profile4_x=yes"
 105 fi
 106
 107 if [[ ${CI_TAGS} == *'win-'* ]];
 108 then
 109         mkdir -p ~/.config/.mono/
 110         wget -qO- https://download.mono-project.com/test/new-certs.tgz| tar zx -C ~/.config/.mono/
 111 fi
 112
 113 if [[ ${CI_TAGS} == *'ccache'* ]];
 114 then
 115         # CCACHE_DIR should be set to a directory outside the chroot which holds the cache
 116         CCACHE=$(which ccache)
 117         if [ "$CCACHE" ]; then
 118                 if [[ ${CI_TAGS} == *'osx-'* ]]; then
 119                         export CC="ccache clang"
 120                         export CXX="ccache clang++"
 121                 else
 122                         export CC="ccache gcc"
 123                         export CXX="ccache g++"
 124                 fi
 125         fi
 126 fi
 127
 128 if [[ ${CI_TAGS} == *'sdks-llvm'* ]]; then
 129         ${TESTCMD} --label=archive --timeout=120m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-llvm-llvm{,win}{32,64} NINJA=
 130         exit 0
 131 fi
 132
 133 if [[ ${CI_TAGS} == *'sdks-ios'* ]];
 134    then
 135         # configuration on our bots: https://github.com/mono/mono/pull/11691#issuecomment-439178459
 136         export XCODE_DIR=/Applications/Xcode101.app/Contents/Developer
 137         export XCODE32_DIR=/Applications/Xcode94.app/Contents/Developer
 138         export MACOS_VERSION=10.14
 139         export IOS_VERSION=12.1
 140         export TVOS_VERSION=12.1
 141         export WATCHOS_VERSION=5.1
 142         export WATCHOS64_32_VERSION=5.1
 143
 144         # make sure we embed the correct path into the PDBs
 145         export MONOTOUCH_MCS_FLAGS=-pathmap:${MONO_REPO_ROOT}/=/Library/Frameworks/Xamarin.iOS.framework/Versions/Current/src/Xamarin.iOS/
 146
 147         echo "DISABLE_ANDROID=1" > sdks/Make.config
 148         echo "DISABLE_WASM=1" >> sdks/Make.config
 149         echo "DISABLE_DESKTOP=1" >> sdks/Make.config
 150         if [[ ${CI_TAGS} == *'cxx'* ]]; then
 151             echo "ENABLE_CXX=1" >> sdks/Make.config
 152         fi
 153         if [[ ${CI_TAGS} == *'debug'* ]]; then
 154             echo "CONFIGURATION=debug" >> sdks/Make.config
 155         fi
 156
 157            export device_test_suites="Mono.Runtime.Tests System.Core"
 158
 159            ${TESTCMD} --label=configure --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds configure-ios NINJA=
 160            ${TESTCMD} --label=build     --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds build-ios     NINJA=
 161            ${TESTCMD} --label=archive   --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-ios   NINJA=
 162
 163         if [[ ${CI_TAGS} != *'no-tests'* ]]; then
 164             ${TESTCMD} --label=run-sim --timeout=20m $gnumake -C sdks/ios run-ios-sim-all
 165             ${TESTCMD} --label=build-ios-dev --timeout=60m $gnumake -C sdks/ios build-ios-dev-all
 166             if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
 167                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-${suite} --timeout=10m $gnumake -C sdks/ios run-ios-dev-${suite}; done
 168             fi
 169             ${TESTCMD} --label=build-ios-dev-llvm --timeout=60m $gnumake -C sdks/ios build-ios-dev-llvm-all
 170             if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
 171                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-llvm-${suite} --timeout=10m $gnumake -C sdks/ios run-ios-dev-${suite}; done
 172             fi
 173             ${TESTCMD} --label=build-ios-dev-interp-only --timeout=60m $gnumake -C sdks/ios build-ios-dev-interp-only-all
 174             if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
 175                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-only-${suite} --timeout=10m $gnumake -C sdks/ios run-ios-dev-${suite}; done
 176             fi
 177             ${TESTCMD} --label=build-ios-dev-interp-mixed --timeout=60m $gnumake -C sdks/ios build-ios-dev-interp-mixed-all
 178             if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
 179                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-mixed-${suite} --timeout=10m $gnumake -C sdks/ios run-ios-dev-${suite}; done
 180             fi
 181         fi
 182            exit 0
 183 fi
 184
 185 if [[ ${CI_TAGS} == *'sdks-mac'* ]];
 186 then
 187     # configuration on our bots: https://github.com/mono/mono/pull/11691#issuecomment-439178459
 188     export XCODE_DIR=/Applications/Xcode101.app/Contents/Developer
 189     export XCODE32_DIR=/Applications/Xcode94.app/Contents/Developer
 190     export MACOS_VERSION=10.14
 191
 192     # make sure we embed the correct path into the PDBs
 193     export XAMMAC_MCS_FLAGS=-pathmap:${MONO_REPO_ROOT}/=/Library/Frameworks/Xamarin.Mac.framework/Versions/Current/src/Xamarin.Mac/
 194
 195     echo "DISABLE_IOS=1" > sdks/Make.config
 196     echo "DISABLE_ANDROID=1" >> sdks/Make.config
 197     echo "DISABLE_WASM=1" >> sdks/Make.config
 198     echo "DISABLE_DESKTOP=1" >> sdks/Make.config
 199     if [[ ${CI_TAGS} == *'cxx'* ]]; then
 200         echo "ENABLE_CXX=1" >> sdks/Make.config
 201     fi
 202     if [[ ${CI_TAGS} == *'debug'* ]]; then
 203         echo "CONFIGURATION=debug" >> sdks/Make.config
 204     fi
 205
 206     ${TESTCMD} --label=configure --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds configure-mac NINJA=
 207     ${TESTCMD} --label=build     --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds build-mac     NINJA=
 208     ${TESTCMD} --label=archive   --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-mac   NINJA=
 209
 210     exit 0
 211 fi
 212
 213 if [[ ${CI_TAGS} == *'sdks-android'* ]];
 214    then
 215         echo "DISABLE_IOS=1" > sdks/Make.config
 216         echo "DISABLE_MAC=1" >> sdks/Make.config
 217         echo "DISABLE_WASM=1" >> sdks/Make.config
 218         echo "DISABLE_DESKTOP=1" >> sdks/Make.config
 219         echo "DISABLE_CCACHE=1" >> sdks/Make.config
 220         if [[ ${CI_TAGS} == *'cxx'* ]]; then
 221             echo "ENABLE_CXX=1" >> sdks/Make.config
 222         fi
 223         if [[ ${CI_TAGS} == *'debug'* ]]; then
 224             echo "CONFIGURATION=debug" >> sdks/Make.config
 225         fi
 226
 227         # TODO: provision-android on Windows.
 228         if [[ ${CI_TAGS} != *'win-'* ]]; then
 229         # For some very strange reasons, `make -C sdks/android accept-android-license` get stuck when invoked through ${TESTCMD}
 230             # but doesn't get stuck when called via the shell, so let's just call it here now.
 231             ${TESTCMD} --label=provision-android --timeout=120m --fatal $gnumake -j ${CI_CPU_COUNT} -C sdks/builds provision-android && $gnumake -C sdks/android accept-android-license
 232         fi
 233         ${TESTCMD} --label=provision-mxe --timeout=240m --fatal $gnumake -j ${CI_CPU_COUNT} -C sdks/builds provision-mxe
 234
 235         ${TESTCMD} --label=configure --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds configure-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
 236         ${TESTCMD} --label=build     --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds build-android     NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
 237         ${TESTCMD} --label=archive   --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android   NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
 238
 239         if [[ ${CI_TAGS} != *'no-tests'* ]]; then
 240             ${TESTCMD} --label=mini --timeout=20m $gnumake -C sdks/android check-Mono.Runtime.Tests
 241             ${TESTCMD} --label=corlib --timeout=20m $gnumake -C sdks/android check-corlib
 242             ${TESTCMD} --label=System --timeout=20m $gnumake -C sdks/android check-System
 243             ${TESTCMD} --label=System.Core --timeout=20m $gnumake -C sdks/android check-System.Core
 244             ${TESTCMD} --label=System.Data --timeout=20m $gnumake -C sdks/android check-System.Data
 245             ${TESTCMD} --label=System.IO.Compression.FileSystem --timeout=20m $gnumake -C sdks/android check-System.IO.Compression.FileSystem
 246             ${TESTCMD} --label=System.IO.Compression --timeout=20m $gnumake -C sdks/android check-System.IO.Compression
 247             ${TESTCMD} --label=System.Json --timeout=20m $gnumake -C sdks/android check-System.Json
 248             ${TESTCMD} --label=System.Net.Http --timeout=20m $gnumake -C sdks/android check-System.Net.Http
 249             ${TESTCMD} --label=System.Numerics --timeout=20m $gnumake -C sdks/android check-System.Numerics
 250             ${TESTCMD} --label=System.Runtime.Serialization --timeout=20m $gnumake -C sdks/android check-System.Runtime.Serialization
 251             ${TESTCMD} --label=System.ServiceModel.Web --timeout=20m $gnumake -C sdks/android check-System.ServiceModel.Web
 252             ${TESTCMD} --label=System.Transactions --timeout=20m $gnumake -C sdks/android check-System.Transactions
 253             ${TESTCMD} --label=System.Xml --timeout=20m $gnumake -C sdks/android check-System.Xml
 254             ${TESTCMD} --label=System.Xml.Linq --timeout=20m $gnumake -C sdks/android check-System.Xml.Linq
 255             ${TESTCMD} --label=Mono.CSharp --timeout=20m $gnumake -C sdks/android check-Mono.CSharp
 256             # ${TESTCMD} --label=Mono.Data.Sqlite --timeout=20m $gnumake -C sdks/android check-Mono.Data.Sqlite
 257             ${TESTCMD} --label=Mono.Data.Tds --timeout=20m $gnumake -C sdks/android check-Mono.Data.Tds
 258             ${TESTCMD} --label=Mono.Security --timeout=20m $gnumake -C sdks/android check-Mono.Security
 259             ${TESTCMD} --label=Mono.Debugger.Soft --timeout=20m $gnumake -C sdks/android check-Mono.Debugger.Soft
 260         fi
 261         exit 0
 262 fi
 263
 264 if [[ ${CI_TAGS} == *'webassembly'* ]] || [[ ${CI_TAGS} == *'wasm'* ]];
 265    then
 266         echo "DISABLE_ANDROID=1" > sdks/Make.config
 267         echo "DISABLE_IOS=1" >> sdks/Make.config
 268         echo "DISABLE_MAC=1" >> sdks/Make.config
 269         echo "DISABLE_DESKTOP=1" >> sdks/Make.config
 270         if [[ ${CI_TAGS} == *'cxx'* ]]; then
 271             echo "ENABLE_CXX=1" >> sdks/Make.config
 272         fi
 273         if [[ ${CI_TAGS} == *'debug'* ]]; then
 274             echo "CONFIGURATION=debug" >> sdks/Make.config
 275         fi
 276         echo "ENABLE_WASM_THREADS=1" >> sdks/Make.config
 277
 278            export aot_test_suites="System.Core"
 279            export mixed_test_suites="System.Core"
 280
 281            ${TESTCMD} --label=provision --timeout=20m --fatal $gnumake --output-sync=recurse --trace -C sdks/builds provision-wasm
 282
 283            ${TESTCMD} --label=configure --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds configure-wasm NINJA=
 284            ${TESTCMD} --label=build     --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds build-wasm     NINJA=
 285            ${TESTCMD} --label=archive   --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-wasm   NINJA=
 286
 287         if [[ ${CI_TAGS} != *'no-tests'* ]]; then
 288             ${TESTCMD} --label=wasm-build --timeout=20m --fatal $gnumake -j ${CI_CPU_COUNT} -C sdks/wasm build
 289             ${TESTCMD} --label=ch-mini-test --timeout=20m $gnumake -C sdks/wasm run-ch-mini
 290             ${TESTCMD} --label=v8-mini-test --timeout=20m $gnumake -C sdks/wasm run-v8-mini
 291             ${TESTCMD} --label=sm-mini-test --timeout=20m $gnumake -C sdks/wasm run-sm-mini
 292             ${TESTCMD} --label=jsc-mini-test --timeout=20m $gnumake -C sdks/wasm run-jsc-mini
 293             #The following tests are not passing yet, so enabling them would make us perma-red
 294             #${TESTCMD} --label=mini-corlib --timeout=20m $gnu$gnumake -C sdks/wasm run-all-corlib
 295             #${TESTCMD} --label=mini-system --timeout=20m $gnu$gnumake -C sdks/wasm run-all-system
 296             ${TESTCMD} --label=ch-system-core --timeout=20m $gnumake -C sdks/wasm run-ch-system-core
 297             ${TESTCMD} --label=v8-system-core --timeout=20m $gnumake -C sdks/wasm run-v8-system-core
 298             ${TESTCMD} --label=sm-system-core --timeout=20m $gnumake -C sdks/wasm run-sm-system-core
 299             ${TESTCMD} --label=jsc-system-core --timeout=20m $gnumake -C sdks/wasm run-jsc-system-core
 300             # disable for now until https://github.com/mono/mono/pull/13622 goes in
 301             #${TESTCMD} --label=debugger --timeout=20m $gnumake -C sdks/wasm test-debugger
 302             ${TESTCMD} --label=browser --timeout=20m $gnumake -C sdks/wasm run-browser-tests
 303             ${TESTCMD} --label=browser-threads --timeout=20m $gnumake -C sdks/wasm run-browser-threads-tests
 304             ${TESTCMD} --label=v8-corlib --timeout=20m $gnumake -C sdks/wasm run-v8-corlib
 305             ${TESTCMD} --label=aot-mini --timeout=20m $gnumake -j ${CI_CPU_COUNT} -C sdks/wasm run-aot-mini
 306             ${TESTCMD} --label=build-aot-all --timeout=20m $gnumake -j ${CI_CPU_COUNT} -C sdks/wasm build-aot-all
 307             for suite in ${aot_test_suites}; do ${TESTCMD} --label=run-aot-${suite} --timeout=10m $gnumake -C sdks/wasm run-aot-${suite}; done
 308             for suite in ${mixed_test_suites}; do ${TESTCMD} --label=run-aot-mixed-${suite} --timeout=10m $gnumake -C sdks/wasm run-aot-mixed-${suite}; done
 309             #${TESTCMD} --label=check-aot --timeout=20m $gnumake -C sdks/wasm check-aot
 310             ${TESTCMD} --label=package --timeout=20m $gnumake -C sdks/wasm package
 311         fi
 312            exit 0
 313 fi
 314
 315
 316 if [[ ${CI_TAGS} != *'mac-sdk'* ]]; # Mac SDK builds Mono itself
 317         then
 318         echo ./autogen.sh CFLAGS="$CFLAGS $EXTRA_CFLAGS" CXXFLAGS="$CXXFLAGS $EXTRA_CXXFLAGS" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" $EXTRA_CONF_FLAGS
 319         ${TESTCMD} --label=configure --timeout=60m --fatal ./autogen.sh CFLAGS="$CFLAGS $EXTRA_CFLAGS" CXXFLAGS="$CXXFLAGS $EXTRA_CXXFLAGS" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" $EXTRA_CONF_FLAGS
 320 fi
 321 if [[ ${CI_TAGS} == *'win-i386'* ]];
 322     then
 323         # only build boehm on w32 (only windows platform supporting boehm).
 324     ${TESTCMD} --label=make-msvc --timeout=60m --fatal ./msvc/run-msbuild.sh "build" "${PLATFORM}" "release" "/p:PlatformToolset=v140 /p:MONO_TARGET_GC=boehm ${MSBUILD_CXX}"
 325 fi
 326 if [[ ${CI_TAGS} == *'win-'* ]];
 327     then
 328     ${TESTCMD} --label=make-msvc-sgen --timeout=60m --fatal ./msvc/run-msbuild.sh "build" "${PLATFORM}" "release" "/p:PlatformToolset=v140 /p:MONO_TARGET_GC=sgen ${MSBUILD_CXX}"
 329 fi
 330
 331 if [[ ${CI_TAGS} == *'win-amd64'* ]];
 332     then
 333     # The AOT compiler on Windows requires Visual Studio's clang.exe and link.exe.
 334     # Depending on codegen (JIT/LLVM) it might also need platform specific libraries.
 335     # Use a wrapper script that will make sure to setup full VS MSVC environment if
 336     # needed when running mono-sgen.exe as AOT compiler.
 337     export MONO_EXECUTABLE_WRAPPER="${MONO_REPO_ROOT}/msvc/build/sgen/x64/bin/Release/mono-sgen-msvc.sh"
 338 fi
 339
 340 if [[ ${CI_TAGS} == *'monolite'* ]]; then make get-monolite-latest; fi
 341
 342 make_parallelism="-j ${CI_CPU_COUNT}"
 343 if [[ ${CI_TAGS} == *'linux-ppc64el'* ]]; then make_parallelism=-j1; fi
 344
 345 make_continue=
 346 if [[ ${CI_TAGS} == *'checked-all'* ]]; then make_continue=-k; fi
 347
 348 if [[ ${CI_TAGS} != *'mac-sdk'* ]]; # Mac SDK builds Mono itself
 349     then
 350     build_error=0
 351     ${TESTCMD} --label=make --timeout=${make_timeout} --fatal make ${make_parallelism} ${make_continue} -w V=1 || build_error=1
 352     helix_send_build_done_event "build/source/$MONO_HELIX_TYPE/" $build_error
 353
 354     if [[ ${build_error} != 0 ]]; then
 355         echo "ERROR: The Mono build failed."
 356         ${MONO_REPO_ROOT}/scripts/ci/run-upload-sentry.sh
 357         exit ${build_error}
 358     fi
 359 fi
 360
 361 if [[ ${CI_TAGS} == *'checked-coop'* ]]; then export MONO_CHECK_MODE=gc,thread; fi
 362 if [[ ${CI_TAGS} == *'checked-all'* ]]; then export MONO_CHECK_MODE=all; fi
 363
 364 if [[ ${CI_TAGS} == *'hardened-runtime'* ]]; then codesign -s - -fv -o runtime --entitlements ${MONO_REPO_ROOT}/mono/mini/mac-entitlements.plist ${MONO_REPO_ROOT}/mono/mini/mono-sgen; fi
 365
 366 export MONO_ENV_OPTIONS="$MONO_ENV_OPTIONS $MONO_TEST_ENV_OPTIONS"
 367
 368 if   [[ ${CI_TAGS} == *'acceptance-tests'* ]];         then ${MONO_REPO_ROOT}/scripts/ci/run-test-acceptance-tests.sh;
 369 elif [[ ${CI_TAGS} == *'microbench'* ]];               then ${MONO_REPO_ROOT}/scripts/ci/run-test-microbench.sh;
 370 elif [[ ${CI_TAGS} == *'profiler-stress-tests'* ]];    then ${MONO_REPO_ROOT}/scripts/ci/run-test-profiler-stress-tests.sh;
 371 elif [[ ${CI_TAGS} == *'stress-tests'* ]];             then ${MONO_REPO_ROOT}/scripts/ci/run-test-stress-tests.sh;
 372 elif [[ ${CI_TAGS} == *'interpreter'* ]];              then ${MONO_REPO_ROOT}/scripts/ci/run-test-interpreter.sh;
 373 elif [[ ${CI_TAGS} == *'mcs-compiler'* ]];             then ${MONO_REPO_ROOT}/scripts/ci/run-test-mcs.sh;
 374 elif [[ ${CI_TAGS} == *'mac-sdk'* ]];                  then ${MONO_REPO_ROOT}/scripts/ci/run-test-mac-sdk.sh;
 375 elif [[ ${CI_TAGS} == *'helix-tests'* ]];              then ${MONO_REPO_ROOT}/scripts/ci/run-test-helix.sh;
 376 elif [[ ${CI_TAGS} == *'compile-msbuild-source'* ]];   then ${MONO_REPO_ROOT}/scripts/ci/run-test-msbuild.sh;
 377 elif [[ ${CI_TAGS} == *'make-install'* ]];             then ${MONO_REPO_ROOT}/scripts/ci/run-test-make-install.sh;
 378 elif [[ ${CI_TAGS} == *'compiler-server-tests'* ]];          then ${MONO_REPO_ROOT}/scripts/ci/run-test-compiler-server.sh;
 379 elif [[ ${CI_TAGS} == *'no-tests'* ]];                 then echo "Skipping tests.";
 380 else make check-ci;
 381 fi
 382
 383 if [[ $CI_TAGS == *'apidiff'* ]]; then
 384     if ${TESTCMD} --label=apidiff --timeout=15m --fatal make -w -C mcs -j ${CI_CPU_COUNT} mono-api-diff
 385     then report_github_status "success" "API Diff" "No public API changes found." || true
 386     else report_github_status "error" "API Diff" "The public API changed." "$BUILD_URL/Public_20API_20Diff/" || true
 387     fi
 388 fi
 389 if [[ $CI_TAGS == *'csprojdiff'* ]]; then
 390     make update-solution-files
 391     if ${TESTCMD} --label=csprojdiff --timeout=5m --fatal make -w -C mcs mono-csproj-diff
 392     then report_github_status "success" "Project Files Diff" "No csproj file changes found." || true
 393     else report_github_status "error" "Project Files Diff" "The csproj files changed." "$BUILD_URL/Project_20Files_20Diff/" || true
 394     fi
 395 fi