scripts/ci/run-jenkins.sh

   1 #!/bin/bash -e
   2
   3 export MONO_REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../" && pwd )"
   4 export TESTCMD=${MONO_REPO_ROOT}/scripts/ci/run-step.sh
   5 export CI=1
   6 export CI_PR=$([[ ${CI_TAGS} == *'pull-request'* ]] && echo 1 || true)
   7 export CI_CPU_COUNT=$(getconf _NPROCESSORS_ONLN || echo 4)
   8 export TEST_HARNESS_VERBOSE=1
   9
  10 # workaround for acceptance-tests submodules leaving files behind since Jenkins only does "git clean -xdf" (no second 'f')
  11 # which won't clean untracked .git repos (remove once https://github.com/jenkinsci/git-plugin/pull/449 is available)
  12 for dir in acceptance-tests/external/*; do [ -d "$dir" ] && (cd "$dir" && echo "Cleaning $dir" && git clean -xdff); done
  13
  14 source ${MONO_REPO_ROOT}/scripts/ci/util.sh
  15
  16 helix_set_env_vars
  17 helix_send_build_start_event "build/source/$MONO_HELIX_TYPE/"
  18
  19 make_timeout=300m
  20
  21 if [[ ${CI_TAGS} == *'clang-sanitizer'* ]]; then
  22         export CC="clang"
  23         export CXX="clang++"
  24         export CFLAGS="$CFLAGS -g -O1 -fsanitize=thread -fsanitize-blacklist=${MONO_REPO_ROOT}/scripts/ci/clang-thread-sanitizer-blacklist -mllvm -tsan-instrument-atomics=false"
  25         export LDFLAGS="-fsanitize=thread"
  26         # TSAN_OPTIONS are used by programs that were compiled with Clang's ThreadSanitizer
  27         # see https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags for more details
  28         export TSAN_OPTIONS="history_size=7:exitcode=0:force_seq_cst_atomics=1"
  29         make_timeout=30m
  30 fi
  31
  32 if [[ ${CI_TAGS} == *'win-'* ]]; then
  33     # Passing -ggdb3 on Cygwin breaks linking against libmonosgen-x.y.dll
  34     export CFLAGS="$CFLAGS -g -O2"
  35 else
  36     export EXTRA_CFLAGS="$EXTRA_CFLAGS -ggdb3 -O2"
  37 fi
  38
  39 if [[ $CI_TAGS == *'collect-coverage'* ]]; then
  40     # Collect coverage for further use by lcov and similar tools.
  41     # Coverage must be collected with -O0 and debug information.
  42     export CFLAGS="$CFLAGS -ggdb3 --coverage -O0"
  43 fi
  44
  45 if [[ $CI_TAGS == *'retry-flaky-tests'* ]]; then
  46     export MONO_FLAKY_TEST_RETRIES=5
  47 fi
  48
  49 # We don't want to have to maintain symbolification blobs for CI
  50 export EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-crash-privacy=no "
  51
  52 if [[ ${CI_TAGS} == *'osx-i386'* ]]; then EXTRA_CFLAGS="$EXTRA_CFLAGS -m32 -arch i386 -mmacosx-version-min=10.9"; EXTRA_LDFLAGS="$EXTRA_LDFLAGS -m32 -arch i386"; EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-libgdiplus=/Library/Frameworks/Mono.framework/Versions/Current/lib/libgdiplus.dylib --host=i386-apple-darwin13.0.0 --build=i386-apple-darwin13.0.0"; fi
  53 if [[ ${CI_TAGS} == *'osx-amd64'* ]]; then EXTRA_CFLAGS="$EXTRA_CFLAGS -m64 -arch x86_64 -mmacosx-version-min=10.9"; EXTRA_LDFLAGS="$EXTRA_LDFLAGS -m64 -arch x86_64" EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-libgdiplus=/Library/Frameworks/Mono.framework/Versions/Current/lib/libgdiplus.dylib"; fi
  54 if [[ ${CI_TAGS} == *'win-i386'* ]]; then PLATFORM=Win32; EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --host=i686-w64-mingw32 --enable-btls"; export MONO_EXECUTABLE="${MONO_REPO_ROOT}/msvc/build/sgen/Win32/bin/Release/mono-sgen.exe"; fi
  55 if [[ ${CI_TAGS} == *'win-amd64'* ]]; then PLATFORM=x64; EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --host=x86_64-w64-mingw32 --disable-boehm --enable-btls"; export MONO_EXECUTABLE="${MONO_REPO_ROOT}/msvc/build/sgen/x64/bin/Release/mono-sgen.exe"; fi
  56
  57 if   [[ ${CI_TAGS} == *'coop-suspend'* ]];   then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --disable-hybrid-suspend --enable-cooperative-suspend";
  58 elif [[ ${CI_TAGS} == *'hybrid-suspend'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-hybrid-suspend";
  59 elif [[ ${CI_TAGS} == *'preemptive-suspend'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --disable-hybrid-suspend";
  60 fi
  61
  62 if [[ ${CI_TAGS} == *'checked-coop'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-checked-build=gc,thread"; fi
  63 if [[ ${CI_TAGS} == *'checked-all'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-checked-build=all"; fi
  64
  65 if [[ ${CI_TAGS} == *'mcs-compiler'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-csc=mcs"; fi
  66 if [[ ${CI_TAGS} == *'disable-mcs-build'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --disable-mcs-build"; fi
  67
  68 if   [[ ${CI_TAGS} == *'fullaot_llvm'* ]];       then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=fullaot_llvm ";
  69 elif [[ ${CI_TAGS} == *'hybridaot_llvm'* ]];     then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=hybridaot_llvm";
  70 elif [[ ${CI_TAGS} == *'aot_llvm'* ]];           then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=aot_llvm ";
  71 elif [[ ${CI_TAGS} == *'jit_llvm'* ]];           then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes"; export MONO_ENV_OPTIONS="$MONO_ENV_OPTIONS --llvm";
  72 elif [[ ${CI_TAGS} == *'fullaotinterp_llvm'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=fullaotinterp_llvm";
  73 elif [[ ${CI_TAGS} == *'fullaotinterp'* ]];      then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=fullaotinterp";
  74 elif [[ ${CI_TAGS} == *'fullaot'* ]];            then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=fullaot";
  75 elif [[ ${CI_TAGS} == *'hybridaot'* ]];          then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=hybridaot";
  76 elif [[ ${CI_TAGS} == *'winaot'* ]];             then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=winaot";
  77 elif [[ ${CI_TAGS} == *'aot'* ]];                then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=aot";
  78 elif [[ ${CI_TAGS} == *'bitcode'* ]];            then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=bitcode"; export MONO_ENV_OPTIONS="$MONO_ENV_OPTIONS --aot=clangxx=clang++-6.0";
  79 elif [[ ${CI_TAGS} == *'acceptance-tests'* ]];   then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --prefix=${MONO_REPO_ROOT}/tmp/mono-acceptance-tests --with-sgen-default-concurrent=yes";
  80 elif [[ ${CI_TAGS} == *'all-profiles'* ]]; then
  81     # only enable build of the additional profiles on one config to save time
  82     EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=all"
  83     # when building profiles like monotouch/monodroid which don't build System.Drawing.dll in the Mono repo we need
  84     # to build the facades against _something_ to satisfy the typeforwards. In CI we can cheat a little and pass
  85     # them System.Drawing.dll from the 'build' profile since we don't test those profiles here (we just ensure they compile).
  86     export EXTERNAL_FACADE_DRAWING_REFERENCE=${MONO_REPO_ROOT}/mcs/class/lib/build/System.Drawing.dll
  87 fi
  88
  89 if [ -x "/usr/bin/dpkg-architecture" ];
  90         then
  91         EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS --host=`/usr/bin/dpkg-architecture -qDEB_HOST_GNU_TYPE`"
  92         #force build arch = dpkg arch, sometimes misdetected
  93         mkdir -p ~/.config/.mono/
  94         wget -qO- https://download.mono-project.com/test/new-certs.tgz| tar zx -C ~/.config/.mono/
  95 fi
  96
  97 if [[ ${CI_TAGS} == *'cxx'* ]]; then
  98         EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS -enable-cxx"
  99         MSBUILD_CXX="/p:MONO_COMPILE_AS_CPP=true"
 100 fi
 101
 102 if [[ ${CI_TAGS} == *'microbench'* ]]; then
 103         EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS --with-profile4_x=yes"
 104 fi
 105
 106 if [[ ${CI_TAGS} == *'win-'* ]];
 107 then
 108         mkdir -p ~/.config/.mono/
 109         wget -qO- https://download.mono-project.com/test/new-certs.tgz| tar zx -C ~/.config/.mono/
 110 fi
 111
 112 if [[ ${CI_TAGS} == *'sdks-llvm'* ]]; then
 113         ${TESTCMD} --label=archive --timeout=120m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-llvm-llvm{,win}{32,64} NINJA=
 114         if [[ ${CI_TAGS} == *'osx-amd64'* ]]; then
 115                 ${TESTCMD} --label=archive-llvm36 --timeout=60m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-llvm36-llvm32 NINJA=
 116         fi
 117         exit 0
 118 fi
 119
 120 if [[ ${CI_TAGS} == *'sdks-ios'* ]];
 121    then
 122            # configuration on our bots: https://github.com/mono/mono/pull/11691#issuecomment-439178459
 123            export XCODE_DIR=/Applications/Xcode101.app/Contents/Developer
 124            export XCODE32_DIR=/Applications/Xcode94.app/Contents/Developer
 125
 126            echo "DISABLE_ANDROID=1" > sdks/Make.config
 127            echo "DISABLE_WASM=1" >> sdks/Make.config
 128            echo "DISABLE_DESKTOP=1" >> sdks/Make.config
 129            if [[ ${CI_TAGS} == *'cxx'* ]]; then
 130                    echo "ENABLE_CXX=1" >> sdks/Make.config
 131            fi
 132            export device_test_suites="Mono.Runtime.Tests System.Core"
 133
 134            ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-ios NINJA=
 135
 136         if [[ ${CI_TAGS} != *'no-tests'* ]]; then
 137             ${TESTCMD} --label=run-sim --timeout=20m make -C sdks/ios run-ios-sim-all
 138             ${TESTCMD} --label=build-ios-dev --timeout=60m make -C sdks/ios build-ios-dev-all
 139             if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
 140                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
 141             fi
 142             ${TESTCMD} --label=build-ios-dev-llvm --timeout=60m make -C sdks/ios build-ios-dev-llvm-all
 143             if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
 144                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-llvm-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
 145             fi
 146             ${TESTCMD} --label=build-ios-dev-interp-only --timeout=60m make -C sdks/ios build-ios-dev-interp-only-all
 147             if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
 148                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-only-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
 149             fi
 150             ${TESTCMD} --label=build-ios-dev-interp-mixed --timeout=60m make -C sdks/ios build-ios-dev-interp-mixed-all
 151             if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
 152                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-mixed-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
 153             fi
 154         fi
 155            exit 0
 156 fi
 157
 158 if [[ ${CI_TAGS} == *'sdks-android'* ]];
 159    then
 160         echo "DISABLE_IOS=1" > sdks/Make.config
 161         echo "DISABLE_WASM=1" >> sdks/Make.config
 162         echo "DISABLE_DESKTOP=1" >> sdks/Make.config
 163         echo "DISABLE_CCACHE=1" >> sdks/Make.config
 164         if [[ ${CI_TAGS} == *'cxx'* ]]; then
 165             echo "ENABLE_CXX=1" >> sdks/Make.config
 166         fi
 167
 168         # For some very strange reasons, `make -C sdks/android accept-android-license` get stuck when invoked through ${TESTCMD}
 169         # but doesn't get stuck when called via the shell, so let's just call it here now.
 170         ${TESTCMD} --label=provision-android --timeout=120m --fatal make -j ${CI_CPU_COUNT} -C sdks/builds provision-android && make -C sdks/android accept-android-license
 171         ${TESTCMD} --label=provision-mxe --timeout=240m --fatal make -j ${CI_CPU_COUNT} -C sdks/builds provision-mxe
 172         if [[ ${CI_TAGS} != *'debug'* ]]; then
 173             ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
 174         else
 175             ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1 CONFIGURATION=debug
 176         fi
 177
 178         if [[ ${CI_TAGS} != *'no-tests'* ]]; then
 179             ${TESTCMD} --label=mini --timeout=60m make -C sdks/android check-mini
 180             ${TESTCMD} --label=corlib --timeout=60m make -C sdks/android check-corlib
 181             ${TESTCMD} --label=System --timeout=60m make -C sdks/android check-System
 182             ${TESTCMD} --label=System.Core --timeout=60m make -C sdks/android check-System.Core
 183             ${TESTCMD} --label=System.Data --timeout=60m make -C sdks/android check-System.Data
 184             ${TESTCMD} --label=System.IO.Compression.FileSystem --timeout=60m make -C sdks/android check-System.IO.Compression.FileSystem
 185             ${TESTCMD} --label=System.IO.Compression --timeout=60m make -C sdks/android check-System.IO.Compression
 186             ${TESTCMD} --label=System.Json --timeout=60m make -C sdks/android check-System.Json
 187             ${TESTCMD} --label=System.Net.Http --timeout=60m make -C sdks/android check-System.Net.Http
 188             ${TESTCMD} --label=System.Numerics --timeout=60m make -C sdks/android check-System.Numerics
 189             ${TESTCMD} --label=System.Runtime.Serialization --timeout=60m make -C sdks/android check-System.Runtime.Serialization
 190             ${TESTCMD} --label=System.ServiceModel.Web --timeout=60m make -C sdks/android check-System.ServiceModel.Web
 191             ${TESTCMD} --label=System.Transactions --timeout=60m make -C sdks/android check-System.Transactions
 192             ${TESTCMD} --label=System.Xml --timeout=60m make -C sdks/android check-System.Xml
 193             ${TESTCMD} --label=System.Xml.Linq --timeout=60m make -C sdks/android check-System.Xml.Linq
 194             ${TESTCMD} --label=Mono.CSharp --timeout=60m make -C sdks/android check-Mono.CSharp
 195             ${TESTCMD} --label=Mono.Data.Sqlite --timeout=60m make -C sdks/android check-Mono.Data.Sqlite
 196             ${TESTCMD} --label=Mono.Data.Tds --timeout=60m make -C sdks/android check-Mono.Data.Tds
 197             ${TESTCMD} --label=Mono.Security --timeout=60m make -C sdks/android check-Mono.Security
 198         fi
 199         exit 0
 200 fi
 201
 202 if [[ ${CI_TAGS} == *'webassembly'* ]] || [[ ${CI_TAGS} == *'wasm'* ]];
 203    then
 204            echo "DISABLE_ANDROID=1" > sdks/Make.config
 205            echo "DISABLE_IOS=1" >> sdks/Make.config
 206            echo "DISABLE_DESKTOP=1" >> sdks/Make.config
 207            if [[ ${CI_TAGS} == *'cxx'* ]]; then
 208                echo "ENABLE_CXX=1" >> sdks/Make.config
 209            fi
 210
 211            export aot_test_suites="System.Core"
 212
 213            ${TESTCMD} --label=provision --timeout=20m --fatal make --output-sync=recurse --trace -C sdks/builds provision-wasm
 214            ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-wasm NINJA=
 215
 216         if [[ ${CI_TAGS} != *'no-tests'* ]]; then
 217             ${TESTCMD} --label=wasm-build --timeout=60m --fatal make -j ${CI_CPU_COUNT} -C sdks/wasm build
 218             ${TESTCMD} --label=ch-mini-test --timeout=60m make -C sdks/wasm run-ch-mini
 219             ${TESTCMD} --label=v8-mini-test --timeout=60m make -C sdks/wasm run-v8-mini
 220             ${TESTCMD} --label=sm-mini-test --timeout=60m make -C sdks/wasm run-sm-mini
 221             ${TESTCMD} --label=jsc-mini-test --timeout=60m make -C sdks/wasm run-jsc-mini
 222             #The following tests are not passing yet, so enabling them would make us perma-red
 223             #${TESTCMD} --label=mini-corlib --timeout=60m make -C sdks/wasm run-all-corlib
 224             #${TESTCMD} --label=mini-system --timeout=60m make -C sdks/wasm run-all-system
 225             ${TESTCMD} --label=ch-system-core --timeout=60m make -C sdks/wasm run-ch-system-core
 226             ${TESTCMD} --label=v8-system-core --timeout=60m make -C sdks/wasm run-v8-system-core
 227             ${TESTCMD} --label=sm-system-core --timeout=60m make -C sdks/wasm run-sm-system-core
 228             ${TESTCMD} --label=jsc-system-core --timeout=60m make -C sdks/wasm run-jsc-system-core
 229             ${TESTCMD} --label=v8-corlib --timeout=60m make -C sdks/wasm run-v8-corlib
 230             ${TESTCMD} --label=aot-mini --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm run-aot-mini
 231             ${TESTCMD} --label=build-aot-all --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm build-aot-all
 232             for suite in ${aot_test_suites}; do ${TESTCMD} --label=run-aot-${suite} --timeout=10m make -C sdks/wasm run-aot-${suite}; done
 233             #${TESTCMD} --label=check-aot --timeout=60m make -C sdks/wasm check-aot
 234             ${TESTCMD} --label=package --timeout=60m make -C sdks/wasm package
 235         fi
 236            exit 0
 237 fi
 238
 239
 240 if [[ ${CI_TAGS} != *'mac-sdk'* ]]; # Mac SDK builds Mono itself
 241         then
 242         echo ./autogen.sh CFLAGS="$CFLAGS $EXTRA_CFLAGS" CXXFLAGS="$CXXFLAGS $EXTRA_CXXFLAGS" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" $EXTRA_CONF_FLAGS
 243         ${TESTCMD} --label=configure --timeout=60m --fatal ./autogen.sh CFLAGS="$CFLAGS $EXTRA_CFLAGS" CXXFLAGS="$CXXFLAGS $EXTRA_CXXFLAGS" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" $EXTRA_CONF_FLAGS
 244 fi
 245 if [[ ${CI_TAGS} == *'win-i386'* ]];
 246     then
 247         # only build boehm on w32 (only windows platform supporting boehm).
 248     ${TESTCMD} --label=make-msvc --timeout=60m --fatal /cygdrive/c/Program\ Files\ \(x86\)/MSBuild/14.0/Bin/MSBuild.exe /p:PlatformToolset=v140 /p:Platform=${PLATFORM} /p:Configuration=Release ${MSBUILD_CXX} /p:MONO_TARGET_GC=boehm msvc/mono.sln
 249 fi
 250 if [[ ${CI_TAGS} == *'win-'* ]];
 251     then
 252     ${TESTCMD} --label=make-msvc-sgen --timeout=60m --fatal /cygdrive/c/Program\ Files\ \(x86\)/MSBuild/14.0/Bin/MSBuild.exe /p:PlatformToolset=v140 /p:Platform=${PLATFORM} /p:Configuration=Release ${MSBUILD_CXX} /p:MONO_TARGET_GC=sgen msvc/mono.sln
 253 fi
 254
 255 if [[ ${CI_TAGS} == *'winaot'* ]];
 256     then
 257     # The AOT compiler on Windows requires Visual Studio's clang.exe and link.exe in $PATH
 258     # and we must make sure Visual Studio's link.exe comes before Cygwin's link.exe
 259     VC_ROOT="/cygdrive/c/Program Files (x86)/Microsoft Visual Studio 14.0/VC"
 260     export PATH="$VC_ROOT/ClangC2/bin/amd64:$VC_ROOT/bin/amd64":$PATH
 261 fi
 262
 263 if [[ ${CI_TAGS} == *'monolite'* ]]; then make get-monolite-latest; fi
 264
 265 make_parallelism="-j ${CI_CPU_COUNT}"
 266 if [[ ${CI_TAGS} == *'linux-ppc64el'* ]]; then make_parallelism=-j1; fi
 267
 268 make_continue=
 269 if [[ ${CI_TAGS} == *'checked-all'* ]]; then make_continue=-k; fi
 270
 271 if [[ ${CI_TAGS} != *'mac-sdk'* ]]; # Mac SDK builds Mono itself
 272     then
 273     build_error=0
 274     ${TESTCMD} --label=make --timeout=${make_timeout} --fatal make ${make_parallelism} ${make_continue} -w V=1 || build_error=1
 275     helix_send_build_done_event "build/source/$MONO_HELIX_TYPE/" $build_error
 276
 277     if [[ ${build_error} != 0 ]]; then
 278         echo "ERROR: The Mono build failed."
 279         ${MONO_REPO_ROOT}/scripts/ci/run-upload-sentry.sh
 280         exit ${build_error}
 281     fi
 282 fi
 283
 284 if [[ ${CI_TAGS} == *'checked-coop'* ]]; then export MONO_CHECK_MODE=gc,thread; fi
 285 if [[ ${CI_TAGS} == *'checked-all'* ]]; then export MONO_CHECK_MODE=all; fi
 286
 287 export MONO_ENV_OPTIONS="$MONO_ENV_OPTIONS $MONO_TEST_ENV_OPTIONS"
 288
 289 if   [[ ${CI_TAGS} == *'acceptance-tests'* ]];         then ${MONO_REPO_ROOT}/scripts/ci/run-test-acceptance-tests.sh;
 290 elif [[ ${CI_TAGS} == *'microbench'* ]];               then ${MONO_REPO_ROOT}/scripts/ci/run-test-microbench.sh;
 291 elif [[ ${CI_TAGS} == *'profiler-stress-tests'* ]];    then ${MONO_REPO_ROOT}/scripts/ci/run-test-profiler-stress-tests.sh;
 292 elif [[ ${CI_TAGS} == *'stress-tests'* ]];             then ${MONO_REPO_ROOT}/scripts/ci/run-test-stress-tests.sh;
 293 elif [[ ${CI_TAGS} == *'interpreter'* ]];              then ${MONO_REPO_ROOT}/scripts/ci/run-test-interpreter.sh;
 294 elif [[ ${CI_TAGS} == *'mcs-compiler'* ]];             then ${MONO_REPO_ROOT}/scripts/ci/run-test-mcs.sh;
 295 elif [[ ${CI_TAGS} == *'mac-sdk'* ]];                  then ${MONO_REPO_ROOT}/scripts/ci/run-test-mac-sdk.sh;
 296 elif [[ ${CI_TAGS} == *'helix-tests'* ]];              then ${MONO_REPO_ROOT}/scripts/ci/run-test-helix.sh;
 297 elif [[ ${CI_TAGS} == *'no-tests'* ]];                 then echo "Skipping tests.";
 298 else make check-ci;
 299 fi
 300
 301 if [[ $CI_TAGS == *'apidiff'* ]]; then
 302     if ${TESTCMD} --label=apidiff --timeout=15m --fatal make -w -C mcs -j ${CI_CPU_COUNT} mono-api-diff
 303     then report_github_status "success" "API Diff" "No public API changes found." || true
 304     else report_github_status "error" "API Diff" "The public API changed." "$BUILD_URL/Public_20API_20Diff/" || true
 305     fi
 306 fi
 307 if [[ $CI_TAGS == *'csprojdiff'* ]]; then
 308     make update-solution-files
 309     if ${TESTCMD} --label=csprojdiff --timeout=5m --fatal make -w -C mcs mono-csproj-diff
 310     then report_github_status "success" "Project Files Diff" "No csproj file changes found." || true
 311     else report_github_status "error" "Project Files Diff" "The csproj files changed." "$BUILD_URL/Project_20Files_20Diff/" || true
 312     fi
 313 fi