scripts/ci/run-jenkins.sh

   1 #!/bin/bash -e
   2 # -*- mode: shell-script; indent-tabs-mode: nil; -*-
   3
   4 export MONO_REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../" && pwd )"
   5 export TESTCMD=${MONO_REPO_ROOT}/scripts/ci/run-step.sh
   6 export CI=1
   7 export CI_PR=$([[ ${CI_TAGS} == *'pull-request'* ]] && echo 1 || true)
   8 export CI_CPU_COUNT=$(getconf _NPROCESSORS_ONLN || getconf NPROCESSORS_ONLN || echo 4)
   9 export TEST_HARNESS_VERBOSE=1
  10
  11 # workaround for acceptance-tests submodules leaving files behind since Jenkins only does "git clean -xdf" (no second 'f')
  12 # which won't clean untracked .git repos (remove once https://github.com/jenkinsci/git-plugin/pull/449 is available)
  13 for dir in acceptance-tests/external/*; do [ -d "$dir" ] && (cd "$dir" && echo "Cleaning $dir" && git clean -xdff); done
  14
  15 source ${MONO_REPO_ROOT}/scripts/ci/util.sh
  16
  17 if [[ ${CI_TAGS} == *'pull-request'* ]]; then
  18         # Skip lanes which are not affected by the PR
  19         wget -O pr-contents.diff "${ghprbPullLink}.diff"
  20         grep '^diff' pr-contents.diff > pr-files.txt
  21         echo "Files affected by the PR:"
  22         cat pr-files.txt
  23
  24         # FIXME: Add more
  25         skip=false
  26         skip_step=""
  27         if ! grep -q -v -e a/netcore -e a/scripts/ci/pipeline-netcore-runtime.yml pr-files.txt; then
  28                 skip_step="NETCORE"
  29                 skip=true
  30         fi
  31         if ! grep -q -v a/mono/mini/mini-ppc pr-files.txt; then
  32                 skip_step="PPC"
  33                 skip=true
  34         fi
  35         if ! grep -q -v a/scripts/ci/provisioning pr-files.txt; then
  36                 skip_step="CI provisioning scripts"
  37                 skip=true
  38         fi
  39         if ! grep -q -v a/sdks/wasm pr-files.txt; then
  40                 if [[ ${CI_TAGS} == *'webassembly'* ]] || [[ ${CI_TAGS} == *'wasm'* ]]; then
  41                         true
  42                 else
  43                         skip_step="WASM"
  44                         skip=true
  45                 fi
  46         fi
  47         if [ $skip = true ]; then
  48                 ${TESTCMD} --label="Skipped on ${skip_step}." --timeout=60m --fatal sh -c 'exit 0'
  49                 if [[ $CI_TAGS == *'apidiff'* ]]; then report_github_status "success" "API Diff" "Skipped." || true; fi
  50                 if [[ $CI_TAGS == *'csprojdiff'* ]]; then report_github_status "success" "Project Files Diff" "Skipped." || true; fi
  51                 exit 0
  52         fi
  53
  54     rm pr-files.txt
  55 fi
  56
  57 helix_set_env_vars
  58 helix_send_build_start_event "build/source/$MONO_HELIX_TYPE/"
  59
  60 make_timeout=300m
  61 gnumake=$(which gmake || which gnumake || which make)
  62
  63 if [[ ${CI_TAGS} == *'clang-sanitizer'* ]]; then
  64         export CC="clang"
  65         export CXX="clang++"
  66         export CFLAGS="$CFLAGS -g -O1 -fsanitize=thread -fsanitize-blacklist=${MONO_REPO_ROOT}/scripts/ci/clang-thread-sanitizer-blacklist -mllvm -tsan-instrument-atomics=false"
  67         export LDFLAGS="-fsanitize=thread"
  68         # TSAN_OPTIONS are used by programs that were compiled with Clang's ThreadSanitizer
  69         # see https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags for more details
  70         export TSAN_OPTIONS="history_size=7:exitcode=0:force_seq_cst_atomics=1"
  71         make_timeout=30m
  72 fi
  73
  74 if [[ ${CI_TAGS} == *'win-'* ]]; then
  75     # Passing -ggdb3 on Cygwin breaks linking against libmonosgen-x.y.dll
  76     export CFLAGS="$CFLAGS -g -O2"
  77 else
  78     export EXTRA_CFLAGS="$EXTRA_CFLAGS -ggdb3 -O2"
  79 fi
  80
  81 if [[ $CI_TAGS == *'collect-coverage'* ]]; then
  82     # Collect coverage for further use by lcov and similar tools.
  83     # Coverage must be collected with -O0 and debug information.
  84     export CFLAGS="$CFLAGS -ggdb3 --coverage -O0"
  85 fi
  86
  87 if [[ $CI_TAGS == *'retry-flaky-tests'* ]]; then
  88     export MONO_FLAKY_TEST_RETRIES=5
  89 fi
  90
  91 # We don't want to have to maintain symbolification blobs for CI
  92 export EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-crash-privacy=no "
  93
  94 if [[ ${CI_TAGS} == *'osx-i386'* ]]; then EXTRA_CFLAGS="$EXTRA_CFLAGS -m32 -arch i386 -mmacosx-version-min=10.9"; EXTRA_LDFLAGS="$EXTRA_LDFLAGS -m32 -arch i386"; EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-libgdiplus=/Library/Frameworks/Mono.framework/Versions/Current/lib/libgdiplus.dylib --host=i386-apple-darwin13.0.0 --build=i386-apple-darwin13.0.0"; fi
  95 if [[ ${CI_TAGS} == *'osx-amd64'* ]]; then EXTRA_CFLAGS="$EXTRA_CFLAGS -m64 -arch x86_64 -mmacosx-version-min=10.9"; EXTRA_LDFLAGS="$EXTRA_LDFLAGS -m64 -arch x86_64" EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-libgdiplus=/Library/Frameworks/Mono.framework/Versions/Current/lib/libgdiplus.dylib"; fi
  96 if [[ ${CI_TAGS} == *'win-i386'* ]]; then PLATFORM=Win32; EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --host=i686-w64-mingw32 --enable-btls"; export MONO_EXECUTABLE="${MONO_REPO_ROOT}/msvc/build/sgen/Win32/bin/Release/mono-sgen.exe"; fi
  97 if [[ ${CI_TAGS} == *'win-amd64'* && ${CI_TAGS} != *'sdks-android'* ]]; then PLATFORM=x64; EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --host=x86_64-w64-mingw32 --disable-boehm --enable-btls"; export MONO_EXECUTABLE="${MONO_REPO_ROOT}/msvc/build/sgen/x64/bin/Release/mono-sgen.exe"; fi
  98 if [[ ${CI_TAGS} == *'freebsd-amd64'* ]]; then export CC="clang"; export CXX="clang++"; EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --disable-dtrace --disable-boehm ac_cv_header_sys_inotify_h=no ac_cv_func_inotify_init=no ac_cv_func_inotify_add_watch=no ac_cv_func_inotify_rm_watch=no"; fi
  99 if [[ ${CI_TAGS} == *'make-install'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-loadedllvm --prefix=${MONO_REPO_ROOT}/tmp/monoprefix"; fi
 100
 101 if   [[ ${CI_TAGS} == *'coop-suspend'* ]];   then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --disable-hybrid-suspend --enable-cooperative-suspend";
 102 elif [[ ${CI_TAGS} == *'hybrid-suspend'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-hybrid-suspend";
 103 elif [[ ${CI_TAGS} == *'preemptive-suspend'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --disable-hybrid-suspend";
 104 fi
 105
 106 if [[ ${CI_TAGS} == *'checked-coop'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-checked-build=gc,thread"; fi
 107 if [[ ${CI_TAGS} == *'checked-all'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-checked-build=all"; fi
 108
 109 if [[ ${CI_TAGS} == *'mcs-compiler'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-csc=mcs"; fi
 110 if [[ ${CI_TAGS} == *'disable-mcs-build'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --disable-mcs-build"; fi
 111
 112 if   [[ ${CI_TAGS} == *'fullaot_llvm'* ]];       then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=fullaot_llvm ";
 113 elif [[ ${CI_TAGS} == *'hybridaot_llvm'* ]];     then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=hybridaot_llvm";
 114 elif [[ ${CI_TAGS} == *'winaot_llvm'* ]];        then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=winaot_llvm";
 115 elif [[ ${CI_TAGS} == *'aot_llvm'* ]];           then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=aot_llvm ";
 116 elif [[ ${CI_TAGS} == *'jit_llvm'* ]];           then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes"; export MONO_ENV_OPTIONS="$MONO_ENV_OPTIONS --llvm";
 117 elif [[ ${CI_TAGS} == *'fullaotinterp_llvm'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=fullaotinterp_llvm";
 118 elif [[ ${CI_TAGS} == *'fullaotinterp'* ]];      then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=fullaotinterp";
 119 elif [[ ${CI_TAGS} == *'winaotinterp'* ]];       then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=winaotinterp";
 120 elif [[ ${CI_TAGS} == *'winaotinterp_llvm'* ]];  then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --enable-llvm=yes --with-runtime-preset=winaotinterp_llvm";
 121 elif [[ ${CI_TAGS} == *'fullaot'* ]];            then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=fullaot";
 122 elif [[ ${CI_TAGS} == *'hybridaot'* ]];          then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=hybridaot";
 123 elif [[ ${CI_TAGS} == *'winaot'* ]];             then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=winaot";
 124 elif [[ ${CI_TAGS} == *'aot'* ]];                then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=aot";
 125 elif [[ ${CI_TAGS} == *'bitcodeinterp'* ]];      then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=bitcodeinterp"; export PATH="$PATH:${MONO_REPO_ROOT}/llvm/usr/bin";
 126 elif [[ ${CI_TAGS} == *'bitcode'* ]];            then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=bitcode"; export PATH="$PATH:${MONO_REPO_ROOT}/llvm/usr/bin";
 127 elif [[ ${CI_TAGS} == *'acceptance-tests'* ]];   then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --prefix=${MONO_REPO_ROOT}/tmp/mono-acceptance-tests --with-sgen-default-concurrent=yes";
 128 elif [[ ${CI_TAGS} == *'all-profiles'* ]];       then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --with-runtime-preset=all";
 129 elif [[ ${CI_TAGS} == *'compile-msbuild-source'* ]]; then EXTRA_CONF_FLAGS="${EXTRA_CONF_FLAGS} --prefix=/tmp/mono-from-source";
 130 fi
 131
 132 if [ -x "/usr/bin/dpkg-architecture" ];
 133         then
 134         EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS --host=`/usr/bin/dpkg-architecture -qDEB_HOST_GNU_TYPE`"
 135         #force build arch = dpkg arch, sometimes misdetected
 136         mkdir -p ~/.config/.mono/
 137         wget -O- https://download.mono-project.com/test/new-certs.tgz | tar zx -C ~/.config/.mono/
 138 fi
 139
 140 if [[ ${CI_TAGS} == *'cxx'* ]]; then
 141         EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS -enable-cxx"
 142         MSBUILD_CXX="/p:MONO_COMPILE_AS_CPP=true"
 143 fi
 144
 145 if [[ ${CI_TAGS} == *'microbench'* ]]; then
 146         EXTRA_CONF_FLAGS="$EXTRA_CONF_FLAGS --with-profile4_x=yes"
 147 fi
 148
 149 if [[ ${CI_TAGS} == *'win-'* ]];
 150 then
 151         mkdir -p ~/.config/.mono/
 152         wget -qO- https://download.mono-project.com/test/new-certs.tgz| tar zx -C ~/.config/.mono/
 153 fi
 154
 155 if [[ ${CI_TAGS} == *'ccache'* ]];
 156 then
 157         # CCACHE_DIR should be set to a directory outside the chroot which holds the cache
 158         CCACHE=$(which ccache)
 159         if [ "$CCACHE" ]; then
 160                 if [[ ${CI_TAGS} == *'osx-'* ]]; then
 161                         export CC="ccache clang"
 162                         export CXX="ccache clang++"
 163                 else
 164                         export CC="ccache gcc"
 165                         export CXX="ccache g++"
 166                 fi
 167         fi
 168 fi
 169
 170 if [[ ${CI_TAGS} == *'sdks-llvm'* ]]; then
 171         ${TESTCMD} --label=archive --timeout=120m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-llvm-llvm{,win}64 NINJA=
 172         exit 0
 173 fi
 174
 175 if [[ ${CI_TAGS} == *'sdks-ios'* ]];
 176    then
 177         # configuration on our bots
 178         if [[ ${CI_TAGS} == *'xcode113'* ]]; then
 179             export XCODE_DIR=/Applications/Xcode113.app/Contents/Developer
 180             export MACOS_VERSION=10.15
 181             export IOS_VERSION=13.2
 182             export TVOS_VERSION=13.2
 183             export WATCHOS_VERSION=6.1
 184             export WATCHOS64_32_VERSION=6.1
 185         elif [[ ${CI_TAGS} == *'xcode124'* ]]; then
 186             export XCODE_DIR=/Applications/Xcode124.app/Contents/Developer
 187             export MACOS_VERSION=11.1
 188             export IOS_VERSION=14.4
 189             export TVOS_VERSION=14.3
 190             export WATCHOS_VERSION=7.2
 191             export WATCHOS64_32_VERSION=7.2
 192         else
 193             export XCODE_DIR=/Applications/Xcode101.app/Contents/Developer
 194             export MACOS_VERSION=10.14
 195             export IOS_VERSION=12.1
 196             export TVOS_VERSION=12.1
 197             export WATCHOS_VERSION=5.1
 198             export WATCHOS64_32_VERSION=5.1
 199         fi
 200
 201         # retrieve selected Xcode version
 202         /usr/libexec/PlistBuddy -c 'Print :ProductBuildVersion' ${XCODE_DIR}/../version.plist > xcode_version.txt
 203
 204         # make sure we embed the correct path into the PDBs
 205         export MONOTOUCH_MCS_FLAGS=-pathmap:${MONO_REPO_ROOT}/=/Library/Frameworks/Xamarin.iOS.framework/Versions/Current/src/Xamarin.iOS/
 206
 207         echo "ENABLE_IOS=1" > sdks/Make.config
 208         if [[ ${CI_TAGS} == *'cxx'* ]]; then
 209             echo "ENABLE_CXX=1" >> sdks/Make.config
 210         fi
 211         if [[ ${CI_TAGS} == *'debug'* ]]; then
 212             echo "CONFIGURATION=debug" >> sdks/Make.config
 213         fi
 214
 215         ${TESTCMD} --label=configure --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds configure-ios NINJA=
 216         ${TESTCMD} --label=build     --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds build-ios     NINJA=
 217         ${TESTCMD} --label=archive   --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-ios   NINJA=
 218
 219         if [[ ${CI_TAGS} != *'no-tests'* ]]; then
 220             # Simulator runs
 221             export sim_test_suites="Mono.Runtime.Tests corlib System.Core System.Data System.Numerics System.Runtime.Serialization System.Transactions System.IO.Compression System.IO.Compression.FileSystem System.Json System.ComponentModel.DataAnnotations System.Security System.Xml System.Xml.Linq System.ServiceModel.Web Mono.Data.Tds Mono.Security"
 222
 223             ${TESTCMD} --label=build-ios-sim --timeout=20m $gnumake -C sdks/ios build-ios-sim-all
 224             for suite in ${sim_test_suites}; do ${TESTCMD} --label=run-ios-sim-${suite} --timeout=10m $gnumake -C sdks/ios run-ios-sim-${suite}; done
 225
 226             # Device runs
 227             if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
 228                 export device_test_suites="Mono.Runtime.Tests System.Core"
 229
 230                 ${TESTCMD} --label=build-ios-dev --timeout=60m $gnumake -C sdks/ios build-ios-dev-all
 231                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-${suite} --timeout=10m $gnumake -C sdks/ios run-ios-dev-${suite}; done
 232
 233                 ${TESTCMD} --label=build-ios-dev-llvm --timeout=60m $gnumake -C sdks/ios build-ios-dev-llvm-all
 234                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-llvm-${suite} --timeout=10m $gnumake -C sdks/ios run-ios-dev-${suite}; done
 235
 236                 ${TESTCMD} --label=build-ios-dev-interp-only --timeout=60m $gnumake -C sdks/ios build-ios-dev-interp-only-all
 237                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-only-${suite} --timeout=10m $gnumake -C sdks/ios run-ios-dev-${suite}; done
 238
 239                 ${TESTCMD} --label=build-ios-dev-interp-mixed --timeout=60m $gnumake -C sdks/ios build-ios-dev-interp-mixed-all
 240                 for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-mixed-${suite} --timeout=10m $gnumake -C sdks/ios run-ios-dev-${suite}; done
 241             fi
 242         fi
 243         exit 0
 244 fi
 245
 246
 247 if [[ ${CI_TAGS} == *'sdks-maccat'* ]];
 248 then
 249     # configuration on our bots
 250     if [[ ${CI_TAGS} == *'xcode113'* ]]; then
 251         export XCODE_DIR=/Applications/Xcode113.app/Contents/Developer
 252         export MACOS_VERSION=10.15
 253     elif [[ ${CI_TAGS} == *'xcode124'* ]]; then
 254         export XCODE_DIR=/Applications/Xcode124.app/Contents/Developer
 255         export MACOS_VERSION=11.1
 256     else
 257         export XCODE_DIR=/Applications/Xcode101.app/Contents/Developer
 258         export MACOS_VERSION=10.14
 259     fi
 260
 261     # retrieve selected Xcode version
 262     /usr/libexec/PlistBuddy -c 'Print :ProductBuildVersion' ${XCODE_DIR}/../version.plist > xcode_version.txt
 263
 264     # make sure we embed the correct path into the PDBs
 265     export MONOTOUCH_MCS_FLAGS=-pathmap:${MONO_REPO_ROOT}/=/Library/Frameworks/Xamarin.iOS.framework/Versions/Current/src/Xamarin.iOS/
 266
 267     echo "ENABLE_MACCAT=1" > sdks/Make.config
 268     if [[ ${CI_TAGS} == *'cxx'* ]]; then
 269         echo "ENABLE_CXX=1" >> sdks/Make.config
 270     fi
 271     if [[ ${CI_TAGS} == *'debug'* ]]; then
 272         echo "CONFIGURATION=debug" >> sdks/Make.config
 273     fi
 274
 275     ${TESTCMD} --label=configure --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds configure-maccat NINJA=
 276     ${TESTCMD} --label=build     --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds build-maccat     NINJA=
 277     ${TESTCMD} --label=archive   --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-maccat   NINJA=
 278
 279     exit 0
 280 fi
 281
 282 if [[ ${CI_TAGS} == *'sdks-mac'* ]];
 283 then
 284     # configuration on our bots
 285     if [[ ${CI_TAGS} == *'xcode113'* ]]; then
 286         export XCODE_DIR=/Applications/Xcode113.app/Contents/Developer
 287         export MACOS_VERSION=10.15
 288     elif [[ ${CI_TAGS} == *'xcode124'* ]]; then
 289         export XCODE_DIR=/Applications/Xcode124.app/Contents/Developer
 290         export MACOS_VERSION=11.1
 291     else
 292         export XCODE_DIR=/Applications/Xcode101.app/Contents/Developer
 293         export MACOS_VERSION=10.14
 294     fi
 295
 296     # retrieve selected Xcode version
 297     /usr/libexec/PlistBuddy -c 'Print :ProductBuildVersion' ${XCODE_DIR}/../version.plist > xcode_version.txt
 298
 299     # make sure we embed the correct path into the PDBs
 300     export XAMMAC_MCS_FLAGS=-pathmap:${MONO_REPO_ROOT}/=/Library/Frameworks/Xamarin.Mac.framework/Versions/Current/src/Xamarin.Mac/
 301
 302     echo "ENABLE_MAC=1" > sdks/Make.config
 303     if [[ ${CI_TAGS} == *'cxx'* ]]; then
 304         echo "ENABLE_CXX=1" >> sdks/Make.config
 305     fi
 306     if [[ ${CI_TAGS} == *'debug'* ]]; then
 307         echo "CONFIGURATION=debug" >> sdks/Make.config
 308     fi
 309
 310     ${TESTCMD} --label=configure --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds configure-mac NINJA=
 311     ${TESTCMD} --label=build     --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds build-mac     NINJA=
 312     ${TESTCMD} --label=archive   --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-mac   NINJA=
 313
 314     exit 0
 315 fi
 316
 317 if [[ ${CI_TAGS} == *'sdks-android'* ]];
 318    then
 319         echo "ENABLE_ANDROID=1" > sdks/Make.config
 320         echo "DISABLE_CCACHE=1" >> sdks/Make.config
 321         if [[ ${CI_TAGS} == *'cxx'* ]]; then
 322             echo "ENABLE_CXX=1" >> sdks/Make.config
 323         fi
 324         if [[ ${CI_TAGS} == *'debug'* ]]; then
 325             echo "CONFIGURATION=debug" >> sdks/Make.config
 326         fi
 327
 328         # TODO: provision-android on Windows.
 329         if [[ ${CI_TAGS} != *'win-'* ]]; then
 330         # For some very strange reasons, `make -C sdks/android accept-android-license` get stuck when invoked through ${TESTCMD}
 331             # but doesn't get stuck when called via the shell, so let's just call it here now.
 332             ${TESTCMD} --label=provision-android --timeout=120m --fatal $gnumake -j ${CI_CPU_COUNT} -C sdks/builds provision-android && $gnumake -C sdks/android accept-android-license
 333         fi
 334         ${TESTCMD} --label=provision-mxe --timeout=240m --fatal $gnumake -j ${CI_CPU_COUNT} -C sdks/builds provision-mxe
 335
 336         ${TESTCMD} --label=configure --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds configure-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
 337         ${TESTCMD} --label=build     --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds build-android     NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
 338         ${TESTCMD} --label=archive   --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android   NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
 339
 340         if [[ ${CI_TAGS} != *'no-tests'* ]]; then
 341             ${TESTCMD} --label=mini --timeout=20m $gnumake -C sdks/android check-Mono.Runtime.Tests
 342             ${TESTCMD} --label=corlib --timeout=20m $gnumake -C sdks/android check-corlib
 343             ${TESTCMD} --label=System --timeout=20m $gnumake -C sdks/android check-System
 344             ${TESTCMD} --label=System.Core --timeout=20m $gnumake -C sdks/android check-System.Core
 345             ${TESTCMD} --label=System.Data --timeout=20m $gnumake -C sdks/android check-System.Data
 346             ${TESTCMD} --label=System.IO.Compression.FileSystem --timeout=20m $gnumake -C sdks/android check-System.IO.Compression.FileSystem
 347             ${TESTCMD} --label=System.IO.Compression --timeout=20m $gnumake -C sdks/android check-System.IO.Compression
 348             ${TESTCMD} --label=System.Json --timeout=20m $gnumake -C sdks/android check-System.Json
 349             ${TESTCMD} --label=System.Net.Http --timeout=20m $gnumake -C sdks/android check-System.Net.Http
 350             ${TESTCMD} --label=System.Numerics --timeout=20m $gnumake -C sdks/android check-System.Numerics
 351             ${TESTCMD} --label=System.Runtime.Serialization --timeout=20m $gnumake -C sdks/android check-System.Runtime.Serialization
 352             ${TESTCMD} --label=System.ServiceModel.Web --timeout=20m $gnumake -C sdks/android check-System.ServiceModel.Web
 353             ${TESTCMD} --label=System.Transactions --timeout=20m $gnumake -C sdks/android check-System.Transactions
 354             ${TESTCMD} --label=System.Xml --timeout=20m $gnumake -C sdks/android check-System.Xml
 355             ${TESTCMD} --label=System.Xml.Linq --timeout=20m $gnumake -C sdks/android check-System.Xml.Linq
 356             ${TESTCMD} --label=Mono.CSharp --timeout=20m $gnumake -C sdks/android check-Mono.CSharp
 357             # ${TESTCMD} --label=Mono.Data.Sqlite --timeout=20m $gnumake -C sdks/android check-Mono.Data.Sqlite
 358             ${TESTCMD} --label=Mono.Data.Tds --timeout=20m $gnumake -C sdks/android check-Mono.Data.Tds
 359             ${TESTCMD} --label=Mono.Security --timeout=20m $gnumake -C sdks/android check-Mono.Security
 360             ${TESTCMD} --label=Mono.Debugger.Soft --timeout=20m $gnumake -C sdks/android check-Mono.Debugger.Soft
 361         fi
 362         exit 0
 363 fi
 364
 365 if [[ ${CI_TAGS} == *'webassembly'* ]] || [[ ${CI_TAGS} == *'wasm'* ]];
 366    then
 367         echo "ENABLE_WASM=1" > sdks/Make.config
 368
 369         if [[ ${CI_TAGS} != *'osx-amd64'* ]]; then
 370             echo "ENABLE_WINDOWS=1" >> sdks/Make.config
 371             echo "ENABLE_WASM_DYNAMIC_RUNTIME=1" >> sdks/Make.config
 372         fi
 373
 374         if [[ ${CI_TAGS} == *'cxx'* ]]; then
 375             echo "ENABLE_CXX=1" >> sdks/Make.config
 376         fi
 377         if [[ ${CI_TAGS} == *'debug'* ]]; then
 378             echo "CONFIGURATION=debug" >> sdks/Make.config
 379         fi
 380
 381         echo "ENABLE_WASM_THREADS=1" >> sdks/Make.config
 382         #echo "ENABLE_WASM_NETCORE=1" >> sdks/Make.config
 383
 384         export aot_test_suites="System.Core"
 385         export mixed_test_suites="System.Core"
 386         export xunit_test_suites="System.Core corlib System Microsoft.CSharp System.Data System.IO.Compression System.Net.Http.UnitTests System.Numerics System.Runtime.Serialization System.Security System.Xml System.Xml.Linq"
 387
 388         ${TESTCMD} --label=provision --timeout=20m --fatal $gnumake --output-sync=recurse --trace -C sdks/builds provision-wasm
 389
 390         ${TESTCMD} --label=configure --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds configure-wasm NINJA=
 391         ${TESTCMD} --label=build     --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds build-wasm     NINJA=
 392         ${TESTCMD} --label=archive   --timeout=180m --fatal $gnumake -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-wasm   NINJA=
 393
 394         if [[ ${CI_TAGS} != *'no-tests'* ]]; then
 395             ${TESTCMD} --label=wasm-build --timeout=20m --fatal $gnumake -j ${CI_CPU_COUNT} -C sdks/wasm build
 396             ${TESTCMD} --label=mini --timeout=20m $gnumake -C sdks/wasm run-all-mini
 397             ${TESTCMD} --label=v8-corlib --timeout=20m $gnu$gnumake -C sdks/wasm run-v8-corlib
 398             ${TESTCMD} --label=mini-system --timeout=60m $gnu$gnumake -C sdks/wasm run-all-System
 399             ${TESTCMD} --label=system-core --timeout=60m $gnumake -C sdks/wasm run-all-System.Core
 400             for suite in ${xunit_test_suites}; do ${TESTCMD} --label=xunit-${suite} --timeout=30m $gnumake -C sdks/wasm run-${suite}-xunit; done
 401             # disable for now until https://github.com/mono/mono/pull/13622 goes in
 402             ${TESTCMD} --label=debugger --timeout=20m $gnumake -C sdks/wasm run-debugger-tests
 403             ${TESTCMD} --label=browser --timeout=20m $gnumake -C sdks/wasm run-browser-tests
 404             #${TESTCMD} --label=browser-threads --timeout=20m $gnumake -C sdks/wasm run-browser-threads-tests
 405             ${TESTCMD} --label=browser-dynamic --timeout=20m $gnumake -C sdks/wasm run-browser-dynamic-tests
 406             if [[ ${CI_TAGS} == *'osx-amd64'* ]]; then
 407                 ${TESTCMD} --label=browser-safari --timeout=20m $gnumake -C sdks/wasm run-browser-safari-tests
 408             fi
 409             ${TESTCMD} --label=aot-mini --timeout=20m $gnumake -j ${CI_CPU_COUNT} -C sdks/wasm run-aot-mini
 410             ${TESTCMD} --label=build-aot-all --timeout=20m $gnumake -j ${CI_CPU_COUNT} -C sdks/wasm build-aot-all
 411             for suite in ${aot_test_suites}; do ${TESTCMD} --label=run-aot-${suite} --timeout=10m $gnumake -C sdks/wasm run-aot-${suite}; done
 412             for suite in ${mixed_test_suites}; do ${TESTCMD} --label=run-aot-mixed-${suite} --timeout=10m $gnumake -C sdks/wasm run-aot-mixed-${suite}; done
 413             # Requires a net 3.0 sdk
 414             #${TESTCMD} --label=netcore --timeout=20m $gnumake -j ${CI_CPU_COUNT} -C sdks/wasm run-hello-netcore
 415             #${TESTCMD} --label=check-aot --timeout=20m $gnumake -C sdks/wasm check-aot
 416             ${TESTCMD} --label=package --timeout=20m $gnumake -C sdks/wasm package
 417         fi
 418         exit 0
 419 fi
 420
 421
 422 if [[ ${CI_TAGS} != *'mac-sdk'* ]]; # Mac SDK builds Mono itself
 423         then
 424         echo ./autogen.sh CFLAGS="$CFLAGS $EXTRA_CFLAGS" CXXFLAGS="$CXXFLAGS $EXTRA_CXXFLAGS" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" $EXTRA_CONF_FLAGS
 425         ${TESTCMD} --label=configure --timeout=60m --fatal ./autogen.sh CFLAGS="$CFLAGS $EXTRA_CFLAGS" CXXFLAGS="$CXXFLAGS $EXTRA_CXXFLAGS" LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS" $EXTRA_CONF_FLAGS
 426 fi
 427 if [[ ${CI_TAGS} == *'win-i386'* ]];
 428     then
 429         # only build boehm on w32 (only windows platform supporting boehm).
 430     ${TESTCMD} --label=make-msvc --timeout=60m --fatal ./msvc/run-msbuild.sh "build" "${PLATFORM}" "release" "boehm" "/p:PlatformToolset=v140 ${MSBUILD_CXX}"
 431 fi
 432 if [[ ${CI_TAGS} == *'win-'* ]];
 433     then
 434     ${TESTCMD} --label=make-msvc-sgen --timeout=60m --fatal ./msvc/run-msbuild.sh "build" "${PLATFORM}" "release" "sgen" "/p:PlatformToolset=v140 ${MSBUILD_CXX}"
 435 fi
 436
 437 if [[ ${CI_TAGS} == *'win-amd64'* ]];
 438     then
 439     # The AOT compiler on Windows requires Visual Studio's clang.exe and link.exe.
 440     # Depending on codegen (JIT/LLVM) it might also need platform specific libraries.
 441     # Use a wrapper script that will make sure to setup full VS MSVC environment if
 442     # needed when running mono-sgen.exe as AOT compiler.
 443     export MONO_EXECUTABLE_WRAPPER="${MONO_REPO_ROOT}/msvc/build/sgen/x64/bin/Release/mono-sgen-msvc.sh"
 444 fi
 445
 446 if [[ ${CI_TAGS} == *'monolite'* ]]; then make get-monolite-latest; fi
 447
 448 make_parallelism="-j ${CI_CPU_COUNT}"
 449 if [[ ${CI_TAGS} == *'linux-ppc64el'* ]]; then make_parallelism=-j1; fi
 450
 451 make_continue=
 452 if [[ ${CI_TAGS} == *'checked-all'* ]]; then make_continue=-k; fi
 453
 454 if [[ ${CI_TAGS} != *'mac-sdk'* ]]; # Mac SDK builds Mono itself
 455     then
 456     build_error=0
 457     ${TESTCMD} --label=make --timeout=${make_timeout} --fatal make ${make_parallelism} ${make_continue} -w V=1 || build_error=1
 458     helix_send_build_done_event "build/source/$MONO_HELIX_TYPE/" $build_error
 459
 460     if [[ ${build_error} != 0 ]]; then
 461         echo "ERROR: The Mono build failed."
 462         ${MONO_REPO_ROOT}/scripts/ci/run-upload-sentry.sh
 463         exit ${build_error}
 464     fi
 465 fi
 466
 467 if [[ ${CI_TAGS} == *'checked-coop'* ]]; then export MONO_CHECK_MODE=gc,thread; fi
 468 if [[ ${CI_TAGS} == *'checked-all'* ]]; then export MONO_CHECK_MODE=all; fi
 469
 470 if [[ ${CI_TAGS} == *'hardened-runtime'* ]]; then codesign -s - -fv -o runtime --entitlements ${MONO_REPO_ROOT}/mono/mini/mac-entitlements.plist ${MONO_REPO_ROOT}/mono/mini/mono-sgen; fi
 471
 472 export MONO_ENV_OPTIONS="$MONO_ENV_OPTIONS $MONO_TEST_ENV_OPTIONS"
 473
 474 if   [[ ${CI_TAGS} == *'acceptance-tests'* ]];         then ${MONO_REPO_ROOT}/scripts/ci/run-test-acceptance-tests.sh;
 475 elif [[ ${CI_TAGS} == *'microbench'* ]];               then ${MONO_REPO_ROOT}/scripts/ci/run-test-microbench.sh;
 476 elif [[ ${CI_TAGS} == *'profiler-stress-tests'* ]];    then ${MONO_REPO_ROOT}/scripts/ci/run-test-profiler-stress-tests.sh;
 477 elif [[ ${CI_TAGS} == *'stress-tests'* ]];             then ${MONO_REPO_ROOT}/scripts/ci/run-test-stress-tests.sh;
 478 elif [[ ${CI_TAGS} == *'interpreter'* ]];              then ${MONO_REPO_ROOT}/scripts/ci/run-test-interpreter.sh;
 479 elif [[ ${CI_TAGS} == *'mcs-compiler'* ]];             then ${MONO_REPO_ROOT}/scripts/ci/run-test-mcs.sh;
 480 elif [[ ${CI_TAGS} == *'mac-sdk'* ]];                  then ${MONO_REPO_ROOT}/scripts/ci/run-test-mac-sdk.sh;
 481 elif [[ ${CI_TAGS} == *'helix-tests'* ]];              then ${MONO_REPO_ROOT}/scripts/ci/run-test-helix.sh;
 482 elif [[ ${CI_TAGS} == *'compile-msbuild-source'* ]];   then ${MONO_REPO_ROOT}/scripts/ci/run-test-msbuild.sh;
 483 elif [[ ${CI_TAGS} == *'make-install'* ]];             then ${MONO_REPO_ROOT}/scripts/ci/run-test-make-install.sh;
 484 elif [[ ${CI_TAGS} == *'compiler-server-tests'* ]];          then ${MONO_REPO_ROOT}/scripts/ci/run-test-compiler-server.sh;
 485 elif [[ ${CI_TAGS} == *'no-tests'* ]];                 then echo "Skipping tests.";
 486 else make check-ci;
 487 fi
 488
 489 if [[ $CI_TAGS == *'apidiff'* ]]; then
 490     if ${TESTCMD} --label=apidiff --timeout=15m --fatal make -w -C mcs -j ${CI_CPU_COUNT} mono-api-diff
 491     then report_github_status "success" "API Diff" "No public API changes found." || true
 492     else report_github_status "error" "API Diff" "The public API changed." "$BUILD_URL/Public_20API_20Diff/" || true
 493     fi
 494 fi
 495 if [[ $CI_TAGS == *'csprojdiff'* ]]; then
 496     make update-solution-files
 497     if ${TESTCMD} --label=csprojdiff --timeout=5m --fatal make -w -C mcs mono-csproj-diff
 498     then report_github_status "success" "Project Files Diff" "No csproj file changes found." || true
 499     else report_github_status "error" "Project Files Diff" "The csproj files changed." "$BUILD_URL/Project_20Files_20Diff/" || true
 500     fi
 501 fi