runtime/mono-test.sh

   1 #! /bin/sh -e
   2
   3 test_suite="$1"
   4 test_argument_1="$2"
   5 test_argument_2="$3"
   6 xunit_results_path="$(pwd)/testResults.xml"
   7
   8 cd "$(dirname "$0")" || exit 1
   9
  10 r=$(pwd)
  11 MONO_CFG_DIR="$r/_tmpinst/etc"
  12 PATH="$r/_tmpinst/bin:$PATH"
  13 MONO_EXECUTABLE=${MONO_EXECUTABLE:-"$r/mono-sgen"}
  14 MONO_PATH="$r/net_4_x"
  15 export MONO_CFG_DIR MONO_PATH MONO_EXECUTABLE PATH
  16
  17 sed "s,\$test_root_dir,$r,g" "$r/_tmpinst/etc/mono/config.tmpl" > "$r/_tmpinst/etc/mono/config"
  18 sed "s,\$test_root_dir,$r,g" "$r/tests/runtime/tests-config.tmpl" > "$r/tests/runtime/tests-config"
  19
  20 chmod +x "${MONO_EXECUTABLE}"
  21 echo "---------------------------------------------------------------------------------------"
  22 "${MONO_EXECUTABLE}" --version
  23 echo "---------------------------------------------------------------------------------------"
  24
  25 case "$(uname)" in
  26     "Linux")
  27         mkdir -p ~/.config/.mono/
  28         wget -qO- https://download.mono-project.com/test/new-certs.tgz| tar zx -C ~/.config/.mono/
  29         ;;
  30 esac
  31
  32 if [ "$test_suite" = "--xunit" ]; then
  33     cd net_4_x || exit 1
  34     export MONO_PATH="$r/net_4_x/tests:$MONO_PATH"
  35     export REMOTE_EXECUTOR="$r/net_4_x/tests/RemoteExecutorConsoleApp.exe"
  36     case "$test_argument_1" in
  37         *"Mono.Profiler.Log"*)
  38             # necessary for the runtime to find libmono-profiler-log
  39             export LD_LIBRARY_PATH="$r:$LD_LIBRARY_PATH"
  40             export DYLD_LIBRARY_PATH="$r:$LD_LIBRARY_PATH"
  41             ;;
  42         *"System.Net.Http"*)
  43             export MONO_URI_DOTNETRELATIVEORABSOLUTE=true
  44             ;;
  45     esac
  46     case "$(uname)" in
  47         "Darwin")
  48             ADDITIONAL_TRAITS="-notrait category=nonosxtests"
  49             ;;
  50         "Linux")
  51             ADDITIONAL_TRAITS="-notrait category=nonlinuxtests"
  52             ;;
  53     esac
  54
  55     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug "$r/net_4_x/xunit.console.exe" "$r/$test_argument_1" -noappdomain -noshadow -parallel none -xml "${xunit_results_path}" -notrait category=failing -notrait category=nonmonotests -notrait Benchmark=true -notrait category=outerloop $ADDITIONAL_TRAITS
  56     exit $?
  57 fi
  58
  59 if [ "$test_suite" = "--nunit" ]; then
  60     cd net_4_x || exit 1
  61     export MONO_PATH="$r/net_4_x/tests:$MONO_PATH"
  62     export MONO_SYSTEMWEB_CACHEDEPENDENCY_SHARED_FSW=1
  63     case "$test_argument_1" in
  64         *"Microsoft.Build"*)
  65             export TESTING_MONO=a
  66             export MSBuildExtensionsPath="$r/net_4_x/tests/xbuild/extensions"
  67             export XBUILD_FRAMEWORK_FOLDERS_PATH="$r/net_4_x/tests/xbuild/frameworks"
  68             case "$test_argument_1" in
  69                 "xbuild_12"*) export MONO_PATH="$r/xbuild_12:$r/xbuild_12/tests:$MONO_PATH" ;;
  70                 "xbuild_14"*) export MONO_PATH="$r/xbuild_14:$r/xbuild_14/tests:$MONO_PATH" ;;
  71             esac
  72             ;;
  73         *"Mono.Messaging.RabbitMQ"*)
  74             export MONO_MESSAGING_PROVIDER=Mono.Messaging.RabbitMQ.RabbitMQMessagingProvider,Mono.Messaging.RabbitMQ
  75             ;;
  76         *"System.Windows.Forms"*)
  77             sudo apt install -y xvfb xauth
  78             XVFBRUN="xvfb-run -a --"
  79             ADDITIONAL_TEST_EXCLUDES="NotWithXvfb" # TODO: find out why this works on Jenkins?
  80             ;;
  81     esac
  82     case "$test_argument_2" in
  83         "--flaky-test-retries="*)
  84             export MONO_FLAKY_TEST_RETRIES=$(echo "$test_argument_2" | cut -d "=" -f2)
  85             ;;
  86     esac
  87     cp -f "$r/$test_argument_1.nunitlite.config" "$r/net_4_x/nunit-lite-console.exe.config"
  88     MONO_REGISTRY_PATH="$HOME/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $XVFBRUN "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug "$r/net_4_x/nunit-lite-console.exe" "$r/$test_argument_1" -exclude=NotWorking,CAS,$ADDITIONAL_TEST_EXCLUDES -labels -format:xunit -result:"${xunit_results_path}"
  89     exit $?
  90 fi
  91
  92 if [ "$test_suite" = "--verify" ]; then
  93     verifiable_files=$(find net_4_x -maxdepth 1 -name "*.dll" -or -name "*.exe" | grep -v System.Runtime.CompilerServices.Unsafe.dll | grep -v Xunit.NetCore.Extensions.dll)
  94     ok=true
  95     for asm in $verifiable_files; do
  96         echo "$asm"
  97         if [ ! -f "$asm" ]; then continue; fi
  98         if "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --compile-all --verify-all --security=verifiable "$asm"; then
  99             echo "$asm verified OK"
 100         else
 101             echo "$asm verification failed"
 102             ok=false
 103         fi
 104     done;
 105     if [ "$ok" = "true" ]; then
 106         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"verify\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" name=\"Test collection for verify\" time=\"0\"><test name=\"verify.all\" type=\"verify\" method=\"all\" time=\"0\" result=\"Pass\"></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 107         exit 0
 108     else
 109         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"verify\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" name=\"Test collection for verify\" time=\"0\"><test name=\"verify.all\" type=\"verify\" method=\"all\" time=\"0\" result=\"Fail\"><failure exception-type=\"VerifyException\"><message><![CDATA[Verifying framework assemblies failed. Check the log for more details.]]></message></failure></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 110         exit 1
 111     fi
 112 fi
 113
 114 if [ "$test_suite" = "--mcs" ]; then
 115     cd tests/mcs || exit 1
 116     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --verify-all compiler-tester.exe -mode:pos -files:v4 -compiler:"$r/net_4_x/mcs.exe" -reference-dir:"$r/net_4_x" -issues:known-issues-net_4_x -log:net_4_x.log -il:ver-il-net_4_x.xml -resultXml:"${xunit_results_path}" -compiler-options:"-d:NET_4_0;NET_4_5 -debug"
 117     exit $?
 118 fi
 119
 120 if [ "$test_suite" = "--mcs-errors" ]; then
 121     cd tests/mcs-errors || exit 1
 122     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" compiler-tester.exe -mode:neg -files:v4 -compiler:"$r/net_4_x/mcs.exe" -reference-dir:"$r/net_4_x" -issues:known-issues-net_4_x -log:net_4_x.log -resultXml:"${xunit_results_path}" -compiler-options:"-v --break-on-ice -d:NET_4_0;NET_4_5"
 123     exit $?
 124 fi
 125
 126 if [ "$test_suite" = "--aot-test" ]; then
 127     failed=0
 128     passed=0
 129     failed_tests=""
 130     profile="$r/net_4_x"
 131     tmpfile=$(mktemp -t mono_aot_outputXXXXXX) || exit 1
 132     rm -f "test-aot-*.stdout" "test-aot-*.stderr" "${xunit_results_path}.cases.xml"
 133     for assembly in "$profile"/*.dll; do
 134         asm_name=$(basename "$assembly")
 135         echo "... $asm_name"
 136         for conf in "|regular" "--gc=boehm|boehm"; do
 137             name=$(echo $conf | cut -d\| -f 2)
 138             params=$(echo $conf | cut -d\| -f 1)
 139             test_name="${asm_name}|${name}"
 140             echo "  $test_name"
 141             if "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" $params --aot=outfile="$tmpfile" "$assembly" > "test-aot-${name}-${asm_name}.stdout" 2> "test-aot-${name}-${asm_name}.stderr"
 142             then
 143                 passed=$((passed + 1))
 144                 resultstring="Pass"
 145             else \
 146                 failed=$((failed + 1))
 147                 failed_tests="${failed_tests} $test_name"
 148                 resultstring="Fail"
 149             fi
 150             echo "<test name=\"aot-test.$name.$asm_name\" type=\"aot-test.$name\" method=\"$asm_name\" time=\"0\" result=\"$resultstring\">" >> "${xunit_results_path}.cases.xml"
 151             if [ "$resultstring" = "Fail" ]; then
 152                 echo "<failure exception-type=\"AotTestException\"><message><![CDATA[
 153                     STDOUT:
 154                     $(cat "test-aot-${name}-${asm_name}.stdout")
 155                     STDERR:
 156                     $(cat "test-aot-${name}-${asm_name}.stderr")]]></message><stack-trace></stack-trace></failure>" >> "${xunit_results_path}.cases.xml"; fi
 157             echo "</test>" >> "${xunit_results_path}.cases.xml"
 158         done
 159     done
 160     echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>\
 161     <assemblies>\
 162         <assembly name=\"aot-test\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"$((passed + failed))\" passed=\"$passed\" failed=\"$failed\" skipped=\"0\" errors=\"0\" time=\"0\">\
 163             <collection total=\"$((passed + failed))\" passed=\"$passed\" failed=\"$failed\" skipped=\"0\" name=\"Test collection for aot-test\" time=\"0\">\
 164                 $(cat "${xunit_results_path}.cases.xml")
 165             </collection>\
 166         </assembly>\
 167     </assemblies>" > "${xunit_results_path}";
 168     rm "$tmpfile"
 169     echo "${passed} test(s) passed. ${failed} test(s) did not pass."
 170     if [ "${failed}" != 0 ]; then
 171         echo ""
 172         echo "Failed tests:"
 173         echo ""
 174         for i in ${failed_tests}; do
 175             echo "${i}";
 176         done
 177         exit 1
 178     fi
 179     exit 0
 180 fi
 181
 182 if [ "$test_suite" = "--mini" ]; then
 183     cd tests/mini || exit 1
 184
 185     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --regression ./*.exe > regressiontests.out 2>&1
 186     cat regressiontests.out
 187     if grep -q "100% pass" regressiontests.out; then
 188         resultstring="Pass"
 189         failurescount=0
 190         successcount=1
 191     else
 192         resultstring="Fail"
 193         failurescount=1
 194         successcount=0
 195     fi
 196     echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>\
 197         <assemblies>\
 198             <assembly name=\"mini.regression-tests\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"$successcount\" failed=\"$failurescount\" skipped=\"0\" errors=\"0\" time=\"0\">\
 199                 <collection total=\"1\" passed=\"$successcount\" failed=\"$failurescount\" skipped=\"0\" name=\"Test collection for mini.regression-tests\" time=\"0\">\
 200                     <test name=\"mini.regression-tests.all\" type=\"mini.regression-tests\" method=\"all\" time=\"0\" result=\"$resultstring\">" > "${xunit_results_path}"
 201                     if [ "$resultstring" = "Fail" ]; then echo "<failure exception-type=\"MiniRegressionTestsException\"><message><![CDATA[$(cat regressiontests.out)]]></message><stack-trace></stack-trace></failure>" >> "${xunit_results_path}"; fi
 202                 echo "</test>
 203                 </collection>\
 204             </assembly>\
 205         </assemblies>" >> "${xunit_results_path}";
 206     exit $failurescount
 207 fi
 208
 209 if [ "$test_suite" = "--symbolicate" ]; then
 210     cd tests/symbolicate || exit 1
 211
 212     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --aot 2>&1 | grep -q "AOT compilation is not supported" && echo "No AOT support, skipping tests." && exit 0
 213
 214     ok=true
 215     for config in without-aot with-aot with-aot-msym; do
 216         OUT_DIR="$config"
 217         MSYM_DIR="$OUT_DIR/msymdir"
 218         STACKTRACE_FILE="$OUT_DIR/stacktrace.out"
 219         SYMBOLICATE_RAW_FILE="$OUT_DIR/symbolicate_raw.out"
 220         SYMBOLICATE_RESULT_FILE="$OUT_DIR/symbolicate.result"
 221         SYMBOLICATE_EXPECTED_FILE=symbolicate.expected
 222
 223         echo "Checking StackTraceDumper.exe in configuration $config..."
 224         rm -rf "$OUT_DIR"
 225         mkdir -p "$OUT_DIR"
 226         mkdir -p "$MSYM_DIR"
 227
 228         cp StackTraceDumper.exe "$OUT_DIR"
 229         cp StackTraceDumper.pdb "$OUT_DIR"
 230
 231         # store symbols
 232         "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" store-symbols "$MSYM_DIR" "$OUT_DIR"
 233         "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" store-symbols "$MSYM_DIR" "$r/net_4_x"
 234
 235         if [ "$config" = "with-aot" ]; then "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline --aot "$OUT_DIR/StackTraceDumper.exe"; fi
 236         if [ "$config" = "with-aot-msym" ]; then "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline --aot=msym-dir="$MSYM_DIR" "$OUT_DIR/StackTraceDumper.exe"; fi
 237
 238         # check diff
 239         "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline StackTraceDumper.exe > "$STACKTRACE_FILE"
 240         "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" "$MSYM_DIR" "$STACKTRACE_FILE" > "$SYMBOLICATE_RAW_FILE"
 241         tr "\\\\" "/" < "$SYMBOLICATE_RAW_FILE" | sed "s,) .* in .*/mcs/,) in mcs/," | sed "s,) .* in .*/external/,) in external/," | sed '/\[MVID\]/d' | sed '/\[AOTID\]/d' > "$SYMBOLICATE_RESULT_FILE"
 242
 243         DIFF=$(diff -up "$SYMBOLICATE_EXPECTED_FILE" "$SYMBOLICATE_RESULT_FILE")
 244         if [ ! -z "$DIFF" ]; then
 245             echo "ERROR: Symbolicate tests failed."
 246             echo "If $SYMBOLICATE_RESULT_FILE is correct copy it to $SYMBOLICATE_EXPECTED_FILE."
 247             echo "Otherwise runtime sequence points need to be fixed."
 248             echo ""
 249             echo "$DIFF"
 250             ok=false
 251         else
 252             echo "Success."
 253         fi
 254     done
 255
 256     if [ "$ok" = "true" ]; then
 257         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"symbolicate\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" name=\"Test collection for symbolicate\" time=\"0\"><test name=\"symbolicate.all\" type=\"symbolicate\" method=\"all\" time=\"0\" result=\"Pass\"></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 258         exit 0
 259     else
 260         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"symbolicate\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" name=\"Test collection for symbolicate\" time=\"0\"><test name=\"symbolicate.all\" type=\"symbolicate\" method=\"all\" time=\"0\" result=\"Fail\"><failure exception-type=\"SymbolicateException\"><message><![CDATA[Symbolicate tests failed. Check the log for more details.]]></message></failure></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 261         exit 1
 262     fi
 263
 264 fi
 265
 266 if [ "$test_suite" = "--csi" ]; then
 267     cd tests/csi || error 1
 268     echo "Console.WriteLine (\"hello world: \" + DateTime.Now)" > csi-test.csx
 269
 270     ok=true
 271     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" csi.exe csi-test.csx > csi-test-output.txt || ok=false
 272     cat csi-test-output.txt && grep -q "hello world" csi-test-output.txt || ok=false
 273
 274     if [ "$ok" = "true" ]; then
 275         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"csi\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" name=\"Test collection for csi\" time=\"0\"><test name=\"csi.all\" type=\"csi\" method=\"all\" time=\"0\" result=\"Pass\"></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 276         exit 0
 277     else
 278         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"csi\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" name=\"Test collection for csi\" time=\"0\"><test name=\"csi.all\" type=\"csi\" method=\"all\" time=\"0\" result=\"Fail\"><failure exception-type=\"CsiException\"><message><![CDATA[csi.exe tests failed. Check the log for more details.]]></message></failure></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 279         exit 1
 280     fi
 281
 282 fi
 283
 284 if [ "$test_suite" = "--profiler" ]; then
 285     cd tests/profiler || exit 1
 286
 287     chmod +x "$r/mprof-report"
 288     perl ptestrunner.pl out-of-tree xunit "${xunit_results_path}"
 289     exit $?
 290 fi
 291
 292 if [ "$test_suite" = "--runtime" ]; then
 293     cd tests/runtime || exit 1
 294
 295     # TODO: only ported runtest-managed for now
 296     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug test-runner.exe --verbose --xunit "${xunit_results_path}" --config tests-config --runtime "${MONO_EXECUTABLE}" --mono-path "$r/net_4_x" -j a --testsuite-name "runtime" --timeout 300 --disabled "$DISABLED_TESTS" --input-file runtime-test-list.txt
 297     exit $?
 298 fi