runtime/mono-test.sh

   1 #! /bin/sh -e
   2
   3 test_suite="$1"
   4 test_argument_1="$2"
   5 test_argument_2="$3"
   6 xunit_results_path="$(pwd)/testResults.xml"
   7
   8 cd "$(dirname "$0")" || exit 1
   9
  10 r=$(pwd)
  11 MONO_CFG_DIR="$r/_tmpinst/etc"
  12 PATH="$r/_tmpinst/bin:$PATH"
  13 MONO_EXECUTABLE=${MONO_EXECUTABLE:-"$r/mono-sgen"}
  14 MONO_PATH="$r/net_4_x"
  15 export MONO_CFG_DIR MONO_PATH MONO_EXECUTABLE PATH
  16
  17 sed "s,\$test_root_dir,$r,g" "$r/_tmpinst/etc/mono/config.tmpl" > "$r/_tmpinst/etc/mono/config"
  18 sed "s,\$test_root_dir,$r,g" "$r/tests/runtime/tests-config.tmpl" > "$r/tests/runtime/tests-config"
  19
  20 chmod +x "${MONO_EXECUTABLE}"
  21 echo "---------------------------------------------------------------------------------------"
  22 "${MONO_EXECUTABLE}" --version
  23 echo "---------------------------------------------------------------------------------------"
  24
  25 if [ "$test_suite" = "--xunit" ]; then
  26     cd net_4_x || exit 1
  27     export MONO_PATH="$r/net_4_x/tests:$MONO_PATH"
  28     export REMOTE_EXECUTOR="$r/net_4_x/tests/RemoteExecutorConsoleApp.exe"
  29     case "$test_argument_1" in
  30         *"Mono.Profiler.Log"*)
  31             # necessary for the runtime to find libmono-profiler-log
  32             export LD_LIBRARY_PATH="$r:$LD_LIBRARY_PATH"
  33             export DYLD_LIBRARY_PATH="$r:$LD_LIBRARY_PATH"
  34             ;;
  35     esac
  36     case "$(uname)" in
  37         "Darwin")
  38             ADDITIONAL_TRAITS="-notrait category=nonosxtests"
  39             ;;
  40         "Linux")
  41             ADDITIONAL_TRAITS="-notrait category=nonlinuxtests"
  42             ;;
  43     esac
  44
  45     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug "$r/net_4_x/xunit.console.exe" "$r/$test_argument_1" -noappdomain -noshadow -parallel none -xml "${xunit_results_path}" -notrait category=failing -notrait category=nonmonotests -notrait Benchmark=true -notrait category=outerloop $ADDITIONAL_TRAITS
  46     exit $?
  47 fi
  48
  49 if [ "$test_suite" = "--nunit" ]; then
  50     cd net_4_x || exit 1
  51     export MONO_PATH="$r/net_4_x/tests:$MONO_PATH"
  52     export MONO_SYSTEMWEB_CACHEDEPENDENCY_SHARED_FSW=1
  53     case "$test_argument_1" in
  54         *"Microsoft.Build"*)
  55             export TESTING_MONO=a
  56             export MSBuildExtensionsPath="$r/net_4_x/tests/xbuild/extensions"
  57             export XBUILD_FRAMEWORK_FOLDERS_PATH="$r/net_4_x/tests/xbuild/frameworks"
  58             case "$test_argument_1" in
  59                 "xbuild_12"*) export MONO_PATH="$r/xbuild_12:$r/xbuild_12/tests:$MONO_PATH" ;;
  60                 "xbuild_14"*) export MONO_PATH="$r/xbuild_14:$r/xbuild_14/tests:$MONO_PATH" ;;
  61             esac
  62             ;;
  63         *"Mono.Messaging.RabbitMQ"*)
  64             export MONO_MESSAGING_PROVIDER=Mono.Messaging.RabbitMQ.RabbitMQMessagingProvider,Mono.Messaging.RabbitMQ
  65             ;;
  66         *"System.Windows.Forms"*)
  67             sudo apt install -y xvfb xauth
  68             XVFBRUN="xvfb-run -a --"
  69             ADDITIONAL_TEST_EXCLUDES="NotWithXvfb" # TODO: find out why this works on Jenkins?
  70             ;;
  71     esac
  72     case "$test_argument_2" in
  73         "--flaky-test-retries="*)
  74             export MONO_FLAKY_TEST_RETRIES=$(echo "$test_argument_2" | cut -d "=" -f2)
  75             ;;
  76     esac
  77     cp -f "$r/$test_argument_1.nunitlite.config" "$r/net_4_x/nunit-lite-console.exe.config"
  78     MONO_REGISTRY_PATH="$HOME/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $XVFBRUN "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug "$r/net_4_x/nunit-lite-console.exe" "$r/$test_argument_1" -exclude=NotWorking,CAS,$ADDITIONAL_TEST_EXCLUDES -labels -format:xunit -result:"${xunit_results_path}"
  79     exit $?
  80 fi
  81
  82 if [ "$test_suite" = "--verify" ]; then
  83     verifiable_files=$(find net_4_x -maxdepth 1 -name "*.dll" -or -name "*.exe" | grep -v System.Runtime.CompilerServices.Unsafe.dll | grep -v Xunit.NetCore.Extensions.dll)
  84     ok=true
  85     for asm in $verifiable_files; do
  86         echo "$asm"
  87         if [ ! -f "$asm" ]; then continue; fi
  88         if "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --compile-all --verify-all --security=verifiable "$asm"; then
  89             echo "$asm verified OK"
  90         else
  91             echo "$asm verification failed"
  92             ok=false
  93         fi
  94     done;
  95     if [ "$ok" = "true" ]; then
  96         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"verify\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" name=\"Test collection for verify\" time=\"0\"><test name=\"verify.all\" type=\"verify\" method=\"all\" time=\"0\" result=\"Pass\"></test></collection></assembly></assemblies>" > "${xunit_results_path}";
  97         exit 0
  98     else
  99         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"verify\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" name=\"Test collection for verify\" time=\"0\"><test name=\"verify.all\" type=\"verify\" method=\"all\" time=\"0\" result=\"Fail\"><failure exception-type=\"VerifyException\"><message><![CDATA[Verifying framework assemblies failed. Check the log for more details.]]></message></failure></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 100         exit 1
 101     fi
 102 fi
 103
 104 if [ "$test_suite" = "--mcs" ]; then
 105     cd tests/mcs || exit 1
 106     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --verify-all compiler-tester.exe -mode:pos -files:v4 -compiler:"$r/net_4_x/mcs.exe" -reference-dir:"$r/net_4_x" -issues:known-issues-net_4_x -log:net_4_x.log -il:ver-il-net_4_x.xml -resultXml:"${xunit_results_path}" -compiler-options:"-d:NET_4_0;NET_4_5 -debug"
 107     exit $?
 108 fi
 109
 110 if [ "$test_suite" = "--mcs-errors" ]; then
 111     cd tests/mcs-errors || exit 1
 112     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" compiler-tester.exe -mode:neg -files:v4 -compiler:"$r/net_4_x/mcs.exe" -reference-dir:"$r/net_4_x" -issues:known-issues-net_4_x -log:net_4_x.log -resultXml:"${xunit_results_path}" -compiler-options:"-v --break-on-ice -d:NET_4_0;NET_4_5"
 113     exit $?
 114 fi
 115
 116 if [ "$test_suite" = "--aot-test" ]; then
 117     failed=0
 118     passed=0
 119     failed_tests=""
 120     profile="$r/net_4_x"
 121     tmpfile=$(mktemp -t mono_aot_outputXXXXXX) || exit 1
 122     rm -f "test-aot-*.stdout" "test-aot-*.stderr" "${xunit_results_path}.cases.xml"
 123     for assembly in "$profile"/*.dll; do
 124         asm_name=$(basename "$assembly")
 125         echo "... $asm_name"
 126         for conf in "|regular" "--gc=boehm|boehm"; do
 127             name=$(echo $conf | cut -d\| -f 2)
 128             params=$(echo $conf | cut -d\| -f 1)
 129             test_name="${asm_name}|${name}"
 130             echo "  $test_name"
 131             if "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" $params --aot=outfile="$tmpfile" "$assembly" > "test-aot-${name}-${asm_name}.stdout" 2> "test-aot-${name}-${asm_name}.stderr"
 132             then
 133                 passed=$((passed + 1))
 134                 resultstring="Pass"
 135             else \
 136                 failed=$((failed + 1))
 137                 failed_tests="${failed_tests} $test_name"
 138                 resultstring="Fail"
 139             fi
 140             echo "<test name=\"aot-test.$name.$asm_name\" type=\"aot-test.$name\" method=\"$asm_name\" time=\"0\" result=\"$resultstring\">" >> "${xunit_results_path}.cases.xml"
 141             if [ "$resultstring" = "Fail" ]; then
 142                 echo "<failure exception-type=\"AotTestException\"><message><![CDATA[
 143                     STDOUT:
 144                     $(cat "test-aot-${name}-${asm_name}.stdout")
 145                     STDERR:
 146                     $(cat "test-aot-${name}-${asm_name}.stderr")]]></message><stack-trace></stack-trace></failure>" >> "${xunit_results_path}.cases.xml"; fi
 147             echo "</test>" >> "${xunit_results_path}.cases.xml"
 148         done
 149     done
 150     echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>\
 151     <assemblies>\
 152         <assembly name=\"aot-test\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"$((passed + failed))\" passed=\"$passed\" failed=\"$failed\" skipped=\"0\" errors=\"0\" time=\"0\">\
 153             <collection total=\"$((passed + failed))\" passed=\"$passed\" failed=\"$failed\" skipped=\"0\" name=\"Test collection for aot-test\" time=\"0\">\
 154                 $(cat "${xunit_results_path}.cases.xml")
 155             </collection>\
 156         </assembly>\
 157     </assemblies>" > "${xunit_results_path}";
 158     rm "$tmpfile"
 159     echo "${passed} test(s) passed. ${failed} test(s) did not pass."
 160     if [ "${failed}" != 0 ]; then
 161         echo ""
 162         echo "Failed tests:"
 163         echo ""
 164         for i in ${failed_tests}; do
 165             echo "${i}";
 166         done
 167         exit 1
 168     fi
 169     exit 0
 170 fi
 171
 172 if [ "$test_suite" = "--mini" ]; then
 173     cd tests/mini || exit 1
 174
 175     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --regression ./*.exe > regressiontests.out 2>&1
 176     cat regressiontests.out
 177     if grep -q "100% pass" regressiontests.out; then
 178         resultstring="Pass"
 179         failurescount=0
 180         successcount=1
 181     else
 182         resultstring="Fail"
 183         failurescount=1
 184         successcount=0
 185     fi
 186     echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>\
 187         <assemblies>\
 188             <assembly name=\"mini.regression-tests\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"$successcount\" failed=\"$failurescount\" skipped=\"0\" errors=\"0\" time=\"0\">\
 189                 <collection total=\"1\" passed=\"$successcount\" failed=\"$failurescount\" skipped=\"0\" name=\"Test collection for mini.regression-tests\" time=\"0\">\
 190                     <test name=\"mini.regression-tests.all\" type=\"mini.regression-tests\" method=\"all\" time=\"0\" result=\"$resultstring\">" > "${xunit_results_path}"
 191                     if [ "$resultstring" = "Fail" ]; then echo "<failure exception-type=\"MiniRegressionTestsException\"><message><![CDATA[$(cat regressiontests.out)]]></message><stack-trace></stack-trace></failure>" >> "${xunit_results_path}"; fi
 192                 echo "</test>
 193                 </collection>\
 194             </assembly>\
 195         </assemblies>" >> "${xunit_results_path}";
 196     exit $failurescount
 197 fi
 198
 199 if [ "$test_suite" = "--symbolicate" ]; then
 200     cd tests/symbolicate || exit 1
 201
 202     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --aot 2>&1 | grep -q "AOT compilation is not supported" && echo "No AOT support, skipping tests." && exit 0
 203
 204     ok=true
 205     for config in without-aot with-aot with-aot-msym; do
 206         OUT_DIR="$config"
 207         MSYM_DIR="$OUT_DIR/msymdir"
 208         STACKTRACE_FILE="$OUT_DIR/stacktrace.out"
 209         SYMBOLICATE_RAW_FILE="$OUT_DIR/symbolicate_raw.out"
 210         SYMBOLICATE_RESULT_FILE="$OUT_DIR/symbolicate.result"
 211         SYMBOLICATE_EXPECTED_FILE=symbolicate.expected
 212
 213         echo "Checking StackTraceDumper.exe in configuration $config..."
 214         rm -rf "$OUT_DIR"
 215         mkdir -p "$OUT_DIR"
 216         mkdir -p "$MSYM_DIR"
 217
 218         cp StackTraceDumper.exe "$OUT_DIR"
 219         cp StackTraceDumper.pdb "$OUT_DIR"
 220
 221         # store symbols
 222         "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" store-symbols "$MSYM_DIR" "$OUT_DIR"
 223         "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" store-symbols "$MSYM_DIR" "$r/net_4_x"
 224
 225         if [ "$config" = "with-aot" ]; then "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline --aot "$OUT_DIR/StackTraceDumper.exe"; fi
 226         if [ "$config" = "with-aot-msym" ]; then "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline --aot=msym-dir="$MSYM_DIR" "$OUT_DIR/StackTraceDumper.exe"; fi
 227
 228         # check diff
 229         "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline StackTraceDumper.exe > "$STACKTRACE_FILE"
 230         "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" "$MSYM_DIR" "$STACKTRACE_FILE" > "$SYMBOLICATE_RAW_FILE"
 231         tr "\\\\" "/" < "$SYMBOLICATE_RAW_FILE" | sed "s,) .* in .*/mcs/,) in mcs/," | sed "s,) .* in .*/external/,) in external/," | sed '/\[MVID\]/d' | sed '/\[AOTID\]/d' > "$SYMBOLICATE_RESULT_FILE"
 232
 233         DIFF=$(diff -up "$SYMBOLICATE_EXPECTED_FILE" "$SYMBOLICATE_RESULT_FILE")
 234         if [ ! -z "$DIFF" ]; then
 235             echo "ERROR: Symbolicate tests failed."
 236             echo "If $SYMBOLICATE_RESULT_FILE is correct copy it to $SYMBOLICATE_EXPECTED_FILE."
 237             echo "Otherwise runtime sequence points need to be fixed."
 238             echo ""
 239             echo "$DIFF"
 240             ok=false
 241         else
 242             echo "Success."
 243         fi
 244     done
 245
 246     if [ "$ok" = "true" ]; then
 247         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"symbolicate\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" name=\"Test collection for symbolicate\" time=\"0\"><test name=\"symbolicate.all\" type=\"symbolicate\" method=\"all\" time=\"0\" result=\"Pass\"></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 248         exit 0
 249     else
 250         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"symbolicate\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" name=\"Test collection for symbolicate\" time=\"0\"><test name=\"symbolicate.all\" type=\"symbolicate\" method=\"all\" time=\"0\" result=\"Fail\"><failure exception-type=\"SymbolicateException\"><message><![CDATA[Symbolicate tests failed. Check the log for more details.]]></message></failure></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 251         exit 1
 252     fi
 253
 254 fi
 255
 256 if [ "$test_suite" = "--csi" ]; then
 257     cd tests/csi || error 1
 258     echo "Console.WriteLine (\"hello world: \" + DateTime.Now)" > csi-test.csx
 259
 260     ok=true
 261     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" csi.exe csi-test.csx > csi-test-output.txt || ok=false
 262     cat csi-test-output.txt && grep -q "hello world" csi-test-output.txt || ok=false
 263
 264     if [ "$ok" = "true" ]; then
 265         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"csi\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" name=\"Test collection for csi\" time=\"0\"><test name=\"csi.all\" type=\"csi\" method=\"all\" time=\"0\" result=\"Pass\"></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 266         exit 0
 267     else
 268         echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"csi\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" name=\"Test collection for csi\" time=\"0\"><test name=\"csi.all\" type=\"csi\" method=\"all\" time=\"0\" result=\"Fail\"><failure exception-type=\"CsiException\"><message><![CDATA[csi.exe tests failed. Check the log for more details.]]></message></failure></test></collection></assembly></assemblies>" > "${xunit_results_path}";
 269         exit 1
 270     fi
 271
 272 fi
 273
 274 if [ "$test_suite" = "--profiler" ]; then
 275     cd tests/profiler || exit 1
 276
 277     chmod +x "$r/mprof-report"
 278     perl ptestrunner.pl out-of-tree xunit "${xunit_results_path}"
 279     exit $?
 280 fi
 281
 282 if [ "$test_suite" = "--runtime" ]; then
 283     cd tests/runtime || exit 1
 284
 285     # TODO: only ported runtest-managed for now
 286     "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug test-runner.exe --verbose --xunit "${xunit_results_path}" --config tests-config --runtime "${MONO_EXECUTABLE}" --mono-path "$r/net_4_x" -j a --testsuite-name "runtime" --timeout 300 --disabled "$DISABLED_TESTS" $(cat runtime-test-list.txt)
 287     exit $?
 288 fi