w32file-unix.c: In function 'mono_w32file_get_file_size':
[mono-project.git] / runtime / mono-test.sh
blob2afb9177ae09d7f8fc9d663d90075e29afdddc32
1 #! /bin/sh -e
3 test_suite="$1"
4 test_argument_1="$2"
5 test_argument_2="$3"
6 xunit_results_path="$(pwd)/testResults.xml"
8 cd "$(dirname "$0")" || exit 1
10 r=$(pwd)
11 MONO_CFG_DIR="$r/_tmpinst/etc"
12 PATH="$r/_tmpinst/bin:$PATH"
13 MONO_EXECUTABLE=${MONO_EXECUTABLE:-"$r/mono-sgen"}
14 MONO_PATH="$r/net_4_x"
15 export MONO_CFG_DIR MONO_PATH MONO_EXECUTABLE PATH
17 sed "s,\$test_root_dir,$r,g" "$r/_tmpinst/etc/mono/config.tmpl" > "$r/_tmpinst/etc/mono/config"
18 sed "s,\$test_root_dir,$r,g" "$r/tests/runtime/tests-config.tmpl" > "$r/tests/runtime/tests-config"
20 chmod +x "${MONO_EXECUTABLE}"
21 echo "---------------------------------------------------------------------------------------"
22 "${MONO_EXECUTABLE}" --version
23 echo "---------------------------------------------------------------------------------------"
25 case "$(uname)" in
26 "Linux")
27 mkdir -p ~/.config/.mono/
28 wget -qO- https://download.mono-project.com/test/new-certs.tgz| tar zx -C ~/.config/.mono/
30 esac
32 if [ "$test_suite" = "--xunit" ]; then
33 cd net_4_x || exit 1
34 export MONO_PATH="$r/net_4_x/tests:$MONO_PATH"
35 export REMOTE_EXECUTOR="$r/net_4_x/tests/RemoteExecutorConsoleApp.exe"
36 case "$test_argument_1" in
37 *"Mono.Profiler.Log"*)
38 # necessary for the runtime to find libmono-profiler-log
39 export LD_LIBRARY_PATH="$r:$LD_LIBRARY_PATH"
40 export DYLD_LIBRARY_PATH="$r:$LD_LIBRARY_PATH"
42 *"System.Net.Http"*)
43 export MONO_URI_DOTNETRELATIVEORABSOLUTE=true
45 esac
46 case "$(uname)" in
47 "Darwin")
48 ADDITIONAL_TRAITS="-notrait category=nonosxtests"
50 "Linux")
51 ADDITIONAL_TRAITS="-notrait category=nonlinuxtests"
53 esac
55 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug "$r/net_4_x/xunit.console.exe" "$r/$test_argument_1" -noappdomain -noshadow -parallel none -xml "${xunit_results_path}" -notrait category=failing -notrait category=nonmonotests -notrait Benchmark=true -notrait category=outerloop $ADDITIONAL_TRAITS
56 exit $?
59 if [ "$test_suite" = "--nunit" ]; then
60 cd net_4_x || exit 1
61 export MONO_PATH="$r/net_4_x/tests:$MONO_PATH"
62 export MONO_SYSTEMWEB_CACHEDEPENDENCY_SHARED_FSW=1
63 case "$test_argument_1" in
64 *"Microsoft.Build"*)
65 export TESTING_MONO=a
66 export MSBuildExtensionsPath="$r/net_4_x/tests/xbuild/extensions"
67 export XBUILD_FRAMEWORK_FOLDERS_PATH="$r/net_4_x/tests/xbuild/frameworks"
68 case "$test_argument_1" in
69 "xbuild_12"*) export MONO_PATH="$r/xbuild_12:$r/xbuild_12/tests:$MONO_PATH" ;;
70 "xbuild_14"*) export MONO_PATH="$r/xbuild_14:$r/xbuild_14/tests:$MONO_PATH" ;;
71 esac
73 *"Mono.Messaging.RabbitMQ"*)
74 export MONO_MESSAGING_PROVIDER=Mono.Messaging.RabbitMQ.RabbitMQMessagingProvider,Mono.Messaging.RabbitMQ
76 *"System.Windows.Forms"*)
77 sudo apt install -y xvfb xauth
78 XVFBRUN="xvfb-run -a --"
79 ADDITIONAL_TEST_EXCLUDES="NotWithXvfb" # TODO: find out why this works on Jenkins?
81 esac
82 case "$test_argument_2" in
83 "--flaky-test-retries="*)
84 export MONO_FLAKY_TEST_RETRIES=$(echo "$test_argument_2" | cut -d "=" -f2)
86 esac
87 cp -f "$r/$test_argument_1.nunitlite.config" "$r/net_4_x/nunit-lite-console.exe.config"
88 MONO_REGISTRY_PATH="$HOME/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $XVFBRUN "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug "$r/net_4_x/nunit-lite-console.exe" "$r/$test_argument_1" -exclude=NotWorking,CAS,$ADDITIONAL_TEST_EXCLUDES -labels -format:xunit -result:"${xunit_results_path}"
89 exit $?
92 if [ "$test_suite" = "--verify" ]; then
93 verifiable_files=$(find net_4_x -maxdepth 1 -name "*.dll" -or -name "*.exe" | grep -v System.Runtime.CompilerServices.Unsafe.dll | grep -v Xunit.NetCore.Extensions.dll)
94 ok=true
95 for asm in $verifiable_files; do
96 echo "$asm"
97 if [ ! -f "$asm" ]; then continue; fi
98 if "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --compile-all --verify-all --security=verifiable "$asm"; then
99 echo "$asm verified OK"
100 else
101 echo "$asm verification failed"
102 ok=false
104 done;
105 if [ "$ok" = "true" ]; then
106 echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"verify\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" name=\"Test collection for verify\" time=\"0\"><test name=\"verify.all\" type=\"verify\" method=\"all\" time=\"0\" result=\"Pass\"></test></collection></assembly></assemblies>" > "${xunit_results_path}";
107 exit 0
108 else
109 echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"verify\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" name=\"Test collection for verify\" time=\"0\"><test name=\"verify.all\" type=\"verify\" method=\"all\" time=\"0\" result=\"Fail\"><failure exception-type=\"VerifyException\"><message><![CDATA[Verifying framework assemblies failed. Check the log for more details.]]></message></failure></test></collection></assembly></assemblies>" > "${xunit_results_path}";
110 exit 1
114 if [ "$test_suite" = "--mcs" ]; then
115 cd tests/mcs || exit 1
116 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --verify-all compiler-tester.exe -mode:pos -files:v4 -compiler:"$r/net_4_x/mcs.exe" -reference-dir:"$r/net_4_x" -issues:known-issues-net_4_x -log:net_4_x.log -il:ver-il-net_4_x.xml -resultXml:"${xunit_results_path}" -compiler-options:"-d:NET_4_0;NET_4_5 -debug"
117 exit $?
120 if [ "$test_suite" = "--mcs-errors" ]; then
121 cd tests/mcs-errors || exit 1
122 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" compiler-tester.exe -mode:neg -files:v4 -compiler:"$r/net_4_x/mcs.exe" -reference-dir:"$r/net_4_x" -issues:known-issues-net_4_x -log:net_4_x.log -resultXml:"${xunit_results_path}" -compiler-options:"-v --break-on-ice -d:NET_4_0;NET_4_5"
123 exit $?
126 if [ "$test_suite" = "--aot-test" ]; then
127 failed=0
128 passed=0
129 failed_tests=""
130 profile="$r/net_4_x"
131 tmpfile=$(mktemp -t mono_aot_outputXXXXXX) || exit 1
132 rm -f "test-aot-*.stdout" "test-aot-*.stderr" "${xunit_results_path}.cases.xml"
133 for assembly in "$profile"/*.dll; do
134 asm_name=$(basename "$assembly")
135 echo "... $asm_name"
136 for conf in "|regular" "--gc=boehm|boehm"; do
137 name=$(echo $conf | cut -d\| -f 2)
138 params=$(echo $conf | cut -d\| -f 1)
139 test_name="${asm_name}|${name}"
140 echo " $test_name"
141 if "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" $params --aot=outfile="$tmpfile" "$assembly" > "test-aot-${name}-${asm_name}.stdout" 2> "test-aot-${name}-${asm_name}.stderr"
142 then
143 passed=$((passed + 1))
144 resultstring="Pass"
145 else \
146 failed=$((failed + 1))
147 failed_tests="${failed_tests} $test_name"
148 resultstring="Fail"
150 echo "<test name=\"aot-test.$name.$asm_name\" type=\"aot-test.$name\" method=\"$asm_name\" time=\"0\" result=\"$resultstring\">" >> "${xunit_results_path}.cases.xml"
151 if [ "$resultstring" = "Fail" ]; then
152 echo "<failure exception-type=\"AotTestException\"><message><![CDATA[
153 STDOUT:
154 $(cat "test-aot-${name}-${asm_name}.stdout")
155 STDERR:
156 $(cat "test-aot-${name}-${asm_name}.stderr")]]></message><stack-trace></stack-trace></failure>" >> "${xunit_results_path}.cases.xml"; fi
157 echo "</test>" >> "${xunit_results_path}.cases.xml"
158 done
159 done
160 echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>\
161 <assemblies>\
162 <assembly name=\"aot-test\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"$((passed + failed))\" passed=\"$passed\" failed=\"$failed\" skipped=\"0\" errors=\"0\" time=\"0\">\
163 <collection total=\"$((passed + failed))\" passed=\"$passed\" failed=\"$failed\" skipped=\"0\" name=\"Test collection for aot-test\" time=\"0\">\
164 $(cat "${xunit_results_path}.cases.xml")
165 </collection>\
166 </assembly>\
167 </assemblies>" > "${xunit_results_path}";
168 rm "$tmpfile"
169 echo "${passed} test(s) passed. ${failed} test(s) did not pass."
170 if [ "${failed}" != 0 ]; then
171 echo ""
172 echo "Failed tests:"
173 echo ""
174 for i in ${failed_tests}; do
175 echo "${i}";
176 done
177 exit 1
179 exit 0
182 if [ "$test_suite" = "--mini" ]; then
183 cd tests/mini || exit 1
185 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --regression ./*.exe > regressiontests.out 2>&1
186 cat regressiontests.out
187 if grep -q "100% pass" regressiontests.out; then
188 resultstring="Pass"
189 failurescount=0
190 successcount=1
191 else
192 resultstring="Fail"
193 failurescount=1
194 successcount=0
196 echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>\
197 <assemblies>\
198 <assembly name=\"mini.regression-tests\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"$successcount\" failed=\"$failurescount\" skipped=\"0\" errors=\"0\" time=\"0\">\
199 <collection total=\"1\" passed=\"$successcount\" failed=\"$failurescount\" skipped=\"0\" name=\"Test collection for mini.regression-tests\" time=\"0\">\
200 <test name=\"mini.regression-tests.all\" type=\"mini.regression-tests\" method=\"all\" time=\"0\" result=\"$resultstring\">" > "${xunit_results_path}"
201 if [ "$resultstring" = "Fail" ]; then echo "<failure exception-type=\"MiniRegressionTestsException\"><message><![CDATA[$(cat regressiontests.out)]]></message><stack-trace></stack-trace></failure>" >> "${xunit_results_path}"; fi
202 echo "</test>
203 </collection>\
204 </assembly>\
205 </assemblies>" >> "${xunit_results_path}";
206 exit $failurescount
209 if [ "$test_suite" = "--symbolicate" ]; then
210 cd tests/symbolicate || exit 1
212 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --aot 2>&1 | grep -q "AOT compilation is not supported" && echo "No AOT support, skipping tests." && exit 0
214 ok=true
215 for config in without-aot with-aot with-aot-msym; do
216 OUT_DIR="$config"
217 MSYM_DIR="$OUT_DIR/msymdir"
218 STACKTRACE_FILE="$OUT_DIR/stacktrace.out"
219 SYMBOLICATE_RAW_FILE="$OUT_DIR/symbolicate_raw.out"
220 SYMBOLICATE_RESULT_FILE="$OUT_DIR/symbolicate.result"
221 SYMBOLICATE_EXPECTED_FILE=symbolicate.expected
223 echo "Checking StackTraceDumper.exe in configuration $config..."
224 rm -rf "$OUT_DIR"
225 mkdir -p "$OUT_DIR"
226 mkdir -p "$MSYM_DIR"
228 cp StackTraceDumper.exe "$OUT_DIR"
229 cp StackTraceDumper.pdb "$OUT_DIR"
231 # store symbols
232 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" store-symbols "$MSYM_DIR" "$OUT_DIR"
233 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" store-symbols "$MSYM_DIR" "$r/net_4_x"
235 if [ "$config" = "with-aot" ]; then "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline --aot "$OUT_DIR/StackTraceDumper.exe"; fi
236 if [ "$config" = "with-aot-msym" ]; then "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline --aot=msym-dir="$MSYM_DIR" "$OUT_DIR/StackTraceDumper.exe"; fi
238 # check diff
239 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline StackTraceDumper.exe > "$STACKTRACE_FILE"
240 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" "$MSYM_DIR" "$STACKTRACE_FILE" > "$SYMBOLICATE_RAW_FILE"
241 tr "\\\\" "/" < "$SYMBOLICATE_RAW_FILE" | sed "s,) .* in .*/mcs/,) in mcs/," | sed "s,) .* in .*/external/,) in external/," | sed '/\[MVID\]/d' | sed '/\[AOTID\]/d' > "$SYMBOLICATE_RESULT_FILE"
243 DIFF=$(diff -up "$SYMBOLICATE_EXPECTED_FILE" "$SYMBOLICATE_RESULT_FILE")
244 if [ ! -z "$DIFF" ]; then
245 echo "ERROR: Symbolicate tests failed."
246 echo "If $SYMBOLICATE_RESULT_FILE is correct copy it to $SYMBOLICATE_EXPECTED_FILE."
247 echo "Otherwise runtime sequence points need to be fixed."
248 echo ""
249 echo "$DIFF"
250 ok=false
251 else
252 echo "Success."
254 done
256 if [ "$ok" = "true" ]; then
257 echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"symbolicate\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" name=\"Test collection for symbolicate\" time=\"0\"><test name=\"symbolicate.all\" type=\"symbolicate\" method=\"all\" time=\"0\" result=\"Pass\"></test></collection></assembly></assemblies>" > "${xunit_results_path}";
258 exit 0
259 else
260 echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"symbolicate\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" name=\"Test collection for symbolicate\" time=\"0\"><test name=\"symbolicate.all\" type=\"symbolicate\" method=\"all\" time=\"0\" result=\"Fail\"><failure exception-type=\"SymbolicateException\"><message><![CDATA[Symbolicate tests failed. Check the log for more details.]]></message></failure></test></collection></assembly></assemblies>" > "${xunit_results_path}";
261 exit 1
266 if [ "$test_suite" = "--csi" ]; then
267 cd tests/csi || error 1
268 echo "Console.WriteLine (\"hello world: \" + DateTime.Now)" > csi-test.csx
270 ok=true
271 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" csi.exe csi-test.csx > csi-test-output.txt || ok=false
272 cat csi-test-output.txt && grep -q "hello world" csi-test-output.txt || ok=false
274 if [ "$ok" = "true" ]; then
275 echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"csi\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"1\" failed=\"0\" skipped=\"0\" name=\"Test collection for csi\" time=\"0\"><test name=\"csi.all\" type=\"csi\" method=\"all\" time=\"0\" result=\"Pass\"></test></collection></assembly></assemblies>" > "${xunit_results_path}";
276 exit 0
277 else
278 echo "<?xml version=\"1.0\" encoding=\"utf-8\"?><assemblies><assembly name=\"csi\" environment=\"Mono\" test-framework=\"custom\" run-date=\"$(date +%F)\" run-time=\"$(date +%T)\" total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" errors=\"0\" time=\"0\"><collection total=\"1\" passed=\"0\" failed=\"1\" skipped=\"0\" name=\"Test collection for csi\" time=\"0\"><test name=\"csi.all\" type=\"csi\" method=\"all\" time=\"0\" result=\"Fail\"><failure exception-type=\"CsiException\"><message><![CDATA[csi.exe tests failed. Check the log for more details.]]></message></failure></test></collection></assembly></assemblies>" > "${xunit_results_path}";
279 exit 1
284 if [ "$test_suite" = "--profiler" ]; then
285 cd tests/profiler || exit 1
287 chmod +x "$r/mprof-report"
288 perl ptestrunner.pl out-of-tree xunit "${xunit_results_path}"
289 exit $?
292 if [ "$test_suite" = "--runtime" ]; then
293 cd tests/runtime || exit 1
295 # TODO: only ported runtest-managed for now
296 "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug test-runner.exe --verbose --xunit "${xunit_results_path}" --config tests-config --runtime "${MONO_EXECUTABLE}" --mono-path "$r/net_4_x" -j a --testsuite-name "runtime" --timeout 300 --disabled "$DISABLED_TESTS" --input-file runtime-test-list.txt
297 exit $?