From c4087410008da3cb43913bea1b2b44ae802513bb Mon Sep 17 00:00:00 2001 From: =?utf8?q?Alexander=20K=C3=B6plinger?= Date: Mon, 12 Nov 2018 15:15:49 +0100 Subject: [PATCH] [ci] Add targets to bundle tests and add a script to run them in Helix The test-bundle target copies all assets needed for test execution to a specified directory. We upload this directory to Helix and use it to run our mainline tests. --- configure.ac | 1 + libgc/Makefile.am | 2 + mcs/errors/Makefile | 7 + mcs/packages/Makefile | 4 + mcs/tests/Makefile | 8 + mcs/tools/mono-symbolicate/Makefile | 4 + mono/btls/Makefile.am | 4 + mono/mini/Makefile.am.in | 10 ++ mono/native/Makefile.am | 4 +- mono/profiler/Makefile.am | 8 + mono/tests/Makefile.am | 11 +- po/mcs/Makefile.in.in | 2 + runtime/Makefile.am | 25 ++++ runtime/mono-test.sh | 287 ++++++++++++++++++++++++++++++++++++ scripts/ci/run-jenkins.sh | 13 +- scripts/ci/run-test-helix.sh | 35 +++++ scripts/ci/util.sh | 61 ++++++++ support/Makefile.am | 4 + 18 files changed, 483 insertions(+), 7 deletions(-) create mode 100755 runtime/mono-test.sh create mode 100755 scripts/ci/run-test-helix.sh diff --git a/configure.ac b/configure.ac index 6a139b790ea..b414af3bac3 100644 --- a/configure.ac +++ b/configure.ac @@ -21,6 +21,7 @@ AM_INIT_AUTOMAKE([1.9 dist-bzip2 tar-ustar no-dist-gzip foreign subdir-objects] AC_CONFIG_HEADERS([config.h]) AM_MAINTAINER_MODE AM_EXTRA_RECURSIVE_TARGETS([test]) +AM_EXTRA_RECURSIVE_TARGETS([test-bundle]) API_VER=2.0 AC_SUBST(API_VER) diff --git a/libgc/Makefile.am b/libgc/Makefile.am index c2fb48d5437..f1543123141 100644 --- a/libgc/Makefile.am +++ b/libgc/Makefile.am @@ -186,3 +186,5 @@ EXTRA_DIST += configure.host if $(CPP) $< >$@ ; then :; else rm -f $@; fi test: + +test-bundle: diff --git a/mcs/errors/Makefile b/mcs/errors/Makefile index 334d0f1350c..dc1eb33b5da 100644 --- a/mcs/errors/Makefile +++ b/mcs/errors/Makefile @@ -62,6 +62,13 @@ TESTER = MONO_RUNTIME='$(RUNTIME)' $(TEST_RUNTIME) $(TEST_RUNTIME_FLAGS) $(LOCAL test-local: $(TEST_SUPPORT_FILES) +test-bundle: + mkdir -p $(TEST_BUNDLE_PATH)/tests/mcs-errors + cp -L -R dlls $(TEST_BUNDLE_PATH)/tests/mcs-errors/ + cp -L *.cs *.inc *.dll *.snk *.pub $(TEST_BUNDLE_PATH)/tests/mcs-errors/ + cp -L $(topdir)/class/lib/$(PROFILE)/compiler-tester.* $(TEST_BUNDLE_PATH)/tests/mcs-errors/ + cp -L known-issues-$(PROFILE) $(TEST_BUNDLE_PATH)/tests/mcs-errors/known-issues-$(PROFILE) + run-mcs-tests: test-local $(TESTER) -mode:neg -files:$(TEST_PATTERN) -compiler:$(COMPILER) -reference-dir:$(topdir)/class/lib/$(PROFILE) -issues:known-issues-$(PROFILE) -log:$(PROFILE).log $(TESTER_OPTIONS) $(TOPTIONS) diff --git a/mcs/packages/Makefile b/mcs/packages/Makefile index 46c3a5fbc02..95530754d70 100644 --- a/mcs/packages/Makefile +++ b/mcs/packages/Makefile @@ -57,6 +57,10 @@ test-csi: cat csi-test-output.txt && grep -q "hello world" csi-test-output.txt rm csi-test-output.txt +test-bundle: + mkdir -p $(TEST_BUNDLE_PATH)/tests/csi + cp -L $(ROSLYN_CSC_DIR)/* $(TEST_BUNDLE_PATH)/tests/csi/ + endif dist-local: dist-default diff --git a/mcs/tests/Makefile b/mcs/tests/Makefile index bd2944cfdc1..6cbe4dbeb8e 100644 --- a/mcs/tests/Makefile +++ b/mcs/tests/Makefile @@ -86,6 +86,14 @@ run-test-local: test-local qcheck check: run-test-local +test-bundle: + mkdir -p $(TEST_BUNDLE_PATH)/tests/mcs + cp -L -R dlls $(TEST_BUNDLE_PATH)/tests/mcs/ + cp -L *.cs *.xml *.inc *.dll *.snk $(TEST_BUNDLE_PATH)/tests/mcs/ + cp -L $(topdir)/class/lib/$(PROFILE)/compiler-tester.* $(TEST_BUNDLE_PATH)/tests/mcs/ + cp -L $(KNOWN_ISSUES) $(TEST_BUNDLE_PATH)/tests/mcs/$(KNOWN_ISSUES) + cp -L ver-il-$(PROFILE).xml $(TEST_BUNDLE_PATH)/tests/mcs/ver-il-$(PROFILE).xml + endif clean-local: diff --git a/mcs/tools/mono-symbolicate/Makefile b/mcs/tools/mono-symbolicate/Makefile index 02f40193c32..9be64f4beca 100644 --- a/mcs/tools/mono-symbolicate/Makefile +++ b/mcs/tools/mono-symbolicate/Makefile @@ -58,6 +58,10 @@ AOT_SUPPORTED = $(shell $(MONO) --aot 2>&1 | grep -q "AOT compilation is not sup test-local: $(TEST_EXE) +test-bundle: + mkdir -p $(TEST_BUNDLE_PATH)/tests/symbolicate + cp -L $(TEST_EXE) $(TEST_EXE:.exe=.pdb) $(SYMBOLICATE_EXPECTED_FILE) $(TEST_BUNDLE_PATH)/tests/symbolicate/ + $(TEST_EXE): $(TEST_CS) $(CSCOMPILE) $(TEST_CS) -r:$(LIB_PATH)/mscorlib.dll -r:$(LIB_PATH)/System.Core.dll -warn:0 -out:$(TEST_EXE) diff --git a/mono/btls/Makefile.am b/mono/btls/Makefile.am index 9b51fcc32b2..95080871c79 100644 --- a/mono/btls/Makefile.am +++ b/mono/btls/Makefile.am @@ -80,3 +80,7 @@ if HOST_WIN32 else $(install_sh) build-shared/libmono-btls-shared.* "$(DESTDIR)$(libdir)" endif + +test-bundle-local: + mkdir -p $(TEST_BUNDLE_PATH) + cp -L build-shared/libmono-btls-shared$(libsuffix) $(TEST_BUNDLE_PATH)/ diff --git a/mono/mini/Makefile.am.in b/mono/mini/Makefile.am.in index 87cb4c2401d..bf32f69c011 100755 --- a/mono/mini/Makefile.am.in +++ b/mono/mini/Makefile.am.in @@ -805,6 +805,16 @@ testi: mono test.exe test-local: $(regtests) +test-bundle-local: + mkdir -p $(TEST_BUNDLE_PATH)/tests/mini + cp -L $(regtests) TestDriver.dll MemoryIntrinsics.dll generics-variant-types.dll $(TEST_BUNDLE_PATH)/tests/mini/ + cp -L mono-sgen $(TEST_BUNDLE_PATH)/ + chmod +x $(TEST_BUNDLE_PATH)/mono-sgen +if SUPPORT_BOEHM + cp -L mono-boehm $(TEST_BUNDLE_PATH)/ + chmod +x $(TEST_BUNDLE_PATH)/mono-boehm +endif + # ensure the tests are actually correct checktests: $(regtests) for i in $(regtests); do $(MINI_RUNTIME) $$i; done diff --git a/mono/native/Makefile.am b/mono/native/Makefile.am index d1e2caef2d8..bd8b799563c 100644 --- a/mono/native/Makefile.am +++ b/mono/native/Makefile.am @@ -115,4 +115,6 @@ libmono_native_unified_la_LDFLAGS = $(MONO_NATIVE_UNIFIED_LDFLAGS) libmono_native_unified_la_LIBADD = $(MONO_NATIVE_LIBADD) - +test-bundle-local: + mkdir -p $(TEST_BUNDLE_PATH) + cp -L .libs/libmono-native$(libsuffix) $(TEST_BUNDLE_PATH)/ diff --git a/mono/profiler/Makefile.am b/mono/profiler/Makefile.am index a0aebfcb205..16b3dae1a1c 100644 --- a/mono/profiler/Makefile.am +++ b/mono/profiler/Makefile.am @@ -161,6 +161,14 @@ MCS = $(TOOLS_RUNTIME) $(CSC) -lib:$(CLASS) -unsafe -nologo -noconfig -nowarn:01 test-local: $(PLOG_TESTS) +test-bundle-local: + mkdir -p $(TEST_BUNDLE_PATH)/tests/profiler/ + cp -L .libs/libmono-profiler-log$(libsuffix) $(TEST_BUNDLE_PATH)/ + cp -L $(PLOG_TESTS) $(TEST_BUNDLE_PATH)/tests/profiler/ + cp -L ptestrunner.pl $(TEST_BUNDLE_PATH)/tests/profiler/ + cp -L mprof-report $(TEST_BUNDLE_PATH)/ + chmod +x $(TEST_BUNDLE_PATH)/mprof-report + run-test: test MONO_PATH=$(CLASS) perl $(srcdir)/ptestrunner.pl $(top_builddir) nunit TestResult-profiler.xml diff --git a/mono/tests/Makefile.am b/mono/tests/Makefile.am index 02979b57809..c78c45c6d0d 100755 --- a/mono/tests/Makefile.am +++ b/mono/tests/Makefile.am @@ -1996,13 +1996,18 @@ test-env-options: MONO_ENV_OPTIONS="--version" $(RUNTIME) array-init.exe | grep -q Architecture: TESTS_REGULAR = $(TESTS_CS) $(TESTS_IL) $(TESTS_BENCH) - -generate-regular-test-list: - @echo $(TESTS_REGULAR) > $(TEST_LIST_PATH) +TESTS_INCL_DEPS = $(shell find . -type f -name "*.exe" -o -name "*.dll" -o -name "*.netmodule" -o -name "*.config") # Target to precompile the test executables test-local: $(TESTS_REGULAR) $(TESTS_STRESS) $(TESTS_GSHARED) $(TESTSAOT_CS) $(TESTSAOT_IL) $(TESTSAOT_BENCH) $(TESTSAOT_STRESS) $(TESTSAOT_GSHARED) $(TESTS_TAILCALL) $(TESTSAOT_TAILCALL) compile-gac-loading compile-assembly-load-reference test-runner.exe +test-bundle-local: + mkdir -p $(addprefix $(TEST_BUNDLE_PATH)/tests/runtime/,$(sort $(dir $(TESTS_INCL_DEPS)))) + @echo $(TESTS_REGULAR) > $(TEST_BUNDLE_PATH)/tests/runtime/runtime-test-list.txt + sed -e 's,$$mono_libdir,$$test_root_dir,g' tests-config > $(TEST_BUNDLE_PATH)/tests/runtime/tests-config.tmpl + cp -L .libs/libtest$(libsuffix) $(TEST_BUNDLE_PATH)/tests/runtime/ + $(foreach asset,$(TESTS_INCL_DEPS),cp -L $(asset) $(TEST_BUNDLE_PATH)/tests/runtime/$(dir $(asset));) + # Precompile the test assemblies in parallel compile-tests: $(MAKE) -j4 test diff --git a/po/mcs/Makefile.in.in b/po/mcs/Makefile.in.in index 7dd3770a06e..aa1dbf6d36c 100644 --- a/po/mcs/Makefile.in.in +++ b/po/mcs/Makefile.in.in @@ -445,3 +445,5 @@ force: .NOEXPORT: test: + +test-bundle: diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 30df8ca91ea..0ba589b7949 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -147,6 +147,31 @@ mcs-do-run-test-profiles: test-support-files mcs-do-xunit-run-test-profiles: test-support-files cd $(mcs_topdir) && $(MAKE) NO_DIR_CHECK=1 PROFILES='$(test_profiles)' run-xunit-test-profiles +test-bundle-local: + mkdir -p $(TEST_BUNDLE_PATH)/_tmpinst/bin + cp -L mono-test.sh $(TEST_BUNDLE_PATH) + cp -L -R etc $(TEST_BUNDLE_PATH)/_tmpinst + rm $(TEST_BUNDLE_PATH)/_tmpinst/etc/mono/config + sed -e 's,$$mono_libdir,$$test_root_dir,g' -e 's,target="$(libgdiplus_install_loc)",target="$$test_root_dir/mono-libgdiplus$(libsuffix)",g' $(top_builddir)/data/config > $(TEST_BUNDLE_PATH)/_tmpinst/etc/mono/config.tmpl + echo '#! /bin/sh' > $(TEST_BUNDLE_PATH)/_tmpinst/bin/al + echo '#! /bin/sh' > $(TEST_BUNDLE_PATH)/_tmpinst/bin/mcs + echo '#! /bin/sh' > $(TEST_BUNDLE_PATH)/_tmpinst/bin/ilasm + echo '"$$MONO_EXECUTABLE" $$(dirname "$$MONO_EXECUTABLE")/net_4_x/al.exe "$$@"' >> $(TEST_BUNDLE_PATH)/_tmpinst/bin/al + echo '"$$MONO_EXECUTABLE" $$(dirname "$$MONO_EXECUTABLE")/net_4_x/mcs.exe "$$@"' >> $(TEST_BUNDLE_PATH)/_tmpinst/bin/mcs + echo '"$$MONO_EXECUTABLE" $$(dirname "$$MONO_EXECUTABLE")/net_4_x/ilasm.exe "$$@"' >> $(TEST_BUNDLE_PATH)/_tmpinst/bin/ilasm + chmod +x $(TEST_BUNDLE_PATH)/_tmpinst/bin/al + chmod +x $(TEST_BUNDLE_PATH)/_tmpinst/bin/mcs + chmod +x $(TEST_BUNDLE_PATH)/_tmpinst/bin/ilasm + for profile in $(test_profiles); do \ + cp -L -R $(mcs_topdir)/class/lib/$$profile $(TEST_BUNDLE_PATH); \ + done + cp -L $(top_srcdir)/external/xunit-binaries/*.dll $(top_srcdir)/external/xunit-binaries/*.exe $(top_srcdir)/external/xunit-binaries/*.config $(TEST_BUNDLE_PATH)/net_4_x/ + cp -L $(top_srcdir)/external/xunit-binaries/xunit.execution.dotnet.dll $(TEST_BUNDLE_PATH)/net_4_x/tests/ + $(MAKE) -C $(mcs_topdir)/tests test-bundle + $(MAKE) -C $(mcs_topdir)/errors test-bundle + $(MAKE) -C $(mcs_topdir)/packages test-bundle + $(MAKE) -C $(mcs_topdir)/tools/mono-symbolicate test-bundle + if HOST_WIN32 if CROSS_COMPILING cur_dir_cmd = pwd diff --git a/runtime/mono-test.sh b/runtime/mono-test.sh new file mode 100755 index 00000000000..f287fa11915 --- /dev/null +++ b/runtime/mono-test.sh @@ -0,0 +1,287 @@ +#! /bin/sh -e + +test_suite="$1" +test_argument_1="$2" +test_argument_2="$3" +xunit_results_path="$(pwd)/testResults.xml" + +cd "$(dirname "$0")" || exit 1 + +r=$(pwd) +MONO_CFG_DIR="$r/_tmpinst/etc" +PATH="$r/_tmpinst/bin:$PATH" +MONO_EXECUTABLE=${MONO_EXECUTABLE:-"$r/mono-sgen"} +MONO_PATH="$r/net_4_x" +export MONO_CFG_DIR MONO_PATH MONO_EXECUTABLE PATH + +sed "s,\$test_root_dir,$r,g" "$r/_tmpinst/etc/mono/config.tmpl" > "$r/_tmpinst/etc/mono/config" +sed "s,\$test_root_dir,$r,g" "$r/tests/runtime/tests-config.tmpl" > "$r/tests/runtime/tests-config" + +chmod +x "${MONO_EXECUTABLE}" +echo "---------------------------------------------------------------------------------------" +"${MONO_EXECUTABLE}" --version +echo "---------------------------------------------------------------------------------------" + +if [ "$test_suite" = "--xunit" ]; then + cd net_4_x || exit 1 + export MONO_PATH="$r/net_4_x/tests:$MONO_PATH" + export REMOTE_EXECUTOR="$r/net_4_x/tests/RemoteExecutorConsoleApp.exe" + case "$test_argument_1" in + *"Mono.Profiler.Log"*) + # necessary for the runtime to find libmono-profiler-log + export LD_LIBRARY_PATH="$r:$LD_LIBRARY_PATH" + export DYLD_LIBRARY_PATH="$r:$LD_LIBRARY_PATH" + ;; + esac + case "$(uname)" in + "Darwin") + ADDITIONAL_TRAITS="-notrait category=nonosxtests" + ;; + "Linux") + ADDITIONAL_TRAITS="-notrait category=nonlinuxtests" + ;; + esac + + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug "$r/net_4_x/xunit.console.exe" "$r/$test_argument_1" -noappdomain -noshadow -parallel none -xml "${xunit_results_path}" -notrait category=failing -notrait category=nonmonotests -notrait Benchmark=true -notrait category=outerloop $ADDITIONAL_TRAITS + exit $? +fi + +if [ "$test_suite" = "--nunit" ]; then + cd net_4_x || exit 1 + export MONO_PATH="$r/net_4_x/tests:$MONO_PATH" + case "$test_argument_1" in + *"Microsoft.Build"*) + export TESTING_MONO=a + export MSBuildExtensionsPath="$r/net_4_x/tests/xbuild/extensions" + export XBUILD_FRAMEWORK_FOLDERS_PATH="$r/net_4_x/tests/xbuild/frameworks" + case "$test_argument_1" in + "xbuild_12"*) export MONO_PATH="$r/xbuild_12:$r/xbuild_12/tests:$MONO_PATH" ;; + "xbuild_14"*) export MONO_PATH="$r/xbuild_14:$r/xbuild_14/tests:$MONO_PATH" ;; + esac + ;; + *"Mono.Messaging.RabbitMQ"*) + export MONO_MESSAGING_PROVIDER=Mono.Messaging.RabbitMQ.RabbitMQMessagingProvider,Mono.Messaging.RabbitMQ + ;; + *"System.Windows.Forms"*) + sudo apt install -y xvfb xauth + XVFBRUN="xvfb-run -a --" + ADDITIONAL_TEST_EXCLUDES="NotWithXvfb" # TODO: find out why this works on Jenkins? + ;; + esac + case "$test_argument_2" in + "--flaky-test-retries="*) + export MONO_FLAKY_TEST_RETRIES=$(echo "$test_argument_2" | cut -d "=" -f2) + ;; + esac + cp -f "$r/$test_argument_1.nunitlite.config" "$r/net_4_x/nunit-lite-console.exe.config" + MONO_REGISTRY_PATH="$HOME/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $XVFBRUN "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug "$r/net_4_x/nunit-lite-console.exe" "$r/$test_argument_1" -exclude=NotWorking,CAS,$ADDITIONAL_TEST_EXCLUDES -labels -format:xunit -result:"${xunit_results_path}" + exit $? +fi + +if [ "$test_suite" = "--verify" ]; then + verifiable_files=$(find net_4_x -maxdepth 1 -name "*.dll" -or -name "*.exe" | grep -v System.Runtime.CompilerServices.Unsafe.dll | grep -v Xunit.NetCore.Extensions.dll) + ok=true + for asm in $verifiable_files; do + echo "$asm" + if [ ! -f "$asm" ]; then continue; fi + if "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --compile-all --verify-all --security=verifiable "$asm"; then + echo "$asm verified OK" + else + echo "$asm verification failed" + ok=false + fi + done; + if [ "$ok" = "true" ]; then + echo "" > "${xunit_results_path}"; + exit 0 + else + echo "" > "${xunit_results_path}"; + exit 1 + fi +fi + +if [ "$test_suite" = "--mcs" ]; then + cd tests/mcs || exit 1 + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --verify-all compiler-tester.exe -mode:pos -files:v4 -compiler:"$r/net_4_x/mcs.exe" -reference-dir:"$r/net_4_x" -issues:known-issues-net_4_x -log:net_4_x.log -il:ver-il-net_4_x.xml -resultXml:"${xunit_results_path}" -compiler-options:"-d:NET_4_0;NET_4_5 -debug" + exit $? +fi + +if [ "$test_suite" = "--mcs-errors" ]; then + cd tests/mcs-errors || exit 1 + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" compiler-tester.exe -mode:neg -files:v4 -compiler:"$r/net_4_x/mcs.exe" -reference-dir:"$r/net_4_x" -issues:known-issues-net_4_x -log:net_4_x.log -resultXml:"${xunit_results_path}" -compiler-options:"-v --break-on-ice -d:NET_4_0;NET_4_5" + exit $? +fi + +if [ "$test_suite" = "--aot-test" ]; then + failed=0 + passed=0 + failed_tests="" + profile="$r/net_4_x" + tmpfile=$(mktemp -t mono_aot_outputXXXXXX) || exit 1 + rm -f "test-aot-*.stdout" "test-aot-*.stderr" "${xunit_results_path}.cases.xml" + for assembly in "$profile"/*.dll; do + asm_name=$(basename "$assembly") + echo "... $asm_name" + for conf in "|regular" "--gc=boehm|boehm"; do + name=$(echo $conf | cut -d\| -f 2) + params=$(echo $conf | cut -d\| -f 1) + test_name="${asm_name}|${name}" + echo " $test_name" + if "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" $params --aot=outfile="$tmpfile" "$assembly" > "test-aot-${name}-${asm_name}.stdout" 2> "test-aot-${name}-${asm_name}.stderr" + then + passed=$((passed + 1)) + resultstring="Pass" + else \ + failed=$((failed + 1)) + failed_tests="${failed_tests} $test_name" + resultstring="Fail" + fi + echo "" >> "${xunit_results_path}.cases.xml" + if [ "$resultstring" = "Fail" ]; then + echo "" >> "${xunit_results_path}.cases.xml"; fi + echo "" >> "${xunit_results_path}.cases.xml" + done + done + echo "\ + \ + \ + \ + $(cat "${xunit_results_path}.cases.xml") + \ + \ + " > "${xunit_results_path}"; + rm "$tmpfile" + echo "${passed} test(s) passed. ${failed} test(s) did not pass." + if [ "${failed}" != 0 ]; then + echo "" + echo "Failed tests:" + echo "" + for i in ${failed_tests}; do + echo "${i}"; + done + exit 1 + fi + exit 0 +fi + +if [ "$test_suite" = "--mini" ]; then + cd tests/mini || exit 1 + + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --regression ./*.exe > regressiontests.out 2>&1 + cat regressiontests.out + if grep -q "100% pass" regressiontests.out; then + resultstring="Pass" + failurescount=0 + successcount=1 + else + resultstring="Fail" + failurescount=1 + successcount=0 + fi + echo "\ + \ + \ + \ + " > "${xunit_results_path}" + if [ "$resultstring" = "Fail" ]; then echo "" >> "${xunit_results_path}"; fi + echo " + \ + \ + " >> "${xunit_results_path}"; + exit $failurescount +fi + +if [ "$test_suite" = "--symbolicate" ]; then + cd tests/symbolicate || exit 1 + + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --aot 2>&1 | grep -q "AOT compilation is not supported" && echo "No AOT support, skipping tests." && exit 0 + + ok=true + for config in without-aot with-aot with-aot-msym; do + OUT_DIR="$config" + MSYM_DIR="$OUT_DIR/msymdir" + STACKTRACE_FILE="$OUT_DIR/stacktrace.out" + SYMBOLICATE_RAW_FILE="$OUT_DIR/symbolicate_raw.out" + SYMBOLICATE_RESULT_FILE="$OUT_DIR/symbolicate.result" + SYMBOLICATE_EXPECTED_FILE=symbolicate.expected + + echo "Checking StackTraceDumper.exe in configuration $config..." + rm -rf "$OUT_DIR" + mkdir -p "$OUT_DIR" + mkdir -p "$MSYM_DIR" + + cp StackTraceDumper.exe "$OUT_DIR" + cp StackTraceDumper.pdb "$OUT_DIR" + + # store symbols + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" store-symbols "$MSYM_DIR" "$OUT_DIR" + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" store-symbols "$MSYM_DIR" "$r/net_4_x" + + if [ "$config" = "with-aot" ]; then "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline --aot "$OUT_DIR/StackTraceDumper.exe"; fi + if [ "$config" = "with-aot-msym" ]; then "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline --aot=msym-dir="$MSYM_DIR" "$OUT_DIR/StackTraceDumper.exe"; fi + + # check diff + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" -O=-inline StackTraceDumper.exe > "$STACKTRACE_FILE" + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" "$r/net_4_x/mono-symbolicate.exe" "$MSYM_DIR" "$STACKTRACE_FILE" > "$SYMBOLICATE_RAW_FILE" + tr "\\\\" "/" < "$SYMBOLICATE_RAW_FILE" | sed "s,) .* in .*/mcs/,) in mcs/," | sed "s,) .* in .*/external/,) in external/," | sed '/\[MVID\]/d' | sed '/\[AOTID\]/d' > "$SYMBOLICATE_RESULT_FILE" + + DIFF=$(diff -up "$SYMBOLICATE_EXPECTED_FILE" "$SYMBOLICATE_RESULT_FILE") + if [ ! -z "$DIFF" ]; then + echo "ERROR: Symbolicate tests failed." + echo "If $SYMBOLICATE_RESULT_FILE is correct copy it to $SYMBOLICATE_EXPECTED_FILE." + echo "Otherwise runtime sequence points need to be fixed." + echo "" + echo "$DIFF" + ok=false + else + echo "Success." + fi + done + + if [ "$ok" = "true" ]; then + echo "" > "${xunit_results_path}"; + exit 0 + else + echo "" > "${xunit_results_path}"; + exit 1 + fi + +fi + +if [ "$test_suite" = "--csi" ]; then + cd tests/csi || error 1 + echo "Console.WriteLine (\"hello world: \" + DateTime.Now)" > csi-test.csx + + ok=true + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" csi.exe csi-test.csx > csi-test-output.txt || ok=false + cat csi-test-output.txt && grep -q "hello world" csi-test-output.txt || ok=false + + if [ "$ok" = "true" ]; then + echo "" > "${xunit_results_path}"; + exit 0 + else + echo "" > "${xunit_results_path}"; + exit 1 + fi + +fi + +if [ "$test_suite" = "--profiler" ]; then + cd tests/profiler || exit 1 + + chmod +x "$r/mprof-report" + perl ptestrunner.pl out-of-tree xunit "${xunit_results_path}" + exit $? +fi + +if [ "$test_suite" = "--runtime" ]; then + cd tests/runtime || exit 1 + + # TODO: only ported runtest-managed for now + "${MONO_EXECUTABLE}" --config "$r/_tmpinst/etc/mono/config" --debug test-runner.exe --verbose --xunit "${xunit_results_path}" --config tests-config --runtime "${MONO_EXECUTABLE}" --mono-path "$r/net_4_x" -j a --testsuite-name "runtime" --timeout 300 --disabled "$DISABLED_TESTS" $(cat runtime-test-list.txt) + exit $? +fi diff --git a/scripts/ci/run-jenkins.sh b/scripts/ci/run-jenkins.sh index 49931c9ac14..86bef825bb6 100755 --- a/scripts/ci/run-jenkins.sh +++ b/scripts/ci/run-jenkins.sh @@ -13,6 +13,9 @@ git clean -xdff -- acceptance-tests/external || true source ${MONO_REPO_ROOT}/scripts/ci/util.sh +helix_set_env_vars +helix_send_build_start_event "build/source/$MONO_HELIX_TYPE/" + make_timeout=300m if [[ ${CI_TAGS} == *'clang-sanitizer'* ]]; then @@ -257,10 +260,13 @@ if [[ ${CI_TAGS} == *'linux-ppc64el'* ]]; then make_parallelism=-j1; fi make_continue= if [[ ${CI_TAGS} == *'checked-all'* ]]; then make_continue=-k; fi - if [[ ${CI_TAGS} != *'mac-sdk'* ]]; # Mac SDK builds Mono itself - then - ${TESTCMD} --label=make --timeout=${make_timeout} --fatal make ${make_parallelism} ${make_continue} -w V=1 + then + build_error=0 + ${TESTCMD} --label=make --timeout=${make_timeout} --fatal make ${make_parallelism} ${make_continue} -w V=1 || build_error=1 + helix_send_build_done_event "build/source/$MONO_HELIX_TYPE/" $build_error + + if [[ ${build_error} != 0 ]]; then echo "ERROR: The Mono build failed."; exit ${build_error}; fi fi if [[ ${CI_TAGS} == *'checked-coop'* ]]; then export MONO_CHECK_MODE=gc,thread; fi @@ -274,6 +280,7 @@ elif [[ ${CI_TAGS} == *'stress-tests'* ]]; then ${MONO_REPO_ROOT}/sc elif [[ ${CI_TAGS} == *'interpreter'* ]]; then ${MONO_REPO_ROOT}/scripts/ci/run-test-interpreter.sh; elif [[ ${CI_TAGS} == *'mcs-compiler'* ]]; then ${MONO_REPO_ROOT}/scripts/ci/run-test-mcs.sh; elif [[ ${CI_TAGS} == *'mac-sdk'* ]]; then ${MONO_REPO_ROOT}/scripts/ci/run-test-mac-sdk.sh; +elif [[ ${CI_TAGS} == *'helix-tests'* ]]; then ${MONO_REPO_ROOT}/scripts/ci/run-test-helix.sh; elif [[ ${CI_TAGS} == *'no-tests'* ]]; then echo "Skipping tests."; else make check-ci; fi diff --git a/scripts/ci/run-test-helix.sh b/scripts/ci/run-test-helix.sh new file mode 100755 index 00000000000..bfc54f524e9 --- /dev/null +++ b/scripts/ci/run-test-helix.sh @@ -0,0 +1,35 @@ +#!/bin/bash -e + +source ${MONO_REPO_ROOT}/scripts/ci/util.sh + +helix_send_build_start_event "build/tests/$MONO_HELIX_TYPE/" +${TESTCMD} --label=compile-runtime-tests --timeout=20m --fatal make -w -C mono -j ${CI_CPU_COUNT} test +${TESTCMD} --label=compile-bcl-tests --timeout=40m --fatal make -i -w -C runtime -j ${CI_CPU_COUNT} test xunit-test +# TODO: remove the explicit xbuild tests compile step once the nunitlite dependency bug is fixed +${TESTCMD} --label=compile-xbuild_12 --timeout=5m --fatal make -w -C mcs/class -j ${CI_CPU_COUNT} test PROFILE=xbuild_12 +${TESTCMD} --label=compile-xbuild_14 --timeout=5m --fatal make -w -C mcs/class -j ${CI_CPU_COUNT} test PROFILE=xbuild_14 +${TESTCMD} --label=create-test-payload --timeout=5m --fatal make -w test-bundle TEST_BUNDLE_PATH="$MONO_REPO_ROOT/mono-test-bundle" +helix_send_build_done_event "build/tests/$MONO_HELIX_TYPE/" 0 + +# get libgdiplus from CI output +if [[ ${CI_TAGS} == *'linux-amd64'* ]]; then + wget "https://xamjenkinsartifact.blob.core.windows.net/test-libgdiplus-mainline/280/debian-9-amd64/src/.libs/libgdiplus.so" -O "$MONO_REPO_ROOT/mono-test-bundle/mono-libgdiplus.so" +else + echo "Unknown OS, couldn't determine appropriate libgdiplus." + exit 1 +fi + +export MONO_HELIX_TEST_PAYLOAD_DIRECTORY="$MONO_REPO_ROOT/mono-test-bundle" +export MONO_HELIX_CORRELATION_ID_FILE="$MONO_REPO_ROOT/test-correlation-id.txt" +${TESTCMD} --label=upload-helix-tests --timeout=5m --fatal make -w -C mcs/tools/mono-helix-client upload-to-helix + +# test suites which aren't ported to helix yet +${TESTCMD} --label=mini-seq-points --timeout=5m make -w -C mono/mini -k check-seq-points +${TESTCMD} --label=mini-aotcheck --timeout=5m make -j ${CI_CPU_COUNT} -w -C mono/mini -k aotcheck +${TESTCMD} --label=runtime --timeout=20m make -w -C mono/tests test-wrench IGNORE_TEST_JIT=1 V=1 +${TESTCMD} --label=runtime-unit-tests --timeout=5m make -w -C mono/unit-tests -k check +${TESTCMD} --label=monolinker --timeout=10m make -w -C mcs/tools/linker check + +# wait for helix tests to complete +export MONO_HELIX_CORRELATION_ID=$(cat "$MONO_HELIX_CORRELATION_ID_FILE") +${TESTCMD} --label=wait-helix-tests --timeout=40m make -w -C mcs/tools/mono-helix-client wait-for-job-completion diff --git a/scripts/ci/util.sh b/scripts/ci/util.sh index d4b09e8a437..809a750d110 100644 --- a/scripts/ci/util.sh +++ b/scripts/ci/util.sh @@ -9,3 +9,64 @@ function report_github_status { wget -qO- --header "Content-Type: application/json" --post-data "{\"state\": \"$1\", \"context\":\"$2\", \"description\": \"$3\", \"target_url\": \"$4\"}" "https://api.github.com/repos/mono/mono/statuses/${ghprbActualCommit}?access_token=${GITHUB_STATUS_AUTH_TOKEN}" } + +function helix_set_env_vars { + if [[ ${CI_TAGS} != *'helix'* ]]; then return 0; fi; + + if [[ ${CI_TAGS} == *'-i386'* ]]; then export MONO_HELIX_ARCHITECTURE="x86"; + elif [[ ${CI_TAGS} == *'-amd64'* ]]; then export MONO_HELIX_ARCHITECTURE="x64"; + elif [[ ${CI_TAGS} == *'-arm64'* ]]; then export MONO_HELIX_ARCHITECTURE="arm64"; + elif [[ ${CI_TAGS} == *'-armel'* ]]; then export MONO_HELIX_ARCHITECTURE="armel"; + elif [[ ${CI_TAGS} == *'-armhf'* ]]; then export MONO_HELIX_ARCHITECTURE="armhf"; + else echo "Couldn't determine architecture for Helix."; return 1; fi + + if [[ ${CI_TAGS} == *'linux-'* ]]; then export MONO_HELIX_OPERATINGSYSTEM="Debian 9"; + else echo "Couldn't determine operating system for Helix."; return 1; fi + + if [[ ${CI_TAGS} == *'pull-request'* ]]; then + export MONO_HELIX_CREATOR="$ghprbPullAuthorLogin" + export MONO_HELIX_TARGET_QUEUE="debian.9.amd64.open" + export MONO_HELIX_SOURCE="pr/jenkins/mono/mono/$ghprbTargetBranch/" + export MONO_HELIX_BUILD_MONIKER="$(git rev-parse HEAD)" + else + version_number=$(grep AC_INIT configure.ac | sed -e 's/AC_INIT(mono, \[//' -e 's/\],//') + major_ver=$(echo "$version_number" | cut -d . -f 1) + minor_ver=$(echo "$version_number" | cut -d . -f 2) + build_ver=$(echo "$version_number" | cut -d . -f 3) + blame_rev=$(git blame configure.ac HEAD | grep AC_INIT | sed 's/ .*//') + patch_ver=$(git log "$blame_rev"..HEAD --oneline | wc -l | sed 's/ //g') + export MONO_HELIX_CREATOR="monojenkins" + export MONO_HELIX_TARGET_QUEUE="debian.9.amd64" + export MONO_HELIX_SOURCE="official/mono/mono/$MONO_BRANCH/v4/" + export MONO_HELIX_BUILD_MONIKER=$(printf %d.%d.%d.%d "$major_ver" "$minor_ver" "$build_ver" "$patch_ver") + fi +} + +function helix_send_build_start_event { + if [[ ${CI_TAGS} != *'helix-telemetry'* ]]; then return 0; fi; + if [ -z "$MONO_HELIX_API_KEY" ]; then echo "No Helix API key, skipping telementry event."; return 0; fi; + if [ -z "$1" ]; then echo "No type."; return 1; fi; + + mkdir -p "helix-telemetry/${1}" + wget -O- --method="POST" --header='Content-Type: application/json' --header='Accept: application/json' --body-data="{ + \"QueueId\": \"${MONO_HELIX_TARGET_QUEUE}\", + \"Source\": \"${MONO_HELIX_SOURCE}\", + \"Type\": \"${1}\", + \"Build\": \"${MONO_HELIX_BUILD_MONIKER}\", + \"Properties\": { \"architecture\": \"${MONO_HELIX_ARCHITECTURE}\", \"operatingSystem\": \"${MONO_HELIX_OPERATINGSYSTEM}\"} + }" "https://helix.dot.net/api/2018-03-14/telemetry/job?access_token=${MONO_HELIX_API_KEY}" > "helix-telemetry/${1}/job-token.txt" + helix_job_token=$(cat "helix-telemetry/${1}/job-token.txt" | sed 's/"//g') + + wget -O- --method="POST" --header='Accept: application/json' --header="X-Helix-Job-Token: ${helix_job_token}" "https://helix.dot.net/api/2018-03-14/telemetry/job/build?buildUri=${BUILD_URL}" > "helix-telemetry/${1}/build-id.txt" +} + +function helix_send_build_done_event { + if [[ ${CI_TAGS} != *'helix-telemetry'* ]]; then return 0; fi; + if [ -z "$MONO_HELIX_API_KEY" ]; then echo "No Helix API key, skipping telementry event."; return 0; fi; + if [ -z "$1" ]; then echo "No type."; return 1; fi; + if [ -z "$2" ]; then echo "No error count."; return 1; fi; + + helix_job_token=$(cat "helix-telemetry/${1}/job-token.txt" | sed 's/"//g') + helix_build_id=$(cat "helix-telemetry/$1/build-id.txt" | sed 's/"//g') + wget -O- --method="POST" --header='Accept: application/json' --header="X-Helix-Job-Token: ${helix_job_token}" "https://helix.dot.net/api/2018-03-14/telemetry/job/build/${helix_build_id}/finish?errorCount=${2}&warningCount=0" +} diff --git a/support/Makefile.am b/support/Makefile.am index 318bdacda54..4813c345a55 100644 --- a/support/Makefile.am +++ b/support/Makefile.am @@ -129,6 +129,10 @@ libMonoSupportW_la_SOURCES = \ libMonoSupportW_la_LIBADD = \ $(glib_libs) +test-bundle-local: + mkdir -p $(TEST_BUNDLE_PATH) + cp -L .libs/libMonoPosixHelper$(libsuffix) $(TEST_BUNDLE_PATH)/ + # # Use this target to refresh the values in map.[ch] # -- 2.11.4.GIT