mcs/build/tests.make

   1 # -*- makefile -*-
   2 #
   3 # Rules for building unit tests.
   4 #
   5 # Includers of this file must define the following values:
   6 #
   7 #   ASSEMBLY
   8 #   ASSEMBLY_EXT
   9 #   the_assembly
  10
  11 tests_CLEAN_FILES :=
  12
  13 ifndef TEST_COMPILE
  14 TEST_COMPILE = $(subst $(test_remove),,$(CSCOMPILE))
  15 endif
  16
  17 TEST_RUNTIME_WRAPPERS_PATH = $(shell dirname $(RUNTIME))/_tmpinst/bin
  18
  19 ## Unit test support
  20 ifndef NO_TEST
  21
  22 test_nunit_lib = nunitlite.dll
  23 xunit_core := xunit.core xunit.execution.desktop xunit.abstractions xunit.assert Xunit.NetCore.Extensions
  24 xunit_deps := System.Runtime
  25 xunit_src  := $(patsubst %,$(topdir)/../external/xunit-binaries/%,BenchmarkAttribute.cs BenchmarkDiscover.cs) $(topdir)/../mcs/class/test-helpers/PlatformDetection.cs
  26
  27 ifeq ($(USE_XTEST_REMOTE_EXECUTOR), YES)
  28 XTEST_REMOTE_EXECUTOR = $(topdir)/class/lib/$(PROFILE)/RemoteExecutorConsoleApp.exe
  29 xunit_src += $(topdir)/../mcs/class/test-helpers/AdminHelper.cs \
  30 $(topdir)/../mcs/class/test-helpers/RemoteExecutorTestBase.Mono.cs \
  31 $(topdir)/../external/corefx/src/CoreFx.Private.TestUtilities/src/System/IO/FileCleanupTestBase.cs \
  32 $(topdir)/../external/corefx/src/CoreFx.Private.TestUtilities/src/System/Diagnostics/RemoteExecutorTestBase.Process.cs \
  33 $(topdir)/../external/corefx/src/CoreFx.Private.TestUtilities/src/System/Diagnostics/RemoteExecutorTestBase.cs \
  34 $(topdir)/../external/corefx/src/Common/src/System/PasteArguments.cs \
  35 $(topdir)/../external/corefx/src/Common/src/System/PasteArguments.Unix.cs
  36 endif
  37
  38 xunit_class_deps :=
  39
  40 xunit_libs_ref = $(patsubst %,-r:$(topdir)/../external/xunit-binaries/%.dll,$(xunit_core))
  41 xunit_libs_ref += $(patsubst %,-r:$(topdir)/class/lib/$(PROFILE)/Facades/%.dll,$(xunit_deps))
  42
  43 xunit_libs_dep = $(xunit_class_deps:%=$(topdir)/class/lib/$(PROFILE)/$(PARENT_PROFILE)%.dll)
  44 xunit_libs_ref += $(xunit_libs_dep:%=-r:%)
  45
  46 TEST_LIB_MCS_FLAGS = $(patsubst %,-r:$(topdir)/class/lib/$(PROFILE)/%.dll,$(TEST_LIB_REFS) $(DEFAULT_REFERENCES))
  47 XTEST_LIB_MCS_FLAGS = $(patsubst %,-r:$(topdir)/class/lib/$(PROFILE)/%.dll,$(XTEST_LIB_REFS) $(DEFAULT_REFERENCES))
  48
  49 test_nunit_dep = $(test_nunit_lib:%=$(topdir)/class/lib/$(PROFILE)/$(PARENT_PROFILE)%)
  50 test_nunit_ref = $(test_nunit_dep:%=-r:%)
  51 tests_CLEAN_FILES += TestResult*.xml
  52
  53 test_sourcefile = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_test.dll.sources)
  54
  55 ifeq ($(wildcard $(test_sourcefile)),)
  56 test_sourcefile = $(ASSEMBLY:$(ASSEMBLY_EXT)=_test.dll.sources)
  57 endif
  58
  59 test_lib = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_test.dll)
  60
  61 test_sourcefile_excludes = $(test_lib).exclude.sources
  62
  63 test_pdb = $(test_lib:.dll=.pdb)
  64 test_response = $(depsdir)/$(test_lib).response
  65 test_makefrag = $(depsdir)/$(test_lib).makefrag
  66 test_flags = $(test_nunit_ref) $(TEST_MCS_FLAGS) $(TEST_LIB_MCS_FLAGS)
  67 ifndef NO_BUILD
  68 test_flags += -r:$(the_assembly)
  69 test_assembly_dep = $(the_assembly)
  70 endif
  71 tests_CLEAN_FILES += $(ASSEMBLY:$(ASSEMBLY_EXT)=_test*.dll) $(ASSEMBLY:$(ASSEMBLY_EXT)=_test*.pdb) $(test_response) $(test_makefrag)
  72
  73 xtest_sourcefile = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_xtest.dll.sources)
  74 xtest_sourcefile_excludes = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_xtest.dll.exclude.sources)
  75
  76 xunit_test_lib = $(PROFILE)_$(ASSEMBLY:$(ASSEMBLY_EXT)=_xunit-test.dll)
  77
  78 xtest_response = $(depsdir)/$(xunit_test_lib).response
  79 xtest_makefrag = $(depsdir)/$(xunit_test_lib).makefrag
  80 xtest_flags = -r:$(the_assembly) $(xunit_libs_ref) $(XTEST_MCS_FLAGS) $(XTEST_LIB_MCS_FLAGS) /unsafe
  81
  82 ifeq ($(wildcard $(xtest_sourcefile)),)
  83 xtest_sourcefile = $(ASSEMBLY:$(ASSEMBLY_EXT)=_xtest.dll.sources)
  84 tests_CLEAN_FILES += $(xunit_test_lib) $(xtest_response) $(xtest_makefrag)
  85 endif
  86
  87 ifndef HAVE_CS_TESTS
  88 HAVE_CS_TESTS := $(wildcard $(test_sourcefile))
  89 endif
  90
  91 HAVE_CS_XTESTS := $(wildcard $(xtest_sourcefile))
  92
  93 endif # !NO_TEST
  94
  95 ifndef NO_TEST
  96 $(test_nunit_dep): $(topdir)/build/deps/nunit-$(PROFILE).stamp
  97         @if test -f $@; then :; else rm -f $<; $(MAKE) $<; fi
  98
  99 $(topdir)/build/deps/nunit-$(PROFILE).stamp:
 100 ifndef PARENT_PROFILE
 101         cd ${topdir}/tools/nunit-lite && $(MAKE)
 102 endif
 103         echo "stamp" >$@
 104
 105 tests_CLEAN_FILES += $(topdir)/build/deps/nunit-$(PROFILE).stamp
 106
 107 endif
 108
 109 test_assemblies :=
 110
 111 test_lib_dir = $(topdir)/class/lib/$(PROFILE)/tests
 112
 113 test_lib_output = $(topdir)/class/lib/$(PROFILE)/tests/$(test_lib)
 114
 115 ifdef HAVE_CS_TESTS
 116 test_assemblies += $(test_lib_output)
 117
 118 check: run-test
 119 test-local: $(test_assemblies)
 120 run-test-local: run-test-lib
 121 run-test-ondotnet-local: run-test-ondotnet-lib
 122
 123 ifdef TEST_WITH_INTERPRETER
 124 TEST_HARNESS_EXCLUDES = -exclude=$(PLATFORM_TEST_HARNESS_EXCLUDES)$(PROFILE_TEST_HARNESS_EXCLUDES)NotWorking,NotWorkingRuntimeInterpreter,CAS
 125 else
 126 TEST_HARNESS_EXCLUDES = -exclude=$(PLATFORM_TEST_HARNESS_EXCLUDES)$(PROFILE_TEST_HARNESS_EXCLUDES)NotWorking,CAS
 127 endif
 128
 129 TEST_HARNESS_EXCLUDES_ONDOTNET = /exclude:$(PLATFORM_TEST_HARNESS_EXCLUDES)$(PROFILE_TEST_HARNESS_EXCLUDES)NotDotNet,CAS
 130
 131 NOSHADOW_FLAG =
 132
 133 ifdef FIXTURE
 134 FIXTURE_ARG = -test=MonoTests.$(FIXTURE)
 135 endif
 136
 137 ifdef TESTNAME
 138 TESTNAME_ARG = -test=MonoTests.$(TESTNAME)
 139 endif
 140
 141 ifdef TEST_HARNESS_VERBOSE
 142 LABELS_ARG = -labels
 143 endif
 144
 145 ifdef ALWAYS_AOT
 146 test-local-aot-compile: $(topdir)/build/deps/nunit-$(PROFILE).stamp $(test_assemblies)
 147         PATH="$(TEST_RUNTIME_WRAPPERS_PATH):$(PATH)" MONO_REGISTRY_PATH="$(HOME)/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $(TEST_RUNTIME) $(TEST_RUNTIME_FLAGS) $(AOT_BUILD_FLAGS) $(test_assemblies)
 148
 149 else
 150 test-local-aot-compile: $(topdir)/build/deps/nunit-$(PROFILE).stamp $(test_assemblies)
 151
 152 endif # ALWAYS_AOT
 153
 154 ifdef COVERAGE
 155 TEST_COVERAGE_FLAGS = -O=-aot --profile=coverage:output=$(topdir)/class/lib/$(PROFILE_DIRECTORY)/coverage_nunit_$(ASSEMBLY).xml
 156 endif
 157
 158 NUNITLITE_CONFIG_FILE=$(topdir)/class/lib/$(PROFILE)/$(PARENT_PROFILE)nunit-lite-console.exe.config
 159
 160 patch-nunitlite-appconfig:
 161         cp -f $(topdir)/tools/nunit-lite/nunit-lite-console/nunit-lite-console.exe.config.tmpl $(NUNITLITE_CONFIG_FILE)
 162 ifdef TEST_NUNITLITE_APP_CONFIG_GLOBAL
 163         sed -i -e "/__INSERT_CUSTOM_APP_CONFIG_GLOBAL__/r $(TEST_NUNITLITE_APP_CONFIG_GLOBAL)" $(NUNITLITE_CONFIG_FILE)
 164 endif
 165 ifdef TEST_NUNITLITE_APP_CONFIG_RUNTIME
 166         sed -i -e "/__INSERT_CUSTOM_APP_CONFIG_RUNTIME__/r $(TEST_NUNITLITE_APP_CONFIG_RUNTIME)" $(NUNITLITE_CONFIG_FILE)
 167 endif
 168
 169 ifdef PLATFORM_AOT_SUFFIX
 170
 171 DEDUP_DUMMY_CS=$(topdir)/class/lib/$(PROFILE)/DedupInflatedMethods.cs
 172 DEDUP_DUMMY=$(topdir)/class/lib/$(PROFILE)/DedupInflatedMethods.dll
 173
 174 $(DEDUP_DUMMY):
 175         echo " // Empty Assembly \n\n" > $(DEDUP_DUMMY_CS)
 176         $(CSCOMPILE) -t:library -out:$(DEDUP_DUMMY) $(DEDUP_DUMMY_CS)
 177         rm $(DEDUP_DUMMY_CS)
 178
 179 MKBUNDLE_TEST_BIN = $(TEST_HARNESS).static
 180 MKBUNDLE_EXE = $(topdir)/class/lib/$(PROFILE)/mkbundle.exe
 181 # Pattern based on the one in AOT_PROFILE_ASSEMBLIES
 182 # It's easier if you read it backwards.
 183 # What we do here is get the files in the profile directory that end in "test.dll" or are prefixed with nunit (filter)
 184 # and then strip out everything that we expect to live outside the top level (filter-out)
 185 TEST_ASSEMBLIES:=$(sort $(patsubst .//%,%,$(filter-out %.exe.static %.dll.dll %.exe.dll %bare% %plaincore% %secxml% %Facades% %ilasm%,$(filter %.dll,$(wildcard $(topdir)/class/lib/$(PROFILE)/tests/*)))))
 186
 187 $(MKBUNDLE_EXE): $(topdir)/tools/mkbundle/mkbundle.cs
 188         make -C $(topdir)/tools/mkbundle
 189
 190 mkbundle-all-tests:
 191         $(Q_AOT) $(MAKE) -C $(topdir)/class do-test
 192         $(Q_AOT) $(MAKE) -C $(topdir)/tools/mkbundle
 193         $(Q_AOT) $(MAKE) $(MKBUNDLE_TEST_BIN) # recursive make re-computes variables for TEST_ASSEMBLIES
 194
 195 ifdef MKBUNDLE_DEDUP
 196 MKBUNDLE_DEDUP_COND := $(DEDUP_DUMMY)
 197 DEDUP_ARGS=--aot-dedup $(DEDUP_DUMMY)
 198 endif
 199
 200 $(MKBUNDLE_TEST_BIN): $(TEST_ASSEMBLIES) $(TEST_HARNESS) $(MKBUNDLE_EXE) $(MKBUNDLE_DEDUP_COND)
 201         $(Q_AOT) MONO_PATH="$(topdir)/class/lib/$(BUILD_TOOLS_PROFILE)" PKG_CONFIG_PATH="$(topdir)/../data" $(RUNTIME) $(RUNTIME_FLAGS) $(MKBUNDLE_EXE) -L $(topdir)/class/lib/$(PROFILE) -v --deps $(TEST_HARNESS) $(TEST_ASSEMBLIES) -o $(MKBUNDLE_TEST_BIN) --aot-mode $(AOT_MODE) --aot-runtime $(RUNTIME) --aot-args $(AOT_BUILD_ATTRS) --in-tree $(topdir)/.. --managed-linker $(topdir)/class/lib/$(BUILD_TOOLS_PROFILE)/monolinker.exe --config $(topdir)/../data/config --i18n all $(DEDUP_ARGS) --keeptemp
 202
 203 endif # PLATFORM_AOT_SUFFIX
 204
 205 ifneq ($(wildcard $(MKBUNDLE_TEST_BIN)),)
 206 TEST_HARNESS_EXEC=$(MKBUNDLE_TEST_BIN)
 207 TEST_HARNESS_EXCLUDES:=$(TEST_HARNESS_EXCLUDES),StaticLinkedAotNotWorking
 208 else
 209 TEST_HARNESS_EXEC=$(TEST_RUNTIME) $(TEST_RUNTIME_FLAGS) $(TEST_COVERAGE_FLAGS) $(AOT_RUN_FLAGS) $(TEST_HARNESS)
 210 endif
 211
 212 ## FIXME: i18n problem in the 'sed' command below
 213 run-test-lib: test-local test-local-aot-compile patch-nunitlite-appconfig
 214         ok=:; \
 215         PATH="$(TEST_RUNTIME_WRAPPERS_PATH):$(PATH)" MONO_REGISTRY_PATH="$(HOME)/.mono/registry" MONO_TESTS_IN_PROGRESS="yes" $(TEST_HARNESS_EXEC) $(test_assemblies) $(NOSHADOW_FLAG) $(TEST_HARNESS_FLAGS) $(LOCAL_TEST_HARNESS_FLAGS) $(TEST_HARNESS_EXCLUDES) $(LABELS_ARG) -format:nunit2 -result:TestResult-$(PROFILE).xml $(FIXTURE_ARG) $(TESTNAME_ARG)|| ok=false; \
 216         if [ ! -f "TestResult-$(PROFILE).xml" ]; then echo "<?xml version='1.0' encoding='utf-8'?><test-results failures='1' total='1' not-run='0' name='bcl-tests' date='$$(date +%F)' time='$$(date +%T)'><test-suite name='$(strip $(test_assemblies))' success='False' time='0'><results><test-case name='$(notdir $(strip $(test_assemblies))).crash' executed='True' success='False' time='0'><failure><message>The test runner didn't produce a test result XML, probably due to a crash of the runtime. Check the log for more details.</message><stack-trace></stack-trace></failure></test-case></results></test-suite></test-results>" > TestResult-$(PROFILE).xml; fi; \
 217         $$ok
 218
 219 ## Instructs compiler to compile to target .net execution, it can be usefull in rare cases when runtime detection is not possible
 220 run-test-ondotnet-lib: LOCAL_TEST_COMPILER_ONDOTNET_FLAGS:=-d:RUN_ONDOTNET
 221 run-test-ondotnet-lib: test-local
 222         ok=:; \
 223         $(TEST_HARNESS) $(test_assemblies) $(NOSHADOW_FLAG) $(TEST_HARNESS_FLAGS) $(LOCAL_TEST_HARNESS_ONDOTNET_FLAGS) $(TEST_HARNESS_EXCLUDES_ONDOTNET) $(LABELS_ARG) -format:nunit2 -result:TestResult-ondotnet-$(PROFILE).xml $(FIXTURE_ARG) $(TESTNAME_ARG) || ok=false; \
 224         $$ok
 225
 226
 227 endif
 228
 229 TEST_FILES =
 230
 231 ifdef HAVE_CS_TESTS
 232 TEST_FILES += `sed -e '/^$$/d' -e 's,^../,,' -e '/^\#.*$$/d' -et -e 's,^,Test/,' $(test_sourcefile)`
 233 endif
 234
 235 ifdef HAVE_CS_TESTS
 236
 237 $(test_lib_dir):
 238         mkdir -p $@
 239
 240 $(test_lib_output): $(test_assembly_dep) $(test_response) $(test_nunit_dep) $(test_lib_dir)
 241         $(TEST_COMPILE) $(LIBRARY_FLAGS) -target:library -out:$@ $(test_flags) $(LOCAL_TEST_COMPILER_ONDOTNET_FLAGS) @$(test_response)
 242
 243 test_response_preprocessed = $(test_response)_preprocessed
 244
 245 # This handles .excludes/.sources pairs, as well as resolving the
 246 # includes that occur in .sources files
 247 $(test_response_preprocessed): $(test_sourcefile) $(wildcard $(test_sourcefile_excludes))
 248         $(SHELL) $(topdir)/build/gensources.sh $@ '$(test_sourcefile)' '$(test_sourcefile_excludes)'
 249
 250 $(test_response): $(test_response_preprocessed)
 251 #       @echo Creating $@ ...
 252         @sed -e '/^$$/d' -e 's,^,Test/,' $(test_response_preprocessed) | $(PLATFORM_CHANGE_SEPARATOR_CMD) >$@
 253
 254 $(test_makefrag): $(test_response)
 255 #       @echo Creating $@ ...
 256         @sed 's,^,$(test_lib_output): ,' $< >$@
 257
 258 -include $(test_makefrag)
 259
 260 build-test-lib: $(test_lib_output)
 261         @echo Building testing lib
 262
 263 endif
 264
 265
 266 ifdef HAVE_CS_XTESTS
 267
 268 XTEST_HARNESS_PATH := $(topdir)/../external/xunit-binaries
 269 XTEST_HARNESS = $(XTEST_HARNESS_PATH)/xunit.console.exe
 270 XTEST_HARNESS_FLAGS := -noappdomain -noshadow -parallel none -nunit TestResult-$(PROFILE)-xunit.xml
 271 XTEST_TRAIT := -notrait category=failing -notrait category=nonmonotests -notrait Benchmark=true -notrait category=outerloop
 272 # The logic is double inverted so this actually excludes tests not intented for current platform
 273 # best to search for `property name="category"` in the xml output to see what's going on
 274 # https://github.com/dotnet/buildtools/blob/master/src/xunit.netcore.extensions/Discoverers/PlatformSpecificDiscoverer.cs
 275 XTEST_TRAIT_PLATFORM := -notrait category=non$(XTEST_PLATFORM)tests
 276
 277 TEST_MONO_PATH := $(TEST_MONO_PATH)$(PLATFORM_PATH_SEPARATOR)$(XTEST_HARNESS_PATH)
 278
 279 ifdef FIXTURE
 280 XTEST_HARNESS_FLAGS += -class $(FIXTURE)
 281 endif
 282
 283 ifdef TESTNAME
 284 XTEST_HARNESS_FLAGS += -method $(TESTNAME)
 285 endif
 286
 287 ifdef COVERAGE
 288 XTEST_COVERAGE_FLAGS = -O=-aot --profile=coverage:output=$(topdir)/class/lib/$(PROFILE_DIRECTORY)/coverage_xunit_$(ASSEMBLY).xml
 289 endif
 290
 291 check: run-xunit-test-local
 292 xunit-test-local: $(xunit_test_lib)
 293 run-xunit-test-local: run-xunit-test-lib
 294
 295 # cp -rf is a HACK for xunit runner to require xunit.execution.desktop.dll file in local folder on .net only
 296 run-xunit-test-lib: xunit-test-local $(XTEST_REMOTE_EXECUTOR)
 297         @cp -rf $(XTEST_HARNESS_PATH)/xunit.execution.desktop.dll xunit.execution.desktop.dll
 298         ok=:; \
 299         PATH="$(TEST_RUNTIME_WRAPPERS_PATH):$(PATH)" REMOTE_EXECUTOR=$(XTEST_REMOTE_EXECUTOR) $(TEST_RUNTIME) $(TEST_RUNTIME_FLAGS) $(XTEST_COVERAGE_FLAGS) $(AOT_RUN_FLAGS) $(XTEST_HARNESS) $(xunit_test_lib) $(XTEST_HARNESS_FLAGS) $(XTEST_TRAIT) $(XTEST_TRAIT_PLATFORM) || ok=false; \
 300         $$ok
 301         @rm -f xunit.execution.desktop.dll
 302
 303 # Some xunit tests want to be executed in a separate process (see RemoteExecutorTestBase)
 304 $(XTEST_REMOTE_EXECUTOR): $(topdir)/../external/corefx/src/Common/tests/System/Diagnostics/RemoteExecutorConsoleApp/RemoteExecutorConsoleApp.cs
 305         $(TEST_COMPILE) -r:$(topdir)/class/lib/$(PROFILE)/mscorlib.dll $< -out:$@
 306
 307 $(xunit_test_lib): $(the_assembly) $(xtest_response) $(xunit_libs_dep) $(xunit_src)
 308         $(TEST_COMPILE) $(LIBRARY_FLAGS) $(XTEST_LIB_FLAGS) -target:library -out:$@ $(xtest_flags) @$(xtest_response) $(xunit_src)
 309
 310 xtest_response_preprocessed = $(xtest_response)_preprocessed
 311
 312 # This handles .excludes/.sources pairs, as well as resolving the
 313 # includes that occur in .sources files
 314 $(xtest_response): $(xtest_sourcefile) $(wildcard $(xtest_sourcefile_excludes))
 315         $(SHELL) $(topdir)/build/gensources.sh $@ '$(xtest_sourcefile)' '$(xtest_sourcefile_excludes)'
 316
 317 $(xtest_makefrag): $(xtest_response)
 318         @echo Creating $@ ...
 319         @sed 's,^,$(xunit_test_lib): ,' $< >$@
 320
 321 -include $(xtest_makefrag)
 322
 323 endif
 324
 325
 326 .PHONY: patch-nunitlite-appconfig