[netcore] Improve xtest command (#13703)
[mono-project.git] / netcore / Makefile.am
blob72c6ed49d7cc094e767c3ce511c9525383ff6086
2 NETCORESDK_VERSION=3.0.100-preview4-010759
3 NETCORESDK_FILE=dotnet-sdk-$(NETCORESDK_VERSION)-osx-x64.tar.gz
4 URL:=https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$(NETCORESDK_VERSION)/$(NETCORESDK_FILE)
6 $(NETCORESDK_FILE):
7         curl $(URL) --output $(NETCORESDK_FILE)
8         tar -xvf $(NETCORESDK_FILE)
10 build-sample:
11         dotnet build sample/HelloWorld
13 run-sample:
14         COMPlus_DebugWriteToStdErr=1 ./dotnet --fx-version "$(VERSION)" sample/HelloWorld/bin/Debug/netcoreapp3.0/HelloWorld.dll
16 # COREHOST_TRACE=1 
17 VERSION:=3.0.0-preview4-27514-06
18 SHAREDRUNTIME:=shared/Microsoft.NETCore.App/$(VERSION)
20 bcl:
21         $(MAKE) -C ../mcs/class/System.Private.CoreLib
22         cp ../mcs/class/System.Private.CoreLib/bin/x64/System.Private.CoreLib.dll $(SHAREDRUNTIME)
24 runtime:
25         $(MAKE) -C ../mono
26         cp ../mono/mini/.libs/libmonosgen-2.0.dylib $(SHAREDRUNTIME)/libcoreclr.dylib
28 link-mono:
29         cp ../mono/mini/.libs/libmonosgen-2.0.dylib $(SHAREDRUNTIME)/libcoreclr.dylib
30         cp ../mcs/class/System.Private.CoreLib/bin/x64/System.Private.CoreLib.{dll,pdb} $(SHAREDRUNTIME)
32 prepare: $(NETCORESDK_FILE) link-mono
34 nupkg:
35         nuget pack runtime.nuspec -properties VERSION=@VERSION@\;RID=@RID@\;PLATFORM_AOT_SUFFIX=@PLATFORM_AOT_SUFFIX@
37 COREFX_BINDIR=$(COREFX_ROOT)/artifacts/bin
39 check-env:
40         @if [ "x$(COREFX_ROOT)" == "x" ]; then echo "Set COREFX_ROOT to the root of the fully built corefx repo."; exit 1; fi
43 # Running tests:
44 #  - Tests are run using dotnet bin/Debug/netcoreapp3.0/<...>.dll
45 # - One of the test helper assemblies contains pinvokes for System.Globalization.Native, so
46 #   the dylib has to be in the current working directory when running, since the runtime will
47 #   try to load the dylib from the same directory as the assembly which refers to it.
48 #   It has to be symlinked otherwise we load 2 copies of it which fails.
49 # - RemoteExecutorConsoleApp.exe needs to be in the current directory, since
50 #   its executed as: 'dotnet RemoteExecutorConsoleApp.exe <test assembly> ...'
51 # - MONO_PATH needs to be set to the bin dir so the remote executor can find the
52 #   test assembly.
53 # - The RemoteExecutorConsoleApp.runtimeconfig.json needs to be changed to use the proper
54 #   framework version.
57 TEST_SUITES = \
58         System.Runtime.Tests \
59         System.Runtime.CompilerServices.Unsafe.Tests \
60         System.Collections.Tests
62 # The binary directory under $(COREFIX_BINDIR)
63 System.Runtime.CompilerServices.Unsafe.Tests_PROFILE = netstandard-Debug
64 System.Runtime.Tests_PROFILE = netcoreapp-Unix-Debug
65 System.Runtime.Tests_XUNIT_ARGS = @../excludes-System.Runtime.Tests.rsp
66 System.Collections.Tests_PROFILE = netcoreapp-Debug
67 System.Reflection.Emit.Tests_PROFILE = netcoreapp-Debug
68 System.Reflection.Emit.Tests_XUNIT_ARGS = @../excludes-System.Reflection.Emit.Tests.rsp
69 System.Reflection.Emit.ILGeneration.Tests_PROFILE = netcoreapp-Debug
70 System.Reflection.Emit.Lightweight.Tests_PROFILE = netcoreapp-Debug
71 System.Threading.Tests_PROFILE = netcoreapp-Debug
72 System.Threading.Thread.Tests_PROFILE = netcoreapp-Debug
74 build-%: check-env
75         cd gen-xunit-runner && dotnet run ../tests $(COREFX_ROOT) $(COREFX_BINDIR)/$*/$($*_PROFILE)/$*.dll -notrait category=nonosxtests -notrait category=failing -notrait category=Outerloop -notrait category=nonnetcoreapptests $($*_XUNIT_ARGS) $(XUNIT_ARGS)
76         cp $(COREFX_BINDIR)/RemoteExecutorConsoleApp/netstandard-Debug/RemoteExecutorConsoleApp.* tests/$*
77         cp $(COREFX_BINDIR)/System.Runtime.Tests/netcoreapp-Unix-Debug/TestLoadAssembly.dll tests/$*
78         sed -e 's/9.9.9/$(VERSION)/g' < tests/$*/RemoteExecutorConsoleApp.runtimeconfig.json > 2 && mv 2 tests/$*/RemoteExecutorConsoleApp.runtimeconfig.json
79         cd tests/$* && dotnet build
81 run-%: check-env
82         cd tests/$* && MONO_PATH=bin/Debug/netcoreapp3.0 MONO_ENV_OPTIONS="--debug --explicit-null-checks" COMPlus_DebugWriteToStdErr=1 ../../dotnet bin/Debug/netcoreapp3.0/$*-runner.dll
84 all: $(addprefix build-, $(TEST_SUITES))
86 check: $(addprefix run-, $(TEST_SUITES))
88 clean:
89         rm -rf sdk shared host dotnet tests LICENSE.txt ThirdPartyNotices.txt $(NETCORESDK_FILE)
91 # e.g. `make xtest-System.Collections.Tests COREFX_ROOT=/prj/corefx`
92 # '-parallel none -verbose' for debug, `-parallel all` to run all tests in parallel
93 # we need COREFX_ROOT only for xunit.console.deps.json and CoreFx.Private.TestUtilities.dll
94 xtest-%: prepare check-env dl-test-assets
95         ln -sf $(CURDIR)/sdk/$(NETCORESDK_VERSION)/System.Text.Encoding.CodePages.dll assets/extracted/$*/
96         cp $(COREFX_BINDIR)/runtime/netcoreapp-OSX-Debug-x64/xunit.console.deps.json assets/extracted/$*/
97         cp $(COREFX_BINDIR)/runtime/netcoreapp-OSX-Debug-x64/CoreFx.Private.TestUtilities.dll assets/extracted/$*/
98         sed -i -e 's/9.9.9/$(VERSION)/g' assets/extracted/$*/*.runtimeconfig.json
99         cd assets/extracted/$* && \
100         COMPlus_DebugWriteToStdErr=1 $(CURDIR)/./dotnet --fx-version "$(VERSION)" xunit.console.dll $*.dll \
101                 -notrait category=outerloop -notrait category=nonosxtests -notrait category=failing \
102                 -notrait category=nonnetcoreapptests -noappdomain -noshadow -parallel all -verbose \
103                 -html ../../../TestResult-$*.html -nunit ../../TestResult-$*-netcore-xunit.xml \
104                 $(shell if [ -a $(CURDIR)/excludes-$*.rsp ]; then grep -v '^#\|^$$' $(CURDIR)/excludes-$*.rsp; fi;) \
105                 $(FIXTURE) || true
107 # these tests won't be included in `xtestall` (some of them crash runtime, some hang)
108 EXCLUDED_COREFX_TESTS = \
109         Microsoft.CSharp.Tests \
110         Microsoft.XmlSerializer.Generator.Tests \
111         System.ComponentModel.Composition.Tests \
112         System.Diagnostics.StackTrace.Tests \
113         System.Diagnostics.Tracing.Tests \
114         System.Dynamic.Runtime.Tests \
115         System.IO.Ports.Tests \
116         System.Linq.Expressions.Tests \
117         System.Linq.Queryable.Tests \
118         System.Net.Http.Functional.Tests \
119         System.Net.HttpListener.Tests \
120         System.Net.NameResolution.Functional.Tests \
121         System.Net.Sockets.Tests \
122         System.Reflection.Tests \
123         System.Runtime.InteropServices.Tests \
124         System.Runtime.Serialization.Formatters.Tests \
125         System.Runtime.Serialization.Json.Tests \
126         System.Runtime.Serialization.Xml.Tests \
127         System.Threading.Timer.Tests \
128         System.Xml.XmlSerializer.Tests \
129         System.Xml.Xsl.XslCompiledTransformApi.Tests
131 xtestall: $(foreach workingtest, $(foreach test, $(wildcard assets/extracted/*), \
132                 $(filter-out $(EXCLUDED_COREFX_TESTS), $(notdir $(test)))), $(addprefix xtest-, $(workingtest)))
134 FEED_BASE_URL = https://dotnetfeed.blob.core.windows.net/dotnet-core
135 TEST_ASSETS_URL = https://dotnetfeed.blob.core.windows.net/dotnet-core/corefx-tests/4.6.0-preview4.19156.10/OSX.x64/netcoreapp/corefx-test-assets.xml
137 corefx-test-assets.xml:
138         wget $(TEST_ASSETS_URL)
140 dl-test-assets: corefx-test-assets.xml .stamp-dl-test-assets
142 .stamp-dl-test-assets:
143         python dl-test-assets.py corefx-test-assets.xml $(FEED_BASE_URL) assets
144         touch $@