From 2522547cd5ba592e21fdba2e94a9feeb20037daf Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Thu, 28 Mar 2019 19:00:34 +0200 Subject: [PATCH] [netcore] Improve xtest command (#13703) The python script now also extracts downloaded zips and `make xtest-%` is now able to consume these artifacts. E.g. ``` make xtest-System.Collections.Tests COREFX_ROOT=/prj/corefx-repo ``` We still need COREFX_ROOT because of two missing files: `CoreFx.Private.TestUtilities.dll` and `xunit.console.deps.json` maybe I should just add them to our repo? Also this PR fixes `RemoteExecutorConsoleApp` problems for `make xtest-%` --- netcore/.gitignore | 5 ++- netcore/Makefile.am | 50 ++++++++++++++++++---- netcore/dl-test-assets.py | 16 +++++-- .../excludes-System.Diagnostics.Process.Tests.rsp | 1 + ...s-System.Reflection.Emit.ILGeneration.Tests.rsp | 1 + netcore/excludes-System.Runtime.Tests.rsp | 1 + netcore/excludes-System.Threading.Tests.rsp | 2 + netcore/excludes-System.Xml.XmlSchemaSet.Tests.rsp | 1 + 8 files changed, 64 insertions(+), 13 deletions(-) create mode 100644 netcore/excludes-System.Diagnostics.Process.Tests.rsp create mode 100644 netcore/excludes-System.Reflection.Emit.ILGeneration.Tests.rsp create mode 100644 netcore/excludes-System.Threading.Tests.rsp create mode 100644 netcore/excludes-System.Xml.XmlSchemaSet.Tests.rsp diff --git a/netcore/.gitignore b/netcore/.gitignore index 8130516f7da..ff86037dcc9 100644 --- a/netcore/.gitignore +++ b/netcore/.gitignore @@ -6,4 +6,7 @@ sdk dotnet* netcoreapp2.0/ netcoreapp3.0/ -packs/ \ No newline at end of file +packs/ +.stamp-dl-test-assets +assets/ +corefx-test-assets.xml diff --git a/netcore/Makefile.am b/netcore/Makefile.am index 76acc475d34..72c6ed49d7c 100644 --- a/netcore/Makefile.am +++ b/netcore/Makefile.am @@ -88,14 +88,48 @@ check: $(addprefix run-, $(TEST_SUITES)) clean: rm -rf sdk shared host dotnet tests LICENSE.txt ThirdPartyNotices.txt $(NETCORESDK_FILE) -xtest-%: prepare check-env - COMPlus_DebugWriteToStdErr=1 ./dotnet --fx-version "$(VERSION)" \ - $(COREFX_BINDIR)/testhost/netcoreapp-OSX-Debug-x64/shared/Microsoft.NETCore.App/9.9.9/xunit.console.dll \ - $(COREFX_BINDIR)/$*/$($*_PROFILE)/$*.dll -notrait category=outerloop -notrait category=nonosxtests -notrait category=failing \ - -notrait category=nonnetcoreapptests -noappdomain -noshadow -parallel all -nunit TestResult-$*-netcore-xunit.xml \ - $(shell grep -v '^#\|^$$' excludes-$*.rsp) $(FIXTURE) - -xtest-all: $(addprefix xtest-, $(TEST_SUITES)) +# e.g. `make xtest-System.Collections.Tests COREFX_ROOT=/prj/corefx` +# '-parallel none -verbose' for debug, `-parallel all` to run all tests in parallel +# we need COREFX_ROOT only for xunit.console.deps.json and CoreFx.Private.TestUtilities.dll +xtest-%: prepare check-env dl-test-assets + ln -sf $(CURDIR)/sdk/$(NETCORESDK_VERSION)/System.Text.Encoding.CodePages.dll assets/extracted/$*/ + cp $(COREFX_BINDIR)/runtime/netcoreapp-OSX-Debug-x64/xunit.console.deps.json assets/extracted/$*/ + cp $(COREFX_BINDIR)/runtime/netcoreapp-OSX-Debug-x64/CoreFx.Private.TestUtilities.dll assets/extracted/$*/ + sed -i -e 's/9.9.9/$(VERSION)/g' assets/extracted/$*/*.runtimeconfig.json + cd assets/extracted/$* && \ + COMPlus_DebugWriteToStdErr=1 $(CURDIR)/./dotnet --fx-version "$(VERSION)" xunit.console.dll $*.dll \ + -notrait category=outerloop -notrait category=nonosxtests -notrait category=failing \ + -notrait category=nonnetcoreapptests -noappdomain -noshadow -parallel all -verbose \ + -html ../../../TestResult-$*.html -nunit ../../TestResult-$*-netcore-xunit.xml \ + $(shell if [ -a $(CURDIR)/excludes-$*.rsp ]; then grep -v '^#\|^$$' $(CURDIR)/excludes-$*.rsp; fi;) \ + $(FIXTURE) || true + +# these tests won't be included in `xtestall` (some of them crash runtime, some hang) +EXCLUDED_COREFX_TESTS = \ + Microsoft.CSharp.Tests \ + Microsoft.XmlSerializer.Generator.Tests \ + System.ComponentModel.Composition.Tests \ + System.Diagnostics.StackTrace.Tests \ + System.Diagnostics.Tracing.Tests \ + System.Dynamic.Runtime.Tests \ + System.IO.Ports.Tests \ + System.Linq.Expressions.Tests \ + System.Linq.Queryable.Tests \ + System.Net.Http.Functional.Tests \ + System.Net.HttpListener.Tests \ + System.Net.NameResolution.Functional.Tests \ + System.Net.Sockets.Tests \ + System.Reflection.Tests \ + System.Runtime.InteropServices.Tests \ + System.Runtime.Serialization.Formatters.Tests \ + System.Runtime.Serialization.Json.Tests \ + System.Runtime.Serialization.Xml.Tests \ + System.Threading.Timer.Tests \ + System.Xml.XmlSerializer.Tests \ + System.Xml.Xsl.XslCompiledTransformApi.Tests + +xtestall: $(foreach workingtest, $(foreach test, $(wildcard assets/extracted/*), \ + $(filter-out $(EXCLUDED_COREFX_TESTS), $(notdir $(test)))), $(addprefix xtest-, $(workingtest))) FEED_BASE_URL = https://dotnetfeed.blob.core.windows.net/dotnet-core TEST_ASSETS_URL = https://dotnetfeed.blob.core.windows.net/dotnet-core/corefx-tests/4.6.0-preview4.19156.10/OSX.x64/netcoreapp/corefx-test-assets.xml diff --git a/netcore/dl-test-assets.py b/netcore/dl-test-assets.py index a1e28c67a58..357da1c1040 100644 --- a/netcore/dl-test-assets.py +++ b/netcore/dl-test-assets.py @@ -1,8 +1,10 @@ #!/usr/bin/env python import sys +import os import subprocess import xml.etree.ElementTree as ET +import zipfile if len(sys.argv) < 4: print("Usage: dl-test-assets.py ") @@ -17,9 +19,15 @@ root = tree.getroot() for elem in root: if elem.tag != "Blob": continue - print elem.attrib ["Id"] - res = subprocess.call (["wget", "-N", "-P", outdir, base_url + "/" + elem.attrib ["Id"]]) + id = elem.attrib ["Id"] + filename = os.path.basename(id) + # System.IO.Compression.* have a lot of test files (>100mb) + if "System.IO.Compression" in filename: + continue + print filename + res = subprocess.call (["wget", "-N", "-P", outdir, base_url + "/" + id]) if res != 0: - print ("Download failed.") + print ("Download failed for " + id) sys.exit (1) - + with zipfile.ZipFile(outdir + "/" + filename) as zf: + zf.extractall(outdir + "/extracted/" + filename[:-4]) diff --git a/netcore/excludes-System.Diagnostics.Process.Tests.rsp b/netcore/excludes-System.Diagnostics.Process.Tests.rsp new file mode 100644 index 00000000000..47f608eef7c --- /dev/null +++ b/netcore/excludes-System.Diagnostics.Process.Tests.rsp @@ -0,0 +1 @@ +-nomethod System.Diagnostics.Tests.ProcessTests.* \ No newline at end of file diff --git a/netcore/excludes-System.Reflection.Emit.ILGeneration.Tests.rsp b/netcore/excludes-System.Reflection.Emit.ILGeneration.Tests.rsp new file mode 100644 index 00000000000..7849087c371 --- /dev/null +++ b/netcore/excludes-System.Reflection.Emit.ILGeneration.Tests.rsp @@ -0,0 +1 @@ +-noclass System.Reflection.Emit.Tests.CustomAttributeBuilderTests \ No newline at end of file diff --git a/netcore/excludes-System.Runtime.Tests.rsp b/netcore/excludes-System.Runtime.Tests.rsp index 19a43d53eb7..8728b2c4d82 100644 --- a/netcore/excludes-System.Runtime.Tests.rsp +++ b/netcore/excludes-System.Runtime.Tests.rsp @@ -101,3 +101,4 @@ -noclass System.Tests.ArgIteratorTests -nomethod System.Tests.ActivatorTests.CreateInstance_InvalidType_ThrowsNotSupportedException -nomethod System.Tests.ArrayTests.CreateInstance_NotSupportedType_ThrowsNotSupportedException +-noclass System.Reflection.Tests.PointerTests diff --git a/netcore/excludes-System.Threading.Tests.rsp b/netcore/excludes-System.Threading.Tests.rsp new file mode 100644 index 00000000000..45b0e99c0c6 --- /dev/null +++ b/netcore/excludes-System.Threading.Tests.rsp @@ -0,0 +1,2 @@ +System.Threading.Tests.MutexTests.CrossProcess_NamedMutex_ProtectedFileAccessAtomic +System.Threading.Tests.SemaphoreSlimTests.RunSemaphoreSlimTest1_WaitAsync_NegativeCases diff --git a/netcore/excludes-System.Xml.XmlSchemaSet.Tests.rsp b/netcore/excludes-System.Xml.XmlSchemaSet.Tests.rsp new file mode 100644 index 00000000000..0790623b340 --- /dev/null +++ b/netcore/excludes-System.Xml.XmlSchemaSet.Tests.rsp @@ -0,0 +1 @@ +-nomethod System.Xml.Tests.TC_SchemaSet_Misc.* -- 2.11.4.GIT