netcore/Makefile

   1 ifneq ($(wildcard config.make),)
   2
   3 include config.make
   4
   5 NETCORETESTS_VERSION := $(shell cat ../eng/Versions.props | sed -n 's/.*MicrosoftPrivateCoreFxNETCoreAppVersion>\(.*\)<\/MicrosoftPrivateCoreFxNETCoreAppVersion.*/\1/p' )
   6 NETCOREAPP_VERSION := $(shell cat ../eng/Versions.props | sed -n 's/.*MicrosoftNETCoreAppVersion>\(.*\)<\/MicrosoftNETCoreAppVersion.*/\1/p' )
   7 ROSLYN_VERSION:=  $(shell cat ../eng/Versions.props | sed -n 's/.*MicrosoftNetCompilersVersion>\(.*\)<\/MicrosoftNetCompilersVersion.*/\1/p' )
   8
   9 # Should be automated but not sure how for now
  10 # the name is misleading osx version is used for any unix like systems
  11 # the url is manually extracted from coreclr-outerloop (https://dev.azure.com/dnceng/public/_build?definitionId=98&_a=summary) Build artifacts published page
  12 CORECLR_TESTS:=https://dev.azure.com/dnceng/9ee6d478-d288-47f7-aacc-f6e6d082ae6d/_apis/build/builds/368250/artifacts?artifactName=Tests_OSX_x64_checked_outerloop&api-version=5.2-preview.5&%24format=zip
  13
  14 # runtime version used by $(DOTNET) - local .net core sdk to bootstrap stuff
  15 # it doesn't match NETCOREAPP_VERSION
  16 BOOTSTRAP_RUNTIME = $(shell ls ../.dotnet/shared/Microsoft.NETCore.App | tail -1)
  17
  18 ifeq ($(HOST_PLATFORM),win32)
  19 PLATFORM_AOT_SUFFIX := .dll
  20 PLATFORM_AOT_PREFIX :=
  21 NETCORESDK_EXT = zip
  22 UNZIPCMD = python -c "import zipfile,sys; zipfile.ZipFile(sys.argv[1], 'r').extractall()"
  23 XUNIT_INNER_ARGS = -notrait category=nonwindowstests @../../../../CoreFX.issues_windows.rsp
  24 TESTS_PLATFORM = Windows_NT.x64
  25 ON_RUNTIME_EXTRACT = chmod -R 755 {host,shared,./dotnet}
  26 DOTNET := $(shell powershell -ExecutionPolicy Bypass -Command "./init-tools.ps1")/dotnet.exe
  27 DOTNET := "$(subst \,/,$(DOTNET))"
  28 endif
  29
  30 ifeq ($(HOST_PLATFORM),macos)
  31 PLATFORM_AOT_SUFFIX := .dylib
  32 PLATFORM_AOT_PREFIX := lib
  33 OBJDUMP = objdump -x86-asm-syntax=intel --no-show-raw-insn -no-leading-addr -no-leading-headers -d
  34 NETCORESDK_EXT = tar.gz
  35 UNZIPCMD = tar -xvf
  36 XUNIT_INNER_ARGS = -notrait category=nonosxtests @../../../../CoreFX.issues_mac.rsp
  37 TESTS_PLATFORM = OSX.x64
  38 DOTNET := $(shell ./init-tools.sh | tail -1)
  39 endif
  40
  41 ifeq ($(HOST_PLATFORM),linux)
  42 PLATFORM_AOT_SUFFIX := .so
  43 PLATFORM_AOT_PREFIX := lib
  44 OBJDUMP = objdump -M intel --no-show-raw-insn -d
  45 NETCORESDK_EXT = tar.gz
  46 UNZIPCMD = tar -xvf
  47 XUNIT_INNER_ARGS = -notrait category=nonlinuxtests @../../../../CoreFX.issues_linux.rsp
  48 ifeq ($(COREARCH), arm64)
  49 XUNIT_INNER_ARGS += @../../../../CoreFX.issues_linux_arm64.rsp
  50 endif
  51 TESTS_PLATFORM = Linux.x64
  52 DOTNET := $(shell ./init-tools.sh | tail -1)
  53 endif
  54
  55 NETCORESDK_FILE := dotnet-runtime-$(NETCOREAPP_VERSION)-$(RID).$(NETCORESDK_EXT)
  56 NETCORE_URL := https://dotnetcli.blob.core.windows.net/dotnet/Runtime/$(NETCOREAPP_VERSION)/$(NETCORESDK_FILE)
  57 FEED_BASE_URL := https://dotnetfeed.blob.core.windows.net/dotnet-core
  58 TEST_ASSETS_PATH := corefx-tests/$(NETCORETESTS_VERSION)/$(TESTS_PLATFORM)/netcoreapp/corefx-test-assets.xml
  59
  60 # used to calculate exact version number for generating nupkg
  61 BLAME = $(shell git blame ../configure.ac | grep AC_INIT | cut -f1 -d' ')
  62 VERSTUB = .$(shell git rev-list --count $(BLAME)..HEAD)-preview
  63
  64 all: bcl
  65
  66 $(NETCORESDK_FILE):
  67         curl $(NETCORE_URL) --output $(NETCORESDK_FILE)
  68         rm -rf shared/Microsoft.NETCore.App
  69         $(UNZIPCMD) $(NETCORESDK_FILE)
  70         $(ON_RUNTIME_EXTRACT)
  71
  72 update-corefx: corefx/.stamp-dl-corefx-$(NETCORETESTS_VERSION)
  73
  74 corefx/.stamp-dl-corefx-$(NETCORETESTS_VERSION): corefx-restore.csproj
  75         $(DOTNET) build corefx-restore.csproj --runtime $(RID) --packages corefx/packages -p:MicrosoftPrivateCoreFxNETCoreAppVersion=$(NETCORETESTS_VERSION) -p:OutputPath=corefx/restore/ -p:UseSharedCompilation=false
  76         touch $@
  77
  78 update-roslyn: roslyn/packages/microsoft.net.compilers.toolset/$(ROSLYN_VERSION)/microsoft.net.compilers.toolset.nuspec
  79
  80 roslyn/packages/microsoft.net.compilers.toolset/$(ROSLYN_VERSION)/microsoft.net.compilers.toolset.nuspec:
  81         $(DOTNET) restore roslyn-restore.csproj -p:RoslynVersion=$(ROSLYN_VERSION) --packages roslyn/packages -p:OutputPath=roslyn/restore/ -p:UseSharedCompilation=false
  82
  83 run-sample: prepare
  84         $(DOTNET) build sample/HelloWorld
  85         MONO_ENV_OPTIONS="--debug" COMPlus_DebugWriteToStdErr=1 ./dotnet --fx-version "$(NETCOREAPP_VERSION)" sample/HelloWorld/bin/netcoreapp3.0/HelloWorld.dll
  86
  87 run-sample-dbg: prepare
  88         $(DOTNET) build sample/HelloWorld
  89         MONO_ENV_OPTIONS="--debug --debugger-agent=transport=dt_socket,address=127.0.0.1:1235,server=y,suspend=y" COMPlus_DebugWriteToStdErr=1 ./dotnet --fx-version "$(NETCOREAPP_VERSION)" sample/HelloWorld/bin/netcoreapp3.0/HelloWorld.dll
  90
  91 run-sample-coreclr:
  92         cd sample/HelloWorld && $(DOTNET) run
  93
  94 # build any app in "self contained" mode and patch coreclr.dll and corelib with mono bits
  95 # e.g. `make patch-app APP_PATH=/my/aspnetapp`
  96 patch-app: prepare
  97         @if test -z "$(APP_PATH)"; then echo "APP_PATH is not set"; exit 1; fi
  98         cp NuGet.config $(APP_PATH)
  99         cd $(APP_PATH) && $(DOTNET) publish -c Release -r $(RID) -f netcoreapp3.0
 100         cp ../mono/mini/.libs/libmonosgen-2.0$(PLATFORM_AOT_SUFFIX) $(APP_PATH)/bin/Release/netcoreapp3.0/$(RID)/publish/$(PLATFORM_AOT_PREFIX)coreclr$(PLATFORM_AOT_SUFFIX)
 101         cp System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.dll $(APP_PATH)/bin/Release/netcoreapp3.0/$(RID)/publish
 102         cp System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.pdb $(APP_PATH)/bin/Release/netcoreapp3.0/$(RID)/publish
 103
 104 # makes $(DOTNET) to use mono runtime (to run real-world apps using '$(DOTNET) run')
 105 patch-local-dotnet: prepare
 106         cp ../mono/mini/.libs/libmonosgen-2.0$(PLATFORM_AOT_SUFFIX) ../.dotnet/shared/Microsoft.NETCore.App/$(BOOTSTRAP_RUNTIME)/$(PLATFORM_AOT_PREFIX)coreclr$(PLATFORM_AOT_SUFFIX)
 107         cp System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.dll ../.dotnet/shared/Microsoft.NETCore.App/$(BOOTSTRAP_RUNTIME)
 108         cp System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.pdb ../.dotnet/shared/Microsoft.NETCore.App/$(BOOTSTRAP_RUNTIME)
 109
 110 # Precompile BCL libs in $(DOTNET) using LLVM (requires `--enable-llvm` build)
 111 patch-local-dotnet-aot-llvm: patch-local-dotnet
 112         for assembly in ../.dotnet/shared/Microsoft.NETCore.App/$(BOOTSTRAP_RUNTIME)/*.dll; do \
 113                 echo "[AOT] $$assembly"; \
 114                 PATH="../llvm/usr/bin/:$(PATH)" \
 115                 MONO_ENV_OPTIONS="--aot=llvm,mcpu=native" \
 116                 $(DOTNET) $$assembly ; \
 117         done; \
 118
 119 # run 'dotnet/performance' benchmarks
 120 # e.g. 'make run-benchmarks DOTNET_PERF_REPO=/prj/performance'
 121 # you can append BDN parameters at the end, e.g. ` -- --filter Burgers --keepFiles`
 122 # NOTE: we have to delete a couple of System.Drawing files to make it work
 123 run-benchmarks: patch-local-dotnet
 124         @if test -z "$(DOTNET_PERF_REPO)"; then echo "DOTNET_PERF_REPO is not set"; exit 1; fi
 125         cd $(DOTNET_PERF_REPO)/src/benchmarks/micro/ && \
 126         > corefx/System.Drawing/Perf_Color.cs && \
 127         > corefx/System.Drawing/Perf_Graphics_DrawBeziers.cs && \
 128         > corefx/System.Drawing/Perf_Graphics_Transforms.cs && \
 129         > corefx/System.Drawing/Perf_Image_Load.cs && \
 130         $(DOTNET) run -c Release -f netcoreapp5.0 --cli $(CURDIR)/../.dotnet/dotnet
 131
 132 # run HelloWorld using --aot=llvm (requires `--enable-llvm` build)
 133 run-sample-local-dotnet-llvm: patch-local-dotnet
 134         $(DOTNET) build -c Release sample/HelloWorld
 135         PATH="../llvm/usr/bin/:$(PATH)" \
 136         MONO_ENV_OPTIONS="--aot=llvm,mcpu=native" \
 137         $(DOTNET) sample/HelloWorld/bin/netcoreapp3.0/HelloWorld.dll
 138         $(DOTNET) sample/HelloWorld/bin/netcoreapp3.0/HelloWorld.dll
 139
 140 # COREHOST_TRACE=1
 141 SHAREDRUNTIME := shared/Microsoft.NETCore.App/$(NETCOREAPP_VERSION)
 142
 143 System.Private.CoreLib/src/System/Environment.Mono.cs: System.Private.CoreLib/src/System/Environment.Mono.in config.make
 144         test -n '$(MONO_CORLIB_VERSION)'
 145         sed -e 's,@''MONO_CORLIB_VERSION@,$(MONO_CORLIB_VERSION),' $< > $@
 146
 147 CORLIB_BUILD_FLAGS?=-c Release
 148
 149 bcl: update-roslyn System.Private.CoreLib/src/System/Environment.Mono.cs
 150         $(DOTNET) build $(CORETARGETS) $(CORLIB_BUILD_FLAGS) -p:UseSharedCompilation=false -p:BuildArch=$(COREARCH) \
 151         -p:OutputPath=bin/$(COREARCH) \
 152         -p:RoslynPropsFile="../roslyn/packages/microsoft.net.compilers.toolset/$(ROSLYN_VERSION)/build/Microsoft.Net.Compilers.Toolset.props" \
 153         System.Private.CoreLib/System.Private.CoreLib.csproj
 154
 155 debug-bcl:
 156         $(MAKE) bcl CORLIB_BUILD_FLAGS="-c Debug"
 157
 158 runtime:
 159         $(MAKE) -C ../mono
 160
 161 link-mono: $(SHAREDRUNTIME)/.stamp-link-mono
 162
 163 $(SHAREDRUNTIME)/.stamp-link-mono: ../mono/mini/.libs/libmonosgen-2.0$(PLATFORM_AOT_SUFFIX) System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.dll System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.pdb
 164         cp ../mono/mini/.libs/libmonosgen-2.0$(PLATFORM_AOT_SUFFIX) $(SHAREDRUNTIME)/$(PLATFORM_AOT_PREFIX)coreclr$(PLATFORM_AOT_SUFFIX)
 165         cp System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.dll $(SHAREDRUNTIME)
 166         cp System.Private.CoreLib/bin/$(COREARCH)/System.Private.CoreLib.pdb $(SHAREDRUNTIME)
 167         touch $@
 168
 169 prepare: $(NETCORESDK_FILE) update-corefx update-roslyn link-mono
 170
 171 pack-%:
 172         $(DOTNET) pack roslyn-restore.csproj -p:IsPackable=true -p:NuspecFile=$*.nuspec -p:NuspecProperties=\"RID=$(RID)\;VERSION=$(VERSION)$(VERSTUB)\;PLATFORM_AOT_SUFFIX=$(PLATFORM_AOT_SUFFIX)\;COREARCH=$(COREARCH)\;PLATFORM_AOT_PREFIX=$(PLATFORM_AOT_PREFIX)\" --output ../artifacts/ --no-build
 173
 174 nupkg: pack-metapackage pack-runtime
 175 nupkg-llvm: pack-metapackage-llvm pack-runtime-llvm
 176
 177 clean:
 178         rm -rf .configured ../.dotnet sdk shared host dotnet tests obj corefx roslyn LICENSE.txt ThirdPartyNotices.txt $(NETCORESDK_FILE)
 179
 180
 181 update-tests-corefx: corefx/.stamp-dl-corefx-tests-$(NETCORETESTS_VERSION)
 182
 183 corefx/.stamp-dl-corefx-tests-$(NETCORETESTS_VERSION):
 184         rm -rf corefx/tests
 185         $(DOTNET) msbuild corefx-tests-restore.proj -m -t:DownloadAllTests -p:TEST_ASSETS_PATH="$(TEST_ASSETS_PATH)" -p:FEED_BASE_URL="$(FEED_BASE_URL)"
 186         rm -rf corefx/tests/extracted/System.Utf8String.Experimental.Tests
 187         touch $@
 188
 189 #
 190 # Running CoreFX tests
 191 #
 192 # You can run either individual test assembly or run all CoreFX tests. The tests are automatically downloaded
 193 # from the shared location and built locally.
 194 #
 195 # Running all test: `make run-tests-corefx`
 196 #
 197 # Running individual test: MONO_ENV_OPTIONS="--debug" make run-tests-corefx-System.Collections.Tests
 198 #
 199
 200 run-tests-corefx: prepare update-tests-corefx
 201         @rm -f .failures; \
 202         counter=0; \
 203         tests_count=$(words $(dir $(wildcard corefx/tests/extracted/*/))); \
 204         for testdir in corefx/tests/extracted/*; do \
 205                 counter=$$((counter+1)); \
 206                 testname=$$(basename $$testdir); \
 207                 if [ -n "$$USE_TIMEOUT" ]; then timeoutcmd="../scripts/ci/run-step.sh --label=$$testname --timeout=10m --fatal"; fi; \
 208                 $$timeoutcmd $(MAKE) run-tests-corefx-$$testname TEST_COUNTER="($$counter / $$tests_count) " || echo $$testname >> .failures; \
 209         done; \
 210         $(MAKE) xunit-summary; \
 211         if [ -e ".failures" ]; then \
 212                 echo ""; \
 213                 echo "Failures in test suites:"; \
 214                 cat .failures; \
 215                 echo ""; \
 216                 exit 1; \
 217         fi
 218
 219 run-tests-corefx-%: prepare update-tests-corefx
 220         @echo ""
 221         @echo "***************** $* $${TEST_COUNTER}*********************"
 222         @cp corefx/restore/corefx-restore.deps.json corefx/tests/extracted/$*/$*.deps.json
 223         @cp corefx/restore/corefx-restore.runtimeconfig.dev.json corefx/tests/extracted/$*/$*.runtimeconfig.dev.json
 224         @cp corefx/restore/corefx-restore.dll corefx/tests/extracted/$*/corefx-restore.dll
 225         @sed -i -e 's/5.0.0\"/$(NETCOREAPP_VERSION)\"/g' corefx/tests/extracted/$*/*.runtimeconfig.json
 226         cd corefx/tests/extracted/$* && \
 227         MONO_ENV_OPTIONS="--debug $(XUNIT_MONO_ENV_OPTIONS)" COMPlus_DebugWriteToStdErr=1 $(CURDIR)/./dotnet exec \
 228                 --runtimeconfig $*.runtimeconfig.json --additional-deps $*.deps.json \
 229                 --fx-version "$(NETCOREAPP_VERSION)" xunit.console.dll $*.dll \
 230                 -html ../../TestResult-$*.html -xml ../../TestResult-$*-netcore-xunit.xml \
 231                 $(XUNIT_INNER_ARGS) @../../../../CoreFX.issues.rsp \
 232                 $(XUNIT_ARGS)
 233
 234 # build a test assembly in COREFX_ROOT (path to a fully built corefx repo) and copy it to corefx/tests/extracted
 235 # e.g. `make build-test-corefx-System.Runtime COREFX_ROOT=/prj/corefx-master`
 236 build-test-corefx-%:
 237         cd $(COREFX_ROOT)/src/$*/tests && $(DOTNET) msbuild /p:OutputPath=tmp
 238         cp $(COREFX_ROOT)/src/$*/tests/tmp/$*.Tests.{dll,pdb} $(CURDIR)/corefx/tests/extracted/$*.Tests/
 239         rm -rf $(COREFX_ROOT)/src/$*/tests/tmp
 240
 241 xunit-summary:
 242         ./xunit-summary.py "corefx/tests"
 243
 244
 245 update-tests-coreclr: coreclr/.stamp-tests
 246
 247 coreclr/.stamp-tests:
 248         rm -rf coreclr/tests
 249         curl -L "$(CORECLR_TESTS)" --output coreclr/tests.zip
 250         tar -xvf coreclr/tests.zip -C coreclr/
 251         mkdir coreclr/tests
 252         tar -xvf coreclr/Tests_OSX_x64_checked_outerloop/Tests_OSX_x64_checked_outerloop.tar.gz -C coreclr/tests
 253         touch $@
 254
 255 .PHONY: corerun
 256 corerun:
 257         $(MAKE) -C corerun
 258         cp corerun/corerun $(SHAREDRUNTIME)
 259
 260 run-tests-coreclr: prepare update-tests-coreclr corerun
 261         rm -rf coreclr/output.log
 262         @counter=0; \
 263         failures=0; \
 264         test_files=$$(find coreclr/tests -type f -name "*.sh"); \
 265         for testdir in $$test_files; do \
 266                 counter=$$((counter+1)); \
 267                 echo "Running $$testdir"; \
 268                 if sh $$testdir -coreroot=$(realpath $(SHAREDRUNTIME)) >>coreclr/output.log 2>&1 ; then \
 269                         echo "FAILED"; \
 270                         failures=$$((failures+1)); \
 271                 fi; \
 272         done; \
 273         echo "Tests Count: $$counter"; \
 274         echo "Failed: $$failures"
 275 endif
 276
 277 run-tests-coreclr-%: prepare update-tests-coreclr
 278         @echo ""
 279         @echo "***************** $* *********************"
 280         @test_sh=$$(find . -type f -name "$*.sh" | head -n 1); \
 281         if [ ! -z "$$test_sh" ]; then \
 282                 MONO_ENV_OPTIONS="--debug" COMPlus_DebugWriteToStdErr=1 sh $$test_sh -coreroot="$(realpath $(SHAREDRUNTIME))"; \
 283         else \
 284                 echo "Test file $*.sh not found"; \
 285         fi
 286
 287 # dump asm for all methods in BCL using LLVM AOT, e.g.:
 288 #   make dump-asm-bcl DUMPASM_OUT=dumps/before-my-jit-fix
 289 dump-asm-bcl: patch-local-dotnet
 290         @if test -z "$(DUMPASM_OUT)"; then echo "DUMPASM_OUT is not set"; exit 1; fi
 291         mkdir -p $(DUMPASM_OUT)
 292         for assembly in ../.dotnet/shared/Microsoft.NETCore.App/$(BOOTSTRAP_RUNTIME)/*.dll; do \
 293                 printf  "\n[AOT] $$(basename $$assembly):\n\n"; \
 294                 $(MAKE) dump-asm-lib DUMPASM_LIB=$$assembly ; \
 295         done; \
 296         cd ../.dotnet/shared/Microsoft.NETCore.App/$(BOOTSTRAP_RUNTIME) && rm *.dll$(PLATFORM_AOT_SUFFIX)
 297
 298 # dump asm for a specific assembly using LLVM AOT, e.g.:
 299 #   make dump-asm-lib DUMPASM_OUT=dumps/before-my-jit-fix DUMPASM_LIB=/path/to/System.Private.CoreLib.dll
 300 dump-asm-lib: patch-local-dotnet
 301         @if test -z "$(DUMPASM_LIB)"; then echo "DUMPASM_LIB is not set"; exit 1; fi
 302         @if test -z "$(DUMPASM_OUT)"; then echo "DUMPASM_OUT is not set"; exit 1; fi
 303         mkdir -p $(DUMPASM_OUT)
 304         PATH="../llvm/usr/bin/:$(PATH)" \
 305         MONO_ENV_OPTIONS="--aot=llvm,mcpu=native" \
 306         $(DOTNET) $(DUMPASM_LIB)
 307         $(OBJDUMP) "$(DUMPASM_LIB)$(PLATFORM_AOT_SUFFIX)" > "$(DUMPASM_OUT)/$(notdir $(DUMPASM_LIB))$(PLATFORM_AOT_SUFFIX).dasm"
 308
 309 # Generate asm diffs, a typical workflow may look like this:
 310 #
 311 #   make dump-asm-bcl DUMPASM_OUT=dumps/before-my-jit-fix
 312 # (apply some runtime changes)
 313 #   make runtime
 314 #   make dump-asm-bcl DUMPASM_OUT=dumps/after-my-jit-fix
 315 #   make jitdiff BEFORE=dumps/before-my-jit-fix AFTER=dumps/after-my-jit-fix
 316 #
 317 jitdiff:
 318         @if test -z "$(BEFORE)"; then echo "BEFORE is not set"; exit 1; fi
 319         @if test -z "$(AFTER)"; then echo "AFTER is not set"; exit 1; fi
 320         cd tools/jitdiff && $(DOTNET) run -c Release -- "$(BEFORE)" "$(AFTER)"
 321
 322 distdir:
 323 distclean: