testing/testsuite-targets.mk

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # include automation-build.mk to get the path to the binary
   6 TARGET_DEPTH = $(DEPTH)
   7 include $(topsrcdir)/build/binary-location.mk
   8
   9 SYMBOLS_PATH := --symbols-path=$(DIST)/crashreporter-symbols
  10
  11 ifndef TEST_PACKAGE_NAME
  12 TEST_PACKAGE_NAME := $(ANDROID_PACKAGE_NAME)
  13 endif
  14
  15 ifndef NO_FAIL_ON_TEST_ERRORS
  16 define check_test_error_internal
  17   @errors=`grep 'TEST-UNEXPECTED-' $@.log` ;\
  18   if test "$$errors" ; then \
  19           echo '$@ failed:'; \
  20           echo "$$errors"; \
  21           $(if $(1),echo $(1);) \
  22           exit 1; \
  23   fi
  24 endef
  25 CHECK_TEST_ERROR = $(call check_test_error_internal)
  26 CHECK_TEST_ERROR_RERUN = $(call check_test_error_internal,'To rerun your failures please run "make $@-rerun-failures"')
  27 endif
  28
  29 # Usage: |make [EXTRA_TEST_ARGS=...] *test|.
  30 RUN_REFTEST = rm -f ./$@.log && $(PYTHON3) _tests/reftest/runreftest.py \
  31   --extra-profile-file=$(DIST)/plugins \
  32   $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(1) | tee ./$@.log
  33
  34 REMOTE_REFTEST = rm -f ./$@.log && $(PYTHON3) _tests/reftest/remotereftest.py \
  35   --ignore-window-size \
  36   --app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \
  37   --httpd-path=_tests/modules --suite reftest \
  38   --extra-profile-file=$(topsrcdir)/mobile/android/fonts \
  39   $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(1) | tee ./$@.log
  40
  41 ifeq ($(OS_ARCH),WINNT) #{
  42 # GPU-rendered shadow layers are unsupported here
  43 OOP_CONTENT = --setpref=layers.async-pan-zoom.enabled=true --setpref=browser.tabs.remote.autostart=true --setpref=layers.acceleration.disabled=true
  44 GPU_RENDERING =
  45 else
  46 OOP_CONTENT = --setpref=layers.async-pan-zoom.enabled=true --setpref=browser.tabs.remote.autostart=true
  47 GPU_RENDERING = --setpref=layers.acceleration.force-enabled=true
  48 endif #}
  49
  50 reftest: TEST_PATH?=layout/reftests/reftest.list
  51 reftest:
  52         $(call RUN_REFTEST,'$(topsrcdir)/$(TEST_PATH)')
  53         $(CHECK_TEST_ERROR)
  54
  55 reftest-remote: TEST_PATH?=layout/reftests/reftest.list
  56 reftest-remote:
  57         @if [ '${MOZ_HOST_BIN}' = '' ]; then \
  58         echo 'environment variable MOZ_HOST_BIN must be set to a directory containing host xpcshell'; \
  59     elif [ ! -d ${MOZ_HOST_BIN} ]; then \
  60         echo 'MOZ_HOST_BIN does not specify a directory'; \
  61     elif [ ! -f ${MOZ_HOST_BIN}/xpcshell ]; then \
  62         echo 'xpcshell not found in MOZ_HOST_BIN'; \
  63     else \
  64         ln -s $(abspath $(topsrcdir)) _tests/reftest/tests; \
  65         $(call REMOTE_REFTEST,'tests/$(TEST_PATH)'); \
  66         $(CHECK_TEST_ERROR); \
  67     fi
  68
  69 crashtest: TEST_PATH?=testing/crashtest/crashtests.list
  70 crashtest:
  71         $(call RUN_REFTEST,'$(topsrcdir)/$(TEST_PATH)')
  72         $(CHECK_TEST_ERROR)
  73
  74 jstestbrowser: TESTS_PATH?=test-stage/jsreftest/tests/
  75 jstestbrowser:
  76         $(MAKE) -C $(DEPTH)/config
  77         $(MAKE) stage-jstests
  78         $(call RUN_REFTEST,'$(DIST)/$(TESTS_PATH)/jstests.list' --extra-profile-file=$(DIST)/test-stage/jsreftest/tests/user.js)
  79         $(CHECK_TEST_ERROR)
  80
  81 GARBAGE += $(addsuffix .log,$(MOCHITESTS) reftest crashtest jstestbrowser)
  82
  83 REMOTE_CPPUNITTESTS = \
  84         $(PYTHON3) -u $(topsrcdir)/testing/remotecppunittests.py \
  85           --xre-path=$(DEPTH)/dist/bin \
  86           --localLib=$(DEPTH)/dist/fennec \
  87           --deviceIP=${TEST_DEVICE} \
  88           $(TEST_PATH) $(EXTRA_TEST_ARGS)
  89
  90 # Usage: |make [TEST_PATH=...] [EXTRA_TEST_ARGS=...] cppunittests-remote|.
  91 cppunittests-remote:
  92         $(call REMOTE_CPPUNITTESTS);
  93
  94 # Package up the tests and test harnesses
  95 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
  96
  97 PKG_STAGE = $(DIST)/test-stage
  98
  99 stage-all: \
 100   stage-config \
 101   stage-mach \
 102   stage-extensions \
 103   stage-mochitest \
 104   stage-jstests \
 105   test-packages-manifest \
 106   $(NULL)
 107 ifdef MOZ_WEBRTC
 108 stage-all: stage-steeplechase
 109 endif
 110
 111 ifdef COMPILE_ENVIRONMENT
 112 stage-all: stage-cppunittests
 113 endif
 114
 115 TEST_PKGS_TARGZ := \
 116   common \
 117   condprof \
 118   cppunittest \
 119   mochitest \
 120   reftest \
 121   talos \
 122   raptor \
 123   awsy \
 124   xpcshell \
 125   web-platform \
 126   updater-dep \
 127   jsreftest \
 128   jittest \
 129   $(NULL)
 130
 131 ifdef LINK_GTEST_DURING_COMPILE
 132 stage-all: stage-gtest
 133 TEST_PKGS_TARGZ += gtest
 134 endif
 135
 136 PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.$(2)'
 137
 138 test-packages-manifest:
 139         @rm -f $(MOZ_TEST_PACKAGES_FILE)
 140         $(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE))
 141         $(PYTHON3) $(topsrcdir)/build/gen_test_packages_manifest.py \
 142       --jsshell $(JSSHELL_NAME) \
 143       --dest-file '$(MOZ_TEST_PACKAGES_FILE)' \
 144       $(call PKG_ARG,common,zip) \
 145       $(foreach pkg,$(TEST_PKGS_TARGZ),$(call PKG_ARG,$(pkg),tar.gz))
 146
 147 ifdef UPLOAD_PATH
 148 test_archive_dir = $(UPLOAD_PATH)
 149 else
 150 test_archive_dir = $(DIST)/$(PKG_PATH)
 151 endif
 152
 153 package-tests-prepare-dest:
 154         $(NSINSTALL) -D $(test_archive_dir)
 155
 156 download-wpt-manifest:
 157         $(call py_action,download_wpt_manifest)
 158
 159 define package_archive
 160 package-tests-$(1): stage-all package-tests-prepare-dest download-wpt-manifest
 161         $$(call py_action,test_archive, \
 162                 $(1) \
 163                 '$$(abspath $$(test_archive_dir))/$$(PKG_BASENAME).$(1).tests.$(2)')
 164 package-tests: package-tests-$(1)
 165 endef
 166
 167 $(foreach name,$(TEST_PKGS_TARGZ),$(eval $(call package_archive,$(name),tar.gz)))
 168
 169 ifeq ($(MOZ_BUILD_APP),mobile/android)
 170 stage-all: stage-android
 171 endif
 172
 173 # Prepare _tests before any of the other staging/packaging steps.
 174 # make-stage-dir is a prerequisite to all the stage-* targets in testsuite-targets.mk.
 175 make-stage-dir: install-test-files
 176         rm -rf $(PKG_STAGE)
 177         $(NSINSTALL) -D $(PKG_STAGE)
 178         $(NSINSTALL) -D $(PKG_STAGE)/bin
 179         $(NSINSTALL) -D $(PKG_STAGE)/bin/components
 180         $(NSINSTALL) -D $(PKG_STAGE)/certs
 181         $(NSINSTALL) -D $(PKG_STAGE)/config
 182         $(NSINSTALL) -D $(PKG_STAGE)/modules
 183         $(NSINSTALL) -D $(PKG_STAGE)/tools/mach
 184
 185 stage-config: make-stage-dir
 186         $(NSINSTALL) -D $(PKG_STAGE)/config
 187         @(cd $(topsrcdir)/testing/config && tar $(TAR_CREATE_FLAGS) - *) | (cd $(PKG_STAGE)/config && tar -xf -)
 188
 189 stage-mach: make-stage-dir
 190         @(cd $(topsrcdir)/python/mach && tar $(TAR_CREATE_FLAGS) - *) | (cd $(PKG_STAGE)/tools/mach && tar -xf -)
 191         cp $(topsrcdir)/testing/tools/mach_test_package_bootstrap.py $(PKG_STAGE)/tools/mach_bootstrap.py
 192         cp $(topsrcdir)/mach $(PKG_STAGE)
 193
 194 stage-mochitest: make-stage-dir ;
 195
 196 stage-jstests: make-stage-dir
 197         $(MAKE) -C $(DEPTH)/js/src/tests stage-package
 198
 199 ifdef OBJCOPY
 200 ifneq ($(OBJCOPY), :) # see build/autoconf/toolchain.m4:102 for why this is necessary
 201 ifndef PKG_SKIP_STRIP
 202 STRIP_COMPILED_TESTS := 1
 203 endif
 204 endif
 205 endif
 206
 207 stage-gtest: make-stage-dir
 208         $(NSINSTALL) -D $(PKG_STAGE)/gtest/gtest_bin/gtest
 209 ifdef STRIP_COMPILED_TESTS
 210 # The libxul file basename will vary per platform. Fortunately
 211 # dependentlibs.list always lists the library name as its final line, so we
 212 # can get the value from there.
 213         LIBXUL_BASE=`tail -1 $(DIST)/bin/dependentlibs.list` && \
 214         $(OBJCOPY) $(or $(STRIP_FLAGS),--strip-unneeded) \
 215         $(DIST)/bin/gtest/$$LIBXUL_BASE $(PKG_STAGE)/gtest/gtest_bin/gtest/$$LIBXUL_BASE
 216 else
 217         cp -RL $(DIST)/bin/gtest $(PKG_STAGE)/gtest/gtest_bin
 218 endif
 219         cp -RL $(DEPTH)/_tests/gtest $(PKG_STAGE)
 220         cp $(topsrcdir)/testing/gtest/rungtests.py $(PKG_STAGE)/gtest
 221         cp $(topsrcdir)/testing/gtest/remotegtests.py $(PKG_STAGE)/gtest
 222         cp $(topsrcdir)/testing/gtest/mach_test_package_commands.py $(PKG_STAGE)/gtest
 223         cp $(DIST)/bin/dependentlibs.list.gtest $(PKG_STAGE)/gtest
 224         cp $(DEPTH)/mozinfo.json $(PKG_STAGE)/gtest
 225
 226 stage-android: make-stage-dir
 227         $(NSINSTALL) $(topsrcdir)/mobile/android/fonts $(DEPTH)/_tests/reftest
 228         $(NSINSTALL) $(topsrcdir)/mobile/android/fonts $(DEPTH)/_tests/testing/mochitest
 229
 230 ifdef MOZ_COPY_PDBS
 231 CPP_UNIT_TEST_BINS=$(filter-out $(wildcard $(DIST)/cppunittests/*.pdb), $(wildcard $(DIST)/cppunittests/*))
 232 else
 233 CPP_UNIT_TEST_BINS=$(wildcard $(DIST)/cppunittests/*)
 234 endif
 235
 236 stage-cppunittests: make-stage-dir
 237         $(NSINSTALL) -D $(PKG_STAGE)/cppunittest
 238 ifdef STRIP_COMPILED_TESTS
 239         $(foreach bin,$(CPP_UNIT_TEST_BINS),$(OBJCOPY) $(or $(STRIP_FLAGS),--strip-unneeded) $(bin) $(bin:$(DIST)/cppunittests/%=$(PKG_STAGE)/cppunittest/%);)
 240 else
 241         cp -RL $(CPP_UNIT_TEST_BINS) $(PKG_STAGE)/cppunittest
 242 endif
 243 ifdef MOZ_COPY_PDBS
 244         cp -RL $(addsuffix .pdb,$(basename $(CPP_UNIT_TEST_BINS))) $(PKG_STAGE)/cppunittest
 245 endif
 246 ifdef STRIP_COMPILED_TESTS
 247         $(OBJCOPY) $(or $(STRIP_FLAGS),--strip-unneeded) $(DIST)/bin/jsapi-tests$(BIN_SUFFIX) $(PKG_STAGE)/cppunittest/jsapi-tests$(BIN_SUFFIX)
 248 else
 249         cp -RL $(DIST)/bin/jsapi-tests$(BIN_SUFFIX) $(PKG_STAGE)/cppunittest
 250 endif
 251 ifdef MOZ_COPY_PDBS
 252         cp -RL $(DIST)/bin/jsapi-tests.pdb $(PKG_STAGE)/cppunittest
 253 endif
 254
 255 stage-steeplechase: make-stage-dir
 256         $(NSINSTALL) -D $(PKG_STAGE)/steeplechase/
 257         cp -RL $(DEPTH)/_tests/steeplechase $(PKG_STAGE)/steeplechase/tests
 258         cp -RL $(DIST)/xpi-stage/specialpowers $(PKG_STAGE)/steeplechase
 259         cp -RL $(topsrcdir)/testing/profiles/common/user.js $(PKG_STAGE)/steeplechase/prefs_general.js
 260
 261 TEST_EXTENSIONS := \
 262     specialpowers@mozilla.org.xpi \
 263         $(NULL)
 264
 265 stage-extensions: make-stage-dir
 266         $(NSINSTALL) -D $(PKG_STAGE)/extensions/
 267         @$(foreach ext,$(TEST_EXTENSIONS), cp -RL $(DIST)/xpi-stage/$(ext) $(PKG_STAGE)/extensions;)
 268
 269
 270 check::
 271         $(eval cores=$(shell $(PYTHON3) -c 'import multiprocessing; print(multiprocessing.cpu_count())'))
 272         @echo "Starting 'mach python-test' with -j$(cores)"
 273         @$(topsrcdir)/mach --log-no-times python-test -j$(cores) --subsuite default
 274         @echo "Finished 'mach python-test' successfully"
 275         @echo "Starting 'mach python-test' with --python $(PYTHON3) -j$(cores)"
 276         @$(topsrcdir)/mach --log-no-times python-test --python python3 -j$(cores) --subsuite default
 277         @echo "Finished 'mach python-test' with py3 successfully"
 278
 279
 280 .PHONY: \
 281   reftest \
 282   crashtest \
 283   xpcshell-tests \
 284   jstestbrowser \
 285   package-tests \
 286   download-wpt-manifest \
 287   package-tests-prepare-dest \
 288   package-tests-common \
 289   make-stage-dir \
 290   stage-all \
 291   stage-config \
 292   stage-mochitest \
 293   stage-jstests \
 294   stage-android \
 295   stage-steeplechase \
 296   test-packages-manifest \
 297   check \
 298   $(NULL)