testing/testsuite-targets.mk

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # include automation-build.mk to get the path to the binary
   6 TARGET_DEPTH = $(DEPTH)
   7 include $(topsrcdir)/build/binary-location.mk
   8
   9 SYMBOLS_PATH := --symbols-path=$(DIST)/crashreporter-symbols
  10
  11 ifndef TEST_PACKAGE_NAME
  12 TEST_PACKAGE_NAME := $(ANDROID_PACKAGE_NAME)
  13 endif
  14
  15 ifndef NO_FAIL_ON_TEST_ERRORS
  16 define check_test_error_internal
  17   @errors=`grep 'TEST-UNEXPECTED-' $@.log` ;\
  18   if test "$$errors" ; then \
  19           echo '$@ failed:'; \
  20           echo "$$errors"; \
  21           $(if $(1),echo $(1);) \
  22           exit 1; \
  23   fi
  24 endef
  25 CHECK_TEST_ERROR = $(call check_test_error_internal)
  26 CHECK_TEST_ERROR_RERUN = $(call check_test_error_internal,'To rerun your failures please run "make $@-rerun-failures"')
  27 endif
  28
  29 # Usage: |make [EXTRA_TEST_ARGS=...] *test|.
  30 RUN_REFTEST = rm -f ./$@.log && $(PYTHON) _tests/reftest/runreftest.py \
  31   --extra-profile-file=$(DIST)/plugins \
  32   $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(1) | tee ./$@.log
  33
  34 REMOTE_REFTEST = rm -f ./$@.log && $(PYTHON) _tests/reftest/remotereftest.py \
  35   --ignore-window-size \
  36   --app=$(TEST_PACKAGE_NAME) --deviceIP=${TEST_DEVICE} --xre-path=${MOZ_HOST_BIN} \
  37   --httpd-path=_tests/modules --suite reftest \
  38   --extra-profile-file=$(topsrcdir)/mobile/android/fonts \
  39   $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS) $(1) | tee ./$@.log
  40
  41 ifeq ($(OS_ARCH),WINNT) #{
  42 # GPU-rendered shadow layers are unsupported here
  43 OOP_CONTENT = --setpref=layers.async-pan-zoom.enabled=true --setpref=browser.tabs.remote.autostart=true --setpref=layers.acceleration.disabled=true
  44 GPU_RENDERING =
  45 else
  46 OOP_CONTENT = --setpref=layers.async-pan-zoom.enabled=true --setpref=browser.tabs.remote.autostart=true
  47 GPU_RENDERING = --setpref=layers.acceleration.force-enabled=true
  48 endif #}
  49
  50 reftest: TEST_PATH?=layout/reftests/reftest.list
  51 reftest:
  52         $(call RUN_REFTEST,'$(topsrcdir)/$(TEST_PATH)')
  53         $(CHECK_TEST_ERROR)
  54
  55 reftest-remote: TEST_PATH?=layout/reftests/reftest.list
  56 reftest-remote:
  57         @if [ '${MOZ_HOST_BIN}' = '' ]; then \
  58         echo 'environment variable MOZ_HOST_BIN must be set to a directory containing host xpcshell'; \
  59     elif [ ! -d ${MOZ_HOST_BIN} ]; then \
  60         echo 'MOZ_HOST_BIN does not specify a directory'; \
  61     elif [ ! -f ${MOZ_HOST_BIN}/xpcshell ]; then \
  62         echo 'xpcshell not found in MOZ_HOST_BIN'; \
  63     else \
  64         ln -s $(abspath $(topsrcdir)) _tests/reftest/tests; \
  65         $(call REMOTE_REFTEST,'tests/$(TEST_PATH)'); \
  66         $(CHECK_TEST_ERROR); \
  67     fi
  68
  69 crashtest: TEST_PATH?=testing/crashtest/crashtests.list
  70 crashtest:
  71         $(call RUN_REFTEST,'$(topsrcdir)/$(TEST_PATH)')
  72         $(CHECK_TEST_ERROR)
  73
  74 jstestbrowser: TESTS_PATH?=test-stage/jsreftest/tests/
  75 jstestbrowser:
  76         $(MAKE) -C $(DEPTH)/config
  77         $(MAKE) stage-jstests
  78         $(call RUN_REFTEST,'$(DIST)/$(TESTS_PATH)/jstests.list' --extra-profile-file=$(DIST)/test-stage/jsreftest/tests/user.js)
  79         $(CHECK_TEST_ERROR)
  80
  81 GARBAGE += $(addsuffix .log,$(MOCHITESTS) reftest crashtest jstestbrowser)
  82
  83 REMOTE_CPPUNITTESTS = \
  84         $(PYTHON) -u $(topsrcdir)/testing/remotecppunittests.py \
  85           --xre-path=$(DEPTH)/dist/bin \
  86           --localLib=$(DEPTH)/dist/fennec \
  87           --deviceIP=${TEST_DEVICE} \
  88           $(TEST_PATH) $(EXTRA_TEST_ARGS)
  89
  90 # Usage: |make [TEST_PATH=...] [EXTRA_TEST_ARGS=...] cppunittests-remote|.
  91 cppunittests-remote:
  92         $(call REMOTE_CPPUNITTESTS);
  93
  94 # Package up the tests and test harnesses
  95 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
  96
  97 PKG_STAGE = $(DIST)/test-stage
  98
  99 stage-all: \
 100   stage-config \
 101   stage-mach \
 102   stage-extensions \
 103   stage-mochitest \
 104   stage-jstests \
 105   test-packages-manifest \
 106   $(NULL)
 107 ifdef MOZ_WEBRTC
 108 stage-all: stage-steeplechase
 109 endif
 110
 111 ifdef COMPILE_ENVIRONMENT
 112 stage-all: stage-cppunittests
 113 endif
 114
 115 TEST_PKGS_TARGZ := \
 116   common \
 117   condprof \
 118   cppunittest \
 119   mochitest \
 120   reftest \
 121   talos \
 122   raptor \
 123   awsy \
 124   xpcshell \
 125   web-platform \
 126   updater-dep \
 127   $(NULL)
 128
 129 ifdef LINK_GTEST_DURING_COMPILE
 130 stage-all: stage-gtest
 131 TEST_PKGS_TARGZ += gtest
 132 endif
 133
 134 PKG_ARG = --$(1) '$(PKG_BASENAME).$(1).tests.$(2)'
 135
 136 test-packages-manifest:
 137         @rm -f $(MOZ_TEST_PACKAGES_FILE)
 138         $(NSINSTALL) -D $(dir $(MOZ_TEST_PACKAGES_FILE))
 139         $(PYTHON) $(topsrcdir)/build/gen_test_packages_manifest.py \
 140       --jsshell $(JSSHELL_NAME) \
 141       --dest-file '$(MOZ_TEST_PACKAGES_FILE)' \
 142       $(call PKG_ARG,common,zip) \
 143       $(foreach pkg,$(TEST_PKGS_TARGZ),$(call PKG_ARG,$(pkg),tar.gz))
 144
 145 ifdef UPLOAD_PATH
 146 test_archive_dir = $(UPLOAD_PATH)
 147 else
 148 test_archive_dir = $(DIST)/$(PKG_PATH)
 149 endif
 150
 151 package-tests-prepare-dest:
 152         $(NSINSTALL) -D $(test_archive_dir)
 153
 154 download-wpt-manifest:
 155         $(call py_action,download_wpt_manifest)
 156
 157 define package_archive
 158 package-tests-$(1): stage-all package-tests-prepare-dest download-wpt-manifest
 159         $$(call py_action,test_archive, \
 160                 $(1) \
 161                 '$$(abspath $$(test_archive_dir))/$$(PKG_BASENAME).$(1).tests.$(2)')
 162 package-tests: package-tests-$(1)
 163 endef
 164
 165 $(foreach name,$(TEST_PKGS_TARGZ),$(eval $(call package_archive,$(name),tar.gz)))
 166
 167 ifeq ($(MOZ_BUILD_APP),mobile/android)
 168 stage-all: stage-android
 169 endif
 170
 171 # Prepare _tests before any of the other staging/packaging steps.
 172 # make-stage-dir is a prerequisite to all the stage-* targets in testsuite-targets.mk.
 173 make-stage-dir: install-test-files
 174         rm -rf $(PKG_STAGE)
 175         $(NSINSTALL) -D $(PKG_STAGE)
 176         $(NSINSTALL) -D $(PKG_STAGE)/bin
 177         $(NSINSTALL) -D $(PKG_STAGE)/bin/components
 178         $(NSINSTALL) -D $(PKG_STAGE)/certs
 179         $(NSINSTALL) -D $(PKG_STAGE)/config
 180         $(NSINSTALL) -D $(PKG_STAGE)/modules
 181         $(NSINSTALL) -D $(PKG_STAGE)/tools/mach
 182
 183 stage-config: make-stage-dir
 184         $(NSINSTALL) -D $(PKG_STAGE)/config
 185         @(cd $(topsrcdir)/testing/config && tar $(TAR_CREATE_FLAGS) - *) | (cd $(PKG_STAGE)/config && tar -xf -)
 186
 187 stage-mach: make-stage-dir
 188         @(cd $(topsrcdir)/python/mach && tar $(TAR_CREATE_FLAGS) - *) | (cd $(PKG_STAGE)/tools/mach && tar -xf -)
 189         cp $(topsrcdir)/testing/tools/mach_test_package_bootstrap.py $(PKG_STAGE)/tools/mach_bootstrap.py
 190         cp $(topsrcdir)/mach $(PKG_STAGE)
 191
 192 stage-mochitest: make-stage-dir ;
 193
 194 stage-jstests: make-stage-dir
 195         $(MAKE) -C $(DEPTH)/js/src/tests stage-package
 196
 197 ifdef OBJCOPY
 198 ifneq ($(OBJCOPY), :) # see build/autoconf/toolchain.m4:102 for why this is necessary
 199 ifndef PKG_SKIP_STRIP
 200 STRIP_COMPILED_TESTS := 1
 201 endif
 202 endif
 203 endif
 204
 205 stage-gtest: make-stage-dir
 206         $(NSINSTALL) -D $(PKG_STAGE)/gtest/gtest_bin/gtest
 207 ifdef STRIP_COMPILED_TESTS
 208 # The libxul file basename will vary per platform. Fortunately
 209 # dependentlibs.list always lists the library name as its final line, so we
 210 # can get the value from there.
 211         LIBXUL_BASE=`tail -1 $(DIST)/bin/dependentlibs.list` && \
 212         $(OBJCOPY) $(or $(STRIP_FLAGS),--strip-unneeded) \
 213         $(DIST)/bin/gtest/$$LIBXUL_BASE $(PKG_STAGE)/gtest/gtest_bin/gtest/$$LIBXUL_BASE
 214 else
 215         cp -RL $(DIST)/bin/gtest $(PKG_STAGE)/gtest/gtest_bin
 216 endif
 217         cp -RL $(DEPTH)/_tests/gtest $(PKG_STAGE)
 218         cp $(topsrcdir)/testing/gtest/rungtests.py $(PKG_STAGE)/gtest
 219         cp $(topsrcdir)/testing/gtest/remotegtests.py $(PKG_STAGE)/gtest
 220         cp $(DIST)/bin/dependentlibs.list.gtest $(PKG_STAGE)/gtest
 221         cp $(DEPTH)/mozinfo.json $(PKG_STAGE)/gtest
 222
 223 stage-android: make-stage-dir
 224         $(NSINSTALL) $(topsrcdir)/mobile/android/fonts $(DEPTH)/_tests/reftest
 225         $(NSINSTALL) $(topsrcdir)/mobile/android/fonts $(DEPTH)/_tests/testing/mochitest
 226         $(NSINSTALL) -D $(DEPTH)/_tests/reftest/hyphenation
 227         $(NSINSTALL) $(wildcard $(topsrcdir)/intl/locales/*/hyphenation/*.dic) $(DEPTH)/_tests/reftest/hyphenation
 228
 229 ifdef MOZ_COPY_PDBS
 230 CPP_UNIT_TEST_BINS=$(filter-out $(wildcard $(DIST)/cppunittests/*.pdb), $(wildcard $(DIST)/cppunittests/*))
 231 else
 232 CPP_UNIT_TEST_BINS=$(wildcard $(DIST)/cppunittests/*)
 233 endif
 234
 235 stage-cppunittests: make-stage-dir
 236         $(NSINSTALL) -D $(PKG_STAGE)/cppunittest
 237 ifdef STRIP_COMPILED_TESTS
 238         $(foreach bin,$(CPP_UNIT_TEST_BINS),$(OBJCOPY) $(or $(STRIP_FLAGS),--strip-unneeded) $(bin) $(bin:$(DIST)/cppunittests/%=$(PKG_STAGE)/cppunittest/%);)
 239 else
 240         cp -RL $(CPP_UNIT_TEST_BINS) $(PKG_STAGE)/cppunittest
 241 endif
 242 ifdef MOZ_COPY_PDBS
 243         cp -RL $(addsuffix .pdb,$(basename $(CPP_UNIT_TEST_BINS))) $(PKG_STAGE)/cppunittest
 244 endif
 245 ifdef STRIP_COMPILED_TESTS
 246         $(OBJCOPY) $(or $(STRIP_FLAGS),--strip-unneeded) $(DIST)/bin/jsapi-tests$(BIN_SUFFIX) $(PKG_STAGE)/cppunittest/jsapi-tests$(BIN_SUFFIX)
 247 else
 248         cp -RL $(DIST)/bin/jsapi-tests$(BIN_SUFFIX) $(PKG_STAGE)/cppunittest
 249 endif
 250 ifdef MOZ_COPY_PDBS
 251         cp -RL $(DIST)/bin/jsapi-tests.pdb $(PKG_STAGE)/cppunittest
 252 endif
 253
 254 stage-steeplechase: make-stage-dir
 255         $(NSINSTALL) -D $(PKG_STAGE)/steeplechase/
 256         cp -RL $(DEPTH)/_tests/steeplechase $(PKG_STAGE)/steeplechase/tests
 257         cp -RL $(DIST)/xpi-stage/specialpowers $(PKG_STAGE)/steeplechase
 258         cp -RL $(topsrcdir)/testing/profiles/common/user.js $(PKG_STAGE)/steeplechase/prefs_general.js
 259
 260 TEST_EXTENSIONS := \
 261     specialpowers@mozilla.org.xpi \
 262         $(NULL)
 263
 264 stage-extensions: make-stage-dir
 265         $(NSINSTALL) -D $(PKG_STAGE)/extensions/
 266         @$(foreach ext,$(TEST_EXTENSIONS), cp -RL $(DIST)/xpi-stage/$(ext) $(PKG_STAGE)/extensions;)
 267
 268
 269 check::
 270         $(eval cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())'))
 271         @echo "Starting 'mach python-test' with -j$(cores)"
 272         @$(topsrcdir)/mach --log-no-times python-test -j$(cores) --subsuite default
 273         @echo "Finished 'mach python-test' successfully"
 274
 275
 276 .PHONY: \
 277   reftest \
 278   crashtest \
 279   xpcshell-tests \
 280   jstestbrowser \
 281   package-tests \
 282   download-wpt-manifest \
 283   package-tests-prepare-dest \
 284   package-tests-common \
 285   make-stage-dir \
 286   stage-all \
 287   stage-config \
 288   stage-mochitest \
 289   stage-jstests \
 290   stage-android \
 291   stage-steeplechase \
 292   test-packages-manifest \
 293   check \
 294   $(NULL)