1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
10 from argparse
import Namespace
11 from collections
import defaultdict
14 from mach
.decorators
import Command
, CommandArgument
15 from mozbuild
.base
import MachCommandConditions
as conditions
16 from mozbuild
.base
import MozbuildObject
18 here
= os
.path
.abspath(os
.path
.dirname(__file__
))
21 ENG_BUILD_REQUIRED
= """
22 The mochitest command requires an engineering build. It may be the case that
23 VARIANT=user or PRODUCTION=1 were set. Try re-building with VARIANT=eng:
25 $ VARIANT=eng ./build.sh
27 There should be an app called 'test-container.gaiamobile.org' located in
31 SUPPORTED_TESTS_NOT_FOUND
= """
32 The mochitest command could not find any supported tests to run! The
33 following flavors and subsuites were found, but are either not supported on
34 {} builds, or were excluded on the command line:
38 Double check the command line you used, and make sure you are running in
39 context of the proper build. To switch build contexts, either run |mach|
40 from the appropriate objdir, or export the correct mozconfig:
42 $ export MOZCONFIG=path/to/mozconfig
46 The mochitest command could not find any mochitests under the following
51 Please check spelling and make sure there are mochitests living there.
54 SUPPORTED_APPS
= ["firefox", "android", "thunderbird"]
59 class MochitestRunner(MozbuildObject
):
61 """Easily run mochitests.
63 This currently contains just the basics for running mochitests. We may want
64 to hook up result parsing, etc.
67 def __init__(self
, *args
, **kwargs
):
68 MozbuildObject
.__init
__(self
, *args
, **kwargs
)
70 # TODO Bug 794506 remove once mach integrates with virtualenv.
71 build_path
= os
.path
.join(self
.topobjdir
, "build")
72 if build_path
not in sys
.path
:
73 sys
.path
.append(build_path
)
75 self
.tests_dir
= os
.path
.join(self
.topobjdir
, "_tests")
76 self
.mochitest_dir
= os
.path
.join(self
.tests_dir
, "testing", "mochitest")
77 self
.bin_dir
= os
.path
.join(self
.topobjdir
, "dist", "bin")
79 def resolve_tests(self
, test_paths
, test_objects
=None, cwd
=None):
83 from moztest
.resolve
import TestResolver
85 resolver
= self
._spawn
(TestResolver
)
86 tests
= list(resolver
.resolve_tests(paths
=test_paths
, cwd
=cwd
))
89 def run_desktop_test(self
, command_context
, tests
=None, **kwargs
):
90 """Runs a mochitest."""
91 # runtests.py is ambiguous, so we load the file/module manually.
92 if "mochitest" not in sys
.modules
:
95 path
= os
.path
.join(self
.mochitest_dir
, "runtests.py")
96 with
open(path
, "r") as fh
:
97 imp
.load_module("mochitest", fh
, path
, (".py", "r", imp
.PY_SOURCE
))
101 # This is required to make other components happy. Sad, isn't it?
102 os
.chdir(self
.topobjdir
)
104 # Automation installs its own stream handler to stdout. Since we want
105 # all logging to go through us, we just remove their handler.
108 for l
in logging
.getLogger().handlers
109 if isinstance(l
, logging
.StreamHandler
)
111 for handler
in remove_handlers
:
112 logging
.getLogger().removeHandler(handler
)
114 options
= Namespace(**kwargs
)
115 options
.topsrcdir
= self
.topsrcdir
116 options
.topobjdir
= self
.topobjdir
118 from manifestparser
import TestManifest
120 if tests
and not options
.manifestFile
:
121 manifest
= TestManifest()
122 manifest
.tests
.extend(tests
)
123 options
.manifestFile
= manifest
125 # When developing mochitest-plain tests, it's often useful to be able to
126 # refresh the page to pick up modifications. Therefore leave the browser
127 # open if only running a single mochitest-plain test. This behaviour can
128 # be overridden by passing in --keep-open=false.
131 and options
.keep_open
is None
132 and not options
.headless
133 and getattr(options
, "flavor", "plain") == "plain"
135 options
.keep_open
= True
137 # We need this to enable colorization of output.
138 self
.log_manager
.enable_unstructured()
139 result
= mochitest
.run_test_harness(parser
, options
)
140 self
.log_manager
.disable_unstructured()
143 def run_android_test(self
, command_context
, tests
, **kwargs
):
144 host_ret
= verify_host_bin()
150 path
= os
.path
.join(self
.mochitest_dir
, "runtestsremote.py")
151 with
open(path
, "r") as fh
:
152 imp
.load_module("runtestsremote", fh
, path
, (".py", "r", imp
.PY_SOURCE
))
153 import runtestsremote
155 options
= Namespace(**kwargs
)
157 from manifestparser
import TestManifest
159 if tests
and not options
.manifestFile
:
160 manifest
= TestManifest()
161 manifest
.tests
.extend(tests
)
162 options
.manifestFile
= manifest
164 # Firefox for Android doesn't use e10s
165 if options
.app
is not None and "geckoview" not in options
.app
:
167 print("using e10s=False for non-geckoview app")
169 return runtestsremote
.run_test_harness(parser
, options
)
171 def run_geckoview_junit_test(self
, context
, **kwargs
):
172 host_ret
= verify_host_bin()
178 options
= Namespace(**kwargs
)
180 return runjunit
.run_test_harness(parser
, options
)
186 def setup_argument_parser():
187 build_obj
= MozbuildObject
.from_environment(cwd
=here
)
189 build_path
= os
.path
.join(build_obj
.topobjdir
, "build")
190 if build_path
not in sys
.path
:
191 sys
.path
.append(build_path
)
193 mochitest_dir
= os
.path
.join(build_obj
.topobjdir
, "_tests", "testing", "mochitest")
195 with warnings
.catch_warnings():
196 warnings
.simplefilter("ignore")
200 path
= os
.path
.join(build_obj
.topobjdir
, mochitest_dir
, "runtests.py")
201 if not os
.path
.exists(path
):
202 path
= os
.path
.join(here
, "runtests.py")
204 with
open(path
, "r") as fh
:
205 imp
.load_module("mochitest", fh
, path
, (".py", "r", imp
.PY_SOURCE
))
207 from mochitest_options
import MochitestArgumentParser
209 if conditions
.is_android(build_obj
):
210 # On Android, check for a connected device (and offer to start an
211 # emulator if appropriate) before running tests. This check must
212 # be done in this admittedly awkward place because
213 # MochitestArgumentParser initialization fails if no device is found.
214 from mozrunner
.devices
.android_device
import (
216 verify_android_device
,
219 # verify device and xre
220 verify_android_device(build_obj
, install
=InstallIntent
.NO
, xre
=True)
223 parser
= MochitestArgumentParser()
227 def setup_junit_argument_parser():
228 build_obj
= MozbuildObject
.from_environment(cwd
=here
)
230 build_path
= os
.path
.join(build_obj
.topobjdir
, "build")
231 if build_path
not in sys
.path
:
232 sys
.path
.append(build_path
)
234 mochitest_dir
= os
.path
.join(build_obj
.topobjdir
, "_tests", "testing", "mochitest")
236 with warnings
.catch_warnings():
237 warnings
.simplefilter("ignore")
239 # runtests.py contains MochitestDesktop, required by runjunit
242 path
= os
.path
.join(build_obj
.topobjdir
, mochitest_dir
, "runtests.py")
243 if not os
.path
.exists(path
):
244 path
= os
.path
.join(here
, "runtests.py")
246 with
open(path
, "r") as fh
:
247 imp
.load_module("mochitest", fh
, path
, (".py", "r", imp
.PY_SOURCE
))
250 from mozrunner
.devices
.android_device
import (
252 verify_android_device
,
255 verify_android_device(
256 build_obj
, install
=InstallIntent
.NO
, xre
=True, network
=True
260 parser
= runjunit
.JunitArgumentParser()
264 def verify_host_bin():
265 # validate MOZ_HOST_BIN environment variables for Android tests
266 xpcshell_binary
= "xpcshell"
268 xpcshell_binary
= "xpcshell.exe"
269 MOZ_HOST_BIN
= os
.environ
.get("MOZ_HOST_BIN")
272 "environment variable MOZ_HOST_BIN must be set to a directory containing host "
273 "%s" % xpcshell_binary
276 elif not os
.path
.isdir(MOZ_HOST_BIN
):
277 print("$MOZ_HOST_BIN does not specify a directory")
279 elif not os
.path
.isfile(os
.path
.join(MOZ_HOST_BIN
, xpcshell_binary
)):
280 print("$MOZ_HOST_BIN/%s does not exist" % xpcshell_binary
)
288 conditions
=[functools
.partial(conditions
.is_buildapp_in
, apps
=SUPPORTED_APPS
)],
289 description
="Run any flavor of mochitest (integration test).",
290 parser
=setup_argument_parser
,
292 def run_mochitest_general(
293 command_context
, flavor
=None, test_objects
=None, resolve_tests
=True, **kwargs
295 from mochitest_options
import ALL_FLAVORS
296 from mozlog
.commandline
import setup_logging
297 from mozlog
.handlers
import StreamHandler
298 from moztest
.resolve
import get_suite_definition
300 # TODO: This is only strictly necessary while mochitest is using Python
301 # 2 and can be removed once the command is migrated to Python 3.
302 command_context
.activate_virtualenv()
305 for app
in SUPPORTED_APPS
:
306 if conditions
.is_buildapp_in(command_context
, apps
=[app
]):
312 for fname
, fobj
in six
.iteritems(ALL_FLAVORS
):
313 if flavor
in fobj
["aliases"]:
314 if buildapp
not in fobj
["enabled_apps"]:
320 f
for f
, v
in six
.iteritems(ALL_FLAVORS
) if buildapp
in v
["enabled_apps"]
323 from mozbuild
.controller
.building
import BuildDriver
325 command_context
._ensure
_state
_subdir
_exists
(".")
327 test_paths
= kwargs
["test_paths"]
328 kwargs
["test_paths"] = []
330 if kwargs
.get("debugger", None):
333 if not mozdebug
.get_debugger_info(kwargs
.get("debugger")):
336 mochitest
= command_context
._spawn
(MochitestRunner
)
339 tests
= mochitest
.resolve_tests(
340 test_paths
, test_objects
, cwd
=command_context
._mach
_context
.cwd
343 if not kwargs
.get("log"):
344 # Create shared logger
345 format_args
= {"level": command_context
._mach
_context
.settings
["test"]["level"]}
347 format_args
["verbose"] = True
348 format_args
["compact"] = False
350 default_format
= command_context
._mach
_context
.settings
["test"]["format"]
351 kwargs
["log"] = setup_logging(
352 "mach-mochitest", kwargs
, {default_format
: sys
.stdout
}, format_args
354 for handler
in kwargs
["log"].handlers
:
355 if isinstance(handler
, StreamHandler
):
356 handler
.formatter
.inner
.summary_on_shutdown
= True
358 driver
= command_context
._spawn
(BuildDriver
)
359 driver
.install_tests()
361 subsuite
= kwargs
.get("subsuite")
362 if subsuite
== "default":
363 kwargs
["subsuite"] = None
365 suites
= defaultdict(list)
366 is_webrtc_tag_present
= False
369 # Check if we're running a webrtc test so we can enable webrtc
370 # specific test logic later if needed.
371 if "webrtc" in test
.get("tags", ""):
372 is_webrtc_tag_present
= True
374 # Filter out non-mochitests and unsupported flavors.
375 if test
["flavor"] not in ALL_FLAVORS
:
378 key
= (test
["flavor"], test
.get("subsuite", ""))
379 if test
["flavor"] not in flavors
:
383 if subsuite
== "default":
384 # "--subsuite default" means only run tests that don't have a subsuite
385 if test
.get("subsuite"):
388 elif subsuite
and test
.get("subsuite", "") != subsuite
:
392 suites
[key
].append(test
)
394 # Only webrtc mochitests in the media suite need the websocketprocessbridge.
395 if ("mochitest", "media") in suites
and is_webrtc_tag_present
:
399 "websocketprocessbridge",
400 "websocketprocessbridge_requirements_3.txt",
402 command_context
.virtualenv_manager
.activate()
403 command_context
.virtualenv_manager
.install_pip_requirements(
404 req
, require_hashes
=False
407 # sys.executable is used to start the websocketprocessbridge, though for some
408 # reason it doesn't get set when calling `activate_this.py` in the virtualenv.
409 sys
.executable
= command_context
.virtualenv_manager
.python_path
411 # This is a hack to introduce an option in mach to not send
412 # filtered tests to the mochitest harness. Mochitest harness will read
413 # the master manifest in that case.
414 if not resolve_tests
:
415 for flavor
in flavors
:
416 key
= (flavor
, kwargs
.get("subsuite"))
420 # Make it very clear why no tests were found
423 TESTS_NOT_FOUND
.format(
424 "\n".join(sorted(list(test_paths
or test_objects
)))
430 for f
, s
in unsupported
:
431 fobj
= ALL_FLAVORS
[f
]
432 apps
= fobj
["enabled_apps"]
433 name
= fobj
["aliases"][0]
435 name
= "{} --subsuite {}".format(name
, s
)
437 if buildapp
not in apps
:
438 reason
= "requires {}".format(" or ".join(apps
))
440 reason
= "excluded by the command line"
441 msg
.append(" mochitest -f {} ({})".format(name
, reason
))
442 print(SUPPORTED_TESTS_NOT_FOUND
.format(buildapp
, "\n".join(sorted(msg
))))
445 if buildapp
== "android":
446 from mozrunner
.devices
.android_device
import (
449 verify_android_device
,
452 app
= kwargs
.get("app")
454 app
= "org.mozilla.geckoview.test_runner"
455 device_serial
= kwargs
.get("deviceSerial")
456 install
= InstallIntent
.NO
if kwargs
.get("no_install") else InstallIntent
.YES
457 aab
= kwargs
.get("aab")
459 # verify installation
460 verify_android_device(
467 device_serial
=device_serial
,
470 if not kwargs
["adbPath"]:
471 kwargs
["adbPath"] = get_adb_path(command_context
)
473 run_mochitest
= mochitest
.run_android_test
475 run_mochitest
= mochitest
.run_desktop_test
478 for (flavor
, subsuite
), tests
in sorted(suites
.items()):
479 suite_name
, suite
= get_suite_definition(flavor
, subsuite
)
480 if "test_paths" in suite
["kwargs"]:
481 del suite
["kwargs"]["test_paths"]
483 harness_args
= kwargs
.copy()
484 harness_args
.update(suite
["kwargs"])
485 # Pass in the full suite name as defined in moztest/resolve.py in case
486 # chunk-by-runtime is called, in which case runtime information for
487 # specific mochitest suite has to be loaded. See Bug 1637463.
488 harness_args
.update({"suite_name": suite_name
})
490 result
= run_mochitest(
491 command_context
._mach
_context
, tests
=tests
, **harness_args
497 # Halt tests on keyboard interrupt
501 # Only shutdown the logger if we created it
502 if kwargs
["log"].name
== "mach-mochitest":
503 kwargs
["log"].shutdown()
511 conditions
=[conditions
.is_android
],
512 description
="Run remote geckoview junit tests.",
513 parser
=setup_junit_argument_parser
,
517 help="Do not try to install application on device before "
518 + "running (default: False)",
522 def run_junit(command_context
, no_install
, **kwargs
):
523 command_context
._ensure
_state
_subdir
_exists
(".")
525 from mozrunner
.devices
.android_device
import (
528 verify_android_device
,
531 # verify installation
532 app
= kwargs
.get("app")
533 device_serial
= kwargs
.get("deviceSerial")
534 verify_android_device(
536 install
=InstallIntent
.NO
if no_install
else InstallIntent
.YES
,
539 device_serial
=device_serial
,
542 if not kwargs
.get("adbPath"):
543 kwargs
["adbPath"] = get_adb_path(command_context
)
545 if not kwargs
.get("log"):
546 from mozlog
.commandline
import setup_logging
548 format_args
= {"level": command_context
._mach
_context
.settings
["test"]["level"]}
549 default_format
= command_context
._mach
_context
.settings
["test"]["format"]
550 kwargs
["log"] = setup_logging(
551 "mach-mochitest", kwargs
, {default_format
: sys
.stdout
}, format_args
554 mochitest
= command_context
._spawn
(MochitestRunner
)
555 return mochitest
.run_geckoview_junit_test(command_context
._mach
_context
, **kwargs
)