1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
11 from automationutils
import addCommonOptions
, isURL
12 from mozprofile
import DEFAULT_PORTS
14 here
= os
.path
.abspath(os
.path
.dirname(__file__
))
17 from mozbuild
.base
import MozbuildObject
18 build_obj
= MozbuildObject
.from_environment(cwd
=here
)
22 __all__
= ["MochitestOptions", "B2GOptions"]
24 VMWARE_RECORDING_HELPER_BASENAME
= "vmwarerecordinghelper"
26 class MochitestOptions(optparse
.OptionParser
):
27 """Usage instructions for runtests.py.
28 All arguments are optional.
29 If --chrome is specified, chrome tests will be run instead of web content tests.
30 If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests.
31 See <http://mochikit.com/doc/html/MochiKit/Logging.html> for details on the logging levels.
34 LOG_LEVELS
= ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL")
35 LEVEL_STRING
= ", ".join(LOG_LEVELS
)
37 [["--close-when-done"],
38 { "action": "store_true",
39 "dest": "closeWhenDone",
41 "help": "close the application when tests are done running",
48 "help": "absolute path to application, overriding default",
53 "dest": "utilityPath",
54 "default": build_obj
.bindir
if build_obj
is not None else None,
55 "help": "absolute path to directory containing utility programs (xpcshell, ssltunnel, certutil)",
57 [["--certificate-path"],
61 "help": "absolute path to directory containing certificate store to use testing profile",
62 "default": os
.path
.join(build_obj
.topsrcdir
, 'build', 'pgo', 'certs') if build_obj
is not None else None,
65 { "action": "store_true",
67 "help": "start running tests when the application starts",
73 "help": "per-test timeout in seconds",
78 "dest": "totalChunks",
79 "help": "how many chunks to split the tests up into",
85 "help": "which chunk to run",
91 "help": "group tests together in the same chunk that are in the same top chunkByDir directories",
95 { "action": "store_true",
97 "help": "Run each directory in a single browser instance with a fresh profile",
102 "action": "store_true",
103 "help": "randomize test order",
106 [["--console-level"],
109 "dest": "consoleLevel",
110 "choices": LOG_LEVELS
,
112 "help": "one of %s to determine the level of console "
113 "logging" % LEVEL_STRING
,
117 { "action": "store_true",
119 "help": "run chrome Mochitests",
123 { "action": "store_true",
124 "dest": "ipcplugins",
125 "help": "run ipcplugins Mochitests",
132 "help": "start in the given directory's tests",
138 "dest": "bisectChunk",
139 "help": "Specify the failing test name to find the previous tests that may be causing the failure.",
146 "help": "skip over tests until reaching the given test",
153 "help": "don't run any tests after the given one",
156 [["--browser-chrome"],
157 { "action": "store_true",
158 "dest": "browserChrome",
159 "help": "run browser chrome Mochitests",
165 "help": "subsuite of tests to run",
168 [["--webapprt-content"],
169 { "action": "store_true",
170 "dest": "webapprtContent",
171 "help": "run WebappRT content tests",
174 [["--webapprt-chrome"],
175 { "action": "store_true",
176 "dest": "webapprtChrome",
177 "help": "run WebappRT chrome tests",
181 { "action": "store_true",
183 "help": "run accessibility Mochitests",
187 { "action": "append",
189 "dest": "environment",
190 "metavar": "NAME=VALUE",
191 "help": "sets the given variable in the application's "
195 [["--exclude-extension"],
196 { "action": "append",
198 "dest": "extensionsToExclude",
199 "help": "excludes the given extension from being installed "
200 "in the test profile",
204 { "action": "append",
206 "dest": "browserArgs",
208 "help": "provides an argument to the test application",
211 [["--leak-threshold"],
214 "dest": "leakThreshold",
215 "metavar": "THRESHOLD",
216 "help": "fail if the number of bytes leaked through "
217 "refcounted objects (or bytes in classes with "
218 "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater "
219 "than the given number",
222 [["--fatal-assertions"],
223 { "action": "store_true",
224 "dest": "fatalAssertions",
225 "help": "abort testing whenever an assertion is hit "
226 "(requires a debug build to be effective)",
229 [["--extra-profile-file"],
230 { "action": "append",
231 "dest": "extraProfileFiles",
232 "help": "copy specified files/dirs to testing profile",
235 [["--install-extension"],
236 { "action": "append",
237 "dest": "extensionsToInstall",
238 "help": "install the specified extension in the testing profile."
239 "The extension file's name should be <id>.xpi where <id> is"
240 "the extension's id as indicated in its install.rdf."
241 "An optional path can be specified too.",
247 "dest": "profilePath",
248 "help": "Directory where the profile will be stored."
249 "This directory will be deleted after the tests are finished",
252 [["--testing-modules-dir"],
255 "dest": "testingModulesDir",
256 "help": "Directory where testing-only JS modules are located.",
259 [["--use-vmware-recording"],
260 { "action": "store_true",
261 "dest": "vmwareRecording",
262 "help": "enables recording while the application is running "
263 "inside a VMware Workstation 7.0 or later VM",
271 "help": "repeats the test or set of tests the given number of times, ie: repeat: 1 will run the test twice.",
274 [["--run-until-failure"],
275 { "action": "store_true",
276 "dest": "runUntilFailure",
277 "help": "Run tests repeatedly and stops on the first time a test fails. "
278 "Default cap is 30 runs, which can be overwritten with the --repeat parameter.",
281 [["--run-only-tests"],
284 "dest": "runOnlyTests",
285 "help": "JSON list of tests that we only want to run. [DEPRECATED- please use --test-manifest]",
288 [["--test-manifest"],
291 "dest": "testManifest",
292 "help": "JSON list of tests to specify 'runtests'. Old format for mobile specific tests",
298 "dest": "manifestFile",
299 "help": ".ini format of tests to run.",
305 "dest": "failureFile",
306 "help": "Filename of the output file where we can store a .json list of failures to be run in the future with --run-only-tests.",
310 { "action": "store_true",
312 "help": "Delay execution between test files.",
315 [["--metro-immersive"],
316 { "action": "store_true",
317 "dest": "immersiveMode",
318 "help": "launches tests in immersive browser",
326 "help": "path to the httpd.js file",
329 { "action": "append",
332 "dest": "extraPrefs",
333 "metavar": "PREF=VALUE",
334 "help": "defines an extra user preference",
337 { "action": "store_true",
339 "dest": "jsdebugger",
340 "help": "open the browser debugger",
342 [["--debug-on-failure"],
343 { "action": "store_true",
345 "dest": "debugOnFailure",
346 "help": "breaks execution and enters the JS debugger on a test failure. Should be used together with --jsdebugger."
349 { "action": "store_true",
352 "help": "Run tests with electrolysis preferences and test filtering enabled.",
358 "help": "Specifies the path to the directory containing the shared library for DMD.",
360 [["--dump-output-directory"],
363 "dest": "dumpOutputDirectory",
364 "help": "Specifies the directory in which to place dumped memory reports.",
366 [["--dump-about-memory-after-test"],
367 { "action": "store_true",
369 "dest": "dumpAboutMemoryAfterTest",
370 "help": "Produce an about:memory dump after each test in the directory specified "
371 "by --dump-output-directory."
373 [["--dump-dmd-after-test"],
374 { "action": "store_true",
376 "dest": "dumpDMDAfterTest",
377 "help": "Produce a DMD dump after each test in the directory specified "
378 "by --dump-output-directory."
381 { "action": "store_true",
383 "dest": "slowscript",
384 "help": "Do not set the JS_DISABLE_SLOW_SCRIPT_SIGNALS env variable; "
385 "when not set, recoverable but misleading SIGSEGV instances "
386 "may occur in Ion/Odin JIT code."
388 [["--screenshot-on-fail"],
389 { "action": "store_true",
391 "dest": "screenshotOnFail",
392 "help": "Take screenshots on all test failures. Set $MOZ_UPLOAD_DIR to a directory for storing the screenshots."
395 { "action": "store_true",
398 "help": "Do not print test log lines unless a failure occurs."
404 "help": "name of the pidfile to generate",
407 [["--use-test-media-devices"],
408 { "action": "store_true",
410 "dest": "useTestMediaDevices",
411 "help": "Use test media device drivers for media testing.",
417 "help": "Path to fake GMP plugin. Will be deduced from the binary if not passed.",
421 def __init__(self
, **kwargs
):
423 optparse
.OptionParser
.__init
__(self
, **kwargs
)
424 for option
, value
in self
.mochitest_options
:
425 # Allocate new lists so references to original don't get mutated.
426 # allowing multiple uses within a single process.
427 if "default" in value
and isinstance(value
["default"], list):
428 value
["default"] = []
429 self
.add_option(*option
, **value
)
430 addCommonOptions(self
)
431 self
.set_usage(self
.__doc
__)
433 def verifyOptions(self
, options
, mochitest
):
434 """ verify correct options and cleanup paths """
436 mozinfo
.update({"e10s": options
.e10s
}) # for test manifest parsing.
438 if options
.app
is None:
439 if build_obj
is not None:
440 options
.app
= build_obj
.get_binary_path()
442 self
.error("could not find the application path, --appname must be specified")
444 if options
.totalChunks
is not None and options
.thisChunk
is None:
445 self
.error("thisChunk must be specified when totalChunks is specified")
447 if options
.totalChunks
:
448 if not 1 <= options
.thisChunk
<= options
.totalChunks
:
449 self
.error("thisChunk must be between 1 and totalChunks")
451 if options
.xrePath
is None:
452 # default xrePath to the app path if not provided
453 # but only if an app path was explicitly provided
454 if options
.app
!= self
.defaults
['app']:
455 options
.xrePath
= os
.path
.dirname(options
.app
)
457 options
.xrePath
= os
.path
.join(os
.path
.dirname(options
.xrePath
), "Resources")
458 elif build_obj
is not None:
459 # otherwise default to dist/bin
460 options
.xrePath
= build_obj
.bindir
462 self
.error("could not find xre directory, --xre-path must be specified")
464 # allow relative paths
465 options
.xrePath
= mochitest
.getFullPath(options
.xrePath
)
466 if options
.profilePath
:
467 options
.profilePath
= mochitest
.getFullPath(options
.profilePath
)
468 options
.app
= mochitest
.getFullPath(options
.app
)
469 if options
.dmdPath
is not None:
470 options
.dmdPath
= mochitest
.getFullPath(options
.dmdPath
)
472 if not os
.path
.exists(options
.app
):
474 Error: Path %(app)s doesn't exist.
475 Are you executing $objdir/_tests/testing/mochitest/runtests.py?"""
476 self
.error(msg
% {"app": options
.app
})
479 if options
.utilityPath
:
480 options
.utilityPath
= mochitest
.getFullPath(options
.utilityPath
)
483 options
.certPath
= mochitest
.getFullPath(options
.certPath
)
485 if options
.symbolsPath
and not isURL(options
.symbolsPath
):
486 options
.symbolsPath
= mochitest
.getFullPath(options
.symbolsPath
)
488 # Set server information on the options object
489 options
.webServer
= '127.0.0.1'
490 options
.httpPort
= DEFAULT_PORTS
['http']
491 options
.sslPort
= DEFAULT_PORTS
['https']
492 # options.webSocketPort = DEFAULT_PORTS['ws']
493 options
.webSocketPort
= str(9988) # <- http://hg.mozilla.org/mozilla-central/file/b871dfb2186f/build/automation.py.in#l30
494 # The default websocket port is incorrect in mozprofile; it is
495 # set to the SSL proxy setting. See:
496 # see https://bugzilla.mozilla.org/show_bug.cgi?id=916517
498 if options
.vmwareRecording
:
499 if not mozinfo
.isWin
:
500 self
.error("use-vmware-recording is only supported on Windows.")
501 mochitest
.vmwareHelperPath
= os
.path
.join(
502 options
.utilityPath
, VMWARE_RECORDING_HELPER_BASENAME
+ ".dll")
503 if not os
.path
.exists(mochitest
.vmwareHelperPath
):
504 self
.error("%s not found, cannot automate VMware recording." %
505 mochitest
.vmwareHelperPath
)
507 if options
.testManifest
and options
.runOnlyTests
:
508 self
.error("Please use --test-manifest only and not --run-only-tests")
510 if options
.runOnlyTests
:
511 if not os
.path
.exists(os
.path
.abspath(os
.path
.join(here
, options
.runOnlyTests
))):
512 self
.error("unable to find --run-only-tests file '%s'" % options
.runOnlyTests
)
513 options
.runOnly
= True
514 options
.testManifest
= options
.runOnlyTests
515 options
.runOnlyTests
= None
517 if options
.manifestFile
and options
.testManifest
:
518 self
.error("Unable to support both --manifest and --test-manifest/--run-only-tests at the same time")
520 if options
.webapprtContent
and options
.webapprtChrome
:
521 self
.error("Only one of --webapprt-content and --webapprt-chrome may be given.")
523 if options
.jsdebugger
:
524 options
.extraPrefs
+= [
525 "devtools.debugger.remote-enabled=true",
526 "devtools.debugger.chrome-enabled=true",
527 "devtools.chrome.enabled=true",
528 "devtools.debugger.prompt-connection=false"
530 options
.autorun
= False
532 if options
.debugOnFailure
and not options
.jsdebugger
:
533 self
.error("--debug-on-failure should be used together with --jsdebugger.")
535 # Try to guess the testing modules directory.
536 # This somewhat grotesque hack allows the buildbot machines to find the
537 # modules directory without having to configure the buildbot hosts. This
538 # code should never be executed in local runs because the build system
539 # should always set the flag that populates this variable. If buildbot ever
540 # passes this argument, this code can be deleted.
541 if options
.testingModulesDir
is None:
542 possible
= os
.path
.join(here
, os
.path
.pardir
, 'modules')
544 if os
.path
.isdir(possible
):
545 options
.testingModulesDir
= possible
547 # Even if buildbot is updated, we still want this, as the path we pass in
548 # to the app must be absolute and have proper slashes.
549 if options
.testingModulesDir
is not None:
550 options
.testingModulesDir
= os
.path
.normpath(options
.testingModulesDir
)
552 if not os
.path
.isabs(options
.testingModulesDir
):
553 options
.testingModulesDir
= os
.path
.abspath(options
.testingModulesDir
)
555 if not os
.path
.isdir(options
.testingModulesDir
):
556 self
.error('--testing-modules-dir not a directory: %s' %
557 options
.testingModulesDir
)
559 options
.testingModulesDir
= options
.testingModulesDir
.replace('\\', '/')
560 if options
.testingModulesDir
[-1] != '/':
561 options
.testingModulesDir
+= '/'
563 if options
.immersiveMode
:
564 if not mozinfo
.isWin
:
565 self
.error("immersive is only supported on Windows 8 and up.")
566 mochitest
.immersiveHelperPath
= os
.path
.join(
567 options
.utilityPath
, "metrotestharness.exe")
568 if not os
.path
.exists(mochitest
.immersiveHelperPath
):
569 self
.error("%s not found, cannot launch immersive tests." %
570 mochitest
.immersiveHelperPath
)
572 if options
.runUntilFailure
:
573 if not options
.repeat
:
576 if options
.dumpOutputDirectory
is None:
577 options
.dumpOutputDirectory
= tempfile
.gettempdir()
579 if options
.dumpAboutMemoryAfterTest
or options
.dumpDMDAfterTest
:
580 if not os
.path
.isdir(options
.dumpOutputDirectory
):
581 self
.error('--dump-output-directory not a directory: %s' %
582 options
.dumpOutputDirectory
)
584 if options
.useTestMediaDevices
:
585 if not mozinfo
.isLinux
:
586 self
.error('--use-test-media-devices is only supported on Linux currently')
587 for f
in ['/usr/bin/gst-launch-0.10', '/usr/bin/pactl']:
588 if not os
.path
.isfile(f
):
589 self
.error('Missing binary %s required for --use-test-media-devices')
594 class B2GOptions(MochitestOptions
):
600 "help": "path to B2G repo or qemu dir",
604 { "action": "store_true",
606 "help": "Run the tests on a B2G desktop build",
612 "dest": "marionette",
613 "help": "host:port to use when connecting to Marionette",
620 "help": "Architecture of emulator to use: x86 or arm",
627 "help": "Devine wifi configuration for on device mochitest",
634 "help": "Define size of sdcard: 1MB, 50MB...etc",
638 { "action": "store_true",
640 "help": "Pass --no-window to the emulator",
647 "help": "path to adb",
654 "help": "ip address of remote device to test",
660 "dest": "devicePort",
661 "help": "port of remote device to test",
664 [["--remote-logfile"],
667 "dest": "remoteLogFile",
668 "help": "Name of log file on the device relative to the device root. \
669 PLEASE ONLY USE A FILENAME.",
672 [["--remote-webserver"],
675 "dest": "remoteWebServer",
676 "help": "ip address where the remote web server is hosted at",
683 "help": "ip address where the remote web server is hosted at",
690 "help": "ip address where the remote web server is hosted at",
697 "help": "the path to a gecko distribution that should \
698 be installed on the emulator prior to test",
705 "help": "for desktop testing, the path to the \
706 gaia profile to use",
713 "help": "directory to store log files",
720 "help": "Path to busybox binary to install on device",
723 [['--profile-data-dir'],
726 "dest": 'profile_data_dir',
727 "help": "Path to a directory containing preference and other \
728 data to be installed into the profile",
729 "default": os
.path
.join(here
, 'profile_data'),
734 MochitestOptions
.__init
__(self
)
736 for option
in self
.b2g_options
:
737 self
.add_option(*option
[0], **option
[1])
740 defaults
["httpPort"] = DEFAULT_PORTS
['http']
741 defaults
["sslPort"] = DEFAULT_PORTS
['https']
742 defaults
["logFile"] = "mochitest.log"
743 defaults
["autorun"] = True
744 defaults
["closeWhenDone"] = True
745 defaults
["testPath"] = ""
746 defaults
["extensionsToExclude"] = ["specialpowers"]
747 # See dependencies of bug 1038943.
748 defaults
["leakThreshold"] = 5116
749 self
.set_defaults(**defaults
)
751 def verifyRemoteOptions(self
, options
):
752 if options
.remoteWebServer
== None:
754 options
.remoteWebServer
= moznetwork
.get_ip()
756 self
.error("You must specify a --remote-webserver=<ip address>")
757 options
.webServer
= options
.remoteWebServer
759 if options
.geckoPath
and not options
.emulator
:
760 self
.error("You must specify --emulator if you specify --gecko-path")
762 if options
.logdir
and not options
.emulator
:
763 self
.error("You must specify --emulator if you specify --logdir")
765 if not os
.path
.isdir(options
.xrePath
):
766 self
.error("--xre-path '%s' is not a directory" % options
.xrePath
)
767 xpcshell
= os
.path
.join(options
.xrePath
, 'xpcshell')
768 if not os
.access(xpcshell
, os
.F_OK
):
769 self
.error('xpcshell not found at %s' % xpcshell
)
770 if self
.elf_arm(xpcshell
):
771 self
.error('--xre-path points to an ARM version of xpcshell; it '
772 'should instead point to a version that can run on '
775 if options
.pidFile
!= "":
776 f
= open(options
.pidFile
, 'w')
777 f
.write("%s" % os
.getpid())
782 def verifyOptions(self
, options
, mochitest
):
783 # since we are reusing verifyOptions, it will exit if App is not found
785 options
.app
= __file__
786 tempPort
= options
.httpPort
787 tempSSL
= options
.sslPort
788 tempIP
= options
.webServer
789 options
= MochitestOptions
.verifyOptions(self
, options
, mochitest
)
790 options
.webServer
= tempIP
792 options
.sslPort
= tempSSL
793 options
.httpPort
= tempPort
797 def elf_arm(self
, filename
):
798 data
= open(filename
, 'rb').read(20)
799 return data
[:4] == "\x7fELF" and ord(data
[18]) == 40 # EM_ARM