testing/mochitest/mochitest_options.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import json
   6 import os
   7 import sys
   8 import tempfile
   9 from abc import ABCMeta, abstractmethod, abstractproperty
  10 from argparse import SUPPRESS, ArgumentParser
  11 from distutils import spawn
  12 from distutils.util import strtobool
  13 from itertools import chain
  14
  15 import mozinfo
  16 import mozlog
  17 import moznetwork
  18 import six
  19 from mozprofile import DEFAULT_PORTS
  20 from six.moves.urllib.parse import urlparse
  21
  22 here = os.path.abspath(os.path.dirname(__file__))
  23
  24 try:
  25     from mozbuild.base import MachCommandConditions as conditions
  26     from mozbuild.base import MozbuildObject
  27
  28     build_obj = MozbuildObject.from_environment(cwd=here)
  29 except ImportError:
  30     build_obj = None
  31     conditions = None
  32
  33
  34 # Maps test flavors to data needed to run them
  35 ALL_FLAVORS = {
  36     "mochitest": {
  37         "suite": "plain",
  38         "aliases": ("plain", "mochitest"),
  39         "enabled_apps": ("firefox", "android"),
  40         "extra_args": {
  41             "flavor": "plain",
  42         },
  43         "install_subdir": "tests",
  44     },
  45     "chrome": {
  46         "suite": "chrome",
  47         "aliases": ("chrome", "mochitest-chrome"),
  48         "enabled_apps": ("firefox"),
  49         "extra_args": {
  50             "flavor": "chrome",
  51         },
  52     },
  53     "browser-chrome": {
  54         "suite": "browser",
  55         "aliases": ("browser", "browser-chrome", "mochitest-browser-chrome", "bc"),
  56         "enabled_apps": ("firefox", "thunderbird"),
  57         "extra_args": {
  58             "flavor": "browser",
  59         },
  60     },
  61     "a11y": {
  62         "suite": "a11y",
  63         "aliases": ("a11y", "mochitest-a11y", "accessibility"),
  64         "enabled_apps": ("firefox",),
  65         "extra_args": {
  66             "flavor": "a11y",
  67         },
  68     },
  69 }
  70 SUPPORTED_FLAVORS = list(
  71     chain.from_iterable([f["aliases"] for f in ALL_FLAVORS.values()])
  72 )
  73 CANONICAL_FLAVORS = sorted([f["aliases"][0] for f in ALL_FLAVORS.values()])
  74
  75
  76 def get_default_valgrind_suppression_files():
  77     # We are trying to locate files in the source tree.  So if we
  78     # don't know where the source tree is, we must give up.
  79     #
  80     # When this is being run by |mach mochitest --valgrind ...|, it is
  81     # expected that |build_obj| is not None, and so the logic below will
  82     # select the correct suppression files.
  83     #
  84     # When this is run from mozharness, |build_obj| is None, and we expect
  85     # that testing/mozharness/configs/unittests/linux_unittests.py will
  86     # select the correct suppression files (and paths to them) and
  87     # will specify them using the --valgrind-supp-files= flag.  Hence this
  88     # function will not get called when running from mozharness.
  89     #
  90     # Note: keep these Valgrind .sup file names consistent with those
  91     # in testing/mozharness/configs/unittests/linux_unittest.py.
  92     if build_obj is None or build_obj.topsrcdir is None:
  93         return []
  94
  95     supps_path = os.path.join(build_obj.topsrcdir, "build", "valgrind")
  96
  97     rv = []
  98     if mozinfo.os == "linux":
  99         if mozinfo.processor == "x86_64":
 100             rv.append(os.path.join(supps_path, "x86_64-pc-linux-gnu.sup"))
 101             rv.append(os.path.join(supps_path, "cross-architecture.sup"))
 102         elif mozinfo.processor == "x86":
 103             rv.append(os.path.join(supps_path, "i386-pc-linux-gnu.sup"))
 104             rv.append(os.path.join(supps_path, "cross-architecture.sup"))
 105
 106     return rv
 107
 108
 109 @six.add_metaclass(ABCMeta)
 110 class ArgumentContainer:
 111     @abstractproperty
 112     def args(self):
 113         pass
 114
 115     @abstractproperty
 116     def defaults(self):
 117         pass
 118
 119     @abstractmethod
 120     def validate(self, parser, args, context):
 121         pass
 122
 123     def get_full_path(self, path, cwd):
 124         """Get an absolute path relative to cwd."""
 125         return os.path.normpath(os.path.join(cwd, os.path.expanduser(path)))
 126
 127
 128 class MochitestArguments(ArgumentContainer):
 129     """General mochitest arguments."""
 130
 131     LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL")
 132
 133     args = [
 134         [
 135             ["test_paths"],
 136             {
 137                 "nargs": "*",
 138                 "metavar": "TEST",
 139                 "default": [],
 140                 "help": "Test to run. Can be a single test file or a directory of tests "
 141                 "(to run recursively). If omitted, the entire suite is run.",
 142             },
 143         ],
 144         [
 145             ["-f", "--flavor"],
 146             {
 147                 "choices": SUPPORTED_FLAVORS,
 148                 "metavar": "{{{}}}".format(", ".join(CANONICAL_FLAVORS)),
 149                 "default": None,
 150                 "help": "Only run tests of this flavor.",
 151             },
 152         ],
 153         [
 154             ["--keep-open"],
 155             {
 156                 "nargs": "?",
 157                 "type": strtobool,
 158                 "const": "true",
 159                 "default": None,
 160                 "help": "Always keep the browser open after tests complete. Or always close the "
 161                 "browser with --keep-open=false",
 162             },
 163         ],
 164         [
 165             ["--appname"],
 166             {
 167                 "dest": "app",
 168                 "default": None,
 169                 "help": (
 170                     "Override the default binary used to run tests with the path provided, e.g "
 171                     "/usr/bin/firefox. If you have run ./mach package beforehand, you can "
 172                     "specify 'dist' to run tests against the distribution bundle's binary."
 173                 ),
 174             },
 175         ],
 176         [
 177             ["--utility-path"],
 178             {
 179                 "dest": "utilityPath",
 180                 "default": build_obj.bindir if build_obj is not None else None,
 181                 "help": "absolute path to directory containing utility programs "
 182                 "(xpcshell, ssltunnel, certutil)",
 183                 "suppress": True,
 184             },
 185         ],
 186         [
 187             ["--certificate-path"],
 188             {
 189                 "dest": "certPath",
 190                 "default": None,
 191                 "help": "absolute path to directory containing certificate store to use testing profile",  # NOQA: E501
 192                 "suppress": True,
 193             },
 194         ],
 195         [
 196             ["--no-autorun"],
 197             {
 198                 "action": "store_false",
 199                 "dest": "autorun",
 200                 "default": True,
 201                 "help": "Do not start running tests automatically.",
 202             },
 203         ],
 204         [
 205             ["--timeout"],
 206             {
 207                 "type": int,
 208                 "default": None,
 209                 "help": "The per-test timeout in seconds (default: 60 seconds).",
 210             },
 211         ],
 212         [
 213             ["--max-timeouts"],
 214             {
 215                 "type": int,
 216                 "dest": "maxTimeouts",
 217                 "default": None,
 218                 "help": "The maximum number of timeouts permitted before halting testing.",
 219             },
 220         ],
 221         [
 222             ["--total-chunks"],
 223             {
 224                 "type": int,
 225                 "dest": "totalChunks",
 226                 "help": "Total number of chunks to split tests into.",
 227                 "default": None,
 228             },
 229         ],
 230         [
 231             ["--this-chunk"],
 232             {
 233                 "type": int,
 234                 "dest": "thisChunk",
 235                 "help": "If running tests by chunks, the chunk number to run.",
 236                 "default": None,
 237             },
 238         ],
 239         [
 240             ["--chunk-by-runtime"],
 241             {
 242                 "action": "store_true",
 243                 "dest": "chunkByRuntime",
 244                 "help": "Group tests such that each chunk has roughly the same runtime.",
 245                 "default": False,
 246             },
 247         ],
 248         [
 249             ["--chunk-by-dir"],
 250             {
 251                 "type": int,
 252                 "dest": "chunkByDir",
 253                 "help": "Group tests together in the same chunk that are in the same top "
 254                 "chunkByDir directories.",
 255                 "default": 0,
 256             },
 257         ],
 258         [
 259             ["--run-by-manifest"],
 260             {
 261                 "action": "store_true",
 262                 "dest": "runByManifest",
 263                 "help": "Run each manifest in a single browser instance with a fresh profile.",
 264                 "default": False,
 265                 "suppress": True,
 266             },
 267         ],
 268         [
 269             ["--shuffle"],
 270             {
 271                 "action": "store_true",
 272                 "help": "Shuffle execution order of tests.",
 273                 "default": False,
 274             },
 275         ],
 276         [
 277             ["--console-level"],
 278             {
 279                 "dest": "consoleLevel",
 280                 "choices": LOG_LEVELS,
 281                 "default": "INFO",
 282                 "help": "One of {} to determine the level of console logging.".format(
 283                     ", ".join(LOG_LEVELS)
 284                 ),
 285                 "suppress": True,
 286             },
 287         ],
 288         [
 289             ["--bisect-chunk"],
 290             {
 291                 "dest": "bisectChunk",
 292                 "default": None,
 293                 "help": "Specify the failing test name to find the previous tests that may be "
 294                 "causing the failure.",
 295             },
 296         ],
 297         [
 298             ["--start-at"],
 299             {
 300                 "dest": "startAt",
 301                 "default": "",
 302                 "help": "Start running the test sequence at this test.",
 303             },
 304         ],
 305         [
 306             ["--end-at"],
 307             {
 308                 "dest": "endAt",
 309                 "default": "",
 310                 "help": "Stop running the test sequence at this test.",
 311             },
 312         ],
 313         [
 314             ["--subsuite"],
 315             {
 316                 "default": None,
 317                 "help": "Subsuite of tests to run. Unlike tags, subsuites also remove tests from "
 318                 "the default set. Only one can be specified at once.",
 319             },
 320         ],
 321         [
 322             ["--setenv"],
 323             {
 324                 "action": "append",
 325                 "dest": "environment",
 326                 "metavar": "NAME=VALUE",
 327                 "default": [],
 328                 "help": "Sets the given variable in the application's environment.",
 329             },
 330         ],
 331         [
 332             ["--exclude-extension"],
 333             {
 334                 "action": "append",
 335                 "dest": "extensionsToExclude",
 336                 "default": [],
 337                 "help": "Excludes the given extension from being installed in the test profile.",
 338                 "suppress": True,
 339             },
 340         ],
 341         [
 342             ["--browser-arg"],
 343             {
 344                 "action": "append",
 345                 "dest": "browserArgs",
 346                 "default": [],
 347                 "help": "Provides an argument to the test application (e.g Firefox).",
 348                 "suppress": True,
 349             },
 350         ],
 351         [
 352             ["--leak-threshold"],
 353             {
 354                 "type": int,
 355                 "dest": "defaultLeakThreshold",
 356                 "default": 0,
 357                 "help": "Fail if the number of bytes leaked in default processes through "
 358                 "refcounted objects (or bytes in classes with MOZ_COUNT_CTOR and "
 359                 "MOZ_COUNT_DTOR) is greater than the given number.",
 360                 "suppress": True,
 361             },
 362         ],
 363         [
 364             ["--fatal-assertions"],
 365             {
 366                 "action": "store_true",
 367                 "dest": "fatalAssertions",
 368                 "default": False,
 369                 "help": "Abort testing whenever an assertion is hit (requires a debug build to "
 370                 "be effective).",
 371                 "suppress": True,
 372             },
 373         ],
 374         [
 375             ["--extra-profile-file"],
 376             {
 377                 "action": "append",
 378                 "dest": "extraProfileFiles",
 379                 "default": [],
 380                 "help": "Copy specified files/dirs to testing profile. Can be specified more "
 381                 "than once.",
 382                 "suppress": True,
 383             },
 384         ],
 385         [
 386             ["--install-extension"],
 387             {
 388                 "action": "append",
 389                 "dest": "extensionsToInstall",
 390                 "default": [],
 391                 "help": "Install the specified extension in the testing profile. Can be a path "
 392                 "to a .xpi file.",
 393             },
 394         ],
 395         [
 396             ["--profile-path"],
 397             {
 398                 "dest": "profilePath",
 399                 "default": None,
 400                 "help": "Directory where the profile will be stored. This directory will be "
 401                 "deleted after the tests are finished.",
 402                 "suppress": True,
 403             },
 404         ],
 405         [
 406             ["--conditioned-profile"],
 407             {
 408                 "dest": "conditionedProfile",
 409                 "action": "store_true",
 410                 "default": False,
 411                 "help": "Download and run with a full conditioned profile.",
 412             },
 413         ],
 414         [
 415             ["--testing-modules-dir"],
 416             {
 417                 "dest": "testingModulesDir",
 418                 "default": None,
 419                 "help": "Directory where testing-only JS modules are located.",
 420                 "suppress": True,
 421             },
 422         ],
 423         [
 424             ["--repeat"],
 425             {
 426                 "type": int,
 427                 "default": 0,
 428                 "help": "Repeat the tests the given number of times.",
 429             },
 430         ],
 431         [
 432             ["--run-until-failure"],
 433             {
 434                 "action": "store_true",
 435                 "dest": "runUntilFailure",
 436                 "default": False,
 437                 "help": "Run tests repeatedly but stop the first time a test fails. Default cap "
 438                 "is 30 runs, which can be overridden with the --repeat parameter.",
 439             },
 440         ],
 441         [
 442             ["--manifest"],
 443             {
 444                 "dest": "manifestFile",
 445                 "default": None,
 446                 "help": "Path to a manifestparser (.ini formatted) manifest of tests to run.",
 447                 "suppress": True,
 448             },
 449         ],
 450         [
 451             ["--extra-mozinfo-json"],
 452             {
 453                 "dest": "extra_mozinfo_json",
 454                 "default": None,
 455                 "help": "Filter tests based on a given mozinfo file.",
 456                 "suppress": True,
 457             },
 458         ],
 459         [
 460             ["--testrun-manifest-file"],
 461             {
 462                 "dest": "testRunManifestFile",
 463                 "default": "tests.json",
 464                 "help": "Overrides the default filename of the tests.json manifest file that is "
 465                 "generated by the harness and used by SimpleTest. Only useful when running "
 466                 "multiple test runs simulatenously on the same machine.",
 467                 "suppress": True,
 468             },
 469         ],
 470         [
 471             ["--dump-tests"],
 472             {
 473                 "dest": "dump_tests",
 474                 "default": None,
 475                 "help": "Specify path to a filename to dump all the tests that will be run",
 476                 "suppress": True,
 477             },
 478         ],
 479         [
 480             ["--failure-file"],
 481             {
 482                 "dest": "failureFile",
 483                 "default": None,
 484                 "help": "Filename of the output file where we can store a .json list of failures "
 485                 "to be run in the future with --run-only-tests.",
 486                 "suppress": True,
 487             },
 488         ],
 489         [
 490             ["--run-slower"],
 491             {
 492                 "action": "store_true",
 493                 "dest": "runSlower",
 494                 "default": False,
 495                 "help": "Delay execution between tests.",
 496             },
 497         ],
 498         [
 499             ["--httpd-path"],
 500             {
 501                 "dest": "httpdPath",
 502                 "default": None,
 503                 "help": "Path to the httpd.js file.",
 504                 "suppress": True,
 505             },
 506         ],
 507         [
 508             ["--use-http3-server"],
 509             {
 510                 "dest": "useHttp3Server",
 511                 "default": False,
 512                 "help": "Whether to use the Http3 server",
 513                 "action": "store_true",
 514             },
 515         ],
 516         [
 517             ["--use-http2-server"],
 518             {
 519                 "dest": "useHttp2Server",
 520                 "default": False,
 521                 "help": "Whether to use the Http2 server",
 522                 "action": "store_true",
 523             },
 524         ],
 525         [
 526             ["--setpref"],
 527             {
 528                 "action": "append",
 529                 "metavar": "PREF=VALUE",
 530                 "default": [],
 531                 "dest": "extraPrefs",
 532                 "help": "Defines an extra user preference.",
 533             },
 534         ],
 535         [
 536             ["--jsconsole"],
 537             {
 538                 "action": "store_true",
 539                 "default": False,
 540                 "help": "Open the Browser Console.",
 541             },
 542         ],
 543         [
 544             ["--jsdebugger"],
 545             {
 546                 "action": "store_true",
 547                 "default": False,
 548                 "help": "Start the browser JS debugger before running the test.",
 549             },
 550         ],
 551         [
 552             ["--jsdebugger-path"],
 553             {
 554                 "default": None,
 555                 "dest": "jsdebuggerPath",
 556                 "help": "Path to a Firefox binary that will be used to run the toolbox. Should "
 557                 "be used together with --jsdebugger.",
 558             },
 559         ],
 560         [
 561             ["--debug-on-failure"],
 562             {
 563                 "action": "store_true",
 564                 "default": False,
 565                 "dest": "debugOnFailure",
 566                 "help": "Breaks execution and enters the JS debugger on a test failure. Should "
 567                 "be used together with --jsdebugger.",
 568             },
 569         ],
 570         [
 571             ["--disable-e10s"],
 572             {
 573                 "action": "store_false",
 574                 "default": True,
 575                 "dest": "e10s",
 576                 "help": "Run tests with electrolysis preferences and test filtering disabled.",
 577             },
 578         ],
 579         [
 580             ["--enable-a11y-checks"],
 581             {
 582                 "action": "store_true",
 583                 "default": False,
 584                 "dest": "a11y_checks",
 585                 "help": "Run tests with accessibility checks enabled.",
 586             },
 587         ],
 588         [
 589             ["--disable-fission"],
 590             {
 591                 "action": "store_true",
 592                 "default": False,
 593                 "dest": "disable_fission",
 594                 "help": "Run tests with fission (site isolation) disabled.",
 595             },
 596         ],
 597         [
 598             ["--enable-xorigin-tests"],
 599             {
 600                 "action": "store_true",
 601                 "default": False,
 602                 "dest": "xOriginTests",
 603                 "help": "Run tests in a cross origin iframe.",
 604             },
 605         ],
 606         [
 607             ["--store-chrome-manifest"],
 608             {
 609                 "action": "store",
 610                 "help": "Destination path to write a copy of any chrome manifest "
 611                 "written by the harness.",
 612                 "default": None,
 613                 "suppress": True,
 614             },
 615         ],
 616         [
 617             ["--jscov-dir-prefix"],
 618             {
 619                 "action": "store",
 620                 "help": "Directory to store per-test line coverage data as json "
 621                 "(browser-chrome only). To emit lcov formatted data, set "
 622                 "JS_CODE_COVERAGE_OUTPUT_DIR in the environment.",
 623                 "default": None,
 624                 "suppress": True,
 625             },
 626         ],
 627         [
 628             ["--dmd"],
 629             {
 630                 "action": "store_true",
 631                 "default": False,
 632                 "help": "Run tests with DMD active.",
 633             },
 634         ],
 635         [
 636             ["--dump-output-directory"],
 637             {
 638                 "default": None,
 639                 "dest": "dumpOutputDirectory",
 640                 "help": "Specifies the directory in which to place dumped memory reports.",
 641             },
 642         ],
 643         [
 644             ["--dump-about-memory-after-test"],
 645             {
 646                 "action": "store_true",
 647                 "default": False,
 648                 "dest": "dumpAboutMemoryAfterTest",
 649                 "help": "Dump an about:memory log after each test in the directory specified "
 650                 "by --dump-output-directory.",
 651             },
 652         ],
 653         [
 654             ["--dump-dmd-after-test"],
 655             {
 656                 "action": "store_true",
 657                 "default": False,
 658                 "dest": "dumpDMDAfterTest",
 659                 "help": "Dump a DMD log (and an accompanying about:memory log) after each test. "
 660                 "These will be dumped into your default temp directory, NOT the directory "
 661                 "specified by --dump-output-directory. The logs are numbered by test, and "
 662                 "each test will include output that indicates the DMD output filename.",
 663             },
 664         ],
 665         [
 666             ["--screenshot-on-fail"],
 667             {
 668                 "action": "store_true",
 669                 "default": False,
 670                 "dest": "screenshotOnFail",
 671                 "help": "Take screenshots on all test failures. Set $MOZ_UPLOAD_DIR to a directory "  # NOQA: E501
 672                 "for storing the screenshots.",
 673             },
 674         ],
 675         [
 676             ["--quiet"],
 677             {
 678                 "action": "store_true",
 679                 "dest": "quiet",
 680                 "default": False,
 681                 "help": "Do not print test log lines unless a failure occurs.",
 682             },
 683         ],
 684         [
 685             ["--headless"],
 686             {
 687                 "action": "store_true",
 688                 "dest": "headless",
 689                 "default": False,
 690                 "help": "Run tests in headless mode.",
 691             },
 692         ],
 693         [
 694             ["--pidfile"],
 695             {
 696                 "dest": "pidFile",
 697                 "default": "",
 698                 "help": "Name of the pidfile to generate.",
 699                 "suppress": True,
 700             },
 701         ],
 702         [
 703             ["--use-test-media-devices"],
 704             {
 705                 "action": "store_true",
 706                 "default": False,
 707                 "dest": "useTestMediaDevices",
 708                 "help": "Use test media device drivers for media testing.",
 709             },
 710         ],
 711         [
 712             ["--gmp-path"],
 713             {
 714                 "default": None,
 715                 "help": "Path to fake GMP plugin. Will be deduced from the binary if not passed.",
 716                 "suppress": True,
 717             },
 718         ],
 719         [
 720             ["--xre-path"],
 721             {
 722                 "dest": "xrePath",
 723                 "default": None,  # individual scripts will set a sane default
 724                 "help": "Absolute path to directory containing XRE (probably xulrunner).",
 725                 "suppress": True,
 726             },
 727         ],
 728         [
 729             ["--symbols-path"],
 730             {
 731                 "dest": "symbolsPath",
 732                 "default": None,
 733                 "help": "Absolute path to directory containing breakpad symbols, or the URL of a "
 734                 "zip file containing symbols",
 735                 "suppress": True,
 736             },
 737         ],
 738         [
 739             ["--debugger"],
 740             {
 741                 "default": None,
 742                 "help": "Debugger binary to run tests in. Program name or path.",
 743             },
 744         ],
 745         [
 746             ["--debugger-args"],
 747             {
 748                 "dest": "debuggerArgs",
 749                 "default": None,
 750                 "help": "Arguments to pass to the debugger.",
 751             },
 752         ],
 753         [
 754             ["--valgrind"],
 755             {
 756                 "default": None,
 757                 "help": "Valgrind binary to run tests with. Program name or path.",
 758             },
 759         ],
 760         [
 761             ["--valgrind-args"],
 762             {
 763                 "dest": "valgrindArgs",
 764                 "default": None,
 765                 "help": "Comma-separated list of extra arguments to pass to Valgrind.",
 766             },
 767         ],
 768         [
 769             ["--valgrind-supp-files"],
 770             {
 771                 "dest": "valgrindSuppFiles",
 772                 "default": None,
 773                 "help": "Comma-separated list of suppression files to pass to Valgrind.",
 774             },
 775         ],
 776         [
 777             ["--debugger-interactive"],
 778             {
 779                 "action": "store_true",
 780                 "dest": "debuggerInteractive",
 781                 "default": None,
 782                 "help": "Prevents the test harness from redirecting stdout and stderr for "
 783                 "interactive debuggers.",
 784                 "suppress": True,
 785             },
 786         ],
 787         [
 788             ["--tag"],
 789             {
 790                 "action": "append",
 791                 "dest": "test_tags",
 792                 "default": None,
 793                 "help": "Filter out tests that don't have the given tag. Can be used multiple "
 794                 "times in which case the test must contain at least one of the given tags.",
 795             },
 796         ],
 797         [
 798             ["--marionette"],
 799             {
 800                 "default": None,
 801                 "help": "host:port to use when connecting to Marionette",
 802             },
 803         ],
 804         [
 805             ["--marionette-socket-timeout"],
 806             {
 807                 "default": None,
 808                 "help": "Timeout while waiting to receive a message from the marionette server.",
 809                 "suppress": True,
 810             },
 811         ],
 812         [
 813             ["--marionette-startup-timeout"],
 814             {
 815                 "default": None,
 816                 "help": "Timeout while waiting for marionette server startup.",
 817                 "suppress": True,
 818             },
 819         ],
 820         [
 821             ["--cleanup-crashes"],
 822             {
 823                 "action": "store_true",
 824                 "dest": "cleanupCrashes",
 825                 "default": False,
 826                 "help": "Delete pending crash reports before running tests.",
 827                 "suppress": True,
 828             },
 829         ],
 830         [
 831             ["--websocket-process-bridge-port"],
 832             {
 833                 "default": "8191",
 834                 "dest": "websocket_process_bridge_port",
 835                 "help": "Port for websocket/process bridge. Default 8191.",
 836             },
 837         ],
 838         [
 839             ["--failure-pattern-file"],
 840             {
 841                 "default": None,
 842                 "dest": "failure_pattern_file",
 843                 "help": "File describes all failure patterns of the tests.",
 844                 "suppress": True,
 845             },
 846         ],
 847         [
 848             ["--sandbox-read-whitelist"],
 849             {
 850                 "default": [],
 851                 "dest": "sandboxReadWhitelist",
 852                 "action": "append",
 853                 "help": "Path to add to the sandbox whitelist.",
 854                 "suppress": True,
 855             },
 856         ],
 857         [
 858             ["--verify"],
 859             {
 860                 "action": "store_true",
 861                 "default": False,
 862                 "help": "Run tests in verification mode: Run many times in different "
 863                 "ways, to see if there are intermittent failures.",
 864             },
 865         ],
 866         [
 867             ["--verify-fission"],
 868             {
 869                 "action": "store_true",
 870                 "default": False,
 871                 "help": "Run tests once without Fission, once with Fission",
 872             },
 873         ],
 874         [
 875             ["--verify-max-time"],
 876             {
 877                 "type": int,
 878                 "default": 3600,
 879                 "help": "Maximum time, in seconds, to run in --verify mode.",
 880             },
 881         ],
 882         [
 883             ["--profiler"],
 884             {
 885                 "action": "store_true",
 886                 "dest": "profiler",
 887                 "default": False,
 888                 "help": "Run the Firefox Profiler and get a performance profile of the "
 889                 "mochitest. This is useful to find performance issues, and also "
 890                 "to see what exactly the test is doing. To get profiler options run: "
 891                 "`MOZ_PROFILER_HELP=1 ./mach run`",
 892             },
 893         ],
 894         [
 895             ["--profiler-save-only"],
 896             {
 897                 "action": "store_true",
 898                 "dest": "profilerSaveOnly",
 899                 "default": False,
 900                 "help": "Run the Firefox Profiler and save it to the path specified by the "
 901                 "MOZ_UPLOAD_DIR environment variable.",
 902             },
 903         ],
 904         [
 905             ["--run-failures"],
 906             {
 907                 "action": "store",
 908                 "dest": "runFailures",
 909                 "default": "",
 910                 "help": "Run fail-if/skip-if tests that match a keyword given.",
 911             },
 912         ],
 913         [
 914             ["--timeout-as-pass"],
 915             {
 916                 "action": "store_true",
 917                 "dest": "timeoutAsPass",
 918                 "default": False,
 919                 "help": "treat harness level timeouts as passing (used for quarantine jobs).",
 920             },
 921         ],
 922         [
 923             ["--crash-as-pass"],
 924             {
 925                 "action": "store_true",
 926                 "dest": "crashAsPass",
 927                 "default": False,
 928                 "help": "treat harness level crashes as passing (used for quarantine jobs).",
 929             },
 930         ],
 931         [
 932             ["--compare-preferences"],
 933             {
 934                 "action": "store_true",
 935                 "dest": "comparePrefs",
 936                 "default": False,
 937                 "help": "Compare preferences at the end of each test and report changed ones as failures.",
 938             },
 939         ],
 940         [
 941             ["--restart-after-failure"],
 942             {
 943                 "dest": "restartAfterFailure",
 944                 "default": False,
 945                 "help": "Terminate the session on first failure and restart where you left off.",
 946             },
 947         ],
 948     ]
 949
 950     defaults = {
 951         # Bug 1065098 - The gmplugin process fails to produce a leak
 952         # log for some reason.
 953         "ignoreMissingLeaks": ["gmplugin"],
 954         "extensionsToExclude": ["specialpowers"],
 955         # Set server information on the args object
 956         "webServer": "127.0.0.1",
 957         "httpPort": DEFAULT_PORTS["http"],
 958         "sslPort": DEFAULT_PORTS["https"],
 959         "webSocketPort": "9988",
 960         # The default websocket port is incorrect in mozprofile; it is
 961         # set to the SSL proxy setting. See:
 962         # see https://bugzilla.mozilla.org/show_bug.cgi?id=916517
 963         # args.webSocketPort = DEFAULT_PORTS['ws']
 964     }
 965
 966     def validate(self, parser, options, context):
 967         """Validate generic options."""
 968
 969         # and android doesn't use 'app' the same way, so skip validation
 970         if parser.app != "android":
 971             if options.app is None:
 972                 if build_obj:
 973                     from mozbuild.base import BinaryNotFoundException
 974
 975                     try:
 976                         options.app = build_obj.get_binary_path()
 977                     except BinaryNotFoundException as e:
 978                         print("{}\n\n{}\n".format(e, e.help()))
 979                         sys.exit(1)
 980                 else:
 981                     parser.error(
 982                         "could not find the application path, --appname must be specified"
 983                     )
 984             elif options.app == "dist" and build_obj:
 985                 options.app = build_obj.get_binary_path(where="staged-package")
 986
 987             options.app = self.get_full_path(options.app, parser.oldcwd)
 988             if not os.path.exists(options.app):
 989                 parser.error(
 990                     "Error: Path {} doesn't exist. Are you executing "
 991                     "$objdir/_tests/testing/mochitest/runtests.py?".format(options.app)
 992                 )
 993
 994         if options.flavor is None:
 995             options.flavor = "plain"
 996
 997         for value in ALL_FLAVORS.values():
 998             if options.flavor in value["aliases"]:
 999                 options.flavor = value["suite"]
1000                 break
1001
1002         if options.gmp_path is None and options.app and build_obj:
1003             # Need to fix the location of gmp_fake which might not be shipped in the binary
1004             gmp_modules = (
1005                 ("gmp-fake", "1.0"),
1006                 ("gmp-clearkey", "0.1"),
1007                 ("gmp-fakeopenh264", "1.0"),
1008             )
1009             options.gmp_path = os.pathsep.join(
1010                 os.path.join(build_obj.bindir, *p) for p in gmp_modules
1011             )
1012
1013         if options.totalChunks is not None and options.thisChunk is None:
1014             parser.error("thisChunk must be specified when totalChunks is specified")
1015
1016         if options.extra_mozinfo_json:
1017             if not os.path.isfile(options.extra_mozinfo_json):
1018                 parser.error(
1019                     "Error: couldn't find mozinfo.json at '%s'."
1020                     % options.extra_mozinfo_json
1021                 )
1022
1023             options.extra_mozinfo_json = json.load(open(options.extra_mozinfo_json))
1024
1025         if options.totalChunks:
1026             if not 1 <= options.thisChunk <= options.totalChunks:
1027                 parser.error("thisChunk must be between 1 and totalChunks")
1028
1029         if options.chunkByDir and options.chunkByRuntime:
1030             parser.error("can only use one of --chunk-by-dir or --chunk-by-runtime")
1031
1032         if options.xrePath is None:
1033             # default xrePath to the app path if not provided
1034             # but only if an app path was explicitly provided
1035             if options.app != parser.get_default("app"):
1036                 options.xrePath = os.path.dirname(options.app)
1037                 if mozinfo.isMac:
1038                     options.xrePath = os.path.join(
1039                         os.path.dirname(options.xrePath), "Resources"
1040                     )
1041             elif build_obj is not None:
1042                 # otherwise default to dist/bin
1043                 options.xrePath = build_obj.bindir
1044             else:
1045                 parser.error(
1046                     "could not find xre directory, --xre-path must be specified"
1047                 )
1048
1049         # allow relative paths
1050         if options.xrePath:
1051             options.xrePath = self.get_full_path(options.xrePath, parser.oldcwd)
1052
1053         if options.profilePath:
1054             options.profilePath = self.get_full_path(options.profilePath, parser.oldcwd)
1055
1056         if options.utilityPath:
1057             options.utilityPath = self.get_full_path(options.utilityPath, parser.oldcwd)
1058
1059         if options.certPath:
1060             options.certPath = self.get_full_path(options.certPath, parser.oldcwd)
1061         elif build_obj:
1062             options.certPath = os.path.join(
1063                 build_obj.topsrcdir, "build", "pgo", "certs"
1064             )
1065
1066         if options.symbolsPath and len(urlparse(options.symbolsPath).scheme) < 2:
1067             options.symbolsPath = self.get_full_path(options.symbolsPath, parser.oldcwd)
1068         elif not options.symbolsPath and build_obj:
1069             options.symbolsPath = os.path.join(
1070                 build_obj.distdir, "crashreporter-symbols"
1071             )
1072
1073         if options.debugOnFailure and not options.jsdebugger:
1074             parser.error("--debug-on-failure requires --jsdebugger.")
1075
1076         if options.jsdebuggerPath and not options.jsdebugger:
1077             parser.error("--jsdebugger-path requires --jsdebugger.")
1078
1079         if options.debuggerArgs and not options.debugger:
1080             parser.error("--debugger-args requires --debugger.")
1081
1082         if options.valgrind or options.debugger:
1083             # valgrind and some debuggers may cause Gecko to start slowly. Make sure
1084             # marionette waits long enough to connect.
1085             options.marionette_startup_timeout = 900
1086             options.marionette_socket_timeout = 540
1087
1088         if options.store_chrome_manifest:
1089             options.store_chrome_manifest = os.path.abspath(
1090                 options.store_chrome_manifest
1091             )
1092             if not os.path.isdir(os.path.dirname(options.store_chrome_manifest)):
1093                 parser.error(
1094                     "directory for %s does not exist as a destination to copy a "
1095                     "chrome manifest." % options.store_chrome_manifest
1096                 )
1097
1098         if options.jscov_dir_prefix:
1099             options.jscov_dir_prefix = os.path.abspath(options.jscov_dir_prefix)
1100             if not os.path.isdir(options.jscov_dir_prefix):
1101                 parser.error(
1102                     "directory %s does not exist as a destination for coverage "
1103                     "data." % options.jscov_dir_prefix
1104                 )
1105
1106         if options.testingModulesDir is None:
1107             # Try to guess the testing modules directory.
1108             possible = [os.path.join(here, os.path.pardir, "modules")]
1109             if build_obj:
1110                 possible.insert(
1111                     0, os.path.join(build_obj.topobjdir, "_tests", "modules")
1112                 )
1113
1114             for p in possible:
1115                 if os.path.isdir(p):
1116                     options.testingModulesDir = p
1117                     break
1118
1119         # Paths to specialpowers and mochijar from the tests archive.
1120         options.stagedAddons = [
1121             os.path.join(here, "extensions", "specialpowers"),
1122             os.path.join(here, "mochijar"),
1123         ]
1124         if build_obj:
1125             objdir_xpi_stage = os.path.join(build_obj.distdir, "xpi-stage")
1126             if os.path.isdir(objdir_xpi_stage):
1127                 options.stagedAddons = [
1128                     os.path.join(objdir_xpi_stage, "specialpowers"),
1129                     os.path.join(objdir_xpi_stage, "mochijar"),
1130                 ]
1131             plugins_dir = os.path.join(build_obj.distdir, "plugins")
1132             if (
1133                 os.path.isdir(plugins_dir)
1134                 and plugins_dir not in options.extraProfileFiles
1135             ):
1136                 options.extraProfileFiles.append(plugins_dir)
1137
1138         # Even if buildbot is updated, we still want this, as the path we pass in
1139         # to the app must be absolute and have proper slashes.
1140         if options.testingModulesDir is not None:
1141             options.testingModulesDir = os.path.normpath(options.testingModulesDir)
1142
1143             if not os.path.isabs(options.testingModulesDir):
1144                 options.testingModulesDir = os.path.abspath(options.testingModulesDir)
1145
1146             if not os.path.isdir(options.testingModulesDir):
1147                 parser.error(
1148                     "--testing-modules-dir not a directory: %s"
1149                     % options.testingModulesDir
1150                 )
1151
1152             options.testingModulesDir = options.testingModulesDir.replace("\\", "/")
1153             if options.testingModulesDir[-1] != "/":
1154                 options.testingModulesDir += "/"
1155
1156         if options.runUntilFailure:
1157             if not options.repeat:
1158                 options.repeat = 29
1159
1160         if options.dumpOutputDirectory is None:
1161             options.dumpOutputDirectory = tempfile.gettempdir()
1162
1163         if options.dumpAboutMemoryAfterTest or options.dumpDMDAfterTest:
1164             if not os.path.isdir(options.dumpOutputDirectory):
1165                 parser.error(
1166                     "--dump-output-directory not a directory: %s"
1167                     % options.dumpOutputDirectory
1168                 )
1169
1170         if options.useTestMediaDevices:
1171             if not mozinfo.isLinux:
1172                 parser.error(
1173                     "--use-test-media-devices is only supported on Linux currently"
1174                 )
1175
1176             gst01 = spawn.find_executable("gst-launch-0.1")
1177             gst010 = spawn.find_executable("gst-launch-0.10")
1178             gst10 = spawn.find_executable("gst-launch-1.0")
1179             pactl = spawn.find_executable("pactl")
1180
1181             if not (gst01 or gst10 or gst010):
1182                 parser.error(
1183                     "Missing gst-launch-{0.1,0.10,1.0}, required for "
1184                     "--use-test-media-devices"
1185                 )
1186
1187             if not pactl:
1188                 parser.error(
1189                     "Missing binary pactl required for " "--use-test-media-devices"
1190                 )
1191
1192         # The a11y and chrome flavors can't run with e10s.
1193         if options.flavor in ("a11y", "chrome") and options.e10s:
1194             parser.error(
1195                 "mochitest-{} does not support e10s, try again with "
1196                 "--disable-e10s.".format(options.flavor)
1197             )
1198
1199         # If e10s explicitly disabled and no fission option specified, disable fission
1200         if (not options.e10s) and (not options.disable_fission):
1201             options.disable_fission = True
1202
1203         options.leakThresholds = {
1204             "default": options.defaultLeakThreshold,
1205             "tab": options.defaultLeakThreshold,
1206             "forkserver": options.defaultLeakThreshold,
1207             # GMP rarely gets a log, but when it does, it leaks a little.
1208             "gmplugin": 20000,
1209         }
1210
1211         # See the dependencies of bug 1401764.
1212         if mozinfo.isWin:
1213             options.leakThresholds["tab"] = 1000
1214
1215         # XXX We can't normalize test_paths in the non build_obj case here,
1216         # because testRoot depends on the flavor, which is determined by the
1217         # mach command and therefore not finalized yet. Conversely, test paths
1218         # need to be normalized here for the mach case.
1219         if options.test_paths and build_obj:
1220             # Normalize test paths so they are relative to test root
1221             options.test_paths = [
1222                 build_obj._wrap_path_argument(p).relpath() for p in options.test_paths
1223             ]
1224
1225         return options
1226
1227
1228 class AndroidArguments(ArgumentContainer):
1229     """Android specific arguments."""
1230
1231     args = [
1232         [
1233             ["--no-install"],
1234             {
1235                 "action": "store_true",
1236                 "default": False,
1237                 "help": "Skip the installation of the APK.",
1238             },
1239         ],
1240         [
1241             ["--aab"],
1242             {
1243                 "action": "store_true",
1244                 "default": False,
1245                 "help": "Install the test_runner app using AAB.",
1246             },
1247         ],
1248         [
1249             ["--deviceSerial"],
1250             {
1251                 "dest": "deviceSerial",
1252                 "help": "adb serial number of remote device. This is required "
1253                 "when more than one device is connected to the host. "
1254                 "Use 'adb devices' to see connected devices.",
1255                 "default": None,
1256             },
1257         ],
1258         [
1259             ["--adbpath"],
1260             {
1261                 "dest": "adbPath",
1262                 "default": None,
1263                 "help": "Path to adb binary.",
1264                 "suppress": True,
1265             },
1266         ],
1267         [
1268             ["--remote-webserver"],
1269             {
1270                 "dest": "remoteWebServer",
1271                 "default": None,
1272                 "help": "IP address of the remote web server.",
1273             },
1274         ],
1275         [
1276             ["--http-port"],
1277             {
1278                 "dest": "httpPort",
1279                 "default": DEFAULT_PORTS["http"],
1280                 "help": "http port of the remote web server.",
1281                 "suppress": True,
1282             },
1283         ],
1284         [
1285             ["--ssl-port"],
1286             {
1287                 "dest": "sslPort",
1288                 "default": DEFAULT_PORTS["https"],
1289                 "help": "ssl port of the remote web server.",
1290                 "suppress": True,
1291             },
1292         ],
1293         [
1294             ["--remoteTestRoot"],
1295             {
1296                 "dest": "remoteTestRoot",
1297                 "default": None,
1298                 "help": "Remote directory to use as test root "
1299                 "(eg. /data/local/tmp/test_root).",
1300                 "suppress": True,
1301             },
1302         ],
1303         [
1304             ["--enable-coverage"],
1305             {
1306                 "action": "store_true",
1307                 "default": False,
1308                 "help": "Enable collecting code coverage information when running "
1309                 "junit tests.",
1310             },
1311         ],
1312         [
1313             ["--coverage-output-dir"],
1314             {
1315                 "action": "store",
1316                 "default": None,
1317                 "help": "When using --enable-java-coverage, save the code coverage report "
1318                 "files to this directory.",
1319             },
1320         ],
1321     ]
1322
1323     defaults = {
1324         # we don't want to exclude specialpowers on android just yet
1325         "extensionsToExclude": [],
1326         # mochijar doesn't get installed via marionette on android
1327         "extensionsToInstall": [os.path.join(here, "mochijar")],
1328         "logFile": "mochitest.log",
1329         "utilityPath": None,
1330     }
1331
1332     def validate(self, parser, options, context):
1333         """Validate android options."""
1334
1335         if build_obj:
1336             options.log_mach = "-"
1337
1338             objdir_xpi_stage = os.path.join(build_obj.distdir, "xpi-stage")
1339             if os.path.isdir(objdir_xpi_stage):
1340                 options.extensionsToInstall = [
1341                     os.path.join(objdir_xpi_stage, "mochijar"),
1342                     os.path.join(objdir_xpi_stage, "specialpowers"),
1343                 ]
1344
1345         if options.remoteWebServer is None:
1346             options.remoteWebServer = moznetwork.get_ip()
1347
1348         options.webServer = options.remoteWebServer
1349
1350         if options.app is None:
1351             options.app = "org.mozilla.geckoview.test_runner"
1352
1353         if build_obj and "MOZ_HOST_BIN" in os.environ:
1354             options.xrePath = os.environ["MOZ_HOST_BIN"]
1355
1356         # Only reset the xrePath if it wasn't provided
1357         if options.xrePath is None:
1358             options.xrePath = options.utilityPath
1359
1360         if build_obj:
1361             options.topsrcdir = build_obj.topsrcdir
1362
1363         if options.pidFile != "":
1364             f = open(options.pidFile, "w")
1365             f.write("%s" % os.getpid())
1366             f.close()
1367
1368         if options.coverage_output_dir and not options.enable_coverage:
1369             parser.error("--coverage-output-dir must be used with --enable-coverage")
1370         if options.enable_coverage:
1371             if not options.autorun:
1372                 parser.error("--enable-coverage cannot be used with --no-autorun")
1373             if not options.coverage_output_dir:
1374                 parser.error(
1375                     "--coverage-output-dir must be specified when using --enable-coverage"
1376                 )
1377             parent_dir = os.path.dirname(options.coverage_output_dir)
1378             if not os.path.isdir(options.coverage_output_dir):
1379                 parser.error(
1380                     "The directory for the coverage output does not exist: %s"
1381                     % parent_dir
1382                 )
1383
1384         # allow us to keep original application around for cleanup while
1385         # running tests
1386         options.remoteappname = options.app
1387         return options
1388
1389
1390 container_map = {
1391     "generic": [MochitestArguments],
1392     "android": [MochitestArguments, AndroidArguments],
1393 }
1394
1395
1396 class MochitestArgumentParser(ArgumentParser):
1397     """%(prog)s [options] [test paths]"""
1398
1399     _containers = None
1400     context = {}
1401
1402     def __init__(self, app=None, **kwargs):
1403         ArgumentParser.__init__(
1404             self, usage=self.__doc__, conflict_handler="resolve", **kwargs
1405         )
1406
1407         self.oldcwd = os.getcwd()
1408         self.app = app
1409         if not self.app and build_obj:
1410             if conditions.is_android(build_obj):
1411                 self.app = "android"
1412         if not self.app:
1413             # platform can't be determined and app wasn't specified explicitly,
1414             # so just use generic arguments and hope for the best
1415             self.app = "generic"
1416
1417         if self.app not in container_map:
1418             self.error(
1419                 "Unrecognized app '{}'! Must be one of: {}".format(
1420                     self.app, ", ".join(container_map.keys())
1421                 )
1422             )
1423
1424         defaults = {}
1425         for container in self.containers:
1426             defaults.update(container.defaults)
1427             group = self.add_argument_group(
1428                 container.__class__.__name__, container.__doc__
1429             )
1430
1431             for cli, kwargs in container.args:
1432                 # Allocate new lists so references to original don't get mutated.
1433                 # allowing multiple uses within a single process.
1434                 if "default" in kwargs and isinstance(kwargs["default"], list):
1435                     kwargs["default"] = []
1436
1437                 if "suppress" in kwargs:
1438                     if kwargs["suppress"]:
1439                         kwargs["help"] = SUPPRESS
1440                     del kwargs["suppress"]
1441
1442                 group.add_argument(*cli, **kwargs)
1443
1444         self.set_defaults(**defaults)
1445         mozlog.commandline.add_logging_group(self)
1446
1447     @property
1448     def containers(self):
1449         if self._containers:
1450             return self._containers
1451
1452         containers = container_map[self.app]
1453         self._containers = [c() for c in containers]
1454         return self._containers
1455
1456     def validate(self, args):
1457         for container in self.containers:
1458             args = container.validate(self, args, self.context)
1459         return args