testing/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import argparse
   8 import logging
   9 import os
  10 import sys
  11 import subprocess
  12
  13 from mach.decorators import (
  14     CommandArgument,
  15     Command,
  16     SettingsProvider,
  17     SubCommand,
  18 )
  19
  20 from mozbuild.base import (
  21     BuildEnvironmentNotFoundException,
  22     MachCommandConditions as conditions,
  23 )
  24
  25 UNKNOWN_TEST = """
  26 I was unable to find tests from the given argument(s).
  27
  28 You should specify a test directory, filename, test suite name, or
  29 abbreviation.
  30
  31 It's possible my little brain doesn't know about the type of test you are
  32 trying to execute. If you suspect this, please request support by filing
  33 a bug at
  34 https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=General.
  35 """.strip()
  36
  37 UNKNOWN_FLAVOR = """
  38 I know you are trying to run a %s%s test. Unfortunately, I can't run those
  39 tests yet. Sorry!
  40 """.strip()
  41
  42 TEST_HELP = """
  43 Test or tests to run. Tests can be specified by filename, directory, suite
  44 name or suite alias.
  45
  46 The following test suites and aliases are supported: {}
  47 """.strip()
  48
  49
  50 @SettingsProvider
  51 class TestConfig(object):
  52     @classmethod
  53     def config_settings(cls):
  54         from mozlog.commandline import log_formatters
  55         from mozlog.structuredlog import log_levels
  56
  57         format_desc = "The default format to use when running tests with `mach test`."
  58         format_choices = list(log_formatters)
  59         level_desc = "The default log level to use when running tests with `mach test`."
  60         level_choices = [l.lower() for l in log_levels]
  61         return [
  62             ("test.format", "string", format_desc, "mach", {"choices": format_choices}),
  63             ("test.level", "string", level_desc, "info", {"choices": level_choices}),
  64         ]
  65
  66
  67 def get_test_parser():
  68     from mozlog.commandline import add_logging_group
  69     from moztest.resolve import TEST_SUITES
  70
  71     parser = argparse.ArgumentParser()
  72     parser.add_argument(
  73         "what",
  74         default=None,
  75         nargs="+",
  76         help=TEST_HELP.format(", ".join(sorted(TEST_SUITES))),
  77     )
  78     parser.add_argument(
  79         "extra_args",
  80         default=None,
  81         nargs=argparse.REMAINDER,
  82         help="Extra arguments to pass to the underlying test command(s). "
  83         "If an underlying command doesn't recognize the argument, it "
  84         "will fail.",
  85     )
  86     parser.add_argument(
  87         "--debugger",
  88         default=None,
  89         action="store",
  90         nargs="?",
  91         help="Specify a debugger to use.",
  92     )
  93     add_logging_group(parser)
  94     return parser
  95
  96
  97 ADD_TEST_SUPPORTED_SUITES = [
  98     "mochitest-chrome",
  99     "mochitest-plain",
 100     "mochitest-browser-chrome",
 101     "web-platform-tests-testharness",
 102     "web-platform-tests-reftest",
 103     "xpcshell",
 104 ]
 105 ADD_TEST_SUPPORTED_DOCS = ["js", "html", "xhtml", "xul"]
 106
 107 SUITE_SYNONYMS = {
 108     "wpt": "web-platform-tests-testharness",
 109     "wpt-testharness": "web-platform-tests-testharness",
 110     "wpt-reftest": "web-platform-tests-reftest",
 111 }
 112
 113 MISSING_ARG = object()
 114
 115
 116 def create_parser_addtest():
 117     import addtest
 118
 119     parser = argparse.ArgumentParser()
 120     parser.add_argument(
 121         "--suite",
 122         choices=sorted(ADD_TEST_SUPPORTED_SUITES + list(SUITE_SYNONYMS.keys())),
 123         help="suite for the test. "
 124         "If you pass a `test` argument this will be determined "
 125         "based on the filename and the folder it is in",
 126     )
 127     parser.add_argument(
 128         "-o",
 129         "--overwrite",
 130         action="store_true",
 131         help="Overwrite an existing file if it exists.",
 132     )
 133     parser.add_argument(
 134         "--doc",
 135         choices=ADD_TEST_SUPPORTED_DOCS,
 136         help="Document type for the test (if applicable)."
 137         "If you pass a `test` argument this will be determined "
 138         "based on the filename.",
 139     )
 140     parser.add_argument(
 141         "-e",
 142         "--editor",
 143         action="store",
 144         nargs="?",
 145         default=MISSING_ARG,
 146         help="Open the created file(s) in an editor; if a "
 147         "binary is supplied it will be used otherwise the default editor for "
 148         "your environment will be opened",
 149     )
 150
 151     for base_suite in addtest.TEST_CREATORS:
 152         cls = addtest.TEST_CREATORS[base_suite]
 153         if hasattr(cls, "get_parser"):
 154             group = parser.add_argument_group(base_suite)
 155             cls.get_parser(group)
 156
 157     parser.add_argument("test", nargs="?", help=("Test to create."))
 158     return parser
 159
 160
 161 @Command(
 162     "addtest",
 163     category="testing",
 164     description="Generate tests based on templates",
 165     parser=create_parser_addtest,
 166 )
 167 def addtest(
 168     command_context,
 169     suite=None,
 170     test=None,
 171     doc=None,
 172     overwrite=False,
 173     editor=MISSING_ARG,
 174     **kwargs,
 175 ):
 176     import addtest
 177     import io
 178     from moztest.resolve import TEST_SUITES
 179
 180     if not suite and not test:
 181         return create_parser_addtest().parse_args(["--help"])
 182
 183     if suite in SUITE_SYNONYMS:
 184         suite = SUITE_SYNONYMS[suite]
 185
 186     if test:
 187         if not overwrite and os.path.isfile(os.path.abspath(test)):
 188             print("Error: can't generate a test that already exists:", test)
 189             return 1
 190
 191         abs_test = os.path.abspath(test)
 192         if doc is None:
 193             doc = guess_doc(abs_test)
 194         if suite is None:
 195             guessed_suite, err = guess_suite(abs_test)
 196             if err:
 197                 print(err)
 198                 return 1
 199             suite = guessed_suite
 200
 201     else:
 202         test = None
 203         if doc is None:
 204             doc = "html"
 205
 206     if not suite:
 207         print(
 208             "We couldn't automatically determine a suite. "
 209             "Please specify `--suite` with one of the following options:\n{}\n"
 210             "If you'd like to add support to a new suite, please file a bug "
 211             "blocking https://bugzilla.mozilla.org/show_bug.cgi?id=1540285.".format(
 212                 ADD_TEST_SUPPORTED_SUITES
 213             )
 214         )
 215         return 1
 216
 217     if doc not in ADD_TEST_SUPPORTED_DOCS:
 218         print(
 219             "Error: invalid `doc`. Either pass in a test with a valid extension"
 220             "({}) or pass in the `doc` argument".format(ADD_TEST_SUPPORTED_DOCS)
 221         )
 222         return 1
 223
 224     creator_cls = addtest.creator_for_suite(suite)
 225
 226     if creator_cls is None:
 227         print("Sorry, `addtest` doesn't currently know how to add {}".format(suite))
 228         return 1
 229
 230     creator = creator_cls(command_context.topsrcdir, test, suite, doc, **kwargs)
 231
 232     creator.check_args()
 233
 234     paths = []
 235     added_tests = False
 236     for path, template in creator:
 237         if not template:
 238             continue
 239         added_tests = True
 240         if path:
 241             paths.append(path)
 242             print("Adding a test file at {} (suite `{}`)".format(path, suite))
 243
 244             try:
 245                 os.makedirs(os.path.dirname(path))
 246             except OSError:
 247                 pass
 248
 249             with io.open(path, "w", newline="\n") as f:
 250                 f.write(template)
 251         else:
 252             # write to stdout if you passed only suite and doc and not a file path
 253             print(template)
 254
 255     if not added_tests:
 256         return 1
 257
 258     if test:
 259         creator.update_manifest()
 260
 261         # Small hack, should really do this better
 262         if suite.startswith("wpt-"):
 263             suite = "web-platform-tests"
 264
 265         mach_command = TEST_SUITES[suite]["mach_command"]
 266         print(
 267             "Please make sure to add the new test to your commit. "
 268             "You can now run the test with:\n    ./mach {} {}".format(
 269                 mach_command, test
 270             )
 271         )
 272
 273     if editor is not MISSING_ARG:
 274         if editor is not None:
 275             editor = editor
 276         elif "VISUAL" in os.environ:
 277             editor = os.environ["VISUAL"]
 278         elif "EDITOR" in os.environ:
 279             editor = os.environ["EDITOR"]
 280         else:
 281             print("Unable to determine editor; please specify a binary")
 282             editor = None
 283
 284         proc = None
 285         if editor:
 286             import subprocess
 287
 288             proc = subprocess.Popen("%s %s" % (editor, " ".join(paths)), shell=True)
 289
 290         if proc:
 291             proc.wait()
 292
 293     return 0
 294
 295
 296 def guess_doc(abs_test):
 297     filename = os.path.basename(abs_test)
 298     return os.path.splitext(filename)[1].strip(".")
 299
 300
 301 def guess_suite(abs_test):
 302     # If you pass a abs_test, try to detect the type based on the name
 303     # and folder. This detection can be skipped if you pass the `type` arg.
 304     err = None
 305     guessed_suite = None
 306     parent = os.path.dirname(abs_test)
 307     filename = os.path.basename(abs_test)
 308
 309     has_browser_ini = os.path.isfile(os.path.join(parent, "browser.ini"))
 310     has_chrome_ini = os.path.isfile(os.path.join(parent, "chrome.ini"))
 311     has_plain_ini = os.path.isfile(os.path.join(parent, "mochitest.ini"))
 312     has_xpcshell_ini = os.path.isfile(os.path.join(parent, "xpcshell.ini"))
 313
 314     in_wpt_folder = abs_test.startswith(
 315         os.path.abspath(os.path.join("testing", "web-platform"))
 316     )
 317
 318     if in_wpt_folder:
 319         guessed_suite = "web-platform-tests-testharness"
 320         if "/css/" in abs_test:
 321             guessed_suite = "web-platform-tests-reftest"
 322     elif (
 323         filename.startswith("test_")
 324         and has_xpcshell_ini
 325         and guess_doc(abs_test) == "js"
 326     ):
 327         guessed_suite = "xpcshell"
 328     else:
 329         if filename.startswith("browser_") and has_browser_ini:
 330             guessed_suite = "mochitest-browser-chrome"
 331         elif filename.startswith("test_"):
 332             if has_chrome_ini and has_plain_ini:
 333                 err = (
 334                     "Error: directory contains both a chrome.ini and mochitest.ini. "
 335                     "Please set --suite=mochitest-chrome or --suite=mochitest-plain."
 336                 )
 337             elif has_chrome_ini:
 338                 guessed_suite = "mochitest-chrome"
 339             elif has_plain_ini:
 340                 guessed_suite = "mochitest-plain"
 341     return guessed_suite, err
 342
 343
 344 @Command(
 345     "test",
 346     category="testing",
 347     description="Run tests (detects the kind of test and runs it).",
 348     parser=get_test_parser,
 349 )
 350 def test(command_context, what, extra_args, **log_args):
 351     """Run tests from names or paths.
 352
 353     mach test accepts arguments specifying which tests to run. Each argument
 354     can be:
 355
 356     * The path to a test file
 357     * A directory containing tests
 358     * A test suite name
 359     * An alias to a test suite name (codes used on TreeHerder)
 360
 361     When paths or directories are given, they are first resolved to test
 362     files known to the build system.
 363
 364     If resolved tests belong to more than one test type/flavor/harness,
 365     the harness for each relevant type/flavor will be invoked. e.g. if
 366     you specify a directory with xpcshell and browser chrome mochitests,
 367     both harnesses will be invoked.
 368
 369     Warning: `mach test` does not automatically re-build.
 370     Please remember to run `mach build` when necessary.
 371
 372     EXAMPLES
 373
 374     Run all test files in the devtools/client/shared/redux/middleware/xpcshell/
 375     directory:
 376
 377     `./mach test devtools/client/shared/redux/middleware/xpcshell/`
 378
 379     The below command prints a short summary of results instead of
 380     the default more verbose output.
 381     Do not forget the - (minus sign) after --log-grouped!
 382
 383     `./mach test --log-grouped - devtools/client/shared/redux/middleware/xpcshell/`
 384     """
 385     from mozlog.commandline import setup_logging
 386     from mozlog.handlers import StreamHandler
 387     from moztest.resolve import get_suite_definition, TestResolver, TEST_SUITES
 388
 389     resolver = command_context._spawn(TestResolver)
 390     run_suites, run_tests = resolver.resolve_metadata(what)
 391
 392     if not run_suites and not run_tests:
 393         print(UNKNOWN_TEST)
 394         return 1
 395
 396     if log_args.get("debugger", None):
 397         import mozdebug
 398
 399         if not mozdebug.get_debugger_info(log_args.get("debugger")):
 400             sys.exit(1)
 401         extra_args_debugger_notation = "=".join(
 402             ["--debugger", log_args.get("debugger")]
 403         )
 404         if extra_args:
 405             extra_args.append(extra_args_debugger_notation)
 406         else:
 407             extra_args = [extra_args_debugger_notation]
 408
 409     # Create shared logger
 410     format_args = {"level": command_context._mach_context.settings["test"]["level"]}
 411     if not run_suites and len(run_tests) == 1:
 412         format_args["verbose"] = True
 413         format_args["compact"] = False
 414
 415     default_format = command_context._mach_context.settings["test"]["format"]
 416     log = setup_logging(
 417         "mach-test", log_args, {default_format: sys.stdout}, format_args
 418     )
 419     for handler in log.handlers:
 420         if isinstance(handler, StreamHandler):
 421             handler.formatter.inner.summary_on_shutdown = True
 422
 423     status = None
 424     for suite_name in run_suites:
 425         suite = TEST_SUITES[suite_name]
 426         kwargs = suite["kwargs"]
 427         kwargs["log"] = log
 428         kwargs.setdefault("subsuite", None)
 429
 430         if "mach_command" in suite:
 431             res = command_context._mach_context.commands.dispatch(
 432                 suite["mach_command"],
 433                 command_context._mach_context,
 434                 argv=extra_args,
 435                 **kwargs,
 436             )
 437             if res:
 438                 status = res
 439
 440     buckets = {}
 441     for test in run_tests:
 442         key = (test["flavor"], test.get("subsuite", ""))
 443         buckets.setdefault(key, []).append(test)
 444
 445     for (flavor, subsuite), tests in sorted(buckets.items()):
 446         _, m = get_suite_definition(flavor, subsuite)
 447         if "mach_command" not in m:
 448             substr = "-{}".format(subsuite) if subsuite else ""
 449             print(UNKNOWN_FLAVOR % (flavor, substr))
 450             status = 1
 451             continue
 452
 453         kwargs = dict(m["kwargs"])
 454         kwargs["log"] = log
 455         kwargs.setdefault("subsuite", None)
 456
 457         res = command_context._mach_context.commands.dispatch(
 458             m["mach_command"],
 459             command_context._mach_context,
 460             argv=extra_args,
 461             test_objects=tests,
 462             **kwargs,
 463         )
 464         if res:
 465             status = res
 466
 467     log.shutdown()
 468     return status
 469
 470
 471 @Command(
 472     "cppunittest", category="testing", description="Run cpp unit tests (C++ tests)."
 473 )
 474 @CommandArgument(
 475     "--enable-webrender",
 476     action="store_true",
 477     default=False,
 478     dest="enable_webrender",
 479     help="Enable the WebRender compositor in Gecko.",
 480 )
 481 @CommandArgument(
 482     "test_files",
 483     nargs="*",
 484     metavar="N",
 485     help="Test to run. Can be specified as one or more files or "
 486     "directories, or omitted. If omitted, the entire test suite is "
 487     "executed.",
 488 )
 489 def run_cppunit_test(command_context, **params):
 490     from mozlog import commandline
 491
 492     log = params.get("log")
 493     if not log:
 494         log = commandline.setup_logging("cppunittest", {}, {"tbpl": sys.stdout})
 495
 496     # See if we have crash symbols
 497     symbols_path = os.path.join(command_context.distdir, "crashreporter-symbols")
 498     if not os.path.isdir(symbols_path):
 499         symbols_path = None
 500
 501     # If no tests specified, run all tests in main manifest
 502     tests = params["test_files"]
 503     if not tests:
 504         tests = [os.path.join(command_context.distdir, "cppunittests")]
 505         manifest_path = os.path.join(
 506             command_context.topsrcdir, "testing", "cppunittest.ini"
 507         )
 508     else:
 509         manifest_path = None
 510
 511     utility_path = command_context.bindir
 512
 513     if conditions.is_android(command_context):
 514         from mozrunner.devices.android_device import (
 515             verify_android_device,
 516             InstallIntent,
 517         )
 518
 519         verify_android_device(command_context, install=InstallIntent.NO)
 520         return run_android_test(tests, symbols_path, manifest_path, log)
 521
 522     return run_desktop_test(
 523         command_context, tests, symbols_path, manifest_path, utility_path, log
 524     )
 525
 526
 527 def run_desktop_test(
 528     command_context, tests, symbols_path, manifest_path, utility_path, log
 529 ):
 530     import runcppunittests as cppunittests
 531     from mozlog import commandline
 532
 533     parser = cppunittests.CPPUnittestOptions()
 534     commandline.add_logging_group(parser)
 535     options, args = parser.parse_args()
 536
 537     options.symbols_path = symbols_path
 538     options.manifest_path = manifest_path
 539     options.utility_path = utility_path
 540     options.xre_path = command_context.bindir
 541
 542     try:
 543         result = cppunittests.run_test_harness(options, tests)
 544     except Exception as e:
 545         log.error("Caught exception running cpp unit tests: %s" % str(e))
 546         result = False
 547         raise
 548
 549     return 0 if result else 1
 550
 551
 552 def run_android_test(command_context, tests, symbols_path, manifest_path, log):
 553     import remotecppunittests as remotecppunittests
 554     from mozlog import commandline
 555
 556     parser = remotecppunittests.RemoteCPPUnittestOptions()
 557     commandline.add_logging_group(parser)
 558     options, args = parser.parse_args()
 559
 560     if not options.adb_path:
 561         from mozrunner.devices.android_device import get_adb_path
 562
 563         options.adb_path = get_adb_path(command_context)
 564     options.symbols_path = symbols_path
 565     options.manifest_path = manifest_path
 566     options.xre_path = command_context.bindir
 567     options.local_lib = command_context.bindir.replace("bin", "fennec")
 568     for file in os.listdir(os.path.join(command_context.topobjdir, "dist")):
 569         if file.endswith(".apk") and file.startswith("fennec"):
 570             options.local_apk = os.path.join(command_context.topobjdir, "dist", file)
 571             log.info("using APK: " + options.local_apk)
 572             break
 573
 574     try:
 575         result = remotecppunittests.run_test_harness(options, tests)
 576     except Exception as e:
 577         log.error("Caught exception running cpp unit tests: %s" % str(e))
 578         result = False
 579         raise
 580
 581     return 0 if result else 1
 582
 583
 584 def executable_name(name):
 585     return name + ".exe" if sys.platform.startswith("win") else name
 586
 587
 588 @Command(
 589     "jstests",
 590     category="testing",
 591     description="Run SpiderMonkey JS tests in the JS shell.",
 592 )
 593 @CommandArgument("--shell", help="The shell to be used")
 594 @CommandArgument(
 595     "params",
 596     nargs=argparse.REMAINDER,
 597     help="Extra arguments to pass down to the test harness.",
 598 )
 599 def run_jstests(command_context, shell, params):
 600     import subprocess
 601
 602     command_context.virtualenv_manager.ensure()
 603     python = command_context.virtualenv_manager.python_path
 604
 605     js = shell or os.path.join(command_context.bindir, executable_name("js"))
 606     jstest_cmd = [
 607         python,
 608         os.path.join(command_context.topsrcdir, "js", "src", "tests", "jstests.py"),
 609         js,
 610     ] + params
 611
 612     return subprocess.call(jstest_cmd)
 613
 614
 615 @Command(
 616     "jit-test",
 617     category="testing",
 618     description="Run SpiderMonkey jit-tests in the JS shell.",
 619     ok_if_tests_disabled=True,
 620 )
 621 @CommandArgument("--shell", help="The shell to be used")
 622 @CommandArgument(
 623     "--cgc",
 624     action="store_true",
 625     default=False,
 626     help="Run with the SM(cgc) job's env vars",
 627 )
 628 @CommandArgument(
 629     "params",
 630     nargs=argparse.REMAINDER,
 631     help="Extra arguments to pass down to the test harness.",
 632 )
 633 def run_jittests(command_context, shell, cgc, params):
 634     import subprocess
 635
 636     command_context.virtualenv_manager.ensure()
 637     python = command_context.virtualenv_manager.python_path
 638
 639     js = shell or os.path.join(command_context.bindir, executable_name("js"))
 640     jittest_cmd = [
 641         python,
 642         os.path.join(command_context.topsrcdir, "js", "src", "jit-test", "jit_test.py"),
 643         js,
 644     ] + params
 645
 646     env = os.environ.copy()
 647     if cgc:
 648         env["JS_GC_ZEAL"] = "IncrementalMultipleSlices"
 649
 650     return subprocess.call(jittest_cmd, env=env)
 651
 652
 653 @Command("jsapi-tests", category="testing", description="Run SpiderMonkey JSAPI tests.")
 654 @CommandArgument(
 655     "test_name",
 656     nargs="?",
 657     metavar="N",
 658     help="Test to run. Can be a prefix or omitted. If "
 659     "omitted, the entire test suite is executed.",
 660 )
 661 def run_jsapitests(command_context, test_name=None):
 662     import subprocess
 663
 664     jsapi_tests_cmd = [
 665         os.path.join(command_context.bindir, executable_name("jsapi-tests"))
 666     ]
 667     if test_name:
 668         jsapi_tests_cmd.append(test_name)
 669
 670     test_env = os.environ.copy()
 671     test_env["TOPSRCDIR"] = command_context.topsrcdir
 672
 673     result = subprocess.call(jsapi_tests_cmd, env=test_env)
 674     if result != 0:
 675         print(f"jsapi-tests failed, exit code {result}")
 676     return result
 677
 678
 679 def run_check_js_msg(command_context):
 680     import subprocess
 681
 682     command_context.virtualenv_manager.ensure()
 683     python = command_context.virtualenv_manager.python_path
 684
 685     check_cmd = [
 686         python,
 687         os.path.join(command_context.topsrcdir, "config", "check_js_msg_encoding.py"),
 688     ]
 689
 690     return subprocess.call(check_cmd)
 691
 692
 693 def get_jsshell_parser():
 694     from jsshell.benchmark import get_parser
 695
 696     return get_parser()
 697
 698
 699 @Command(
 700     "jsshell-bench",
 701     category="testing",
 702     parser=get_jsshell_parser,
 703     description="Run benchmarks in the SpiderMonkey JS shell.",
 704 )
 705 def run_jsshelltests(command_context, **kwargs):
 706     command_context.activate_virtualenv()
 707     from jsshell import benchmark
 708
 709     return benchmark.run(**kwargs)
 710
 711
 712 @Command(
 713     "cramtest",
 714     category="testing",
 715     description="Mercurial style .t tests for command line applications.",
 716 )
 717 @CommandArgument(
 718     "test_paths",
 719     nargs="*",
 720     metavar="N",
 721     help="Test paths to run. Each path can be a test file or directory. "
 722     "If omitted, the entire suite will be run.",
 723 )
 724 @CommandArgument(
 725     "cram_args",
 726     nargs=argparse.REMAINDER,
 727     help="Extra arguments to pass down to the cram binary. See "
 728     "'./mach python -m cram -- -h' for a list of available options.",
 729 )
 730 def cramtest(command_context, cram_args=None, test_paths=None, test_objects=None):
 731     command_context.activate_virtualenv()
 732     import mozinfo
 733     from manifestparser import TestManifest
 734
 735     if test_objects is None:
 736         from moztest.resolve import TestResolver
 737
 738         resolver = command_context._spawn(TestResolver)
 739         if test_paths:
 740             # If we were given test paths, try to find tests matching them.
 741             test_objects = resolver.resolve_tests(paths=test_paths, flavor="cram")
 742         else:
 743             # Otherwise just run everything in CRAMTEST_MANIFESTS
 744             test_objects = resolver.resolve_tests(flavor="cram")
 745
 746     if not test_objects:
 747         message = "No tests were collected, check spelling of the test paths."
 748         command_context.log(logging.WARN, "cramtest", {}, message)
 749         return 1
 750
 751     mp = TestManifest()
 752     mp.tests.extend(test_objects)
 753     tests = mp.active_tests(disabled=False, **mozinfo.info)
 754
 755     python = command_context.virtualenv_manager.python_path
 756     cmd = [python, "-m", "cram"] + cram_args + [t["relpath"] for t in tests]
 757     return subprocess.call(cmd, cwd=command_context.topsrcdir)
 758
 759
 760 from datetime import date, timedelta
 761
 762
 763 @Command(
 764     "test-info", category="testing", description="Display historical test results."
 765 )
 766 def test_info(command_context):
 767     """
 768     All functions implemented as subcommands.
 769     """
 770
 771
 772 @SubCommand(
 773     "test-info",
 774     "tests",
 775     description="Display historical test result summary for named tests.",
 776 )
 777 @CommandArgument("test_names", nargs=argparse.REMAINDER, help="Test(s) of interest.")
 778 @CommandArgument(
 779     "--start",
 780     default=(date.today() - timedelta(7)).strftime("%Y-%m-%d"),
 781     help="Start date (YYYY-MM-DD)",
 782 )
 783 @CommandArgument(
 784     "--end", default=date.today().strftime("%Y-%m-%d"), help="End date (YYYY-MM-DD)"
 785 )
 786 @CommandArgument(
 787     "--show-info",
 788     action="store_true",
 789     help="Retrieve and display general test information.",
 790 )
 791 @CommandArgument(
 792     "--show-bugs",
 793     action="store_true",
 794     help="Retrieve and display related Bugzilla bugs.",
 795 )
 796 @CommandArgument("--verbose", action="store_true", help="Enable debug logging.")
 797 def test_info_tests(
 798     command_context,
 799     test_names,
 800     start,
 801     end,
 802     show_info,
 803     show_bugs,
 804     verbose,
 805 ):
 806     import testinfo
 807
 808     ti = testinfo.TestInfoTests(verbose)
 809     ti.report(
 810         test_names,
 811         start,
 812         end,
 813         show_info,
 814         show_bugs,
 815     )
 816
 817
 818 @SubCommand(
 819     "test-info",
 820     "report",
 821     description="Generate a json report of test manifests and/or tests "
 822     "categorized by Bugzilla component and optionally filtered "
 823     "by path, component, and/or manifest annotations.",
 824 )
 825 @CommandArgument(
 826     "--components",
 827     default=None,
 828     help="Comma-separated list of Bugzilla components."
 829     " eg. Testing::General,Core::WebVR",
 830 )
 831 @CommandArgument(
 832     "--flavor",
 833     help='Limit results to tests of the specified flavor (eg. "xpcshell").',
 834 )
 835 @CommandArgument(
 836     "--subsuite",
 837     help='Limit results to tests of the specified subsuite (eg. "devtools").',
 838 )
 839 @CommandArgument(
 840     "paths", nargs=argparse.REMAINDER, help="File system paths of interest."
 841 )
 842 @CommandArgument(
 843     "--show-manifests",
 844     action="store_true",
 845     help="Include test manifests in report.",
 846 )
 847 @CommandArgument(
 848     "--show-tests", action="store_true", help="Include individual tests in report."
 849 )
 850 @CommandArgument(
 851     "--show-summary", action="store_true", help="Include summary in report."
 852 )
 853 @CommandArgument(
 854     "--show-annotations",
 855     action="store_true",
 856     help="Include list of manifest annotation conditions in report.",
 857 )
 858 @CommandArgument(
 859     "--filter-values",
 860     help="Comma-separated list of value regular expressions to filter on; "
 861     "displayed tests contain all specified values.",
 862 )
 863 @CommandArgument(
 864     "--filter-keys",
 865     help="Comma-separated list of test keys to filter on, "
 866     'like "skip-if"; only these fields will be searched '
 867     "for filter-values.",
 868 )
 869 @CommandArgument(
 870     "--no-component-report",
 871     action="store_false",
 872     dest="show_components",
 873     default=True,
 874     help="Do not categorize by bugzilla component.",
 875 )
 876 @CommandArgument("--output-file", help="Path to report file.")
 877 @CommandArgument("--verbose", action="store_true", help="Enable debug logging.")
 878 def test_report(
 879     command_context,
 880     components,
 881     flavor,
 882     subsuite,
 883     paths,
 884     show_manifests,
 885     show_tests,
 886     show_summary,
 887     show_annotations,
 888     filter_values,
 889     filter_keys,
 890     show_components,
 891     output_file,
 892     verbose,
 893 ):
 894     import testinfo
 895     from mozbuild import build_commands
 896
 897     try:
 898         command_context.config_environment
 899     except BuildEnvironmentNotFoundException:
 900         print("Looks like configure has not run yet, running it now...")
 901         build_commands.configure(command_context)
 902
 903     ti = testinfo.TestInfoReport(verbose)
 904     ti.report(
 905         components,
 906         flavor,
 907         subsuite,
 908         paths,
 909         show_manifests,
 910         show_tests,
 911         show_summary,
 912         show_annotations,
 913         filter_values,
 914         filter_keys,
 915         show_components,
 916         output_file,
 917     )
 918
 919
 920 @SubCommand(
 921     "test-info",
 922     "report-diff",
 923     description='Compare two reports generated by "test-info reports".',
 924 )
 925 @CommandArgument(
 926     "--before",
 927     default=None,
 928     help="The first (earlier) report file; path to local file or url.",
 929 )
 930 @CommandArgument(
 931     "--after", help="The second (later) report file; path to local file or url."
 932 )
 933 @CommandArgument(
 934     "--output-file",
 935     help="Path to report file to be written. If not specified, report"
 936     "will be written to standard output.",
 937 )
 938 @CommandArgument("--verbose", action="store_true", help="Enable debug logging.")
 939 def test_report_diff(command_context, before, after, output_file, verbose):
 940     import testinfo
 941
 942     ti = testinfo.TestInfoReport(verbose)
 943     ti.report_diff(before, after, output_file)
 944
 945
 946 @Command(
 947     "rusttests",
 948     category="testing",
 949     conditions=[conditions.is_non_artifact_build],
 950     description="Run rust unit tests (via cargo test).",
 951 )
 952 def run_rusttests(command_context, **kwargs):
 953     return command_context._mach_context.commands.dispatch(
 954         "build",
 955         command_context._mach_context,
 956         what=["pre-export", "export", "recurse_rusttests"],
 957     )
 958
 959
 960 @Command(
 961     "fluent-migration-test",
 962     category="testing",
 963     description="Test Fluent migration recipes.",
 964 )
 965 @CommandArgument("test_paths", nargs="*", metavar="N", help="Recipe paths to test.")
 966 def run_migration_tests(command_context, test_paths=None, **kwargs):
 967     if not test_paths:
 968         test_paths = []
 969     command_context.activate_virtualenv()
 970     from test_fluent_migrations import fmt
 971
 972     rv = 0
 973     with_context = []
 974     for to_test in test_paths:
 975         try:
 976             context = fmt.inspect_migration(to_test)
 977             for issue in context["issues"]:
 978                 command_context.log(
 979                     logging.ERROR,
 980                     "fluent-migration-test",
 981                     {
 982                         "error": issue["msg"],
 983                         "file": to_test,
 984                     },
 985                     "ERROR in {file}: {error}",
 986                 )
 987             if context["issues"]:
 988                 continue
 989             with_context.append(
 990                 {
 991                     "to_test": to_test,
 992                     "references": context["references"],
 993                 }
 994             )
 995         except Exception as e:
 996             command_context.log(
 997                 logging.ERROR,
 998                 "fluent-migration-test",
 999                 {"error": str(e), "file": to_test},
1000                 "ERROR in {file}: {error}",
1001             )
1002             rv |= 1
1003     obj_dir = fmt.prepare_object_dir(command_context)
1004     for context in with_context:
1005         rv |= fmt.test_migration(command_context, obj_dir, **context)
1006     return rv