testing/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import argparse
   8 import logging
   9 import os
  10 import sys
  11 import subprocess
  12
  13 from mach.decorators import (
  14     CommandArgument,
  15     Command,
  16     SettingsProvider,
  17     SubCommand,
  18 )
  19
  20 from mozbuild.base import (
  21     BuildEnvironmentNotFoundException,
  22     MachCommandConditions as conditions,
  23 )
  24
  25 UNKNOWN_TEST = """
  26 I was unable to find tests from the given argument(s).
  27
  28 You should specify a test directory, filename, test suite name, or
  29 abbreviation.
  30
  31 It's possible my little brain doesn't know about the type of test you are
  32 trying to execute. If you suspect this, please request support by filing
  33 a bug at
  34 https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=General.
  35 """.strip()
  36
  37 UNKNOWN_FLAVOR = """
  38 I know you are trying to run a %s%s test. Unfortunately, I can't run those
  39 tests yet. Sorry!
  40 """.strip()
  41
  42 TEST_HELP = """
  43 Test or tests to run. Tests can be specified by filename, directory, suite
  44 name or suite alias.
  45
  46 The following test suites and aliases are supported: {}
  47 """.strip()
  48
  49
  50 @SettingsProvider
  51 class TestConfig(object):
  52     @classmethod
  53     def config_settings(cls):
  54         from mozlog.commandline import log_formatters
  55         from mozlog.structuredlog import log_levels
  56
  57         format_desc = "The default format to use when running tests with `mach test`."
  58         format_choices = list(log_formatters)
  59         level_desc = "The default log level to use when running tests with `mach test`."
  60         level_choices = [l.lower() for l in log_levels]
  61         return [
  62             ("test.format", "string", format_desc, "mach", {"choices": format_choices}),
  63             ("test.level", "string", level_desc, "info", {"choices": level_choices}),
  64         ]
  65
  66
  67 def get_test_parser():
  68     from mozlog.commandline import add_logging_group
  69     from moztest.resolve import TEST_SUITES
  70
  71     parser = argparse.ArgumentParser()
  72     parser.add_argument(
  73         "what",
  74         default=None,
  75         nargs="+",
  76         help=TEST_HELP.format(", ".join(sorted(TEST_SUITES))),
  77     )
  78     parser.add_argument(
  79         "extra_args",
  80         default=None,
  81         nargs=argparse.REMAINDER,
  82         help="Extra arguments to pass to the underlying test command(s). "
  83         "If an underlying command doesn't recognize the argument, it "
  84         "will fail.",
  85     )
  86     parser.add_argument(
  87         "--debugger",
  88         default=None,
  89         action="store",
  90         nargs="?",
  91         help="Specify a debugger to use.",
  92     )
  93     add_logging_group(parser)
  94     return parser
  95
  96
  97 ADD_TEST_SUPPORTED_SUITES = [
  98     "mochitest-chrome",
  99     "mochitest-plain",
 100     "mochitest-browser-chrome",
 101     "web-platform-tests-testharness",
 102     "web-platform-tests-reftest",
 103     "xpcshell",
 104 ]
 105 ADD_TEST_SUPPORTED_DOCS = ["js", "html", "xhtml", "xul"]
 106
 107 SUITE_SYNONYMS = {
 108     "wpt": "web-platform-tests-testharness",
 109     "wpt-testharness": "web-platform-tests-testharness",
 110     "wpt-reftest": "web-platform-tests-reftest",
 111 }
 112
 113 MISSING_ARG = object()
 114
 115
 116 def create_parser_addtest():
 117     import addtest
 118
 119     parser = argparse.ArgumentParser()
 120     parser.add_argument(
 121         "--suite",
 122         choices=sorted(ADD_TEST_SUPPORTED_SUITES + list(SUITE_SYNONYMS.keys())),
 123         help="suite for the test. "
 124         "If you pass a `test` argument this will be determined "
 125         "based on the filename and the folder it is in",
 126     )
 127     parser.add_argument(
 128         "-o",
 129         "--overwrite",
 130         action="store_true",
 131         help="Overwrite an existing file if it exists.",
 132     )
 133     parser.add_argument(
 134         "--doc",
 135         choices=ADD_TEST_SUPPORTED_DOCS,
 136         help="Document type for the test (if applicable)."
 137         "If you pass a `test` argument this will be determined "
 138         "based on the filename.",
 139     )
 140     parser.add_argument(
 141         "-e",
 142         "--editor",
 143         action="store",
 144         nargs="?",
 145         default=MISSING_ARG,
 146         help="Open the created file(s) in an editor; if a "
 147         "binary is supplied it will be used otherwise the default editor for "
 148         "your environment will be opened",
 149     )
 150
 151     for base_suite in addtest.TEST_CREATORS:
 152         cls = addtest.TEST_CREATORS[base_suite]
 153         if hasattr(cls, "get_parser"):
 154             group = parser.add_argument_group(base_suite)
 155             cls.get_parser(group)
 156
 157     parser.add_argument("test", nargs="?", help=("Test to create."))
 158     return parser
 159
 160
 161 @Command(
 162     "addtest",
 163     category="testing",
 164     description="Generate tests based on templates",
 165     parser=create_parser_addtest,
 166 )
 167 def addtest(
 168     command_context,
 169     suite=None,
 170     test=None,
 171     doc=None,
 172     overwrite=False,
 173     editor=MISSING_ARG,
 174     **kwargs,
 175 ):
 176     import addtest
 177     import io
 178     from moztest.resolve import TEST_SUITES
 179
 180     if not suite and not test:
 181         return create_parser_addtest().parse_args(["--help"])
 182
 183     if suite in SUITE_SYNONYMS:
 184         suite = SUITE_SYNONYMS[suite]
 185
 186     if test:
 187         if not overwrite and os.path.isfile(os.path.abspath(test)):
 188             print("Error: can't generate a test that already exists:", test)
 189             return 1
 190
 191         abs_test = os.path.abspath(test)
 192         if doc is None:
 193             doc = guess_doc(abs_test)
 194         if suite is None:
 195             guessed_suite, err = guess_suite(abs_test)
 196             if err:
 197                 print(err)
 198                 return 1
 199             suite = guessed_suite
 200
 201     else:
 202         test = None
 203         if doc is None:
 204             doc = "html"
 205
 206     if not suite:
 207         print(
 208             "We couldn't automatically determine a suite. "
 209             "Please specify `--suite` with one of the following options:\n{}\n"
 210             "If you'd like to add support to a new suite, please file a bug "
 211             "blocking https://bugzilla.mozilla.org/show_bug.cgi?id=1540285.".format(
 212                 ADD_TEST_SUPPORTED_SUITES
 213             )
 214         )
 215         return 1
 216
 217     if doc not in ADD_TEST_SUPPORTED_DOCS:
 218         print(
 219             "Error: invalid `doc`. Either pass in a test with a valid extension"
 220             "({}) or pass in the `doc` argument".format(ADD_TEST_SUPPORTED_DOCS)
 221         )
 222         return 1
 223
 224     creator_cls = addtest.creator_for_suite(suite)
 225
 226     if creator_cls is None:
 227         print("Sorry, `addtest` doesn't currently know how to add {}".format(suite))
 228         return 1
 229
 230     creator = creator_cls(command_context.topsrcdir, test, suite, doc, **kwargs)
 231
 232     creator.check_args()
 233
 234     paths = []
 235     added_tests = False
 236     for path, template in creator:
 237         if not template:
 238             continue
 239         added_tests = True
 240         if path:
 241             paths.append(path)
 242             print("Adding a test file at {} (suite `{}`)".format(path, suite))
 243
 244             try:
 245                 os.makedirs(os.path.dirname(path))
 246             except OSError:
 247                 pass
 248
 249             with io.open(path, "w", newline="\n") as f:
 250                 f.write(template)
 251         else:
 252             # write to stdout if you passed only suite and doc and not a file path
 253             print(template)
 254
 255     if not added_tests:
 256         return 1
 257
 258     if test:
 259         creator.update_manifest()
 260
 261         # Small hack, should really do this better
 262         if suite.startswith("wpt-"):
 263             suite = "web-platform-tests"
 264
 265         mach_command = TEST_SUITES[suite]["mach_command"]
 266         print(
 267             "Please make sure to add the new test to your commit. "
 268             "You can now run the test with:\n    ./mach {} {}".format(
 269                 mach_command, test
 270             )
 271         )
 272
 273     if editor is not MISSING_ARG:
 274         if editor is not None:
 275             editor = editor
 276         elif "VISUAL" in os.environ:
 277             editor = os.environ["VISUAL"]
 278         elif "EDITOR" in os.environ:
 279             editor = os.environ["EDITOR"]
 280         else:
 281             print("Unable to determine editor; please specify a binary")
 282             editor = None
 283
 284         proc = None
 285         if editor:
 286             import subprocess
 287
 288             proc = subprocess.Popen("%s %s" % (editor, " ".join(paths)), shell=True)
 289
 290         if proc:
 291             proc.wait()
 292
 293     return 0
 294
 295
 296 def guess_doc(abs_test):
 297     filename = os.path.basename(abs_test)
 298     return os.path.splitext(filename)[1].strip(".")
 299
 300
 301 def guess_suite(abs_test):
 302     # If you pass a abs_test, try to detect the type based on the name
 303     # and folder. This detection can be skipped if you pass the `type` arg.
 304     err = None
 305     guessed_suite = None
 306     parent = os.path.dirname(abs_test)
 307     filename = os.path.basename(abs_test)
 308
 309     has_browser_ini = os.path.isfile(os.path.join(parent, "browser.ini"))
 310     has_chrome_ini = os.path.isfile(os.path.join(parent, "chrome.ini"))
 311     has_plain_ini = os.path.isfile(os.path.join(parent, "mochitest.ini"))
 312     has_xpcshell_ini = os.path.isfile(os.path.join(parent, "xpcshell.ini"))
 313
 314     in_wpt_folder = abs_test.startswith(
 315         os.path.abspath(os.path.join("testing", "web-platform"))
 316     )
 317
 318     if in_wpt_folder:
 319         guessed_suite = "web-platform-tests-testharness"
 320         if "/css/" in abs_test:
 321             guessed_suite = "web-platform-tests-reftest"
 322     elif (
 323         filename.startswith("test_")
 324         and has_xpcshell_ini
 325         and guess_doc(abs_test) == "js"
 326     ):
 327         guessed_suite = "xpcshell"
 328     else:
 329         if filename.startswith("browser_") and has_browser_ini:
 330             guessed_suite = "mochitest-browser-chrome"
 331         elif filename.startswith("test_"):
 332             if has_chrome_ini and has_plain_ini:
 333                 err = (
 334                     "Error: directory contains both a chrome.ini and mochitest.ini. "
 335                     "Please set --suite=mochitest-chrome or --suite=mochitest-plain."
 336                 )
 337             elif has_chrome_ini:
 338                 guessed_suite = "mochitest-chrome"
 339             elif has_plain_ini:
 340                 guessed_suite = "mochitest-plain"
 341     return guessed_suite, err
 342
 343
 344 @Command(
 345     "test",
 346     category="testing",
 347     description="Run tests (detects the kind of test and runs it).",
 348     parser=get_test_parser,
 349 )
 350 def test(command_context, what, extra_args, **log_args):
 351     """Run tests from names or paths.
 352
 353     mach test accepts arguments specifying which tests to run. Each argument
 354     can be:
 355
 356     * The path to a test file
 357     * A directory containing tests
 358     * A test suite name
 359     * An alias to a test suite name (codes used on TreeHerder)
 360
 361     When paths or directories are given, they are first resolved to test
 362     files known to the build system.
 363
 364     If resolved tests belong to more than one test type/flavor/harness,
 365     the harness for each relevant type/flavor will be invoked. e.g. if
 366     you specify a directory with xpcshell and browser chrome mochitests,
 367     both harnesses will be invoked.
 368
 369     Warning: `mach test` does not automatically re-build.
 370     Please remember to run `mach build` when necessary.
 371
 372     EXAMPLES
 373
 374     Run all test files in the devtools/client/shared/redux/middleware/xpcshell/
 375     directory:
 376
 377     `./mach test devtools/client/shared/redux/middleware/xpcshell/`
 378
 379     The below command prints a short summary of results instead of
 380     the default more verbose output.
 381     Do not forget the - (minus sign) after --log-grouped!
 382
 383     `./mach test --log-grouped - devtools/client/shared/redux/middleware/xpcshell/`
 384     """
 385     from mozlog.commandline import setup_logging
 386     from mozlog.handlers import StreamHandler
 387     from moztest.resolve import get_suite_definition, TestResolver, TEST_SUITES
 388
 389     resolver = command_context._spawn(TestResolver)
 390     run_suites, run_tests = resolver.resolve_metadata(what)
 391
 392     if not run_suites and not run_tests:
 393         print(UNKNOWN_TEST)
 394         return 1
 395
 396     if log_args.get("debugger", None):
 397         import mozdebug
 398
 399         if not mozdebug.get_debugger_info(log_args.get("debugger")):
 400             sys.exit(1)
 401         extra_args_debugger_notation = "=".join(
 402             ["--debugger", log_args.get("debugger")]
 403         )
 404         if extra_args:
 405             extra_args.append(extra_args_debugger_notation)
 406         else:
 407             extra_args = [extra_args_debugger_notation]
 408
 409     # Create shared logger
 410     format_args = {"level": command_context._mach_context.settings["test"]["level"]}
 411     if not run_suites and len(run_tests) == 1:
 412         format_args["verbose"] = True
 413         format_args["compact"] = False
 414
 415     default_format = command_context._mach_context.settings["test"]["format"]
 416     log = setup_logging(
 417         "mach-test", log_args, {default_format: sys.stdout}, format_args
 418     )
 419     for handler in log.handlers:
 420         if isinstance(handler, StreamHandler):
 421             handler.formatter.inner.summary_on_shutdown = True
 422
 423     status = None
 424     for suite_name in run_suites:
 425         suite = TEST_SUITES[suite_name]
 426         kwargs = suite["kwargs"]
 427         kwargs["log"] = log
 428         kwargs.setdefault("subsuite", None)
 429
 430         if "mach_command" in suite:
 431             res = command_context._mach_context.commands.dispatch(
 432                 suite["mach_command"],
 433                 command_context._mach_context,
 434                 argv=extra_args,
 435                 **kwargs,
 436             )
 437             if res:
 438                 status = res
 439
 440     buckets = {}
 441     for test in run_tests:
 442         key = (test["flavor"], test.get("subsuite", ""))
 443         buckets.setdefault(key, []).append(test)
 444
 445     for (flavor, subsuite), tests in sorted(buckets.items()):
 446         _, m = get_suite_definition(flavor, subsuite)
 447         if "mach_command" not in m:
 448             substr = "-{}".format(subsuite) if subsuite else ""
 449             print(UNKNOWN_FLAVOR % (flavor, substr))
 450             status = 1
 451             continue
 452
 453         kwargs = dict(m["kwargs"])
 454         kwargs["log"] = log
 455         kwargs.setdefault("subsuite", None)
 456
 457         res = command_context._mach_context.commands.dispatch(
 458             m["mach_command"],
 459             command_context._mach_context,
 460             argv=extra_args,
 461             test_objects=tests,
 462             **kwargs,
 463         )
 464         if res:
 465             status = res
 466
 467     log.shutdown()
 468     return status
 469
 470
 471 @Command(
 472     "cppunittest", category="testing", description="Run cpp unit tests (C++ tests)."
 473 )
 474 @CommandArgument(
 475     "test_files",
 476     nargs="*",
 477     metavar="N",
 478     help="Test to run. Can be specified as one or more files or "
 479     "directories, or omitted. If omitted, the entire test suite is "
 480     "executed.",
 481 )
 482 def run_cppunit_test(command_context, **params):
 483     from mozlog import commandline
 484
 485     log = params.get("log")
 486     if not log:
 487         log = commandline.setup_logging("cppunittest", {}, {"tbpl": sys.stdout})
 488
 489     # See if we have crash symbols
 490     symbols_path = os.path.join(command_context.distdir, "crashreporter-symbols")
 491     if not os.path.isdir(symbols_path):
 492         symbols_path = None
 493
 494     # If no tests specified, run all tests in main manifest
 495     tests = params["test_files"]
 496     if not tests:
 497         tests = [os.path.join(command_context.distdir, "cppunittests")]
 498         manifest_path = os.path.join(
 499             command_context.topsrcdir, "testing", "cppunittest.ini"
 500         )
 501     else:
 502         manifest_path = None
 503
 504     utility_path = command_context.bindir
 505
 506     if conditions.is_android(command_context):
 507         from mozrunner.devices.android_device import (
 508             verify_android_device,
 509             InstallIntent,
 510         )
 511
 512         verify_android_device(command_context, install=InstallIntent.NO)
 513         return run_android_test(tests, symbols_path, manifest_path, log)
 514
 515     return run_desktop_test(
 516         command_context, tests, symbols_path, manifest_path, utility_path, log
 517     )
 518
 519
 520 def run_desktop_test(
 521     command_context, tests, symbols_path, manifest_path, utility_path, log
 522 ):
 523     import runcppunittests as cppunittests
 524     from mozlog import commandline
 525
 526     parser = cppunittests.CPPUnittestOptions()
 527     commandline.add_logging_group(parser)
 528     options, args = parser.parse_args()
 529
 530     options.symbols_path = symbols_path
 531     options.manifest_path = manifest_path
 532     options.utility_path = utility_path
 533     options.xre_path = command_context.bindir
 534
 535     try:
 536         result = cppunittests.run_test_harness(options, tests)
 537     except Exception as e:
 538         log.error("Caught exception running cpp unit tests: %s" % str(e))
 539         result = False
 540         raise
 541
 542     return 0 if result else 1
 543
 544
 545 def run_android_test(command_context, tests, symbols_path, manifest_path, log):
 546     import remotecppunittests as remotecppunittests
 547     from mozlog import commandline
 548
 549     parser = remotecppunittests.RemoteCPPUnittestOptions()
 550     commandline.add_logging_group(parser)
 551     options, args = parser.parse_args()
 552
 553     if not options.adb_path:
 554         from mozrunner.devices.android_device import get_adb_path
 555
 556         options.adb_path = get_adb_path(command_context)
 557     options.symbols_path = symbols_path
 558     options.manifest_path = manifest_path
 559     options.xre_path = command_context.bindir
 560     options.local_lib = command_context.bindir.replace("bin", "fennec")
 561     for file in os.listdir(os.path.join(command_context.topobjdir, "dist")):
 562         if file.endswith(".apk") and file.startswith("fennec"):
 563             options.local_apk = os.path.join(command_context.topobjdir, "dist", file)
 564             log.info("using APK: " + options.local_apk)
 565             break
 566
 567     try:
 568         result = remotecppunittests.run_test_harness(options, tests)
 569     except Exception as e:
 570         log.error("Caught exception running cpp unit tests: %s" % str(e))
 571         result = False
 572         raise
 573
 574     return 0 if result else 1
 575
 576
 577 def executable_name(name):
 578     return name + ".exe" if sys.platform.startswith("win") else name
 579
 580
 581 @Command(
 582     "jstests",
 583     category="testing",
 584     description="Run SpiderMonkey JS tests in the JS shell.",
 585 )
 586 @CommandArgument("--shell", help="The shell to be used")
 587 @CommandArgument(
 588     "params",
 589     nargs=argparse.REMAINDER,
 590     help="Extra arguments to pass down to the test harness.",
 591 )
 592 def run_jstests(command_context, shell, params):
 593     import subprocess
 594
 595     command_context.virtualenv_manager.ensure()
 596     python = command_context.virtualenv_manager.python_path
 597
 598     js = shell or os.path.join(command_context.bindir, executable_name("js"))
 599     jstest_cmd = [
 600         python,
 601         os.path.join(command_context.topsrcdir, "js", "src", "tests", "jstests.py"),
 602         js,
 603     ] + params
 604
 605     return subprocess.call(jstest_cmd)
 606
 607
 608 @Command(
 609     "jit-test",
 610     category="testing",
 611     description="Run SpiderMonkey jit-tests in the JS shell.",
 612     ok_if_tests_disabled=True,
 613 )
 614 @CommandArgument("--shell", help="The shell to be used")
 615 @CommandArgument(
 616     "--cgc",
 617     action="store_true",
 618     default=False,
 619     help="Run with the SM(cgc) job's env vars",
 620 )
 621 @CommandArgument(
 622     "params",
 623     nargs=argparse.REMAINDER,
 624     help="Extra arguments to pass down to the test harness.",
 625 )
 626 def run_jittests(command_context, shell, cgc, params):
 627     import subprocess
 628
 629     command_context.virtualenv_manager.ensure()
 630     python = command_context.virtualenv_manager.python_path
 631
 632     js = shell or os.path.join(command_context.bindir, executable_name("js"))
 633     jittest_cmd = [
 634         python,
 635         os.path.join(command_context.topsrcdir, "js", "src", "jit-test", "jit_test.py"),
 636         js,
 637     ] + params
 638
 639     env = os.environ.copy()
 640     if cgc:
 641         env["JS_GC_ZEAL"] = "IncrementalMultipleSlices"
 642
 643     return subprocess.call(jittest_cmd, env=env)
 644
 645
 646 @Command("jsapi-tests", category="testing", description="Run SpiderMonkey JSAPI tests.")
 647 @CommandArgument(
 648     "test_name",
 649     nargs="?",
 650     metavar="N",
 651     help="Test to run. Can be a prefix or omitted. If "
 652     "omitted, the entire test suite is executed.",
 653 )
 654 def run_jsapitests(command_context, test_name=None):
 655     import subprocess
 656
 657     jsapi_tests_cmd = [
 658         os.path.join(command_context.bindir, executable_name("jsapi-tests"))
 659     ]
 660     if test_name:
 661         jsapi_tests_cmd.append(test_name)
 662
 663     test_env = os.environ.copy()
 664     test_env["TOPSRCDIR"] = command_context.topsrcdir
 665
 666     result = subprocess.call(jsapi_tests_cmd, env=test_env)
 667     if result != 0:
 668         print(f"jsapi-tests failed, exit code {result}")
 669     return result
 670
 671
 672 def run_check_js_msg(command_context):
 673     import subprocess
 674
 675     command_context.virtualenv_manager.ensure()
 676     python = command_context.virtualenv_manager.python_path
 677
 678     check_cmd = [
 679         python,
 680         os.path.join(command_context.topsrcdir, "config", "check_js_msg_encoding.py"),
 681     ]
 682
 683     return subprocess.call(check_cmd)
 684
 685
 686 def get_jsshell_parser():
 687     from jsshell.benchmark import get_parser
 688
 689     return get_parser()
 690
 691
 692 @Command(
 693     "jsshell-bench",
 694     category="testing",
 695     parser=get_jsshell_parser,
 696     description="Run benchmarks in the SpiderMonkey JS shell.",
 697 )
 698 def run_jsshelltests(command_context, **kwargs):
 699     from jsshell import benchmark
 700
 701     return benchmark.run(**kwargs)
 702
 703
 704 @Command(
 705     "cramtest",
 706     category="testing",
 707     description="Mercurial style .t tests for command line applications.",
 708 )
 709 @CommandArgument(
 710     "test_paths",
 711     nargs="*",
 712     metavar="N",
 713     help="Test paths to run. Each path can be a test file or directory. "
 714     "If omitted, the entire suite will be run.",
 715 )
 716 @CommandArgument(
 717     "cram_args",
 718     nargs=argparse.REMAINDER,
 719     help="Extra arguments to pass down to the cram binary. See "
 720     "'./mach python -m cram -- -h' for a list of available options.",
 721 )
 722 def cramtest(command_context, cram_args=None, test_paths=None, test_objects=None):
 723     command_context.activate_virtualenv()
 724     import mozinfo
 725     from manifestparser import TestManifest
 726
 727     if test_objects is None:
 728         from moztest.resolve import TestResolver
 729
 730         resolver = command_context._spawn(TestResolver)
 731         if test_paths:
 732             # If we were given test paths, try to find tests matching them.
 733             test_objects = resolver.resolve_tests(paths=test_paths, flavor="cram")
 734         else:
 735             # Otherwise just run everything in CRAMTEST_MANIFESTS
 736             test_objects = resolver.resolve_tests(flavor="cram")
 737
 738     if not test_objects:
 739         message = "No tests were collected, check spelling of the test paths."
 740         command_context.log(logging.WARN, "cramtest", {}, message)
 741         return 1
 742
 743     mp = TestManifest()
 744     mp.tests.extend(test_objects)
 745     tests = mp.active_tests(disabled=False, **mozinfo.info)
 746
 747     python = command_context.virtualenv_manager.python_path
 748     cmd = [python, "-m", "cram"] + cram_args + [t["relpath"] for t in tests]
 749     return subprocess.call(cmd, cwd=command_context.topsrcdir)
 750
 751
 752 from datetime import date, timedelta
 753
 754
 755 @Command(
 756     "test-info", category="testing", description="Display historical test results."
 757 )
 758 def test_info(command_context):
 759     """
 760     All functions implemented as subcommands.
 761     """
 762
 763
 764 @SubCommand(
 765     "test-info",
 766     "tests",
 767     description="Display historical test result summary for named tests.",
 768 )
 769 @CommandArgument("test_names", nargs=argparse.REMAINDER, help="Test(s) of interest.")
 770 @CommandArgument(
 771     "--start",
 772     default=(date.today() - timedelta(7)).strftime("%Y-%m-%d"),
 773     help="Start date (YYYY-MM-DD)",
 774 )
 775 @CommandArgument(
 776     "--end", default=date.today().strftime("%Y-%m-%d"), help="End date (YYYY-MM-DD)"
 777 )
 778 @CommandArgument(
 779     "--show-info",
 780     action="store_true",
 781     help="Retrieve and display general test information.",
 782 )
 783 @CommandArgument(
 784     "--show-bugs",
 785     action="store_true",
 786     help="Retrieve and display related Bugzilla bugs.",
 787 )
 788 @CommandArgument("--verbose", action="store_true", help="Enable debug logging.")
 789 def test_info_tests(
 790     command_context,
 791     test_names,
 792     start,
 793     end,
 794     show_info,
 795     show_bugs,
 796     verbose,
 797 ):
 798     import testinfo
 799
 800     ti = testinfo.TestInfoTests(verbose)
 801     ti.report(
 802         test_names,
 803         start,
 804         end,
 805         show_info,
 806         show_bugs,
 807     )
 808
 809
 810 @SubCommand(
 811     "test-info",
 812     "report",
 813     description="Generate a json report of test manifests and/or tests "
 814     "categorized by Bugzilla component and optionally filtered "
 815     "by path, component, and/or manifest annotations.",
 816 )
 817 @CommandArgument(
 818     "--components",
 819     default=None,
 820     help="Comma-separated list of Bugzilla components."
 821     " eg. Testing::General,Core::WebVR",
 822 )
 823 @CommandArgument(
 824     "--flavor",
 825     help='Limit results to tests of the specified flavor (eg. "xpcshell").',
 826 )
 827 @CommandArgument(
 828     "--subsuite",
 829     help='Limit results to tests of the specified subsuite (eg. "devtools").',
 830 )
 831 @CommandArgument(
 832     "paths", nargs=argparse.REMAINDER, help="File system paths of interest."
 833 )
 834 @CommandArgument(
 835     "--show-manifests",
 836     action="store_true",
 837     help="Include test manifests in report.",
 838 )
 839 @CommandArgument(
 840     "--show-tests", action="store_true", help="Include individual tests in report."
 841 )
 842 @CommandArgument(
 843     "--show-summary", action="store_true", help="Include summary in report."
 844 )
 845 @CommandArgument(
 846     "--show-annotations",
 847     action="store_true",
 848     help="Include list of manifest annotation conditions in report.",
 849 )
 850 @CommandArgument(
 851     "--filter-values",
 852     help="Comma-separated list of value regular expressions to filter on; "
 853     "displayed tests contain all specified values.",
 854 )
 855 @CommandArgument(
 856     "--filter-keys",
 857     help="Comma-separated list of test keys to filter on, "
 858     'like "skip-if"; only these fields will be searched '
 859     "for filter-values.",
 860 )
 861 @CommandArgument(
 862     "--no-component-report",
 863     action="store_false",
 864     dest="show_components",
 865     default=True,
 866     help="Do not categorize by bugzilla component.",
 867 )
 868 @CommandArgument("--output-file", help="Path to report file.")
 869 @CommandArgument("--verbose", action="store_true", help="Enable debug logging.")
 870 def test_report(
 871     command_context,
 872     components,
 873     flavor,
 874     subsuite,
 875     paths,
 876     show_manifests,
 877     show_tests,
 878     show_summary,
 879     show_annotations,
 880     filter_values,
 881     filter_keys,
 882     show_components,
 883     output_file,
 884     verbose,
 885 ):
 886     import testinfo
 887     from mozbuild import build_commands
 888
 889     try:
 890         command_context.config_environment
 891     except BuildEnvironmentNotFoundException:
 892         print("Looks like configure has not run yet, running it now...")
 893         build_commands.configure(command_context)
 894
 895     ti = testinfo.TestInfoReport(verbose)
 896     ti.report(
 897         components,
 898         flavor,
 899         subsuite,
 900         paths,
 901         show_manifests,
 902         show_tests,
 903         show_summary,
 904         show_annotations,
 905         filter_values,
 906         filter_keys,
 907         show_components,
 908         output_file,
 909     )
 910
 911
 912 @SubCommand(
 913     "test-info",
 914     "report-diff",
 915     description='Compare two reports generated by "test-info reports".',
 916 )
 917 @CommandArgument(
 918     "--before",
 919     default=None,
 920     help="The first (earlier) report file; path to local file or url.",
 921 )
 922 @CommandArgument(
 923     "--after", help="The second (later) report file; path to local file or url."
 924 )
 925 @CommandArgument(
 926     "--output-file",
 927     help="Path to report file to be written. If not specified, report"
 928     "will be written to standard output.",
 929 )
 930 @CommandArgument("--verbose", action="store_true", help="Enable debug logging.")
 931 def test_report_diff(command_context, before, after, output_file, verbose):
 932     import testinfo
 933
 934     ti = testinfo.TestInfoReport(verbose)
 935     ti.report_diff(before, after, output_file)
 936
 937
 938 @SubCommand(
 939     "test-info",
 940     "failure-report",
 941     description="Display failure line groupings and frequencies for "
 942     "single tracking intermittent bugs.",
 943 )
 944 @CommandArgument(
 945     "--start",
 946     default=(date.today() - timedelta(30)).strftime("%Y-%m-%d"),
 947     help="Start date (YYYY-MM-DD)",
 948 )
 949 @CommandArgument(
 950     "--end", default=date.today().strftime("%Y-%m-%d"), help="End date (YYYY-MM-DD)"
 951 )
 952 @CommandArgument(
 953     "--bugid",
 954     default=None,
 955     help="bugid for treeherder intermittent failures data query.",
 956 )
 957 def test_info_failures(
 958     command_context,
 959     start,
 960     end,
 961     bugid,
 962 ):
 963     import requests
 964
 965     # bugid comes in as a string, we need an int:
 966     try:
 967         bugid = int(bugid)
 968     except ValueError:
 969         bugid = None
 970     if not bugid:
 971         print("Please enter a valid bugid (i.e. '1760132')")
 972         return
 973
 974     # get bug info
 975     url = "https://bugzilla.mozilla.org/rest/bug?include_fields=summary&id=%s" % bugid
 976     r = requests.get(url, headers={"User-agent": "mach-test-info/1.0"})
 977     if r.status_code != 200:
 978         print("%s error retrieving url: %s" % (r.status_code, url))
 979
 980     data = r.json()
 981     if not data:
 982         print("unable to get bugzilla information for %s" % bugid)
 983         return
 984
 985     summary = data["bugs"][0]["summary"]
 986     parts = summary.split("|")
 987     if not summary.endswith("single tracking bug") or len(parts) != 2:
 988         print("this query only works with single tracking bugs")
 989         return
 990
 991     testname = parts[0].strip().split(" ")[-1]
 992
 993     # now query treeherder to get details about annotations
 994     url = "https://treeherder.mozilla.org/api/failuresbybug/"
 995     url += "?startday=%s&endday=%s&tree=trunk&bug=%s" % (start, end, bugid)
 996     r = requests.get(url, headers={"User-agent": "mach-test-info/1.0"})
 997     r.raise_for_status()
 998
 999     data = r.json()
1000     if len(data) == 0:
1001         print("no failures were found for given bugid, please ensure bug is")
1002         print("accessible via: https://treeherder.mozilla.org/intermittent-failures")
1003         return
1004
1005     jobs = {}
1006     lines = {}
1007     for failure in data:
1008         # config = platform/buildtype
1009         # testsuite (<suite>[-variant][-fis][-e10s|1proc][-<chunk>])
1010         # lines - group by patterns that contain test name
1011         config = "%s/%s" % (failure["platform"], failure["build_type"])
1012
1013         if "-e10s" in failure["test_suite"]:
1014             parts = failure["test_suite"].split("-e10s")
1015             sv = parts[0].split("-")
1016             suite = sv[0]
1017             variant = ""
1018             if len(sv) > 1:
1019                 variant = "-%s" % "-".join(sv[1:])
1020         elif "-1proc" in failure["test_suite"]:
1021             parts = failure["test_suite"].split("-1proc")
1022             sv = parts[0].split("-")
1023             suite = sv[0]
1024             variant = ""
1025             if len(sv) > 1:
1026                 variant = "-%s" % "-".join(sv[1:])
1027         else:
1028             print("unable to parse test suite: %s" % failure["test_suite"])
1029             print("no `-e10s` or `-1proc` found")
1030
1031         job = "%s-%s%s" % (config, suite, variant)
1032         if job not in jobs.keys():
1033             jobs[job] = 0
1034         jobs[job] += 1
1035
1036         # lines - sum(hash) of all lines where we match testname
1037         hvalue = 0
1038         for line in failure["lines"]:
1039             if len(line.split(testname)) <= 1:
1040                 continue
1041             # strip off timestamp and mozharness status
1042             parts = line.split("TEST-UNEXPECTED")
1043             l = "TEST-UNEXPECTED%s" % parts[-1]
1044
1045             # only keep 25 characters of the failure, often longer is random numbers
1046             parts = l.split(testname)
1047             l = "%s%s%s" % (parts[0], testname, parts[1][:25])
1048
1049             hvalue += hash(l)
1050
1051         if not hvalue:
1052             continue
1053
1054         if hvalue not in lines.keys():
1055             lines[hvalue] = {"lines": failure["lines"], "config": []}
1056         lines[hvalue]["config"].append(job)
1057
1058     for h in lines.keys():
1059         print("%s errors with:" % (len(lines[h]["config"])))
1060         for l in lines[h]["lines"]:
1061             print(l)
1062         print("")
1063
1064         for job in jobs:
1065             count = len([x for x in lines[h]["config"] if x == job])
1066             if count > 0:
1067                 print("  %s: %s" % (job, count))
1068
1069
1070 @Command(
1071     "rusttests",
1072     category="testing",
1073     conditions=[conditions.is_non_artifact_build],
1074     description="Run rust unit tests (via cargo test).",
1075 )
1076 def run_rusttests(command_context, **kwargs):
1077     return command_context._mach_context.commands.dispatch(
1078         "build",
1079         command_context._mach_context,
1080         what=["pre-export", "export", "recurse_rusttests"],
1081     )
1082
1083
1084 @Command(
1085     "fluent-migration-test",
1086     category="testing",
1087     description="Test Fluent migration recipes.",
1088 )
1089 @CommandArgument("test_paths", nargs="*", metavar="N", help="Recipe paths to test.")
1090 def run_migration_tests(command_context, test_paths=None, **kwargs):
1091     if not test_paths:
1092         test_paths = []
1093     command_context.activate_virtualenv()
1094     from test_fluent_migrations import fmt
1095
1096     rv = 0
1097     with_context = []
1098     for to_test in test_paths:
1099         try:
1100             context = fmt.inspect_migration(to_test)
1101             for issue in context["issues"]:
1102                 command_context.log(
1103                     logging.ERROR,
1104                     "fluent-migration-test",
1105                     {
1106                         "error": issue["msg"],
1107                         "file": to_test,
1108                     },
1109                     "ERROR in {file}: {error}",
1110                 )
1111             if context["issues"]:
1112                 continue
1113             with_context.append(
1114                 {
1115                     "to_test": to_test,
1116                     "references": context["references"],
1117                 }
1118             )
1119         except Exception as e:
1120             command_context.log(
1121                 logging.ERROR,
1122                 "fluent-migration-test",
1123                 {"error": str(e), "file": to_test},
1124                 "ERROR in {file}: {error}",
1125             )
1126             rv |= 1
1127     obj_dir = fmt.prepare_object_dir(command_context)
1128     for context in with_context:
1129         rv |= fmt.test_migration(command_context, obj_dir, **context)
1130     return rv