layout/tools/reftest/runreftest.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 """
   6 Runs the reftest test harness.
   7 """
   8 import json
   9 import multiprocessing
  10 import os
  11 import platform
  12 import posixpath
  13 import re
  14 import shutil
  15 import signal
  16 import subprocess
  17 import sys
  18 import tempfile
  19 import threading
  20 from collections import defaultdict
  21 from datetime import datetime, timedelta
  22
  23 SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
  24 if SCRIPT_DIRECTORY not in sys.path:
  25     sys.path.insert(0, SCRIPT_DIRECTORY)
  26
  27 import mozcrash
  28 import mozdebug
  29 import mozfile
  30 import mozinfo
  31 import mozleak
  32 import mozlog
  33 import mozprocess
  34 import mozprofile
  35 import mozrunner
  36 from manifestparser import TestManifest
  37 from manifestparser import filters as mpf
  38 from mozrunner.utils import get_stack_fixer_function, test_environment
  39 from mozscreenshot import dump_screen, printstatus
  40 from six import reraise, string_types
  41 from six.moves import range
  42
  43 try:
  44     from marionette_driver.addons import Addons
  45     from marionette_driver.marionette import Marionette
  46 except ImportError as e:  # noqa
  47     # Defer ImportError until attempt to use Marionette.
  48     # Python 3 deletes the exception once the except block
  49     # is exited. Save a version to raise later.
  50     e_save = ImportError(str(e))
  51
  52     def reraise_(*args, **kwargs):
  53         raise (e_save)  # noqa
  54
  55     Marionette = reraise_
  56
  57 import reftestcommandline
  58 from output import OutputHandler, ReftestFormatter
  59
  60 here = os.path.abspath(os.path.dirname(__file__))
  61
  62 try:
  63     from mozbuild.base import MozbuildObject
  64
  65     build_obj = MozbuildObject.from_environment(cwd=here)
  66 except ImportError:
  67     build_obj = None
  68
  69
  70 def categoriesToRegex(categoryList):
  71     return "\\(" + ", ".join(["(?P<%s>\\d+) %s" % c for c in categoryList]) + "\\)"
  72
  73
  74 summaryLines = [
  75     ("Successful", [("pass", "pass"), ("loadOnly", "load only")]),
  76     (
  77         "Unexpected",
  78         [
  79             ("fail", "unexpected fail"),
  80             ("pass", "unexpected pass"),
  81             ("asserts", "unexpected asserts"),
  82             ("fixedAsserts", "unexpected fixed asserts"),
  83             ("failedLoad", "failed load"),
  84             ("exception", "exception"),
  85         ],
  86     ),
  87     (
  88         "Known problems",
  89         [
  90             ("knownFail", "known fail"),
  91             ("knownAsserts", "known asserts"),
  92             ("random", "random"),
  93             ("skipped", "skipped"),
  94             ("slow", "slow"),
  95         ],
  96     ),
  97 ]
  98
  99
 100 if sys.version_info[0] == 3:
 101
 102     def reraise_(tp_, value_, tb_=None):
 103         if value_ is None:
 104             value_ = tp_()
 105         if value_.__traceback__ is not tb_:
 106             raise value_.with_traceback(tb_)
 107         raise value_
 108
 109
 110 else:
 111     exec("def reraise_(tp_, value_, tb_=None):\n    raise tp_, value_, tb_\n")
 112
 113
 114 def update_mozinfo():
 115     """walk up directories to find mozinfo.json update the info"""
 116     # TODO: This should go in a more generic place, e.g. mozinfo
 117
 118     path = SCRIPT_DIRECTORY
 119     dirs = set()
 120     while path != os.path.expanduser("~"):
 121         if path in dirs:
 122             break
 123         dirs.add(path)
 124         path = os.path.split(path)[0]
 125     mozinfo.find_and_update_from_json(*dirs)
 126
 127
 128 # Python's print is not threadsafe.
 129 printLock = threading.Lock()
 130
 131
 132 class ReftestThread(threading.Thread):
 133     def __init__(self, cmdargs):
 134         threading.Thread.__init__(self)
 135         self.cmdargs = cmdargs
 136         self.summaryMatches = {}
 137         self.retcode = -1
 138         for text, _ in summaryLines:
 139             self.summaryMatches[text] = None
 140
 141     def run(self):
 142         with printLock:
 143             print("Starting thread with", self.cmdargs)
 144             sys.stdout.flush()
 145         process = subprocess.Popen(self.cmdargs, stdout=subprocess.PIPE)
 146         for chunk in self.chunkForMergedOutput(process.stdout):
 147             with printLock:
 148                 print(chunk, end=" ")
 149                 sys.stdout.flush()
 150         self.retcode = process.wait()
 151
 152     def chunkForMergedOutput(self, logsource):
 153         """Gather lines together that should be printed as one atomic unit.
 154         Individual test results--anything between 'REFTEST TEST-START' and
 155         'REFTEST TEST-END' lines--are an atomic unit.  Lines with data from
 156         summaries are parsed and the data stored for later aggregation.
 157         Other lines are considered their own atomic units and are permitted
 158         to intermix freely."""
 159         testStartRegex = re.compile("^REFTEST TEST-START")
 160         testEndRegex = re.compile("^REFTEST TEST-END")
 161         summaryHeadRegex = re.compile("^REFTEST INFO \\| Result summary:")
 162         summaryRegexFormatString = (
 163             "^REFTEST INFO \\| (?P<message>{text}): (?P<total>\\d+) {regex}"
 164         )
 165         summaryRegexStrings = [
 166             summaryRegexFormatString.format(
 167                 text=text, regex=categoriesToRegex(categories)
 168             )
 169             for (text, categories) in summaryLines
 170         ]
 171         summaryRegexes = [re.compile(regex) for regex in summaryRegexStrings]
 172
 173         for line in logsource:
 174             if testStartRegex.search(line) is not None:
 175                 chunkedLines = [line]
 176                 for lineToBeChunked in logsource:
 177                     chunkedLines.append(lineToBeChunked)
 178                     if testEndRegex.search(lineToBeChunked) is not None:
 179                         break
 180                 yield "".join(chunkedLines)
 181                 continue
 182
 183             haveSuppressedSummaryLine = False
 184             for regex in summaryRegexes:
 185                 match = regex.search(line)
 186                 if match is not None:
 187                     self.summaryMatches[match.group("message")] = match
 188                     haveSuppressedSummaryLine = True
 189                     break
 190             if haveSuppressedSummaryLine:
 191                 continue
 192
 193             if summaryHeadRegex.search(line) is None:
 194                 yield line
 195
 196
 197 class ReftestResolver(object):
 198     def defaultManifest(self, suite):
 199         return {
 200             "reftest": "reftest.list",
 201             "crashtest": "crashtests.list",
 202             "jstestbrowser": "jstests.list",
 203         }[suite]
 204
 205     def directoryManifest(self, suite, path):
 206         return os.path.join(path, self.defaultManifest(suite))
 207
 208     def findManifest(self, suite, test_file, subdirs=True):
 209         """Return a tuple of (manifest-path, filter-string) for running test_file.
 210
 211         test_file is a path to a test or a manifest file
 212         """
 213         rv = []
 214         default_manifest = self.defaultManifest(suite)
 215         relative_path = None
 216         if not os.path.isabs(test_file):
 217             relative_path = test_file
 218             test_file = self.absManifestPath(test_file)
 219
 220         if os.path.isdir(test_file):
 221             for dirpath, dirnames, filenames in os.walk(test_file):
 222                 if default_manifest in filenames:
 223                     rv.append((os.path.join(dirpath, default_manifest), None))
 224                     # We keep recursing into subdirectories which means that in the case
 225                     # of include directives we get the same manifest multiple times.
 226                     # However reftest.js will only read each manifest once
 227
 228             if (
 229                 len(rv) == 0
 230                 and relative_path
 231                 and suite == "jstestbrowser"
 232                 and build_obj
 233             ):
 234                 # The relative path can be from staging area.
 235                 staged_js_dir = os.path.join(
 236                     build_obj.topobjdir, "dist", "test-stage", "jsreftest"
 237                 )
 238                 staged_file = os.path.join(staged_js_dir, "tests", relative_path)
 239                 return self.findManifest(suite, staged_file, subdirs)
 240         elif test_file.endswith(".list"):
 241             if os.path.exists(test_file):
 242                 rv = [(test_file, None)]
 243         else:
 244             dirname, pathname = os.path.split(test_file)
 245             found = True
 246             while not os.path.exists(os.path.join(dirname, default_manifest)):
 247                 dirname, suffix = os.path.split(dirname)
 248                 pathname = posixpath.join(suffix, pathname)
 249                 if os.path.dirname(dirname) == dirname:
 250                     found = False
 251                     break
 252             if found:
 253                 rv = [
 254                     (
 255                         os.path.join(dirname, default_manifest),
 256                         r".*%s(?:[#?].*)?$" % pathname.replace("?", "\?"),
 257                     )
 258                 ]
 259
 260         return rv
 261
 262     def absManifestPath(self, path):
 263         return os.path.normpath(os.path.abspath(path))
 264
 265     def manifestURL(self, options, path):
 266         return "file://%s" % path
 267
 268     def resolveManifests(self, options, tests):
 269         suite = options.suite
 270         manifests = {}
 271         for testPath in tests:
 272             for manifest, filter_str in self.findManifest(suite, testPath):
 273                 if manifest not in manifests:
 274                     manifests[manifest] = set()
 275                 manifests[manifest].add(filter_str)
 276         manifests_by_url = {}
 277         for key in manifests.keys():
 278             id = os.path.relpath(
 279                 os.path.abspath(os.path.dirname(key)), options.topsrcdir
 280             )
 281             id = id.replace(os.sep, posixpath.sep)
 282             if None in manifests[key]:
 283                 manifests[key] = (None, id)
 284             else:
 285                 manifests[key] = ("|".join(list(manifests[key])), id)
 286             url = self.manifestURL(options, key)
 287             manifests_by_url[url] = manifests[key]
 288         return manifests_by_url
 289
 290
 291 class RefTest(object):
 292     oldcwd = os.getcwd()
 293     resolver_cls = ReftestResolver
 294     use_marionette = True
 295
 296     def __init__(self, suite):
 297         update_mozinfo()
 298         self.lastTestSeen = None
 299         self.lastTest = None
 300         self.haveDumpedScreen = False
 301         self.resolver = self.resolver_cls()
 302         self.log = None
 303         self.outputHandler = None
 304         self.testDumpFile = os.path.join(tempfile.gettempdir(), "reftests.json")
 305         self.currentManifest = "No test started"
 306
 307         self.run_by_manifest = True
 308         if suite in ("crashtest", "jstestbrowser"):
 309             self.run_by_manifest = False
 310
 311     def _populate_logger(self, options):
 312         if self.log:
 313             return
 314
 315         self.log = getattr(options, "log", None)
 316         if self.log:
 317             return
 318
 319         mozlog.commandline.log_formatters["tbpl"] = (
 320             ReftestFormatter,
 321             "Reftest specific formatter for the"
 322             "benefit of legacy log parsers and"
 323             "tools such as the reftest analyzer",
 324         )
 325         fmt_options = {}
 326         if not options.log_tbpl_level and os.environ.get("MOZ_REFTEST_VERBOSE"):
 327             options.log_tbpl_level = fmt_options["level"] = "debug"
 328         self.log = mozlog.commandline.setup_logging(
 329             "reftest harness", options, {"tbpl": sys.stdout}, fmt_options
 330         )
 331
 332     def getFullPath(self, path):
 333         "Get an absolute path relative to self.oldcwd."
 334         return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
 335
 336     def createReftestProfile(
 337         self,
 338         options,
 339         tests=None,
 340         manifests=None,
 341         server="localhost",
 342         port=0,
 343         profile_to_clone=None,
 344         prefs=None,
 345     ):
 346         """Sets up a profile for reftest.
 347
 348         :param options: Object containing command line options
 349         :param tests: List of test objects to run
 350         :param manifests: List of manifest files to parse (only takes effect
 351                           if tests were not passed in)
 352         :param server: Server name to use for http tests
 353         :param profile_to_clone: Path to a profile to use as the basis for the
 354                                  test profile
 355         :param prefs: Extra preferences to set in the profile
 356         """
 357         locations = mozprofile.permissions.ServerLocations()
 358         locations.add_host(server, scheme="http", port=port)
 359         locations.add_host(server, scheme="https", port=port)
 360
 361         sandbox_whitelist_paths = options.sandboxReadWhitelist
 362         if platform.system() == "Linux" or platform.system() in (
 363             "Windows",
 364             "Microsoft",
 365         ):
 366             # Trailing slashes are needed to indicate directories on Linux and Windows
 367             sandbox_whitelist_paths = map(
 368                 lambda p: os.path.join(p, ""), sandbox_whitelist_paths
 369             )
 370
 371         addons = []
 372         if not self.use_marionette:
 373             addons.append(options.reftestExtensionPath)
 374
 375         if options.specialPowersExtensionPath is not None:
 376             if not self.use_marionette:
 377                 addons.append(options.specialPowersExtensionPath)
 378
 379         # Install distributed extensions, if application has any.
 380         distExtDir = os.path.join(
 381             options.app[: options.app.rfind(os.sep)], "distribution", "extensions"
 382         )
 383         if os.path.isdir(distExtDir):
 384             for f in os.listdir(distExtDir):
 385                 addons.append(os.path.join(distExtDir, f))
 386
 387         # Install custom extensions.
 388         for f in options.extensionsToInstall:
 389             addons.append(self.getFullPath(f))
 390
 391         kwargs = {
 392             "addons": addons,
 393             "locations": locations,
 394             "whitelistpaths": sandbox_whitelist_paths,
 395         }
 396         if profile_to_clone:
 397             profile = mozprofile.Profile.clone(profile_to_clone, **kwargs)
 398         else:
 399             profile = mozprofile.Profile(**kwargs)
 400
 401         # First set prefs from the base profiles under testing/profiles.
 402
 403         # In test packages used in CI, the profile_data directory is installed
 404         # in the SCRIPT_DIRECTORY.
 405         profile_data_dir = os.path.join(SCRIPT_DIRECTORY, "profile_data")
 406         # If possible, read profile data from topsrcdir. This prevents us from
 407         # requiring a re-build to pick up newly added extensions in the
 408         # <profile>/extensions directory.
 409         if build_obj:
 410             path = os.path.join(build_obj.topsrcdir, "testing", "profiles")
 411             if os.path.isdir(path):
 412                 profile_data_dir = path
 413         # Still not found? Look for testing/profiles relative to layout/tools/reftest.
 414         if not os.path.isdir(profile_data_dir):
 415             path = os.path.abspath(
 416                 os.path.join(SCRIPT_DIRECTORY, "..", "..", "..", "testing", "profiles")
 417             )
 418             if os.path.isdir(path):
 419                 profile_data_dir = path
 420
 421         with open(os.path.join(profile_data_dir, "profiles.json"), "r") as fh:
 422             base_profiles = json.load(fh)["reftest"]
 423
 424         for name in base_profiles:
 425             path = os.path.join(profile_data_dir, name)
 426             profile.merge(path)
 427
 428         # Second set preferences for communication between our command line
 429         # arguments and the reftest harness. Preferences that are required for
 430         # reftest to work should instead be set under srcdir/testing/profiles.
 431         prefs = prefs or {}
 432         prefs["reftest.timeout"] = options.timeout * 1000
 433         if options.logFile:
 434             prefs["reftest.logFile"] = options.logFile
 435         if options.ignoreWindowSize:
 436             prefs["reftest.ignoreWindowSize"] = True
 437         if options.shuffle:
 438             prefs["reftest.shuffle"] = True
 439         if options.repeat:
 440             prefs["reftest.repeat"] = options.repeat
 441         if options.runUntilFailure:
 442             prefs["reftest.runUntilFailure"] = True
 443             if not options.repeat:
 444                 prefs["reftest.repeat"] = 30
 445         if options.verify:
 446             prefs["reftest.verify"] = True
 447         if options.cleanupCrashes:
 448             prefs["reftest.cleanupPendingCrashes"] = True
 449         prefs["reftest.focusFilterMode"] = options.focusFilterMode
 450         prefs["reftest.logLevel"] = options.log_tbpl_level or "info"
 451         prefs["reftest.suite"] = options.suite
 452         prefs["gfx.font_rendering.ahem_antialias_none"] = True
 453         # Run the "deferred" font-loader immediately, because if it finishes
 454         # mid-test, the extra reflow that is triggered can disrupt the test.
 455         prefs["gfx.font_loader.delay"] = 0
 456         # Ensure bundled fonts are activated, even if not enabled by default
 457         # on the platform, so that tests can rely on them.
 458         prefs["gfx.bundled-fonts.activate"] = 1
 459         # Disable dark scrollbars because it's semi-transparent.
 460         prefs["widget.disable-dark-scrollbar"] = True
 461         prefs["reftest.isCoverageBuild"] = mozinfo.info.get("ccov", False)
 462
 463         # config specific flags
 464         prefs["sandbox.apple_silicon"] = mozinfo.info.get("apple_silicon", False)
 465
 466         # Set tests to run or manifests to parse.
 467         if tests:
 468             testlist = os.path.join(profile.profile, "reftests.json")
 469             with open(testlist, "w") as fh:
 470                 json.dump(tests, fh)
 471             prefs["reftest.tests"] = testlist
 472         elif manifests:
 473             prefs["reftest.manifests"] = json.dumps(manifests)
 474
 475         # Unconditionally update the e10s pref, default True
 476         prefs["browser.tabs.remote.autostart"] = True
 477         if not options.e10s:
 478             prefs["browser.tabs.remote.autostart"] = False
 479
 480         # default fission to True
 481         prefs["fission.autostart"] = True
 482         if options.disableFission:
 483             prefs["fission.autostart"] = False
 484
 485         if not self.run_by_manifest:
 486             if options.totalChunks:
 487                 prefs["reftest.totalChunks"] = options.totalChunks
 488             if options.thisChunk:
 489                 prefs["reftest.thisChunk"] = options.thisChunk
 490
 491         # Bug 1262954: For winXP + e10s disable acceleration
 492         if (
 493             platform.system() in ("Windows", "Microsoft")
 494             and "5.1" in platform.version()
 495             and options.e10s
 496         ):
 497             prefs["layers.acceleration.disabled"] = True
 498
 499         # Bug 1300355: Disable canvas cache for win7 as it uses
 500         # too much memory and causes OOMs.
 501         if (
 502             platform.system() in ("Windows", "Microsoft")
 503             and "6.1" in platform.version()
 504         ):
 505             prefs["reftest.nocache"] = True
 506
 507         if options.marionette:
 508             # options.marionette can specify host:port
 509             port = options.marionette.split(":")[1]
 510             prefs["marionette.port"] = int(port)
 511
 512         # Enable tracing output for detailed failures in case of
 513         # failing connection attempts, and hangs (bug 1397201)
 514         prefs["remote.log.level"] = "Trace"
 515
 516         # Third, set preferences passed in via the command line.
 517         for v in options.extraPrefs:
 518             thispref = v.split("=")
 519             if len(thispref) < 2:
 520                 print("Error: syntax error in --setpref=" + v)
 521                 sys.exit(1)
 522             prefs[thispref[0]] = thispref[1].strip()
 523
 524         for pref in prefs:
 525             prefs[pref] = mozprofile.Preferences.cast(prefs[pref])
 526         profile.set_preferences(prefs)
 527
 528         if os.path.join(here, "chrome") not in options.extraProfileFiles:
 529             options.extraProfileFiles.append(os.path.join(here, "chrome"))
 530
 531         self.copyExtraFilesToProfile(options, profile)
 532
 533         self.log.info(
 534             "Running with e10s: {}".format(prefs["browser.tabs.remote.autostart"])
 535         )
 536         self.log.info("Running with fission: {}".format(prefs["fission.autostart"]))
 537
 538         return profile
 539
 540     def environment(self, **kwargs):
 541         kwargs["log"] = self.log
 542         return test_environment(**kwargs)
 543
 544     def buildBrowserEnv(self, options, profileDir):
 545         browserEnv = self.environment(
 546             xrePath=options.xrePath, debugger=options.debugger
 547         )
 548         browserEnv["XPCOM_DEBUG_BREAK"] = "stack"
 549         if options.topsrcdir:
 550             browserEnv["MOZ_DEVELOPER_REPO_DIR"] = options.topsrcdir
 551         if hasattr(options, "topobjdir"):
 552             browserEnv["MOZ_DEVELOPER_OBJ_DIR"] = options.topobjdir
 553
 554         if mozinfo.info["asan"]:
 555             # Disable leak checking for reftests for now
 556             if "ASAN_OPTIONS" in browserEnv:
 557                 browserEnv["ASAN_OPTIONS"] += ":detect_leaks=0"
 558             else:
 559                 browserEnv["ASAN_OPTIONS"] = "detect_leaks=0"
 560
 561         # Set environment defaults for jstestbrowser. Keep in sync with the
 562         # defaults used in js/src/tests/lib/tests.py.
 563         if options.suite == "jstestbrowser":
 564             browserEnv["TZ"] = "PST8PDT"
 565             browserEnv["LC_ALL"] = "en_US.UTF-8"
 566
 567         for v in options.environment:
 568             ix = v.find("=")
 569             if ix <= 0:
 570                 print("Error: syntax error in --setenv=" + v)
 571                 return None
 572             browserEnv[v[:ix]] = v[ix + 1 :]
 573
 574         # Enable leaks detection to its own log file.
 575         self.leakLogFile = os.path.join(profileDir, "runreftest_leaks.log")
 576         browserEnv["XPCOM_MEM_BLOAT_LOG"] = self.leakLogFile
 577
 578         # TODO: this is always defined (as part of --enable-webrender which is default)
 579         #       can we make this default in the browser?
 580         browserEnv["MOZ_ACCELERATED"] = "1"
 581
 582         if options.headless:
 583             browserEnv["MOZ_HEADLESS"] = "1"
 584
 585         return browserEnv
 586
 587     def cleanup(self, profileDir):
 588         if profileDir:
 589             shutil.rmtree(profileDir, True)
 590
 591     def verifyTests(self, tests, options):
 592         """
 593         Support --verify mode: Run test(s) many times in a variety of
 594         configurations/environments in an effort to find intermittent
 595         failures.
 596         """
 597
 598         self._populate_logger(options)
 599
 600         # Number of times to repeat test(s) when running with --repeat
 601         VERIFY_REPEAT = 10
 602         # Number of times to repeat test(s) when running test in separate browser
 603         VERIFY_REPEAT_SINGLE_BROWSER = 5
 604
 605         def step1():
 606             options.repeat = VERIFY_REPEAT
 607             options.runUntilFailure = True
 608             result = self.runTests(tests, options)
 609             return result
 610
 611         def step2():
 612             options.repeat = 0
 613             options.runUntilFailure = False
 614             for i in range(VERIFY_REPEAT_SINGLE_BROWSER):
 615                 result = self.runTests(tests, options)
 616                 if result != 0:
 617                     break
 618             return result
 619
 620         def step3():
 621             options.repeat = VERIFY_REPEAT
 622             options.runUntilFailure = True
 623             options.environment.append("MOZ_CHAOSMODE=0xfb")
 624             result = self.runTests(tests, options)
 625             options.environment.remove("MOZ_CHAOSMODE=0xfb")
 626             return result
 627
 628         def step4():
 629             options.repeat = 0
 630             options.runUntilFailure = False
 631             options.environment.append("MOZ_CHAOSMODE=0xfb")
 632             for i in range(VERIFY_REPEAT_SINGLE_BROWSER):
 633                 result = self.runTests(tests, options)
 634                 if result != 0:
 635                     break
 636             options.environment.remove("MOZ_CHAOSMODE=0xfb")
 637             return result
 638
 639         steps = [
 640             ("1. Run each test %d times in one browser." % VERIFY_REPEAT, step1),
 641             (
 642                 "2. Run each test %d times in a new browser each time."
 643                 % VERIFY_REPEAT_SINGLE_BROWSER,
 644                 step2,
 645             ),
 646             (
 647                 "3. Run each test %d times in one browser, in chaos mode."
 648                 % VERIFY_REPEAT,
 649                 step3,
 650             ),
 651             (
 652                 "4. Run each test %d times in a new browser each time, in chaos mode."
 653                 % VERIFY_REPEAT_SINGLE_BROWSER,
 654                 step4,
 655             ),
 656         ]
 657
 658         stepResults = {}
 659         for (descr, step) in steps:
 660             stepResults[descr] = "not run / incomplete"
 661
 662         startTime = datetime.now()
 663         maxTime = timedelta(seconds=options.verify_max_time)
 664         finalResult = "PASSED"
 665         for (descr, step) in steps:
 666             if (datetime.now() - startTime) > maxTime:
 667                 self.log.info("::: Test verification is taking too long: Giving up!")
 668                 self.log.info(
 669                     "::: So far, all checks passed, but not all checks were run."
 670                 )
 671                 break
 672             self.log.info(":::")
 673             self.log.info('::: Running test verification step "%s"...' % descr)
 674             self.log.info(":::")
 675             result = step()
 676             if result != 0:
 677                 stepResults[descr] = "FAIL"
 678                 finalResult = "FAILED!"
 679                 break
 680             stepResults[descr] = "Pass"
 681
 682         self.log.info(":::")
 683         self.log.info("::: Test verification summary for:")
 684         self.log.info(":::")
 685         for test in tests:
 686             self.log.info("::: " + test)
 687         self.log.info(":::")
 688         for descr in sorted(stepResults.keys()):
 689             self.log.info("::: %s : %s" % (descr, stepResults[descr]))
 690         self.log.info(":::")
 691         self.log.info("::: Test verification %s" % finalResult)
 692         self.log.info(":::")
 693
 694         return result
 695
 696     def runTests(self, tests, options, cmdargs=None):
 697         cmdargs = cmdargs or []
 698         self._populate_logger(options)
 699         self.outputHandler = OutputHandler(
 700             self.log, options.utilityPath, options.symbolsPath
 701         )
 702
 703         if options.cleanupCrashes:
 704             mozcrash.cleanup_pending_crash_reports()
 705
 706         manifests = self.resolver.resolveManifests(options, tests)
 707         if options.filter:
 708             manifests[""] = (options.filter, None)
 709
 710         if not getattr(options, "runTestsInParallel", False):
 711             return self.runSerialTests(manifests, options, cmdargs)
 712
 713         cpuCount = multiprocessing.cpu_count()
 714
 715         # We have the directive, technology, and machine to run multiple test instances.
 716         # Experimentation says that reftests are not overly CPU-intensive, so we can run
 717         # multiple jobs per CPU core.
 718         #
 719         # Our Windows machines in automation seem to get upset when we run a lot of
 720         # simultaneous tests on them, so tone things down there.
 721         if sys.platform == "win32":
 722             jobsWithoutFocus = cpuCount
 723         else:
 724             jobsWithoutFocus = 2 * cpuCount
 725
 726         totalJobs = jobsWithoutFocus + 1
 727         perProcessArgs = [sys.argv[:] for i in range(0, totalJobs)]
 728
 729         host = "localhost"
 730         port = 2828
 731         if options.marionette:
 732             host, port = options.marionette.split(":")
 733
 734         # First job is only needs-focus tests.  Remaining jobs are
 735         # non-needs-focus and chunked.
 736         perProcessArgs[0].insert(-1, "--focus-filter-mode=needs-focus")
 737         for (chunkNumber, jobArgs) in enumerate(perProcessArgs[1:], start=1):
 738             jobArgs[-1:-1] = [
 739                 "--focus-filter-mode=non-needs-focus",
 740                 "--total-chunks=%d" % jobsWithoutFocus,
 741                 "--this-chunk=%d" % chunkNumber,
 742                 "--marionette=%s:%d" % (host, port),
 743             ]
 744             port += 1
 745
 746         for jobArgs in perProcessArgs:
 747             try:
 748                 jobArgs.remove("--run-tests-in-parallel")
 749             except Exception:
 750                 pass
 751             jobArgs[0:0] = [sys.executable, "-u"]
 752
 753         threads = [ReftestThread(args) for args in perProcessArgs[1:]]
 754         for t in threads:
 755             t.start()
 756
 757         while True:
 758             # The test harness in each individual thread will be doing timeout
 759             # handling on its own, so we shouldn't need to worry about any of
 760             # the threads hanging for arbitrarily long.
 761             for t in threads:
 762                 t.join(10)
 763             if not any(t.is_alive() for t in threads):
 764                 break
 765
 766         # Run the needs-focus tests serially after the other ones, so we don't
 767         # have to worry about races between the needs-focus tests *actually*
 768         # needing focus and the dummy windows in the non-needs-focus tests
 769         # trying to focus themselves.
 770         focusThread = ReftestThread(perProcessArgs[0])
 771         focusThread.start()
 772         focusThread.join()
 773
 774         # Output the summaries that the ReftestThread filters suppressed.
 775         summaryObjects = [defaultdict(int) for s in summaryLines]
 776         for t in threads:
 777             for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):
 778                 threadMatches = t.summaryMatches[text]
 779                 for (attribute, description) in categories:
 780                     amount = int(threadMatches.group(attribute) if threadMatches else 0)
 781                     summaryObj[attribute] += amount
 782                 amount = int(threadMatches.group("total") if threadMatches else 0)
 783                 summaryObj["total"] += amount
 784
 785         print("REFTEST INFO | Result summary:")
 786         for (summaryObj, (text, categories)) in zip(summaryObjects, summaryLines):
 787             details = ", ".join(
 788                 [
 789                     "%d %s" % (summaryObj[attribute], description)
 790                     for (attribute, description) in categories
 791                 ]
 792             )
 793             print(
 794                 "REFTEST INFO | "
 795                 + text
 796                 + ": "
 797                 + str(summaryObj["total"])
 798                 + " ("
 799                 + details
 800                 + ")"
 801             )
 802
 803         return int(any(t.retcode != 0 for t in threads))
 804
 805     def handleTimeout(self, timeout, proc, utilityPath, debuggerInfo):
 806         """handle process output timeout"""
 807         # TODO: bug 913975 : _processOutput should call self.processOutputLine
 808         # one more time one timeout (I think)
 809         self.log.error(
 810             "%s | application timed out after %d seconds with no output"
 811             % (self.lastTestSeen, int(timeout))
 812         )
 813         self.log.warning("Force-terminating active process(es).")
 814         self.killAndGetStack(
 815             proc, utilityPath, debuggerInfo, dump_screen=not debuggerInfo
 816         )
 817
 818     def dumpScreen(self, utilityPath):
 819         if self.haveDumpedScreen:
 820             self.log.info(
 821                 "Not taking screenshot here: see the one that was previously logged"
 822             )
 823             return
 824         self.haveDumpedScreen = True
 825         dump_screen(utilityPath, self.log)
 826
 827     def killAndGetStack(self, process, utilityPath, debuggerInfo, dump_screen=False):
 828         """
 829         Kill the process, preferrably in a way that gets us a stack trace.
 830         Also attempts to obtain a screenshot before killing the process
 831         if specified.
 832         """
 833
 834         if dump_screen:
 835             self.dumpScreen(utilityPath)
 836
 837         if mozinfo.info.get("crashreporter", True) and not debuggerInfo:
 838             if mozinfo.isWin:
 839                 # We should have a "crashinject" program in our utility path
 840                 crashinject = os.path.normpath(
 841                     os.path.join(utilityPath, "crashinject.exe")
 842                 )
 843                 if os.path.exists(crashinject):
 844                     status = subprocess.Popen([crashinject, str(process.pid)]).wait()
 845                     printstatus("crashinject", status)
 846                     if status == 0:
 847                         return
 848             else:
 849                 try:
 850                     process.kill(sig=signal.SIGABRT)
 851                 except OSError:
 852                     # https://bugzilla.mozilla.org/show_bug.cgi?id=921509
 853                     self.log.info("Can't trigger Breakpad, process no longer exists")
 854                 return
 855         self.log.info("Can't trigger Breakpad, just killing process")
 856         process.kill()
 857
 858     def runApp(
 859         self,
 860         options,
 861         cmdargs=None,
 862         timeout=None,
 863         debuggerInfo=None,
 864         symbolsPath=None,
 865         valgrindPath=None,
 866         valgrindArgs=None,
 867         valgrindSuppFiles=None,
 868         **profileArgs
 869     ):
 870
 871         if cmdargs is None:
 872             cmdargs = []
 873         cmdargs = cmdargs[:]
 874
 875         if self.use_marionette:
 876             cmdargs.append("-marionette")
 877
 878         binary = options.app
 879         profile = self.createReftestProfile(options, **profileArgs)
 880
 881         # browser environment
 882         env = self.buildBrowserEnv(options, profile.profile)
 883
 884         def timeoutHandler():
 885             self.handleTimeout(timeout, proc, options.utilityPath, debuggerInfo)
 886
 887         interactive = False
 888         debug_args = None
 889         if debuggerInfo:
 890             interactive = debuggerInfo.interactive
 891             debug_args = [debuggerInfo.path] + debuggerInfo.args
 892
 893         def record_last_test(message):
 894             """Records the last test seen by this harness for the benefit of crash logging."""
 895
 896             def testid(test):
 897                 if " " in test:
 898                     return test.split(" ")[0]
 899                 return test
 900
 901             if message["action"] == "test_start":
 902                 self.lastTestSeen = testid(message["test"])
 903             elif message["action"] == "test_end":
 904                 if self.lastTest and message["test"] == self.lastTest:
 905                     self.lastTestSeen = self.currentManifest
 906                 else:
 907                     self.lastTestSeen = "{} (finished)".format(testid(message["test"]))
 908
 909         self.log.add_handler(record_last_test)
 910
 911         kp_kwargs = {
 912             "kill_on_timeout": False,
 913             "cwd": SCRIPT_DIRECTORY,
 914             "onTimeout": [timeoutHandler],
 915             "processOutputLine": [self.outputHandler],
 916         }
 917
 918         if mozinfo.isWin or mozinfo.isMac:
 919             # Prevents log interleaving on Windows at the expense of losing
 920             # true log order. See bug 798300 and bug 1324961 for more details.
 921             kp_kwargs["processStderrLine"] = [self.outputHandler]
 922
 923         if interactive:
 924             # If an interactive debugger is attached,
 925             # don't use timeouts, and don't capture ctrl-c.
 926             timeout = None
 927             signal.signal(signal.SIGINT, lambda sigid, frame: None)
 928
 929         runner_cls = mozrunner.runners.get(
 930             mozinfo.info.get("appname", "firefox"), mozrunner.Runner
 931         )
 932         runner = runner_cls(
 933             profile=profile,
 934             binary=binary,
 935             process_class=mozprocess.ProcessHandlerMixin,
 936             cmdargs=cmdargs,
 937             env=env,
 938             process_args=kp_kwargs,
 939         )
 940         runner.start(
 941             debug_args=debug_args, interactive=interactive, outputTimeout=timeout
 942         )
 943         proc = runner.process_handler
 944         self.outputHandler.proc_name = "GECKO({})".format(proc.pid)
 945
 946         # Used to defer a possible IOError exception from Marionette
 947         marionette_exception = None
 948
 949         if self.use_marionette:
 950             marionette_args = {
 951                 "socket_timeout": options.marionette_socket_timeout,
 952                 "startup_timeout": options.marionette_startup_timeout,
 953                 "symbols_path": options.symbolsPath,
 954             }
 955             if options.marionette:
 956                 host, port = options.marionette.split(":")
 957                 marionette_args["host"] = host
 958                 marionette_args["port"] = int(port)
 959
 960             try:
 961                 marionette = Marionette(**marionette_args)
 962                 marionette.start_session()
 963
 964                 addons = Addons(marionette)
 965                 if options.specialPowersExtensionPath:
 966                     addons.install(options.specialPowersExtensionPath, temp=True)
 967
 968                 addons.install(options.reftestExtensionPath, temp=True)
 969
 970                 marionette.delete_session()
 971             except IOError:
 972                 # Any IOError as thrown by Marionette means that something is
 973                 # wrong with the process, like a crash or the socket is no
 974                 # longer open. We defer raising this specific error so that
 975                 # post-test checks for leaks and crashes are performed and
 976                 # reported first.
 977                 marionette_exception = sys.exc_info()
 978
 979         status = runner.wait()
 980         runner.process_handler = None
 981         self.outputHandler.proc_name = None
 982
 983         crashed = mozcrash.log_crashes(
 984             self.log,
 985             os.path.join(profile.profile, "minidumps"),
 986             options.symbolsPath,
 987             test=self.lastTestSeen,
 988         )
 989
 990         if crashed:
 991             # log suite_end to wrap up, this is usually done with in in-browser harness
 992             if not self.outputHandler.results:
 993                 # TODO: while .results is a defaultdict(int), it is proxied via log_actions as data, not type
 994                 self.outputHandler.results = {
 995                     "Pass": 0,
 996                     "LoadOnly": 0,
 997                     "Exception": 0,
 998                     "FailedLoad": 0,
 999                     "UnexpectedFail": 1,
1000                     "UnexpectedPass": 0,
1001                     "AssertionUnexpected": 0,
1002                     "AssertionUnexpectedFixed": 0,
1003                     "KnownFail": 0,
1004                     "AssertionKnown": 0,
1005                     "Random": 0,
1006                     "Skip": 0,
1007                     "Slow": 0,
1008                 }
1009             self.log.suite_end(extra={"results": self.outputHandler.results})
1010
1011         if not status and crashed:
1012             status = 1
1013
1014         if status and not crashed:
1015             msg = (
1016                 "TEST-UNEXPECTED-FAIL | %s | application terminated with exit code %s"
1017                 % (self.lastTestSeen, status)
1018             )
1019             # use process_output so message is logged verbatim
1020             self.log.process_output(None, msg)
1021
1022         runner.cleanup()
1023         self.cleanup(profile.profile)
1024
1025         if marionette_exception is not None:
1026             exc, value, tb = marionette_exception
1027             raise reraise(exc, value, tb)
1028
1029         self.log.info("Process mode: {}".format("e10s" if options.e10s else "non-e10s"))
1030         return status
1031
1032     def getActiveTests(self, manifests, options, testDumpFile=None):
1033         # These prefs will cause reftest.jsm to parse the manifests,
1034         # dump the resulting tests to a file, and exit.
1035         prefs = {
1036             "reftest.manifests": json.dumps(manifests),
1037             "reftest.manifests.dumpTests": testDumpFile or self.testDumpFile,
1038         }
1039         cmdargs = []
1040         self.runApp(options, cmdargs=cmdargs, prefs=prefs)
1041
1042         if not os.path.isfile(self.testDumpFile):
1043             print("Error: parsing manifests failed!")
1044             sys.exit(1)
1045
1046         with open(self.testDumpFile, "r") as fh:
1047             tests = json.load(fh)
1048
1049         if os.path.isfile(self.testDumpFile):
1050             mozfile.remove(self.testDumpFile)
1051
1052         for test in tests:
1053             # Name and path are expected by manifestparser, but not used in reftest.
1054             test["name"] = test["path"] = test["url1"]
1055
1056         mp = TestManifest(strict=False)
1057         mp.tests = tests
1058
1059         filters = []
1060         if options.totalChunks:
1061             filters.append(
1062                 mpf.chunk_by_manifest(options.thisChunk, options.totalChunks)
1063             )
1064
1065         tests = mp.active_tests(exists=False, filters=filters)
1066         return tests
1067
1068     def runSerialTests(self, manifests, options, cmdargs=None):
1069         debuggerInfo = None
1070         if options.debugger:
1071             debuggerInfo = mozdebug.get_debugger_info(
1072                 options.debugger, options.debuggerArgs, options.debuggerInteractive
1073             )
1074
1075         def run(**kwargs):
1076             if kwargs.get("tests"):
1077                 self.lastTest = kwargs["tests"][-1]["identifier"]
1078                 if not isinstance(self.lastTest, string_types):
1079                     self.lastTest = " ".join(self.lastTest)
1080
1081             status = self.runApp(
1082                 options,
1083                 manifests=manifests,
1084                 cmdargs=cmdargs,
1085                 # We generally want the JS harness or marionette
1086                 # to handle timeouts if they can.
1087                 # The default JS harness timeout is currently
1088                 # 300 seconds (default options.timeout).
1089                 # The default Marionette socket timeout is
1090                 # currently 360 seconds.
1091                 # Give the JS harness extra time to deal with
1092                 # its own timeouts and try to usually exceed
1093                 # the 360 second marionette socket timeout.
1094                 # See bug 479518 and bug 1414063.
1095                 timeout=options.timeout + 70.0,
1096                 debuggerInfo=debuggerInfo,
1097                 symbolsPath=options.symbolsPath,
1098                 **kwargs
1099             )
1100
1101             # do not process leak log when we crash/assert
1102             if status == 0:
1103                 mozleak.process_leak_log(
1104                     self.leakLogFile,
1105                     leak_thresholds=options.leakThresholds,
1106                     stack_fixer=get_stack_fixer_function(
1107                         options.utilityPath, options.symbolsPath
1108                     ),
1109                 )
1110             return status
1111
1112         if not self.run_by_manifest:
1113             return run()
1114
1115         tests = self.getActiveTests(manifests, options)
1116         tests_by_manifest = defaultdict(list)
1117         ids_by_manifest = defaultdict(list)
1118         for t in tests:
1119             tests_by_manifest[t["manifest"]].append(t)
1120             test_id = t["identifier"]
1121             if not isinstance(test_id, string_types):
1122                 test_id = " ".join(test_id)
1123             ids_by_manifest[t["manifestID"]].append(test_id)
1124
1125         self.log.suite_start(ids_by_manifest, name=options.suite)
1126
1127         overall = 0
1128         status = -1
1129         for manifest, tests in tests_by_manifest.items():
1130             self.log.info("Running tests in {}".format(manifest))
1131             self.currentManifest = manifest
1132             status = run(tests=tests)
1133             overall = overall or status
1134         if status == -1:
1135             # we didn't run anything
1136             overall = 1
1137
1138         self.log.suite_end(extra={"results": self.outputHandler.results})
1139         return overall
1140
1141     def copyExtraFilesToProfile(self, options, profile):
1142         "Copy extra files or dirs specified on the command line to the testing profile."
1143         profileDir = profile.profile
1144         for f in options.extraProfileFiles:
1145             abspath = self.getFullPath(f)
1146             if os.path.isfile(abspath):
1147                 if os.path.basename(abspath) == "user.js":
1148                     extra_prefs = mozprofile.Preferences.read_prefs(abspath)
1149                     profile.set_preferences(extra_prefs)
1150                 elif os.path.basename(abspath).endswith(".dic"):
1151                     hyphDir = os.path.join(profileDir, "hyphenation")
1152                     if not os.path.exists(hyphDir):
1153                         os.makedirs(hyphDir)
1154                     shutil.copy2(abspath, hyphDir)
1155                 else:
1156                     shutil.copy2(abspath, profileDir)
1157             elif os.path.isdir(abspath):
1158                 dest = os.path.join(profileDir, os.path.basename(abspath))
1159                 shutil.copytree(abspath, dest)
1160             else:
1161                 self.log.warning(
1162                     "runreftest.py | Failed to copy %s to profile" % abspath
1163                 )
1164                 continue
1165
1166
1167 def run_test_harness(parser, options):
1168     reftest = RefTest(options.suite)
1169     parser.validate(options, reftest)
1170
1171     # We have to validate options.app here for the case when the mach
1172     # command is able to find it after argument parsing. This can happen
1173     # when running from a tests archive.
1174     if not options.app:
1175         parser.error("could not find the application path, --appname must be specified")
1176
1177     options.app = reftest.getFullPath(options.app)
1178     if not os.path.exists(options.app):
1179         parser.error(
1180             "Error: Path %(app)s doesn't exist. Are you executing "
1181             "$objdir/_tests/reftest/runreftest.py?" % {"app": options.app}
1182         )
1183
1184     if options.xrePath is None:
1185         options.xrePath = os.path.dirname(options.app)
1186
1187     if options.verify:
1188         result = reftest.verifyTests(options.tests, options)
1189     else:
1190         result = reftest.runTests(options.tests, options)
1191
1192     return result
1193
1194
1195 if __name__ == "__main__":
1196     parser = reftestcommandline.DesktopArgumentsParser()
1197     options = parser.parse_args()
1198     sys.exit(run_test_harness(parser, options))