build/automationutils.py

   1 #
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 from __future__ import with_statement
   7 import logging
   8 from operator import itemgetter
   9 import os
  10 import platform
  11 import re
  12 import signal
  13 import subprocess
  14 import sys
  15 import tempfile
  16 from urlparse import urlparse
  17 import zipfile
  18 import mozinfo
  19
  20 __all__ = [
  21   "ZipFileReader",
  22   "addCommonOptions",
  23   "dumpLeakLog",
  24   "isURL",
  25   "processLeakLog",
  26   "replaceBackSlashes",
  27   'KeyValueParseError',
  28   'parseKeyValue',
  29   'systemMemory',
  30   'environment',
  31   'dumpScreen',
  32   "ShutdownLeaks",
  33   "setAutomationLog",
  34   ]
  35
  36 log = logging.getLogger()
  37 def resetGlobalLog():
  38   while log.handlers:
  39     log.removeHandler(log.handlers[0])
  40   handler = logging.StreamHandler(sys.stdout)
  41   log.setLevel(logging.INFO)
  42   log.addHandler(handler)
  43 resetGlobalLog()
  44
  45 def setAutomationLog(alt_logger):
  46   global log
  47   log = alt_logger
  48
  49 class ZipFileReader(object):
  50   """
  51   Class to read zip files in Python 2.5 and later. Limited to only what we
  52   actually use.
  53   """
  54
  55   def __init__(self, filename):
  56     self._zipfile = zipfile.ZipFile(filename, "r")
  57
  58   def __del__(self):
  59     self._zipfile.close()
  60
  61   def _getnormalizedpath(self, path):
  62     """
  63     Gets a normalized path from 'path' (or the current working directory if
  64     'path' is None). Also asserts that the path exists.
  65     """
  66     if path is None:
  67       path = os.curdir
  68     path = os.path.normpath(os.path.expanduser(path))
  69     assert os.path.isdir(path)
  70     return path
  71
  72   def _extractname(self, name, path):
  73     """
  74     Extracts a file with the given name from the zip file to the given path.
  75     Also creates any directories needed along the way.
  76     """
  77     filename = os.path.normpath(os.path.join(path, name))
  78     if name.endswith("/"):
  79       os.makedirs(filename)
  80     else:
  81       path = os.path.split(filename)[0]
  82       if not os.path.isdir(path):
  83         os.makedirs(path)
  84       with open(filename, "wb") as dest:
  85         dest.write(self._zipfile.read(name))
  86
  87   def namelist(self):
  88     return self._zipfile.namelist()
  89
  90   def read(self, name):
  91     return self._zipfile.read(name)
  92
  93   def extract(self, name, path = None):
  94     if hasattr(self._zipfile, "extract"):
  95       return self._zipfile.extract(name, path)
  96
  97     # This will throw if name is not part of the zip file.
  98     self._zipfile.getinfo(name)
  99
 100     self._extractname(name, self._getnormalizedpath(path))
 101
 102   def extractall(self, path = None):
 103     if hasattr(self._zipfile, "extractall"):
 104       return self._zipfile.extractall(path)
 105
 106     path = self._getnormalizedpath(path)
 107
 108     for name in self._zipfile.namelist():
 109       self._extractname(name, path)
 110
 111 def isURL(thing):
 112   """Return True if |thing| looks like a URL."""
 113   # We want to download URLs like http://... but not Windows paths like c:\...
 114   return len(urlparse(thing).scheme) >= 2
 115
 116 # Python does not provide strsignal() even in the very latest 3.x.
 117 # This is a reasonable fake.
 118 def strsig(n):
 119   # Signal numbers run 0 through NSIG-1; an array with NSIG members
 120   # has exactly that many slots
 121   _sigtbl = [None]*signal.NSIG
 122   for k in dir(signal):
 123     if k.startswith("SIG") and not k.startswith("SIG_") and k != "SIGCLD" and k != "SIGPOLL":
 124       _sigtbl[getattr(signal, k)] = k
 125   # Realtime signals mostly have no names
 126   if hasattr(signal, "SIGRTMIN") and hasattr(signal, "SIGRTMAX"):
 127     for r in range(signal.SIGRTMIN+1, signal.SIGRTMAX+1):
 128       _sigtbl[r] = "SIGRTMIN+" + str(r - signal.SIGRTMIN)
 129   # Fill in any remaining gaps
 130   for i in range(signal.NSIG):
 131     if _sigtbl[i] is None:
 132       _sigtbl[i] = "unrecognized signal, number " + str(i)
 133   if n < 0 or n >= signal.NSIG:
 134     return "out-of-range signal, number "+str(n)
 135   return _sigtbl[n]
 136
 137 def printstatus(status, name = ""):
 138   # 'status' is the exit status
 139   if os.name != 'posix':
 140     # Windows error codes are easier to look up if printed in hexadecimal
 141     if status < 0:
 142       status += 2**32
 143     print "TEST-INFO | %s: exit status %x\n" % (name, status)
 144   elif os.WIFEXITED(status):
 145     print "TEST-INFO | %s: exit %d\n" % (name, os.WEXITSTATUS(status))
 146   elif os.WIFSIGNALED(status):
 147     # The python stdlib doesn't appear to have strsignal(), alas
 148     print "TEST-INFO | {}: killed by {}".format(name,strsig(os.WTERMSIG(status)))
 149   else:
 150     # This is probably a can't-happen condition on Unix, but let's be defensive
 151     print "TEST-INFO | %s: undecodable exit status %04x\n" % (name, status)
 152
 153 def addCommonOptions(parser, defaults={}):
 154   parser.add_option("--xre-path",
 155                     action = "store", type = "string", dest = "xrePath",
 156                     # individual scripts will set a sane default
 157                     default = None,
 158                     help = "absolute path to directory containing XRE (probably xulrunner)")
 159   if 'SYMBOLS_PATH' not in defaults:
 160     defaults['SYMBOLS_PATH'] = None
 161   parser.add_option("--symbols-path",
 162                     action = "store", type = "string", dest = "symbolsPath",
 163                     default = defaults['SYMBOLS_PATH'],
 164                     help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
 165   parser.add_option("--debugger",
 166                     action = "store", dest = "debugger",
 167                     help = "use the given debugger to launch the application")
 168   parser.add_option("--debugger-args",
 169                     action = "store", dest = "debuggerArgs",
 170                     help = "pass the given args to the debugger _before_ "
 171                            "the application on the command line")
 172   parser.add_option("--debugger-interactive",
 173                     action = "store_true", dest = "debuggerInteractive",
 174                     help = "prevents the test harness from redirecting "
 175                         "stdout and stderr for interactive debuggers")
 176
 177 def dumpLeakLog(leakLogFile, filter = False):
 178   """Process the leak log, without parsing it.
 179
 180   Use this function if you want the raw log only.
 181   Use it preferably with the |XPCOM_MEM_LEAK_LOG| environment variable.
 182   """
 183
 184   # Don't warn (nor "info") if the log file is not there.
 185   if not os.path.exists(leakLogFile):
 186     return
 187
 188   with open(leakLogFile, "r") as leaks:
 189     leakReport = leaks.read()
 190
 191   # Only |XPCOM_MEM_LEAK_LOG| reports can be actually filtered out.
 192   # Only check whether an actual leak was reported.
 193   if filter and not "0 TOTAL " in leakReport:
 194     return
 195
 196   # Simply copy the log.
 197   log.info(leakReport.rstrip("\n"))
 198
 199 def processSingleLeakFile(leakLogFileName, processType, leakThreshold):
 200   """Process a single leak log.
 201   """
 202
 203   #                  Per-Inst  Leaked      Total  Rem ...
 204   #   0 TOTAL              17     192  419115886    2 ...
 205   # 833 nsTimerImpl        60     120      24726    2 ...
 206   lineRe = re.compile(r"^\s*\d+\s+(?P<name>\S+)\s+"
 207                       r"(?P<size>-?\d+)\s+(?P<bytesLeaked>-?\d+)\s+"
 208                       r"-?\d+\s+(?P<numLeaked>-?\d+)")
 209
 210   processString = ""
 211   if processType:
 212     # eg 'plugin'
 213     processString = " %s process:" % processType
 214
 215   crashedOnPurpose = False
 216   totalBytesLeaked = None
 217   leakAnalysis = []
 218   leakedObjectNames = []
 219   with open(leakLogFileName, "r") as leaks:
 220     for line in leaks:
 221       if line.find("purposefully crash") > -1:
 222         crashedOnPurpose = True
 223       matches = lineRe.match(line)
 224       if not matches:
 225         # eg: the leak table header row
 226         log.info(line.rstrip())
 227         continue
 228       name = matches.group("name")
 229       size = int(matches.group("size"))
 230       bytesLeaked = int(matches.group("bytesLeaked"))
 231       numLeaked = int(matches.group("numLeaked"))
 232       # Output the raw line from the leak log table if it is the TOTAL row,
 233       # or is for an object row that has been leaked.
 234       if numLeaked != 0 or name == "TOTAL":
 235         log.info(line.rstrip())
 236       # Analyse the leak log, but output later or it will interrupt the leak table
 237       if name == "TOTAL":
 238         totalBytesLeaked = bytesLeaked
 239       if size < 0 or bytesLeaked < 0 or numLeaked < 0:
 240         leakAnalysis.append("TEST-UNEXPECTED-FAIL | leakcheck |%s negative leaks caught!"
 241                             % processString)
 242         continue
 243       if name != "TOTAL" and numLeaked != 0:
 244         leakedObjectNames.append(name)
 245         leakAnalysis.append("TEST-INFO | leakcheck |%s leaked %d %s (%s bytes)"
 246                             % (processString, numLeaked, name, bytesLeaked))
 247   log.info('\n'.join(leakAnalysis))
 248
 249   if totalBytesLeaked is None:
 250     # We didn't see a line with name 'TOTAL'
 251     if crashedOnPurpose:
 252       log.info("TEST-INFO | leakcheck |%s deliberate crash and thus no leak log"
 253                % processString)
 254     else:
 255       # TODO: This should be a TEST-UNEXPECTED-FAIL, but was changed to a warning
 256       # due to too many intermittent failures (see bug 831223).
 257       log.info("WARNING | leakcheck |%s missing output line for total leaks!"
 258                % processString)
 259     return
 260
 261   if totalBytesLeaked == 0:
 262     log.info("TEST-PASS | leakcheck |%s no leaks detected!" % processString)
 263     return
 264
 265   # totalBytesLeaked was seen and is non-zero.
 266   if totalBytesLeaked > leakThreshold:
 267     if processType and processType == "tab":
 268       # For now, ignore tab process leaks. See bug 1051230.
 269       log.info("WARNING | leakcheck | ignoring leaks in tab process")
 270       prefix = "WARNING"
 271     else:
 272       # Fail the run if we're over the threshold (which defaults to 0)
 273       prefix = "TEST-UNEXPECTED-FAIL"
 274   else:
 275     prefix = "WARNING"
 276   # Create a comma delimited string of the first N leaked objects found,
 277   # to aid with bug summary matching in TBPL. Note: The order of the objects
 278   # had no significance (they're sorted alphabetically).
 279   maxSummaryObjects = 5
 280   leakedObjectSummary = ', '.join(leakedObjectNames[:maxSummaryObjects])
 281   if len(leakedObjectNames) > maxSummaryObjects:
 282     leakedObjectSummary += ', ...'
 283   log.info("%s | leakcheck |%s %d bytes leaked (%s)"
 284            % (prefix, processString, totalBytesLeaked, leakedObjectSummary))
 285
 286 def processLeakLog(leakLogFile, leakThreshold = 0):
 287   """Process the leak log, including separate leak logs created
 288   by child processes.
 289
 290   Use this function if you want an additional PASS/FAIL summary.
 291   It must be used with the |XPCOM_MEM_BLOAT_LOG| environment variable.
 292   """
 293
 294   if not os.path.exists(leakLogFile):
 295     log.info("WARNING | leakcheck | refcount logging is off, so leaks can't be detected!")
 296     return
 297
 298   if leakThreshold != 0:
 299     log.info("TEST-INFO | leakcheck | threshold set at %d bytes" % leakThreshold)
 300
 301   (leakLogFileDir, leakFileBase) = os.path.split(leakLogFile)
 302   fileNameRegExp = re.compile(r".*?_([a-z]*)_pid\d*$")
 303   if leakFileBase[-4:] == ".log":
 304     leakFileBase = leakFileBase[:-4]
 305     fileNameRegExp = re.compile(r".*?_([a-z]*)_pid\d*.log$")
 306
 307   for fileName in os.listdir(leakLogFileDir):
 308     if fileName.find(leakFileBase) != -1:
 309       thisFile = os.path.join(leakLogFileDir, fileName)
 310       processType = None
 311       m = fileNameRegExp.search(fileName)
 312       if m:
 313         processType = m.group(1)
 314       processSingleLeakFile(thisFile, processType, leakThreshold)
 315
 316 def replaceBackSlashes(input):
 317   return input.replace('\\', '/')
 318
 319 class KeyValueParseError(Exception):
 320   """error when parsing strings of serialized key-values"""
 321   def __init__(self, msg, errors=()):
 322     self.errors = errors
 323     Exception.__init__(self, msg)
 324
 325 def parseKeyValue(strings, separator='=', context='key, value: '):
 326   """
 327   parse string-serialized key-value pairs in the form of
 328   `key = value`. Returns a list of 2-tuples.
 329   Note that whitespace is not stripped.
 330   """
 331
 332   # syntax check
 333   missing = [string for string in strings if separator not in string]
 334   if missing:
 335     raise KeyValueParseError("Error: syntax error in %s" % (context,
 336                                                             ','.join(missing)),
 337                                                             errors=missing)
 338   return [string.split(separator, 1) for string in strings]
 339
 340 def systemMemory():
 341   """
 342   Returns total system memory in kilobytes.
 343   Works only on unix-like platforms where `free` is in the path.
 344   """
 345   return int(os.popen("free").readlines()[1].split()[1])
 346
 347 def environment(xrePath, env=None, crashreporter=True, debugger=False, dmdPath=None, lsanPath=None):
 348   """populate OS environment variables for mochitest"""
 349
 350   env = os.environ.copy() if env is None else env
 351
 352   assert os.path.isabs(xrePath)
 353
 354   if mozinfo.isMac:
 355     ldLibraryPath = os.path.join(os.path.dirname(xrePath), "MacOS")
 356   else:
 357     ldLibraryPath = xrePath
 358
 359   envVar = None
 360   dmdLibrary = None
 361   preloadEnvVar = None
 362   if 'toolkit' in mozinfo.info and mozinfo.info['toolkit'] == "gonk":
 363     # Skip all of this, it's only valid for the host.
 364     pass
 365   elif mozinfo.isUnix:
 366     envVar = "LD_LIBRARY_PATH"
 367     env['MOZILLA_FIVE_HOME'] = xrePath
 368     dmdLibrary = "libdmd.so"
 369     preloadEnvVar = "LD_PRELOAD"
 370   elif mozinfo.isMac:
 371     envVar = "DYLD_LIBRARY_PATH"
 372     dmdLibrary = "libdmd.dylib"
 373     preloadEnvVar = "DYLD_INSERT_LIBRARIES"
 374   elif mozinfo.isWin:
 375     envVar = "PATH"
 376     dmdLibrary = "dmd.dll"
 377     preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB"
 378   if envVar:
 379     envValue = ((env.get(envVar), str(ldLibraryPath))
 380                 if mozinfo.isWin
 381                 else (ldLibraryPath, dmdPath, env.get(envVar)))
 382     env[envVar] = os.path.pathsep.join([path for path in envValue if path])
 383
 384   if dmdPath and dmdLibrary and preloadEnvVar:
 385     env['DMD'] = '1'
 386     env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary)
 387
 388   # crashreporter
 389   env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
 390   env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
 391   env['NS_TRACE_MALLOC_DISABLE_STACKS'] = '1'
 392
 393   if crashreporter and not debugger:
 394     env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
 395     env['MOZ_CRASHREPORTER'] = '1'
 396   else:
 397     env['MOZ_CRASHREPORTER_DISABLE'] = '1'
 398
 399   # Crash on non-local network connections.
 400   env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'
 401
 402   # Set WebRTC logging in case it is not set yet
 403   env.setdefault('NSPR_LOG_MODULES', 'signaling:5,mtransport:5,datachannel:5')
 404   env.setdefault('R_LOG_LEVEL', '6')
 405   env.setdefault('R_LOG_DESTINATION', 'stderr')
 406   env.setdefault('R_LOG_VERBOSE', '1')
 407
 408   # ASan specific environment stuff
 409   asan = bool(mozinfo.info.get("asan"))
 410   if asan and (mozinfo.isLinux or mozinfo.isMac):
 411     try:
 412       # Symbolizer support
 413       llvmsym = os.path.join(xrePath, "llvm-symbolizer")
 414       if os.path.isfile(llvmsym):
 415         env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 416         log.info("INFO | runtests.py | ASan using symbolizer at %s" % llvmsym)
 417       else:
 418         log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find ASan symbolizer at %s" % llvmsym)
 419
 420       totalMemory = systemMemory()
 421
 422       # Only 4 GB RAM or less available? Use custom ASan options to reduce
 423       # the amount of resources required to do the tests. Standard options
 424       # will otherwise lead to OOM conditions on the current test slaves.
 425       message = "INFO | runtests.py | ASan running in %s configuration"
 426       asanOptions = []
 427       if totalMemory <= 1024 * 1024 * 4:
 428         message = message % 'low-memory'
 429         asanOptions = ['quarantine_size=50331648', 'malloc_context_size=5']
 430       else:
 431         message = message % 'default memory'
 432
 433       if lsanPath:
 434         log.info("LSan enabled.")
 435         asanOptions.append('detect_leaks=1')
 436         lsanOptions = ["exitcode=0"]
 437         suppressionsFile = os.path.join(lsanPath, 'lsan_suppressions.txt')
 438         if os.path.exists(suppressionsFile):
 439           log.info("LSan using suppression file " + suppressionsFile)
 440           lsanOptions.append("suppressions=" + suppressionsFile)
 441         else:
 442           log.info("WARNING | runtests.py | LSan suppressions file does not exist! " + suppressionsFile)
 443         env["LSAN_OPTIONS"] = ':'.join(lsanOptions)
 444         # Run shutdown GCs and CCs to avoid spurious leaks.
 445         env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
 446
 447       if len(asanOptions):
 448         env['ASAN_OPTIONS'] = ':'.join(asanOptions)
 449
 450     except OSError,err:
 451       log.info("Failed determine available memory, disabling ASan low-memory configuration: %s" % err.strerror)
 452     except:
 453       log.info("Failed determine available memory, disabling ASan low-memory configuration")
 454     else:
 455       log.info(message)
 456
 457   return env
 458
 459 def dumpScreen(utilityPath):
 460   """dumps a screenshot of the entire screen to a directory specified by
 461   the MOZ_UPLOAD_DIR environment variable"""
 462
 463   # Need to figure out which OS-dependent tool to use
 464   if mozinfo.isUnix:
 465     utility = [os.path.join(utilityPath, "screentopng")]
 466     utilityname = "screentopng"
 467   elif mozinfo.isMac:
 468     utility = ['/usr/sbin/screencapture', '-C', '-x', '-t', 'png']
 469     utilityname = "screencapture"
 470   elif mozinfo.isWin:
 471     utility = [os.path.join(utilityPath, "screenshot.exe")]
 472     utilityname = "screenshot"
 473
 474   # Get dir where to write the screenshot file
 475   parent_dir = os.environ.get('MOZ_UPLOAD_DIR', None)
 476   if not parent_dir:
 477     log.info('Failed to retrieve MOZ_UPLOAD_DIR env var')
 478     return
 479
 480   # Run the capture
 481   try:
 482     tmpfd, imgfilename = tempfile.mkstemp(prefix='mozilla-test-fail-screenshot_', suffix='.png', dir=parent_dir)
 483     os.close(tmpfd)
 484     returncode = subprocess.call(utility + [imgfilename])
 485     printstatus(returncode, utilityname)
 486   except OSError, err:
 487     log.info("Failed to start %s for screenshot: %s" %
 488              utility[0], err.strerror)
 489     return
 490
 491 class ShutdownLeaks(object):
 492   """
 493   Parses the mochitest run log when running a debug build, assigns all leaked
 494   DOM windows (that are still around after test suite shutdown, despite running
 495   the GC) to the tests that created them and prints leak statistics.
 496   """
 497
 498   def __init__(self, logger):
 499     self.logger = logger
 500     self.tests = []
 501     self.leakedWindows = {}
 502     self.leakedDocShells = set()
 503     self.currentTest = None
 504     self.seenShutdown = False
 505
 506   def log(self, message):
 507     if message['action'] == 'log':
 508         line = message['message']
 509         if line[2:11] == "DOMWINDOW":
 510           self._logWindow(line)
 511         elif line[2:10] == "DOCSHELL":
 512           self._logDocShell(line)
 513         elif line.startswith("TEST-START | Shutdown"):
 514           self.seenShutdown = True
 515     elif message['action'] == 'test_start':
 516       fileName = message['test'].replace("chrome://mochitests/content/browser/", "")
 517       self.currentTest = {"fileName": fileName, "windows": set(), "docShells": set()}
 518     elif message['action'] == 'test_end':
 519       # don't track a test if no windows or docShells leaked
 520       if self.currentTest and (self.currentTest["windows"] or self.currentTest["docShells"]):
 521         self.tests.append(self.currentTest)
 522       self.currentTest = None
 523
 524   def process(self):
 525     if not self.seenShutdown:
 526       self.logger("TEST-UNEXPECTED-FAIL | ShutdownLeaks | process() called before end of test suite")
 527
 528     for test in self._parseLeakingTests():
 529       for url, count in self._zipLeakedWindows(test["leakedWindows"]):
 530         self.logger("TEST-UNEXPECTED-FAIL | %s | leaked %d window(s) until shutdown [url = %s]" % (test["fileName"], count, url))
 531
 532       if test["leakedDocShells"]:
 533         self.logger("TEST-UNEXPECTED-FAIL | %s | leaked %d docShell(s) until shutdown" % (test["fileName"], len(test["leakedDocShells"])))
 534
 535   def _logWindow(self, line):
 536     created = line[:2] == "++"
 537     pid = self._parseValue(line, "pid")
 538     serial = self._parseValue(line, "serial")
 539
 540     # log line has invalid format
 541     if not pid or not serial:
 542       self.logger("TEST-UNEXPECTED-FAIL | ShutdownLeaks | failed to parse line <%s>" % line)
 543       return
 544
 545     key = pid + "." + serial
 546
 547     if self.currentTest:
 548       windows = self.currentTest["windows"]
 549       if created:
 550         windows.add(key)
 551       else:
 552         windows.discard(key)
 553     elif self.seenShutdown and not created:
 554       self.leakedWindows[key] = self._parseValue(line, "url")
 555
 556   def _logDocShell(self, line):
 557     created = line[:2] == "++"
 558     pid = self._parseValue(line, "pid")
 559     id = self._parseValue(line, "id")
 560
 561     # log line has invalid format
 562     if not pid or not id:
 563       self.logger("TEST-UNEXPECTED-FAIL | ShutdownLeaks | failed to parse line <%s>" % line)
 564       return
 565
 566     key = pid + "." + id
 567
 568     if self.currentTest:
 569       docShells = self.currentTest["docShells"]
 570       if created:
 571         docShells.add(key)
 572       else:
 573         docShells.discard(key)
 574     elif self.seenShutdown and not created:
 575       self.leakedDocShells.add(key)
 576
 577   def _parseValue(self, line, name):
 578     match = re.search("\[%s = (.+?)\]" % name, line)
 579     if match:
 580       return match.group(1)
 581     return None
 582
 583   def _parseLeakingTests(self):
 584     leakingTests = []
 585
 586     for test in self.tests:
 587       test["leakedWindows"] = [self.leakedWindows[id] for id in test["windows"] if id in self.leakedWindows]
 588       test["leakedDocShells"] = [id for id in test["docShells"] if id in self.leakedDocShells]
 589       test["leakCount"] = len(test["leakedWindows"]) + len(test["leakedDocShells"])
 590
 591       if test["leakCount"]:
 592         leakingTests.append(test)
 593
 594     return sorted(leakingTests, key=itemgetter("leakCount"), reverse=True)
 595
 596   def _zipLeakedWindows(self, leakedWindows):
 597     counts = []
 598     counted = set()
 599
 600     for url in leakedWindows:
 601       if not url in counted:
 602         counts.append((url, leakedWindows.count(url)))
 603         counted.add(url)
 604
 605     return sorted(counts, key=itemgetter(1), reverse=True)
 606
 607
 608 class LSANLeaks(object):
 609   """
 610   Parses the log when running an LSAN build, looking for interesting stack frames
 611   in allocation stacks, and prints out reports.
 612   """
 613
 614   def __init__(self, logger):
 615     self.logger = logger
 616     self.inReport = False
 617     self.foundFrames = set([])
 618     self.recordMoreFrames = None
 619     self.currStack = None
 620     self.maxNumRecordedFrames = 4
 621
 622     # Don't various allocation-related stack frames, as they do not help much to
 623     # distinguish different leaks.
 624     unescapedSkipList = [
 625       "malloc", "js_malloc", "malloc_", "__interceptor_malloc", "moz_malloc", "moz_xmalloc",
 626       "calloc", "js_calloc", "calloc_", "__interceptor_calloc", "moz_calloc", "moz_xcalloc",
 627       "realloc","js_realloc", "realloc_", "__interceptor_realloc", "moz_realloc", "moz_xrealloc",
 628       "new",
 629       "js::MallocProvider",
 630     ]
 631     self.skipListRegExp = re.compile("^" + "|".join([re.escape(f) for f in unescapedSkipList]) + "$")
 632
 633     self.startRegExp = re.compile("==\d+==ERROR: LeakSanitizer: detected memory leaks")
 634     self.stackFrameRegExp = re.compile("    #\d+ 0x[0-9a-f]+ in ([^(</]+)")
 635     self.sysLibStackFrameRegExp = re.compile("    #\d+ 0x[0-9a-f]+ \(([^+]+)\+0x[0-9a-f]+\)")
 636
 637
 638   def log(self, line):
 639     if re.match(self.startRegExp, line):
 640       self.inReport = True
 641       return
 642
 643     if not self.inReport:
 644       return
 645
 646     if line.startswith("Direct leak"):
 647       self._finishStack()
 648       self.recordMoreFrames = True
 649       self.currStack = []
 650       return
 651
 652     if line.startswith("Indirect leak"):
 653       self._finishStack()
 654       # Only report direct leaks, in the hope that they are less flaky.
 655       self.recordMoreFrames = False
 656       return
 657
 658     if line.startswith("SUMMARY: AddressSanitizer"):
 659       self._finishStack()
 660       self.inReport = False
 661       return
 662
 663     if not self.recordMoreFrames:
 664       return
 665
 666     stackFrame = re.match(self.stackFrameRegExp, line)
 667     if stackFrame:
 668       # Split the frame to remove any return types.
 669       frame = stackFrame.group(1).split()[-1]
 670       if not re.match(self.skipListRegExp, frame):
 671         self._recordFrame(frame)
 672       return
 673
 674     sysLibStackFrame = re.match(self.sysLibStackFrameRegExp, line)
 675     if sysLibStackFrame:
 676       # System library stack frames will never match the skip list,
 677       # so don't bother checking if they do.
 678       self._recordFrame(sysLibStackFrame.group(1))
 679
 680     # If we don't match either of these, just ignore the frame.
 681     # We'll end up with "unknown stack" if everything is ignored.
 682
 683   def process(self):
 684     for f in self.foundFrames:
 685       self.logger("TEST-UNEXPECTED-FAIL | LeakSanitizer | leak at " + f)
 686
 687   def _finishStack(self):
 688     if self.recordMoreFrames and len(self.currStack) == 0:
 689       self.currStack = ["unknown stack"]
 690     if self.currStack:
 691       self.foundFrames.add(", ".join(self.currStack))
 692       self.currStack = None
 693     self.recordMoreFrames = False
 694     self.numRecordedFrames = 0
 695
 696   def _recordFrame(self, frame):
 697     self.currStack.append(frame)
 698     self.numRecordedFrames += 1
 699     if self.numRecordedFrames >= self.maxNumRecordedFrames:
 700       self.recordMoreFrames = False