build/automation.py.in

   1 #
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 from __future__ import with_statement
   7 import logging
   8 import os
   9 import re
  10 import select
  11 import signal
  12 import subprocess
  13 import sys
  14 import tempfile
  15 from datetime import datetime, timedelta
  16
  17 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
  18 sys.path.insert(0, SCRIPT_DIR)
  19
  20 # --------------------------------------------------------------
  21 # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
  22 # These paths refer to relative locations to test.zip, not the OBJDIR or SRCDIR
  23 here = os.path.dirname(os.path.realpath(__file__))
  24 mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase'))
  25
  26 if os.path.isdir(mozbase):
  27     for package in os.listdir(mozbase):
  28         package_path = os.path.join(mozbase, package)
  29         if package_path not in sys.path:
  30             sys.path.append(package_path)
  31
  32 import mozcrash
  33 from mozscreenshot import printstatus, dump_screen
  34
  35
  36 # ---------------------------------------------------------------
  37
  38 _DEFAULT_WEB_SERVER = "127.0.0.1"
  39 _DEFAULT_HTTP_PORT = 8888
  40 _DEFAULT_SSL_PORT = 4443
  41 _DEFAULT_WEBSOCKET_PORT = 9988
  42
  43 #expand _DIST_BIN = __XPC_BIN_PATH__
  44 #expand _IS_WIN32 = len("__WIN32__") != 0
  45 #expand _IS_MAC = __IS_MAC__ != 0
  46 #expand _IS_LINUX = __IS_LINUX__ != 0
  47 #ifdef IS_CYGWIN
  48 #expand _IS_CYGWIN = __IS_CYGWIN__ == 1
  49 #else
  50 _IS_CYGWIN = False
  51 #endif
  52 #expand _BIN_SUFFIX = __BIN_SUFFIX__
  53
  54 #expand _CERTS_SRC_DIR = __CERTS_SRC_DIR__
  55 #expand _IS_TEST_BUILD = __IS_TEST_BUILD__
  56 #expand _IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
  57 #expand _CRASHREPORTER = __CRASHREPORTER__ == 1
  58 #expand _IS_ASAN = __IS_ASAN__ == 1
  59
  60
  61 if _IS_WIN32:
  62   import ctypes, ctypes.wintypes, time, msvcrt
  63 else:
  64   import errno
  65
  66 def resetGlobalLog(log):
  67   while _log.handlers:
  68     _log.removeHandler(_log.handlers[0])
  69   handler = logging.StreamHandler(log)
  70   _log.setLevel(logging.INFO)
  71   _log.addHandler(handler)
  72
  73 # We use the logging system here primarily because it'll handle multiple
  74 # threads, which is needed to process the output of the server and application
  75 # processes simultaneously.
  76 _log = logging.getLogger()
  77 resetGlobalLog(sys.stdout)
  78
  79
  80 #################
  81 # PROFILE SETUP #
  82 #################
  83
  84 class Automation(object):
  85   """
  86   Runs the browser from a script, and provides useful utilities
  87   for setting up the browser environment.
  88   """
  89
  90   DIST_BIN = _DIST_BIN
  91   IS_WIN32 = _IS_WIN32
  92   IS_MAC = _IS_MAC
  93   IS_LINUX = _IS_LINUX
  94   IS_CYGWIN = _IS_CYGWIN
  95   BIN_SUFFIX = _BIN_SUFFIX
  96
  97   UNIXISH = not IS_WIN32 and not IS_MAC
  98
  99   CERTS_SRC_DIR = _CERTS_SRC_DIR
 100   IS_TEST_BUILD = _IS_TEST_BUILD
 101   IS_DEBUG_BUILD = _IS_DEBUG_BUILD
 102   CRASHREPORTER = _CRASHREPORTER
 103   IS_ASAN = _IS_ASAN
 104
 105   # timeout, in seconds
 106   DEFAULT_TIMEOUT = 60.0
 107   DEFAULT_WEB_SERVER = _DEFAULT_WEB_SERVER
 108   DEFAULT_HTTP_PORT = _DEFAULT_HTTP_PORT
 109   DEFAULT_SSL_PORT = _DEFAULT_SSL_PORT
 110   DEFAULT_WEBSOCKET_PORT = _DEFAULT_WEBSOCKET_PORT
 111
 112   def __init__(self):
 113     self.log = _log
 114     self.lastTestSeen = "automation.py"
 115     self.haveDumpedScreen = False
 116
 117   def setServerInfo(self,
 118                     webServer = _DEFAULT_WEB_SERVER,
 119                     httpPort = _DEFAULT_HTTP_PORT,
 120                     sslPort = _DEFAULT_SSL_PORT,
 121                     webSocketPort = _DEFAULT_WEBSOCKET_PORT):
 122     self.webServer = webServer
 123     self.httpPort = httpPort
 124     self.sslPort = sslPort
 125     self.webSocketPort = webSocketPort
 126
 127   @property
 128   def __all__(self):
 129     return [
 130            "UNIXISH",
 131            "IS_WIN32",
 132            "IS_MAC",
 133            "log",
 134            "runApp",
 135            "Process",
 136            "DIST_BIN",
 137            "CERTS_SRC_DIR",
 138            "environment",
 139            "IS_TEST_BUILD",
 140            "IS_DEBUG_BUILD",
 141            "DEFAULT_TIMEOUT",
 142           ]
 143
 144   class Process(subprocess.Popen):
 145     """
 146     Represents our view of a subprocess.
 147     It adds a kill() method which allows it to be stopped explicitly.
 148     """
 149
 150     def __init__(self,
 151                  args,
 152                  bufsize=0,
 153                  executable=None,
 154                  stdin=None,
 155                  stdout=None,
 156                  stderr=None,
 157                  preexec_fn=None,
 158                  close_fds=False,
 159                  shell=False,
 160                  cwd=None,
 161                  env=None,
 162                  universal_newlines=False,
 163                  startupinfo=None,
 164                  creationflags=0):
 165       _log.info("INFO | automation.py | Launching: %s", subprocess.list2cmdline(args))
 166       subprocess.Popen.__init__(self, args, bufsize, executable,
 167                                 stdin, stdout, stderr,
 168                                 preexec_fn, close_fds,
 169                                 shell, cwd, env,
 170                                 universal_newlines, startupinfo, creationflags)
 171       self.log = _log
 172
 173     def kill(self):
 174       if Automation().IS_WIN32:
 175         import platform
 176         pid = "%i" % self.pid
 177         subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()
 178       else:
 179         os.kill(self.pid, signal.SIGKILL)
 180
 181   def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, lsanPath=None, ubsanPath=None):
 182     if xrePath == None:
 183       xrePath = self.DIST_BIN
 184     if env == None:
 185       env = dict(os.environ)
 186
 187     ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath))
 188     if self.UNIXISH or self.IS_MAC:
 189       envVar = "LD_LIBRARY_PATH"
 190       if self.IS_MAC:
 191         envVar = "DYLD_LIBRARY_PATH"
 192       if envVar in env:
 193         ldLibraryPath = ldLibraryPath + ":" + env[envVar]
 194       env[envVar] = ldLibraryPath
 195     elif self.IS_WIN32:
 196       env["PATH"] = env["PATH"] + ";" + str(ldLibraryPath)
 197
 198     if crashreporter and not debugger:
 199       env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
 200       env['MOZ_CRASHREPORTER'] = '1'
 201     else:
 202       env['MOZ_CRASHREPORTER_DISABLE'] = '1'
 203
 204     # Crash on non-local network connections by default.
 205     # MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
 206     # enable non-local connections for the purposes of local testing.  Don't
 207     # override the user's choice here.  See bug 1049688.
 208     env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1')
 209
 210     env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
 211     env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
 212
 213     # Set WebRTC logging in case it is not set yet
 214     env.setdefault('MOZ_LOG', 'signaling:3,mtransport:4,DataChannel:4,jsep:4,MediaPipelineFactory:4')
 215     env.setdefault('R_LOG_LEVEL', '6')
 216     env.setdefault('R_LOG_DESTINATION', 'stderr')
 217     env.setdefault('R_LOG_VERBOSE', '1')
 218
 219     # ASan specific environment stuff
 220     if self.IS_ASAN and (self.IS_LINUX or self.IS_MAC):
 221       # Symbolizer support
 222       llvmsym = os.path.join(xrePath, "llvm-symbolizer")
 223       if os.path.isfile(llvmsym):
 224         env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 225         self.log.info("INFO | automation.py | ASan using symbolizer at %s", llvmsym)
 226       else:
 227         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Failed to find ASan symbolizer at %s", llvmsym)
 228
 229       try:
 230         totalMemory = int(os.popen("free").readlines()[1].split()[1])
 231
 232         # Only 4 GB RAM or less available? Use custom ASan options to reduce
 233         # the amount of resources required to do the tests. Standard options
 234         # will otherwise lead to OOM conditions on the current test slaves.
 235         if totalMemory <= 1024 * 1024 * 4:
 236           self.log.info("INFO | automation.py | ASan running in low-memory configuration")
 237           env["ASAN_OPTIONS"] = "quarantine_size=50331648:malloc_context_size=5"
 238         else:
 239           self.log.info("INFO | automation.py | ASan running in default memory configuration")
 240       except OSError,err:
 241         self.log.info("Failed determine available memory, disabling ASan low-memory configuration: %s", err.strerror)
 242       except:
 243         self.log.info("Failed determine available memory, disabling ASan low-memory configuration")
 244
 245     return env
 246
 247   def killPid(self, pid):
 248     try:
 249       os.kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM))
 250     except WindowsError:
 251       self.log.info("Failed to kill process %d." % pid)
 252
 253   if IS_WIN32:
 254     PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
 255     GetLastError = ctypes.windll.kernel32.GetLastError
 256
 257     def readWithTimeout(self, f, timeout):
 258       """
 259       Try to read a line of output from the file object |f|. |f| must be a
 260       pipe, like the |stdout| member of a subprocess.Popen object created
 261       with stdout=PIPE. Returns a tuple (line, did_timeout), where |did_timeout|
 262       is True if the read timed out, and False otherwise. If no output is
 263       received within |timeout| seconds, returns a blank line.
 264       """
 265
 266       if timeout is None:
 267         timeout = 0
 268
 269       x = msvcrt.get_osfhandle(f.fileno())
 270       l = ctypes.c_long()
 271       done = time.time() + timeout
 272
 273       buffer = ""
 274       while timeout == 0 or time.time() < done:
 275         if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0:
 276           err = self.GetLastError()
 277           if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE
 278             return ('', False)
 279           else:
 280             self.log.error("readWithTimeout got error: %d", err)
 281         # read a character at a time, checking for eol. Return once we get there.
 282         index = 0
 283         while index < l.value:
 284           char = f.read(1)
 285           buffer += char
 286           if char == '\n':
 287             return (buffer, False)
 288           index = index + 1
 289         time.sleep(0.01)
 290       return (buffer, True)
 291
 292     def isPidAlive(self, pid):
 293       STILL_ACTIVE = 259
 294       PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
 295       pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)
 296       if not pHandle:
 297         return False
 298       pExitCode = ctypes.wintypes.DWORD()
 299       ctypes.windll.kernel32.GetExitCodeProcess(pHandle, ctypes.byref(pExitCode))
 300       ctypes.windll.kernel32.CloseHandle(pHandle)
 301       return pExitCode.value == STILL_ACTIVE
 302
 303   else:
 304
 305     def readWithTimeout(self, f, timeout):
 306       """Try to read a line of output from the file object |f|. If no output
 307       is received within |timeout| seconds, return a blank line.
 308       Returns a tuple (line, did_timeout), where |did_timeout| is True
 309       if the read timed out, and False otherwise."""
 310       (r, w, e) = select.select([f], [], [], timeout)
 311       if len(r) == 0:
 312         return ('', True)
 313       return (f.readline(), False)
 314
 315     def isPidAlive(self, pid):
 316       try:
 317         # kill(pid, 0) checks for a valid PID without actually sending a signal
 318         # The method throws OSError if the PID is invalid, which we catch below.
 319         os.kill(pid, 0)
 320
 321         # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if
 322         # the process terminates before we get to this point.
 323         wpid, wstatus = os.waitpid(pid, os.WNOHANG)
 324         return wpid == 0
 325       except OSError, err:
 326         # Catch the errors we might expect from os.kill/os.waitpid,
 327         # and re-raise any others
 328         if err.errno == errno.ESRCH or err.errno == errno.ECHILD:
 329           return False
 330         raise
 331
 332   def dumpScreen(self, utilityPath):
 333     if self.haveDumpedScreen:
 334       self.log.info("Not taking screenshot here: see the one that was previously logged")
 335       return
 336
 337     self.haveDumpedScreen = True;
 338     dump_screen(utilityPath, self.log)
 339
 340
 341   def killAndGetStack(self, processPID, utilityPath, debuggerInfo):
 342     """Kill the process, preferrably in a way that gets us a stack trace.
 343        Also attempts to obtain a screenshot before killing the process."""
 344     if not debuggerInfo:
 345       self.dumpScreen(utilityPath)
 346     self.killAndGetStackNoScreenshot(processPID, utilityPath, debuggerInfo)
 347
 348   def killAndGetStackNoScreenshot(self, processPID, utilityPath, debuggerInfo):
 349     """Kill the process, preferrably in a way that gets us a stack trace."""
 350     if self.CRASHREPORTER and not debuggerInfo:
 351       if not self.IS_WIN32:
 352         # ABRT will get picked up by Breakpad's signal handler
 353         os.kill(processPID, signal.SIGABRT)
 354         return
 355       else:
 356         # We should have a "crashinject" program in our utility path
 357         crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe"))
 358         if os.path.exists(crashinject):
 359           status = subprocess.Popen([crashinject, str(processPID)]).wait()
 360           printstatus("crashinject", status)
 361           if status == 0:
 362             return
 363     self.log.info("Can't trigger Breakpad, just killing process")
 364     self.killPid(processPID)
 365
 366   def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath, outputHandler=None):
 367     """ Look for timeout or crashes and return the status after the process terminates """
 368     stackFixerFunction = None
 369     didTimeout = False
 370     hitMaxTime = False
 371     if proc.stdout is None:
 372       self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection")
 373     else:
 374       logsource = proc.stdout
 375
 376       if self.IS_DEBUG_BUILD and symbolsPath and os.path.exists(symbolsPath):
 377         # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad symbol files)
 378         # This method is preferred for Tinderbox builds, since native symbols may have been stripped.
 379         sys.path.insert(0, utilityPath)
 380         import fix_stack_using_bpsyms as stackFixerModule
 381         stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line, symbolsPath)
 382         del sys.path[0]
 383       elif self.IS_DEBUG_BUILD and self.IS_MAC:
 384         # Run each line through a function in fix_macosx_stack.py (uses atos)
 385         sys.path.insert(0, utilityPath)
 386         import fix_macosx_stack as stackFixerModule
 387         stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line)
 388         del sys.path[0]
 389       elif self.IS_DEBUG_BUILD and self.IS_LINUX:
 390         # Run each line through a function in fix_linux_stack.py (uses addr2line)
 391         # This method is preferred for developer machines, so we don't have to run "make buildsymbols".
 392         sys.path.insert(0, utilityPath)
 393         import fix_linux_stack as stackFixerModule
 394         stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line)
 395         del sys.path[0]
 396
 397       # With metro browser runs this script launches the metro test harness which launches the browser.
 398       # The metro test harness hands back the real browser process id via log output which we need to
 399       # pick up on and parse out. This variable tracks the real browser process id if we find it.
 400       browserProcessId = -1
 401
 402       (line, didTimeout) = self.readWithTimeout(logsource, timeout)
 403       while line != "" and not didTimeout:
 404         if stackFixerFunction:
 405           line = stackFixerFunction(line)
 406
 407         if outputHandler is None:
 408             self.log.info(line.rstrip().decode("UTF-8", "ignore"))
 409         else:
 410             outputHandler(line)
 411
 412         if "TEST-START" in line and "|" in line:
 413           self.lastTestSeen = line.split("|")[1].strip()
 414         if not debuggerInfo and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:
 415           self.dumpScreen(utilityPath)
 416
 417         (line, didTimeout) = self.readWithTimeout(logsource, timeout)
 418
 419         if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime):
 420           # Kill the application.
 421           hitMaxTime = True
 422           self.log.info("TEST-UNEXPECTED-FAIL | %s | application ran for longer than allowed maximum time of %d seconds", self.lastTestSeen, int(maxTime))
 423           self.log.error("Force-terminating active process(es).");
 424           self.killAndGetStack(proc.pid, utilityPath, debuggerInfo)
 425       if didTimeout:
 426         if line:
 427           self.log.info(line.rstrip().decode("UTF-8", "ignore"))
 428         self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout))
 429         self.log.error("Force-terminating active process(es).");
 430         if browserProcessId == -1:
 431           browserProcessId = proc.pid
 432         self.killAndGetStack(browserProcessId, utilityPath, debuggerInfo)
 433
 434     status = proc.wait()
 435     printstatus("Main app process", status)
 436     if status == 0:
 437       self.lastTestSeen = "Main app process exited normally"
 438     if status != 0 and not didTimeout and not hitMaxTime:
 439       self.log.info("TEST-UNEXPECTED-FAIL | %s | Exited with code %d during test run", self.lastTestSeen, status)
 440     return status
 441
 442   def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
 443     """ build the application command line """
 444
 445     cmd = os.path.abspath(app)
 446     if self.IS_MAC and os.path.exists(cmd + "-bin"):
 447       # Prefer 'app-bin' in case 'app' is a shell script.
 448       # We can remove this hack once bug 673899 etc are fixed.
 449       cmd += "-bin"
 450
 451     args = []
 452
 453     if debuggerInfo:
 454       args.extend(debuggerInfo.args)
 455       args.append(cmd)
 456       cmd = os.path.abspath(debuggerInfo.path)
 457
 458     if self.IS_MAC:
 459       args.append("-foreground")
 460
 461     if self.IS_CYGWIN:
 462       profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
 463     else:
 464       profileDirectory = profileDir + "/"
 465
 466     args.extend(("-no-remote", "-profile", profileDirectory))
 467     if testURL is not None:
 468       args.append((testURL))
 469     args.extend(extraArgs)
 470     return cmd, args
 471
 472   def checkForZombies(self, processLog, utilityPath, debuggerInfo):
 473     """ Look for hung processes """
 474     if not os.path.exists(processLog):
 475       self.log.info('Automation Error: PID log not found: %s', processLog)
 476       # Whilst no hung process was found, the run should still display as a failure
 477       return True
 478
 479     foundZombie = False
 480     self.log.info('INFO | zombiecheck | Reading PID log: %s', processLog)
 481     processList = []
 482     pidRE = re.compile(r'launched child process (\d+)$')
 483     processLogFD = open(processLog)
 484     for line in processLogFD:
 485       self.log.info(line.rstrip())
 486       m = pidRE.search(line)
 487       if m:
 488         processList.append(int(m.group(1)))
 489     processLogFD.close()
 490
 491     for processPID in processList:
 492       self.log.info("INFO | zombiecheck | Checking for orphan process with PID: %d", processPID)
 493       if self.isPidAlive(processPID):
 494         foundZombie = True
 495         self.log.info("TEST-UNEXPECTED-FAIL | zombiecheck | child process %d still alive after shutdown", processPID)
 496         self.killAndGetStack(processPID, utilityPath, debuggerInfo)
 497     return foundZombie
 498
 499   def checkForCrashes(self, minidumpDir, symbolsPath):
 500     return mozcrash.check_for_crashes(minidumpDir, symbolsPath, test_name=self.lastTestSeen)
 501
 502   def runApp(self, testURL, env, app, profileDir, extraArgs, utilityPath = None,
 503              xrePath = None, certPath = None,
 504              debuggerInfo = None, symbolsPath = None,
 505              timeout = -1, maxTime = None, onLaunch = None,
 506              detectShutdownLeaks = False, screenshotOnFail=False, testPath=None, bisectChunk=None,
 507              valgrindPath=None, valgrindArgs=None, valgrindSuppFiles=None, outputHandler=None, e10s=True):
 508     """
 509     Run the app, log the duration it took to execute, return the status code.
 510     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
 511     """
 512
 513     if utilityPath == None:
 514       utilityPath = self.DIST_BIN
 515     if xrePath == None:
 516       xrePath = self.DIST_BIN
 517     if certPath == None:
 518       certPath = self.CERTS_SRC_DIR
 519     if timeout == -1:
 520       timeout = self.DEFAULT_TIMEOUT
 521
 522     # copy env so we don't munge the caller's environment
 523     env = dict(env);
 524     tmpfd, processLog = tempfile.mkstemp(suffix='pidlog')
 525     os.close(tmpfd)
 526     env["MOZ_PROCESS_LOG"] = processLog
 527
 528
 529     cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs)
 530     startTime = datetime.now()
 531
 532     if debuggerInfo and debuggerInfo.interactive:
 533       # If an interactive debugger is attached, don't redirect output,
 534       # don't use timeouts, and don't capture ctrl-c.
 535       timeout = None
 536       maxTime = None
 537       outputPipe = None
 538       signal.signal(signal.SIGINT, lambda sigid, frame: None)
 539     else:
 540       outputPipe = subprocess.PIPE
 541
 542     self.lastTestSeen = "automation.py"
 543     proc = self.Process([cmd] + args,
 544                  env = self.environment(env, xrePath = xrePath,
 545                                    crashreporter = not debuggerInfo),
 546                  stdout = outputPipe,
 547                  stderr = subprocess.STDOUT)
 548     self.log.info("INFO | automation.py | Application pid: %d", proc.pid)
 549
 550     if onLaunch is not None:
 551       # Allow callers to specify an onLaunch callback to be fired after the
 552       # app is launched.
 553       onLaunch()
 554
 555     status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath,
 556                                 outputHandler=outputHandler)
 557     self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime))
 558
 559     # Do a final check for zombie child processes.
 560     zombieProcesses = self.checkForZombies(processLog, utilityPath, debuggerInfo)
 561
 562     crashed = self.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath)
 563
 564     if crashed or zombieProcesses:
 565       status = 1
 566
 567     if os.path.exists(processLog):
 568       os.unlink(processLog)
 569
 570     return status, self.lastTestSeen
 571
 572   def elf_arm(self, filename):
 573     data = open(filename, 'rb').read(20)
 574     return data[:4] == "\x7fELF" and ord(data[18]) == 40 # EM_ARM
 575