build/automation.py.in

   1 #
   2 # This Source Code Form is subject to the terms of the Mozilla Public
   3 # License, v. 2.0. If a copy of the MPL was not distributed with this
   4 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   5
   6 from __future__ import with_statement
   7 import codecs
   8 import itertools
   9 import json
  10 import logging
  11 import os
  12 import re
  13 import select
  14 import shutil
  15 import signal
  16 import subprocess
  17 import sys
  18 import threading
  19 import tempfile
  20 import sqlite3
  21 from datetime import datetime, timedelta
  22 from string import Template
  23
  24 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
  25 sys.path.insert(0, SCRIPT_DIR)
  26 import automationutils
  27
  28 # --------------------------------------------------------------
  29 # TODO: this is a hack for mozbase without virtualenv, remove with bug 849900
  30 # These paths refer to relative locations to test.zip, not the OBJDIR or SRCDIR
  31 here = os.path.dirname(os.path.realpath(__file__))
  32 mozbase = os.path.realpath(os.path.join(os.path.dirname(here), 'mozbase'))
  33
  34 if os.path.isdir(mozbase):
  35     for package in os.listdir(mozbase):
  36         package_path = os.path.join(mozbase, package)
  37         if package_path not in sys.path:
  38             sys.path.append(package_path)
  39
  40 import mozcrash
  41 from mozprofile import Profile, Preferences
  42 from mozprofile.permissions import ServerLocations
  43
  44 # ---------------------------------------------------------------
  45
  46 _DEFAULT_PREFERENCE_FILE = os.path.join(SCRIPT_DIR, 'prefs_general.js')
  47 _DEFAULT_APPS_FILE = os.path.join(SCRIPT_DIR, 'webapps_mochitest.json')
  48
  49 _DEFAULT_WEB_SERVER = "127.0.0.1"
  50 _DEFAULT_HTTP_PORT = 8888
  51 _DEFAULT_SSL_PORT = 4443
  52 _DEFAULT_WEBSOCKET_PORT = 9988
  53
  54 # from nsIPrincipal.idl
  55 _APP_STATUS_NOT_INSTALLED = 0
  56 _APP_STATUS_INSTALLED     = 1
  57 _APP_STATUS_PRIVILEGED    = 2
  58 _APP_STATUS_CERTIFIED     = 3
  59
  60 #expand _DIST_BIN = __XPC_BIN_PATH__
  61 #expand _IS_WIN32 = len("__WIN32__") != 0
  62 #expand _IS_MAC = __IS_MAC__ != 0
  63 #expand _IS_LINUX = __IS_LINUX__ != 0
  64 #ifdef IS_CYGWIN
  65 #expand _IS_CYGWIN = __IS_CYGWIN__ == 1
  66 #else
  67 _IS_CYGWIN = False
  68 #endif
  69 #expand _IS_CAMINO = __IS_CAMINO__ != 0
  70 #expand _BIN_SUFFIX = __BIN_SUFFIX__
  71 #expand _PERL = __PERL__
  72
  73 #expand _DEFAULT_APP = "./" + __BROWSER_PATH__
  74 #expand _CERTS_SRC_DIR = __CERTS_SRC_DIR__
  75 #expand _IS_TEST_BUILD = __IS_TEST_BUILD__
  76 #expand _IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
  77 #expand _CRASHREPORTER = __CRASHREPORTER__ == 1
  78 #expand _IS_ASAN = __IS_ASAN__ == 1
  79
  80
  81 if _IS_WIN32:
  82   import ctypes, ctypes.wintypes, time, msvcrt
  83 else:
  84   import errno
  85
  86
  87 def getGlobalLog():
  88   return _log
  89
  90 def resetGlobalLog(log):
  91   while _log.handlers:
  92     _log.removeHandler(_log.handlers[0])
  93   handler = logging.StreamHandler(log)
  94   _log.setLevel(logging.INFO)
  95   _log.addHandler(handler)
  96
  97 # We use the logging system here primarily because it'll handle multiple
  98 # threads, which is needed to process the output of the server and application
  99 # processes simultaneously.
 100 _log = logging.getLogger()
 101 resetGlobalLog(sys.stdout)
 102
 103
 104 #################
 105 # PROFILE SETUP #
 106 #################
 107
 108 class SyntaxError(Exception):
 109   "Signifies a syntax error on a particular line in server-locations.txt."
 110
 111   def __init__(self, lineno, msg = None):
 112     self.lineno = lineno
 113     self.msg = msg
 114
 115   def __str__(self):
 116     s = "Syntax error on line " + str(self.lineno)
 117     if self.msg:
 118       s += ": %s." % self.msg
 119     else:
 120       s += "."
 121     return s
 122
 123
 124 class Location:
 125   "Represents a location line in server-locations.txt."
 126
 127   def __init__(self, scheme, host, port, options):
 128     self.scheme = scheme
 129     self.host = host
 130     self.port = port
 131     self.options = options
 132
 133 class Automation(object):
 134   """
 135   Runs the browser from a script, and provides useful utilities
 136   for setting up the browser environment.
 137   """
 138
 139   DIST_BIN = _DIST_BIN
 140   IS_WIN32 = _IS_WIN32
 141   IS_MAC = _IS_MAC
 142   IS_LINUX = _IS_LINUX
 143   IS_CYGWIN = _IS_CYGWIN
 144   IS_CAMINO = _IS_CAMINO
 145   BIN_SUFFIX = _BIN_SUFFIX
 146   PERL = _PERL
 147
 148   UNIXISH = not IS_WIN32 and not IS_MAC
 149
 150   DEFAULT_APP = _DEFAULT_APP
 151   CERTS_SRC_DIR = _CERTS_SRC_DIR
 152   IS_TEST_BUILD = _IS_TEST_BUILD
 153   IS_DEBUG_BUILD = _IS_DEBUG_BUILD
 154   CRASHREPORTER = _CRASHREPORTER
 155   IS_ASAN = _IS_ASAN
 156
 157   # timeout, in seconds
 158   DEFAULT_TIMEOUT = 60.0
 159   DEFAULT_WEB_SERVER = _DEFAULT_WEB_SERVER
 160   DEFAULT_HTTP_PORT = _DEFAULT_HTTP_PORT
 161   DEFAULT_SSL_PORT = _DEFAULT_SSL_PORT
 162   DEFAULT_WEBSOCKET_PORT = _DEFAULT_WEBSOCKET_PORT
 163
 164   def __init__(self):
 165     self.log = _log
 166     self.lastTestSeen = "automation.py"
 167     self.haveDumpedScreen = False
 168
 169   def setServerInfo(self,
 170                     webServer = _DEFAULT_WEB_SERVER,
 171                     httpPort = _DEFAULT_HTTP_PORT,
 172                     sslPort = _DEFAULT_SSL_PORT,
 173                     webSocketPort = _DEFAULT_WEBSOCKET_PORT):
 174     self.webServer = webServer
 175     self.httpPort = httpPort
 176     self.sslPort = sslPort
 177     self.webSocketPort = webSocketPort
 178
 179   @property
 180   def __all__(self):
 181     return [
 182            "UNIXISH",
 183            "IS_WIN32",
 184            "IS_MAC",
 185            "log",
 186            "runApp",
 187            "Process",
 188            "addCommonOptions",
 189            "initializeProfile",
 190            "DIST_BIN",
 191            "DEFAULT_APP",
 192            "CERTS_SRC_DIR",
 193            "environment",
 194            "IS_TEST_BUILD",
 195            "IS_DEBUG_BUILD",
 196            "DEFAULT_TIMEOUT",
 197           ]
 198
 199   class Process(subprocess.Popen):
 200     """
 201     Represents our view of a subprocess.
 202     It adds a kill() method which allows it to be stopped explicitly.
 203     """
 204
 205     def __init__(self,
 206                  args,
 207                  bufsize=0,
 208                  executable=None,
 209                  stdin=None,
 210                  stdout=None,
 211                  stderr=None,
 212                  preexec_fn=None,
 213                  close_fds=False,
 214                  shell=False,
 215                  cwd=None,
 216                  env=None,
 217                  universal_newlines=False,
 218                  startupinfo=None,
 219                  creationflags=0):
 220       _log.info("INFO | automation.py | Launching: %s", subprocess.list2cmdline(args))
 221       subprocess.Popen.__init__(self, args, bufsize, executable,
 222                                 stdin, stdout, stderr,
 223                                 preexec_fn, close_fds,
 224                                 shell, cwd, env,
 225                                 universal_newlines, startupinfo, creationflags)
 226       self.log = _log
 227
 228     def kill(self):
 229       if Automation().IS_WIN32:
 230         import platform
 231         pid = "%i" % self.pid
 232         if platform.release() == "2000":
 233           # Windows 2000 needs 'kill.exe' from the
 234           #'Windows 2000 Resource Kit tools'. (See bug 475455.)
 235           try:
 236             subprocess.Popen(["kill", "-f", pid]).wait()
 237           except:
 238             self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid)
 239         else:
 240           # Windows XP and later.
 241           subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()
 242       else:
 243         os.kill(self.pid, signal.SIGKILL)
 244
 245   def readLocations(self, locationsPath = "server-locations.txt"):
 246     """
 247     Reads the locations at which the Mochitest HTTP server is available from
 248     server-locations.txt.
 249     """
 250
 251     locationFile = codecs.open(locationsPath, "r", "UTF-8")
 252
 253     # Perhaps more detail than necessary, but it's the easiest way to make sure
 254     # we get exactly the format we want.  See server-locations.txt for the exact
 255     # format guaranteed here.
 256     lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"
 257                       r"://"
 258                       r"(?P<host>"
 259                         r"\d+\.\d+\.\d+\.\d+"
 260                         r"|"
 261                         r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*"
 262                         r"[a-z](?:[-a-z0-9]*[a-z0-9])?"
 263                       r")"
 264                       r":"
 265                       r"(?P<port>\d+)"
 266                       r"(?:"
 267                       r"\s+"
 268                       r"(?P<options>\S+(?:,\S+)*)"
 269                       r")?$")
 270     locations = []
 271     lineno = 0
 272     seenPrimary = False
 273     for line in locationFile:
 274       lineno += 1
 275       if line.startswith("#") or line == "\n":
 276         continue
 277
 278       match = lineRe.match(line)
 279       if not match:
 280         raise SyntaxError(lineno)
 281
 282       options = match.group("options")
 283       if options:
 284         options = options.split(",")
 285         if "primary" in options:
 286           if seenPrimary:
 287             raise SyntaxError(lineno, "multiple primary locations")
 288           seenPrimary = True
 289       else:
 290         options = []
 291
 292       locations.append(Location(match.group("scheme"), match.group("host"),
 293                                 match.group("port"), options))
 294
 295     if not seenPrimary:
 296       raise SyntaxError(lineno + 1, "missing primary location")
 297
 298     return locations
 299
 300   def setupPermissionsDatabase(self, profileDir, permissions):
 301     # Included for reftest compatibility;
 302     # see https://bugzilla.mozilla.org/show_bug.cgi?id=688667
 303
 304     # Open database and create table
 305     permDB = sqlite3.connect(os.path.join(profileDir, "permissions.sqlite"))
 306     cursor = permDB.cursor();
 307
 308     cursor.execute("PRAGMA user_version=3");
 309
 310     # SQL copied from nsPermissionManager.cpp
 311     cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (
 312       id INTEGER PRIMARY KEY,
 313       host TEXT,
 314       type TEXT,
 315       permission INTEGER,
 316       expireType INTEGER,
 317       expireTime INTEGER,
 318       appId INTEGER,
 319       isInBrowserElement INTEGER)""")
 320
 321     # Insert desired permissions
 322     for perm in permissions.keys():
 323       for host,allow in permissions[perm]:
 324         cursor.execute("INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0, 0, 0)",
 325                        (host, perm, 1 if allow else 2))
 326
 327     # Commit and close
 328     permDB.commit()
 329     cursor.close()
 330
 331   def initializeProfile(self, profileDir,
 332                               extraPrefs=None,
 333                               useServerLocations=False,
 334                               prefsPath=_DEFAULT_PREFERENCE_FILE,
 335                               appsPath=_DEFAULT_APPS_FILE,
 336                               addons=None):
 337     " Sets up the standard testing profile."
 338
 339     extraPrefs = extraPrefs or []
 340
 341     # create the profile
 342     prefs = {}
 343     locations = None
 344     if useServerLocations:
 345         locations = ServerLocations()
 346         locations.read(os.path.abspath('server-locations.txt'), True)
 347     else:
 348       prefs['network.proxy.type'] = 0
 349
 350     prefs.update(Preferences.read_prefs(prefsPath))
 351
 352     for v in extraPrefs:
 353       thispref = v.split("=", 1)
 354       if len(thispref) < 2:
 355         print "Error: syntax error in --setpref=" + v
 356         sys.exit(1)
 357       prefs[thispref[0]] = thispref[1]
 358
 359
 360     interpolation = {"server": "%s:%s" % (self.webServer, self.httpPort)}
 361     prefs = json.loads(json.dumps(prefs) % interpolation)
 362     for pref in prefs:
 363         prefs[pref] = Preferences.cast(prefs[pref])
 364
 365     # load apps
 366     apps = None
 367     if appsPath and os.path.exists(appsPath):
 368         with open(appsPath, 'r') as apps_file:
 369             apps = json.load(apps_file)
 370
 371     proxy = {'remote': str(self.webServer),
 372              'http': str(self.httpPort),
 373              'https': str(self.sslPort),
 374     # use SSL port for legacy compatibility; see
 375     # - https://bugzilla.mozilla.org/show_bug.cgi?id=688667#c66
 376     # - https://bugzilla.mozilla.org/show_bug.cgi?id=899221
 377     #             'ws': str(self.webSocketPort)
 378              'ws': str(self.sslPort)
 379              }
 380
 381     # return profile object
 382     profile = Profile(profile=profileDir,
 383                       addons=addons,
 384                       locations=locations,
 385                       preferences=prefs,
 386                       restore=False,
 387                       apps=apps,
 388                       proxy=proxy)
 389     return profile
 390
 391   def addCommonOptions(self, parser):
 392     "Adds command-line options which are common to mochitest and reftest."
 393
 394     parser.add_option("--setpref",
 395                       action = "append", type = "string",
 396                       default = [],
 397                       dest = "extraPrefs", metavar = "PREF=VALUE",
 398                       help = "defines an extra user preference")
 399
 400   def fillCertificateDB(self, profileDir, certPath, utilityPath, xrePath):
 401     pwfilePath = os.path.join(profileDir, ".crtdbpw")
 402     pwfile = open(pwfilePath, "w")
 403     pwfile.write("\n")
 404     pwfile.close()
 405
 406     # Create head of the ssltunnel configuration file
 407     sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")
 408     sslTunnelConfig = open(sslTunnelConfigPath, "w")
 409
 410     sslTunnelConfig.write("httpproxy:1\n")
 411     sslTunnelConfig.write("certdbdir:%s\n" % certPath)
 412     sslTunnelConfig.write("forward:127.0.0.1:%s\n" % self.httpPort)
 413     sslTunnelConfig.write("websocketserver:%s:%s\n" % (self.webServer, self.webSocketPort))
 414     sslTunnelConfig.write("listen:*:%s:pgo server certificate\n" % self.sslPort)
 415
 416     # Configure automatic certificate and bind custom certificates, client authentication
 417     locations = self.readLocations()
 418     locations.pop(0)
 419     for loc in locations:
 420       if loc.scheme == "https" and "nocert" not in loc.options:
 421         customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")
 422         clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")
 423         redirRE      = re.compile("^redir=(?P<redirhost>[0-9a-zA-Z_ .]+)")
 424         for option in loc.options:
 425           match = customCertRE.match(option)
 426           if match:
 427             customcert = match.group("nickname");
 428             sslTunnelConfig.write("listen:%s:%s:%s:%s\n" %
 429                       (loc.host, loc.port, self.sslPort, customcert))
 430
 431           match = clientAuthRE.match(option)
 432           if match:
 433             clientauth = match.group("clientauth");
 434             sslTunnelConfig.write("clientauth:%s:%s:%s:%s\n" %
 435                       (loc.host, loc.port, self.sslPort, clientauth))
 436
 437           match = redirRE.match(option)
 438           if match:
 439             redirhost = match.group("redirhost")
 440             sslTunnelConfig.write("redirhost:%s:%s:%s:%s\n" %
 441                       (loc.host, loc.port, self.sslPort, redirhost))
 442
 443     sslTunnelConfig.close()
 444
 445     # Pre-create the certification database for the profile
 446     env = self.environment(xrePath = xrePath)
 447     certutil = os.path.join(utilityPath, "certutil" + self.BIN_SUFFIX)
 448     pk12util = os.path.join(utilityPath, "pk12util" + self.BIN_SUFFIX)
 449
 450     status = self.Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait()
 451     automationutils.printstatus(status, "certutil")
 452     if status != 0:
 453       return status
 454
 455     # Walk the cert directory and add custom CAs and client certs
 456     files = os.listdir(certPath)
 457     for item in files:
 458       root, ext = os.path.splitext(item)
 459       if ext == ".ca":
 460         trustBits = "CT,,"
 461         if root.endswith("-object"):
 462           trustBits = "CT,,CT"
 463         status = self.Process([certutil, "-A", "-i", os.path.join(certPath, item),
 464                     "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits],
 465                     env = env).wait()
 466         automationutils.printstatus(status, "certutil")
 467       if ext == ".client":
 468         status = self.Process([pk12util, "-i", os.path.join(certPath, item), "-w",
 469                     pwfilePath, "-d", profileDir],
 470                     env = env).wait()
 471         automationutils.printstatus(status, "pk12util")
 472
 473     os.unlink(pwfilePath)
 474     return 0
 475
 476   def environment(self, env=None, xrePath=None, crashreporter=True, debugger=False, dmdPath=None, lsanPath=None):
 477     if xrePath == None:
 478       xrePath = self.DIST_BIN
 479     if env == None:
 480       env = dict(os.environ)
 481
 482     ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath))
 483     dmdLibrary = None
 484     preloadEnvVar = None
 485     if self.UNIXISH or self.IS_MAC:
 486       envVar = "LD_LIBRARY_PATH"
 487       preloadEnvVar = "LD_PRELOAD"
 488       if self.IS_MAC:
 489         envVar = "DYLD_LIBRARY_PATH"
 490         dmdLibrary = "libdmd.dylib"
 491       else: # unixish
 492         env['MOZILLA_FIVE_HOME'] = xrePath
 493         dmdLibrary = "libdmd.so"
 494       if envVar in env:
 495         ldLibraryPath = ldLibraryPath + ":" + env[envVar]
 496       env[envVar] = ldLibraryPath
 497     elif self.IS_WIN32:
 498       env["PATH"] = env["PATH"] + ";" + str(ldLibraryPath)
 499       dmdLibrary = "dmd.dll"
 500       preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB"
 501
 502     if dmdPath and dmdLibrary and preloadEnvVar:
 503       env['DMD'] = '1'
 504       env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary)
 505
 506     if crashreporter and not debugger:
 507       env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
 508       env['MOZ_CRASHREPORTER'] = '1'
 509     else:
 510       env['MOZ_CRASHREPORTER_DISABLE'] = '1'
 511
 512     # Crash on non-local network connections.
 513     env['MOZ_DISABLE_NONLOCAL_CONNECTIONS'] = '1'
 514
 515     env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
 516     env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
 517     env['NS_TRACE_MALLOC_DISABLE_STACKS'] = '1'
 518
 519     # Set WebRTC logging in case it is not set yet
 520     env.setdefault('NSPR_LOG_MODULES', 'signaling:5,mtransport:5,datachannel:5')
 521     env.setdefault('R_LOG_LEVEL', '6')
 522     env.setdefault('R_LOG_DESTINATION', 'stderr')
 523     env.setdefault('R_LOG_VERBOSE', '1')
 524
 525     # ASan specific environment stuff
 526     if self.IS_ASAN and (self.IS_LINUX or self.IS_MAC):
 527       # Symbolizer support
 528       llvmsym = os.path.join(xrePath, "llvm-symbolizer")
 529       if os.path.isfile(llvmsym):
 530         env["ASAN_SYMBOLIZER_PATH"] = llvmsym
 531         self.log.info("INFO | automation.py | ASan using symbolizer at %s", llvmsym)
 532       else:
 533         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Failed to find ASan symbolizer at %s", llvmsym)
 534
 535       try:
 536         totalMemory = int(os.popen("free").readlines()[1].split()[1])
 537
 538         # Only 4 GB RAM or less available? Use custom ASan options to reduce
 539         # the amount of resources required to do the tests. Standard options
 540         # will otherwise lead to OOM conditions on the current test slaves.
 541         if totalMemory <= 1024 * 1024 * 4:
 542           self.log.info("INFO | automation.py | ASan running in low-memory configuration")
 543           env["ASAN_OPTIONS"] = "quarantine_size=50331648:malloc_context_size=5"
 544         else:
 545           self.log.info("INFO | automation.py | ASan running in default memory configuration")
 546       except OSError,err:
 547         self.log.info("Failed determine available memory, disabling ASan low-memory configuration: %s", err.strerror)
 548       except:
 549         self.log.info("Failed determine available memory, disabling ASan low-memory configuration")
 550
 551     return env
 552
 553   def killPid(self, pid):
 554     try:
 555       os.kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM))
 556     except WindowsError:
 557       self.log.info("Failed to kill process %d." % pid)
 558
 559   if IS_WIN32:
 560     PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
 561     GetLastError = ctypes.windll.kernel32.GetLastError
 562
 563     def readWithTimeout(self, f, timeout):
 564       """
 565       Try to read a line of output from the file object |f|. |f| must be a
 566       pipe, like the |stdout| member of a subprocess.Popen object created
 567       with stdout=PIPE. Returns a tuple (line, did_timeout), where |did_timeout|
 568       is True if the read timed out, and False otherwise. If no output is
 569       received within |timeout| seconds, returns a blank line.
 570       """
 571
 572       if timeout is None:
 573         timeout = 0
 574
 575       x = msvcrt.get_osfhandle(f.fileno())
 576       l = ctypes.c_long()
 577       done = time.time() + timeout
 578
 579       buffer = ""
 580       while timeout == 0 or time.time() < done:
 581         if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0:
 582           err = self.GetLastError()
 583           if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE
 584             return ('', False)
 585           else:
 586             self.log.error("readWithTimeout got error: %d", err)
 587         # read a character at a time, checking for eol. Return once we get there.
 588         index = 0
 589         while index < l.value:
 590           char = f.read(1)
 591           buffer += char
 592           if char == '\n':
 593             return (buffer, False)
 594           index = index + 1
 595         time.sleep(0.01)
 596       return (buffer, True)
 597
 598     def isPidAlive(self, pid):
 599       STILL_ACTIVE = 259
 600       PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
 601       pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)
 602       if not pHandle:
 603         return False
 604       pExitCode = ctypes.wintypes.DWORD()
 605       ctypes.windll.kernel32.GetExitCodeProcess(pHandle, ctypes.byref(pExitCode))
 606       ctypes.windll.kernel32.CloseHandle(pHandle)
 607       return pExitCode.value == STILL_ACTIVE
 608
 609   else:
 610
 611     def readWithTimeout(self, f, timeout):
 612       """Try to read a line of output from the file object |f|. If no output
 613       is received within |timeout| seconds, return a blank line.
 614       Returns a tuple (line, did_timeout), where |did_timeout| is True
 615       if the read timed out, and False otherwise."""
 616       (r, w, e) = select.select([f], [], [], timeout)
 617       if len(r) == 0:
 618         return ('', True)
 619       return (f.readline(), False)
 620
 621     def isPidAlive(self, pid):
 622       try:
 623         # kill(pid, 0) checks for a valid PID without actually sending a signal
 624         # The method throws OSError if the PID is invalid, which we catch below.
 625         os.kill(pid, 0)
 626
 627         # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if
 628         # the process terminates before we get to this point.
 629         wpid, wstatus = os.waitpid(pid, os.WNOHANG)
 630         return wpid == 0
 631       except OSError, err:
 632         # Catch the errors we might expect from os.kill/os.waitpid,
 633         # and re-raise any others
 634         if err.errno == errno.ESRCH or err.errno == errno.ECHILD:
 635           return False
 636         raise
 637
 638   def dumpScreen(self, utilityPath):
 639     if self.haveDumpedScreen:
 640       self.log.info("Not taking screenshot here: see the one that was previously logged")
 641       return
 642
 643     self.haveDumpedScreen = True;
 644     automationutils.dumpScreen(utilityPath)
 645
 646
 647   def killAndGetStack(self, processPID, utilityPath, debuggerInfo):
 648     """Kill the process, preferrably in a way that gets us a stack trace.
 649        Also attempts to obtain a screenshot before killing the process."""
 650     if not debuggerInfo:
 651       self.dumpScreen(utilityPath)
 652     self.killAndGetStackNoScreenshot(processPID, utilityPath, debuggerInfo)
 653
 654   def killAndGetStackNoScreenshot(self, processPID, utilityPath, debuggerInfo):
 655     """Kill the process, preferrably in a way that gets us a stack trace."""
 656     if self.CRASHREPORTER and not debuggerInfo:
 657       if not self.IS_WIN32:
 658         # ABRT will get picked up by Breakpad's signal handler
 659         os.kill(processPID, signal.SIGABRT)
 660         return
 661       else:
 662         # We should have a "crashinject" program in our utility path
 663         crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe"))
 664         if os.path.exists(crashinject):
 665           status = subprocess.Popen([crashinject, str(processPID)]).wait()
 666           automationutils.printstatus(status, "crashinject")
 667           if status == 0:
 668             return
 669     self.log.info("Can't trigger Breakpad, just killing process")
 670     self.killPid(processPID)
 671
 672   def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath):
 673     """ Look for timeout or crashes and return the status after the process terminates """
 674     stackFixerProcess = None
 675     stackFixerFunction = None
 676     didTimeout = False
 677     hitMaxTime = False
 678     if proc.stdout is None:
 679       self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection")
 680     else:
 681       logsource = proc.stdout
 682
 683       if self.IS_DEBUG_BUILD and symbolsPath and os.path.exists(symbolsPath):
 684         # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad symbol files)
 685         # This method is preferred for Tinderbox builds, since native symbols may have been stripped.
 686         sys.path.insert(0, utilityPath)
 687         import fix_stack_using_bpsyms as stackFixerModule
 688         stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line, symbolsPath)
 689         del sys.path[0]
 690       elif self.IS_DEBUG_BUILD and self.IS_MAC and False:
 691         # Run each line through a function in fix_macosx_stack.py (uses atos)
 692         sys.path.insert(0, utilityPath)
 693         import fix_macosx_stack as stackFixerModule
 694         stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line)
 695         del sys.path[0]
 696       elif self.IS_DEBUG_BUILD and self.IS_LINUX:
 697         # Run logsource through fix-linux-stack.pl (uses addr2line)
 698         # This method is preferred for developer machines, so we don't have to run "make buildsymbols".
 699         stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, "fix-linux-stack.pl")],
 700                                          stdin=logsource,
 701                                          stdout=subprocess.PIPE)
 702         logsource = stackFixerProcess.stdout
 703
 704       # With metro browser runs this script launches the metro test harness which launches the browser.
 705       # The metro test harness hands back the real browser process id via log output which we need to
 706       # pick up on and parse out. This variable tracks the real browser process id if we find it.
 707       browserProcessId = -1
 708
 709       (line, didTimeout) = self.readWithTimeout(logsource, timeout)
 710       while line != "" and not didTimeout:
 711         if stackFixerFunction:
 712           line = stackFixerFunction(line)
 713         self.log.info(line.rstrip().decode("UTF-8", "ignore"))
 714         if "TEST-START" in line and "|" in line:
 715           self.lastTestSeen = line.split("|")[1].strip()
 716         if not debuggerInfo and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:
 717           self.dumpScreen(utilityPath)
 718
 719         (line, didTimeout) = self.readWithTimeout(logsource, timeout)
 720
 721         if "METRO_BROWSER_PROCESS" in line:
 722           index = line.find("=")
 723           if index:
 724             browserProcessId = line[index+1:].rstrip()
 725             self.log.info("INFO | automation.py | metro browser sub process id detected: %s", browserProcessId)
 726
 727         if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime):
 728           # Kill the application, but continue reading from stack fixer so as not to deadlock on stackFixerProcess.wait().
 729           hitMaxTime = True
 730           self.log.info("TEST-UNEXPECTED-FAIL | %s | application ran for longer than allowed maximum time of %d seconds", self.lastTestSeen, int(maxTime))
 731           self.killAndGetStack(proc.pid, utilityPath, debuggerInfo)
 732       if didTimeout:
 733         if line:
 734           self.log.info(line.rstrip().decode("UTF-8", "ignore"))
 735         self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout))
 736         if browserProcessId == -1:
 737           browserProcessId = proc.pid
 738         self.killAndGetStack(browserProcessId, utilityPath, debuggerInfo)
 739
 740     status = proc.wait()
 741     automationutils.printstatus(status, "Main app process")
 742     if status == 0:
 743       self.lastTestSeen = "Main app process exited normally"
 744     if status != 0 and not didTimeout and not hitMaxTime:
 745       self.log.info("TEST-UNEXPECTED-FAIL | %s | Exited with code %d during test run", self.lastTestSeen, status)
 746     if stackFixerProcess is not None:
 747       fixerStatus = stackFixerProcess.wait()
 748       automationutils.printstatus(status, "stackFixerProcess")
 749       if fixerStatus != 0 and not didTimeout and not hitMaxTime:
 750         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Stack fixer process exited with code %d during test run", fixerStatus)
 751     return status
 752
 753   def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
 754     """ build the application command line """
 755
 756     cmd = os.path.abspath(app)
 757     if self.IS_MAC and not self.IS_CAMINO and os.path.exists(cmd + "-bin"):
 758       # Prefer 'app-bin' in case 'app' is a shell script.
 759       # We can remove this hack once bug 673899 etc are fixed.
 760       cmd += "-bin"
 761
 762     args = []
 763
 764     if debuggerInfo:
 765       args.extend(debuggerInfo["args"])
 766       args.append(cmd)
 767       cmd = os.path.abspath(debuggerInfo["path"])
 768
 769     if self.IS_MAC:
 770       args.append("-foreground")
 771
 772     if self.IS_CYGWIN:
 773       profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
 774     else:
 775       profileDirectory = profileDir + "/"
 776
 777     args.extend(("-no-remote", "-profile", profileDirectory))
 778     if testURL is not None:
 779       if self.IS_CAMINO:
 780         args.extend(("-url", testURL))
 781       else:
 782         args.append((testURL))
 783     args.extend(extraArgs)
 784     return cmd, args
 785
 786   def checkForZombies(self, processLog, utilityPath, debuggerInfo):
 787     """ Look for hung processes """
 788     if not os.path.exists(processLog):
 789       self.log.info('Automation Error: PID log not found: %s', processLog)
 790       # Whilst no hung process was found, the run should still display as a failure
 791       return True
 792
 793     foundZombie = False
 794     self.log.info('INFO | zombiecheck | Reading PID log: %s', processLog)
 795     processList = []
 796     pidRE = re.compile(r'launched child process (\d+)$')
 797     processLogFD = open(processLog)
 798     for line in processLogFD:
 799       self.log.info(line.rstrip())
 800       m = pidRE.search(line)
 801       if m:
 802         processList.append(int(m.group(1)))
 803     processLogFD.close()
 804
 805     for processPID in processList:
 806       self.log.info("INFO | zombiecheck | Checking for orphan process with PID: %d", processPID)
 807       if self.isPidAlive(processPID):
 808         foundZombie = True
 809         self.log.info("TEST-UNEXPECTED-FAIL | zombiecheck | child process %d still alive after shutdown", processPID)
 810         self.killAndGetStack(processPID, utilityPath, debuggerInfo)
 811     return foundZombie
 812
 813   def checkForCrashes(self, minidumpDir, symbolsPath):
 814     return mozcrash.check_for_crashes(minidumpDir, symbolsPath, test_name=self.lastTestSeen)
 815
 816   def runApp(self, testURL, env, app, profileDir, extraArgs,
 817              runSSLTunnel = False, utilityPath = None,
 818              xrePath = None, certPath = None,
 819              debuggerInfo = None, symbolsPath = None,
 820              timeout = -1, maxTime = None, onLaunch = None,
 821              detectShutdownLeaks = False, screenshotOnFail=False, testPath=None, bisectChunk=None):
 822     """
 823     Run the app, log the duration it took to execute, return the status code.
 824     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
 825     """
 826
 827     if utilityPath == None:
 828       utilityPath = self.DIST_BIN
 829     if xrePath == None:
 830       xrePath = self.DIST_BIN
 831     if certPath == None:
 832       certPath = self.CERTS_SRC_DIR
 833     if timeout == -1:
 834       timeout = self.DEFAULT_TIMEOUT
 835
 836     # copy env so we don't munge the caller's environment
 837     env = dict(env);
 838     env["NO_EM_RESTART"] = "1"
 839     tmpfd, processLog = tempfile.mkstemp(suffix='pidlog')
 840     os.close(tmpfd)
 841     env["MOZ_PROCESS_LOG"] = processLog
 842
 843     if self.IS_TEST_BUILD and runSSLTunnel:
 844       # create certificate database for the profile
 845       certificateStatus = self.fillCertificateDB(profileDir, certPath, utilityPath, xrePath)
 846       if certificateStatus != 0:
 847         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Certificate integration failed")
 848         return certificateStatus
 849
 850       # start ssltunnel to provide https:// URLs capability
 851       ssltunnel = os.path.join(utilityPath, "ssltunnel" + self.BIN_SUFFIX)
 852       ssltunnelProcess = self.Process([ssltunnel,
 853                                os.path.join(profileDir, "ssltunnel.cfg")],
 854                                env = self.environment(xrePath = xrePath))
 855       self.log.info("INFO | automation.py | SSL tunnel pid: %d", ssltunnelProcess.pid)
 856
 857     cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs)
 858     startTime = datetime.now()
 859
 860     if debuggerInfo and debuggerInfo["interactive"]:
 861       # If an interactive debugger is attached, don't redirect output,
 862       # don't use timeouts, and don't capture ctrl-c.
 863       timeout = None
 864       maxTime = None
 865       outputPipe = None
 866       signal.signal(signal.SIGINT, lambda sigid, frame: None)
 867     else:
 868       outputPipe = subprocess.PIPE
 869
 870     self.lastTestSeen = "automation.py"
 871     proc = self.Process([cmd] + args,
 872                  env = self.environment(env, xrePath = xrePath,
 873                                    crashreporter = not debuggerInfo),
 874                  stdout = outputPipe,
 875                  stderr = subprocess.STDOUT)
 876     self.log.info("INFO | automation.py | Application pid: %d", proc.pid)
 877
 878     if onLaunch is not None:
 879       # Allow callers to specify an onLaunch callback to be fired after the
 880       # app is launched.
 881       onLaunch()
 882
 883     status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath)
 884     self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime))
 885
 886     # Do a final check for zombie child processes.
 887     zombieProcesses = self.checkForZombies(processLog, utilityPath, debuggerInfo)
 888
 889     crashed = self.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath)
 890
 891     if crashed or zombieProcesses:
 892       status = 1
 893
 894     if os.path.exists(processLog):
 895       os.unlink(processLog)
 896
 897     if self.IS_TEST_BUILD and runSSLTunnel:
 898       ssltunnelProcess.kill()
 899
 900     return status
 901
 902   def getExtensionIDFromRDF(self, rdfSource):
 903     """
 904     Retrieves the extension id from an install.rdf file (or string).
 905     """
 906     from xml.dom.minidom import parse, parseString, Node
 907
 908     if isinstance(rdfSource, file):
 909       document = parse(rdfSource)
 910     else:
 911       document = parseString(rdfSource)
 912
 913     # Find the <em:id> element. There can be multiple <em:id> tags
 914     # within <em:targetApplication> tags, so we have to check this way.
 915     for rdfChild in document.documentElement.childNodes:
 916       if rdfChild.nodeType == Node.ELEMENT_NODE and rdfChild.tagName == "Description":
 917         for descChild in rdfChild.childNodes:
 918           if descChild.nodeType == Node.ELEMENT_NODE and descChild.tagName == "em:id":
 919             return descChild.childNodes[0].data
 920
 921     return None
 922
 923   def installExtension(self, extensionSource, profileDir, extensionID = None):
 924     """
 925     Copies an extension into the extensions directory of the given profile.
 926     extensionSource - the source location of the extension files.  This can be either
 927                       a directory or a path to an xpi file.
 928     profileDir      - the profile directory we are copying into.  We will create the
 929                       "extensions" directory there if it doesn't exist.
 930     extensionID     - the id of the extension to be used as the containing directory for the
 931                       extension, if extensionSource is a directory, i.e.
 932                   this is the name of the folder in the <profileDir>/extensions/<extensionID>
 933     """
 934     if not os.path.isdir(profileDir):
 935       self.log.info("INFO | automation.py | Cannot install extension, invalid profileDir at: %s", profileDir)
 936       return
 937
 938     installRDFFilename = "install.rdf"
 939
 940     extensionsRootDir = os.path.join(profileDir, "extensions", "staged")
 941     if not os.path.isdir(extensionsRootDir):
 942       os.makedirs(extensionsRootDir)
 943
 944     if os.path.isfile(extensionSource):
 945       reader = automationutils.ZipFileReader(extensionSource)
 946
 947       for filename in reader.namelist():
 948         # Sanity check the zip file.
 949         if os.path.isabs(filename):
 950           self.log.info("INFO | automation.py | Cannot install extension, bad files in xpi")
 951           return
 952
 953         # We may need to dig the extensionID out of the zip file...
 954         if extensionID is None and filename == installRDFFilename:
 955           extensionID = self.getExtensionIDFromRDF(reader.read(filename))
 956
 957       # We must know the extensionID now.
 958       if extensionID is None:
 959         self.log.info("INFO | automation.py | Cannot install extension, missing extensionID")
 960         return
 961
 962       # Make the extension directory.
 963       extensionDir = os.path.join(extensionsRootDir, extensionID)
 964       os.mkdir(extensionDir)
 965
 966       # Extract all files.
 967       reader.extractall(extensionDir)
 968
 969     elif os.path.isdir(extensionSource):
 970       if extensionID is None:
 971         filename = os.path.join(extensionSource, installRDFFilename)
 972         if os.path.isfile(filename):
 973           with open(filename, "r") as installRDF:
 974             extensionID = self.getExtensionIDFromRDF(installRDF)
 975
 976         if extensionID is None:
 977           self.log.info("INFO | automation.py | Cannot install extension, missing extensionID")
 978           return
 979
 980       # Copy extension tree into its own directory.
 981       # "destination directory must not already exist".
 982       shutil.copytree(extensionSource, os.path.join(extensionsRootDir, extensionID))
 983
 984     else:
 985       self.log.info("INFO | automation.py | Cannot install extension, invalid extensionSource at: %s", extensionSource)
 986
 987   def elf_arm(self, filename):
 988     data = open(filename, 'rb').read(20)
 989     return data[:4] == "\x7fELF" and ord(data[18]) == 40 # EM_ARM
 990