build/automation.py.in

   1 #
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   4 #
   5 # The contents of this file are subject to the Mozilla Public License Version
   6 # 1.1 (the "License"); you may not use this file except in compliance with
   7 # the License. You may obtain a copy of the License at
   8 # http://www.mozilla.org/MPL/
   9 #
  10 # Software distributed under the License is distributed on an "AS IS" basis,
  11 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12 # for the specific language governing rights and limitations under the
  13 # License.
  14 #
  15 # The Original Code is mozilla.org code.
  16 #
  17 # The Initial Developer of the Original Code is
  18 # Mozilla Foundation.
  19 # Portions created by the Initial Developer are Copyright (C) 2008
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #   Robert Sayre <sayrer@gmail.com>
  24 #   Jeff Walden <jwalden+bmo@mit.edu>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 import codecs
  41 from datetime import datetime, timedelta
  42 import itertools
  43 import logging
  44 import os
  45 import re
  46 import select
  47 import shutil
  48 import signal
  49 import subprocess
  50 import sys
  51 import threading
  52 import tempfile
  53
  54
  55 #expand _DIST_BIN = __XPC_BIN_PATH__
  56 #expand _IS_WIN32 = len("__WIN32__") != 0
  57 #expand _IS_MAC = __IS_MAC__ != 0
  58 #expand _IS_LINUX = __IS_LINUX__ != 0
  59 #ifdef IS_CYGWIN
  60 #expand _IS_CYGWIN = __IS_CYGWIN__ == 1
  61 #else
  62 _IS_CYGWIN = False
  63 #endif
  64 #expand _IS_CAMINO = __IS_CAMINO__ != 0
  65 #expand _BIN_SUFFIX = __BIN_SUFFIX__
  66 #expand _PERL = __PERL__
  67
  68 #expand _DEFAULT_APP = "./" + __BROWSER_PATH__
  69 #expand _CERTS_SRC_DIR = __CERTS_SRC_DIR__
  70 #expand _IS_TEST_BUILD = __IS_TEST_BUILD__
  71 #expand _IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
  72 #expand _CRASHREPORTER = __CRASHREPORTER__ == 1
  73
  74 #################
  75 # PROFILE SETUP #
  76 #################
  77
  78 class SyntaxError(Exception):
  79   "Signifies a syntax error on a particular line in server-locations.txt."
  80
  81   def __init__(self, lineno, msg = None):
  82     self.lineno = lineno
  83     self.msg = msg
  84
  85   def __str__(self):
  86     s = "Syntax error on line " + str(self.lineno)
  87     if self.msg:
  88       s += ": %s." % self.msg
  89     else:
  90       s += "."
  91     return s
  92
  93
  94 class Location:
  95   "Represents a location line in server-locations.txt."
  96
  97   def __init__(self, scheme, host, port, options):
  98     self.scheme = scheme
  99     self.host = host
 100     self.port = port
 101     self.options = options
 102
 103 class Automation(object):
 104   """
 105   Runs the browser from a script, and provides useful utilities
 106   for setting up the browser environment.
 107   """
 108
 109   DIST_BIN = _DIST_BIN
 110   IS_WIN32 = _IS_WIN32
 111   IS_MAC = _IS_MAC
 112   IS_LINUX = _IS_LINUX
 113   IS_CYGWIN = _IS_CYGWIN
 114   IS_CAMINO = _IS_CAMINO
 115   BIN_SUFFIX = _BIN_SUFFIX
 116   PERL = _PERL
 117
 118   UNIXISH = not IS_WIN32 and not IS_MAC
 119
 120   DEFAULT_APP = _DEFAULT_APP
 121   CERTS_SRC_DIR = _CERTS_SRC_DIR
 122   IS_TEST_BUILD = _IS_TEST_BUILD
 123   IS_DEBUG_BUILD = _IS_DEBUG_BUILD
 124   CRASHREPORTER = _CRASHREPORTER
 125
 126   SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 127   sys.path.insert(0, SCRIPT_DIR)
 128   automationutils = __import__('automationutils')
 129
 130   # timeout, in seconds
 131   DEFAULT_TIMEOUT = 60.0
 132
 133   log = logging.getLogger()
 134
 135   def __init__(self):
 136
 137     # We use the logging system here primarily because it'll handle multiple
 138     # threads, which is needed to process the output of the server and application
 139     # processes simultaneously.
 140     handler = logging.StreamHandler(sys.stdout)
 141     self.log.setLevel(logging.INFO)
 142     self.log.addHandler(handler)
 143
 144   @property
 145   def __all__(self):
 146     return [
 147            "UNIXISH",
 148            "IS_WIN32",
 149            "IS_MAC",
 150            "log",
 151            "runApp",
 152            "Process",
 153            "addCommonOptions",
 154            "initializeProfile",
 155            "DIST_BIN",
 156            "DEFAULT_APP",
 157            "CERTS_SRC_DIR",
 158            "environment",
 159            "IS_TEST_BUILD",
 160            "IS_DEBUG_BUILD",
 161            "DEFAULT_TIMEOUT",
 162           ]
 163
 164   class Process(subprocess.Popen):
 165     """
 166     Represents our view of a subprocess.
 167     It adds a kill() method which allows it to be stopped explicitly.
 168     """
 169
 170     def kill(self):
 171       if Automation().IS_WIN32:
 172         import platform
 173         pid = "%i" % self.pid
 174         if platform.release() == "2000":
 175           # Windows 2000 needs 'kill.exe' from the
 176           #'Windows 2000 Resource Kit tools'. (See bug 475455.)
 177           try:
 178             subprocess.Popen(["kill", "-f", pid]).wait()
 179           except:
 180             self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid)
 181         else:
 182           # Windows XP and later.
 183           subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()
 184       else:
 185         os.kill(self.pid, signal.SIGKILL)
 186
 187   def readLocations(self, locationsPath = "server-locations.txt"):
 188     """
 189     Reads the locations at which the Mochitest HTTP server is available from
 190     server-locations.txt.
 191     """
 192
 193     locationFile = codecs.open(locationsPath, "r", "UTF-8")
 194
 195     # Perhaps more detail than necessary, but it's the easiest way to make sure
 196     # we get exactly the format we want.  See server-locations.txt for the exact
 197     # format guaranteed here.
 198     lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"
 199                       r"://"
 200                       r"(?P<host>"
 201                         r"\d+\.\d+\.\d+\.\d+"
 202                         r"|"
 203                         r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*"
 204                         r"[a-z](?:[-a-z0-9]*[a-z0-9])?"
 205                       r")"
 206                       r":"
 207                       r"(?P<port>\d+)"
 208                       r"(?:"
 209                       r"\s+"
 210                       r"(?P<options>\S+(?:,\S+)*)"
 211                       r")?$")
 212     locations = []
 213     lineno = 0
 214     seenPrimary = False
 215     for line in locationFile:
 216       lineno += 1
 217       if line.startswith("#") or line == "\n":
 218         continue
 219
 220       match = lineRe.match(line)
 221       if not match:
 222         raise SyntaxError(lineno)
 223
 224       options = match.group("options")
 225       if options:
 226         options = options.split(",")
 227         if "primary" in options:
 228           if seenPrimary:
 229             raise SyntaxError(lineno, "multiple primary locations")
 230           seenPrimary = True
 231       else:
 232         options = []
 233
 234       locations.append(Location(match.group("scheme"), match.group("host"),
 235                                 match.group("port"), options))
 236
 237     if not seenPrimary:
 238       raise SyntaxError(lineno + 1, "missing primary location")
 239
 240     return locations
 241
 242
 243   def initializeProfile(self, profileDir, extraPrefs = []):
 244     "Sets up the standard testing profile."
 245
 246     # Start with a clean slate.
 247     shutil.rmtree(profileDir, True)
 248     os.mkdir(profileDir)
 249
 250     prefs = []
 251
 252     part = """\
 253 user_pref("browser.dom.window.dump.enabled", true);
 254 user_pref("dom.allow_scripts_to_close_windows", true);
 255 user_pref("dom.disable_open_during_load", false);
 256 user_pref("dom.max_script_run_time", 0); // no slow script dialogs
 257 user_pref("dom.max_chrome_script_run_time", 0);
 258 user_pref("dom.popup_maximum", -1);
 259 user_pref("signed.applets.codebase_principal_support", true);
 260 user_pref("security.warn_submit_insecure", false);
 261 user_pref("browser.shell.checkDefaultBrowser", false);
 262 user_pref("shell.checkDefaultClient", false);
 263 user_pref("browser.warnOnQuit", false);
 264 user_pref("accessibility.typeaheadfind.autostart", false);
 265 user_pref("javascript.options.showInConsole", true);
 266 user_pref("layout.debug.enable_data_xbl", true);
 267 user_pref("browser.EULA.override", true);
 268 user_pref("javascript.options.jit.content", true);
 269 user_pref("gfx.color_management.force_srgb", true);
 270 user_pref("network.manage-offline-status", false);
 271 user_pref("test.mousescroll", true);
 272 user_pref("security.default_personal_cert", "Select Automatically"); // Need to client auth test be w/o any dialogs
 273 user_pref("network.http.prompt-temp-redirect", false);
 274 user_pref("media.cache_size", 100);
 275 user_pref("security.warn_viewing_mixed", false);
 276
 277 user_pref("geo.wifi.uri", "http://localhost:8888/tests/dom/tests/mochitest/geolocation/network_geolocation.sjs");
 278 user_pref("geo.wifi.testing", true);
 279
 280 user_pref("camino.warn_when_closing", false); // Camino-only, harmless to others
 281
 282 // Make url-classifier updates so rare that they won't affect tests
 283 user_pref("urlclassifier.updateinterval", 172800);
 284 // Point the url-classifier to the local testing server for fast failures
 285 user_pref("browser.safebrowsing.provider.0.gethashURL", "http://localhost:8888/safebrowsing-dummy/gethash");
 286 user_pref("browser.safebrowsing.provider.0.keyURL", "http://localhost:8888/safebrowsing-dummy/newkey");
 287 user_pref("browser.safebrowsing.provider.0.lookupURL", "http://localhost:8888/safebrowsing-dummy/lookup");
 288 user_pref("browser.safebrowsing.provider.0.updateURL", "http://localhost:8888/safebrowsing-dummy/update");
 289 """
 290
 291     prefs.append(part)
 292
 293     locations = self.readLocations()
 294
 295     # Grant God-power to all the privileged servers on which tests run.
 296     privileged = filter(lambda loc: "privileged" in loc.options, locations)
 297     for (i, l) in itertools.izip(itertools.count(1), privileged):
 298       part = """
 299 user_pref("capability.principal.codebase.p%(i)d.granted",
 300           "UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite \
 301            UniversalPreferencesRead UniversalPreferencesWrite \
 302            UniversalFileRead");
 303 user_pref("capability.principal.codebase.p%(i)d.id", "%(origin)s");
 304 user_pref("capability.principal.codebase.p%(i)d.subjectName", "");
 305 """  % { "i": i,
 306          "origin": (l.scheme + "://" + l.host + ":" + l.port) }
 307       prefs.append(part)
 308
 309     # We need to proxy every server but the primary one.
 310     origins = ["'%s://%s:%s'" % (l.scheme, l.host, l.port)
 311               for l in filter(lambda l: "primary" not in l.options, locations)]
 312     origins = ", ".join(origins)
 313
 314     pacURL = """data:text/plain,
 315 function FindProxyForURL(url, host)
 316 {
 317   var origins = [%(origins)s];
 318   var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
 319                          '://' +
 320                          '(?:[^/@]*@)?' +
 321                          '(.*?)' +
 322                          '(?::(\\\\\\\\d+))?/');
 323   var matches = regex.exec(url);
 324   if (!matches)
 325     return 'DIRECT';
 326   var isHttp = matches[1] == 'http';
 327   var isHttps = matches[1] == 'https';
 328   if (!matches[3])
 329   {
 330     if (isHttp) matches[3] = '80';
 331     if (isHttps) matches[3] = '443';
 332   }
 333
 334   var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
 335   if (origins.indexOf(origin) < 0)
 336     return 'DIRECT';
 337   if (isHttp)
 338     return 'PROXY 127.0.0.1:8888';
 339   if (isHttps)
 340     return 'PROXY 127.0.0.1:4443';
 341   return 'DIRECT';
 342 }""" % { "origins": origins }
 343     pacURL = "".join(pacURL.splitlines())
 344
 345     part = """
 346 user_pref("network.proxy.type", 2);
 347 user_pref("network.proxy.autoconfig_url", "%(pacURL)s");
 348
 349 user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless to others
 350 """ % {"pacURL": pacURL}
 351     prefs.append(part)
 352
 353     for v in extraPrefs:
 354       thispref = v.split("=")
 355       if len(thispref) < 2:
 356         print "Error: syntax error in --setpref=" + v
 357         sys.exit(1)
 358       part = 'user_pref("%s", %s);\n' % (thispref[0], thispref[1])
 359       prefs.append(part)
 360
 361     # write the preferences
 362     prefsFile = open(profileDir + "/" + "user.js", "a")
 363     prefsFile.write("".join(prefs))
 364     prefsFile.close()
 365
 366   def addCommonOptions(self, parser):
 367     "Adds command-line options which are common to mochitest and reftest."
 368
 369     parser.add_option("--setpref",
 370                       action = "append", type = "string",
 371                       default = [],
 372                       dest = "extraPrefs", metavar = "PREF=VALUE",
 373                       help = "defines an extra user preference")
 374
 375   def fillCertificateDB(self, profileDir, certPath, utilityPath, xrePath):
 376     pwfilePath = os.path.join(profileDir, ".crtdbpw")
 377
 378     pwfile = open(pwfilePath, "w")
 379     pwfile.write("\n")
 380     pwfile.close()
 381
 382     # Create head of the ssltunnel configuration file
 383     sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")
 384     sslTunnelConfig = open(sslTunnelConfigPath, "w")
 385
 386     sslTunnelConfig.write("httpproxy:1\n")
 387     sslTunnelConfig.write("certdbdir:%s\n" % certPath)
 388     sslTunnelConfig.write("forward:127.0.0.1:8888\n")
 389     sslTunnelConfig.write("listen:*:4443:pgo server certificate\n")
 390
 391     # Configure automatic certificate and bind custom certificates, client authentication
 392     locations = self.readLocations()
 393     locations.pop(0)
 394     for loc in locations:
 395       if loc.scheme == "https" and "nocert" not in loc.options:
 396         customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")
 397         clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")
 398         for option in loc.options:
 399           match = customCertRE.match(option)
 400           if match:
 401             customcert = match.group("nickname");
 402             sslTunnelConfig.write("listen:%s:%s:4443:%s\n" %
 403                       (loc.host, loc.port, customcert))
 404
 405           match = clientAuthRE.match(option)
 406           if match:
 407             clientauth = match.group("clientauth");
 408             sslTunnelConfig.write("clientauth:%s:%s:4443:%s\n" %
 409                       (loc.host, loc.port, clientauth))
 410
 411     sslTunnelConfig.close()
 412
 413     # Pre-create the certification database for the profile
 414     env = self.environment(xrePath = xrePath)
 415     certutil = os.path.join(utilityPath, "certutil" + self.BIN_SUFFIX)
 416     pk12util = os.path.join(utilityPath, "pk12util" + self.BIN_SUFFIX)
 417
 418     status = self.Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait()
 419     if status != 0:
 420       return status
 421
 422     # Walk the cert directory and add custom CAs and client certs
 423     files = os.listdir(certPath)
 424     for item in files:
 425       root, ext = os.path.splitext(item)
 426       if ext == ".ca":
 427         trustBits = "CT,,"
 428         if root.endswith("-object"):
 429           trustBits = "CT,,CT"
 430         self.Process([certutil, "-A", "-i", os.path.join(certPath, item),
 431                     "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits],
 432                     env = env).wait()
 433       if ext == ".client":
 434         self.Process([pk12util, "-i", os.path.join(certPath, item), "-w",
 435                     pwfilePath, "-d", profileDir],
 436                     env = env).wait()
 437
 438     os.unlink(pwfilePath)
 439     return 0
 440
 441   def environment(self, env = None, xrePath = None, crashreporter = True):
 442     if xrePath == None:
 443       xrePath = self.DIST_BIN
 444     if env == None:
 445       env = dict(os.environ)
 446
 447     ldLibraryPath = os.path.abspath(os.path.join(self.SCRIPT_DIR, xrePath))
 448     if self.UNIXISH or self.IS_MAC:
 449       envVar = "LD_LIBRARY_PATH"
 450       if self.IS_MAC:
 451         envVar = "DYLD_LIBRARY_PATH"
 452       else: # unixish
 453         env['MOZILLA_FIVE_HOME'] = xrePath
 454       if envVar in env:
 455         ldLibraryPath = ldLibraryPath + ":" + env[envVar]
 456       env[envVar] = ldLibraryPath
 457     elif self.IS_WIN32:
 458       env["PATH"] = env["PATH"] + ";" + ldLibraryPath
 459
 460     if crashreporter:
 461       env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
 462       env['MOZ_CRASHREPORTER'] = '1'
 463     else:
 464       env['MOZ_CRASHREPORTER_DISABLE'] = '1'
 465
 466     env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
 467     env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
 468     return env
 469
 470   if IS_WIN32:
 471     ctypes = __import__('ctypes')
 472     wintypes = __import__('ctypes.wintypes')
 473     time = __import__('time')
 474     msvcrt = __import__('msvcrt')
 475     PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
 476     GetLastError = ctypes.windll.kernel32.GetLastError
 477
 478     def readWithTimeout(self, f, timeout):
 479       """Try to read a line of output from the file object |f|.
 480       |f| must be a  pipe, like the |stdout| member of a subprocess.Popen
 481       object created with stdout=PIPE. If no output
 482       is received within |timeout| seconds, return a blank line.
 483       Returns a tuple (line, did_timeout), where |did_timeout| is True
 484       if the read timed out, and False otherwise."""
 485       if timeout is None:
 486         # shortcut to allow callers to pass in "None" for no timeout.
 487         return (f.readline(), False)
 488       x = self.msvcrt.get_osfhandle(f.fileno())
 489       l = self.ctypes.c_long()
 490       done = self.time.time() + timeout
 491       while self.time.time() < done:
 492         if self.PeekNamedPipe(x, None, 0, None, self.ctypes.byref(l), None) == 0:
 493           err = self.GetLastError()
 494           if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE
 495             return ('', False)
 496           else:
 497             log.error("readWithTimeout got error: %d", err)
 498         if l.value > 0:
 499           # we're assuming that the output is line-buffered,
 500           # which is not unreasonable
 501           return (f.readline(), False)
 502         self.time.sleep(0.01)
 503       return ('', True)
 504
 505     def isPidAlive(self, pid):
 506       STILL_ACTIVE = 259
 507       PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
 508       pHandle = self.ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)
 509       if not pHandle:
 510         return False
 511       pExitCode = ctypes.wintypes.DWORD()
 512       self.ctypes.windll.kernel32.GetExitCodeProcess(pHandle, self.ctypes.byref(pExitCode))
 513       self.ctypes.windll.kernel32.CloseHandle(pHandle)
 514       if (pExitCode.value == STILL_ACTIVE):
 515         return True
 516       else:
 517         return False
 518
 519     def killPid(self, pid):
 520       PROCESS_TERMINATE = 0x0001
 521       pHandle = self.ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, 0, pid)
 522       if not pHandle:
 523         return
 524       success = self.ctypes.windll.kernel32.TerminateProcess(pHandle, 1)
 525       self.ctypes.windll.kernel32.CloseHandle(pHandle)
 526
 527   else:
 528     errno = __import__('errno')
 529
 530     def readWithTimeout(self, f, timeout):
 531       """Try to read a line of output from the file object |f|. If no output
 532       is received within |timeout| seconds, return a blank line.
 533       Returns a tuple (line, did_timeout), where |did_timeout| is True
 534       if the read timed out, and False otherwise."""
 535       (r, w, e) = select.select([f], [], [], timeout)
 536       if len(r) == 0:
 537         return ('', True)
 538       return (f.readline(), False)
 539
 540     def isPidAlive(self, pid):
 541       try:
 542         # kill(pid, 0) checks for a valid PID without actually sending a signal
 543         # The method throws OSError if the PID is invalid, which we catch below.
 544         os.kill(pid, 0)
 545
 546         # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if
 547         # the process terminates before we get to this point.
 548         wpid, wstatus = os.waitpid(pid, os.WNOHANG)
 549         if wpid == 0:
 550           return True
 551
 552         return False
 553       except OSError, err:
 554         # Catch the errors we might expect from os.kill/os.waitpid,
 555         # and re-raise any others
 556         if err.errno == self.errno.ESRCH or err.errno == self.errno.ECHILD:
 557           return False
 558         raise
 559
 560     def killPid(self, pid):
 561       os.kill(pid, signal.SIGKILL)
 562
 563   def triggerBreakpad(self, proc, utilityPath):
 564     """Attempt to kill this process in a way that triggers Breakpad crash
 565     reporting, if we know how for this platform. Otherwise just .kill() it."""
 566     if self.CRASHREPORTER:
 567       if self.UNIXISH:
 568         # SEGV will get picked up by Breakpad's signal handler
 569         os.kill(proc.pid, signal.SIGSEGV)
 570         return
 571       elif self.IS_WIN32:
 572         # We should have a "crashinject" program in our utility path
 573         crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe"))
 574         if os.path.exists(crashinject) and subprocess.Popen([crashinject, str(proc.pid)]).wait() == 0:
 575           return
 576     #TODO: kill the process such that it triggers Breakpad on OS X (bug 525296)
 577     self.log.info("Can't trigger Breakpad, just killing process")
 578     proc.kill()
 579
 580   def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime):
 581     """ Look for timeout or crashes and return the status after the process terminates """
 582     stackFixerProcess = None
 583     stackFixerModule = None
 584     didTimeout = False
 585     if proc.stdout is None:
 586       self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection")
 587     else:
 588       logsource = proc.stdout
 589       if self.IS_DEBUG_BUILD and self.IS_LINUX:
 590         # Run logsource through fix-linux-stack.pl
 591         stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, "fix-linux-stack.pl")],
 592                                          stdin=logsource,
 593                                          stdout=subprocess.PIPE)
 594         logsource = stackFixerProcess.stdout
 595
 596       if self.IS_DEBUG_BUILD and self.IS_MAC:
 597         # Import fix_macosx_stack.py from utilityPath
 598         sys.path.insert(0, utilityPath)
 599         import fix_macosx_stack as stackFixerModule
 600         del sys.path[0]
 601
 602       (line, didTimeout) = self.readWithTimeout(logsource, timeout)
 603       hitMaxTime = False
 604       while line != "" and not didTimeout:
 605         if stackFixerModule:
 606           line = stackFixerModule.fixSymbols(line)
 607         self.log.info(line.rstrip())
 608         (line, didTimeout) = self.readWithTimeout(logsource, timeout)
 609         if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime):
 610           # Kill the application, but continue reading from stack fixer so as not to deadlock on stackFixerProcess.wait().
 611           hitMaxTime = True
 612           self.log.info("TEST-UNEXPECTED-FAIL | automation.py | application ran for longer than allowed maximum time of %d seconds", int(maxTime))
 613           self.triggerBreakpad(proc, utilityPath)
 614       if didTimeout:
 615         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | application timed out after %d seconds with no output", int(timeout))
 616         self.triggerBreakpad(proc, utilityPath)
 617
 618     status = proc.wait()
 619     if status != 0 and not didTimeout and not hitMaxTime:
 620       self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Exited with code %d during test run", status)
 621     if stackFixerProcess is not None:
 622       fixerStatus = stackFixerProcess.wait()
 623       if fixerStatus != 0 and not didTimeout and not hitMaxTime:
 624         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Stack fixer process exited with code %d during test run", fixerStatus)
 625     return status
 626
 627   def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
 628     """ build the application command line """
 629
 630     cmd = app
 631     if self.IS_MAC and not self.IS_CAMINO and not cmd.endswith("-bin"):
 632       cmd += "-bin"
 633     cmd = os.path.abspath(cmd)
 634
 635     args = []
 636
 637     if debuggerInfo:
 638       args.extend(debuggerInfo["args"])
 639       args.append(cmd)
 640       cmd = os.path.abspath(debuggerInfo["path"])
 641
 642     if self.IS_MAC:
 643       args.append("-foreground")
 644
 645     if self.IS_CYGWIN:
 646       profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
 647     else:
 648       profileDirectory = profileDir + "/"
 649
 650     args.extend(("-no-remote", "-profile", profileDirectory))
 651     if testURL is not None:
 652       if self.IS_CAMINO:
 653         args.extend(("-url", testURL))
 654       else:
 655         args.append((testURL))
 656     args.extend(extraArgs)
 657     return cmd, args
 658
 659   def checkForZombies(self, processLog):
 660     """ Look for hung processes """
 661     if not os.path.exists(processLog):
 662       self.log.info('INFO | automation.py | PID log not found: %s', processLog)
 663     else:
 664       self.log.info('INFO | automation.py | Reading PID log: %s', processLog)
 665       processList = []
 666       pidRE = re.compile(r'launched child process (\d+)$')
 667       processLogFD = open(processLog)
 668       for line in processLogFD:
 669         self.log.info(line.rstrip())
 670         m = pidRE.search(line)
 671         if m:
 672           processList.append(int(m.group(1)))
 673       processLogFD.close()
 674
 675       for processPID in processList:
 676         self.log.info("INFO | automation.py | Checking for orphan process with PID: %d", processPID)
 677         if self.isPidAlive(processPID):
 678           self.log.info("TEST-UNEXPECTED-FAIL | automation.py | child process %d still alive after shutdown", processPID)
 679           self.killPid(processPID)
 680
 681   def runApp(self, testURL, env, app, profileDir, extraArgs,
 682              runSSLTunnel = False, utilityPath = None,
 683              xrePath = None, certPath = None,
 684              debuggerInfo = None, symbolsPath = None,
 685              timeout = -1, maxTime = None):
 686     """
 687     Run the app, log the duration it took to execute, return the status code.
 688     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
 689     """
 690
 691     if utilityPath == None:
 692       utilityPath = self.DIST_BIN
 693     if xrePath == None:
 694       xrePath = self.DIST_BIN
 695     if certPath == None:
 696       certPath = self.CERTS_SRC_DIR
 697     if timeout == -1:
 698       timeout = self.DEFAULT_TIMEOUT
 699
 700     # copy env so we don't munge the caller's environment
 701     env = dict(env);
 702     env["NO_EM_RESTART"] = "1"
 703     tmpfd, processLog = tempfile.mkstemp(suffix='pidlog')
 704     os.close(tmpfd)
 705     env["MOZ_PROCESS_LOG"] = processLog
 706
 707     if self.IS_TEST_BUILD and runSSLTunnel:
 708       # create certificate database for the profile
 709       certificateStatus = self.fillCertificateDB(profileDir, certPath, utilityPath, xrePath)
 710       if certificateStatus != 0:
 711         self.log.info("TEST-UNEXPECTED FAIL | automation.py | Certificate integration failed")
 712         return certificateStatus
 713
 714       # start ssltunnel to provide https:// URLs capability
 715       ssltunnel = os.path.join(utilityPath, "ssltunnel" + self.BIN_SUFFIX)
 716       ssltunnelProcess = self.Process([ssltunnel,
 717                                os.path.join(profileDir, "ssltunnel.cfg")],
 718                                env = self.environment(xrePath = xrePath))
 719       self.log.info("INFO | automation.py | SSL tunnel pid: %d", ssltunnelProcess.pid)
 720
 721     cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs)
 722     startTime = datetime.now()
 723
 724     # Don't redirect stdout and stderr if an interactive debugger is attached
 725     if debuggerInfo and debuggerInfo["interactive"]:
 726       outputPipe = None
 727     else:
 728       outputPipe = subprocess.PIPE
 729
 730     proc = self.Process([cmd] + args,
 731                  env = self.environment(env, xrePath = xrePath,
 732                                    crashreporter = not debuggerInfo),
 733                  stdout = outputPipe,
 734                  stderr = subprocess.STDOUT)
 735     self.log.info("INFO | automation.py | Application pid: %d", proc.pid)
 736
 737     status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime)
 738     self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime))
 739
 740     # Do a final check for zombie child processes.
 741     self.checkForZombies(processLog)
 742     self.automationutils.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath)
 743
 744     if os.path.exists(processLog):
 745       os.unlink(processLog)
 746
 747     if self.IS_TEST_BUILD and runSSLTunnel:
 748       ssltunnelProcess.kill()
 749
 750     return status