build/automation.py.in

   1 #
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   4 #
   5 # The contents of this file are subject to the Mozilla Public License Version
   6 # 1.1 (the "License"); you may not use this file except in compliance with
   7 # the License. You may obtain a copy of the License at
   8 # http://www.mozilla.org/MPL/
   9 #
  10 # Software distributed under the License is distributed on an "AS IS" basis,
  11 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12 # for the specific language governing rights and limitations under the
  13 # License.
  14 #
  15 # The Original Code is mozilla.org code.
  16 #
  17 # The Initial Developer of the Original Code is
  18 # Mozilla Foundation.
  19 # Portions created by the Initial Developer are Copyright (C) 2008
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #   Robert Sayre <sayrer@gmail.com>
  24 #   Jeff Walden <jwalden+bmo@mit.edu>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 import codecs
  41 from datetime import datetime, timedelta
  42 import itertools
  43 import logging
  44 import os
  45 import re
  46 import select
  47 import shutil
  48 import signal
  49 import subprocess
  50 import sys
  51 import threading
  52 import tempfile
  53
  54 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
  55 sys.path.insert(0, SCRIPT_DIR)
  56 import automationutils
  57
  58 _DEFAULT_WEB_SERVER = "127.0.0.1"
  59 _DEFAULT_HTTP_PORT = 8888
  60 _DEFAULT_SSL_PORT = 4443
  61
  62 #expand _DIST_BIN = __XPC_BIN_PATH__
  63 #expand _IS_WIN32 = len("__WIN32__") != 0
  64 #expand _IS_MAC = __IS_MAC__ != 0
  65 #expand _IS_LINUX = __IS_LINUX__ != 0
  66 #ifdef IS_CYGWIN
  67 #expand _IS_CYGWIN = __IS_CYGWIN__ == 1
  68 #else
  69 _IS_CYGWIN = False
  70 #endif
  71 #expand _IS_CAMINO = __IS_CAMINO__ != 0
  72 #expand _BIN_SUFFIX = __BIN_SUFFIX__
  73 #expand _PERL = __PERL__
  74
  75 #expand _DEFAULT_APP = "./" + __BROWSER_PATH__
  76 #expand _CERTS_SRC_DIR = __CERTS_SRC_DIR__
  77 #expand _IS_TEST_BUILD = __IS_TEST_BUILD__
  78 #expand _IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
  79 #expand _CRASHREPORTER = __CRASHREPORTER__ == 1
  80
  81
  82 if _IS_WIN32:
  83   import ctypes, ctypes.wintypes, time, msvcrt
  84 else:
  85   import errno
  86
  87
  88 # We use the logging system here primarily because it'll handle multiple
  89 # threads, which is needed to process the output of the server and application
  90 # processes simultaneously.
  91 _log = logging.getLogger()
  92 handler = logging.StreamHandler(sys.stdout)
  93 _log.setLevel(logging.INFO)
  94 _log.addHandler(handler)
  95
  96
  97 #################
  98 # PROFILE SETUP #
  99 #################
 100
 101 class SyntaxError(Exception):
 102   "Signifies a syntax error on a particular line in server-locations.txt."
 103
 104   def __init__(self, lineno, msg = None):
 105     self.lineno = lineno
 106     self.msg = msg
 107
 108   def __str__(self):
 109     s = "Syntax error on line " + str(self.lineno)
 110     if self.msg:
 111       s += ": %s." % self.msg
 112     else:
 113       s += "."
 114     return s
 115
 116
 117 class Location:
 118   "Represents a location line in server-locations.txt."
 119
 120   def __init__(self, scheme, host, port, options):
 121     self.scheme = scheme
 122     self.host = host
 123     self.port = port
 124     self.options = options
 125
 126 class Automation(object):
 127   """
 128   Runs the browser from a script, and provides useful utilities
 129   for setting up the browser environment.
 130   """
 131
 132   DIST_BIN = _DIST_BIN
 133   IS_WIN32 = _IS_WIN32
 134   IS_MAC = _IS_MAC
 135   IS_LINUX = _IS_LINUX
 136   IS_CYGWIN = _IS_CYGWIN
 137   IS_CAMINO = _IS_CAMINO
 138   BIN_SUFFIX = _BIN_SUFFIX
 139   PERL = _PERL
 140
 141   UNIXISH = not IS_WIN32 and not IS_MAC
 142
 143   DEFAULT_APP = _DEFAULT_APP
 144   CERTS_SRC_DIR = _CERTS_SRC_DIR
 145   IS_TEST_BUILD = _IS_TEST_BUILD
 146   IS_DEBUG_BUILD = _IS_DEBUG_BUILD
 147   CRASHREPORTER = _CRASHREPORTER
 148
 149   # timeout, in seconds
 150   DEFAULT_TIMEOUT = 60.0
 151   DEFAULT_WEB_SERVER = _DEFAULT_WEB_SERVER
 152   DEFAULT_HTTP_PORT = _DEFAULT_HTTP_PORT
 153   DEFAULT_SSL_PORT = _DEFAULT_SSL_PORT
 154
 155   def __init__(self):
 156     self.log = _log
 157
 158   def setServerInfo(self, webServer = _DEFAULT_WEB_SERVER, httpPort = _DEFAULT_HTTP_PORT, sslPort = _DEFAULT_SSL_PORT):
 159     self.webServer = webServer
 160     self.httpPort = httpPort
 161     self.sslPort = sslPort
 162
 163   @property
 164   def __all__(self):
 165     return [
 166            "UNIXISH",
 167            "IS_WIN32",
 168            "IS_MAC",
 169            "log",
 170            "runApp",
 171            "Process",
 172            "addCommonOptions",
 173            "initializeProfile",
 174            "DIST_BIN",
 175            "DEFAULT_APP",
 176            "CERTS_SRC_DIR",
 177            "environment",
 178            "IS_TEST_BUILD",
 179            "IS_DEBUG_BUILD",
 180            "DEFAULT_TIMEOUT",
 181           ]
 182
 183   class Process(subprocess.Popen):
 184     """
 185     Represents our view of a subprocess.
 186     It adds a kill() method which allows it to be stopped explicitly.
 187     """
 188
 189     def __init__(self,
 190                  args,
 191                  bufsize=0,
 192                  executable=None,
 193                  stdin=None,
 194                  stdout=None,
 195                  stderr=None,
 196                  preexec_fn=None,
 197                  close_fds=False,
 198                  shell=False,
 199                  cwd=None,
 200                  env=None,
 201                  universal_newlines=False,
 202                  startupinfo=None,
 203                  creationflags=0):
 204       subprocess.Popen.__init__(self, args, bufsize, executable,
 205                                 stdin, stdout, stderr,
 206                                 preexec_fn, close_fds,
 207                                 shell, cwd, env,
 208                                 universal_newlines, startupinfo, creationflags)
 209       self.log = _log
 210
 211     def kill(self):
 212       if Automation().IS_WIN32:
 213         import platform
 214         pid = "%i" % self.pid
 215         if platform.release() == "2000":
 216           # Windows 2000 needs 'kill.exe' from the
 217           #'Windows 2000 Resource Kit tools'. (See bug 475455.)
 218           try:
 219             subprocess.Popen(["kill", "-f", pid]).wait()
 220           except:
 221             self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid)
 222         else:
 223           # Windows XP and later.
 224           subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()
 225       else:
 226         os.kill(self.pid, signal.SIGKILL)
 227
 228   def readLocations(self, locationsPath = "server-locations.txt"):
 229     """
 230     Reads the locations at which the Mochitest HTTP server is available from
 231     server-locations.txt.
 232     """
 233
 234     locationFile = codecs.open(locationsPath, "r", "UTF-8")
 235
 236     # Perhaps more detail than necessary, but it's the easiest way to make sure
 237     # we get exactly the format we want.  See server-locations.txt for the exact
 238     # format guaranteed here.
 239     lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"
 240                       r"://"
 241                       r"(?P<host>"
 242                         r"\d+\.\d+\.\d+\.\d+"
 243                         r"|"
 244                         r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*"
 245                         r"[a-z](?:[-a-z0-9]*[a-z0-9])?"
 246                       r")"
 247                       r":"
 248                       r"(?P<port>\d+)"
 249                       r"(?:"
 250                       r"\s+"
 251                       r"(?P<options>\S+(?:,\S+)*)"
 252                       r")?$")
 253     locations = []
 254     lineno = 0
 255     seenPrimary = False
 256     for line in locationFile:
 257       lineno += 1
 258       if line.startswith("#") or line == "\n":
 259         continue
 260
 261       match = lineRe.match(line)
 262       if not match:
 263         raise SyntaxError(lineno)
 264
 265       options = match.group("options")
 266       if options:
 267         options = options.split(",")
 268         if "primary" in options:
 269           if seenPrimary:
 270             raise SyntaxError(lineno, "multiple primary locations")
 271           seenPrimary = True
 272       else:
 273         options = []
 274
 275       locations.append(Location(match.group("scheme"), match.group("host"),
 276                                 match.group("port"), options))
 277
 278     if not seenPrimary:
 279       raise SyntaxError(lineno + 1, "missing primary location")
 280
 281     return locations
 282
 283   def initializeProfile(self, profileDir, extraPrefs = [], useServerLocations = False):
 284     " Sets up the standard testing profile."
 285
 286     prefs = []
 287     # Start with a clean slate.
 288     shutil.rmtree(profileDir, True)
 289     os.mkdir(profileDir)
 290
 291     part = """\
 292 user_pref("browser.dom.window.dump.enabled", true);
 293 user_pref("dom.allow_scripts_to_close_windows", true);
 294 user_pref("dom.disable_open_during_load", false);
 295 user_pref("dom.max_script_run_time", 0); // no slow script dialogs
 296 user_pref("dom.max_chrome_script_run_time", 0);
 297 user_pref("dom.popup_maximum", -1);
 298 user_pref("signed.applets.codebase_principal_support", true);
 299 user_pref("security.warn_submit_insecure", false);
 300 user_pref("browser.shell.checkDefaultBrowser", false);
 301 user_pref("shell.checkDefaultClient", false);
 302 user_pref("browser.warnOnQuit", false);
 303 user_pref("accessibility.typeaheadfind.autostart", false);
 304 user_pref("javascript.options.showInConsole", true);
 305 user_pref("layout.debug.enable_data_xbl", true);
 306 user_pref("browser.EULA.override", true);
 307 user_pref("javascript.options.jit.content", true);
 308 user_pref("gfx.color_management.force_srgb", true);
 309 user_pref("network.manage-offline-status", false);
 310 user_pref("test.mousescroll", true);
 311 user_pref("security.default_personal_cert", "Select Automatically"); // Need to client auth test be w/o any dialogs
 312 user_pref("network.http.prompt-temp-redirect", false);
 313 user_pref("media.cache_size", 100);
 314 user_pref("security.warn_viewing_mixed", false);
 315
 316 user_pref("geo.wifi.uri", "http://%(server)s/tests/dom/tests/mochitest/geolocation/network_geolocation.sjs");
 317 user_pref("geo.wifi.testing", true);
 318
 319 user_pref("camino.warn_when_closing", false); // Camino-only, harmless to others
 320
 321 // Make url-classifier updates so rare that they won't affect tests
 322 user_pref("urlclassifier.updateinterval", 172800);
 323 // Point the url-classifier to the local testing server for fast failures
 324 user_pref("browser.safebrowsing.provider.0.gethashURL", "http://%(server)s/safebrowsing-dummy/gethash");
 325 user_pref("browser.safebrowsing.provider.0.keyURL", "http://%(server)s/safebrowsing-dummy/newkey");
 326 user_pref("browser.safebrowsing.provider.0.updateURL", "http://%(server)s/safebrowsing-dummy/update");
 327 """ % { "server" : self.webServer + ":" + str(self.httpPort) }
 328     prefs.append(part)
 329
 330     if useServerLocations == False:
 331       part = """
 332 user_pref("capability.principal.codebase.p1.granted",
 333           "UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite \
 334            UniversalPreferencesRead UniversalPreferencesWrite \
 335            UniversalFileRead");
 336 user_pref("capability.principal.codebase.p1.id", "%(origin)s");
 337 user_pref("capability.principal.codebase.p1.subjectName", "");
 338 """  % { "origin": "http://" + self.webServer + ":" + str(self.httpPort) }
 339       prefs.append(part)
 340     else:
 341       locations = self.readLocations()
 342
 343       # Grant God-power to all the privileged servers on which tests run.
 344       privileged = filter(lambda loc: "privileged" in loc.options, locations)
 345       for (i, l) in itertools.izip(itertools.count(1), privileged):
 346         part = """
 347 user_pref("capability.principal.codebase.p%(i)d.granted",
 348           "UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite \
 349            UniversalPreferencesRead UniversalPreferencesWrite \
 350            UniversalFileRead");
 351 user_pref("capability.principal.codebase.p%(i)d.id", "%(origin)s");
 352 user_pref("capability.principal.codebase.p%(i)d.subjectName", "");
 353 """  % { "i": i,
 354          "origin": (l.scheme + "://" + l.host + ":" + str(l.port)) }
 355         prefs.append(part)
 356
 357       # We need to proxy every server but the primary one.
 358       origins = ["'%s://%s:%s'" % (l.scheme, l.host, l.port)
 359                 for l in filter(lambda l: "primary" not in l.options, locations)]
 360       origins = ", ".join(origins)
 361
 362       pacURL = """data:text/plain,
 363 function FindProxyForURL(url, host)
 364 {
 365   var origins = [%(origins)s];
 366   var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
 367                          '://' +
 368                          '(?:[^/@]*@)?' +
 369                          '(.*?)' +
 370                          '(?::(\\\\\\\\d+))?/');
 371   var matches = regex.exec(url);
 372   if (!matches)
 373     return 'DIRECT';
 374   var isHttp = matches[1] == 'http';
 375   var isHttps = matches[1] == 'https';
 376   if (!matches[3])
 377   {
 378     if (isHttp) matches[3] = '80';
 379     if (isHttps) matches[3] = '443';
 380   }
 381
 382   var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
 383   if (origins.indexOf(origin) < 0)
 384     return 'DIRECT';
 385   if (isHttp)
 386     return 'PROXY %(remote)s:%(httpport)s';
 387   if (isHttps)
 388     return 'PROXY %(remote)s:%(sslport)s';
 389   return 'DIRECT';
 390 }""" % { "origins": origins,
 391          "remote":  self.webServer,
 392          "httpport":self.httpPort,
 393          "sslport": self.sslPort }
 394       pacURL = "".join(pacURL.splitlines())
 395
 396       part += """
 397 user_pref("network.proxy.type", 2);
 398 user_pref("network.proxy.autoconfig_url", "%(pacURL)s");
 399
 400 user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless to others
 401 """ % {"pacURL": pacURL}
 402       prefs.append(part)
 403
 404     for v in extraPrefs:
 405       thispref = v.split("=")
 406       if len(thispref) < 2:
 407         print "Error: syntax error in --setpref=" + v
 408         sys.exit(1)
 409       part = 'user_pref("%s", %s);\n' % (thispref[0], thispref[1])
 410       prefs.append(part)
 411
 412     # write the preferences
 413     prefsFile = open(profileDir + "/" + "user.js", "a")
 414     prefsFile.write("".join(prefs))
 415     prefsFile.close()
 416
 417   def addCommonOptions(self, parser):
 418     "Adds command-line options which are common to mochitest and reftest."
 419
 420     parser.add_option("--setpref",
 421                       action = "append", type = "string",
 422                       default = [],
 423                       dest = "extraPrefs", metavar = "PREF=VALUE",
 424                       help = "defines an extra user preference")
 425
 426   def fillCertificateDB(self, profileDir, certPath, utilityPath, xrePath):
 427     pwfilePath = os.path.join(profileDir, ".crtdbpw")
 428
 429     pwfile = open(pwfilePath, "w")
 430     pwfile.write("\n")
 431     pwfile.close()
 432
 433     # Create head of the ssltunnel configuration file
 434     sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")
 435     sslTunnelConfig = open(sslTunnelConfigPath, "w")
 436
 437     sslTunnelConfig.write("httpproxy:1\n")
 438     sslTunnelConfig.write("certdbdir:%s\n" % certPath)
 439     sslTunnelConfig.write("forward:127.0.0.1:%s\n" % self.httpPort)
 440     sslTunnelConfig.write("listen:*:%s:pgo server certificate\n" % self.sslPort)
 441
 442     # Configure automatic certificate and bind custom certificates, client authentication
 443     locations = self.readLocations()
 444     locations.pop(0)
 445     for loc in locations:
 446       if loc.scheme == "https" and "nocert" not in loc.options:
 447         customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")
 448         clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")
 449         for option in loc.options:
 450           match = customCertRE.match(option)
 451           if match:
 452             customcert = match.group("nickname");
 453             sslTunnelConfig.write("listen:%s:%s:%s:%s\n" %
 454                       (loc.host, loc.port, self.sslPort, customcert))
 455
 456           match = clientAuthRE.match(option)
 457           if match:
 458             clientauth = match.group("clientauth");
 459             sslTunnelConfig.write("clientauth:%s:%s:%s:%s\n" %
 460                       (loc.host, loc.port, self.sslPort, clientauth))
 461
 462     sslTunnelConfig.close()
 463
 464     # Pre-create the certification database for the profile
 465     env = self.environment(xrePath = xrePath)
 466     certutil = os.path.join(utilityPath, "certutil" + self.BIN_SUFFIX)
 467     pk12util = os.path.join(utilityPath, "pk12util" + self.BIN_SUFFIX)
 468
 469     status = self.Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait()
 470     if status != 0:
 471       return status
 472
 473     # Walk the cert directory and add custom CAs and client certs
 474     files = os.listdir(certPath)
 475     for item in files:
 476       root, ext = os.path.splitext(item)
 477       if ext == ".ca":
 478         trustBits = "CT,,"
 479         if root.endswith("-object"):
 480           trustBits = "CT,,CT"
 481         self.Process([certutil, "-A", "-i", os.path.join(certPath, item),
 482                     "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits],
 483                     env = env).wait()
 484       if ext == ".client":
 485         self.Process([pk12util, "-i", os.path.join(certPath, item), "-w",
 486                     pwfilePath, "-d", profileDir],
 487                     env = env).wait()
 488
 489     os.unlink(pwfilePath)
 490     return 0
 491
 492   def environment(self, env = None, xrePath = None, crashreporter = True):
 493     if xrePath == None:
 494       xrePath = self.DIST_BIN
 495     if env == None:
 496       env = dict(os.environ)
 497
 498     ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath))
 499     if self.UNIXISH or self.IS_MAC:
 500       envVar = "LD_LIBRARY_PATH"
 501       if self.IS_MAC:
 502         envVar = "DYLD_LIBRARY_PATH"
 503       else: # unixish
 504         env['MOZILLA_FIVE_HOME'] = xrePath
 505       if envVar in env:
 506         ldLibraryPath = ldLibraryPath + ":" + env[envVar]
 507       env[envVar] = ldLibraryPath
 508     elif self.IS_WIN32:
 509       env["PATH"] = env["PATH"] + ";" + ldLibraryPath
 510
 511     if crashreporter:
 512       env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
 513       env['MOZ_CRASHREPORTER'] = '1'
 514     else:
 515       env['MOZ_CRASHREPORTER_DISABLE'] = '1'
 516
 517     env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
 518     env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
 519     return env
 520
 521   if IS_WIN32:
 522     PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
 523     GetLastError = ctypes.windll.kernel32.GetLastError
 524
 525     def readWithTimeout(self, f, timeout):
 526       """Try to read a line of output from the file object |f|.
 527       |f| must be a  pipe, like the |stdout| member of a subprocess.Popen
 528       object created with stdout=PIPE. If no output
 529       is received within |timeout| seconds, return a blank line.
 530       Returns a tuple (line, did_timeout), where |did_timeout| is True
 531       if the read timed out, and False otherwise."""
 532       if timeout is None:
 533         # shortcut to allow callers to pass in "None" for no timeout.
 534         return (f.readline(), False)
 535       x = msvcrt.get_osfhandle(f.fileno())
 536       l = ctypes.c_long()
 537       done = time.time() + timeout
 538       while time.time() < done:
 539         if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0:
 540           err = self.GetLastError()
 541           if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE
 542             return ('', False)
 543           else:
 544             log.error("readWithTimeout got error: %d", err)
 545         if l.value > 0:
 546           # we're assuming that the output is line-buffered,
 547           # which is not unreasonable
 548           return (f.readline(), False)
 549         time.sleep(0.01)
 550       return ('', True)
 551
 552     def isPidAlive(self, pid):
 553       STILL_ACTIVE = 259
 554       PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
 555       pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)
 556       if not pHandle:
 557         return False
 558       pExitCode = ctypes.wintypes.DWORD()
 559       ctypes.windll.kernel32.GetExitCodeProcess(pHandle, self.ctypes.byref(pExitCode))
 560       ctypes.windll.kernel32.CloseHandle(pHandle)
 561       if (pExitCode.value == STILL_ACTIVE):
 562         return True
 563       else:
 564         return False
 565
 566     def killPid(self, pid):
 567       PROCESS_TERMINATE = 0x0001
 568       pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, 0, pid)
 569       if not pHandle:
 570         return
 571       success = ctypes.windll.kernel32.TerminateProcess(pHandle, 1)
 572       ctypes.windll.kernel32.CloseHandle(pHandle)
 573
 574   else:
 575
 576     def readWithTimeout(self, f, timeout):
 577       """Try to read a line of output from the file object |f|. If no output
 578       is received within |timeout| seconds, return a blank line.
 579       Returns a tuple (line, did_timeout), where |did_timeout| is True
 580       if the read timed out, and False otherwise."""
 581       (r, w, e) = select.select([f], [], [], timeout)
 582       if len(r) == 0:
 583         return ('', True)
 584       return (f.readline(), False)
 585
 586     def isPidAlive(self, pid):
 587       try:
 588         # kill(pid, 0) checks for a valid PID without actually sending a signal
 589         # The method throws OSError if the PID is invalid, which we catch below.
 590         os.kill(pid, 0)
 591
 592         # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if
 593         # the process terminates before we get to this point.
 594         wpid, wstatus = os.waitpid(pid, os.WNOHANG)
 595         if wpid == 0:
 596           return True
 597
 598         return False
 599       except OSError, err:
 600         # Catch the errors we might expect from os.kill/os.waitpid,
 601         # and re-raise any others
 602         if err.errno == errno.ESRCH or err.errno == errno.ECHILD:
 603           return False
 604         raise
 605
 606     def killPid(self, pid):
 607       os.kill(pid, signal.SIGKILL)
 608
 609   def killAndGetStack(self, proc, utilityPath, debuggerInfo):
 610     """Kill the process, preferrably in a way that gets us a stack trace."""
 611     if self.CRASHREPORTER and not debuggerInfo:
 612       if self.UNIXISH:
 613         # ABRT will get picked up by Breakpad's signal handler
 614         os.kill(proc.pid, signal.SIGABRT)
 615         return
 616       elif self.IS_WIN32:
 617         # We should have a "crashinject" program in our utility path
 618         crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe"))
 619         if os.path.exists(crashinject) and subprocess.Popen([crashinject, str(proc.pid)]).wait() == 0:
 620           return
 621       #TODO: kill the process such that it triggers Breakpad on OS X (bug 525296)
 622     self.log.info("Can't trigger Breakpad, just killing process")
 623     proc.kill()
 624
 625   def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo):
 626     """ Look for timeout or crashes and return the status after the process terminates """
 627     stackFixerProcess = None
 628     stackFixerModule = None
 629     didTimeout = False
 630     if proc.stdout is None:
 631       self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection")
 632     else:
 633       logsource = proc.stdout
 634       if self.IS_DEBUG_BUILD and self.IS_LINUX:
 635         # Run logsource through fix-linux-stack.pl
 636         stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, "fix-linux-stack.pl")],
 637                                          stdin=logsource,
 638                                          stdout=subprocess.PIPE)
 639         logsource = stackFixerProcess.stdout
 640
 641       if self.IS_DEBUG_BUILD and self.IS_MAC:
 642         # Import fix_macosx_stack.py from utilityPath
 643         sys.path.insert(0, utilityPath)
 644         import fix_macosx_stack as stackFixerModule
 645         del sys.path[0]
 646
 647       (line, didTimeout) = self.readWithTimeout(logsource, timeout)
 648       hitMaxTime = False
 649       while line != "" and not didTimeout:
 650         if stackFixerModule:
 651           line = stackFixerModule.fixSymbols(line)
 652         self.log.info(line.rstrip())
 653         (line, didTimeout) = self.readWithTimeout(logsource, timeout)
 654         if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime):
 655           # Kill the application, but continue reading from stack fixer so as not to deadlock on stackFixerProcess.wait().
 656           hitMaxTime = True
 657           self.log.info("TEST-UNEXPECTED-FAIL | automation.py | application ran for longer than allowed maximum time of %d seconds", int(maxTime))
 658           self.killAndGetStack(proc, utilityPath, debuggerInfo)
 659       if didTimeout:
 660         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | application timed out after %d seconds with no output", int(timeout))
 661         self.killAndGetStack(proc, utilityPath, debuggerInfo)
 662
 663     status = proc.wait()
 664     if status != 0 and not didTimeout and not hitMaxTime:
 665       self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Exited with code %d during test run", status)
 666     if stackFixerProcess is not None:
 667       fixerStatus = stackFixerProcess.wait()
 668       if fixerStatus != 0 and not didTimeout and not hitMaxTime:
 669         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Stack fixer process exited with code %d during test run", fixerStatus)
 670     return status
 671
 672   def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
 673     """ build the application command line """
 674
 675     cmd = app
 676     if self.IS_MAC and not self.IS_CAMINO and not cmd.endswith("-bin"):
 677       cmd += "-bin"
 678     cmd = os.path.abspath(cmd)
 679
 680     args = []
 681
 682     if debuggerInfo:
 683       args.extend(debuggerInfo["args"])
 684       args.append(cmd)
 685       cmd = os.path.abspath(debuggerInfo["path"])
 686
 687     if self.IS_MAC:
 688       args.append("-foreground")
 689
 690     if self.IS_CYGWIN:
 691       profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
 692     else:
 693       profileDirectory = profileDir + "/"
 694
 695     args.extend(("-no-remote", "-profile", profileDirectory))
 696     if testURL is not None:
 697       if self.IS_CAMINO:
 698         args.extend(("-url", testURL))
 699       else:
 700         args.append((testURL))
 701     args.extend(extraArgs)
 702     return cmd, args
 703
 704   def checkForZombies(self, processLog):
 705     """ Look for hung processes """
 706     if not os.path.exists(processLog):
 707       self.log.info('INFO | automation.py | PID log not found: %s', processLog)
 708     else:
 709       self.log.info('INFO | automation.py | Reading PID log: %s', processLog)
 710       processList = []
 711       pidRE = re.compile(r'launched child process (\d+)$')
 712       processLogFD = open(processLog)
 713       for line in processLogFD:
 714         self.log.info(line.rstrip())
 715         m = pidRE.search(line)
 716         if m:
 717           processList.append(int(m.group(1)))
 718       processLogFD.close()
 719
 720       for processPID in processList:
 721         self.log.info("INFO | automation.py | Checking for orphan process with PID: %d", processPID)
 722         if self.isPidAlive(processPID):
 723           self.log.info("TEST-UNEXPECTED-FAIL | automation.py | child process %d still alive after shutdown", processPID)
 724           self.killPid(processPID)
 725
 726   def runApp(self, testURL, env, app, profileDir, extraArgs,
 727              runSSLTunnel = False, utilityPath = None,
 728              xrePath = None, certPath = None,
 729              debuggerInfo = None, symbolsPath = None,
 730              timeout = -1, maxTime = None):
 731     """
 732     Run the app, log the duration it took to execute, return the status code.
 733     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
 734     """
 735
 736     if utilityPath == None:
 737       utilityPath = self.DIST_BIN
 738     if xrePath == None:
 739       xrePath = self.DIST_BIN
 740     if certPath == None:
 741       certPath = self.CERTS_SRC_DIR
 742     if timeout == -1:
 743       timeout = self.DEFAULT_TIMEOUT
 744
 745     # copy env so we don't munge the caller's environment
 746     env = dict(env);
 747     env["NO_EM_RESTART"] = "1"
 748     tmpfd, processLog = tempfile.mkstemp(suffix='pidlog')
 749     os.close(tmpfd)
 750     env["MOZ_PROCESS_LOG"] = processLog
 751
 752     if self.IS_TEST_BUILD and runSSLTunnel:
 753       # create certificate database for the profile
 754       certificateStatus = self.fillCertificateDB(profileDir, certPath, utilityPath, xrePath)
 755       if certificateStatus != 0:
 756         self.log.info("TEST-UNEXPECTED FAIL | automation.py | Certificate integration failed")
 757         return certificateStatus
 758
 759       # start ssltunnel to provide https:// URLs capability
 760       ssltunnel = os.path.join(utilityPath, "ssltunnel" + self.BIN_SUFFIX)
 761       ssltunnelProcess = self.Process([ssltunnel,
 762                                os.path.join(profileDir, "ssltunnel.cfg")],
 763                                env = self.environment(xrePath = xrePath))
 764       self.log.info("INFO | automation.py | SSL tunnel pid: %d", ssltunnelProcess.pid)
 765
 766     cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs)
 767     startTime = datetime.now()
 768
 769     if debuggerInfo and debuggerInfo["interactive"]:
 770       # If an interactive debugger is attached, don't redirect output
 771       # and don't use timeouts.
 772       timeout = None
 773       maxTime = None
 774       outputPipe = None
 775     else:
 776       outputPipe = subprocess.PIPE
 777
 778     proc = self.Process([cmd] + args,
 779                  env = self.environment(env, xrePath = xrePath,
 780                                    crashreporter = not debuggerInfo),
 781                  stdout = outputPipe,
 782                  stderr = subprocess.STDOUT)
 783     self.log.info("INFO | automation.py | Application pid: %d", proc.pid)
 784
 785     status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo)
 786     self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime))
 787
 788     # Do a final check for zombie child processes.
 789     self.checkForZombies(processLog)
 790     automationutils.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath)
 791
 792     if os.path.exists(processLog):
 793       os.unlink(processLog)
 794
 795     if self.IS_TEST_BUILD and runSSLTunnel:
 796       ssltunnelProcess.kill()
 797
 798     return status