build/automation.py.in

   1 #
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   4 #
   5 # The contents of this file are subject to the Mozilla Public License Version
   6 # 1.1 (the "License"); you may not use this file except in compliance with
   7 # the License. You may obtain a copy of the License at
   8 # http://www.mozilla.org/MPL/
   9 #
  10 # Software distributed under the License is distributed on an "AS IS" basis,
  11 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12 # for the specific language governing rights and limitations under the
  13 # License.
  14 #
  15 # The Original Code is mozilla.org code.
  16 #
  17 # The Initial Developer of the Original Code is
  18 # Mozilla Foundation.
  19 # Portions created by the Initial Developer are Copyright (C) 2008
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #   Robert Sayre <sayrer@gmail.com>
  24 #   Jeff Walden <jwalden+bmo@mit.edu>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 from __future__ import with_statement
  41 import codecs
  42 from datetime import datetime, timedelta
  43 import itertools
  44 import logging
  45 import os
  46 import re
  47 import select
  48 import shutil
  49 import signal
  50 import subprocess
  51 import sys
  52 import threading
  53 import tempfile
  54 import sqlite3
  55
  56 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
  57 sys.path.insert(0, SCRIPT_DIR)
  58 import automationutils
  59
  60 _DEFAULT_WEB_SERVER = "127.0.0.1"
  61 _DEFAULT_HTTP_PORT = 8888
  62 _DEFAULT_SSL_PORT = 4443
  63 _DEFAULT_WEBSOCKET_PORT = 9988
  64
  65 #expand _DIST_BIN = __XPC_BIN_PATH__
  66 #expand _IS_WIN32 = len("__WIN32__") != 0
  67 #expand _IS_MAC = __IS_MAC__ != 0
  68 #expand _IS_LINUX = __IS_LINUX__ != 0
  69 #ifdef IS_CYGWIN
  70 #expand _IS_CYGWIN = __IS_CYGWIN__ == 1
  71 #else
  72 _IS_CYGWIN = False
  73 #endif
  74 #expand _IS_CAMINO = __IS_CAMINO__ != 0
  75 #expand _BIN_SUFFIX = __BIN_SUFFIX__
  76 #expand _PERL = __PERL__
  77
  78 #expand _DEFAULT_APP = "./" + __BROWSER_PATH__
  79 #expand _CERTS_SRC_DIR = __CERTS_SRC_DIR__
  80 #expand _IS_TEST_BUILD = __IS_TEST_BUILD__
  81 #expand _IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
  82 #expand _CRASHREPORTER = __CRASHREPORTER__ == 1
  83
  84
  85 if _IS_WIN32:
  86   import ctypes, ctypes.wintypes, time, msvcrt
  87 else:
  88   import errno
  89
  90
  91 # We use the logging system here primarily because it'll handle multiple
  92 # threads, which is needed to process the output of the server and application
  93 # processes simultaneously.
  94 _log = logging.getLogger()
  95 handler = logging.StreamHandler(sys.stdout)
  96 _log.setLevel(logging.INFO)
  97 _log.addHandler(handler)
  98
  99
 100 #################
 101 # PROFILE SETUP #
 102 #################
 103
 104 class SyntaxError(Exception):
 105   "Signifies a syntax error on a particular line in server-locations.txt."
 106
 107   def __init__(self, lineno, msg = None):
 108     self.lineno = lineno
 109     self.msg = msg
 110
 111   def __str__(self):
 112     s = "Syntax error on line " + str(self.lineno)
 113     if self.msg:
 114       s += ": %s." % self.msg
 115     else:
 116       s += "."
 117     return s
 118
 119
 120 class Location:
 121   "Represents a location line in server-locations.txt."
 122
 123   def __init__(self, scheme, host, port, options):
 124     self.scheme = scheme
 125     self.host = host
 126     self.port = port
 127     self.options = options
 128
 129 class Automation(object):
 130   """
 131   Runs the browser from a script, and provides useful utilities
 132   for setting up the browser environment.
 133   """
 134
 135   DIST_BIN = _DIST_BIN
 136   IS_WIN32 = _IS_WIN32
 137   IS_MAC = _IS_MAC
 138   IS_LINUX = _IS_LINUX
 139   IS_CYGWIN = _IS_CYGWIN
 140   IS_CAMINO = _IS_CAMINO
 141   BIN_SUFFIX = _BIN_SUFFIX
 142   PERL = _PERL
 143
 144   UNIXISH = not IS_WIN32 and not IS_MAC
 145
 146   DEFAULT_APP = _DEFAULT_APP
 147   CERTS_SRC_DIR = _CERTS_SRC_DIR
 148   IS_TEST_BUILD = _IS_TEST_BUILD
 149   IS_DEBUG_BUILD = _IS_DEBUG_BUILD
 150   CRASHREPORTER = _CRASHREPORTER
 151
 152   # timeout, in seconds
 153   DEFAULT_TIMEOUT = 60.0
 154   DEFAULT_WEB_SERVER = _DEFAULT_WEB_SERVER
 155   DEFAULT_HTTP_PORT = _DEFAULT_HTTP_PORT
 156   DEFAULT_SSL_PORT = _DEFAULT_SSL_PORT
 157   DEFAULT_WEBSOCKET_PORT = _DEFAULT_WEBSOCKET_PORT
 158
 159   def __init__(self):
 160     self.log = _log
 161     self.lastTestSeen = "automation.py"
 162     self.haveDumpedScreen = False
 163
 164   def setServerInfo(self,
 165                     webServer = _DEFAULT_WEB_SERVER,
 166                     httpPort = _DEFAULT_HTTP_PORT,
 167                     sslPort = _DEFAULT_SSL_PORT,
 168                     webSocketPort = _DEFAULT_WEBSOCKET_PORT):
 169     self.webServer = webServer
 170     self.httpPort = httpPort
 171     self.sslPort = sslPort
 172     self.webSocketPort = webSocketPort
 173
 174   @property
 175   def __all__(self):
 176     return [
 177            "UNIXISH",
 178            "IS_WIN32",
 179            "IS_MAC",
 180            "log",
 181            "runApp",
 182            "Process",
 183            "addCommonOptions",
 184            "initializeProfile",
 185            "DIST_BIN",
 186            "DEFAULT_APP",
 187            "CERTS_SRC_DIR",
 188            "environment",
 189            "IS_TEST_BUILD",
 190            "IS_DEBUG_BUILD",
 191            "DEFAULT_TIMEOUT",
 192           ]
 193
 194   class Process(subprocess.Popen):
 195     """
 196     Represents our view of a subprocess.
 197     It adds a kill() method which allows it to be stopped explicitly.
 198     """
 199
 200     def __init__(self,
 201                  args,
 202                  bufsize=0,
 203                  executable=None,
 204                  stdin=None,
 205                  stdout=None,
 206                  stderr=None,
 207                  preexec_fn=None,
 208                  close_fds=False,
 209                  shell=False,
 210                  cwd=None,
 211                  env=None,
 212                  universal_newlines=False,
 213                  startupinfo=None,
 214                  creationflags=0):
 215       args = automationutils.wrapCommand(args)
 216       print "args: %s" % args
 217       subprocess.Popen.__init__(self, args, bufsize, executable,
 218                                 stdin, stdout, stderr,
 219                                 preexec_fn, close_fds,
 220                                 shell, cwd, env,
 221                                 universal_newlines, startupinfo, creationflags)
 222       self.log = _log
 223
 224     def kill(self):
 225       if Automation().IS_WIN32:
 226         import platform
 227         pid = "%i" % self.pid
 228         if platform.release() == "2000":
 229           # Windows 2000 needs 'kill.exe' from the
 230           #'Windows 2000 Resource Kit tools'. (See bug 475455.)
 231           try:
 232             subprocess.Popen(["kill", "-f", pid]).wait()
 233           except:
 234             self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid)
 235         else:
 236           # Windows XP and later.
 237           subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()
 238       else:
 239         os.kill(self.pid, signal.SIGKILL)
 240
 241   def readLocations(self, locationsPath = "server-locations.txt"):
 242     """
 243     Reads the locations at which the Mochitest HTTP server is available from
 244     server-locations.txt.
 245     """
 246
 247     locationFile = codecs.open(locationsPath, "r", "UTF-8")
 248
 249     # Perhaps more detail than necessary, but it's the easiest way to make sure
 250     # we get exactly the format we want.  See server-locations.txt for the exact
 251     # format guaranteed here.
 252     lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"
 253                       r"://"
 254                       r"(?P<host>"
 255                         r"\d+\.\d+\.\d+\.\d+"
 256                         r"|"
 257                         r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*"
 258                         r"[a-z](?:[-a-z0-9]*[a-z0-9])?"
 259                       r")"
 260                       r":"
 261                       r"(?P<port>\d+)"
 262                       r"(?:"
 263                       r"\s+"
 264                       r"(?P<options>\S+(?:,\S+)*)"
 265                       r")?$")
 266     locations = []
 267     lineno = 0
 268     seenPrimary = False
 269     for line in locationFile:
 270       lineno += 1
 271       if line.startswith("#") or line == "\n":
 272         continue
 273
 274       match = lineRe.match(line)
 275       if not match:
 276         raise SyntaxError(lineno)
 277
 278       options = match.group("options")
 279       if options:
 280         options = options.split(",")
 281         if "primary" in options:
 282           if seenPrimary:
 283             raise SyntaxError(lineno, "multiple primary locations")
 284           seenPrimary = True
 285       else:
 286         options = []
 287
 288       locations.append(Location(match.group("scheme"), match.group("host"),
 289                                 match.group("port"), options))
 290
 291     if not seenPrimary:
 292       raise SyntaxError(lineno + 1, "missing primary location")
 293
 294     return locations
 295
 296   def setupPermissionsDatabase(self, profileDir, permissions):
 297     # Open database and create table
 298     permDB = sqlite3.connect(os.path.join(profileDir, "permissions.sqlite"))
 299     cursor = permDB.cursor();
 300     # SQL copied from nsPermissionManager.cpp
 301     cursor.execute("""CREATE TABLE moz_hosts (
 302        id INTEGER PRIMARY KEY,
 303        host TEXT,
 304        type TEXT,
 305        permission INTEGER,
 306        expireType INTEGER,
 307        expireTime INTEGER)""")
 308
 309     # Insert desired permissions
 310     c = 0
 311     for perm in permissions.keys():
 312       for host,allow in permissions[perm]:
 313         c += 1
 314         cursor.execute("INSERT INTO moz_hosts values(?, ?, ?, ?, 0, 0)",
 315                        (c, host, perm, 1 if allow else 2))
 316
 317     # Commit and close
 318     permDB.commit()
 319     cursor.close()
 320
 321   def initializeProfile(self, profileDir, extraPrefs = [], useServerLocations = False):
 322     " Sets up the standard testing profile."
 323
 324     prefs = []
 325     # Start with a clean slate.
 326     shutil.rmtree(profileDir, True)
 327     os.mkdir(profileDir)
 328
 329     # Set up permissions database
 330     locations = self.readLocations()
 331     self.setupPermissionsDatabase(profileDir,
 332       {'allowXULXBL':[(l.host, 'noxul' not in l.options) for l in locations]});
 333
 334     part = """\
 335 user_pref("browser.console.showInPanel", true);
 336 user_pref("browser.dom.window.dump.enabled", true);
 337 user_pref("browser.firstrun.show.localepicker", false);
 338 user_pref("browser.firstrun.show.uidiscovery", false);
 339 user_pref("browser.ui.layout.tablet", 0); // force tablet UI off
 340 user_pref("dom.allow_scripts_to_close_windows", true);
 341 user_pref("dom.disable_open_during_load", false);
 342 user_pref("dom.max_script_run_time", 0); // no slow script dialogs
 343 user_pref("hangmonitor.timeout", 0); // no hang monitor
 344 user_pref("dom.max_chrome_script_run_time", 0);
 345 user_pref("dom.popup_maximum", -1);
 346 user_pref("dom.send_after_paint_to_content", true);
 347 user_pref("dom.successive_dialog_time_limit", 0);
 348 user_pref("signed.applets.codebase_principal_support", true);
 349 user_pref("security.warn_submit_insecure", false);
 350 user_pref("browser.shell.checkDefaultBrowser", false);
 351 user_pref("shell.checkDefaultClient", false);
 352 user_pref("browser.warnOnQuit", false);
 353 user_pref("accessibility.typeaheadfind.autostart", false);
 354 user_pref("javascript.options.showInConsole", true);
 355 user_pref("devtools.errorconsole.enabled", true);
 356 user_pref("layout.debug.enable_data_xbl", true);
 357 user_pref("browser.EULA.override", true);
 358 user_pref("javascript.options.jit_hardening", true);
 359 user_pref("gfx.color_management.force_srgb", true);
 360 user_pref("network.manage-offline-status", false);
 361 user_pref("test.mousescroll", true);
 362 user_pref("security.default_personal_cert", "Select Automatically"); // Need to client auth test be w/o any dialogs
 363 user_pref("network.http.prompt-temp-redirect", false);
 364 user_pref("media.cache_size", 100);
 365 user_pref("security.warn_viewing_mixed", false);
 366 user_pref("app.update.enabled", false);
 367 user_pref("browser.panorama.experienced_first_run", true); // Assume experienced
 368 user_pref("dom.w3c_touch_events.enabled", true);
 369 user_pref("toolkit.telemetry.prompted", 2);
 370 // Existing tests assume there is no font size inflation.
 371 user_pref("font.size.inflation.emPerLine", 0);
 372 user_pref("font.size.inflation.minTwips", 0);
 373
 374 // Only load extensions from the application and user profile
 375 // AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_APPLICATION
 376 user_pref("extensions.enabledScopes", 5);
 377 // Disable metadata caching for installed add-ons by default
 378 user_pref("extensions.getAddons.cache.enabled", false);
 379 // Disable intalling any distribution add-ons
 380 user_pref("extensions.installDistroAddons", false);
 381
 382 user_pref("extensions.testpilot.runStudies", false);
 383
 384 user_pref("geo.wifi.uri", "http://%(server)s/tests/dom/tests/mochitest/geolocation/network_geolocation.sjs");
 385 user_pref("geo.wifi.testing", true);
 386 user_pref("geo.ignore.location_filter", true);
 387
 388 user_pref("camino.warn_when_closing", false); // Camino-only, harmless to others
 389
 390 // Make url-classifier updates so rare that they won't affect tests
 391 user_pref("urlclassifier.updateinterval", 172800);
 392 // Point the url-classifier to the local testing server for fast failures
 393 user_pref("browser.safebrowsing.provider.0.gethashURL", "http://%(server)s/safebrowsing-dummy/gethash");
 394 user_pref("browser.safebrowsing.provider.0.keyURL", "http://%(server)s/safebrowsing-dummy/newkey");
 395 user_pref("browser.safebrowsing.provider.0.updateURL", "http://%(server)s/safebrowsing-dummy/update");
 396 // Point update checks to the local testing server for fast failures
 397 user_pref("extensions.update.url", "http://%(server)s/extensions-dummy/updateURL");
 398 user_pref("extensions.update.background.url", "http://%(server)s/extensions-dummy/updateBackgroundURL");
 399 user_pref("extensions.blocklist.url", "http://%(server)s/extensions-dummy/blocklistURL");
 400 user_pref("extensions.hotfix.url", "http://%(server)s/extensions-dummy/hotfixURL");
 401 // Make sure opening about:addons won't hit the network
 402 user_pref("extensions.webservice.discoverURL", "http://%(server)s/extensions-dummy/discoveryURL");
 403 // Make sure AddonRepository won't hit the network
 404 user_pref("extensions.getAddons.maxResults", 0);
 405 user_pref("extensions.getAddons.get.url", "http://%(server)s/extensions-dummy/repositoryGetURL");
 406 user_pref("extensions.getAddons.getWithPerformance.url", "http://%(server)s/extensions-dummy/repositoryGetWithPerformanceURL");
 407 user_pref("extensions.getAddons.search.browseURL", "http://%(server)s/extensions-dummy/repositoryBrowseURL");
 408 user_pref("extensions.getAddons.search.url", "http://%(server)s/extensions-dummy/repositorySearchURL");
 409 """ % { "server" : self.webServer + ":" + str(self.httpPort) }
 410     prefs.append(part)
 411
 412     if useServerLocations == False:
 413       part = """
 414 user_pref("capability.principal.codebase.p1.granted", "UniversalXPConnect");
 415 user_pref("capability.principal.codebase.p1.id", "%(origin)s");
 416 user_pref("capability.principal.codebase.p1.subjectName", "");
 417 """  % { "origin": "http://" + self.webServer + ":" + str(self.httpPort) }
 418       prefs.append(part)
 419     else:
 420       # Grant God-power to all the privileged servers on which tests run.
 421       privileged = filter(lambda loc: "privileged" in loc.options, locations)
 422       for (i, l) in itertools.izip(itertools.count(1), privileged):
 423         part = """
 424 user_pref("capability.principal.codebase.p%(i)d.granted", "UniversalXPConnect");
 425 user_pref("capability.principal.codebase.p%(i)d.id", "%(origin)s");
 426 user_pref("capability.principal.codebase.p%(i)d.subjectName", "");
 427 """  % { "i": i,
 428          "origin": (l.scheme + "://" + l.host + ":" + str(l.port)) }
 429         prefs.append(part)
 430
 431       # We need to proxy every server but the primary one.
 432       origins = ["'%s://%s:%s'" % (l.scheme, l.host, l.port)
 433                 for l in filter(lambda l: "primary" not in l.options, locations)]
 434       origins = ", ".join(origins)
 435
 436       pacURL = """data:text/plain,
 437 function FindProxyForURL(url, host)
 438 {
 439   var origins = [%(origins)s];
 440   var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
 441                          '://' +
 442                          '(?:[^/@]*@)?' +
 443                          '(.*?)' +
 444                          '(?::(\\\\\\\\d+))?/');
 445   var matches = regex.exec(url);
 446   if (!matches)
 447     return 'DIRECT';
 448   var isHttp = matches[1] == 'http';
 449   var isHttps = matches[1] == 'https';
 450   var isWebSocket = matches[1] == 'ws';
 451   var isWebSocketSSL = matches[1] == 'wss';
 452   if (!matches[3])
 453   {
 454     if (isHttp | isWebSocket) matches[3] = '80';
 455     if (isHttps | isWebSocketSSL) matches[3] = '443';
 456   }
 457   if (isWebSocket)
 458     matches[1] = 'http';
 459   if (isWebSocketSSL)
 460     matches[1] = 'https';
 461
 462   var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
 463   if (origins.indexOf(origin) < 0)
 464     return 'DIRECT';
 465   if (isHttp)
 466     return 'PROXY %(remote)s:%(httpport)s';
 467   if (isHttps || isWebSocket || isWebSocketSSL)
 468     return 'PROXY %(remote)s:%(sslport)s';
 469   return 'DIRECT';
 470 }""" % { "origins": origins,
 471          "remote":  self.webServer,
 472          "httpport":self.httpPort,
 473          "sslport": self.sslPort }
 474       pacURL = "".join(pacURL.splitlines())
 475
 476       part += """
 477 user_pref("network.proxy.type", 2);
 478 user_pref("network.proxy.autoconfig_url", "%(pacURL)s");
 479
 480 user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless to others
 481 """ % {"pacURL": pacURL}
 482       prefs.append(part)
 483
 484     for v in extraPrefs:
 485       thispref = v.split("=", 1)
 486       if len(thispref) < 2:
 487         print "Error: syntax error in --setpref=" + v
 488         sys.exit(1)
 489       part = 'user_pref("%s", %s);\n' % (thispref[0], thispref[1])
 490       prefs.append(part)
 491
 492     # write the preferences
 493     prefsFile = open(profileDir + "/" + "user.js", "a")
 494     prefsFile.write("".join(prefs))
 495     prefsFile.close()
 496
 497   def addCommonOptions(self, parser):
 498     "Adds command-line options which are common to mochitest and reftest."
 499
 500     parser.add_option("--setpref",
 501                       action = "append", type = "string",
 502                       default = [],
 503                       dest = "extraPrefs", metavar = "PREF=VALUE",
 504                       help = "defines an extra user preference")
 505
 506   def fillCertificateDB(self, profileDir, certPath, utilityPath, xrePath):
 507     pwfilePath = os.path.join(profileDir, ".crtdbpw")
 508
 509     pwfile = open(pwfilePath, "w")
 510     pwfile.write("\n")
 511     pwfile.close()
 512
 513     # Create head of the ssltunnel configuration file
 514     sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")
 515     sslTunnelConfig = open(sslTunnelConfigPath, "w")
 516
 517     sslTunnelConfig.write("httpproxy:1\n")
 518     sslTunnelConfig.write("certdbdir:%s\n" % certPath)
 519     sslTunnelConfig.write("forward:127.0.0.1:%s\n" % self.httpPort)
 520     sslTunnelConfig.write("websocketserver:%s:%s\n" % (self.webServer, self.webSocketPort))
 521     sslTunnelConfig.write("listen:*:%s:pgo server certificate\n" % self.sslPort)
 522
 523     # Configure automatic certificate and bind custom certificates, client authentication
 524     locations = self.readLocations()
 525     locations.pop(0)
 526     for loc in locations:
 527       if loc.scheme == "https" and "nocert" not in loc.options:
 528         customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")
 529         clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")
 530         redirRE      = re.compile("^redir=(?P<redirhost>[0-9a-zA-Z_ .]+)")
 531         for option in loc.options:
 532           match = customCertRE.match(option)
 533           if match:
 534             customcert = match.group("nickname");
 535             sslTunnelConfig.write("listen:%s:%s:%s:%s\n" %
 536                       (loc.host, loc.port, self.sslPort, customcert))
 537
 538           match = clientAuthRE.match(option)
 539           if match:
 540             clientauth = match.group("clientauth");
 541             sslTunnelConfig.write("clientauth:%s:%s:%s:%s\n" %
 542                       (loc.host, loc.port, self.sslPort, clientauth))
 543
 544           match = redirRE.match(option)
 545           if match:
 546             redirhost = match.group("redirhost")
 547             sslTunnelConfig.write("redirhost:%s:%s:%s:%s\n" %
 548                       (loc.host, loc.port, self.sslPort, redirhost))
 549
 550     sslTunnelConfig.close()
 551
 552     # Pre-create the certification database for the profile
 553     env = self.environment(xrePath = xrePath)
 554     certutil = os.path.join(utilityPath, "certutil" + self.BIN_SUFFIX)
 555     pk12util = os.path.join(utilityPath, "pk12util" + self.BIN_SUFFIX)
 556
 557     status = self.Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait()
 558     if status != 0:
 559       return status
 560
 561     # Walk the cert directory and add custom CAs and client certs
 562     files = os.listdir(certPath)
 563     for item in files:
 564       root, ext = os.path.splitext(item)
 565       if ext == ".ca":
 566         trustBits = "CT,,"
 567         if root.endswith("-object"):
 568           trustBits = "CT,,CT"
 569         self.Process([certutil, "-A", "-i", os.path.join(certPath, item),
 570                     "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits],
 571                     env = env).wait()
 572       if ext == ".client":
 573         self.Process([pk12util, "-i", os.path.join(certPath, item), "-w",
 574                     pwfilePath, "-d", profileDir],
 575                     env = env).wait()
 576
 577     os.unlink(pwfilePath)
 578     return 0
 579
 580   def environment(self, env = None, xrePath = None, crashreporter = True):
 581     if xrePath == None:
 582       xrePath = self.DIST_BIN
 583     if env == None:
 584       env = dict(os.environ)
 585
 586     ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath))
 587     if self.UNIXISH or self.IS_MAC:
 588       envVar = "LD_LIBRARY_PATH"
 589       if self.IS_MAC:
 590         envVar = "DYLD_LIBRARY_PATH"
 591       else: # unixish
 592         env['MOZILLA_FIVE_HOME'] = xrePath
 593       if envVar in env:
 594         ldLibraryPath = ldLibraryPath + ":" + env[envVar]
 595       env[envVar] = ldLibraryPath
 596     elif self.IS_WIN32:
 597       env["PATH"] = env["PATH"] + ";" + ldLibraryPath
 598
 599     if crashreporter:
 600       env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
 601       env['MOZ_CRASHREPORTER'] = '1'
 602     else:
 603       env['MOZ_CRASHREPORTER_DISABLE'] = '1'
 604
 605     env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
 606     env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
 607     env['NS_TRACE_MALLOC_DISABLE_STACKS'] = '1'
 608     return env
 609
 610   if IS_WIN32:
 611     PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
 612     GetLastError = ctypes.windll.kernel32.GetLastError
 613
 614     def readWithTimeout(self, f, timeout):
 615       """Try to read a line of output from the file object |f|.
 616       |f| must be a  pipe, like the |stdout| member of a subprocess.Popen
 617       object created with stdout=PIPE. If no output
 618       is received within |timeout| seconds, return a blank line.
 619       Returns a tuple (line, did_timeout), where |did_timeout| is True
 620       if the read timed out, and False otherwise."""
 621       if timeout is None:
 622         # shortcut to allow callers to pass in "None" for no timeout.
 623         return (f.readline(), False)
 624       x = msvcrt.get_osfhandle(f.fileno())
 625       l = ctypes.c_long()
 626       done = time.time() + timeout
 627       while time.time() < done:
 628         if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0:
 629           err = self.GetLastError()
 630           if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE
 631             return ('', False)
 632           else:
 633             log.error("readWithTimeout got error: %d", err)
 634         if l.value > 0:
 635           # we're assuming that the output is line-buffered,
 636           # which is not unreasonable
 637           return (f.readline(), False)
 638         time.sleep(0.01)
 639       return ('', True)
 640
 641     def isPidAlive(self, pid):
 642       STILL_ACTIVE = 259
 643       PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
 644       pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid)
 645       if not pHandle:
 646         return False
 647       pExitCode = ctypes.wintypes.DWORD()
 648       ctypes.windll.kernel32.GetExitCodeProcess(pHandle, ctypes.byref(pExitCode))
 649       ctypes.windll.kernel32.CloseHandle(pHandle)
 650       return pExitCode.value == STILL_ACTIVE
 651
 652     def killPid(self, pid):
 653       PROCESS_TERMINATE = 0x0001
 654       pHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_TERMINATE, 0, pid)
 655       if not pHandle:
 656         return
 657       success = ctypes.windll.kernel32.TerminateProcess(pHandle, 1)
 658       ctypes.windll.kernel32.CloseHandle(pHandle)
 659
 660   else:
 661
 662     def readWithTimeout(self, f, timeout):
 663       """Try to read a line of output from the file object |f|. If no output
 664       is received within |timeout| seconds, return a blank line.
 665       Returns a tuple (line, did_timeout), where |did_timeout| is True
 666       if the read timed out, and False otherwise."""
 667       (r, w, e) = select.select([f], [], [], timeout)
 668       if len(r) == 0:
 669         return ('', True)
 670       return (f.readline(), False)
 671
 672     def isPidAlive(self, pid):
 673       try:
 674         # kill(pid, 0) checks for a valid PID without actually sending a signal
 675         # The method throws OSError if the PID is invalid, which we catch below.
 676         os.kill(pid, 0)
 677
 678         # Wait on it to see if it's a zombie. This can throw OSError.ECHILD if
 679         # the process terminates before we get to this point.
 680         wpid, wstatus = os.waitpid(pid, os.WNOHANG)
 681         return wpid == 0
 682       except OSError, err:
 683         # Catch the errors we might expect from os.kill/os.waitpid,
 684         # and re-raise any others
 685         if err.errno == errno.ESRCH or err.errno == errno.ECHILD:
 686           return False
 687         raise
 688
 689     def killPid(self, pid):
 690       os.kill(pid, signal.SIGKILL)
 691
 692   def dumpScreen(self, utilityPath):
 693     self.haveDumpedScreen = True;
 694
 695     # Need to figure out what tool and whether it write to a file or stdout
 696     if self.UNIXISH:
 697       utility = [os.path.join(utilityPath, "screentopng")]
 698       imgoutput = 'stdout'
 699     elif self.IS_MAC:
 700       utility = ['/usr/sbin/screencapture', '-C', '-x', '-t', 'png']
 701       imgoutput = 'file'
 702     elif self.IS_WIN32:
 703       utility = [os.path.join(utilityPath, "screenshot.exe")]
 704       imgoutput = 'file'
 705
 706     # Run the capture correctly for the type of capture
 707     try:
 708       if imgoutput == 'file':
 709         tmpfd, imgfilename = tempfile.mkstemp(prefix='mozilla-test-fail_')
 710         os.close(tmpfd)
 711         dumper = self.Process(utility + [imgfilename])
 712       elif imgoutput == 'stdout':
 713         dumper = self.Process(utility, bufsize=-1,
 714                               stdout=subprocess.PIPE, close_fds=True)
 715     except OSError, err:
 716       self.log.info("Failed to start %s for screenshot: %s",
 717                     utility[0], err.strerror)
 718       return
 719
 720     # Check whether the capture utility ran successfully
 721     dumper_out, dumper_err = dumper.communicate()
 722     if dumper.returncode != 0:
 723       self.log.info("%s exited with code %d", utility, dumper.returncode)
 724       return
 725
 726     try:
 727       if imgoutput == 'stdout':
 728         image = dumper_out
 729       elif imgoutput == 'file':
 730         with open(imgfilename, 'rb') as imgfile:
 731           image = imgfile.read()
 732     except IOError, err:
 733         self.log.info("Failed to read image from %s", imgoutput)
 734
 735     import base64
 736     encoded = base64.b64encode(image)
 737     self.log.info("SCREENSHOT: data:image/png;base64,%s", encoded)
 738
 739   def killAndGetStack(self, proc, utilityPath, debuggerInfo):
 740     """Kill the process, preferrably in a way that gets us a stack trace."""
 741     if not debuggerInfo:
 742       if self.haveDumpedScreen:
 743         self.log.info("Not taking screenshot here: see the one that was previously logged")
 744       else:
 745         self.dumpScreen(utilityPath)
 746
 747     if self.CRASHREPORTER and not debuggerInfo:
 748       if self.UNIXISH:
 749         # ABRT will get picked up by Breakpad's signal handler
 750         os.kill(proc.pid, signal.SIGABRT)
 751         return
 752       elif self.IS_WIN32:
 753         # We should have a "crashinject" program in our utility path
 754         crashinject = os.path.normpath(os.path.join(utilityPath, "crashinject.exe"))
 755         if os.path.exists(crashinject) and subprocess.Popen([crashinject, str(proc.pid)]).wait() == 0:
 756           return
 757       #TODO: kill the process such that it triggers Breakpad on OS X (bug 525296)
 758     self.log.info("Can't trigger Breakpad, just killing process")
 759     proc.kill()
 760
 761   def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath, logger):
 762     """ Look for timeout or crashes and return the status after the process terminates """
 763     stackFixerProcess = None
 764     stackFixerFunction = None
 765     didTimeout = False
 766     hitMaxTime = False
 767     if proc.stdout is None:
 768       self.log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection")
 769     else:
 770       logsource = proc.stdout
 771
 772       if self.IS_DEBUG_BUILD and (self.IS_MAC or self.IS_LINUX) and symbolsPath and os.path.exists(symbolsPath):
 773         # Run each line through a function in fix_stack_using_bpsyms.py (uses breakpad symbol files)
 774         # This method is preferred for Tinderbox builds, since native symbols may have been stripped.
 775         sys.path.insert(0, utilityPath)
 776         import fix_stack_using_bpsyms as stackFixerModule
 777         stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line, symbolsPath)
 778         del sys.path[0]
 779       elif self.IS_DEBUG_BUILD and self.IS_MAC and False:
 780         # Run each line through a function in fix_macosx_stack.py (uses atos)
 781         sys.path.insert(0, utilityPath)
 782         import fix_macosx_stack as stackFixerModule
 783         stackFixerFunction = lambda line: stackFixerModule.fixSymbols(line)
 784         del sys.path[0]
 785       elif self.IS_DEBUG_BUILD and self.IS_LINUX:
 786         # Run logsource through fix-linux-stack.pl (uses addr2line)
 787         # This method is preferred for developer machines, so we don't have to run "make buildsymbols".
 788         stackFixerProcess = self.Process([self.PERL, os.path.join(utilityPath, "fix-linux-stack.pl")],
 789                                          stdin=logsource,
 790                                          stdout=subprocess.PIPE)
 791         logsource = stackFixerProcess.stdout
 792
 793       (line, didTimeout) = self.readWithTimeout(logsource, timeout)
 794       while line != "" and not didTimeout:
 795         self.log.info(line.rstrip().decode("UTF-8", "ignore"))
 796         if logger:
 797           logger.log(line)
 798         if "TEST-START" in line and "|" in line:
 799           self.lastTestSeen = line.split("|")[1].strip()
 800         if stackFixerFunction:
 801           line = stackFixerFunction(line)
 802         if not debuggerInfo and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:
 803           if self.haveDumpedScreen:
 804             self.log.info("Not taking screenshot here: see the one that was previously logged")
 805           else:
 806             self.dumpScreen(utilityPath)
 807
 808         (line, didTimeout) = self.readWithTimeout(logsource, timeout)
 809         if not hitMaxTime and maxTime and datetime.now() - startTime > timedelta(seconds = maxTime):
 810           # Kill the application, but continue reading from stack fixer so as not to deadlock on stackFixerProcess.wait().
 811           hitMaxTime = True
 812           self.log.info("TEST-UNEXPECTED-FAIL | %s | application ran for longer than allowed maximum time of %d seconds", self.lastTestSeen, int(maxTime))
 813           self.killAndGetStack(proc, utilityPath, debuggerInfo)
 814       if didTimeout:
 815         self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout))
 816         self.killAndGetStack(proc, utilityPath, debuggerInfo)
 817
 818     status = proc.wait()
 819     if status == 0:
 820       self.lastTestSeen = "Main app process exited normally"
 821     if status != 0 and not didTimeout and not hitMaxTime:
 822       self.log.info("TEST-UNEXPECTED-FAIL | %s | Exited with code %d during test run", self.lastTestSeen, status)
 823     if stackFixerProcess is not None:
 824       fixerStatus = stackFixerProcess.wait()
 825       if fixerStatus != 0 and not didTimeout and not hitMaxTime:
 826         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Stack fixer process exited with code %d during test run", fixerStatus)
 827     return status
 828
 829   def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
 830     """ build the application command line """
 831
 832     cmd = os.path.abspath(app)
 833     if self.IS_MAC and not self.IS_CAMINO and os.path.exists(cmd + "-bin"):
 834       # Prefer 'app-bin' in case 'app' is a shell script.
 835       # We can remove this hack once bug 673899 etc are fixed.
 836       cmd += "-bin"
 837
 838     args = []
 839
 840     if debuggerInfo:
 841       args.extend(debuggerInfo["args"])
 842       args.append(cmd)
 843       cmd = os.path.abspath(debuggerInfo["path"])
 844
 845     if self.IS_MAC:
 846       args.append("-foreground")
 847
 848     if self.IS_CYGWIN:
 849       profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
 850     else:
 851       profileDirectory = profileDir + "/"
 852
 853     args.extend(("-no-remote", "-profile", profileDirectory))
 854     if testURL is not None:
 855       if self.IS_CAMINO:
 856         args.extend(("-url", testURL))
 857       else:
 858         args.append((testURL))
 859     args.extend(extraArgs)
 860     return cmd, args
 861
 862   def checkForZombies(self, processLog):
 863     """ Look for hung processes """
 864     if not os.path.exists(processLog):
 865       self.log.info('INFO | automation.py | PID log not found: %s', processLog)
 866     else:
 867       self.log.info('INFO | automation.py | Reading PID log: %s', processLog)
 868       processList = []
 869       pidRE = re.compile(r'launched child process (\d+)$')
 870       processLogFD = open(processLog)
 871       for line in processLogFD:
 872         self.log.info(line.rstrip())
 873         m = pidRE.search(line)
 874         if m:
 875           processList.append(int(m.group(1)))
 876       processLogFD.close()
 877
 878       for processPID in processList:
 879         self.log.info("INFO | automation.py | Checking for orphan process with PID: %d", processPID)
 880         if self.isPidAlive(processPID):
 881           self.log.info("TEST-UNEXPECTED-FAIL | automation.py | child process %d still alive after shutdown", processPID)
 882           self.killPid(processPID)
 883
 884   def checkForCrashes(self, profileDir, symbolsPath):
 885     automationutils.checkForCrashes(os.path.join(profileDir, "minidumps"), symbolsPath, self.lastTestSeen)
 886
 887   def runApp(self, testURL, env, app, profileDir, extraArgs,
 888              runSSLTunnel = False, utilityPath = None,
 889              xrePath = None, certPath = None, logger = None,
 890              debuggerInfo = None, symbolsPath = None,
 891              timeout = -1, maxTime = None):
 892     """
 893     Run the app, log the duration it took to execute, return the status code.
 894     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
 895     """
 896
 897     if utilityPath == None:
 898       utilityPath = self.DIST_BIN
 899     if xrePath == None:
 900       xrePath = self.DIST_BIN
 901     if certPath == None:
 902       certPath = self.CERTS_SRC_DIR
 903     if timeout == -1:
 904       timeout = self.DEFAULT_TIMEOUT
 905
 906     # copy env so we don't munge the caller's environment
 907     env = dict(env);
 908     env["NO_EM_RESTART"] = "1"
 909     tmpfd, processLog = tempfile.mkstemp(suffix='pidlog')
 910     os.close(tmpfd)
 911     env["MOZ_PROCESS_LOG"] = processLog
 912
 913     if self.IS_TEST_BUILD and runSSLTunnel:
 914       # create certificate database for the profile
 915       certificateStatus = self.fillCertificateDB(profileDir, certPath, utilityPath, xrePath)
 916       if certificateStatus != 0:
 917         self.log.info("TEST-UNEXPECTED-FAIL | automation.py | Certificate integration failed")
 918         return certificateStatus
 919
 920       # start ssltunnel to provide https:// URLs capability
 921       ssltunnel = os.path.join(utilityPath, "ssltunnel" + self.BIN_SUFFIX)
 922       ssltunnelProcess = self.Process([ssltunnel,
 923                                os.path.join(profileDir, "ssltunnel.cfg")],
 924                                env = self.environment(xrePath = xrePath))
 925       self.log.info("INFO | automation.py | SSL tunnel pid: %d", ssltunnelProcess.pid)
 926
 927     cmd, args = self.buildCommandLine(app, debuggerInfo, profileDir, testURL, extraArgs)
 928     startTime = datetime.now()
 929
 930     if debuggerInfo and debuggerInfo["interactive"]:
 931       # If an interactive debugger is attached, don't redirect output,
 932       # don't use timeouts, and don't capture ctrl-c.
 933       timeout = None
 934       maxTime = None
 935       outputPipe = None
 936       signal.signal(signal.SIGINT, lambda sigid, frame: None)
 937     else:
 938       outputPipe = subprocess.PIPE
 939
 940     self.lastTestSeen = "automation.py"
 941     proc = self.Process([cmd] + args,
 942                  env = self.environment(env, xrePath = xrePath,
 943                                    crashreporter = not debuggerInfo),
 944                  stdout = outputPipe,
 945                  stderr = subprocess.STDOUT)
 946     self.log.info("INFO | automation.py | Application pid: %d", proc.pid)
 947
 948     status = self.waitForFinish(proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsPath, logger)
 949     self.log.info("INFO | automation.py | Application ran for: %s", str(datetime.now() - startTime))
 950
 951     # Do a final check for zombie child processes.
 952     self.checkForZombies(processLog)
 953     self.checkForCrashes(profileDir, symbolsPath)
 954
 955     if os.path.exists(processLog):
 956       os.unlink(processLog)
 957
 958     if self.IS_TEST_BUILD and runSSLTunnel:
 959       ssltunnelProcess.kill()
 960
 961     return status
 962
 963   def getExtensionIDFromRDF(self, rdfSource):
 964     """
 965     Retrieves the extension id from an install.rdf file (or string).
 966     """
 967     from xml.dom.minidom import parse, parseString, Node
 968
 969     if isinstance(rdfSource, file):
 970       document = parse(rdfSource)
 971     else:
 972       document = parseString(rdfSource)
 973
 974     # Find the <em:id> element. There can be multiple <em:id> tags
 975     # within <em:targetApplication> tags, so we have to check this way.
 976     for rdfChild in document.documentElement.childNodes:
 977       if rdfChild.nodeType == Node.ELEMENT_NODE and rdfChild.tagName == "Description":
 978         for descChild in rdfChild.childNodes:
 979           if descChild.nodeType == Node.ELEMENT_NODE and descChild.tagName == "em:id":
 980             return descChild.childNodes[0].data
 981
 982     return None
 983
 984   def installExtension(self, extensionSource, profileDir, extensionID = None):
 985     """
 986     Copies an extension into the extensions directory of the given profile.
 987     extensionSource - the source location of the extension files.  This can be either
 988                       a directory or a path to an xpi file.
 989     profileDir      - the profile directory we are copying into.  We will create the
 990                       "extensions" directory there if it doesn't exist.
 991     extensionID     - the id of the extension to be used as the containing directory for the
 992                       extension, if extensionSource is a directory, i.e.
 993                   this is the name of the folder in the <profileDir>/extensions/<extensionID>
 994     """
 995     if not os.path.isdir(profileDir):
 996       self.log.info("INFO | automation.py | Cannot install extension, invalid profileDir at: %s", profileDir)
 997       return
 998
 999     installRDFFilename = "install.rdf"
1000
1001     extensionsRootDir = os.path.join(profileDir, "extensions", "staged")
1002     if not os.path.isdir(extensionsRootDir):
1003       os.makedirs(extensionsRootDir)
1004
1005     if os.path.isfile(extensionSource):
1006       reader = automationutils.ZipFileReader(extensionSource)
1007
1008       for filename in reader.namelist():
1009         # Sanity check the zip file.
1010         if os.path.isabs(filename):
1011           self.log.info("INFO | automation.py | Cannot install extension, bad files in xpi")
1012           return
1013
1014         # We may need to dig the extensionID out of the zip file...
1015         if extensionID is None and filename == installRDFFilename:
1016           extensionID = self.getExtensionIDFromRDF(reader.read(filename))
1017
1018       # We must know the extensionID now.
1019       if extensionID is None:
1020         self.log.info("INFO | automation.py | Cannot install extension, missing extensionID")
1021         return
1022
1023       # Make the extension directory.
1024       extensionDir = os.path.join(extensionsRootDir, extensionID)
1025       os.mkdir(extensionDir)
1026
1027       # Extract all files.
1028       reader.extractall(extensionDir)
1029
1030     elif os.path.isdir(extensionSource):
1031       if extensionID is None:
1032         filename = os.path.join(extensionSource, installRDFFilename)
1033         if os.path.isfile(filename):
1034           with open(filename, "r") as installRDF:
1035             extensionID = self.getExtensionIDFromRDF(installRDF)
1036
1037         if extensionID is None:
1038           self.log.info("INFO | automation.py | Cannot install extension, missing extensionID")
1039           return
1040
1041       # Copy extension tree into its own directory.
1042       # "destination directory must not already exist".
1043       shutil.copytree(extensionSource, os.path.join(extensionsRootDir, extensionID))
1044
1045     else:
1046       self.log.info("INFO | automation.py | Cannot install extension, invalid extensionSource at: %s", extensionSource)