build/automation.py.in

   1 #
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   4 #
   5 # The contents of this file are subject to the Mozilla Public License Version
   6 # 1.1 (the "License"); you may not use this file except in compliance with
   7 # the License. You may obtain a copy of the License at
   8 # http://www.mozilla.org/MPL/
   9 #
  10 # Software distributed under the License is distributed on an "AS IS" basis,
  11 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12 # for the specific language governing rights and limitations under the
  13 # License.
  14 #
  15 # The Original Code is mozilla.org code.
  16 #
  17 # The Initial Developer of the Original Code is
  18 # Mozilla Foundation.
  19 # Portions created by the Initial Developer are Copyright (C) 2008
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #   Robert Sayre <sayrer@gmail.com>
  24 #   Jeff Walden <jwalden+bmo@mit.edu>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 import codecs
  41 from datetime import datetime
  42 import itertools
  43 import logging
  44 import os
  45 import re
  46 import shutil
  47 import signal
  48 import subprocess
  49 import sys
  50 import threading
  51
  52 """
  53 Runs the browser from a script, and provides useful utilities
  54 for setting up the browser environment.
  55 """
  56
  57 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
  58
  59 __all__ = [
  60            "UNIXISH",
  61            "IS_WIN32",
  62            "IS_MAC",
  63            "runApp",
  64            "Process",
  65            "initializeProfile",
  66            "DIST_BIN",
  67            "DEFAULT_APP",
  68            "CERTS_SRC_DIR",
  69            "environment",
  70           ]
  71
  72 # These are generated in mozilla/build/Makefile.in
  73 #expand DIST_BIN = __XPC_BIN_PATH__
  74 #expand IS_WIN32 = len("__WIN32__") != 0
  75 #expand IS_MAC = __IS_MAC__ != 0
  76 #ifdef IS_CYGWIN
  77 #expand IS_CYGWIN = __IS_CYGWIN__ == 1
  78 #else
  79 IS_CYGWIN = False
  80 #endif
  81 #expand IS_CAMINO = __IS_CAMINO__ != 0
  82 #expand BIN_SUFFIX = __BIN_SUFFIX__
  83
  84 UNIXISH = not IS_WIN32 and not IS_MAC
  85
  86 #expand DEFAULT_APP = "./" + __BROWSER_PATH__
  87 #expand CERTS_SRC_DIR = __CERTS_SRC_DIR__
  88 #expand IS_TEST_BUILD = __IS_TEST_BUILD__
  89 #expand IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
  90
  91 ###########
  92 # LOGGING #
  93 ###########
  94
  95 # We use the logging system here primarily because it'll handle multiple
  96 # threads, which is needed to process the output of the server and application
  97 # processes simultaneously.
  98 log = logging.getLogger()
  99 handler = logging.StreamHandler(sys.stdout)
 100 log.setLevel(logging.INFO)
 101 log.addHandler(handler)
 102
 103
 104 #################
 105 # SUBPROCESSING #
 106 #################
 107
 108 class Process(subprocess.Popen):
 109   """
 110   Represents our view of a subprocess.
 111   It adds a kill() method which allows it to be stopped explicitly.
 112   """
 113
 114   def kill(self):
 115     if IS_WIN32:
 116       import platform
 117       pid = "%i" % self.pid
 118       if platform.release() == "2000":
 119         # Windows 2000 needs 'kill.exe' from the 'Windows 2000 Resource Kit tools'. (See bug 475455.)
 120         try:
 121           subprocess.Popen(["kill", "-f", pid]).wait()
 122         except:
 123           log.info("TEST-UNEXPECTED-FAIL | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid)
 124       else:
 125         # Windows XP and later.
 126         subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()
 127     else:
 128       os.kill(self.pid, signal.SIGKILL)
 129
 130
 131 #################
 132 # PROFILE SETUP #
 133 #################
 134
 135 class SyntaxError(Exception):
 136   "Signifies a syntax error on a particular line in server-locations.txt."
 137
 138   def __init__(self, lineno, msg = None):
 139     self.lineno = lineno
 140     self.msg = msg
 141
 142   def __str__(self):
 143     s = "Syntax error on line " + str(self.lineno)
 144     if self.msg:
 145       s += ": %s." % self.msg
 146     else:
 147       s += "."
 148     return s
 149
 150
 151 class Location:
 152   "Represents a location line in server-locations.txt."
 153
 154   def __init__(self, scheme, host, port, options):
 155     self.scheme = scheme
 156     self.host = host
 157     self.port = port
 158     self.options = options
 159
 160
 161 def readLocations(locationsPath = "server-locations.txt"):
 162   """
 163   Reads the locations at which the Mochitest HTTP server is available from
 164   server-locations.txt.
 165   """
 166
 167   locationFile = codecs.open(locationsPath, "r", "UTF-8")
 168
 169   # Perhaps more detail than necessary, but it's the easiest way to make sure
 170   # we get exactly the format we want.  See server-locations.txt for the exact
 171   # format guaranteed here.
 172   lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"
 173                       r"://"
 174                       r"(?P<host>"
 175                         r"\d+\.\d+\.\d+\.\d+"
 176                         r"|"
 177                         r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*"
 178                         r"[a-z](?:[-a-z0-9]*[a-z0-9])?"
 179                       r")"
 180                       r":"
 181                       r"(?P<port>\d+)"
 182                       r"(?:"
 183                       r"\s+"
 184                       r"(?P<options>\S+(?:,\S+)*)"
 185                       r")?$")
 186   locations = []
 187   lineno = 0
 188   seenPrimary = False
 189   for line in locationFile:
 190     lineno += 1
 191     if line.startswith("#") or line == "\n":
 192       continue
 193
 194     match = lineRe.match(line)
 195     if not match:
 196       raise SyntaxError(lineno)
 197
 198     options = match.group("options")
 199     if options:
 200       options = options.split(",")
 201       if "primary" in options:
 202         if seenPrimary:
 203           raise SyntaxError(lineno, "multiple primary locations")
 204         seenPrimary = True
 205     else:
 206       options = []
 207
 208     locations.append(Location(match.group("scheme"), match.group("host"),
 209                               match.group("port"), options))
 210
 211   if not seenPrimary:
 212     raise SyntaxError(lineno + 1, "missing primary location")
 213
 214   return locations
 215
 216
 217 def initializeProfile(profileDir):
 218   "Sets up the standard testing profile."
 219
 220   # Start with a clean slate.
 221   shutil.rmtree(profileDir, True)
 222   os.mkdir(profileDir)
 223
 224   prefs = []
 225
 226   part = """\
 227 user_pref("browser.dom.window.dump.enabled", true);
 228 user_pref("dom.allow_scripts_to_close_windows", true);
 229 user_pref("dom.disable_open_during_load", false);
 230 user_pref("dom.max_script_run_time", 0); // no slow script dialogs
 231 user_pref("signed.applets.codebase_principal_support", true);
 232 user_pref("security.warn_submit_insecure", false);
 233 user_pref("browser.shell.checkDefaultBrowser", false);
 234 user_pref("shell.checkDefaultClient", false);
 235 user_pref("browser.warnOnQuit", false);
 236 user_pref("accessibility.typeaheadfind.autostart", false);
 237 user_pref("javascript.options.showInConsole", true);
 238 user_pref("layout.debug.enable_data_xbl", true);
 239 user_pref("browser.EULA.override", true);
 240 user_pref("javascript.options.jit.content", true);
 241 user_pref("gfx.color_management.force_srgb", true);
 242 user_pref("network.manage-offline-status", false);
 243 user_pref("security.default_personal_cert", "Select Automatically"); // Need to client auth test be w/o any dialogs
 244
 245 user_pref("camino.warn_when_closing", false); // Camino-only, harmless to others
 246 """
 247
 248   prefs.append(part)
 249
 250   # Increase the max script run time 10-fold for debug builds
 251   if (IS_DEBUG_BUILD):
 252     prefs.append("""\
 253 user_pref("dom.max_script_run_time", 100);
 254 user_pref("dom.max_chrome_script_run_time", 200);
 255 """)
 256
 257   locations = readLocations()
 258
 259   # Grant God-power to all the privileged servers on which tests run.
 260   privileged = filter(lambda loc: "privileged" in loc.options, locations)
 261   for (i, l) in itertools.izip(itertools.count(1), privileged):
 262     part = """
 263 user_pref("capability.principal.codebase.p%(i)d.granted",
 264           "UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite \
 265            UniversalPreferencesRead UniversalPreferencesWrite \
 266            UniversalFileRead");
 267 user_pref("capability.principal.codebase.p%(i)d.id", "%(origin)s");
 268 user_pref("capability.principal.codebase.p%(i)d.subjectName", "");
 269 """  % { "i": i,
 270          "origin": (l.scheme + "://" + l.host + ":" + l.port) }
 271     prefs.append(part)
 272
 273   # We need to proxy every server but the primary one.
 274   origins = ["'%s://%s:%s'" % (l.scheme, l.host, l.port)
 275              for l in filter(lambda l: "primary" not in l.options, locations)]
 276   origins = ", ".join(origins)
 277
 278   pacURL = """data:text/plain,
 279 function FindProxyForURL(url, host)
 280 {
 281   var origins = [%(origins)s];
 282   var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
 283                          '://' +
 284                          '(?:[^/@]*@)?' +
 285                          '(.*?)' +
 286                          '(?::(\\\\\\\\d+))?/');
 287   var matches = regex.exec(url);
 288   if (!matches)
 289     return 'DIRECT';
 290   var isHttp = matches[1] == 'http';
 291   var isHttps = matches[1] == 'https';
 292   if (!matches[3])
 293   {
 294     if (isHttp) matches[3] = '80';
 295     if (isHttps) matches[3] = '443';
 296   }
 297
 298   var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
 299   if (origins.indexOf(origin) < 0)
 300     return 'DIRECT';
 301   if (isHttp)
 302     return 'PROXY 127.0.0.1:8888';
 303   if (isHttps)
 304     return 'PROXY 127.0.0.1:4443';
 305   return 'DIRECT';
 306 }""" % { "origins": origins }
 307   pacURL = "".join(pacURL.splitlines())
 308
 309   part = """
 310 user_pref("network.proxy.type", 2);
 311 user_pref("network.proxy.autoconfig_url", "%(pacURL)s");
 312
 313 user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless to others
 314 """ % {"pacURL": pacURL}
 315   prefs.append(part)
 316
 317   # write the preferences
 318   prefsFile = open(profileDir + "/" + "user.js", "a")
 319   prefsFile.write("".join(prefs))
 320   prefsFile.close()
 321
 322 def fillCertificateDB(profileDir, certPath, utilityPath, xrePath):
 323   pwfilePath = os.path.join(profileDir, ".crtdbpw")
 324
 325   pwfile = open(pwfilePath, "w")
 326   pwfile.write("\n")
 327   pwfile.close()
 328
 329   # Create head of the ssltunnel configuration file
 330   sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")
 331   sslTunnelConfig = open(sslTunnelConfigPath, "w")
 332
 333   sslTunnelConfig.write("httpproxy:1\n")
 334   sslTunnelConfig.write("certdbdir:%s\n" % certPath)
 335   sslTunnelConfig.write("forward:127.0.0.1:8888\n")
 336   sslTunnelConfig.write("listen:*:4443:pgo server certificate\n")
 337
 338   # Configure automatic certificate and bind custom certificates, client authentication
 339   locations = readLocations()
 340   locations.pop(0)
 341   for loc in locations:
 342     if loc.scheme == "https" and "nocert" not in loc.options:
 343       customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")
 344       clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")
 345       for option in loc.options:
 346         match = customCertRE.match(option)
 347         if match:
 348           customcert = match.group("nickname");
 349           sslTunnelConfig.write("listen:%s:%s:4443:%s\n" %
 350               (loc.host, loc.port, customcert))
 351
 352         match = clientAuthRE.match(option)
 353         if match:
 354           clientauth = match.group("clientauth");
 355           sslTunnelConfig.write("clientauth:%s:%s:4443:%s\n" %
 356               (loc.host, loc.port, clientauth))
 357
 358   sslTunnelConfig.close()
 359
 360   # Pre-create the certification database for the profile
 361   env = environment(xrePath = xrePath)
 362   certutil = os.path.join(utilityPath, "certutil" + BIN_SUFFIX)
 363   pk12util = os.path.join(utilityPath, "pk12util" + BIN_SUFFIX)
 364
 365   status = Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait()
 366   if status != 0:
 367     return status
 368
 369   # Walk the cert directory and add custom CAs and client certs
 370   files = os.listdir(certPath)
 371   for item in files:
 372     root, ext = os.path.splitext(item)
 373     if ext == ".ca":
 374       trustBits = "CT,,"
 375       if root.endswith("-object"):
 376         trustBits = "CT,,CT"
 377       Process([certutil, "-A", "-i", os.path.join(certPath, item),
 378         "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits],
 379         env = env).wait()
 380     if ext == ".client":
 381       Process([pk12util, "-i", os.path.join(certPath, item), "-w",
 382         pwfilePath, "-d", profileDir],
 383         env = env).wait()
 384
 385   os.unlink(pwfilePath)
 386   return 0
 387
 388 def environment(env = None, xrePath = DIST_BIN):
 389   if env == None:
 390     env = dict(os.environ)
 391
 392   ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath))
 393   if UNIXISH or IS_MAC:
 394     envVar = "LD_LIBRARY_PATH"
 395     if IS_MAC:
 396       envVar = "DYLD_LIBRARY_PATH"
 397     if envVar in env:
 398       ldLibraryPath = ldLibraryPath + ":" + env[envVar]
 399     env[envVar] = ldLibraryPath
 400   elif IS_WIN32:
 401     env["PATH"] = env["PATH"] + ";" + ldLibraryPath
 402
 403   return env
 404
 405 ###############
 406 # RUN THE APP #
 407 ###############
 408
 409 def runApp(testURL, env, app, profileDir, extraArgs, runSSLTunnel = False, utilityPath = DIST_BIN, xrePath = DIST_BIN, certPath = CERTS_SRC_DIR):
 410   "Run the app, returning a tuple containing the status code and the time at which it was started."
 411   if IS_TEST_BUILD and runSSLTunnel:
 412     # create certificate database for the profile
 413     certificateStatus = fillCertificateDB(profileDir, certPath, utilityPath, xrePath)
 414     if certificateStatus != 0:
 415       log.info("TEST-UNEXPECTED FAIL | Certificate integration failed")
 416       return certificateStatus
 417
 418     # start ssltunnel to provide https:// URLs capability
 419     ssltunnel = os.path.join(utilityPath, "ssltunnel" + BIN_SUFFIX)
 420     ssltunnelProcess = Process([ssltunnel, os.path.join(profileDir, "ssltunnel.cfg")], env = environment(xrePath = xrePath))
 421     log.info("SSL tunnel pid: %d", ssltunnelProcess.pid)
 422
 423   "Run the app, returning the time at which it was started."
 424   # mark the start
 425   start = datetime.now()
 426
 427   # now run with the profile we created
 428   cmd = app
 429   if IS_MAC and not IS_CAMINO and not cmd.endswith("-bin"):
 430     cmd += "-bin"
 431   cmd = os.path.abspath(cmd)
 432
 433   args = []
 434   if IS_MAC:
 435     args.append("-foreground")
 436
 437   if IS_CYGWIN:
 438     profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
 439   else:
 440     profileDirectory = profileDir + "/"
 441
 442   args.extend(("-no-remote", "-profile", profileDirectory))
 443   if testURL is not None:
 444     if IS_CAMINO:
 445       args.extend(("-url", testURL))
 446     else:
 447       args.append((testURL))
 448   args.extend(extraArgs)
 449   proc = Process([cmd] + args, env = environment(env), stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
 450   log.info("Application pid: %d", proc.pid)
 451   line = proc.stdout.readline()
 452   while line != "":
 453     log.info(line.rstrip())
 454     line = proc.stdout.readline()
 455   status = proc.wait()
 456   if status != 0:
 457     log.info("TEST-UNEXPECTED-FAIL | Exited with code %d during test run", status)
 458
 459   if IS_TEST_BUILD and runSSLTunnel:
 460     ssltunnelProcess.kill()
 461
 462   return (status, start)