build/automation.py.in

   1 #
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   4 #
   5 # The contents of this file are subject to the Mozilla Public License Version
   6 # 1.1 (the "License"); you may not use this file except in compliance with
   7 # the License. You may obtain a copy of the License at
   8 # http://www.mozilla.org/MPL/
   9 #
  10 # Software distributed under the License is distributed on an "AS IS" basis,
  11 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12 # for the specific language governing rights and limitations under the
  13 # License.
  14 #
  15 # The Original Code is mozilla.org code.
  16 #
  17 # The Initial Developer of the Original Code is
  18 # Mozilla Foundation.
  19 # Portions created by the Initial Developer are Copyright (C) 2008
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #   Robert Sayre <sayrer@gmail.com>
  24 #   Jeff Walden <jwalden+bmo@mit.edu>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 import codecs
  41 from datetime import datetime
  42 import itertools
  43 import logging
  44 import os
  45 import re
  46 import shutil
  47 import signal
  48 import subprocess
  49 import sys
  50 import threading
  51
  52 """
  53 Runs the browser from a script, and provides useful utilities
  54 for setting up the browser environment.
  55 """
  56
  57 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
  58
  59 __all__ = [
  60            "UNIXISH",
  61            "IS_WIN32",
  62            "IS_MAC",
  63            "runApp",
  64            "Process",
  65            "initializeProfile",
  66            "DIST_BIN",
  67            "DEFAULT_APP",
  68            "CERTS_SRC_DIR",
  69            "environment",
  70           ]
  71
  72 # These are generated in mozilla/build/Makefile.in
  73 #expand DIST_BIN = __XPC_BIN_PATH__
  74 #expand IS_WIN32 = len("__WIN32__") != 0
  75 #expand IS_MAC = __IS_MAC__ != 0
  76 #ifdef IS_CYGWIN
  77 #expand IS_CYGWIN = __IS_CYGWIN__ == 1
  78 #else
  79 IS_CYGWIN = False
  80 #endif
  81 #expand IS_CAMINO = __IS_CAMINO__ != 0
  82 #expand BIN_SUFFIX = __BIN_SUFFIX__
  83
  84 UNIXISH = not IS_WIN32 and not IS_MAC
  85
  86 #expand DEFAULT_APP = "./" + __BROWSER_PATH__
  87 #expand CERTS_SRC_DIR = __CERTS_SRC_DIR__
  88 #expand IS_TEST_BUILD = __IS_TEST_BUILD__
  89 #expand IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
  90
  91 ###########
  92 # LOGGING #
  93 ###########
  94
  95 # We use the logging system here primarily because it'll handle multiple
  96 # threads, which is needed to process the output of the server and application
  97 # processes simultaneously.
  98 log = logging.getLogger()
  99 handler = logging.StreamHandler(sys.stdout)
 100 log.setLevel(logging.INFO)
 101 log.addHandler(handler)
 102
 103
 104 #################
 105 # SUBPROCESSING #
 106 #################
 107
 108 class Process(subprocess.Popen):
 109   """
 110   Represents our view of a subprocess.
 111   It adds a kill() method which allows it to be stopped explicitly.
 112   """
 113
 114   def kill(self):
 115     if IS_WIN32:
 116       import platform
 117       pid = "%i" % self.pid
 118       if platform.release() == "2000":
 119         # Windows 2000 needs 'kill.exe' from the 'Windows 2000 Resource Kit tools'. (See bug 475455.)
 120         try:
 121           subprocess.Popen(["kill", "-f", pid]).wait()
 122         except:
 123           log.info("TEST-UNEXPECTED-FAIL | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid)
 124       else:
 125         # Windows XP and later.
 126         subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()
 127     else:
 128       os.kill(self.pid, signal.SIGKILL)
 129
 130
 131 #################
 132 # PROFILE SETUP #
 133 #################
 134
 135 class SyntaxError(Exception):
 136   "Signifies a syntax error on a particular line in server-locations.txt."
 137
 138   def __init__(self, lineno, msg = None):
 139     self.lineno = lineno
 140     self.msg = msg
 141
 142   def __str__(self):
 143     s = "Syntax error on line " + str(self.lineno)
 144     if self.msg:
 145       s += ": %s." % self.msg
 146     else:
 147       s += "."
 148     return s
 149
 150
 151 class Location:
 152   "Represents a location line in server-locations.txt."
 153
 154   def __init__(self, scheme, host, port, options):
 155     self.scheme = scheme
 156     self.host = host
 157     self.port = port
 158     self.options = options
 159
 160
 161 def readLocations(locationsPath = "server-locations.txt"):
 162   """
 163   Reads the locations at which the Mochitest HTTP server is available from
 164   server-locations.txt.
 165   """
 166
 167   locationFile = codecs.open(locationsPath, "r", "UTF-8")
 168
 169   # Perhaps more detail than necessary, but it's the easiest way to make sure
 170   # we get exactly the format we want.  See server-locations.txt for the exact
 171   # format guaranteed here.
 172   lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"
 173                       r"://"
 174                       r"(?P<host>"
 175                         r"\d+\.\d+\.\d+\.\d+"
 176                         r"|"
 177                         r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*"
 178                         r"[a-z](?:[-a-z0-9]*[a-z0-9])?"
 179                       r")"
 180                       r":"
 181                       r"(?P<port>\d+)"
 182                       r"(?:"
 183                       r"\s+"
 184                       r"(?P<options>\S+(?:,\S+)*)"
 185                       r")?$")
 186   locations = []
 187   lineno = 0
 188   seenPrimary = False
 189   for line in locationFile:
 190     lineno += 1
 191     if line.startswith("#") or line == "\n":
 192       continue
 193
 194     match = lineRe.match(line)
 195     if not match:
 196       raise SyntaxError(lineno)
 197
 198     options = match.group("options")
 199     if options:
 200       options = options.split(",")
 201       if "primary" in options:
 202         if seenPrimary:
 203           raise SyntaxError(lineno, "multiple primary locations")
 204         seenPrimary = True
 205     else:
 206       options = []
 207
 208     locations.append(Location(match.group("scheme"), match.group("host"),
 209                               match.group("port"), options))
 210
 211   if not seenPrimary:
 212     raise SyntaxError(lineno + 1, "missing primary location")
 213
 214   return locations
 215
 216
 217 def initializeProfile(profileDir):
 218   "Sets up the standard testing profile."
 219
 220   # Start with a clean slate.
 221   shutil.rmtree(profileDir, True)
 222   os.mkdir(profileDir)
 223
 224   prefs = []
 225
 226   part = """\
 227 user_pref("browser.dom.window.dump.enabled", true);
 228 user_pref("dom.allow_scripts_to_close_windows", true);
 229 user_pref("dom.disable_open_during_load", false);
 230 user_pref("dom.max_script_run_time", 0); // no slow script dialogs
 231 user_pref("signed.applets.codebase_principal_support", true);
 232 user_pref("security.warn_submit_insecure", false);
 233 user_pref("browser.shell.checkDefaultBrowser", false);
 234 user_pref("shell.checkDefaultClient", false);
 235 user_pref("browser.warnOnQuit", false);
 236 user_pref("accessibility.typeaheadfind.autostart", false);
 237 user_pref("javascript.options.showInConsole", true);
 238 user_pref("layout.debug.enable_data_xbl", true);
 239 user_pref("browser.EULA.override", true);
 240 user_pref("javascript.options.jit.content", true);
 241 user_pref("gfx.color_management.force_srgb", true);
 242 user_pref("network.manage-offline-status", false);
 243 user_pref("security.default_personal_cert", "Select Automatically"); // Need to client auth test be w/o any dialogs
 244 user_pref("network.http.prompt-temp-redirect", false);
 245
 246 user_pref("camino.warn_when_closing", false); // Camino-only, harmless to others
 247 """
 248
 249   prefs.append(part)
 250
 251   # Increase the max script run time 10-fold for debug builds
 252   if (IS_DEBUG_BUILD):
 253     prefs.append("""\
 254 user_pref("dom.max_script_run_time", 100);
 255 user_pref("dom.max_chrome_script_run_time", 200);
 256 """)
 257
 258   locations = readLocations()
 259
 260   # Grant God-power to all the privileged servers on which tests run.
 261   privileged = filter(lambda loc: "privileged" in loc.options, locations)
 262   for (i, l) in itertools.izip(itertools.count(1), privileged):
 263     part = """
 264 user_pref("capability.principal.codebase.p%(i)d.granted",
 265           "UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite \
 266            UniversalPreferencesRead UniversalPreferencesWrite \
 267            UniversalFileRead");
 268 user_pref("capability.principal.codebase.p%(i)d.id", "%(origin)s");
 269 user_pref("capability.principal.codebase.p%(i)d.subjectName", "");
 270 """  % { "i": i,
 271          "origin": (l.scheme + "://" + l.host + ":" + l.port) }
 272     prefs.append(part)
 273
 274   # We need to proxy every server but the primary one.
 275   origins = ["'%s://%s:%s'" % (l.scheme, l.host, l.port)
 276              for l in filter(lambda l: "primary" not in l.options, locations)]
 277   origins = ", ".join(origins)
 278
 279   pacURL = """data:text/plain,
 280 function FindProxyForURL(url, host)
 281 {
 282   var origins = [%(origins)s];
 283   var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
 284                          '://' +
 285                          '(?:[^/@]*@)?' +
 286                          '(.*?)' +
 287                          '(?::(\\\\\\\\d+))?/');
 288   var matches = regex.exec(url);
 289   if (!matches)
 290     return 'DIRECT';
 291   var isHttp = matches[1] == 'http';
 292   var isHttps = matches[1] == 'https';
 293   if (!matches[3])
 294   {
 295     if (isHttp) matches[3] = '80';
 296     if (isHttps) matches[3] = '443';
 297   }
 298
 299   var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
 300   if (origins.indexOf(origin) < 0)
 301     return 'DIRECT';
 302   if (isHttp)
 303     return 'PROXY 127.0.0.1:8888';
 304   if (isHttps)
 305     return 'PROXY 127.0.0.1:4443';
 306   return 'DIRECT';
 307 }""" % { "origins": origins }
 308   pacURL = "".join(pacURL.splitlines())
 309
 310   part = """
 311 user_pref("network.proxy.type", 2);
 312 user_pref("network.proxy.autoconfig_url", "%(pacURL)s");
 313
 314 user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless to others
 315 """ % {"pacURL": pacURL}
 316   prefs.append(part)
 317
 318   # write the preferences
 319   prefsFile = open(profileDir + "/" + "user.js", "a")
 320   prefsFile.write("".join(prefs))
 321   prefsFile.close()
 322
 323 def fillCertificateDB(profileDir, certPath, utilityPath, xrePath):
 324   pwfilePath = os.path.join(profileDir, ".crtdbpw")
 325
 326   pwfile = open(pwfilePath, "w")
 327   pwfile.write("\n")
 328   pwfile.close()
 329
 330   # Create head of the ssltunnel configuration file
 331   sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")
 332   sslTunnelConfig = open(sslTunnelConfigPath, "w")
 333
 334   sslTunnelConfig.write("httpproxy:1\n")
 335   sslTunnelConfig.write("certdbdir:%s\n" % certPath)
 336   sslTunnelConfig.write("forward:127.0.0.1:8888\n")
 337   sslTunnelConfig.write("listen:*:4443:pgo server certificate\n")
 338
 339   # Configure automatic certificate and bind custom certificates, client authentication
 340   locations = readLocations()
 341   locations.pop(0)
 342   for loc in locations:
 343     if loc.scheme == "https" and "nocert" not in loc.options:
 344       customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")
 345       clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")
 346       for option in loc.options:
 347         match = customCertRE.match(option)
 348         if match:
 349           customcert = match.group("nickname");
 350           sslTunnelConfig.write("listen:%s:%s:4443:%s\n" %
 351               (loc.host, loc.port, customcert))
 352
 353         match = clientAuthRE.match(option)
 354         if match:
 355           clientauth = match.group("clientauth");
 356           sslTunnelConfig.write("clientauth:%s:%s:4443:%s\n" %
 357               (loc.host, loc.port, clientauth))
 358
 359   sslTunnelConfig.close()
 360
 361   # Pre-create the certification database for the profile
 362   env = environment(xrePath = xrePath)
 363   certutil = os.path.join(utilityPath, "certutil" + BIN_SUFFIX)
 364   pk12util = os.path.join(utilityPath, "pk12util" + BIN_SUFFIX)
 365
 366   status = Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait()
 367   if status != 0:
 368     return status
 369
 370   # Walk the cert directory and add custom CAs and client certs
 371   files = os.listdir(certPath)
 372   for item in files:
 373     root, ext = os.path.splitext(item)
 374     if ext == ".ca":
 375       trustBits = "CT,,"
 376       if root.endswith("-object"):
 377         trustBits = "CT,,CT"
 378       Process([certutil, "-A", "-i", os.path.join(certPath, item),
 379         "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits],
 380         env = env).wait()
 381     if ext == ".client":
 382       Process([pk12util, "-i", os.path.join(certPath, item), "-w",
 383         pwfilePath, "-d", profileDir],
 384         env = env).wait()
 385
 386   os.unlink(pwfilePath)
 387   return 0
 388
 389 def environment(env = None, xrePath = DIST_BIN):
 390   if env == None:
 391     env = dict(os.environ)
 392
 393   ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath))
 394   if UNIXISH or IS_MAC:
 395     envVar = "LD_LIBRARY_PATH"
 396     if IS_MAC:
 397       envVar = "DYLD_LIBRARY_PATH"
 398     if envVar in env:
 399       ldLibraryPath = ldLibraryPath + ":" + env[envVar]
 400     env[envVar] = ldLibraryPath
 401   elif IS_WIN32:
 402     env["PATH"] = env["PATH"] + ";" + ldLibraryPath
 403
 404   return env
 405
 406 ###############
 407 # RUN THE APP #
 408 ###############
 409
 410 def runApp(testURL, env, app, profileDir, extraArgs, runSSLTunnel = False, utilityPath = DIST_BIN, xrePath = DIST_BIN, certPath = CERTS_SRC_DIR):
 411   "Run the app, returning a tuple containing the status code and the time at which it was started."
 412   if IS_TEST_BUILD and runSSLTunnel:
 413     # create certificate database for the profile
 414     certificateStatus = fillCertificateDB(profileDir, certPath, utilityPath, xrePath)
 415     if certificateStatus != 0:
 416       log.info("TEST-UNEXPECTED FAIL | Certificate integration failed")
 417       return certificateStatus
 418
 419     # start ssltunnel to provide https:// URLs capability
 420     ssltunnel = os.path.join(utilityPath, "ssltunnel" + BIN_SUFFIX)
 421     ssltunnelProcess = Process([ssltunnel, os.path.join(profileDir, "ssltunnel.cfg")], env = environment(xrePath = xrePath))
 422     log.info("SSL tunnel pid: %d", ssltunnelProcess.pid)
 423
 424   "Run the app, returning the time at which it was started."
 425   # mark the start
 426   start = datetime.now()
 427
 428   # now run with the profile we created
 429   cmd = app
 430   if IS_MAC and not IS_CAMINO and not cmd.endswith("-bin"):
 431     cmd += "-bin"
 432   cmd = os.path.abspath(cmd)
 433
 434   args = []
 435   if IS_MAC:
 436     args.append("-foreground")
 437
 438   if IS_CYGWIN:
 439     profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
 440   else:
 441     profileDirectory = profileDir + "/"
 442
 443   args.extend(("-no-remote", "-profile", profileDirectory))
 444   if testURL is not None:
 445     if IS_CAMINO:
 446       args.extend(("-url", testURL))
 447     else:
 448       args.append((testURL))
 449   args.extend(extraArgs)
 450   proc = Process([cmd] + args, env = environment(env), stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
 451   log.info("Application pid: %d", proc.pid)
 452   line = proc.stdout.readline()
 453   while line != "":
 454     log.info(line.rstrip())
 455     line = proc.stdout.readline()
 456   status = proc.wait()
 457   if status != 0:
 458     log.info("TEST-UNEXPECTED-FAIL | Exited with code %d during test run", status)
 459
 460   if IS_TEST_BUILD and runSSLTunnel:
 461     ssltunnelProcess.kill()
 462
 463   return (status, start)