build/automation.py.in

   1 #
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   4 #
   5 # The contents of this file are subject to the Mozilla Public License Version
   6 # 1.1 (the "License"); you may not use this file except in compliance with
   7 # the License. You may obtain a copy of the License at
   8 # http://www.mozilla.org/MPL/
   9 #
  10 # Software distributed under the License is distributed on an "AS IS" basis,
  11 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12 # for the specific language governing rights and limitations under the
  13 # License.
  14 #
  15 # The Original Code is mozilla.org code.
  16 #
  17 # The Initial Developer of the Original Code is
  18 # Mozilla Foundation.
  19 # Portions created by the Initial Developer are Copyright (C) 2008
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #   Robert Sayre <sayrer@gmail.com>
  24 #   Jeff Walden <jwalden+bmo@mit.edu>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 import codecs
  41 from datetime import datetime
  42 import itertools
  43 import logging
  44 import os
  45 import re
  46 import shutil
  47 import signal
  48 import subprocess
  49 import sys
  50 import threading
  51 import glob
  52
  53 """
  54 Runs the browser from a script, and provides useful utilities
  55 for setting up the browser environment.
  56 """
  57
  58 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
  59
  60 __all__ = [
  61            "UNIXISH",
  62            "IS_WIN32",
  63            "IS_MAC",
  64            "runApp",
  65            "Process",
  66            "initializeProfile",
  67            "DIST_BIN",
  68            "DEFAULT_APP",
  69            "CERTS_SRC_DIR",
  70            "environment",
  71           ]
  72
  73 # These are generated in mozilla/build/Makefile.in
  74 #expand DIST_BIN = __XPC_BIN_PATH__
  75 #expand IS_WIN32 = len("__WIN32__") != 0
  76 #expand IS_MAC = __IS_MAC__ != 0
  77 #ifdef IS_CYGWIN
  78 #expand IS_CYGWIN = __IS_CYGWIN__ == 1
  79 #else
  80 IS_CYGWIN = False
  81 #endif
  82 #expand IS_CAMINO = __IS_CAMINO__ != 0
  83 #expand BIN_SUFFIX = __BIN_SUFFIX__
  84
  85 UNIXISH = not IS_WIN32 and not IS_MAC
  86
  87 #expand DEFAULT_APP = "./" + __BROWSER_PATH__
  88 #expand CERTS_SRC_DIR = __CERTS_SRC_DIR__
  89 #expand IS_TEST_BUILD = __IS_TEST_BUILD__
  90 #expand IS_DEBUG_BUILD = __IS_DEBUG_BUILD__
  91 #expand SYMBOLS_PATH = __SYMBOLS_PATH__
  92
  93 ###########
  94 # LOGGING #
  95 ###########
  96
  97 # We use the logging system here primarily because it'll handle multiple
  98 # threads, which is needed to process the output of the server and application
  99 # processes simultaneously.
 100 log = logging.getLogger()
 101 handler = logging.StreamHandler(sys.stdout)
 102 log.setLevel(logging.INFO)
 103 log.addHandler(handler)
 104
 105
 106 #################
 107 # SUBPROCESSING #
 108 #################
 109
 110 class Process(subprocess.Popen):
 111   """
 112   Represents our view of a subprocess.
 113   It adds a kill() method which allows it to be stopped explicitly.
 114   """
 115
 116   def kill(self):
 117     if IS_WIN32:
 118       import platform
 119       pid = "%i" % self.pid
 120       if platform.release() == "2000":
 121         # Windows 2000 needs 'kill.exe' from the 'Windows 2000 Resource Kit tools'. (See bug 475455.)
 122         try:
 123           subprocess.Popen(["kill", "-f", pid]).wait()
 124         except:
 125           log.info("TEST-UNEXPECTED-FAIL | (automation.py) | Missing 'kill' utility to kill process with pid=%s. Kill it manually!", pid)
 126       else:
 127         # Windows XP and later.
 128         subprocess.Popen(["taskkill", "/F", "/PID", pid]).wait()
 129     else:
 130       os.kill(self.pid, signal.SIGKILL)
 131
 132
 133 #################
 134 # PROFILE SETUP #
 135 #################
 136
 137 class SyntaxError(Exception):
 138   "Signifies a syntax error on a particular line in server-locations.txt."
 139
 140   def __init__(self, lineno, msg = None):
 141     self.lineno = lineno
 142     self.msg = msg
 143
 144   def __str__(self):
 145     s = "Syntax error on line " + str(self.lineno)
 146     if self.msg:
 147       s += ": %s." % self.msg
 148     else:
 149       s += "."
 150     return s
 151
 152
 153 class Location:
 154   "Represents a location line in server-locations.txt."
 155
 156   def __init__(self, scheme, host, port, options):
 157     self.scheme = scheme
 158     self.host = host
 159     self.port = port
 160     self.options = options
 161
 162
 163 def readLocations(locationsPath = "server-locations.txt"):
 164   """
 165   Reads the locations at which the Mochitest HTTP server is available from
 166   server-locations.txt.
 167   """
 168
 169   locationFile = codecs.open(locationsPath, "r", "UTF-8")
 170
 171   # Perhaps more detail than necessary, but it's the easiest way to make sure
 172   # we get exactly the format we want.  See server-locations.txt for the exact
 173   # format guaranteed here.
 174   lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"
 175                       r"://"
 176                       r"(?P<host>"
 177                         r"\d+\.\d+\.\d+\.\d+"
 178                         r"|"
 179                         r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*"
 180                         r"[a-z](?:[-a-z0-9]*[a-z0-9])?"
 181                       r")"
 182                       r":"
 183                       r"(?P<port>\d+)"
 184                       r"(?:"
 185                       r"\s+"
 186                       r"(?P<options>\S+(?:,\S+)*)"
 187                       r")?$")
 188   locations = []
 189   lineno = 0
 190   seenPrimary = False
 191   for line in locationFile:
 192     lineno += 1
 193     if line.startswith("#") or line == "\n":
 194       continue
 195
 196     match = lineRe.match(line)
 197     if not match:
 198       raise SyntaxError(lineno)
 199
 200     options = match.group("options")
 201     if options:
 202       options = options.split(",")
 203       if "primary" in options:
 204         if seenPrimary:
 205           raise SyntaxError(lineno, "multiple primary locations")
 206         seenPrimary = True
 207     else:
 208       options = []
 209
 210     locations.append(Location(match.group("scheme"), match.group("host"),
 211                               match.group("port"), options))
 212
 213   if not seenPrimary:
 214     raise SyntaxError(lineno + 1, "missing primary location")
 215
 216   return locations
 217
 218
 219 def initializeProfile(profileDir):
 220   "Sets up the standard testing profile."
 221
 222   # Start with a clean slate.
 223   shutil.rmtree(profileDir, True)
 224   os.mkdir(profileDir)
 225
 226   prefs = []
 227
 228   part = """\
 229 user_pref("browser.dom.window.dump.enabled", true);
 230 user_pref("dom.allow_scripts_to_close_windows", true);
 231 user_pref("dom.disable_open_during_load", false);
 232 user_pref("dom.max_script_run_time", 0); // no slow script dialogs
 233 user_pref("signed.applets.codebase_principal_support", true);
 234 user_pref("security.warn_submit_insecure", false);
 235 user_pref("browser.shell.checkDefaultBrowser", false);
 236 user_pref("shell.checkDefaultClient", false);
 237 user_pref("browser.warnOnQuit", false);
 238 user_pref("accessibility.typeaheadfind.autostart", false);
 239 user_pref("javascript.options.showInConsole", true);
 240 user_pref("layout.debug.enable_data_xbl", true);
 241 user_pref("browser.EULA.override", true);
 242 user_pref("javascript.options.jit.content", true);
 243 user_pref("gfx.color_management.force_srgb", true);
 244 user_pref("network.manage-offline-status", false);
 245 user_pref("security.default_personal_cert", "Select Automatically"); // Need to client auth test be w/o any dialogs
 246 user_pref("network.http.prompt-temp-redirect", false);
 247 user_pref("svg.smil.enabled", true); // Needed for SMIL mochitests until bug 482402 lands
 248
 249 user_pref("camino.warn_when_closing", false); // Camino-only, harmless to others
 250 """
 251
 252   prefs.append(part)
 253
 254   # Increase the max script run time 10-fold for debug builds
 255   if (IS_DEBUG_BUILD):
 256     prefs.append("""\
 257 user_pref("dom.max_script_run_time", 100);
 258 user_pref("dom.max_chrome_script_run_time", 200);
 259 """)
 260
 261   locations = readLocations()
 262
 263   # Grant God-power to all the privileged servers on which tests run.
 264   privileged = filter(lambda loc: "privileged" in loc.options, locations)
 265   for (i, l) in itertools.izip(itertools.count(1), privileged):
 266     part = """
 267 user_pref("capability.principal.codebase.p%(i)d.granted",
 268           "UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite \
 269            UniversalPreferencesRead UniversalPreferencesWrite \
 270            UniversalFileRead");
 271 user_pref("capability.principal.codebase.p%(i)d.id", "%(origin)s");
 272 user_pref("capability.principal.codebase.p%(i)d.subjectName", "");
 273 """  % { "i": i,
 274          "origin": (l.scheme + "://" + l.host + ":" + l.port) }
 275     prefs.append(part)
 276
 277   # We need to proxy every server but the primary one.
 278   origins = ["'%s://%s:%s'" % (l.scheme, l.host, l.port)
 279              for l in filter(lambda l: "primary" not in l.options, locations)]
 280   origins = ", ".join(origins)
 281
 282   pacURL = """data:text/plain,
 283 function FindProxyForURL(url, host)
 284 {
 285   var origins = [%(origins)s];
 286   var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
 287                          '://' +
 288                          '(?:[^/@]*@)?' +
 289                          '(.*?)' +
 290                          '(?::(\\\\\\\\d+))?/');
 291   var matches = regex.exec(url);
 292   if (!matches)
 293     return 'DIRECT';
 294   var isHttp = matches[1] == 'http';
 295   var isHttps = matches[1] == 'https';
 296   if (!matches[3])
 297   {
 298     if (isHttp) matches[3] = '80';
 299     if (isHttps) matches[3] = '443';
 300   }
 301
 302   var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
 303   if (origins.indexOf(origin) < 0)
 304     return 'DIRECT';
 305   if (isHttp)
 306     return 'PROXY 127.0.0.1:8888';
 307   if (isHttps)
 308     return 'PROXY 127.0.0.1:4443';
 309   return 'DIRECT';
 310 }""" % { "origins": origins }
 311   pacURL = "".join(pacURL.splitlines())
 312
 313   part = """
 314 user_pref("network.proxy.type", 2);
 315 user_pref("network.proxy.autoconfig_url", "%(pacURL)s");
 316
 317 user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless to others
 318 """ % {"pacURL": pacURL}
 319   prefs.append(part)
 320
 321   # write the preferences
 322   prefsFile = open(profileDir + "/" + "user.js", "a")
 323   prefsFile.write("".join(prefs))
 324   prefsFile.close()
 325
 326 def fillCertificateDB(profileDir, certPath, utilityPath, xrePath):
 327   pwfilePath = os.path.join(profileDir, ".crtdbpw")
 328
 329   pwfile = open(pwfilePath, "w")
 330   pwfile.write("\n")
 331   pwfile.close()
 332
 333   # Create head of the ssltunnel configuration file
 334   sslTunnelConfigPath = os.path.join(profileDir, "ssltunnel.cfg")
 335   sslTunnelConfig = open(sslTunnelConfigPath, "w")
 336
 337   sslTunnelConfig.write("httpproxy:1\n")
 338   sslTunnelConfig.write("certdbdir:%s\n" % certPath)
 339   sslTunnelConfig.write("forward:127.0.0.1:8888\n")
 340   sslTunnelConfig.write("listen:*:4443:pgo server certificate\n")
 341
 342   # Configure automatic certificate and bind custom certificates, client authentication
 343   locations = readLocations()
 344   locations.pop(0)
 345   for loc in locations:
 346     if loc.scheme == "https" and "nocert" not in loc.options:
 347       customCertRE = re.compile("^cert=(?P<nickname>[0-9a-zA-Z_ ]+)")
 348       clientAuthRE = re.compile("^clientauth=(?P<clientauth>[a-z]+)")
 349       for option in loc.options:
 350         match = customCertRE.match(option)
 351         if match:
 352           customcert = match.group("nickname");
 353           sslTunnelConfig.write("listen:%s:%s:4443:%s\n" %
 354               (loc.host, loc.port, customcert))
 355
 356         match = clientAuthRE.match(option)
 357         if match:
 358           clientauth = match.group("clientauth");
 359           sslTunnelConfig.write("clientauth:%s:%s:4443:%s\n" %
 360               (loc.host, loc.port, clientauth))
 361
 362   sslTunnelConfig.close()
 363
 364   # Pre-create the certification database for the profile
 365   env = environment(xrePath = xrePath)
 366   certutil = os.path.join(utilityPath, "certutil" + BIN_SUFFIX)
 367   pk12util = os.path.join(utilityPath, "pk12util" + BIN_SUFFIX)
 368
 369   status = Process([certutil, "-N", "-d", profileDir, "-f", pwfilePath], env = env).wait()
 370   if status != 0:
 371     return status
 372
 373   # Walk the cert directory and add custom CAs and client certs
 374   files = os.listdir(certPath)
 375   for item in files:
 376     root, ext = os.path.splitext(item)
 377     if ext == ".ca":
 378       trustBits = "CT,,"
 379       if root.endswith("-object"):
 380         trustBits = "CT,,CT"
 381       Process([certutil, "-A", "-i", os.path.join(certPath, item),
 382         "-d", profileDir, "-f", pwfilePath, "-n", root, "-t", trustBits],
 383         env = env).wait()
 384     if ext == ".client":
 385       Process([pk12util, "-i", os.path.join(certPath, item), "-w",
 386         pwfilePath, "-d", profileDir],
 387         env = env).wait()
 388
 389   os.unlink(pwfilePath)
 390   return 0
 391
 392 def environment(env = None, xrePath = DIST_BIN):
 393   if env == None:
 394     env = dict(os.environ)
 395
 396   ldLibraryPath = os.path.abspath(os.path.join(SCRIPT_DIR, xrePath))
 397   if UNIXISH or IS_MAC:
 398     envVar = "LD_LIBRARY_PATH"
 399     if IS_MAC:
 400       envVar = "DYLD_LIBRARY_PATH"
 401     if envVar in env:
 402       ldLibraryPath = ldLibraryPath + ":" + env[envVar]
 403     env[envVar] = ldLibraryPath
 404   elif IS_WIN32:
 405     env["PATH"] = env["PATH"] + ";" + ldLibraryPath
 406
 407   env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
 408   env['MOZ_CRASHREPORTER'] = '1'
 409
 410   return env
 411
 412 def checkForCrashes(profileDir, symbolsPath):
 413   stackwalkPath = os.environ.get('MINIDUMP_STACKWALK', None)
 414
 415   foundCrash = False
 416   dumps = glob.glob(os.path.join(profileDir, 'minidumps', '*.dmp'))
 417   for d in dumps:
 418     log.info("TEST-UNEXPECTED-FAIL | (automation.py) | Browser crashed (minidump found)")
 419     if symbolsPath and stackwalkPath:
 420       nullfd = open(os.devnull, 'w')
 421       # eat minidump_stackwalk errors
 422       subprocess.call([stackwalkPath, d, symbolsPath], stderr=nullfd)
 423       nullfd.close()
 424     os.remove(d)
 425     foundCrash = True
 426   return foundCrash
 427
 428 ###############
 429 # RUN THE APP #
 430 ###############
 431
 432 def processLeakLog(leakLogFile, leakThreshold):
 433   "Process the leak log."
 434
 435   if not os.path.exists(leakLogFile):
 436     log.info("WARNING refcount logging is off, so leaks can't be detected!")
 437   else:
 438     #                  Per-Inst  Leaked      Total  Rem ...
 439     #   0 TOTAL              17     192  419115886    2 ...
 440     # 833 nsTimerImpl        60     120      24726    2 ...
 441     lineRe = re.compile(r"^\s*\d+\s+(?P<name>\S+)\s+"
 442                         r"(?P<size>-?\d+)\s+(?P<bytesLeaked>-?\d+)\s+"
 443                         r"\d+\s+(?P<numLeaked>-?\d+)")
 444
 445     leaks = open(leakLogFile, "r")
 446     for line in leaks:
 447       matches = lineRe.match(line)
 448       if (matches and
 449           int(matches.group("numLeaked")) == 0 and
 450           matches.group("name") != "TOTAL"):
 451         continue
 452       log.info(line.rstrip())
 453     leaks.close()
 454
 455     leaks = open(leakLogFile, "r")
 456     seenTotal = False
 457     prefix = "TEST-PASS"
 458     for line in leaks:
 459       matches = lineRe.match(line)
 460       if not matches:
 461         continue
 462       name = matches.group("name")
 463       size = int(matches.group("size"))
 464       bytesLeaked = int(matches.group("bytesLeaked"))
 465       numLeaked = int(matches.group("numLeaked"))
 466       if size < 0 or bytesLeaked < 0 or numLeaked < 0:
 467         log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | negative leaks caught!")
 468       if "TOTAL" == name:
 469         seenTotal = True
 470         # Check for leaks.
 471         if bytesLeaked < 0 or bytesLeaked > leakThreshold:
 472           prefix = "TEST-UNEXPECTED-FAIL"
 473           leakLog = "TEST-UNEXPECTED-FAIL | runtests-leaks | leaked" \
 474                     " %d bytes during test execution" % bytesLeaked
 475         elif bytesLeaked > 0:
 476           leakLog = "TEST-PASS | runtests-leaks | WARNING leaked" \
 477                     " %d bytes during test execution" % bytesLeaked
 478         else:
 479           leakLog = "TEST-PASS | runtests-leaks | no leaks detected!"
 480         # Remind the threshold if it is not 0, which is the default/goal.
 481         if leakThreshold != 0:
 482           leakLog += " (threshold set at %d bytes)" % leakThreshold
 483         # Log the information.
 484         log.info(leakLog)
 485       else:
 486         if numLeaked != 0:
 487           if abs(numLeaked) > 1:
 488             instance = "instances"
 489             rest = " each (%s bytes total)" % matches.group("bytesLeaked")
 490           else:
 491             instance = "instance"
 492             rest = ""
 493           log.info("%(prefix)s | runtests-leaks | leaked %(numLeaked)d %(instance)s of %(name)s "
 494                    "with size %(size)s bytes%(rest)s" %
 495                    { "prefix": prefix,
 496                      "numLeaked": numLeaked,
 497                      "instance": instance,
 498                      "name": name,
 499                      "size": matches.group("size"),
 500                      "rest": rest })
 501     if not seenTotal:
 502       log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | missing output line for total leaks!")
 503     leaks.close()
 504
 505 def runApp(testURL, env, app, profileDir, extraArgs,
 506            runSSLTunnel = False, utilityPath = DIST_BIN,
 507            xrePath = DIST_BIN, certPath = CERTS_SRC_DIR,
 508            debuggerInfo = None, symbolsPath = SYMBOLS_PATH):
 509   "Run the app, log the duration it took to execute, return the status code."
 510
 511   if IS_TEST_BUILD and runSSLTunnel:
 512     # create certificate database for the profile
 513     certificateStatus = fillCertificateDB(profileDir, certPath, utilityPath, xrePath)
 514     if certificateStatus != 0:
 515       log.info("TEST-UNEXPECTED FAIL | (automation.py) | Certificate integration failed")
 516       return certificateStatus
 517
 518     # start ssltunnel to provide https:// URLs capability
 519     ssltunnel = os.path.join(utilityPath, "ssltunnel" + BIN_SUFFIX)
 520     ssltunnelProcess = Process([ssltunnel, os.path.join(profileDir, "ssltunnel.cfg")], env = environment(xrePath = xrePath))
 521     log.info("INFO | (automation.py) | SSL tunnel pid: %d", ssltunnelProcess.pid)
 522
 523   # now run with the profile we created
 524   cmd = app
 525   if IS_MAC and not IS_CAMINO and not cmd.endswith("-bin"):
 526     cmd += "-bin"
 527   cmd = os.path.abspath(cmd)
 528
 529   args = []
 530
 531   if debuggerInfo:
 532     args.extend(debuggerInfo["args"])
 533     args.append(cmd)
 534     cmd = os.path.abspath(debuggerInfo["path"])
 535
 536   if IS_MAC:
 537     args.append("-foreground")
 538
 539   if IS_CYGWIN:
 540     profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
 541   else:
 542     profileDirectory = profileDir + "/"
 543
 544   args.extend(("-no-remote", "-profile", profileDirectory))
 545   if testURL is not None:
 546     if IS_CAMINO:
 547       args.extend(("-url", testURL))
 548     else:
 549       args.append((testURL))
 550   args.extend(extraArgs)
 551
 552   startTime = datetime.now()
 553
 554   # Don't redirect stdout and stderr if an interactive debugger is attached
 555   if debuggerInfo and debuggerInfo["interactive"]:
 556     outputPipe = None
 557   else:
 558     outputPipe = subprocess.PIPE
 559
 560   proc = Process([cmd] + args, env = environment(env), stdout = outputPipe, stderr = subprocess.STDOUT)
 561   log.info("INFO | (automation.py) | Application pid: %d", proc.pid)
 562
 563   if outputPipe is None:
 564     log.info("TEST-INFO: Not logging stdout or stderr due to debugger connection")
 565   else:
 566     line = proc.stdout.readline()
 567     while line != "":
 568       log.info(line.rstrip())
 569       line = proc.stdout.readline()
 570
 571   status = proc.wait()
 572   if status != 0:
 573     log.info("TEST-UNEXPECTED-FAIL | (automation.py) | Exited with code %d during test run", status)
 574   log.info("INFO | (automation.py) | Application ran for: %s", str(datetime.now() - startTime))
 575
 576   if checkForCrashes(profileDir, symbolsPath):
 577     status = -1
 578
 579   if IS_TEST_BUILD and runSSLTunnel:
 580     ssltunnelProcess.kill()
 581
 582   return status