testing/xpcshell/runxpcshelltests.py

   1 #!/usr/bin/env python
   2 #
   3 # ***** BEGIN LICENSE BLOCK *****
   4 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5 #
   6 # The contents of this file are subject to the Mozilla Public License Version
   7 # 1.1 (the "License"); you may not use this file except in compliance with
   8 # the License. You may obtain a copy of the License at
   9 # http://www.mozilla.org/MPL/
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13 # for the specific language governing rights and limitations under the
  14 # License.
  15 #
  16 # The Original Code is mozilla.org code.
  17 #
  18 # The Initial Developer of the Original Code is The Mozilla Foundation
  19 # Portions created by the Initial Developer are Copyright (C) 2009
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #  Serge Gautherie <sgautherie.bz@free.fr>
  24 #  Ted Mielczarek <ted.mielczarek@gmail.com>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK ***** */
  39
  40 import re, sys, os, os.path, logging, shutil
  41 import tempfile
  42 from glob import glob
  43 from optparse import OptionParser
  44 from subprocess import Popen, PIPE, STDOUT
  45 from tempfile import mkdtemp
  46
  47 from automationutils import addCommonOptions, checkForCrashes, dumpLeakLog
  48
  49 # Init logging
  50 log = logging.getLogger()
  51 handler = logging.StreamHandler(sys.stdout)
  52 log.setLevel(logging.INFO)
  53 log.addHandler(handler)
  54
  55 def readManifest(manifest):
  56   """Given a manifest file containing a list of test directories,
  57   return a list of absolute paths to the directories contained within."""
  58   manifestdir = os.path.dirname(manifest)
  59   testdirs = []
  60   try:
  61     f = open(manifest, "r")
  62     for line in f:
  63       dir = line.rstrip()
  64       path = os.path.join(manifestdir, dir)
  65       if os.path.isdir(path):
  66         testdirs.append(path)
  67     f.close()
  68   except:
  69     pass # just eat exceptions
  70   return testdirs
  71
  72 def runTests(xpcshell, testdirs=[], xrePath=None, testPath=None,
  73              manifest=None, interactive=False, symbolsPath=None):
  74   """Run the tests in |testdirs| using the |xpcshell| executable.
  75
  76   |xrePath|, if provided, is the path to the XRE to use.
  77   |testPath|, if provided, indicates a single path and/or test to run.
  78   |manifest|, if provided, is a file containing a list of
  79     test directories to run.
  80   |interactive|, if set to True, indicates to provide an xpcshell prompt
  81     instead of automatically executing the test.
  82   |symbolsPath|, if provided is the path to a directory containing
  83     breakpad symbols for processing crashes in tests.
  84   """
  85
  86   if not testdirs and not manifest:
  87     # nothing to test!
  88     print >>sys.stderr, "Error: No test dirs or test manifest specified!"
  89     return False
  90
  91   passCount = 0
  92   failCount = 0
  93
  94   testharnessdir = os.path.dirname(os.path.abspath(__file__))
  95   xpcshell = os.path.abspath(xpcshell)
  96   # we assume that httpd.js lives in components/ relative to xpcshell
  97   httpdJSPath = os.path.join(os.path.dirname(xpcshell), "components", "httpd.js").replace("\\", "/");
  98
  99   env = dict(os.environ)
 100   # Make assertions fatal
 101   env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 102   # Don't launch the crash reporter client
 103   env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
 104
 105   # Enable leaks (only) detection to its own log file.
 106   # Each test will overwrite it.
 107   leakLogFile = os.path.join(tempfile.gettempdir(), "runxpcshelltests_leaks.log")
 108   env["XPCOM_MEM_LEAK_LOG"] = leakLogFile
 109
 110   if xrePath is None:
 111     xrePath = os.path.dirname(xpcshell)
 112   else:
 113     xrePath = os.path.abspath(xrePath)
 114   if sys.platform == 'win32':
 115     env["PATH"] = env["PATH"] + ";" + xrePath
 116   elif sys.platform in ('os2emx', 'os2knix'):
 117     os.environ["BEGINLIBPATH"] = xrePath + ";" + env["BEGINLIBPATH"]
 118     os.environ["LIBPATHSTRICT"] = "T"
 119   elif sys.platform == 'osx':
 120     env["DYLD_LIBRARY_PATH"] = xrePath
 121   else: # unix or linux?
 122     env["LD_LIBRARY_PATH"] = xrePath
 123
 124   # xpcsRunArgs: <head.js> function to call to run the test.
 125   # pStdout, pStderr: Parameter values for later |Popen()| call.
 126   if interactive:
 127     xpcsRunArgs = [
 128       '-e', 'print("To start the test, type |_execute_test();|.");',
 129       '-i']
 130     pStdout = None
 131     pStderr = None
 132   else:
 133     xpcsRunArgs = ['-e', '_execute_test();']
 134     if sys.platform == 'os2emx':
 135       pStdout = None
 136     else:
 137       pStdout = PIPE
 138     pStderr = STDOUT
 139
 140   # <head.js> has to be loaded by xpchell: it can't load itself.
 141   xpcsCmd = [xpcshell, '-g', xrePath, '-j', '-s'] + \
 142             ['-e', 'const _HTTPD_JS_PATH = "%s";' % httpdJSPath,
 143              '-f', os.path.join(testharnessdir, 'head.js')]
 144
 145   # |testPath| will be the optional path only, or |None|.
 146   # |singleFile| will be the optional test only, or |None|.
 147   singleFile = None
 148   if testPath:
 149     if testPath.endswith('.js'):
 150       # Split into path and file.
 151       if testPath.find('/') == -1:
 152         # Test only.
 153         singleFile = testPath
 154         testPath = None
 155       else:
 156         # Both path and test.
 157         # Reuse |testPath| temporarily.
 158         testPath = testPath.rsplit('/', 1)
 159         singleFile = testPath[1]
 160         testPath = testPath[0]
 161     else:
 162       # Path only.
 163       # Simply remove optional ending separator.
 164       testPath = testPath.rstrip("/")
 165
 166   if manifest is not None:
 167     testdirs = readManifest(os.path.abspath(manifest))
 168
 169   # Process each test directory individually.
 170   for testdir in testdirs:
 171     if testPath and not testdir.endswith(testPath):
 172       continue
 173
 174     testdir = os.path.abspath(testdir)
 175
 176     # get the list of head and tail files from the directory
 177     testHeadFiles = []
 178     for f in sorted(glob(os.path.join(testdir, "head_*.js"))):
 179       if os.path.isfile(f):
 180         testHeadFiles += [f]
 181     testTailFiles = []
 182     # Tails are executed in the reverse order, to "match" heads order,
 183     # as in "h1-h2-h3 then t3-t2-t1".
 184     for f in reversed(sorted(glob(os.path.join(testdir, "tail_*.js")))):
 185       if os.path.isfile(f):
 186         testTailFiles += [f]
 187
 188     # if a single test file was specified, we only want to execute that test
 189     testfiles = sorted(glob(os.path.join(testdir, "test_*.js")))
 190     if singleFile:
 191       if singleFile in [os.path.basename(x) for x in testfiles]:
 192         testfiles = [os.path.join(testdir, singleFile)]
 193       else: # not in this dir? skip it
 194         continue
 195
 196     cmdH = ", ".join(['"' + f.replace('\\', '/') + '"'
 197                        for f in testHeadFiles])
 198     cmdT = ", ".join(['"' + f.replace('\\', '/') + '"'
 199                        for f in testTailFiles])
 200     cmdH = xpcsCmd + \
 201            ['-e', 'const _HEAD_FILES = [%s];' % cmdH] + \
 202            ['-e', 'const _TAIL_FILES = [%s];' % cmdT]
 203
 204     # Now execute each test individually.
 205     for test in testfiles:
 206       # The test file will have to be loaded after the head files.
 207       cmdT = ['-e', 'const _TEST_FILE = ["%s"];' %
 208                       os.path.join(testdir, test).replace('\\', '/')]
 209       # create a temp dir that the JS harness can stick a profile in
 210       profd = mkdtemp()
 211       env["XPCSHELL_TEST_PROFILE_DIR"] = profd
 212
 213       proc = Popen(cmdH + cmdT + xpcsRunArgs,
 214                    stdout=pStdout, stderr=pStderr, env=env, cwd=testdir)
 215       # |stderr == None| as |pStderr| was either |None| or redirected to |stdout|.
 216       stdout, stderr = proc.communicate()
 217
 218       shutil.rmtree(profd, True)
 219
 220       if interactive:
 221         # not sure what else to do here...
 222         return True
 223
 224       if proc.returncode != 0 or (stdout is not None and re.search("^TEST-UNEXPECTED-FAIL", stdout, re.MULTILINE)):
 225         print """TEST-UNEXPECTED-FAIL | %s | test failed (with xpcshell return code: %d), see following log:
 226   >>>>>>>
 227   %s
 228   <<<<<<<""" % (test, proc.returncode, stdout)
 229         checkForCrashes(testdir, symbolsPath, testName=test)
 230         failCount += 1
 231       else:
 232         print "TEST-PASS | %s | test passed" % test
 233         passCount += 1
 234
 235       dumpLeakLog(leakLogFile, True)
 236
 237       if stdout is not None:
 238         try:
 239           f = open(test + '.log', 'w')
 240           f.write(stdout)
 241
 242           if os.path.exists(leakLogFile):
 243             leaks = open(leakLogFile, "r")
 244             f.write(leaks.read())
 245             leaks.close()
 246         finally:
 247           if f:
 248             f.close()
 249
 250       # Remove the leak detection file (here) so it can't "leak" to the next test.
 251       # The file is not there if leak logging was not enabled in the xpcshell build.
 252       if os.path.exists(leakLogFile):
 253         os.remove(leakLogFile)
 254
 255   if passCount == 0 and failCount == 0:
 256     print "TEST-UNEXPECTED-FAIL | runxpcshelltests.py | No tests run. Did you pass an invalid --test-path?"
 257     failCount = 1
 258
 259   print """INFO | Result summary:
 260 INFO | Passed: %d
 261 INFO | Failed: %d""" % (passCount, failCount)
 262
 263   return failCount == 0
 264
 265 def main():
 266   """Process command line arguments and call runTests() to do the real work."""
 267   parser = OptionParser()
 268
 269   addCommonOptions(parser)
 270   parser.add_option("--test-path",
 271                     action="store", type="string", dest="testPath",
 272                     default=None, help="single path and/or test filename to test")
 273   parser.add_option("--interactive",
 274                     action="store_true", dest="interactive", default=False,
 275                     help="don't automatically run tests, drop to an xpcshell prompt")
 276   parser.add_option("--manifest",
 277                     action="store", type="string", dest="manifest",
 278                     default=None, help="Manifest of test directories to use")
 279   options, args = parser.parse_args()
 280
 281   if len(args) < 2 and options.manifest is None or \
 282      (len(args) < 1 and options.manifest is not None):
 283     print >>sys.stderr, """Usage: %s <path to xpcshell> <test dirs>
 284   or: %s --manifest=test.manifest <path to xpcshell>""" % (sys.argv[0],
 285                                                            sys.argv[0])
 286     sys.exit(1)
 287
 288   if options.interactive and not options.testPath:
 289     print >>sys.stderr, "Error: You must specify a test filename in interactive mode!"
 290     sys.exit(1)
 291
 292   if not runTests(args[0], testdirs=args[1:],
 293                   xrePath=options.xrePath,
 294                   testPath=options.testPath,
 295                   interactive=options.interactive,
 296                   manifest=options.manifest,
 297                   symbolsPath=options.symbolsPath):
 298     sys.exit(1)
 299
 300 if __name__ == '__main__':
 301   main()