testing/xpcshell/selftest.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6
   7 from __future__ import with_statement
   8 import sys, os, unittest, tempfile, shutil
   9 import mozinfo
  10
  11 from StringIO import StringIO
  12 from xml.etree.ElementTree import ElementTree
  13
  14 from mozbuild.base import MozbuildObject
  15 os.environ.pop('MOZ_OBJDIR', None)
  16 build_obj = MozbuildObject.from_environment()
  17
  18 from runxpcshelltests import XPCShellTests
  19
  20 mozinfo.find_and_update_from_json()
  21
  22 objdir = build_obj.topobjdir.encode("utf-8")
  23
  24 if mozinfo.isMac:
  25   from buildconfig import substs
  26   xpcshellBin = os.path.join(objdir, "dist", substs['MOZ_MACBUNDLE_NAME'], "Contents", "MacOS", "xpcshell")
  27 else:
  28   xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
  29   if sys.platform == "win32":
  30     xpcshellBin += ".exe"
  31
  32 SIMPLE_PASSING_TEST = "function run_test() { do_check_true(true); }"
  33 SIMPLE_FAILING_TEST = "function run_test() { do_check_true(false); }"
  34
  35 ADD_TEST_SIMPLE = '''
  36 function run_test() { run_next_test(); }
  37
  38 add_test(function test_simple() {
  39   do_check_true(true);
  40   run_next_test();
  41 });
  42 '''
  43
  44 ADD_TEST_FAILING = '''
  45 function run_test() { run_next_test(); }
  46
  47 add_test(function test_failing() {
  48   do_check_true(false);
  49   run_next_test();
  50 });
  51 '''
  52
  53 CHILD_TEST_PASSING = '''
  54 function run_test () { run_next_test(); }
  55
  56 add_test(function test_child_simple () {
  57   run_test_in_child("test_pass.js");
  58   run_next_test();
  59 });
  60 '''
  61
  62 CHILD_TEST_FAILING = '''
  63 function run_test () { run_next_test(); }
  64
  65 add_test(function test_child_simple () {
  66   run_test_in_child("test_fail.js");
  67   run_next_test();
  68 });
  69 '''
  70
  71 CHILD_TEST_HANG = '''
  72 function run_test () { run_next_test(); }
  73
  74 add_test(function test_child_simple () {
  75   do_test_pending("hang test");
  76   do_load_child_test_harness();
  77   sendCommand("_log('child_test_start', {_message: 'CHILD-TEST-STARTED'}); " +
  78               + "const _TEST_FILE=['test_pass.js']; _execute_test(); ",
  79               do_test_finished);
  80   run_next_test();
  81 });
  82 '''
  83
  84 ADD_TASK_SINGLE = '''
  85 Components.utils.import("resource://gre/modules/Promise.jsm");
  86
  87 function run_test() { run_next_test(); }
  88
  89 add_task(function test_task() {
  90   yield Promise.resolve(true);
  91   yield Promise.resolve(false);
  92 });
  93 '''
  94
  95 ADD_TASK_MULTIPLE = '''
  96 Components.utils.import("resource://gre/modules/Promise.jsm");
  97
  98 function run_test() { run_next_test(); }
  99
 100 add_task(function test_task() {
 101   yield Promise.resolve(true);
 102 });
 103
 104 add_task(function test_2() {
 105   yield Promise.resolve(true);
 106 });
 107 '''
 108
 109 ADD_TASK_REJECTED = '''
 110 Components.utils.import("resource://gre/modules/Promise.jsm");
 111
 112 function run_test() { run_next_test(); }
 113
 114 add_task(function test_failing() {
 115   yield Promise.reject(new Error("I fail."));
 116 });
 117 '''
 118
 119 ADD_TASK_FAILURE_INSIDE = '''
 120 Components.utils.import("resource://gre/modules/Promise.jsm");
 121
 122 function run_test() { run_next_test(); }
 123
 124 add_task(function test() {
 125   let result = yield Promise.resolve(false);
 126
 127   do_check_true(result);
 128 });
 129 '''
 130
 131 ADD_TASK_RUN_NEXT_TEST = '''
 132 function run_test() { run_next_test(); }
 133
 134 add_task(function () {
 135   Assert.ok(true);
 136
 137   run_next_test();
 138 });
 139 '''
 140
 141 ADD_TASK_STACK_TRACE = '''
 142 Components.utils.import("resource://gre/modules/Promise.jsm", this);
 143
 144 function run_test() { run_next_test(); }
 145
 146 add_task(function* this_test_will_fail() {
 147   for (let i = 0; i < 10; ++i) {
 148     yield Promise.resolve();
 149   }
 150   Assert.ok(false);
 151 });
 152 '''
 153
 154 ADD_TASK_STACK_TRACE_WITHOUT_STAR = '''
 155 Components.utils.import("resource://gre/modules/Promise.jsm", this);
 156
 157 function run_test() { run_next_test(); }
 158
 159 add_task(function this_test_will_fail() {
 160   for (let i = 0; i < 10; ++i) {
 161     yield Promise.resolve();
 162   }
 163   Assert.ok(false);
 164 });
 165 '''
 166
 167 ADD_TEST_THROW_STRING = '''
 168 function run_test() {do_throw("Passing a string to do_throw")};
 169 '''
 170
 171 ADD_TEST_THROW_OBJECT = '''
 172 let error = {
 173   message: "Error object",
 174   fileName: "failure.js",
 175   stack: "ERROR STACK",
 176   toString: function() {return this.message;}
 177 };
 178 function run_test() {do_throw(error)};
 179 '''
 180
 181 ADD_TEST_REPORT_OBJECT = '''
 182 let error = {
 183   message: "Error object",
 184   fileName: "failure.js",
 185   stack: "ERROR STACK",
 186   toString: function() {return this.message;}
 187 };
 188 function run_test() {do_report_unexpected_exception(error)};
 189 '''
 190
 191 # A test for genuine JS-generated Error objects
 192 ADD_TEST_REPORT_REF_ERROR = '''
 193 function run_test() {
 194   let obj = {blah: 0};
 195   try {
 196     obj.noSuchFunction();
 197   }
 198   catch (error) {
 199     do_report_unexpected_exception(error);
 200   }
 201 };
 202 '''
 203
 204 # A test for failure to load a test due to a syntax error
 205 LOAD_ERROR_SYNTAX_ERROR = '''
 206 function run_test(
 207 '''
 208
 209 # A test for failure to load a test due to an error other than a syntax error
 210 LOAD_ERROR_OTHER_ERROR = '''
 211 function run_test() {
 212     yield "foo";
 213     return "foo"; // can't use return in a generator!
 214 };
 215 '''
 216
 217 # A test for asynchronous cleanup functions
 218 ASYNC_CLEANUP = '''
 219 function run_test() {
 220   Components.utils.import("resource://gre/modules/Promise.jsm", this);
 221
 222   // The list of checkpoints in the order we encounter them.
 223   let checkpoints = [];
 224
 225   // Cleanup tasks, in reverse order
 226   do_register_cleanup(function cleanup_checkout() {
 227     do_check_eq(checkpoints.join(""), "1234");
 228     do_print("At this stage, the test has succeeded");
 229     do_throw("Throwing an error to force displaying the log");
 230   });
 231
 232   do_register_cleanup(function sync_cleanup_2() {
 233     checkpoints.push(4);
 234   });
 235
 236   do_register_cleanup(function async_cleanup_2() {
 237     let deferred = Promise.defer();
 238     do_execute_soon(deferred.resolve);
 239     return deferred.promise.then(function() {
 240       checkpoints.push(3);
 241     });
 242   });
 243
 244   do_register_cleanup(function sync_cleanup() {
 245     checkpoints.push(2);
 246   });
 247
 248   do_register_cleanup(function async_cleanup() {
 249     let deferred = Promise.defer();
 250     do_execute_soon(deferred.resolve);
 251     return deferred.promise.then(function() {
 252       checkpoints.push(1);
 253     });
 254   });
 255
 256 }
 257 '''
 258
 259
 260 class XPCShellTestsTests(unittest.TestCase):
 261     """
 262     Yes, these are unit tests for a unit test harness.
 263     """
 264     def setUp(self):
 265         self.log = StringIO()
 266         self.tempdir = tempfile.mkdtemp()
 267         self.x = XPCShellTests(log=self.log)
 268
 269     def tearDown(self):
 270         shutil.rmtree(self.tempdir)
 271
 272     def writeFile(self, name, contents):
 273         """
 274         Write |contents| to a file named |name| in the temp directory,
 275         and return the full path to the file.
 276         """
 277         fullpath = os.path.join(self.tempdir, name)
 278         with open(fullpath, "w") as f:
 279             f.write(contents)
 280         return fullpath
 281
 282     def writeManifest(self, tests):
 283         """
 284         Write an xpcshell.ini in the temp directory and set
 285         self.manifest to its pathname. |tests| is a list containing
 286         either strings (for test names), or tuples with a test name
 287         as the first element and manifest conditions as the following
 288         elements.
 289         """
 290         testlines = []
 291         for t in tests:
 292             testlines.append("[%s]" % (t if isinstance(t, basestring)
 293                                        else t[0]))
 294             if isinstance(t, tuple):
 295                 testlines.extend(t[1:])
 296         self.manifest = self.writeFile("xpcshell.ini", """
 297 [DEFAULT]
 298 head =
 299 tail =
 300
 301 """ + "\n".join(testlines))
 302
 303     def assertTestResult(self, expected, shuffle=False, xunitFilename=None, verbose=False):
 304         """
 305         Assert that self.x.runTests with manifest=self.manifest
 306         returns |expected|.
 307         """
 308         self.assertEquals(expected,
 309                           self.x.runTests(xpcshellBin,
 310                                           manifest=self.manifest,
 311                                           mozInfo=mozinfo.info,
 312                                           shuffle=shuffle,
 313                                           testsRootDir=self.tempdir,
 314                                           verbose=verbose,
 315                                           xunitFilename=xunitFilename,
 316                                           sequential=True),
 317                           msg="""Tests should have %s, log:
 318 ========
 319 %s
 320 ========
 321 """ % ("passed" if expected else "failed", self.log.getvalue()))
 322
 323     def _assertLog(self, s, expected):
 324         l = self.log.getvalue()
 325         self.assertEqual(expected, s in l,
 326                          msg="""Value %s %s in log:
 327 ========
 328 %s
 329 ========""" % (s, "expected" if expected else "not expected", l))
 330
 331     def assertInLog(self, s):
 332         """
 333         Assert that the string |s| is contained in self.log.
 334         """
 335         self._assertLog(s, True)
 336
 337     def assertNotInLog(self, s):
 338         """
 339         Assert that the string |s| is not contained in self.log.
 340         """
 341         self._assertLog(s, False)
 342
 343     def testPass(self):
 344         """
 345         Check that a simple test without any manifest conditions passes.
 346         """
 347         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 348         self.writeManifest(["test_basic.js"])
 349
 350         self.assertTestResult(True)
 351         self.assertEquals(1, self.x.testCount)
 352         self.assertEquals(1, self.x.passCount)
 353         self.assertEquals(0, self.x.failCount)
 354         self.assertEquals(0, self.x.todoCount)
 355         self.assertInLog("TEST-PASS")
 356         self.assertNotInLog("TEST-UNEXPECTED-FAIL")
 357
 358     def testFail(self):
 359         """
 360         Check that a simple failing test without any manifest conditions fails.
 361         """
 362         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 363         self.writeManifest(["test_basic.js"])
 364
 365         self.assertTestResult(False)
 366         self.assertEquals(1, self.x.testCount)
 367         self.assertEquals(0, self.x.passCount)
 368         self.assertEquals(1, self.x.failCount)
 369         self.assertEquals(0, self.x.todoCount)
 370         self.assertInLog("TEST-UNEXPECTED-FAIL")
 371         self.assertNotInLog("TEST-PASS")
 372
 373     @unittest.skipIf(build_obj.defines.get('MOZ_B2G'),
 374                      'selftests with child processes fail on b2g desktop builds')
 375     def testChildPass(self):
 376         """
 377         Check that a simple test running in a child process passes.
 378         """
 379         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 380         self.writeFile("test_child_pass.js", CHILD_TEST_PASSING)
 381         self.writeManifest(["test_child_pass.js"])
 382
 383         self.assertTestResult(True, verbose=True)
 384         self.assertEquals(1, self.x.testCount)
 385         self.assertEquals(1, self.x.passCount)
 386         self.assertEquals(0, self.x.failCount)
 387         self.assertEquals(0, self.x.todoCount)
 388         self.assertInLog("TEST-PASS")
 389         self.assertInLog("CHILD-TEST-STARTED")
 390         self.assertInLog("CHILD-TEST-COMPLETED")
 391         self.assertNotInLog("TEST-UNEXPECTED-FAIL")
 392
 393
 394     @unittest.skipIf(build_obj.defines.get('MOZ_B2G'),
 395                      'selftests with child processes fail on b2g desktop builds')
 396     def testChildFail(self):
 397         """
 398         Check that a simple failing test running in a child process fails.
 399         """
 400         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
 401         self.writeFile("test_child_fail.js", CHILD_TEST_FAILING)
 402         self.writeManifest(["test_child_fail.js"])
 403
 404         self.assertTestResult(False)
 405         self.assertEquals(1, self.x.testCount)
 406         self.assertEquals(0, self.x.passCount)
 407         self.assertEquals(1, self.x.failCount)
 408         self.assertEquals(0, self.x.todoCount)
 409         self.assertInLog("TEST-UNEXPECTED-FAIL")
 410         self.assertInLog("CHILD-TEST-STARTED")
 411         self.assertInLog("CHILD-TEST-COMPLETED")
 412         self.assertNotInLog("TEST-PASS")
 413
 414     @unittest.skipIf(build_obj.defines.get('MOZ_B2G'),
 415                      'selftests with child processes fail on b2g desktop builds')
 416     def testChildHang(self):
 417         """
 418         Check that incomplete output from a child process results in a
 419         test failure.
 420         """
 421         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 422         self.writeFile("test_child_hang.js", CHILD_TEST_HANG)
 423         self.writeManifest(["test_child_hang.js"])
 424
 425         self.assertTestResult(False)
 426         self.assertEquals(1, self.x.testCount)
 427         self.assertEquals(0, self.x.passCount)
 428         self.assertEquals(1, self.x.failCount)
 429         self.assertEquals(0, self.x.todoCount)
 430         self.assertInLog("TEST-UNEXPECTED-FAIL")
 431         self.assertInLog("CHILD-TEST-STARTED")
 432         self.assertNotInLog("CHILD-TEST-COMPLETED")
 433         self.assertNotInLog("TEST-PASS")
 434
 435     def testSyntaxError(self):
 436         """
 437         Check that running a test file containing a syntax error produces
 438         a test failure and expected output.
 439         """
 440         self.writeFile("test_syntax_error.js", '"')
 441         self.writeManifest(["test_syntax_error.js"])
 442
 443         self.assertTestResult(False, verbose=True)
 444         self.assertEquals(1, self.x.testCount)
 445         self.assertEquals(0, self.x.passCount)
 446         self.assertEquals(1, self.x.failCount)
 447         self.assertEquals(0, self.x.todoCount)
 448         self.assertInLog("TEST-UNEXPECTED-FAIL")
 449         self.assertNotInLog("TEST-PASS")
 450
 451     def testPassFail(self):
 452         """
 453         Check that running more than one test works.
 454         """
 455         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 456         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
 457         self.writeManifest(["test_pass.js", "test_fail.js"])
 458
 459         self.assertTestResult(False)
 460         self.assertEquals(2, self.x.testCount)
 461         self.assertEquals(1, self.x.passCount)
 462         self.assertEquals(1, self.x.failCount)
 463         self.assertEquals(0, self.x.todoCount)
 464         self.assertInLog("TEST-PASS")
 465         self.assertInLog("TEST-UNEXPECTED-FAIL")
 466
 467     def testSkip(self):
 468         """
 469         Check that a simple failing test skipped in the manifest does
 470         not cause failure.
 471         """
 472         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 473         self.writeManifest([("test_basic.js", "skip-if = true")])
 474         self.assertTestResult(True)
 475         self.assertEquals(1, self.x.testCount)
 476         self.assertEquals(0, self.x.passCount)
 477         self.assertEquals(0, self.x.failCount)
 478         self.assertEquals(0, self.x.todoCount)
 479         self.assertNotInLog("TEST-UNEXPECTED-FAIL")
 480         self.assertNotInLog("TEST-PASS")
 481
 482     def testKnownFail(self):
 483         """
 484         Check that a simple failing test marked as known-fail in the manifest
 485         does not cause failure.
 486         """
 487         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 488         self.writeManifest([("test_basic.js", "fail-if = true")])
 489         self.assertTestResult(True)
 490         self.assertEquals(1, self.x.testCount)
 491         self.assertEquals(0, self.x.passCount)
 492         self.assertEquals(0, self.x.failCount)
 493         self.assertEquals(1, self.x.todoCount)
 494         self.assertInLog("TEST-KNOWN-FAIL")
 495         # This should be suppressed because the harness doesn't include
 496         # the full log from the xpcshell run when things pass.
 497         self.assertNotInLog("TEST-UNEXPECTED-FAIL")
 498         self.assertNotInLog("TEST-PASS")
 499
 500     def testUnexpectedPass(self):
 501         """
 502         Check that a simple failing test marked as known-fail in the manifest
 503         that passes causes an unexpected pass.
 504         """
 505         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 506         self.writeManifest([("test_basic.js", "fail-if = true")])
 507         self.assertTestResult(False)
 508         self.assertEquals(1, self.x.testCount)
 509         self.assertEquals(0, self.x.passCount)
 510         self.assertEquals(1, self.x.failCount)
 511         self.assertEquals(0, self.x.todoCount)
 512         # From the outer (Python) harness
 513         self.assertInLog("TEST-UNEXPECTED-PASS")
 514         self.assertNotInLog("TEST-KNOWN-FAIL")
 515         # From the inner (JS) harness
 516         self.assertInLog("TEST-PASS")
 517
 518     def testReturnNonzero(self):
 519         """
 520         Check that a test where xpcshell returns nonzero fails.
 521         """
 522         self.writeFile("test_error.js", "throw 'foo'")
 523         self.writeManifest(["test_error.js"])
 524
 525         self.assertTestResult(False)
 526         self.assertEquals(1, self.x.testCount)
 527         self.assertEquals(0, self.x.passCount)
 528         self.assertEquals(1, self.x.failCount)
 529         self.assertEquals(0, self.x.todoCount)
 530         self.assertInLog("TEST-UNEXPECTED-FAIL")
 531         self.assertNotInLog("TEST-PASS")
 532
 533     def testAddTestSimple(self):
 534         """
 535         Ensure simple add_test() works.
 536         """
 537         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
 538         self.writeManifest(["test_add_test_simple.js"])
 539
 540         self.assertTestResult(True)
 541         self.assertEquals(1, self.x.testCount)
 542         self.assertEquals(1, self.x.passCount)
 543         self.assertEquals(0, self.x.failCount)
 544
 545     def testLogCorrectFileName(self):
 546         """
 547         Make sure a meaningful filename and line number is logged
 548         by a passing test.
 549         """
 550         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
 551         self.writeManifest(["test_add_test_simple.js"])
 552
 553         self.assertTestResult(True, verbose=True)
 554         self.assertInLog("true == true")
 555         self.assertNotInLog("[do_check_true :")
 556         self.assertInLog("[test_simple : 5]")
 557
 558     def testAddTestFailing(self):
 559         """
 560         Ensure add_test() with a failing test is reported.
 561         """
 562         self.writeFile("test_add_test_failing.js", ADD_TEST_FAILING)
 563         self.writeManifest(["test_add_test_failing.js"])
 564
 565         self.assertTestResult(False)
 566         self.assertEquals(1, self.x.testCount)
 567         self.assertEquals(0, self.x.passCount)
 568         self.assertEquals(1, self.x.failCount)
 569
 570     def testAddTaskTestSingle(self):
 571         """
 572         Ensure add_test_task() with a single passing test works.
 573         """
 574         self.writeFile("test_add_task_simple.js", ADD_TASK_SINGLE)
 575         self.writeManifest(["test_add_task_simple.js"])
 576
 577         self.assertTestResult(True)
 578         self.assertEquals(1, self.x.testCount)
 579         self.assertEquals(1, self.x.passCount)
 580         self.assertEquals(0, self.x.failCount)
 581
 582     def testAddTaskTestMultiple(self):
 583         """
 584         Ensure multiple calls to add_test_task() work as expected.
 585         """
 586         self.writeFile("test_add_task_multiple.js",
 587             ADD_TASK_MULTIPLE)
 588         self.writeManifest(["test_add_task_multiple.js"])
 589
 590         self.assertTestResult(True)
 591         self.assertEquals(1, self.x.testCount)
 592         self.assertEquals(1, self.x.passCount)
 593         self.assertEquals(0, self.x.failCount)
 594
 595     def testAddTaskTestRejected(self):
 596         """
 597         Ensure rejected task reports as failure.
 598         """
 599         self.writeFile("test_add_task_rejected.js",
 600             ADD_TASK_REJECTED)
 601         self.writeManifest(["test_add_task_rejected.js"])
 602
 603         self.assertTestResult(False)
 604         self.assertEquals(1, self.x.testCount)
 605         self.assertEquals(0, self.x.passCount)
 606         self.assertEquals(1, self.x.failCount)
 607
 608     def testAddTaskTestFailureInside(self):
 609         """
 610         Ensure tests inside task are reported as failures.
 611         """
 612         self.writeFile("test_add_task_failure_inside.js",
 613             ADD_TASK_FAILURE_INSIDE)
 614         self.writeManifest(["test_add_task_failure_inside.js"])
 615
 616         self.assertTestResult(False)
 617         self.assertEquals(1, self.x.testCount)
 618         self.assertEquals(0, self.x.passCount)
 619         self.assertEquals(1, self.x.failCount)
 620
 621     def testAddTaskRunNextTest(self):
 622         """
 623         Calling run_next_test() from inside add_task() results in failure.
 624         """
 625         self.writeFile("test_add_task_run_next_test.js",
 626             ADD_TASK_RUN_NEXT_TEST)
 627         self.writeManifest(["test_add_task_run_next_test.js"])
 628
 629         self.assertTestResult(False)
 630         self.assertEquals(1, self.x.testCount)
 631         self.assertEquals(0, self.x.passCount)
 632         self.assertEquals(1, self.x.failCount)
 633
 634     def testAddTaskStackTrace(self):
 635         """
 636         Ensuring that calling Assert.ok(false) from inside add_task()
 637         results in a human-readable stack trace.
 638         """
 639         self.writeFile("test_add_task_stack_trace.js",
 640             ADD_TASK_STACK_TRACE)
 641         self.writeManifest(["test_add_task_stack_trace.js"])
 642
 643         self.assertTestResult(False)
 644         self.assertInLog("this_test_will_fail")
 645         self.assertInLog("run_next_test")
 646         self.assertInLog("run_test")
 647         self.assertNotInLog("Task.jsm")
 648
 649     def testAddTaskStackTraceWithoutStar(self):
 650         """
 651         Ensuring that calling Assert.ok(false) from inside add_task()
 652         results in a human-readable stack trace. This variant uses deprecated
 653         `function()` syntax instead of now standard `function*()`.
 654         """
 655         self.writeFile("test_add_task_stack_trace_without_star.js",
 656             ADD_TASK_STACK_TRACE)
 657         self.writeManifest(["test_add_task_stack_trace_without_star.js"])
 658
 659         self.assertTestResult(False)
 660         self.assertInLog("this_test_will_fail")
 661         self.assertInLog("run_next_test")
 662         self.assertInLog("run_test")
 663         self.assertNotInLog("Task.jsm")
 664
 665     def testMissingHeadFile(self):
 666         """
 667         Ensure that missing head file results in fatal error.
 668         """
 669         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 670         self.writeManifest([("test_basic.js", "head = missing.js")])
 671
 672         raised = False
 673
 674         try:
 675             # The actual return value is never checked because we raise.
 676             self.assertTestResult(True)
 677         except Exception, ex:
 678             raised = True
 679             self.assertEquals(ex.message[0:9], "head file")
 680
 681         self.assertTrue(raised)
 682
 683     def testMissingTailFile(self):
 684         """
 685         Ensure that missing tail file results in fatal error.
 686         """
 687         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 688         self.writeManifest([("test_basic.js", "tail = missing.js")])
 689
 690         raised = False
 691
 692         try:
 693             self.assertTestResult(True)
 694         except Exception, ex:
 695             raised = True
 696             self.assertEquals(ex.message[0:9], "tail file")
 697
 698         self.assertTrue(raised)
 699
 700     def testRandomExecution(self):
 701         """
 702         Check that random execution doesn't break.
 703         """
 704         manifest = []
 705         for i in range(0, 10):
 706             filename = "test_pass_%d.js" % i
 707             self.writeFile(filename, SIMPLE_PASSING_TEST)
 708             manifest.append(filename)
 709
 710         self.writeManifest(manifest)
 711         self.assertTestResult(True, shuffle=True)
 712         self.assertEquals(10, self.x.testCount)
 713         self.assertEquals(10, self.x.passCount)
 714
 715     def testXunitOutput(self):
 716         """
 717         Check that Xunit XML files are written.
 718         """
 719         self.writeFile("test_00.js", SIMPLE_PASSING_TEST)
 720         self.writeFile("test_01.js", SIMPLE_FAILING_TEST)
 721         self.writeFile("test_02.js", SIMPLE_PASSING_TEST)
 722
 723         manifest = [
 724             "test_00.js",
 725             "test_01.js",
 726             ("test_02.js", "skip-if = true")
 727         ]
 728
 729         self.writeManifest(manifest)
 730
 731         filename = os.path.join(self.tempdir, "xunit.xml")
 732
 733         self.assertTestResult(False, xunitFilename=filename)
 734
 735         self.assertTrue(os.path.exists(filename))
 736         self.assertTrue(os.path.getsize(filename) > 0)
 737
 738         tree = ElementTree()
 739         tree.parse(filename)
 740         suite = tree.getroot()
 741
 742         self.assertTrue(suite is not None)
 743         self.assertEqual(suite.get("tests"), "3")
 744         self.assertEqual(suite.get("failures"), "1")
 745         self.assertEqual(suite.get("skip"), "1")
 746
 747         testcases = suite.findall("testcase")
 748         self.assertEqual(len(testcases), 3)
 749
 750         for testcase in testcases:
 751             attributes = testcase.keys()
 752             self.assertTrue("classname" in attributes)
 753             self.assertTrue("name" in attributes)
 754             self.assertTrue("time" in attributes)
 755
 756         self.assertTrue(testcases[1].find("failure") is not None)
 757         self.assertTrue(testcases[2].find("skipped") is not None)
 758
 759     def testDoThrowString(self):
 760         """
 761         Check that do_throw produces reasonable messages when the
 762         input is a string instead of an object
 763         """
 764         self.writeFile("test_error.js", ADD_TEST_THROW_STRING)
 765         self.writeManifest(["test_error.js"])
 766
 767         self.assertTestResult(False)
 768         self.assertInLog("TEST-UNEXPECTED-FAIL")
 769         self.assertInLog("Passing a string to do_throw")
 770         self.assertNotInLog("TEST-PASS")
 771
 772     def testDoThrowForeignObject(self):
 773         """
 774         Check that do_throw produces reasonable messages when the
 775         input is a generic object with 'filename', 'message' and 'stack' attributes
 776         but 'object instanceof Error' returns false
 777         """
 778         self.writeFile("test_error.js", ADD_TEST_THROW_OBJECT)
 779         self.writeManifest(["test_error.js"])
 780
 781         self.assertTestResult(False)
 782         self.assertInLog("TEST-UNEXPECTED-FAIL")
 783         self.assertInLog("failure.js")
 784         self.assertInLog("Error object")
 785         self.assertInLog("ERROR STACK")
 786         self.assertNotInLog("TEST-PASS")
 787
 788     def testDoReportForeignObject(self):
 789         """
 790         Check that do_report_unexpected_exception produces reasonable messages when the
 791         input is a generic object with 'filename', 'message' and 'stack' attributes
 792         but 'object instanceof Error' returns false
 793         """
 794         self.writeFile("test_error.js", ADD_TEST_REPORT_OBJECT)
 795         self.writeManifest(["test_error.js"])
 796
 797         self.assertTestResult(False)
 798         self.assertInLog("TEST-UNEXPECTED-FAIL")
 799         self.assertInLog("failure.js")
 800         self.assertInLog("Error object")
 801         self.assertInLog("ERROR STACK")
 802         self.assertNotInLog("TEST-PASS")
 803
 804     def testDoReportRefError(self):
 805         """
 806         Check that do_report_unexpected_exception produces reasonable messages when the
 807         input is a JS-generated Error
 808         """
 809         self.writeFile("test_error.js", ADD_TEST_REPORT_REF_ERROR)
 810         self.writeManifest(["test_error.js"])
 811
 812         self.assertTestResult(False)
 813         self.assertInLog("TEST-UNEXPECTED-FAIL")
 814         self.assertInLog("test_error.js")
 815         self.assertInLog("obj.noSuchFunction is not a function")
 816         self.assertInLog("run_test@")
 817         self.assertNotInLog("TEST-PASS")
 818
 819     def testDoReportSyntaxError(self):
 820         """
 821         Check that attempting to load a test file containing a syntax error
 822         generates details of the error in the log
 823         """
 824         self.writeFile("test_error.js", LOAD_ERROR_SYNTAX_ERROR)
 825         self.writeManifest(["test_error.js"])
 826
 827         self.assertTestResult(False)
 828         self.assertInLog("TEST-UNEXPECTED-FAIL")
 829         self.assertInLog("test_error.js")
 830         self.assertInLog("test_error.js contains SyntaxError")
 831         self.assertInLog("Diagnostic: SyntaxError: missing formal parameter at")
 832         self.assertInLog("test_error.js:3")
 833         self.assertNotInLog("TEST-PASS")
 834
 835     def testDoReportNonSyntaxError(self):
 836         """
 837         Check that attempting to load a test file containing an error other
 838         than a syntax error generates details of the error in the log
 839         """
 840         self.writeFile("test_error.js", LOAD_ERROR_OTHER_ERROR)
 841         self.writeManifest(["test_error.js"])
 842
 843         self.assertTestResult(False)
 844         self.assertInLog("TEST-UNEXPECTED-FAIL")
 845         self.assertInLog("Diagnostic: TypeError: generator function run_test returns a value at")
 846         self.assertInLog("test_error.js:4")
 847         self.assertNotInLog("TEST-PASS")
 848
 849     def testAsyncCleanup(self):
 850         """
 851         Check that do_register_cleanup handles nicely cleanup tasks that
 852         return a promise
 853         """
 854         self.writeFile("test_asyncCleanup.js", ASYNC_CLEANUP)
 855         self.writeManifest(["test_asyncCleanup.js"])
 856         self.assertTestResult(False)
 857         self.assertInLog("\"1234\" == \"1234\"")
 858         self.assertInLog("At this stage, the test has succeeded")
 859         self.assertInLog("Throwing an error to force displaying the log")
 860
 861 if __name__ == "__main__":
 862     unittest.main()