testing/xpcshell/selftest.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6
   7 from __future__ import absolute_import, print_function
   8
   9 import mozinfo
  10 import os
  11 import pprint
  12 import re
  13 import shutil
  14 import six
  15 import sys
  16 import tempfile
  17 import unittest
  18
  19 from mozlog import structured
  20
  21 from runxpcshelltests import XPCShellTests
  22
  23
  24 TEST_PASS_STRING = "TEST-PASS"
  25 TEST_FAIL_STRING = "TEST-UNEXPECTED-FAIL"
  26
  27 SIMPLE_PASSING_TEST = "function run_test() { Assert.ok(true); }"
  28 SIMPLE_FAILING_TEST = "function run_test() { Assert.ok(false); }"
  29 SIMPLE_PREFCHECK_TEST = """
  30 function run_test() {
  31   const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
  32   Assert.ok(Services.prefs.getBoolPref("fake.pref.to.test"));
  33 }
  34 """
  35
  36 SIMPLE_UNCAUGHT_REJECTION_TEST = """
  37 function run_test() {
  38   Promise.reject(new Error("Test rejection."));
  39   Assert.ok(true);
  40 }
  41 """
  42
  43 SIMPLE_UNCAUGHT_REJECTION_JSM_TEST = """
  44 Components.utils.import("resource://gre/modules/Promise.jsm");
  45
  46 Promise.reject(new Error("Test rejection."));
  47
  48 function run_test() {
  49   Assert.ok(true);
  50 }
  51 """
  52
  53 ADD_TEST_SIMPLE = """
  54 function run_test() { run_next_test(); }
  55
  56 add_test(function test_simple() {
  57   Assert.ok(true);
  58   run_next_test();
  59 });
  60 """
  61
  62 ADD_TEST_FAILING = """
  63 function run_test() { run_next_test(); }
  64
  65 add_test(function test_failing() {
  66   Assert.ok(false);
  67   run_next_test();
  68 });
  69 """
  70
  71 ADD_TEST_UNCAUGHT_REJECTION = """
  72 function run_test() { run_next_test(); }
  73
  74 add_test(function test_uncaught_rejection() {
  75   Promise.reject(new Error("Test rejection."));
  76   run_next_test();
  77 });
  78 """
  79
  80 ADD_TEST_UNCAUGHT_REJECTION_JSM = """
  81 Components.utils.import("resource://gre/modules/Promise.jsm");
  82
  83 function run_test() { run_next_test(); }
  84
  85 add_test(function test_uncaught_rejection() {
  86   Promise.reject(new Error("Test rejection."));
  87   run_next_test();
  88 });
  89 """
  90
  91 CHILD_TEST_PASSING = """
  92 function run_test () { run_next_test(); }
  93
  94 add_test(function test_child_simple () {
  95   run_test_in_child("test_pass.js");
  96   run_next_test();
  97 });
  98 """
  99
 100 CHILD_TEST_FAILING = """
 101 function run_test () { run_next_test(); }
 102
 103 add_test(function test_child_simple () {
 104   run_test_in_child("test_fail.js");
 105   run_next_test();
 106 });
 107 """
 108
 109 CHILD_HARNESS_SIMPLE = """
 110 function run_test () { run_next_test(); }
 111
 112 add_test(function test_child_assert () {
 113   do_load_child_test_harness();
 114   do_test_pending("test child assertion");
 115   sendCommand("Assert.ok(true);", do_test_finished);
 116   run_next_test();
 117 });
 118 """
 119
 120 CHILD_TEST_HANG = """
 121 function run_test () { run_next_test(); }
 122
 123 add_test(function test_child_simple () {
 124   do_test_pending("hang test");
 125   do_load_child_test_harness();
 126   sendCommand("_testLogger.info('CHILD-TEST-STARTED'); " +
 127               + "const _TEST_FILE=['test_pass.js']; _execute_test(); ",
 128               do_test_finished);
 129   run_next_test();
 130 });
 131 """
 132
 133 SIMPLE_LOOPING_TEST = """
 134 function run_test () { run_next_test(); }
 135
 136 add_test(function test_loop () {
 137   do_test_pending()
 138 });
 139 """
 140
 141 PASSING_TEST_UNICODE = b"""
 142 function run_test () { run_next_test(); }
 143
 144 add_test(function test_unicode_print () {
 145   Assert.equal("\u201c\u201d", "\u201c\u201d");
 146   run_next_test();
 147 });
 148 """
 149
 150 ADD_TASK_SINGLE = """
 151 Components.utils.import("resource://gre/modules/Promise.jsm");
 152
 153 function run_test() { run_next_test(); }
 154
 155 add_task(async function test_task() {
 156   await Promise.resolve(true);
 157   await Promise.resolve(false);
 158 });
 159 """
 160
 161 ADD_TASK_MULTIPLE = """
 162 Components.utils.import("resource://gre/modules/Promise.jsm");
 163
 164 function run_test() { run_next_test(); }
 165
 166 add_task(async function test_task() {
 167   await Promise.resolve(true);
 168 });
 169
 170 add_task(async function test_2() {
 171   await Promise.resolve(true);
 172 });
 173 """
 174
 175 ADD_TASK_REJECTED = """
 176 Components.utils.import("resource://gre/modules/Promise.jsm");
 177
 178 function run_test() { run_next_test(); }
 179
 180 add_task(async function test_failing() {
 181   await Promise.reject(new Error("I fail."));
 182 });
 183 """
 184
 185 ADD_TASK_FAILURE_INSIDE = """
 186 Components.utils.import("resource://gre/modules/Promise.jsm");
 187
 188 function run_test() { run_next_test(); }
 189
 190 add_task(async function test() {
 191   let result = await Promise.resolve(false);
 192
 193   Assert.ok(result);
 194 });
 195 """
 196
 197 ADD_TASK_RUN_NEXT_TEST = """
 198 function run_test() { run_next_test(); }
 199
 200 add_task(function () {
 201   Assert.ok(true);
 202
 203   run_next_test();
 204 });
 205 """
 206
 207 ADD_TASK_STACK_TRACE = """
 208 Components.utils.import("resource://gre/modules/Promise.jsm", this);
 209
 210 function run_test() { run_next_test(); }
 211
 212 add_task(async function this_test_will_fail() {
 213   for (let i = 0; i < 10; ++i) {
 214     await Promise.resolve();
 215   }
 216   Assert.ok(false);
 217 });
 218 """
 219
 220 ADD_TASK_SKIP = """
 221 add_task(async function skipMeNot1() {
 222   Assert.ok(true, "Well well well.");
 223 });
 224
 225 add_task(async function skipMe1() {
 226   Assert.ok(false, "Not skipped after all.");
 227 }).skip();
 228
 229 add_task(async function skipMeNot2() {
 230   Assert.ok(true, "Well well well.");
 231 });
 232
 233 add_task(async function skipMeNot3() {
 234   Assert.ok(true, "Well well well.");
 235 });
 236
 237 add_task(async function skipMe2() {
 238   Assert.ok(false, "Not skipped after all.");
 239 }).skip();
 240 """
 241
 242 ADD_TASK_SKIPALL = """
 243 add_task(async function skipMe1() {
 244   Assert.ok(false, "Not skipped after all.");
 245 });
 246
 247 add_task(async function skipMe2() {
 248   Assert.ok(false, "Not skipped after all.");
 249 }).skip();
 250
 251 add_task(async function skipMe3() {
 252   Assert.ok(false, "Not skipped after all.");
 253 }).only();
 254
 255 add_task(async function skipMeNot() {
 256   Assert.ok(true, "Well well well.");
 257 }).only();
 258
 259 add_task(async function skipMe4() {
 260   Assert.ok(false, "Not skipped after all.");
 261 });
 262 """
 263
 264 ADD_TEST_THROW_STRING = """
 265 function run_test() {do_throw("Passing a string to do_throw")};
 266 """
 267
 268 ADD_TEST_THROW_OBJECT = """
 269 let error = {
 270   message: "Error object",
 271   fileName: "failure.js",
 272   stack: "ERROR STACK",
 273   toString: function() {return this.message;}
 274 };
 275 function run_test() {do_throw(error)};
 276 """
 277
 278 ADD_TEST_REPORT_OBJECT = """
 279 let error = {
 280   message: "Error object",
 281   fileName: "failure.js",
 282   stack: "ERROR STACK",
 283   toString: function() {return this.message;}
 284 };
 285 function run_test() {do_report_unexpected_exception(error)};
 286 """
 287
 288 ADD_TEST_VERBOSE = """
 289 function run_test() {info("a message from info")};
 290 """
 291
 292 # A test for genuine JS-generated Error objects
 293 ADD_TEST_REPORT_REF_ERROR = """
 294 function run_test() {
 295   let obj = {blah: 0};
 296   try {
 297     obj.noSuchFunction();
 298   }
 299   catch (error) {
 300     do_report_unexpected_exception(error);
 301   }
 302 };
 303 """
 304
 305 # A test for failure to load a test due to a syntax error
 306 LOAD_ERROR_SYNTAX_ERROR = """
 307 function run_test(
 308 """
 309
 310 # A test for failure to load a test due to an error other than a syntax error
 311 LOAD_ERROR_OTHER_ERROR = """
 312 "use strict";
 313 no_such_var = "foo"; // assignment to undeclared variable
 314 """
 315
 316 # A test that crashes outright.
 317 TEST_CRASHING = """
 318 function run_test () {
 319   Components.utils.import("resource://gre/modules/ctypes.jsm", this);
 320   let zero = new ctypes.intptr_t(8);
 321   let badptr = ctypes.cast(zero, ctypes.PointerType(ctypes.int32_t));
 322   badptr.contents;
 323 }
 324 """
 325
 326 # A test for asynchronous cleanup functions
 327 ASYNC_CLEANUP = """
 328 function run_test() {
 329   Components.utils.import("resource://gre/modules/Promise.jsm", this);
 330
 331   // The list of checkpoints in the order we encounter them.
 332   let checkpoints = [];
 333
 334   // Cleanup tasks, in reverse order
 335   registerCleanupFunction(function cleanup_checkout() {
 336     Assert.equal(checkpoints.join(""), "123456");
 337     info("At this stage, the test has succeeded");
 338     do_throw("Throwing an error to force displaying the log");
 339   });
 340
 341   registerCleanupFunction(function sync_cleanup_2() {
 342     checkpoints.push(6);
 343   });
 344
 345   registerCleanupFunction(async function async_cleanup_4() {
 346     await undefined;
 347     checkpoints.push(5);
 348   });
 349
 350   registerCleanupFunction(async function async_cleanup_3() {
 351     await undefined;
 352     checkpoints.push(4);
 353   });
 354
 355   registerCleanupFunction(function async_cleanup_2() {
 356     let deferred = Promise.defer();
 357     executeSoon(deferred.resolve);
 358     return deferred.promise.then(function() {
 359       checkpoints.push(3);
 360     });
 361   });
 362
 363   registerCleanupFunction(function sync_cleanup() {
 364     checkpoints.push(2);
 365   });
 366
 367   registerCleanupFunction(function async_cleanup() {
 368     let deferred = Promise.defer();
 369     executeSoon(deferred.resolve);
 370     return deferred.promise.then(function() {
 371       checkpoints.push(1);
 372     });
 373   });
 374
 375 }
 376 """
 377
 378 # A test to check that add_test() tests run without run_test()
 379 NO_RUN_TEST_ADD_TEST = """
 380 add_test(function no_run_test_add_test() {
 381   Assert.ok(true);
 382   run_next_test();
 383 });
 384 """
 385
 386 # A test to check that add_task() tests run without run_test()
 387 NO_RUN_TEST_ADD_TASK = """
 388 add_task(function no_run_test_add_task() {
 389   Assert.ok(true);
 390 });
 391 """
 392
 393 # A test to check that both add_task() and add_test() work without run_test()
 394 NO_RUN_TEST_ADD_TEST_ADD_TASK = """
 395 add_test(function no_run_test_add_test() {
 396   Assert.ok(true);
 397   run_next_test();
 398 });
 399
 400 add_task(function no_run_test_add_task() {
 401   Assert.ok(true);
 402 });
 403 """
 404
 405 # A test to check that an empty test file without run_test(),
 406 # add_test() or add_task() works.
 407 NO_RUN_TEST_EMPTY_TEST = """
 408 // This is an empty test file.
 409 """
 410
 411 NO_RUN_TEST_ADD_TEST_FAIL = """
 412 add_test(function no_run_test_add_test_fail() {
 413   Assert.ok(false);
 414   run_next_test();
 415 });
 416 """
 417
 418 NO_RUN_TEST_ADD_TASK_FAIL = """
 419 add_task(function no_run_test_add_task_fail() {
 420   Assert.ok(false);
 421 });
 422 """
 423
 424 NO_RUN_TEST_ADD_TASK_MULTIPLE = """
 425 Components.utils.import("resource://gre/modules/Promise.jsm");
 426
 427 add_task(async function test_task() {
 428   await Promise.resolve(true);
 429 });
 430
 431 add_task(async function test_2() {
 432   await Promise.resolve(true);
 433 });
 434 """
 435
 436 LOAD_MOZINFO = """
 437 function run_test() {
 438   Assert.notEqual(typeof mozinfo, undefined);
 439   Assert.notEqual(typeof mozinfo.os, undefined);
 440 }
 441 """
 442
 443 CHILD_MOZINFO = """
 444 function run_test () { run_next_test(); }
 445
 446 add_test(function test_child_mozinfo () {
 447   run_test_in_child("test_mozinfo.js");
 448   run_next_test();
 449 });
 450 """
 451
 452 HEADLESS_TRUE = """
 453 add_task(function headless_true() {
 454   let env = Cc["@mozilla.org/process/environment;1"].getService(
 455     Ci.nsIEnvironment
 456   );
 457   Assert.equal(env.get("MOZ_HEADLESS"), "1", "Check MOZ_HEADLESS");
 458   Assert.equal(env.get("DISPLAY"), "77", "Check DISPLAY");
 459 });
 460 """
 461
 462 HEADLESS_FALSE = """
 463 add_task(function headless_false() {
 464   let env = Cc["@mozilla.org/process/environment;1"].getService(
 465     Ci.nsIEnvironment
 466   );
 467   Assert.notEqual(env.get("MOZ_HEADLESS"), "1", "Check MOZ_HEADLESS");
 468   Assert.notEqual(env.get("DISPLAY"), "77", "Check DISPLAY");
 469 });
 470 """
 471
 472
 473 class XPCShellTestsTests(unittest.TestCase):
 474     """
 475     Yes, these are unit tests for a unit test harness.
 476     """
 477
 478     def __init__(self, name):
 479         super(XPCShellTestsTests, self).__init__(name)
 480         from buildconfig import substs
 481         from mozbuild.base import MozbuildObject
 482
 483         os.environ.pop("MOZ_OBJDIR", None)
 484         self.build_obj = MozbuildObject.from_environment()
 485
 486         objdir = self.build_obj.topobjdir
 487         self.testing_modules = os.path.join(objdir, "_tests", "modules")
 488
 489         if mozinfo.isMac:
 490             self.xpcshellBin = os.path.join(
 491                 objdir,
 492                 "dist",
 493                 substs["MOZ_MACBUNDLE_NAME"],
 494                 "Contents",
 495                 "MacOS",
 496                 "xpcshell",
 497             )
 498         else:
 499             self.xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
 500         if sys.platform == "win32":
 501             self.xpcshellBin += ".exe"
 502         self.utility_path = os.path.join(objdir, "dist", "bin")
 503         self.symbols_path = None
 504         candidate_path = os.path.join(self.build_obj.distdir, "crashreporter-symbols")
 505         if os.path.isdir(candidate_path):
 506             self.symbols_path = candidate_path
 507
 508     def setUp(self):
 509         self.log = six.StringIO()
 510         self.tempdir = tempfile.mkdtemp()
 511         logger = structured.commandline.setup_logging(
 512             "selftest%s" % id(self), {}, {"tbpl": self.log}
 513         )
 514         self.x = XPCShellTests(logger)
 515         self.x.harness_timeout = 30 if not mozinfo.info["ccov"] else 60
 516
 517     def tearDown(self):
 518         shutil.rmtree(self.tempdir)
 519         self.x.shutdownNode()
 520
 521     def writeFile(self, name, contents, mode="w"):
 522         """
 523         Write |contents| to a file named |name| in the temp directory,
 524         and return the full path to the file.
 525         """
 526         fullpath = os.path.join(self.tempdir, name)
 527         with open(fullpath, mode) as f:
 528             f.write(contents)
 529         return fullpath
 530
 531     def writeManifest(self, tests, prefs=[]):
 532         """
 533         Write an xpcshell.ini in the temp directory and set
 534         self.manifest to its pathname. |tests| is a list containing
 535         either strings (for test names), or tuples with a test name
 536         as the first element and manifest conditions as the following
 537         elements. |prefs| is an optional list of prefs in the form of
 538         "prefname=prefvalue" strings.
 539         """
 540         testlines = []
 541         for t in tests:
 542             testlines.append("[%s]" % (t if isinstance(t, six.string_types) else t[0]))
 543             if isinstance(t, tuple):
 544                 testlines.extend(t[1:])
 545         prefslines = []
 546         for p in prefs:
 547             # Append prefs lines as indented inside "prefs=" manifest option.
 548             prefslines.append("  %s" % p)
 549
 550         self.manifest = self.writeFile(
 551             "xpcshell.ini",
 552             """
 553 [DEFAULT]
 554 head =
 555 tail =
 556 prefs =
 557 """
 558             + "\n".join(prefslines)
 559             + "\n"
 560             + "\n".join(testlines),
 561         )
 562
 563     def assertTestResult(self, expected, shuffle=False, verbose=False, headless=False):
 564         """
 565         Assert that self.x.runTests with manifest=self.manifest
 566         returns |expected|.
 567         """
 568         kwargs = {}
 569         kwargs["xpcshell"] = self.xpcshellBin
 570         kwargs["symbolsPath"] = self.symbols_path
 571         kwargs["manifest"] = self.manifest
 572         kwargs["mozInfo"] = mozinfo.info
 573         kwargs["shuffle"] = shuffle
 574         kwargs["verbose"] = verbose
 575         kwargs["headless"] = headless
 576         kwargs["sequential"] = True
 577         kwargs["testingModulesDir"] = self.testing_modules
 578         kwargs["utility_path"] = self.utility_path
 579         self.assertEquals(
 580             expected,
 581             self.x.runTests(kwargs),
 582             msg="""Tests should have %s, log:
 583 ========
 584 %s
 585 ========
 586 """
 587             % ("passed" if expected else "failed", self.log.getvalue()),
 588         )
 589
 590     def _assertLog(self, s, expected):
 591         l = self.log.getvalue()
 592         self.assertEqual(
 593             expected,
 594             s in l,
 595             msg="""Value %s %s in log:
 596 ========
 597 %s
 598 ========"""
 599             % (s, "expected" if expected else "not expected", l),
 600         )
 601
 602     def assertInLog(self, s):
 603         """
 604         Assert that the string |s| is contained in self.log.
 605         """
 606         self._assertLog(s, True)
 607
 608     def assertNotInLog(self, s):
 609         """
 610         Assert that the string |s| is not contained in self.log.
 611         """
 612         self._assertLog(s, False)
 613
 614     def testPass(self):
 615         """
 616         Check that a simple test without any manifest conditions passes.
 617         """
 618         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 619         self.writeManifest(["test_basic.js"])
 620
 621         self.assertTestResult(True)
 622         self.assertEquals(1, self.x.testCount)
 623         self.assertEquals(1, self.x.passCount)
 624         self.assertEquals(0, self.x.failCount)
 625         self.assertEquals(0, self.x.todoCount)
 626         self.assertInLog(TEST_PASS_STRING)
 627         self.assertNotInLog(TEST_FAIL_STRING)
 628
 629     def testFail(self):
 630         """
 631         Check that a simple failing test without any manifest conditions fails.
 632         """
 633         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 634         self.writeManifest(["test_basic.js"])
 635
 636         self.assertTestResult(False)
 637         self.assertEquals(1, self.x.testCount)
 638         self.assertEquals(0, self.x.passCount)
 639         self.assertEquals(1, self.x.failCount)
 640         self.assertEquals(0, self.x.todoCount)
 641         self.assertInLog(TEST_FAIL_STRING)
 642         self.assertNotInLog(TEST_PASS_STRING)
 643
 644     def testPrefsInManifestVerbose(self):
 645         """
 646         Check prefs configuration option is supported in xpcshell manifests.
 647         """
 648         self.writeFile("test_prefs.js", SIMPLE_PREFCHECK_TEST)
 649         self.writeManifest(tests=["test_prefs.js"], prefs=["fake.pref.to.test=true"])
 650
 651         self.assertTestResult(True, verbose=True)
 652         self.assertInLog(TEST_PASS_STRING)
 653         self.assertNotInLog(TEST_FAIL_STRING)
 654         self.assertEquals(1, self.x.testCount)
 655         self.assertEquals(1, self.x.passCount)
 656         self.assertInLog("Per-test extra prefs will be set:")
 657         self.assertInLog("fake.pref.to.test=true")
 658
 659     def testPrefsInManifestNonVerbose(self):
 660         """
 661         Check prefs configuration are not logged in non verbose mode.
 662         """
 663         self.writeFile("test_prefs.js", SIMPLE_PREFCHECK_TEST)
 664         self.writeManifest(tests=["test_prefs.js"], prefs=["fake.pref.to.test=true"])
 665
 666         self.assertTestResult(True, verbose=False)
 667         self.assertNotInLog("Per-test extra prefs will be set:")
 668         self.assertNotInLog("fake.pref.to.test=true")
 669
 670     @unittest.skipIf(
 671         mozinfo.isWin or not mozinfo.info.get("debug"),
 672         "We don't have a stack fixer on hand for windows.",
 673     )
 674     def testAssertStack(self):
 675         """
 676         When an assertion is hit, we should produce a useful stack.
 677         """
 678         self.writeFile(
 679             "test_assert.js",
 680             """
 681           add_test(function test_asserts_immediately() {
 682             Components.classes["@mozilla.org/xpcom/debug;1"]
 683                       .getService(Components.interfaces.nsIDebug2)
 684                       .assertion("foo", "assertion failed", "test.js", 1)
 685             run_next_test();
 686           });
 687         """,
 688         )
 689
 690         self.writeManifest(["test_assert.js"])
 691         self.assertTestResult(False)
 692
 693         self.assertInLog("###!!! ASSERTION")
 694         log_lines = self.log.getvalue().splitlines()
 695         line_pat = "#\d\d:"
 696         unknown_pat = "#\d\d\: \?\?\?\[.* \+0x[a-f0-9]+\]"
 697         self.assertFalse(
 698             any(re.search(unknown_pat, line) for line in log_lines),
 699             "An stack frame without symbols was found in\n%s"
 700             % pprint.pformat(log_lines),
 701         )
 702         self.assertTrue(
 703             any(re.search(line_pat, line) for line in log_lines),
 704             "No line resembling a stack frame was found in\n%s"
 705             % pprint.pformat(log_lines),
 706         )
 707
 708     def testChildPass(self):
 709         """
 710         Check that a simple test running in a child process passes.
 711         """
 712         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 713         self.writeFile("test_child_pass.js", CHILD_TEST_PASSING)
 714         self.writeManifest(["test_child_pass.js"])
 715
 716         self.assertTestResult(True, verbose=True)
 717         self.assertEquals(1, self.x.testCount)
 718         self.assertEquals(1, self.x.passCount)
 719         self.assertEquals(0, self.x.failCount)
 720         self.assertEquals(0, self.x.todoCount)
 721         self.assertInLog(TEST_PASS_STRING)
 722         self.assertInLog("CHILD-TEST-STARTED")
 723         self.assertInLog("CHILD-TEST-COMPLETED")
 724         self.assertNotInLog(TEST_FAIL_STRING)
 725
 726     def testChildFail(self):
 727         """
 728         Check that a simple failing test running in a child process fails.
 729         """
 730         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
 731         self.writeFile("test_child_fail.js", CHILD_TEST_FAILING)
 732         self.writeManifest(["test_child_fail.js"])
 733
 734         self.assertTestResult(False)
 735         self.assertEquals(1, self.x.testCount)
 736         self.assertEquals(0, self.x.passCount)
 737         self.assertEquals(1, self.x.failCount)
 738         self.assertEquals(0, self.x.todoCount)
 739         self.assertInLog(TEST_FAIL_STRING)
 740         self.assertInLog("CHILD-TEST-STARTED")
 741         self.assertInLog("CHILD-TEST-COMPLETED")
 742         self.assertNotInLog(TEST_PASS_STRING)
 743
 744     def testChildHang(self):
 745         """
 746         Check that incomplete output from a child process results in a
 747         test failure.
 748         """
 749         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 750         self.writeFile("test_child_hang.js", CHILD_TEST_HANG)
 751         self.writeManifest(["test_child_hang.js"])
 752
 753         self.assertTestResult(False)
 754         self.assertEquals(1, self.x.testCount)
 755         self.assertEquals(0, self.x.passCount)
 756         self.assertEquals(1, self.x.failCount)
 757         self.assertEquals(0, self.x.todoCount)
 758         self.assertInLog(TEST_FAIL_STRING)
 759         self.assertInLog("CHILD-TEST-STARTED")
 760         self.assertNotInLog("CHILD-TEST-COMPLETED")
 761         self.assertNotInLog(TEST_PASS_STRING)
 762
 763     def testChild(self):
 764         """
 765         Checks that calling do_load_child_test_harness without run_test_in_child
 766         results in a usable test state. This test has a spurious failure when
 767         run using |mach python-test|. See bug 1103226.
 768         """
 769         self.writeFile("test_child_assertions.js", CHILD_HARNESS_SIMPLE)
 770         self.writeManifest(["test_child_assertions.js"])
 771
 772         self.assertTestResult(True)
 773         self.assertEquals(1, self.x.testCount)
 774         self.assertEquals(1, self.x.passCount)
 775         self.assertEquals(0, self.x.failCount)
 776         self.assertEquals(0, self.x.todoCount)
 777         self.assertInLog(TEST_PASS_STRING)
 778         self.assertNotInLog(TEST_FAIL_STRING)
 779
 780     def testSkipForAddTest(self):
 781         """
 782         Check that add_test is skipped if |skip_if| condition is true
 783         """
 784         self.writeFile(
 785             "test_skip.js",
 786             """
 787 add_test({
 788   skip_if: () => true,
 789 }, function test_should_be_skipped() {
 790   Assert.ok(false);
 791   run_next_test();
 792 });
 793 """,
 794         )
 795         self.writeManifest(["test_skip.js"])
 796         self.assertTestResult(True, verbose=True)
 797         self.assertEquals(1, self.x.testCount)
 798         self.assertEquals(1, self.x.passCount)
 799         self.assertEquals(0, self.x.failCount)
 800         self.assertEquals(0, self.x.todoCount)
 801         self.assertInLog(TEST_PASS_STRING)
 802         self.assertInLog("TEST-SKIP")
 803         self.assertNotInLog(TEST_FAIL_STRING)
 804
 805     def testNotSkipForAddTask(self):
 806         """
 807         Check that add_task is not skipped if |skip_if| condition is false
 808         """
 809         self.writeFile(
 810             "test_not_skip.js",
 811             """
 812 add_task({
 813   skip_if: () => false,
 814 }, function test_should_not_be_skipped() {
 815   Assert.ok(true);
 816 });
 817 """,
 818         )
 819         self.writeManifest(["test_not_skip.js"])
 820         self.assertTestResult(True, verbose=True)
 821         self.assertEquals(1, self.x.testCount)
 822         self.assertEquals(1, self.x.passCount)
 823         self.assertEquals(0, self.x.failCount)
 824         self.assertEquals(0, self.x.todoCount)
 825         self.assertInLog(TEST_PASS_STRING)
 826         self.assertNotInLog("TEST-SKIP")
 827         self.assertNotInLog(TEST_FAIL_STRING)
 828
 829     def testSkipForAddTask(self):
 830         """
 831         Check that add_task is skipped if |skip_if| condition is true
 832         """
 833         self.writeFile(
 834             "test_skip.js",
 835             """
 836 add_task({
 837   skip_if: () => true,
 838 }, function test_should_be_skipped() {
 839   Assert.ok(false);
 840 });
 841 """,
 842         )
 843         self.writeManifest(["test_skip.js"])
 844         self.assertTestResult(True, verbose=True)
 845         self.assertEquals(1, self.x.testCount)
 846         self.assertEquals(1, self.x.passCount)
 847         self.assertEquals(0, self.x.failCount)
 848         self.assertEquals(0, self.x.todoCount)
 849         self.assertInLog(TEST_PASS_STRING)
 850         self.assertInLog("TEST-SKIP")
 851         self.assertNotInLog(TEST_FAIL_STRING)
 852
 853     def testNotSkipForAddTest(self):
 854         """
 855         Check that add_test is not skipped if |skip_if| condition is false
 856         """
 857         self.writeFile(
 858             "test_not_skip.js",
 859             """
 860 add_test({
 861   skip_if: () => false,
 862 }, function test_should_not_be_skipped() {
 863   Assert.ok(true);
 864   run_next_test();
 865 });
 866 """,
 867         )
 868         self.writeManifest(["test_not_skip.js"])
 869         self.assertTestResult(True, verbose=True)
 870         self.assertEquals(1, self.x.testCount)
 871         self.assertEquals(1, self.x.passCount)
 872         self.assertEquals(0, self.x.failCount)
 873         self.assertEquals(0, self.x.todoCount)
 874         self.assertInLog(TEST_PASS_STRING)
 875         self.assertNotInLog("TEST-SKIP")
 876         self.assertNotInLog(TEST_FAIL_STRING)
 877
 878     def testSyntaxError(self):
 879         """
 880         Check that running a test file containing a syntax error produces
 881         a test failure and expected output.
 882         """
 883         self.writeFile("test_syntax_error.js", '"')
 884         self.writeManifest(["test_syntax_error.js"])
 885
 886         self.assertTestResult(False, verbose=True)
 887         self.assertEquals(1, self.x.testCount)
 888         self.assertEquals(0, self.x.passCount)
 889         self.assertEquals(1, self.x.failCount)
 890         self.assertEquals(0, self.x.todoCount)
 891         self.assertInLog(TEST_FAIL_STRING)
 892         self.assertNotInLog(TEST_PASS_STRING)
 893
 894     def testUnicodeInAssertMethods(self):
 895         """
 896         Check that passing unicode characters through an assertion method works.
 897         """
 898         self.writeFile("test_unicode_assert.js", PASSING_TEST_UNICODE, mode="wb")
 899         self.writeManifest(["test_unicode_assert.js"])
 900
 901         self.assertTestResult(True, verbose=True)
 902
 903     @unittest.skipIf(
 904         "MOZ_AUTOMATION" in os.environ,
 905         "Timeout code path occasionally times out (bug 1098121)",
 906     )
 907     def testHangingTimeout(self):
 908         """
 909         Check that a test that never finishes results in the correct error log.
 910         """
 911         self.writeFile("test_loop.js", SIMPLE_LOOPING_TEST)
 912         self.writeManifest(["test_loop.js"])
 913
 914         old_timeout = self.x.harness_timeout
 915         self.x.harness_timeout = 1
 916
 917         self.assertTestResult(False)
 918         self.assertEquals(1, self.x.testCount)
 919         self.assertEquals(1, self.x.failCount)
 920         self.assertEquals(0, self.x.passCount)
 921         self.assertEquals(0, self.x.todoCount)
 922         self.assertInLog("TEST-UNEXPECTED-TIMEOUT")
 923
 924         self.x.harness_timeout = old_timeout
 925
 926     def testPassFail(self):
 927         """
 928         Check that running more than one test works.
 929         """
 930         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 931         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
 932         self.writeManifest(["test_pass.js", "test_fail.js"])
 933
 934         self.assertTestResult(False)
 935         self.assertEquals(2, self.x.testCount)
 936         self.assertEquals(1, self.x.passCount)
 937         self.assertEquals(1, self.x.failCount)
 938         self.assertEquals(0, self.x.todoCount)
 939         self.assertInLog(TEST_PASS_STRING)
 940         self.assertInLog(TEST_FAIL_STRING)
 941
 942     def testSkip(self):
 943         """
 944         Check that a simple failing test skipped in the manifest does
 945         not cause failure.
 946         """
 947         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 948         self.writeManifest([("test_basic.js", "skip-if = true")])
 949         self.assertTestResult(True)
 950         self.assertEquals(1, self.x.testCount)
 951         self.assertEquals(0, self.x.passCount)
 952         self.assertEquals(0, self.x.failCount)
 953         self.assertEquals(0, self.x.todoCount)
 954         self.assertNotInLog(TEST_FAIL_STRING)
 955         self.assertNotInLog(TEST_PASS_STRING)
 956
 957     def testKnownFail(self):
 958         """
 959         Check that a simple failing test marked as known-fail in the manifest
 960         does not cause failure.
 961         """
 962         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 963         self.writeManifest([("test_basic.js", "fail-if = true")])
 964         self.assertTestResult(True)
 965         self.assertEquals(1, self.x.testCount)
 966         self.assertEquals(0, self.x.passCount)
 967         self.assertEquals(0, self.x.failCount)
 968         self.assertEquals(1, self.x.todoCount)
 969         self.assertInLog("TEST-FAIL")
 970         # This should be suppressed because the harness doesn't include
 971         # the full log from the xpcshell run when things pass.
 972         self.assertNotInLog(TEST_FAIL_STRING)
 973         self.assertNotInLog(TEST_PASS_STRING)
 974
 975     def testUnexpectedPass(self):
 976         """
 977         Check that a simple failing test marked as known-fail in the manifest
 978         that passes causes an unexpected pass.
 979         """
 980         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 981         self.writeManifest([("test_basic.js", "fail-if = true")])
 982         self.assertTestResult(False)
 983         self.assertEquals(1, self.x.testCount)
 984         self.assertEquals(0, self.x.passCount)
 985         self.assertEquals(1, self.x.failCount)
 986         self.assertEquals(0, self.x.todoCount)
 987         # From the outer (Python) harness
 988         self.assertInLog("TEST-UNEXPECTED-PASS")
 989         self.assertNotInLog("TEST-KNOWN-FAIL")
 990
 991     def testReturnNonzero(self):
 992         """
 993         Check that a test where xpcshell returns nonzero fails.
 994         """
 995         self.writeFile("test_error.js", "throw 'foo'")
 996         self.writeManifest(["test_error.js"])
 997
 998         self.assertTestResult(False)
 999         self.assertEquals(1, self.x.testCount)
1000         self.assertEquals(0, self.x.passCount)
1001         self.assertEquals(1, self.x.failCount)
1002         self.assertEquals(0, self.x.todoCount)
1003         self.assertInLog(TEST_FAIL_STRING)
1004         self.assertNotInLog(TEST_PASS_STRING)
1005
1006     def testUncaughtRejection(self):
1007         """
1008         Ensure a simple test with an uncaught rejection is reported.
1009         """
1010         self.writeFile(
1011             "test_simple_uncaught_rejection.js", SIMPLE_UNCAUGHT_REJECTION_TEST
1012         )
1013         self.writeManifest(["test_simple_uncaught_rejection.js"])
1014
1015         self.assertTestResult(False)
1016         self.assertInLog(TEST_FAIL_STRING)
1017         self.assertInLog("test_simple_uncaught_rejection.js:3:18")
1018         self.assertInLog("Test rejection.")
1019         self.assertEquals(1, self.x.testCount)
1020         self.assertEquals(0, self.x.passCount)
1021         self.assertEquals(1, self.x.failCount)
1022
1023     def testUncaughtRejectionJSM(self):
1024         """
1025         Ensure a simple test with an uncaught rejection from Promise.jsm is reported.
1026         """
1027         self.writeFile(
1028             "test_simple_uncaught_rejection_jsm.js", SIMPLE_UNCAUGHT_REJECTION_JSM_TEST
1029         )
1030         self.writeManifest(["test_simple_uncaught_rejection_jsm.js"])
1031
1032         self.assertTestResult(False)
1033         self.assertInLog(TEST_FAIL_STRING)
1034         self.assertInLog("test_simple_uncaught_rejection_jsm.js:4:16")
1035         self.assertInLog("Test rejection.")
1036         self.assertEquals(1, self.x.testCount)
1037         self.assertEquals(0, self.x.passCount)
1038         self.assertEquals(1, self.x.failCount)
1039
1040     def testAddTestSimple(self):
1041         """
1042         Ensure simple add_test() works.
1043         """
1044         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
1045         self.writeManifest(["test_add_test_simple.js"])
1046
1047         self.assertTestResult(True)
1048         self.assertEquals(1, self.x.testCount)
1049         self.assertEquals(1, self.x.passCount)
1050         self.assertEquals(0, self.x.failCount)
1051
1052     def testCrashLogging(self):
1053         """
1054         Test that a crashing test process logs a failure.
1055         """
1056         self.writeFile("test_crashes.js", TEST_CRASHING)
1057         self.writeManifest(["test_crashes.js"])
1058
1059         self.assertTestResult(False)
1060         self.assertEquals(1, self.x.testCount)
1061         self.assertEquals(0, self.x.passCount)
1062         self.assertEquals(1, self.x.failCount)
1063         if mozinfo.info.get("crashreporter"):
1064             self.assertInLog("\nPROCESS-CRASH")
1065
1066     def testLogCorrectFileName(self):
1067         """
1068         Make sure a meaningful filename and line number is logged
1069         by a passing test.
1070         """
1071         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
1072         self.writeManifest(["test_add_test_simple.js"])
1073
1074         self.assertTestResult(True, verbose=True)
1075         self.assertInLog("true == true")
1076         self.assertNotInLog("[Assert.ok :")
1077         self.assertInLog("[test_simple : 5]")
1078
1079     def testAddTestFailing(self):
1080         """
1081         Ensure add_test() with a failing test is reported.
1082         """
1083         self.writeFile("test_add_test_failing.js", ADD_TEST_FAILING)
1084         self.writeManifest(["test_add_test_failing.js"])
1085
1086         self.assertTestResult(False)
1087         self.assertEquals(1, self.x.testCount)
1088         self.assertEquals(0, self.x.passCount)
1089         self.assertEquals(1, self.x.failCount)
1090
1091     def testAddTestUncaughtRejection(self):
1092         """
1093         Ensure add_test() with an uncaught rejection is reported.
1094         """
1095         self.writeFile(
1096             "test_add_test_uncaught_rejection.js", ADD_TEST_UNCAUGHT_REJECTION
1097         )
1098         self.writeManifest(["test_add_test_uncaught_rejection.js"])
1099
1100         self.assertTestResult(False)
1101         self.assertEquals(1, self.x.testCount)
1102         self.assertEquals(0, self.x.passCount)
1103         self.assertEquals(1, self.x.failCount)
1104
1105     def testAddTestUncaughtRejectionJSM(self):
1106         """
1107         Ensure add_test() with an uncaught rejection from Promise.jsm is reported.
1108         """
1109         self.writeFile(
1110             "test_add_test_uncaught_rejection_jsm.js", ADD_TEST_UNCAUGHT_REJECTION_JSM
1111         )
1112         self.writeManifest(["test_add_test_uncaught_rejection_jsm.js"])
1113
1114         self.assertTestResult(False)
1115         self.assertEquals(1, self.x.testCount)
1116         self.assertEquals(0, self.x.passCount)
1117         self.assertEquals(1, self.x.failCount)
1118
1119     def testAddTaskTestSingle(self):
1120         """
1121         Ensure add_test_task() with a single passing test works.
1122         """
1123         self.writeFile("test_add_task_simple.js", ADD_TASK_SINGLE)
1124         self.writeManifest(["test_add_task_simple.js"])
1125
1126         self.assertTestResult(True)
1127         self.assertEquals(1, self.x.testCount)
1128         self.assertEquals(1, self.x.passCount)
1129         self.assertEquals(0, self.x.failCount)
1130
1131     def testAddTaskTestMultiple(self):
1132         """
1133         Ensure multiple calls to add_test_task() work as expected.
1134         """
1135         self.writeFile("test_add_task_multiple.js", ADD_TASK_MULTIPLE)
1136         self.writeManifest(["test_add_task_multiple.js"])
1137
1138         self.assertTestResult(True)
1139         self.assertEquals(1, self.x.testCount)
1140         self.assertEquals(1, self.x.passCount)
1141         self.assertEquals(0, self.x.failCount)
1142
1143     def testAddTaskTestRejected(self):
1144         """
1145         Ensure rejected task reports as failure.
1146         """
1147         self.writeFile("test_add_task_rejected.js", ADD_TASK_REJECTED)
1148         self.writeManifest(["test_add_task_rejected.js"])
1149
1150         self.assertTestResult(False)
1151         self.assertEquals(1, self.x.testCount)
1152         self.assertEquals(0, self.x.passCount)
1153         self.assertEquals(1, self.x.failCount)
1154
1155     def testAddTaskTestFailureInside(self):
1156         """
1157         Ensure tests inside task are reported as failures.
1158         """
1159         self.writeFile("test_add_task_failure_inside.js", ADD_TASK_FAILURE_INSIDE)
1160         self.writeManifest(["test_add_task_failure_inside.js"])
1161
1162         self.assertTestResult(False)
1163         self.assertEquals(1, self.x.testCount)
1164         self.assertEquals(0, self.x.passCount)
1165         self.assertEquals(1, self.x.failCount)
1166
1167     def testAddTaskRunNextTest(self):
1168         """
1169         Calling run_next_test() from inside add_task() results in failure.
1170         """
1171         self.writeFile("test_add_task_run_next_test.js", ADD_TASK_RUN_NEXT_TEST)
1172         self.writeManifest(["test_add_task_run_next_test.js"])
1173
1174         self.assertTestResult(False)
1175         self.assertEquals(1, self.x.testCount)
1176         self.assertEquals(0, self.x.passCount)
1177         self.assertEquals(1, self.x.failCount)
1178
1179     def testAddTaskStackTrace(self):
1180         """
1181         Ensuring that calling Assert.ok(false) from inside add_task()
1182         results in a human-readable stack trace.
1183         """
1184         self.writeFile("test_add_task_stack_trace.js", ADD_TASK_STACK_TRACE)
1185         self.writeManifest(["test_add_task_stack_trace.js"])
1186
1187         self.assertTestResult(False)
1188         self.assertInLog("this_test_will_fail")
1189         self.assertInLog("run_next_test")
1190         self.assertInLog("run_test")
1191         self.assertNotInLog("Task.jsm")
1192
1193     def testAddTaskSkip(self):
1194         self.writeFile("test_tasks_skip.js", ADD_TASK_SKIP)
1195         self.writeManifest(["test_tasks_skip.js"])
1196
1197         self.assertTestResult(True)
1198         self.assertEquals(1, self.x.testCount)
1199         self.assertEquals(1, self.x.passCount)
1200         self.assertEquals(0, self.x.failCount)
1201
1202     def testAddTaskSkipAll(self):
1203         self.writeFile("test_tasks_skipall.js", ADD_TASK_SKIPALL)
1204         self.writeManifest(["test_tasks_skipall.js"])
1205
1206         self.assertTestResult(True)
1207         self.assertEquals(1, self.x.testCount)
1208         self.assertEquals(1, self.x.passCount)
1209         self.assertEquals(0, self.x.failCount)
1210
1211     def testMissingHeadFile(self):
1212         """
1213         Ensure that missing head file results in fatal error.
1214         """
1215         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
1216         self.writeManifest([("test_basic.js", "head = missing.js")])
1217
1218         raised = False
1219
1220         try:
1221             # The actual return value is never checked because we raise.
1222             self.assertTestResult(True)
1223         except Exception as ex:
1224             raised = True
1225             self.assertEquals(str(ex)[0:9], "head file")
1226
1227         self.assertTrue(raised)
1228
1229     def testRandomExecution(self):
1230         """
1231         Check that random execution doesn't break.
1232         """
1233         manifest = []
1234         for i in range(0, 10):
1235             filename = "test_pass_%d.js" % i
1236             self.writeFile(filename, SIMPLE_PASSING_TEST)
1237             manifest.append(filename)
1238
1239         self.writeManifest(manifest)
1240         self.assertTestResult(True, shuffle=True)
1241         self.assertEquals(10, self.x.testCount)
1242         self.assertEquals(10, self.x.passCount)
1243
1244     def testDoThrowString(self):
1245         """
1246         Check that do_throw produces reasonable messages when the
1247         input is a string instead of an object
1248         """
1249         self.writeFile("test_error.js", ADD_TEST_THROW_STRING)
1250         self.writeManifest(["test_error.js"])
1251
1252         self.assertTestResult(False)
1253         self.assertInLog(TEST_FAIL_STRING)
1254         self.assertInLog("Passing a string to do_throw")
1255         self.assertNotInLog(TEST_PASS_STRING)
1256
1257     def testDoThrowForeignObject(self):
1258         """
1259         Check that do_throw produces reasonable messages when the
1260         input is a generic object with 'filename', 'message' and 'stack' attributes
1261         but 'object instanceof Error' returns false
1262         """
1263         self.writeFile("test_error.js", ADD_TEST_THROW_OBJECT)
1264         self.writeManifest(["test_error.js"])
1265
1266         self.assertTestResult(False)
1267         self.assertInLog(TEST_FAIL_STRING)
1268         self.assertInLog("failure.js")
1269         self.assertInLog("Error object")
1270         self.assertInLog("ERROR STACK")
1271         self.assertNotInLog(TEST_PASS_STRING)
1272
1273     def testDoReportForeignObject(self):
1274         """
1275         Check that do_report_unexpected_exception produces reasonable messages when the
1276         input is a generic object with 'filename', 'message' and 'stack' attributes
1277         but 'object instanceof Error' returns false
1278         """
1279         self.writeFile("test_error.js", ADD_TEST_REPORT_OBJECT)
1280         self.writeManifest(["test_error.js"])
1281
1282         self.assertTestResult(False)
1283         self.assertInLog(TEST_FAIL_STRING)
1284         self.assertInLog("failure.js")
1285         self.assertInLog("Error object")
1286         self.assertInLog("ERROR STACK")
1287         self.assertNotInLog(TEST_PASS_STRING)
1288
1289     def testDoReportRefError(self):
1290         """
1291         Check that do_report_unexpected_exception produces reasonable messages when the
1292         input is a JS-generated Error
1293         """
1294         self.writeFile("test_error.js", ADD_TEST_REPORT_REF_ERROR)
1295         self.writeManifest(["test_error.js"])
1296
1297         self.assertTestResult(False)
1298         self.assertInLog(TEST_FAIL_STRING)
1299         self.assertInLog("test_error.js")
1300         self.assertInLog("obj.noSuchFunction is not a function")
1301         self.assertInLog("run_test@")
1302         self.assertNotInLog(TEST_PASS_STRING)
1303
1304     def testDoReportSyntaxError(self):
1305         """
1306         Check that attempting to load a test file containing a syntax error
1307         generates details of the error in the log
1308         """
1309         self.writeFile("test_error.js", LOAD_ERROR_SYNTAX_ERROR)
1310         self.writeManifest(["test_error.js"])
1311
1312         self.assertTestResult(False)
1313         self.assertInLog(TEST_FAIL_STRING)
1314         self.assertInLog("test_error.js:3")
1315         self.assertNotInLog(TEST_PASS_STRING)
1316
1317     def testDoReportNonSyntaxError(self):
1318         """
1319         Check that attempting to load a test file containing an error other
1320         than a syntax error generates details of the error in the log
1321         """
1322         self.writeFile("test_error.js", LOAD_ERROR_OTHER_ERROR)
1323         self.writeManifest(["test_error.js"])
1324
1325         self.assertTestResult(False)
1326         self.assertInLog(TEST_FAIL_STRING)
1327         self.assertInLog("ReferenceError: assignment to undeclared variable")
1328         self.assertInLog("test_error.js:3")
1329         self.assertNotInLog(TEST_PASS_STRING)
1330
1331     def testDoPrintWhenVerboseNotExplicit(self):
1332         """
1333         Check that info() and similar calls that generate output do
1334         not have the output when not run verbosely.
1335         """
1336         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1337         self.writeManifest(["test_verbose.js"])
1338
1339         self.assertTestResult(True)
1340         self.assertNotInLog("a message from info")
1341
1342     def testDoPrintWhenVerboseExplicit(self):
1343         """
1344         Check that info() and similar calls that generate output have the
1345         output shown when run verbosely.
1346         """
1347         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1348         self.writeManifest(["test_verbose.js"])
1349         self.assertTestResult(True, verbose=True)
1350         self.assertInLog("a message from info")
1351
1352     def testDoPrintWhenVerboseInManifest(self):
1353         """
1354         Check that info() and similar calls that generate output have the
1355         output shown when 'verbose = true' is in the manifest, even when
1356         not run verbosely.
1357         """
1358         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1359         self.writeManifest([("test_verbose.js", "verbose = true")])
1360
1361         self.assertTestResult(True)
1362         self.assertInLog("a message from info")
1363
1364     def testAsyncCleanup(self):
1365         """
1366         Check that registerCleanupFunction handles nicely async cleanup tasks
1367         """
1368         self.writeFile("test_asyncCleanup.js", ASYNC_CLEANUP)
1369         self.writeManifest(["test_asyncCleanup.js"])
1370         self.assertTestResult(False)
1371         self.assertInLog('"123456" == "123456"')
1372         self.assertInLog("At this stage, the test has succeeded")
1373         self.assertInLog("Throwing an error to force displaying the log")
1374
1375     def testNoRunTestAddTest(self):
1376         """
1377         Check that add_test() works fine without run_test() in the test file.
1378         """
1379         self.writeFile("test_noRunTestAddTest.js", NO_RUN_TEST_ADD_TEST)
1380         self.writeManifest(["test_noRunTestAddTest.js"])
1381
1382         self.assertTestResult(True)
1383         self.assertEquals(1, self.x.testCount)
1384         self.assertEquals(1, self.x.passCount)
1385         self.assertEquals(0, self.x.failCount)
1386         self.assertInLog(TEST_PASS_STRING)
1387         self.assertNotInLog(TEST_FAIL_STRING)
1388
1389     def testNoRunTestAddTask(self):
1390         """
1391         Check that add_task() works fine without run_test() in the test file.
1392         """
1393         self.writeFile("test_noRunTestAddTask.js", NO_RUN_TEST_ADD_TASK)
1394         self.writeManifest(["test_noRunTestAddTask.js"])
1395
1396         self.assertTestResult(True)
1397         self.assertEquals(1, self.x.testCount)
1398         self.assertEquals(1, self.x.passCount)
1399         self.assertEquals(0, self.x.failCount)
1400         self.assertInLog(TEST_PASS_STRING)
1401         self.assertNotInLog(TEST_FAIL_STRING)
1402
1403     def testNoRunTestAddTestAddTask(self):
1404         """
1405         Check that both add_test() and add_task() work without run_test()
1406         in the test file.
1407         """
1408         self.writeFile("test_noRunTestAddTestAddTask.js", NO_RUN_TEST_ADD_TEST_ADD_TASK)
1409         self.writeManifest(["test_noRunTestAddTestAddTask.js"])
1410
1411         self.assertTestResult(True)
1412         self.assertEquals(1, self.x.testCount)
1413         self.assertEquals(1, self.x.passCount)
1414         self.assertEquals(0, self.x.failCount)
1415         self.assertInLog(TEST_PASS_STRING)
1416         self.assertNotInLog(TEST_FAIL_STRING)
1417
1418     def testNoRunTestEmptyTest(self):
1419         """
1420         Check that the test passes on an empty file that contains neither
1421         run_test() nor add_test(), add_task().
1422         """
1423         self.writeFile("test_noRunTestEmptyTest.js", NO_RUN_TEST_EMPTY_TEST)
1424         self.writeManifest(["test_noRunTestEmptyTest.js"])
1425
1426         self.assertTestResult(True)
1427         self.assertEquals(1, self.x.testCount)
1428         self.assertEquals(1, self.x.passCount)
1429         self.assertEquals(0, self.x.failCount)
1430         self.assertInLog(TEST_PASS_STRING)
1431         self.assertNotInLog(TEST_FAIL_STRING)
1432
1433     def testNoRunTestAddTestFail(self):
1434         """
1435         Check that test fails on using add_test() without run_test().
1436         """
1437         self.writeFile("test_noRunTestAddTestFail.js", NO_RUN_TEST_ADD_TEST_FAIL)
1438         self.writeManifest(["test_noRunTestAddTestFail.js"])
1439
1440         self.assertTestResult(False)
1441         self.assertEquals(1, self.x.testCount)
1442         self.assertEquals(0, self.x.passCount)
1443         self.assertEquals(1, self.x.failCount)
1444         self.assertInLog(TEST_FAIL_STRING)
1445         self.assertNotInLog(TEST_PASS_STRING)
1446
1447     def testNoRunTestAddTaskFail(self):
1448         """
1449         Check that test fails on using add_task() without run_test().
1450         """
1451         self.writeFile("test_noRunTestAddTaskFail.js", NO_RUN_TEST_ADD_TASK_FAIL)
1452         self.writeManifest(["test_noRunTestAddTaskFail.js"])
1453
1454         self.assertTestResult(False)
1455         self.assertEquals(1, self.x.testCount)
1456         self.assertEquals(0, self.x.passCount)
1457         self.assertEquals(1, self.x.failCount)
1458         self.assertInLog(TEST_FAIL_STRING)
1459         self.assertNotInLog(TEST_PASS_STRING)
1460
1461     def testNoRunTestAddTaskMultiple(self):
1462         """
1463         Check that multple add_task() tests work without run_test().
1464         """
1465         self.writeFile(
1466             "test_noRunTestAddTaskMultiple.js", NO_RUN_TEST_ADD_TASK_MULTIPLE
1467         )
1468         self.writeManifest(["test_noRunTestAddTaskMultiple.js"])
1469
1470         self.assertTestResult(True)
1471         self.assertEquals(1, self.x.testCount)
1472         self.assertEquals(1, self.x.passCount)
1473         self.assertEquals(0, self.x.failCount)
1474         self.assertInLog(TEST_PASS_STRING)
1475         self.assertNotInLog(TEST_FAIL_STRING)
1476
1477     def testMozinfo(self):
1478         """
1479         Check that mozinfo.json is loaded
1480         """
1481         self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
1482         self.writeManifest(["test_mozinfo.js"])
1483         self.assertTestResult(True)
1484         self.assertEquals(1, self.x.testCount)
1485         self.assertEquals(1, self.x.passCount)
1486         self.assertEquals(0, self.x.failCount)
1487         self.assertEquals(0, self.x.todoCount)
1488         self.assertInLog(TEST_PASS_STRING)
1489         self.assertNotInLog(TEST_FAIL_STRING)
1490
1491     def testChildMozinfo(self):
1492         """
1493         Check that mozinfo.json is loaded in child process
1494         """
1495         self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
1496         self.writeFile("test_child_mozinfo.js", CHILD_MOZINFO)
1497         self.writeManifest(["test_child_mozinfo.js"])
1498         self.assertTestResult(True)
1499         self.assertEquals(1, self.x.testCount)
1500         self.assertEquals(1, self.x.passCount)
1501         self.assertEquals(0, self.x.failCount)
1502         self.assertEquals(0, self.x.todoCount)
1503         self.assertInLog(TEST_PASS_STRING)
1504         self.assertNotInLog(TEST_FAIL_STRING)
1505
1506     def testNotHeadlessByDefault(self):
1507         """
1508         Check that the default is not headless.
1509         """
1510         self.writeFile("test_notHeadlessByDefault.js", HEADLESS_FALSE)
1511         self.writeManifest(["test_notHeadlessByDefault.js"])
1512         self.assertTestResult(True)
1513
1514     def testHeadlessWhenHeadlessExplicit(self):
1515         """
1516         Check that explicitly requesting headless works when the manifest doesn't override.
1517         """
1518         self.writeFile("test_headlessWhenExplicit.js", HEADLESS_TRUE)
1519         self.writeManifest(["test_headlessWhenExplicit.js"])
1520         self.assertTestResult(True, headless=True)
1521
1522     def testHeadlessWhenHeadlessTrueInManifest(self):
1523         """
1524         Check that enabling headless in the manifest alone works.
1525         """
1526         self.writeFile("test_headlessWhenTrueInManifest.js", HEADLESS_TRUE)
1527         self.writeManifest([("test_headlessWhenTrueInManifest.js", "headless = true")])
1528         self.assertTestResult(True)
1529
1530     def testNotHeadlessWhenHeadlessFalseInManifest(self):
1531         """
1532         Check that the manifest entry overrides the explicit default.
1533         """
1534         self.writeFile("test_notHeadlessWhenFalseInManifest.js", HEADLESS_FALSE)
1535         self.writeManifest(
1536             [("test_notHeadlessWhenFalseInManifest.js", "headless = false")]
1537         )
1538         self.assertTestResult(True, headless=True)
1539
1540
1541 if __name__ == "__main__":
1542     import mozunit
1543
1544     mozinfo.find_and_update_from_json()
1545     mozunit.main()