testing/xpcshell/selftest.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6
   7 import os
   8 import pprint
   9 import re
  10 import shutil
  11 import sys
  12 import tempfile
  13 import unittest
  14
  15 import mozinfo
  16 import six
  17 from mozlog import structured
  18 from runxpcshelltests import XPCShellTests
  19
  20 TEST_PASS_STRING = "TEST-PASS"
  21 TEST_FAIL_STRING = "TEST-UNEXPECTED-FAIL"
  22
  23 SIMPLE_PASSING_TEST = "function run_test() { Assert.ok(true); }"
  24 SIMPLE_FAILING_TEST = "function run_test() { Assert.ok(false); }"
  25 SIMPLE_PREFCHECK_TEST = """
  26 function run_test() {
  27   Assert.ok(Services.prefs.getBoolPref("fake.pref.to.test"));
  28 }
  29 """
  30
  31 SIMPLE_UNCAUGHT_REJECTION_TEST = """
  32 function run_test() {
  33   Promise.reject(new Error("Test rejection."));
  34   Assert.ok(true);
  35 }
  36 """
  37
  38 ADD_TEST_SIMPLE = """
  39 function run_test() { run_next_test(); }
  40
  41 add_test(function test_simple() {
  42   Assert.ok(true);
  43   run_next_test();
  44 });
  45 """
  46
  47 ADD_TEST_FAILING = """
  48 function run_test() { run_next_test(); }
  49
  50 add_test(function test_failing() {
  51   Assert.ok(false);
  52   run_next_test();
  53 });
  54 """
  55
  56 ADD_TEST_UNCAUGHT_REJECTION = """
  57 function run_test() { run_next_test(); }
  58
  59 add_test(function test_uncaught_rejection() {
  60   Promise.reject(new Error("Test rejection."));
  61   run_next_test();
  62 });
  63 """
  64
  65 CHILD_TEST_PASSING = """
  66 function run_test () { run_next_test(); }
  67
  68 add_test(function test_child_simple () {
  69   run_test_in_child("test_pass.js");
  70   run_next_test();
  71 });
  72 """
  73
  74 CHILD_TEST_FAILING = """
  75 function run_test () { run_next_test(); }
  76
  77 add_test(function test_child_simple () {
  78   run_test_in_child("test_fail.js");
  79   run_next_test();
  80 });
  81 """
  82
  83 CHILD_HARNESS_SIMPLE = """
  84 function run_test () { run_next_test(); }
  85
  86 add_test(function test_child_assert () {
  87   do_load_child_test_harness();
  88   do_test_pending("test child assertion");
  89   sendCommand("Assert.ok(true);", do_test_finished);
  90   run_next_test();
  91 });
  92 """
  93
  94 CHILD_TEST_HANG = """
  95 function run_test () { run_next_test(); }
  96
  97 add_test(function test_child_simple () {
  98   do_test_pending("hang test");
  99   do_load_child_test_harness();
 100   sendCommand("_testLogger.info('CHILD-TEST-STARTED'); " +
 101               + "const _TEST_FILE=['test_pass.js']; _execute_test(); ",
 102               do_test_finished);
 103   run_next_test();
 104 });
 105 """
 106
 107 SIMPLE_LOOPING_TEST = """
 108 function run_test () { run_next_test(); }
 109
 110 add_test(function test_loop () {
 111   do_test_pending()
 112 });
 113 """
 114
 115 PASSING_TEST_UNICODE = b"""
 116 function run_test () { run_next_test(); }
 117
 118 add_test(function test_unicode_print () {
 119   Assert.equal("\u201c\u201d", "\u201c\u201d");
 120   run_next_test();
 121 });
 122 """
 123
 124 ADD_TASK_SINGLE = """
 125 function run_test() { run_next_test(); }
 126
 127 add_task(async function test_task() {
 128   await Promise.resolve(true);
 129   await Promise.resolve(false);
 130 });
 131 """
 132
 133 ADD_TASK_MULTIPLE = """
 134 function run_test() { run_next_test(); }
 135
 136 add_task(async function test_task() {
 137   await Promise.resolve(true);
 138 });
 139
 140 add_task(async function test_2() {
 141   await Promise.resolve(true);
 142 });
 143 """
 144
 145 ADD_TASK_REJECTED = """
 146 function run_test() { run_next_test(); }
 147
 148 add_task(async function test_failing() {
 149   await Promise.reject(new Error("I fail."));
 150 });
 151 """
 152
 153 ADD_TASK_REJECTED_UNDEFINED = """
 154 function run_test() { run_next_test(); }
 155
 156 add_task(async function test_failing() {
 157   await Promise.reject();
 158 });
 159 """
 160
 161 ADD_TASK_FAILURE_INSIDE = """
 162 function run_test() { run_next_test(); }
 163
 164 add_task(async function test() {
 165   let result = await Promise.resolve(false);
 166
 167   Assert.ok(result);
 168 });
 169 """
 170
 171 ADD_TASK_RUN_NEXT_TEST = """
 172 function run_test() { run_next_test(); }
 173
 174 add_task(function () {
 175   Assert.ok(true);
 176
 177   run_next_test();
 178 });
 179 """
 180
 181 ADD_TASK_STACK_TRACE = """
 182 function run_test() { run_next_test(); }
 183
 184 add_task(async function this_test_will_fail() {
 185   for (let i = 0; i < 10; ++i) {
 186     await Promise.resolve();
 187   }
 188   Assert.ok(false);
 189 });
 190 """
 191
 192 ADD_TASK_SKIP = """
 193 add_task(async function skipMeNot1() {
 194   Assert.ok(true, "Well well well.");
 195 });
 196
 197 add_task(async function skipMe1() {
 198   Assert.ok(false, "Not skipped after all.");
 199 }).skip();
 200
 201 add_task(async function skipMeNot2() {
 202   Assert.ok(true, "Well well well.");
 203 });
 204
 205 add_task(async function skipMeNot3() {
 206   Assert.ok(true, "Well well well.");
 207 });
 208
 209 add_task(async function skipMe2() {
 210   Assert.ok(false, "Not skipped after all.");
 211 }).skip();
 212 """
 213
 214 ADD_TASK_SKIPALL = """
 215 add_task(async function skipMe1() {
 216   Assert.ok(false, "Not skipped after all.");
 217 });
 218
 219 add_task(async function skipMe2() {
 220   Assert.ok(false, "Not skipped after all.");
 221 }).skip();
 222
 223 add_task(async function skipMe3() {
 224   Assert.ok(false, "Not skipped after all.");
 225 }).only();
 226
 227 add_task(async function skipMeNot() {
 228   Assert.ok(true, "Well well well.");
 229 }).only();
 230
 231 add_task(async function skipMe4() {
 232   Assert.ok(false, "Not skipped after all.");
 233 });
 234 """
 235
 236 ADD_TEST_THROW_STRING = """
 237 function run_test() {do_throw("Passing a string to do_throw")};
 238 """
 239
 240 ADD_TEST_THROW_OBJECT = """
 241 let error = {
 242   message: "Error object",
 243   fileName: "failure.js",
 244   stack: "ERROR STACK",
 245   toString: function() {return this.message;}
 246 };
 247 function run_test() {do_throw(error)};
 248 """
 249
 250 ADD_TEST_REPORT_OBJECT = """
 251 let error = {
 252   message: "Error object",
 253   fileName: "failure.js",
 254   stack: "ERROR STACK",
 255   toString: function() {return this.message;}
 256 };
 257 function run_test() {do_report_unexpected_exception(error)};
 258 """
 259
 260 ADD_TEST_VERBOSE = """
 261 function run_test() {info("a message from info")};
 262 """
 263
 264 # A test for genuine JS-generated Error objects
 265 ADD_TEST_REPORT_REF_ERROR = """
 266 function run_test() {
 267   let obj = {blah: 0};
 268   try {
 269     obj.noSuchFunction();
 270   }
 271   catch (error) {
 272     do_report_unexpected_exception(error);
 273   }
 274 };
 275 """
 276
 277 # A test for failure to load a test due to a syntax error
 278 LOAD_ERROR_SYNTAX_ERROR = """
 279 function run_test(
 280 """
 281
 282 # A test for failure to load a test due to an error other than a syntax error
 283 LOAD_ERROR_OTHER_ERROR = """
 284 "use strict";
 285 no_such_var = "foo"; // assignment to undeclared variable
 286 """
 287
 288 # A test that crashes outright.
 289 TEST_CRASHING = """
 290 function run_test () {
 291   const { ctypes } = ChromeUtils.import("resource://gre/modules/ctypes.jsm");
 292   let zero = new ctypes.intptr_t(8);
 293   let badptr = ctypes.cast(zero, ctypes.PointerType(ctypes.int32_t));
 294   badptr.contents;
 295 }
 296 """
 297
 298 # A test for asynchronous cleanup functions
 299 ASYNC_CLEANUP = """
 300 function run_test() {
 301   let { PromiseUtils } = ChromeUtils.importESModule(
 302     "resource://gre/modules/PromiseUtils.sys.mjs"
 303   );
 304
 305   // The list of checkpoints in the order we encounter them.
 306   let checkpoints = [];
 307
 308   // Cleanup tasks, in reverse order
 309   registerCleanupFunction(function cleanup_checkout() {
 310     Assert.equal(checkpoints.join(""), "123456");
 311     info("At this stage, the test has succeeded");
 312     do_throw("Throwing an error to force displaying the log");
 313   });
 314
 315   registerCleanupFunction(function sync_cleanup_2() {
 316     checkpoints.push(6);
 317   });
 318
 319   registerCleanupFunction(async function async_cleanup_4() {
 320     await undefined;
 321     checkpoints.push(5);
 322   });
 323
 324   registerCleanupFunction(async function async_cleanup_3() {
 325     await undefined;
 326     checkpoints.push(4);
 327   });
 328
 329   registerCleanupFunction(function async_cleanup_2() {
 330     let deferred = PromiseUtils.defer();
 331     executeSoon(deferred.resolve);
 332     return deferred.promise.then(function() {
 333       checkpoints.push(3);
 334     });
 335   });
 336
 337   registerCleanupFunction(function sync_cleanup() {
 338     checkpoints.push(2);
 339   });
 340
 341   registerCleanupFunction(function async_cleanup() {
 342     let deferred = PromiseUtils.defer();
 343     executeSoon(deferred.resolve);
 344     return deferred.promise.then(function() {
 345       checkpoints.push(1);
 346     });
 347   });
 348
 349 }
 350 """
 351
 352 # A test to check that add_test() tests run without run_test()
 353 NO_RUN_TEST_ADD_TEST = """
 354 add_test(function no_run_test_add_test() {
 355   Assert.ok(true);
 356   run_next_test();
 357 });
 358 """
 359
 360 # A test to check that add_task() tests run without run_test()
 361 NO_RUN_TEST_ADD_TASK = """
 362 add_task(function no_run_test_add_task() {
 363   Assert.ok(true);
 364 });
 365 """
 366
 367 # A test to check that both add_task() and add_test() work without run_test()
 368 NO_RUN_TEST_ADD_TEST_ADD_TASK = """
 369 add_test(function no_run_test_add_test() {
 370   Assert.ok(true);
 371   run_next_test();
 372 });
 373
 374 add_task(function no_run_test_add_task() {
 375   Assert.ok(true);
 376 });
 377 """
 378
 379 # A test to check that an empty test file without run_test(),
 380 # add_test() or add_task() works.
 381 NO_RUN_TEST_EMPTY_TEST = """
 382 // This is an empty test file.
 383 """
 384
 385 NO_RUN_TEST_ADD_TEST_FAIL = """
 386 add_test(function no_run_test_add_test_fail() {
 387   Assert.ok(false);
 388   run_next_test();
 389 });
 390 """
 391
 392 NO_RUN_TEST_ADD_TASK_FAIL = """
 393 add_task(function no_run_test_add_task_fail() {
 394   Assert.ok(false);
 395 });
 396 """
 397
 398 NO_RUN_TEST_ADD_TASK_MULTIPLE = """
 399 add_task(async function test_task() {
 400   await Promise.resolve(true);
 401 });
 402
 403 add_task(async function test_2() {
 404   await Promise.resolve(true);
 405 });
 406 """
 407
 408 LOAD_MOZINFO = """
 409 function run_test() {
 410   Assert.notEqual(typeof mozinfo, undefined);
 411   Assert.notEqual(typeof mozinfo.os, undefined);
 412 }
 413 """
 414
 415 CHILD_MOZINFO = """
 416 function run_test () { run_next_test(); }
 417
 418 add_test(function test_child_mozinfo () {
 419   run_test_in_child("test_mozinfo.js");
 420   run_next_test();
 421 });
 422 """
 423
 424 HEADLESS_TRUE = """
 425 add_task(function headless_true() {
 426   Assert.equal(Services.env.get("MOZ_HEADLESS"), "1", "Check MOZ_HEADLESS");
 427   Assert.equal(Services.env.get("DISPLAY"), "77", "Check DISPLAY");
 428 });
 429 """
 430
 431 HEADLESS_FALSE = """
 432 add_task(function headless_false() {
 433   Assert.notEqual(Services.env.get("MOZ_HEADLESS"), "1", "Check MOZ_HEADLESS");
 434   Assert.notEqual(Services.env.get("DISPLAY"), "77", "Check DISPLAY");
 435 });
 436 """
 437
 438
 439 class XPCShellTestsTests(unittest.TestCase):
 440     """
 441     Yes, these are unit tests for a unit test harness.
 442     """
 443
 444     def __init__(self, name):
 445         super(XPCShellTestsTests, self).__init__(name)
 446         from buildconfig import substs
 447         from mozbuild.base import MozbuildObject
 448
 449         os.environ.pop("MOZ_OBJDIR", None)
 450         self.build_obj = MozbuildObject.from_environment()
 451
 452         objdir = self.build_obj.topobjdir
 453         self.testing_modules = os.path.join(objdir, "_tests", "modules")
 454
 455         if mozinfo.isMac:
 456             self.xpcshellBin = os.path.join(
 457                 objdir,
 458                 "dist",
 459                 substs["MOZ_MACBUNDLE_NAME"],
 460                 "Contents",
 461                 "MacOS",
 462                 "xpcshell",
 463             )
 464         else:
 465             self.xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
 466
 467         if sys.platform == "win32":
 468             self.xpcshellBin += ".exe"
 469         self.utility_path = os.path.join(objdir, "dist", "bin")
 470         self.symbols_path = None
 471         candidate_path = os.path.join(self.build_obj.distdir, "crashreporter-symbols")
 472         if os.path.isdir(candidate_path):
 473             self.symbols_path = candidate_path
 474
 475     def setUp(self):
 476         self.log = six.StringIO()
 477         self.tempdir = tempfile.mkdtemp()
 478         logger = structured.commandline.setup_logging(
 479             "selftest%s" % id(self), {}, {"tbpl": self.log}
 480         )
 481         self.x = XPCShellTests(logger)
 482         self.x.harness_timeout = 30 if not mozinfo.info["ccov"] else 60
 483
 484     def tearDown(self):
 485         shutil.rmtree(self.tempdir)
 486         self.x.shutdownNode()
 487
 488     def writeFile(self, name, contents, mode="w"):
 489         """
 490         Write |contents| to a file named |name| in the temp directory,
 491         and return the full path to the file.
 492         """
 493         fullpath = os.path.join(self.tempdir, name)
 494         with open(fullpath, mode) as f:
 495             f.write(contents)
 496         return fullpath
 497
 498     def writeManifest(self, tests, prefs=[]):
 499         """
 500         Write an xpcshell.ini in the temp directory and set
 501         self.manifest to its pathname. |tests| is a list containing
 502         either strings (for test names), or tuples with a test name
 503         as the first element and manifest conditions as the following
 504         elements. |prefs| is an optional list of prefs in the form of
 505         "prefname=prefvalue" strings.
 506         """
 507         testlines = []
 508         for t in tests:
 509             testlines.append("[%s]" % (t if isinstance(t, six.string_types) else t[0]))
 510             if isinstance(t, tuple):
 511                 testlines.extend(t[1:])
 512         prefslines = []
 513         for p in prefs:
 514             # Append prefs lines as indented inside "prefs=" manifest option.
 515             prefslines.append("  %s" % p)
 516
 517         self.manifest = self.writeFile(
 518             "xpcshell.ini",
 519             """
 520 [DEFAULT]
 521 head =
 522 tail =
 523 prefs =
 524 """
 525             + "\n".join(prefslines)
 526             + "\n"
 527             + "\n".join(testlines),
 528         )
 529
 530     def assertTestResult(self, expected, shuffle=False, verbose=False, headless=False):
 531         """
 532         Assert that self.x.runTests with manifest=self.manifest
 533         returns |expected|.
 534         """
 535         kwargs = {}
 536         kwargs["app_binary"] = self.app_binary
 537         kwargs["xpcshell"] = self.xpcshellBin
 538         kwargs["symbolsPath"] = self.symbols_path
 539         kwargs["manifest"] = self.manifest
 540         kwargs["mozInfo"] = mozinfo.info
 541         kwargs["shuffle"] = shuffle
 542         kwargs["verbose"] = verbose
 543         kwargs["headless"] = headless
 544         kwargs["sequential"] = True
 545         kwargs["testingModulesDir"] = self.testing_modules
 546         kwargs["utility_path"] = self.utility_path
 547         kwargs["repeat"] = 0
 548         self.assertEqual(
 549             expected,
 550             self.x.runTests(kwargs),
 551             msg="""Tests should have %s, log:
 552 ========
 553 %s
 554 ========
 555 """
 556             % ("passed" if expected else "failed", self.log.getvalue()),
 557         )
 558
 559     def _assertLog(self, s, expected):
 560         l = self.log.getvalue()
 561         self.assertEqual(
 562             expected,
 563             s in l,
 564             msg="""Value %s %s in log:
 565 ========
 566 %s
 567 ========"""
 568             % (s, "expected" if expected else "not expected", l),
 569         )
 570
 571     def assertInLog(self, s):
 572         """
 573         Assert that the string |s| is contained in self.log.
 574         """
 575         self._assertLog(s, True)
 576
 577     def assertNotInLog(self, s):
 578         """
 579         Assert that the string |s| is not contained in self.log.
 580         """
 581         self._assertLog(s, False)
 582
 583     def testPass(self):
 584         """
 585         Check that a simple test without any manifest conditions passes.
 586         """
 587         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 588         self.writeManifest(["test_basic.js"])
 589
 590         self.assertTestResult(True)
 591         self.assertEqual(1, self.x.testCount)
 592         self.assertEqual(1, self.x.passCount)
 593         self.assertEqual(0, self.x.failCount)
 594         self.assertEqual(0, self.x.todoCount)
 595         self.assertInLog(TEST_PASS_STRING)
 596         self.assertNotInLog(TEST_FAIL_STRING)
 597
 598     def testFail(self):
 599         """
 600         Check that a simple failing test without any manifest conditions fails.
 601         """
 602         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 603         self.writeManifest(["test_basic.js"])
 604
 605         self.assertTestResult(False)
 606         self.assertEqual(1, self.x.testCount)
 607         self.assertEqual(0, self.x.passCount)
 608         self.assertEqual(1, self.x.failCount)
 609         self.assertEqual(0, self.x.todoCount)
 610         self.assertInLog(TEST_FAIL_STRING)
 611         self.assertNotInLog(TEST_PASS_STRING)
 612
 613     def testPrefsInManifestVerbose(self):
 614         """
 615         Check prefs configuration option is supported in xpcshell manifests.
 616         """
 617         self.writeFile("test_prefs.js", SIMPLE_PREFCHECK_TEST)
 618         self.writeManifest(tests=["test_prefs.js"], prefs=["fake.pref.to.test=true"])
 619
 620         self.assertTestResult(True, verbose=True)
 621         self.assertInLog(TEST_PASS_STRING)
 622         self.assertNotInLog(TEST_FAIL_STRING)
 623         self.assertEqual(1, self.x.testCount)
 624         self.assertEqual(1, self.x.passCount)
 625         self.assertInLog("Per-test extra prefs will be set:")
 626         self.assertInLog("fake.pref.to.test=true")
 627
 628     def testPrefsInManifestNonVerbose(self):
 629         """
 630         Check prefs configuration are not logged in non verbose mode.
 631         """
 632         self.writeFile("test_prefs.js", SIMPLE_PREFCHECK_TEST)
 633         self.writeManifest(tests=["test_prefs.js"], prefs=["fake.pref.to.test=true"])
 634
 635         self.assertTestResult(True, verbose=False)
 636         self.assertNotInLog("Per-test extra prefs will be set:")
 637         self.assertNotInLog("fake.pref.to.test=true")
 638
 639     @unittest.skipIf(
 640         mozinfo.isWin or not mozinfo.info.get("debug"),
 641         "We don't have a stack fixer on hand for windows.",
 642     )
 643     def testAssertStack(self):
 644         """
 645         When an assertion is hit, we should produce a useful stack.
 646         """
 647         self.writeFile(
 648             "test_assert.js",
 649             """
 650           add_test(function test_asserts_immediately() {
 651             Components.classes["@mozilla.org/xpcom/debug;1"]
 652                       .getService(Components.interfaces.nsIDebug2)
 653                       .assertion("foo", "assertion failed", "test.js", 1)
 654             run_next_test();
 655           });
 656         """,
 657         )
 658
 659         self.writeManifest(["test_assert.js"])
 660         self.assertTestResult(False)
 661
 662         self.assertInLog("###!!! ASSERTION")
 663         log_lines = self.log.getvalue().splitlines()
 664         line_pat = "#\d\d:"
 665         unknown_pat = "#\d\d\: \?\?\?\[.* \+0x[a-f0-9]+\]"
 666         self.assertFalse(
 667             any(re.search(unknown_pat, line) for line in log_lines),
 668             "An stack frame without symbols was found in\n%s"
 669             % pprint.pformat(log_lines),
 670         )
 671         self.assertTrue(
 672             any(re.search(line_pat, line) for line in log_lines),
 673             "No line resembling a stack frame was found in\n%s"
 674             % pprint.pformat(log_lines),
 675         )
 676
 677     def testChildPass(self):
 678         """
 679         Check that a simple test running in a child process passes.
 680         """
 681         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 682         self.writeFile("test_child_pass.js", CHILD_TEST_PASSING)
 683         self.writeManifest(["test_child_pass.js"])
 684
 685         self.assertTestResult(True, verbose=True)
 686         self.assertEqual(1, self.x.testCount)
 687         self.assertEqual(1, self.x.passCount)
 688         self.assertEqual(0, self.x.failCount)
 689         self.assertEqual(0, self.x.todoCount)
 690         self.assertInLog(TEST_PASS_STRING)
 691         self.assertInLog("CHILD-TEST-STARTED")
 692         self.assertInLog("CHILD-TEST-COMPLETED")
 693         self.assertNotInLog(TEST_FAIL_STRING)
 694
 695     def testChildFail(self):
 696         """
 697         Check that a simple failing test running in a child process fails.
 698         """
 699         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
 700         self.writeFile("test_child_fail.js", CHILD_TEST_FAILING)
 701         self.writeManifest(["test_child_fail.js"])
 702
 703         self.assertTestResult(False)
 704         self.assertEqual(1, self.x.testCount)
 705         self.assertEqual(0, self.x.passCount)
 706         self.assertEqual(1, self.x.failCount)
 707         self.assertEqual(0, self.x.todoCount)
 708         self.assertInLog(TEST_FAIL_STRING)
 709         self.assertInLog("CHILD-TEST-STARTED")
 710         self.assertInLog("CHILD-TEST-COMPLETED")
 711         self.assertNotInLog(TEST_PASS_STRING)
 712
 713     def testChildHang(self):
 714         """
 715         Check that incomplete output from a child process results in a
 716         test failure.
 717         """
 718         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 719         self.writeFile("test_child_hang.js", CHILD_TEST_HANG)
 720         self.writeManifest(["test_child_hang.js"])
 721
 722         self.assertTestResult(False)
 723         self.assertEqual(1, self.x.testCount)
 724         self.assertEqual(0, self.x.passCount)
 725         self.assertEqual(1, self.x.failCount)
 726         self.assertEqual(0, self.x.todoCount)
 727         self.assertInLog(TEST_FAIL_STRING)
 728         self.assertInLog("CHILD-TEST-STARTED")
 729         self.assertNotInLog("CHILD-TEST-COMPLETED")
 730         self.assertNotInLog(TEST_PASS_STRING)
 731
 732     def testChild(self):
 733         """
 734         Checks that calling do_load_child_test_harness without run_test_in_child
 735         results in a usable test state. This test has a spurious failure when
 736         run using |mach python-test|. See bug 1103226.
 737         """
 738         self.writeFile("test_child_assertions.js", CHILD_HARNESS_SIMPLE)
 739         self.writeManifest(["test_child_assertions.js"])
 740
 741         self.assertTestResult(True)
 742         self.assertEqual(1, self.x.testCount)
 743         self.assertEqual(1, self.x.passCount)
 744         self.assertEqual(0, self.x.failCount)
 745         self.assertEqual(0, self.x.todoCount)
 746         self.assertInLog(TEST_PASS_STRING)
 747         self.assertNotInLog(TEST_FAIL_STRING)
 748
 749     def testSkipForAddTest(self):
 750         """
 751         Check that add_test is skipped if |skip_if| condition is true
 752         """
 753         self.writeFile(
 754             "test_skip.js",
 755             """
 756 add_test({
 757   skip_if: () => true,
 758 }, function test_should_be_skipped() {
 759   Assert.ok(false);
 760   run_next_test();
 761 });
 762 """,
 763         )
 764         self.writeManifest(["test_skip.js"])
 765         self.assertTestResult(True, verbose=True)
 766         self.assertEqual(1, self.x.testCount)
 767         self.assertEqual(1, self.x.passCount)
 768         self.assertEqual(0, self.x.failCount)
 769         self.assertEqual(0, self.x.todoCount)
 770         self.assertInLog(TEST_PASS_STRING)
 771         self.assertInLog("TEST-SKIP")
 772         self.assertNotInLog(TEST_FAIL_STRING)
 773
 774     def testNotSkipForAddTask(self):
 775         """
 776         Check that add_task is not skipped if |skip_if| condition is false
 777         """
 778         self.writeFile(
 779             "test_not_skip.js",
 780             """
 781 add_task({
 782   skip_if: () => false,
 783 }, function test_should_not_be_skipped() {
 784   Assert.ok(true);
 785 });
 786 """,
 787         )
 788         self.writeManifest(["test_not_skip.js"])
 789         self.assertTestResult(True, verbose=True)
 790         self.assertEqual(1, self.x.testCount)
 791         self.assertEqual(1, self.x.passCount)
 792         self.assertEqual(0, self.x.failCount)
 793         self.assertEqual(0, self.x.todoCount)
 794         self.assertInLog(TEST_PASS_STRING)
 795         self.assertNotInLog("TEST-SKIP")
 796         self.assertNotInLog(TEST_FAIL_STRING)
 797
 798     def testSkipForAddTask(self):
 799         """
 800         Check that add_task is skipped if |skip_if| condition is true
 801         """
 802         self.writeFile(
 803             "test_skip.js",
 804             """
 805 add_task({
 806   skip_if: () => true,
 807 }, function test_should_be_skipped() {
 808   Assert.ok(false);
 809 });
 810 """,
 811         )
 812         self.writeManifest(["test_skip.js"])
 813         self.assertTestResult(True, verbose=True)
 814         self.assertEqual(1, self.x.testCount)
 815         self.assertEqual(1, self.x.passCount)
 816         self.assertEqual(0, self.x.failCount)
 817         self.assertEqual(0, self.x.todoCount)
 818         self.assertInLog(TEST_PASS_STRING)
 819         self.assertInLog("TEST-SKIP")
 820         self.assertNotInLog(TEST_FAIL_STRING)
 821
 822     def testNotSkipForAddTest(self):
 823         """
 824         Check that add_test is not skipped if |skip_if| condition is false
 825         """
 826         self.writeFile(
 827             "test_not_skip.js",
 828             """
 829 add_test({
 830   skip_if: () => false,
 831 }, function test_should_not_be_skipped() {
 832   Assert.ok(true);
 833   run_next_test();
 834 });
 835 """,
 836         )
 837         self.writeManifest(["test_not_skip.js"])
 838         self.assertTestResult(True, verbose=True)
 839         self.assertEqual(1, self.x.testCount)
 840         self.assertEqual(1, self.x.passCount)
 841         self.assertEqual(0, self.x.failCount)
 842         self.assertEqual(0, self.x.todoCount)
 843         self.assertInLog(TEST_PASS_STRING)
 844         self.assertNotInLog("TEST-SKIP")
 845         self.assertNotInLog(TEST_FAIL_STRING)
 846
 847     def testSyntaxError(self):
 848         """
 849         Check that running a test file containing a syntax error produces
 850         a test failure and expected output.
 851         """
 852         self.writeFile("test_syntax_error.js", '"')
 853         self.writeManifest(["test_syntax_error.js"])
 854
 855         self.assertTestResult(False, verbose=True)
 856         self.assertEqual(1, self.x.testCount)
 857         self.assertEqual(0, self.x.passCount)
 858         self.assertEqual(1, self.x.failCount)
 859         self.assertEqual(0, self.x.todoCount)
 860         self.assertInLog(TEST_FAIL_STRING)
 861         self.assertNotInLog(TEST_PASS_STRING)
 862
 863     def testUnicodeInAssertMethods(self):
 864         """
 865         Check that passing unicode characters through an assertion method works.
 866         """
 867         self.writeFile("test_unicode_assert.js", PASSING_TEST_UNICODE, mode="wb")
 868         self.writeManifest(["test_unicode_assert.js"])
 869
 870         self.assertTestResult(True, verbose=True)
 871
 872     @unittest.skipIf(
 873         "MOZ_AUTOMATION" in os.environ,
 874         "Timeout code path occasionally times out (bug 1098121)",
 875     )
 876     def testHangingTimeout(self):
 877         """
 878         Check that a test that never finishes results in the correct error log.
 879         """
 880         self.writeFile("test_loop.js", SIMPLE_LOOPING_TEST)
 881         self.writeManifest(["test_loop.js"])
 882
 883         old_timeout = self.x.harness_timeout
 884         self.x.harness_timeout = 1
 885
 886         self.assertTestResult(False)
 887         self.assertEqual(1, self.x.testCount)
 888         self.assertEqual(1, self.x.failCount)
 889         self.assertEqual(0, self.x.passCount)
 890         self.assertEqual(0, self.x.todoCount)
 891         self.assertInLog("TEST-UNEXPECTED-TIMEOUT")
 892
 893         self.x.harness_timeout = old_timeout
 894
 895     def testPassFail(self):
 896         """
 897         Check that running more than one test works.
 898         """
 899         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 900         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
 901         self.writeManifest(["test_pass.js", "test_fail.js"])
 902
 903         self.assertTestResult(False)
 904         self.assertEqual(2, self.x.testCount)
 905         self.assertEqual(1, self.x.passCount)
 906         self.assertEqual(1, self.x.failCount)
 907         self.assertEqual(0, self.x.todoCount)
 908         self.assertInLog(TEST_PASS_STRING)
 909         self.assertInLog(TEST_FAIL_STRING)
 910
 911     def testSkip(self):
 912         """
 913         Check that a simple failing test skipped in the manifest does
 914         not cause failure.
 915         """
 916         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 917         self.writeManifest([("test_basic.js", "skip-if = true")])
 918         self.assertTestResult(True)
 919         self.assertEqual(1, self.x.testCount)
 920         self.assertEqual(0, self.x.passCount)
 921         self.assertEqual(0, self.x.failCount)
 922         self.assertEqual(0, self.x.todoCount)
 923         self.assertNotInLog(TEST_FAIL_STRING)
 924         self.assertNotInLog(TEST_PASS_STRING)
 925
 926     def testKnownFail(self):
 927         """
 928         Check that a simple failing test marked as known-fail in the manifest
 929         does not cause failure.
 930         """
 931         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 932         self.writeManifest([("test_basic.js", "fail-if = true")])
 933         self.assertTestResult(True)
 934         self.assertEqual(1, self.x.testCount)
 935         self.assertEqual(0, self.x.passCount)
 936         self.assertEqual(0, self.x.failCount)
 937         self.assertEqual(1, self.x.todoCount)
 938         self.assertInLog("TEST-FAIL")
 939         # This should be suppressed because the harness doesn't include
 940         # the full log from the xpcshell run when things pass.
 941         self.assertNotInLog(TEST_FAIL_STRING)
 942         self.assertNotInLog(TEST_PASS_STRING)
 943
 944     def testUnexpectedPass(self):
 945         """
 946         Check that a simple failing test marked as known-fail in the manifest
 947         that passes causes an unexpected pass.
 948         """
 949         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 950         self.writeManifest([("test_basic.js", "fail-if = true")])
 951         self.assertTestResult(False)
 952         self.assertEqual(1, self.x.testCount)
 953         self.assertEqual(0, self.x.passCount)
 954         self.assertEqual(1, self.x.failCount)
 955         self.assertEqual(0, self.x.todoCount)
 956         # From the outer (Python) harness
 957         self.assertInLog("TEST-UNEXPECTED-PASS")
 958         self.assertNotInLog("TEST-KNOWN-FAIL")
 959
 960     def testReturnNonzero(self):
 961         """
 962         Check that a test where xpcshell returns nonzero fails.
 963         """
 964         self.writeFile("test_error.js", "throw 'foo'")
 965         self.writeManifest(["test_error.js"])
 966
 967         self.assertTestResult(False)
 968         self.assertEqual(1, self.x.testCount)
 969         self.assertEqual(0, self.x.passCount)
 970         self.assertEqual(1, self.x.failCount)
 971         self.assertEqual(0, self.x.todoCount)
 972         self.assertInLog(TEST_FAIL_STRING)
 973         self.assertNotInLog(TEST_PASS_STRING)
 974
 975     def testUncaughtRejection(self):
 976         """
 977         Ensure a simple test with an uncaught rejection is reported.
 978         """
 979         self.writeFile(
 980             "test_simple_uncaught_rejection.js", SIMPLE_UNCAUGHT_REJECTION_TEST
 981         )
 982         self.writeManifest(["test_simple_uncaught_rejection.js"])
 983
 984         self.assertTestResult(False)
 985         self.assertInLog(TEST_FAIL_STRING)
 986         self.assertInLog("test_simple_uncaught_rejection.js:3:18")
 987         self.assertInLog("Test rejection.")
 988         self.assertEqual(1, self.x.testCount)
 989         self.assertEqual(0, self.x.passCount)
 990         self.assertEqual(1, self.x.failCount)
 991
 992     def testAddTestSimple(self):
 993         """
 994         Ensure simple add_test() works.
 995         """
 996         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
 997         self.writeManifest(["test_add_test_simple.js"])
 998
 999         self.assertTestResult(True)
1000         self.assertEqual(1, self.x.testCount)
1001         self.assertEqual(1, self.x.passCount)
1002         self.assertEqual(0, self.x.failCount)
1003
1004     def testCrashLogging(self):
1005         """
1006         Test that a crashing test process logs a failure.
1007         """
1008         self.writeFile("test_crashes.js", TEST_CRASHING)
1009         self.writeManifest(["test_crashes.js"])
1010
1011         self.assertTestResult(False)
1012         self.assertEqual(1, self.x.testCount)
1013         self.assertEqual(0, self.x.passCount)
1014         self.assertEqual(1, self.x.failCount)
1015         if mozinfo.info.get("crashreporter"):
1016             self.assertInLog("\nPROCESS-CRASH")
1017
1018     def testLogCorrectFileName(self):
1019         """
1020         Make sure a meaningful filename and line number is logged
1021         by a passing test.
1022         """
1023         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
1024         self.writeManifest(["test_add_test_simple.js"])
1025
1026         self.assertTestResult(True, verbose=True)
1027         self.assertInLog("true == true")
1028         self.assertNotInLog("[Assert.ok :")
1029         self.assertInLog("[test_simple : 5]")
1030
1031     def testAddTestFailing(self):
1032         """
1033         Ensure add_test() with a failing test is reported.
1034         """
1035         self.writeFile("test_add_test_failing.js", ADD_TEST_FAILING)
1036         self.writeManifest(["test_add_test_failing.js"])
1037
1038         self.assertTestResult(False)
1039         self.assertEqual(1, self.x.testCount)
1040         self.assertEqual(0, self.x.passCount)
1041         self.assertEqual(1, self.x.failCount)
1042
1043     def testAddTestUncaughtRejection(self):
1044         """
1045         Ensure add_test() with an uncaught rejection is reported.
1046         """
1047         self.writeFile(
1048             "test_add_test_uncaught_rejection.js", ADD_TEST_UNCAUGHT_REJECTION
1049         )
1050         self.writeManifest(["test_add_test_uncaught_rejection.js"])
1051
1052         self.assertTestResult(False)
1053         self.assertEqual(1, self.x.testCount)
1054         self.assertEqual(0, self.x.passCount)
1055         self.assertEqual(1, self.x.failCount)
1056
1057     def testAddTaskTestSingle(self):
1058         """
1059         Ensure add_test_task() with a single passing test works.
1060         """
1061         self.writeFile("test_add_task_simple.js", ADD_TASK_SINGLE)
1062         self.writeManifest(["test_add_task_simple.js"])
1063
1064         self.assertTestResult(True)
1065         self.assertEqual(1, self.x.testCount)
1066         self.assertEqual(1, self.x.passCount)
1067         self.assertEqual(0, self.x.failCount)
1068
1069     def testAddTaskTestMultiple(self):
1070         """
1071         Ensure multiple calls to add_test_task() work as expected.
1072         """
1073         self.writeFile("test_add_task_multiple.js", ADD_TASK_MULTIPLE)
1074         self.writeManifest(["test_add_task_multiple.js"])
1075
1076         self.assertTestResult(True)
1077         self.assertEqual(1, self.x.testCount)
1078         self.assertEqual(1, self.x.passCount)
1079         self.assertEqual(0, self.x.failCount)
1080
1081     def testAddTaskTestRejected(self):
1082         """
1083         Ensure rejected task reports as failure.
1084         """
1085         self.writeFile("test_add_task_rejected.js", ADD_TASK_REJECTED)
1086         self.writeManifest(["test_add_task_rejected.js"])
1087
1088         self.assertTestResult(False)
1089         self.assertEqual(1, self.x.testCount)
1090         self.assertEqual(0, self.x.passCount)
1091         self.assertEqual(1, self.x.failCount)
1092
1093     def testAddTaskTestRejectedUndefined(self):
1094         """
1095         Ensure rejected task with undefined reason reports as failure and does not hang.
1096         """
1097         self.writeFile(
1098             "test_add_task_rejected_undefined.js", ADD_TASK_REJECTED_UNDEFINED
1099         )
1100         self.writeManifest(["test_add_task_rejected_undefined.js"])
1101
1102         self.assertTestResult(False)
1103         self.assertEqual(1, self.x.testCount)
1104         self.assertEqual(0, self.x.passCount)
1105         self.assertEqual(1, self.x.failCount)
1106         self.assertNotInLog("TEST-UNEXPECTED-TIMEOUT")
1107
1108     def testAddTaskTestFailureInside(self):
1109         """
1110         Ensure tests inside task are reported as failures.
1111         """
1112         self.writeFile("test_add_task_failure_inside.js", ADD_TASK_FAILURE_INSIDE)
1113         self.writeManifest(["test_add_task_failure_inside.js"])
1114
1115         self.assertTestResult(False)
1116         self.assertEqual(1, self.x.testCount)
1117         self.assertEqual(0, self.x.passCount)
1118         self.assertEqual(1, self.x.failCount)
1119
1120     def testAddTaskRunNextTest(self):
1121         """
1122         Calling run_next_test() from inside add_task() results in failure.
1123         """
1124         self.writeFile("test_add_task_run_next_test.js", ADD_TASK_RUN_NEXT_TEST)
1125         self.writeManifest(["test_add_task_run_next_test.js"])
1126
1127         self.assertTestResult(False)
1128         self.assertEqual(1, self.x.testCount)
1129         self.assertEqual(0, self.x.passCount)
1130         self.assertEqual(1, self.x.failCount)
1131
1132     def testAddTaskStackTrace(self):
1133         """
1134         Ensuring that calling Assert.ok(false) from inside add_task()
1135         results in a human-readable stack trace.
1136         """
1137         self.writeFile("test_add_task_stack_trace.js", ADD_TASK_STACK_TRACE)
1138         self.writeManifest(["test_add_task_stack_trace.js"])
1139
1140         self.assertTestResult(False)
1141         self.assertInLog("this_test_will_fail")
1142         self.assertInLog("run_next_test")
1143         self.assertInLog("run_test")
1144         self.assertNotInLog("Task.jsm")
1145
1146     def testAddTaskSkip(self):
1147         self.writeFile("test_tasks_skip.js", ADD_TASK_SKIP)
1148         self.writeManifest(["test_tasks_skip.js"])
1149
1150         self.assertTestResult(True)
1151         self.assertEqual(1, self.x.testCount)
1152         self.assertEqual(1, self.x.passCount)
1153         self.assertEqual(0, self.x.failCount)
1154
1155     def testAddTaskSkipAll(self):
1156         self.writeFile("test_tasks_skipall.js", ADD_TASK_SKIPALL)
1157         self.writeManifest(["test_tasks_skipall.js"])
1158
1159         self.assertTestResult(True)
1160         self.assertEqual(1, self.x.testCount)
1161         self.assertEqual(1, self.x.passCount)
1162         self.assertEqual(0, self.x.failCount)
1163
1164     def testMissingHeadFile(self):
1165         """
1166         Ensure that missing head file results in fatal failure.
1167         """
1168         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
1169         self.writeManifest([("test_basic.js", "head = missing.js")])
1170
1171         raised = False
1172
1173         try:
1174             # The actual return value is never checked because we raise.
1175             self.assertTestResult(True)
1176         except Exception as ex:
1177             raised = True
1178             self.assertEqual(str(ex)[0:9], "head file")
1179
1180         self.assertTrue(raised)
1181
1182     def testRandomExecution(self):
1183         """
1184         Check that random execution doesn't break.
1185         """
1186         manifest = []
1187         for i in range(0, 10):
1188             filename = "test_pass_%d.js" % i
1189             self.writeFile(filename, SIMPLE_PASSING_TEST)
1190             manifest.append(filename)
1191
1192         self.writeManifest(manifest)
1193         self.assertTestResult(True, shuffle=True)
1194         self.assertEqual(10, self.x.testCount)
1195         self.assertEqual(10, self.x.passCount)
1196
1197     def testDoThrowString(self):
1198         """
1199         Check that do_throw produces reasonable messages when the
1200         input is a string instead of an object
1201         """
1202         self.writeFile("test_error.js", ADD_TEST_THROW_STRING)
1203         self.writeManifest(["test_error.js"])
1204
1205         self.assertTestResult(False)
1206         self.assertInLog(TEST_FAIL_STRING)
1207         self.assertInLog("Passing a string to do_throw")
1208         self.assertNotInLog(TEST_PASS_STRING)
1209
1210     def testDoThrowForeignObject(self):
1211         """
1212         Check that do_throw produces reasonable messages when the
1213         input is a generic object with 'filename', 'message' and 'stack' attributes
1214         but 'object instanceof Error' returns false
1215         """
1216         self.writeFile("test_error.js", ADD_TEST_THROW_OBJECT)
1217         self.writeManifest(["test_error.js"])
1218
1219         self.assertTestResult(False)
1220         self.assertInLog(TEST_FAIL_STRING)
1221         self.assertInLog("failure.js")
1222         self.assertInLog("Error object")
1223         self.assertInLog("ERROR STACK")
1224         self.assertNotInLog(TEST_PASS_STRING)
1225
1226     def testDoReportForeignObject(self):
1227         """
1228         Check that do_report_unexpected_exception produces reasonable messages when the
1229         input is a generic object with 'filename', 'message' and 'stack' attributes
1230         but 'object instanceof Error' returns false
1231         """
1232         self.writeFile("test_error.js", ADD_TEST_REPORT_OBJECT)
1233         self.writeManifest(["test_error.js"])
1234
1235         self.assertTestResult(False)
1236         self.assertInLog(TEST_FAIL_STRING)
1237         self.assertInLog("failure.js")
1238         self.assertInLog("Error object")
1239         self.assertInLog("ERROR STACK")
1240         self.assertNotInLog(TEST_PASS_STRING)
1241
1242     def testDoReportRefError(self):
1243         """
1244         Check that do_report_unexpected_exception produces reasonable messages when the
1245         input is a JS-generated Error
1246         """
1247         self.writeFile("test_error.js", ADD_TEST_REPORT_REF_ERROR)
1248         self.writeManifest(["test_error.js"])
1249
1250         self.assertTestResult(False)
1251         self.assertInLog(TEST_FAIL_STRING)
1252         self.assertInLog("test_error.js")
1253         self.assertInLog("obj.noSuchFunction is not a function")
1254         self.assertInLog("run_test@")
1255         self.assertNotInLog(TEST_PASS_STRING)
1256
1257     def testDoReportSyntaxError(self):
1258         """
1259         Check that attempting to load a test file containing a syntax error
1260         generates details of the error in the log
1261         """
1262         self.writeFile("test_error.js", LOAD_ERROR_SYNTAX_ERROR)
1263         self.writeManifest(["test_error.js"])
1264
1265         self.assertTestResult(False)
1266         self.assertInLog(TEST_FAIL_STRING)
1267         self.assertInLog("test_error.js:3")
1268         self.assertNotInLog(TEST_PASS_STRING)
1269
1270     def testDoReportNonSyntaxError(self):
1271         """
1272         Check that attempting to load a test file containing an error other
1273         than a syntax error generates details of the error in the log
1274         """
1275         self.writeFile("test_error.js", LOAD_ERROR_OTHER_ERROR)
1276         self.writeManifest(["test_error.js"])
1277
1278         self.assertTestResult(False)
1279         self.assertInLog(TEST_FAIL_STRING)
1280         self.assertInLog("ReferenceError: assignment to undeclared variable")
1281         self.assertInLog("test_error.js:3")
1282         self.assertNotInLog(TEST_PASS_STRING)
1283
1284     def testDoPrintWhenVerboseNotExplicit(self):
1285         """
1286         Check that info() and similar calls that generate output do
1287         not have the output when not run verbosely.
1288         """
1289         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1290         self.writeManifest(["test_verbose.js"])
1291
1292         self.assertTestResult(True)
1293         self.assertNotInLog("a message from info")
1294
1295     def testDoPrintWhenVerboseExplicit(self):
1296         """
1297         Check that info() and similar calls that generate output have the
1298         output shown when run verbosely.
1299         """
1300         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1301         self.writeManifest(["test_verbose.js"])
1302         self.assertTestResult(True, verbose=True)
1303         self.assertInLog("a message from info")
1304
1305     def testDoPrintWhenVerboseInManifest(self):
1306         """
1307         Check that info() and similar calls that generate output have the
1308         output shown when 'verbose = true' is in the manifest, even when
1309         not run verbosely.
1310         """
1311         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1312         self.writeManifest([("test_verbose.js", "verbose = true")])
1313
1314         self.assertTestResult(True)
1315         self.assertInLog("a message from info")
1316
1317     def testAsyncCleanup(self):
1318         """
1319         Check that registerCleanupFunction handles nicely async cleanup tasks
1320         """
1321         self.writeFile("test_asyncCleanup.js", ASYNC_CLEANUP)
1322         self.writeManifest(["test_asyncCleanup.js"])
1323         self.assertTestResult(False)
1324         self.assertInLog('"123456" == "123456"')
1325         self.assertInLog("At this stage, the test has succeeded")
1326         self.assertInLog("Throwing an error to force displaying the log")
1327
1328     def testNoRunTestAddTest(self):
1329         """
1330         Check that add_test() works fine without run_test() in the test file.
1331         """
1332         self.writeFile("test_noRunTestAddTest.js", NO_RUN_TEST_ADD_TEST)
1333         self.writeManifest(["test_noRunTestAddTest.js"])
1334
1335         self.assertTestResult(True)
1336         self.assertEqual(1, self.x.testCount)
1337         self.assertEqual(1, self.x.passCount)
1338         self.assertEqual(0, self.x.failCount)
1339         self.assertInLog(TEST_PASS_STRING)
1340         self.assertNotInLog(TEST_FAIL_STRING)
1341
1342     def testNoRunTestAddTask(self):
1343         """
1344         Check that add_task() works fine without run_test() in the test file.
1345         """
1346         self.writeFile("test_noRunTestAddTask.js", NO_RUN_TEST_ADD_TASK)
1347         self.writeManifest(["test_noRunTestAddTask.js"])
1348
1349         self.assertTestResult(True)
1350         self.assertEqual(1, self.x.testCount)
1351         self.assertEqual(1, self.x.passCount)
1352         self.assertEqual(0, self.x.failCount)
1353         self.assertInLog(TEST_PASS_STRING)
1354         self.assertNotInLog(TEST_FAIL_STRING)
1355
1356     def testNoRunTestAddTestAddTask(self):
1357         """
1358         Check that both add_test() and add_task() work without run_test()
1359         in the test file.
1360         """
1361         self.writeFile("test_noRunTestAddTestAddTask.js", NO_RUN_TEST_ADD_TEST_ADD_TASK)
1362         self.writeManifest(["test_noRunTestAddTestAddTask.js"])
1363
1364         self.assertTestResult(True)
1365         self.assertEqual(1, self.x.testCount)
1366         self.assertEqual(1, self.x.passCount)
1367         self.assertEqual(0, self.x.failCount)
1368         self.assertInLog(TEST_PASS_STRING)
1369         self.assertNotInLog(TEST_FAIL_STRING)
1370
1371     def testNoRunTestEmptyTest(self):
1372         """
1373         Check that the test passes on an empty file that contains neither
1374         run_test() nor add_test(), add_task().
1375         """
1376         self.writeFile("test_noRunTestEmptyTest.js", NO_RUN_TEST_EMPTY_TEST)
1377         self.writeManifest(["test_noRunTestEmptyTest.js"])
1378
1379         self.assertTestResult(True)
1380         self.assertEqual(1, self.x.testCount)
1381         self.assertEqual(1, self.x.passCount)
1382         self.assertEqual(0, self.x.failCount)
1383         self.assertInLog(TEST_PASS_STRING)
1384         self.assertNotInLog(TEST_FAIL_STRING)
1385
1386     def testNoRunTestAddTestFail(self):
1387         """
1388         Check that test fails on using add_test() without run_test().
1389         """
1390         self.writeFile("test_noRunTestAddTestFail.js", NO_RUN_TEST_ADD_TEST_FAIL)
1391         self.writeManifest(["test_noRunTestAddTestFail.js"])
1392
1393         self.assertTestResult(False)
1394         self.assertEqual(1, self.x.testCount)
1395         self.assertEqual(0, self.x.passCount)
1396         self.assertEqual(1, self.x.failCount)
1397         self.assertInLog(TEST_FAIL_STRING)
1398         self.assertNotInLog(TEST_PASS_STRING)
1399
1400     def testNoRunTestAddTaskFail(self):
1401         """
1402         Check that test fails on using add_task() without run_test().
1403         """
1404         self.writeFile("test_noRunTestAddTaskFail.js", NO_RUN_TEST_ADD_TASK_FAIL)
1405         self.writeManifest(["test_noRunTestAddTaskFail.js"])
1406
1407         self.assertTestResult(False)
1408         self.assertEqual(1, self.x.testCount)
1409         self.assertEqual(0, self.x.passCount)
1410         self.assertEqual(1, self.x.failCount)
1411         self.assertInLog(TEST_FAIL_STRING)
1412         self.assertNotInLog(TEST_PASS_STRING)
1413
1414     def testNoRunTestAddTaskMultiple(self):
1415         """
1416         Check that multple add_task() tests work without run_test().
1417         """
1418         self.writeFile(
1419             "test_noRunTestAddTaskMultiple.js", NO_RUN_TEST_ADD_TASK_MULTIPLE
1420         )
1421         self.writeManifest(["test_noRunTestAddTaskMultiple.js"])
1422
1423         self.assertTestResult(True)
1424         self.assertEqual(1, self.x.testCount)
1425         self.assertEqual(1, self.x.passCount)
1426         self.assertEqual(0, self.x.failCount)
1427         self.assertInLog(TEST_PASS_STRING)
1428         self.assertNotInLog(TEST_FAIL_STRING)
1429
1430     def testMozinfo(self):
1431         """
1432         Check that mozinfo.json is loaded
1433         """
1434         self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
1435         self.writeManifest(["test_mozinfo.js"])
1436         self.assertTestResult(True)
1437         self.assertEqual(1, self.x.testCount)
1438         self.assertEqual(1, self.x.passCount)
1439         self.assertEqual(0, self.x.failCount)
1440         self.assertEqual(0, self.x.todoCount)
1441         self.assertInLog(TEST_PASS_STRING)
1442         self.assertNotInLog(TEST_FAIL_STRING)
1443
1444     def testChildMozinfo(self):
1445         """
1446         Check that mozinfo.json is loaded in child process
1447         """
1448         self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
1449         self.writeFile("test_child_mozinfo.js", CHILD_MOZINFO)
1450         self.writeManifest(["test_child_mozinfo.js"])
1451         self.assertTestResult(True)
1452         self.assertEqual(1, self.x.testCount)
1453         self.assertEqual(1, self.x.passCount)
1454         self.assertEqual(0, self.x.failCount)
1455         self.assertEqual(0, self.x.todoCount)
1456         self.assertInLog(TEST_PASS_STRING)
1457         self.assertNotInLog(TEST_FAIL_STRING)
1458
1459     def testNotHeadlessByDefault(self):
1460         """
1461         Check that the default is not headless.
1462         """
1463         self.writeFile("test_notHeadlessByDefault.js", HEADLESS_FALSE)
1464         self.writeManifest(["test_notHeadlessByDefault.js"])
1465         self.assertTestResult(True)
1466
1467     def testHeadlessWhenHeadlessExplicit(self):
1468         """
1469         Check that explicitly requesting headless works when the manifest doesn't override.
1470         """
1471         self.writeFile("test_headlessWhenExplicit.js", HEADLESS_TRUE)
1472         self.writeManifest(["test_headlessWhenExplicit.js"])
1473         self.assertTestResult(True, headless=True)
1474
1475     def testHeadlessWhenHeadlessTrueInManifest(self):
1476         """
1477         Check that enabling headless in the manifest alone works.
1478         """
1479         self.writeFile("test_headlessWhenTrueInManifest.js", HEADLESS_TRUE)
1480         self.writeManifest([("test_headlessWhenTrueInManifest.js", "headless = true")])
1481         self.assertTestResult(True)
1482
1483     def testNotHeadlessWhenHeadlessFalseInManifest(self):
1484         """
1485         Check that the manifest entry overrides the explicit default.
1486         """
1487         self.writeFile("test_notHeadlessWhenFalseInManifest.js", HEADLESS_FALSE)
1488         self.writeManifest(
1489             [("test_notHeadlessWhenFalseInManifest.js", "headless = false")]
1490         )
1491         self.assertTestResult(True, headless=True)
1492
1493
1494 if __name__ == "__main__":
1495     import mozunit
1496
1497     mozinfo.find_and_update_from_json()
1498     mozunit.main()