testing/xpcshell/selftest.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6
   7 import os
   8 import pprint
   9 import re
  10 import shutil
  11 import sys
  12 import tempfile
  13 import unittest
  14
  15 import mozinfo
  16 import six
  17 from mozlog import structured
  18 from runxpcshelltests import XPCShellTests
  19
  20 TEST_PASS_STRING = "TEST-PASS"
  21 TEST_FAIL_STRING = "TEST-UNEXPECTED-FAIL"
  22
  23 SIMPLE_PASSING_TEST = "function run_test() { Assert.ok(true); }"
  24 SIMPLE_FAILING_TEST = "function run_test() { Assert.ok(false); }"
  25 SIMPLE_PREFCHECK_TEST = """
  26 function run_test() {
  27   Assert.ok(Services.prefs.getBoolPref("fake.pref.to.test"));
  28 }
  29 """
  30
  31 SIMPLE_UNCAUGHT_REJECTION_TEST = """
  32 function run_test() {
  33   Promise.reject(new Error("Test rejection."));
  34   Assert.ok(true);
  35 }
  36 """
  37
  38 ADD_TEST_SIMPLE = """
  39 function run_test() { run_next_test(); }
  40
  41 add_test(function test_simple() {
  42   Assert.ok(true);
  43   run_next_test();
  44 });
  45 """
  46
  47 ADD_TEST_FAILING = """
  48 function run_test() { run_next_test(); }
  49
  50 add_test(function test_failing() {
  51   Assert.ok(false);
  52   run_next_test();
  53 });
  54 """
  55
  56 ADD_TEST_UNCAUGHT_REJECTION = """
  57 function run_test() { run_next_test(); }
  58
  59 add_test(function test_uncaught_rejection() {
  60   Promise.reject(new Error("Test rejection."));
  61   run_next_test();
  62 });
  63 """
  64
  65 CHILD_TEST_PASSING = """
  66 function run_test () { run_next_test(); }
  67
  68 add_test(function test_child_simple () {
  69   run_test_in_child("test_pass.js");
  70   run_next_test();
  71 });
  72 """
  73
  74 CHILD_TEST_FAILING = """
  75 function run_test () { run_next_test(); }
  76
  77 add_test(function test_child_simple () {
  78   run_test_in_child("test_fail.js");
  79   run_next_test();
  80 });
  81 """
  82
  83 CHILD_HARNESS_SIMPLE = """
  84 function run_test () { run_next_test(); }
  85
  86 add_test(function test_child_assert () {
  87   do_load_child_test_harness();
  88   do_test_pending("test child assertion");
  89   sendCommand("Assert.ok(true);", do_test_finished);
  90   run_next_test();
  91 });
  92 """
  93
  94 CHILD_TEST_HANG = """
  95 function run_test () { run_next_test(); }
  96
  97 add_test(function test_child_simple () {
  98   do_test_pending("hang test");
  99   do_load_child_test_harness();
 100   sendCommand("_testLogger.info('CHILD-TEST-STARTED'); " +
 101               + "const _TEST_FILE=['test_pass.js']; _execute_test(); ",
 102               do_test_finished);
 103   run_next_test();
 104 });
 105 """
 106
 107 SIMPLE_LOOPING_TEST = """
 108 function run_test () { run_next_test(); }
 109
 110 add_test(function test_loop () {
 111   do_test_pending()
 112 });
 113 """
 114
 115 PASSING_TEST_UNICODE = rb"""
 116 function run_test () { run_next_test(); }
 117
 118 add_test(function test_unicode_print () {
 119   Assert.equal("\u201c\u201d", "\u201c\u201d");
 120   run_next_test();
 121 });
 122 """
 123
 124 ADD_TASK_SINGLE = """
 125 function run_test() { run_next_test(); }
 126
 127 add_task(async function test_task() {
 128   await Promise.resolve(true);
 129   await Promise.resolve(false);
 130 });
 131 """
 132
 133 ADD_TASK_MULTIPLE = """
 134 function run_test() { run_next_test(); }
 135
 136 add_task(async function test_task() {
 137   await Promise.resolve(true);
 138 });
 139
 140 add_task(async function test_2() {
 141   await Promise.resolve(true);
 142 });
 143 """
 144
 145 ADD_TASK_REJECTED = """
 146 function run_test() { run_next_test(); }
 147
 148 add_task(async function test_failing() {
 149   await Promise.reject(new Error("I fail."));
 150 });
 151 """
 152
 153 ADD_TASK_REJECTED_UNDEFINED = """
 154 function run_test() { run_next_test(); }
 155
 156 add_task(async function test_failing() {
 157   await Promise.reject();
 158 });
 159 """
 160
 161 ADD_TASK_FAILURE_INSIDE = """
 162 function run_test() { run_next_test(); }
 163
 164 add_task(async function test() {
 165   let result = await Promise.resolve(false);
 166
 167   Assert.ok(result);
 168 });
 169 """
 170
 171 ADD_TASK_RUN_NEXT_TEST = """
 172 function run_test() { run_next_test(); }
 173
 174 add_task(function () {
 175   Assert.ok(true);
 176
 177   run_next_test();
 178 });
 179 """
 180
 181 ADD_TASK_STACK_TRACE = """
 182 function run_test() { run_next_test(); }
 183
 184 add_task(async function this_test_will_fail() {
 185   for (let i = 0; i < 10; ++i) {
 186     await Promise.resolve();
 187   }
 188   Assert.ok(false);
 189 });
 190 """
 191
 192 ADD_TASK_SKIP = """
 193 add_task(async function skipMeNot1() {
 194   Assert.ok(true, "Well well well.");
 195 });
 196
 197 add_task(async function skipMe1() {
 198   Assert.ok(false, "Not skipped after all.");
 199 }).skip();
 200
 201 add_task(async function skipMeNot2() {
 202   Assert.ok(true, "Well well well.");
 203 });
 204
 205 add_task(async function skipMeNot3() {
 206   Assert.ok(true, "Well well well.");
 207 });
 208
 209 add_task(async function skipMe2() {
 210   Assert.ok(false, "Not skipped after all.");
 211 }).skip();
 212 """
 213
 214 ADD_TASK_SKIPALL = """
 215 add_task(async function skipMe1() {
 216   Assert.ok(false, "Not skipped after all.");
 217 });
 218
 219 add_task(async function skipMe2() {
 220   Assert.ok(false, "Not skipped after all.");
 221 }).skip();
 222
 223 add_task(async function skipMe3() {
 224   Assert.ok(false, "Not skipped after all.");
 225 }).only();
 226
 227 add_task(async function skipMeNot() {
 228   Assert.ok(true, "Well well well.");
 229 }).only();
 230
 231 add_task(async function skipMe4() {
 232   Assert.ok(false, "Not skipped after all.");
 233 });
 234 """
 235
 236 ADD_TEST_THROW_STRING = """
 237 function run_test() {do_throw("Passing a string to do_throw")};
 238 """
 239
 240 ADD_TEST_THROW_OBJECT = """
 241 let error = {
 242   message: "Error object",
 243   fileName: "failure.js",
 244   stack: "ERROR STACK",
 245   toString: function() {return this.message;}
 246 };
 247 function run_test() {do_throw(error)};
 248 """
 249
 250 ADD_TEST_REPORT_OBJECT = """
 251 let error = {
 252   message: "Error object",
 253   fileName: "failure.js",
 254   stack: "ERROR STACK",
 255   toString: function() {return this.message;}
 256 };
 257 function run_test() {do_report_unexpected_exception(error)};
 258 """
 259
 260 ADD_TEST_VERBOSE = """
 261 function run_test() {info("a message from info")};
 262 """
 263
 264 # A test for genuine JS-generated Error objects
 265 ADD_TEST_REPORT_REF_ERROR = """
 266 function run_test() {
 267   let obj = {blah: 0};
 268   try {
 269     obj.noSuchFunction();
 270   }
 271   catch (error) {
 272     do_report_unexpected_exception(error);
 273   }
 274 };
 275 """
 276
 277 # A test for failure to load a test due to a syntax error
 278 LOAD_ERROR_SYNTAX_ERROR = """
 279 function run_test(
 280 """
 281
 282 # A test for failure to load a test due to an error other than a syntax error
 283 LOAD_ERROR_OTHER_ERROR = """
 284 "use strict";
 285 no_such_var = "foo"; // assignment to undeclared variable
 286 """
 287
 288 # A test that crashes outright.
 289 TEST_CRASHING = """
 290 function run_test () {
 291   const { ctypes } = ChromeUtils.importESModule(
 292     "resource://gre/modules/ctypes.sys.mjs"
 293   );
 294   let zero = new ctypes.intptr_t(8);
 295   let badptr = ctypes.cast(zero, ctypes.PointerType(ctypes.int32_t));
 296   badptr.contents;
 297 }
 298 """
 299
 300 # A test for asynchronous cleanup functions
 301 ASYNC_CLEANUP = """
 302 function run_test() {
 303   // The list of checkpoints in the order we encounter them.
 304   let checkpoints = [];
 305
 306   // Cleanup tasks, in reverse order
 307   registerCleanupFunction(function cleanup_checkout() {
 308     Assert.equal(checkpoints.join(""), "123456");
 309     info("At this stage, the test has succeeded");
 310     do_throw("Throwing an error to force displaying the log");
 311   });
 312
 313   registerCleanupFunction(function sync_cleanup_2() {
 314     checkpoints.push(6);
 315   });
 316
 317   registerCleanupFunction(async function async_cleanup_4() {
 318     await undefined;
 319     checkpoints.push(5);
 320   });
 321
 322   registerCleanupFunction(async function async_cleanup_3() {
 323     await undefined;
 324     checkpoints.push(4);
 325   });
 326
 327   registerCleanupFunction(function async_cleanup_2() {
 328     let deferred = Promise.withResolvers();
 329     executeSoon(deferred.resolve);
 330     return deferred.promise.then(function() {
 331       checkpoints.push(3);
 332     });
 333   });
 334
 335   registerCleanupFunction(function sync_cleanup() {
 336     checkpoints.push(2);
 337   });
 338
 339   registerCleanupFunction(function async_cleanup() {
 340     let deferred = Promise.withResolvers();
 341     executeSoon(deferred.resolve);
 342     return deferred.promise.then(function() {
 343       checkpoints.push(1);
 344     });
 345   });
 346
 347 }
 348 """
 349
 350 # A test to check that add_test() tests run without run_test()
 351 NO_RUN_TEST_ADD_TEST = """
 352 add_test(function no_run_test_add_test() {
 353   Assert.ok(true);
 354   run_next_test();
 355 });
 356 """
 357
 358 # A test to check that add_task() tests run without run_test()
 359 NO_RUN_TEST_ADD_TASK = """
 360 add_task(function no_run_test_add_task() {
 361   Assert.ok(true);
 362 });
 363 """
 364
 365 # A test to check that both add_task() and add_test() work without run_test()
 366 NO_RUN_TEST_ADD_TEST_ADD_TASK = """
 367 add_test(function no_run_test_add_test() {
 368   Assert.ok(true);
 369   run_next_test();
 370 });
 371
 372 add_task(function no_run_test_add_task() {
 373   Assert.ok(true);
 374 });
 375 """
 376
 377 # A test to check that an empty test file without run_test(),
 378 # add_test() or add_task() works.
 379 NO_RUN_TEST_EMPTY_TEST = """
 380 // This is an empty test file.
 381 """
 382
 383 NO_RUN_TEST_ADD_TEST_FAIL = """
 384 add_test(function no_run_test_add_test_fail() {
 385   Assert.ok(false);
 386   run_next_test();
 387 });
 388 """
 389
 390 NO_RUN_TEST_ADD_TASK_FAIL = """
 391 add_task(function no_run_test_add_task_fail() {
 392   Assert.ok(false);
 393 });
 394 """
 395
 396 NO_RUN_TEST_ADD_TASK_MULTIPLE = """
 397 add_task(async function test_task() {
 398   await Promise.resolve(true);
 399 });
 400
 401 add_task(async function test_2() {
 402   await Promise.resolve(true);
 403 });
 404 """
 405
 406 LOAD_MOZINFO = """
 407 function run_test() {
 408   Assert.notEqual(typeof mozinfo, undefined);
 409   Assert.notEqual(typeof mozinfo.os, undefined);
 410 }
 411 """
 412
 413 CHILD_MOZINFO = """
 414 function run_test () { run_next_test(); }
 415
 416 add_test(function test_child_mozinfo () {
 417   run_test_in_child("test_mozinfo.js");
 418   run_next_test();
 419 });
 420 """
 421
 422 HEADLESS_TRUE = """
 423 add_task(function headless_true() {
 424   Assert.equal(Services.env.get("MOZ_HEADLESS"), "1", "Check MOZ_HEADLESS");
 425   Assert.equal(Services.env.get("DISPLAY"), "77", "Check DISPLAY");
 426 });
 427 """
 428
 429 HEADLESS_FALSE = """
 430 add_task(function headless_false() {
 431   Assert.notEqual(Services.env.get("MOZ_HEADLESS"), "1", "Check MOZ_HEADLESS");
 432   Assert.notEqual(Services.env.get("DISPLAY"), "77", "Check DISPLAY");
 433 });
 434 """
 435
 436
 437 class XPCShellTestsTests(unittest.TestCase):
 438     """
 439     Yes, these are unit tests for a unit test harness.
 440     """
 441
 442     def __init__(self, name):
 443         super(XPCShellTestsTests, self).__init__(name)
 444         from buildconfig import substs
 445         from mozbuild.base import MozbuildObject
 446
 447         os.environ.pop("MOZ_OBJDIR", None)
 448         self.build_obj = MozbuildObject.from_environment()
 449
 450         objdir = self.build_obj.topobjdir
 451         self.testing_modules = os.path.join(objdir, "_tests", "modules")
 452
 453         if mozinfo.isMac:
 454             self.xpcshellBin = os.path.join(
 455                 objdir,
 456                 "dist",
 457                 substs["MOZ_MACBUNDLE_NAME"],
 458                 "Contents",
 459                 "MacOS",
 460                 "xpcshell",
 461             )
 462         else:
 463             self.xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
 464
 465         if sys.platform == "win32":
 466             self.xpcshellBin += ".exe"
 467         self.utility_path = os.path.join(objdir, "dist", "bin")
 468         self.symbols_path = None
 469         candidate_path = os.path.join(self.build_obj.distdir, "crashreporter-symbols")
 470         if os.path.isdir(candidate_path):
 471             self.symbols_path = candidate_path
 472
 473     def setUp(self):
 474         self.log = six.StringIO()
 475         self.tempdir = tempfile.mkdtemp()
 476         logger = structured.commandline.setup_logging(
 477             "selftest%s" % id(self), {}, {"tbpl": self.log}
 478         )
 479         self.x = XPCShellTests(logger)
 480         self.x.harness_timeout = 30 if not mozinfo.info["ccov"] else 60
 481
 482     def tearDown(self):
 483         shutil.rmtree(self.tempdir)
 484         self.x.shutdownNode()
 485
 486     def writeFile(self, name, contents, mode="w"):
 487         """
 488         Write |contents| to a file named |name| in the temp directory,
 489         and return the full path to the file.
 490         """
 491         fullpath = os.path.join(self.tempdir, name)
 492         with open(fullpath, mode) as f:
 493             f.write(contents)
 494         return fullpath
 495
 496     def writeManifest(self, tests, prefs=[]):
 497         """
 498         Write an xpcshell.ini in the temp directory and set
 499         self.manifest to its pathname. |tests| is a list containing
 500         either strings (for test names), or tuples with a test name
 501         as the first element and manifest conditions as the following
 502         elements. |prefs| is an optional list of prefs in the form of
 503         "prefname=prefvalue" strings.
 504         """
 505         testlines = []
 506         for t in tests:
 507             testlines.append(
 508                 '["%s"]' % (t if isinstance(t, six.string_types) else t[0])
 509             )
 510             if isinstance(t, tuple):
 511                 testlines.extend(t[1:])
 512         prefslines = []
 513         for p in prefs:
 514             # Append prefs lines as indented inside "prefs=" manifest option.
 515             prefslines.append('  "%s",' % p)
 516
 517         val = """
 518 [DEFAULT]
 519 head = ""
 520 tail = ""
 521 prefs = [
 522 """
 523         val += "\n".join(prefslines)
 524         val += "]\n"
 525         val += "\n".join(testlines)
 526         self.manifest = self.writeFile("xpcshell.toml", val)
 527
 528     def assertTestResult(self, expected, shuffle=False, verbose=False, headless=False):
 529         """
 530         Assert that self.x.runTests with manifest=self.manifest
 531         returns |expected|.
 532         """
 533         kwargs = {}
 534         kwargs["app_binary"] = self.app_binary
 535         kwargs["xpcshell"] = self.xpcshellBin
 536         kwargs["symbolsPath"] = self.symbols_path
 537         kwargs["manifest"] = self.manifest
 538         kwargs["mozInfo"] = mozinfo.info
 539         kwargs["shuffle"] = shuffle
 540         kwargs["verbose"] = verbose
 541         kwargs["headless"] = headless
 542         kwargs["sequential"] = True
 543         kwargs["testingModulesDir"] = self.testing_modules
 544         kwargs["utility_path"] = self.utility_path
 545         kwargs["repeat"] = 0
 546         self.assertEqual(
 547             expected,
 548             self.x.runTests(kwargs),
 549             msg="""Tests should have %s, log:
 550 ========
 551 %s
 552 ========
 553 """
 554             % ("passed" if expected else "failed", self.log.getvalue()),
 555         )
 556
 557     def _assertLog(self, s, expected):
 558         l = self.log.getvalue()
 559         self.assertEqual(
 560             expected,
 561             s in l,
 562             msg="""Value %s %s in log:
 563 ========
 564 %s
 565 ========"""
 566             % (s, "expected" if expected else "not expected", l),
 567         )
 568
 569     def assertInLog(self, s):
 570         """
 571         Assert that the string |s| is contained in self.log.
 572         """
 573         self._assertLog(s, True)
 574
 575     def assertNotInLog(self, s):
 576         """
 577         Assert that the string |s| is not contained in self.log.
 578         """
 579         self._assertLog(s, False)
 580
 581     def testPass(self):
 582         """
 583         Check that a simple test without any manifest conditions passes.
 584         """
 585         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 586         self.writeManifest(["test_basic.js"])
 587
 588         self.assertTestResult(True)
 589         self.assertEqual(1, self.x.testCount)
 590         self.assertEqual(1, self.x.passCount)
 591         self.assertEqual(0, self.x.failCount)
 592         self.assertEqual(0, self.x.todoCount)
 593         self.assertInLog(TEST_PASS_STRING)
 594         self.assertNotInLog(TEST_FAIL_STRING)
 595
 596     def testFail(self):
 597         """
 598         Check that a simple failing test without any manifest conditions fails.
 599         """
 600         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 601         self.writeManifest(["test_basic.js"])
 602
 603         self.assertTestResult(False)
 604         self.assertEqual(1, self.x.testCount)
 605         self.assertEqual(0, self.x.passCount)
 606         self.assertEqual(1, self.x.failCount)
 607         self.assertEqual(0, self.x.todoCount)
 608         self.assertInLog(TEST_FAIL_STRING)
 609         self.assertNotInLog(TEST_PASS_STRING)
 610
 611     def testPrefsInManifestVerbose(self):
 612         """
 613         Check prefs configuration option is supported in xpcshell manifests.
 614         """
 615         self.writeFile("test_prefs.js", SIMPLE_PREFCHECK_TEST)
 616         self.writeManifest(tests=["test_prefs.js"], prefs=["fake.pref.to.test=true"])
 617
 618         self.assertTestResult(True, verbose=True)
 619         self.assertInLog(TEST_PASS_STRING)
 620         self.assertNotInLog(TEST_FAIL_STRING)
 621         self.assertEqual(1, self.x.testCount)
 622         self.assertEqual(1, self.x.passCount)
 623         self.assertInLog("Per-test extra prefs will be set:")
 624         self.assertInLog("fake.pref.to.test=true")
 625
 626     def testPrefsInManifestNonVerbose(self):
 627         """
 628         Check prefs configuration are not logged in non verbose mode.
 629         """
 630         self.writeFile("test_prefs.js", SIMPLE_PREFCHECK_TEST)
 631         self.writeManifest(tests=["test_prefs.js"], prefs=["fake.pref.to.test=true"])
 632
 633         self.assertTestResult(True, verbose=False)
 634         self.assertNotInLog("Per-test extra prefs will be set:")
 635         self.assertNotInLog("fake.pref.to.test=true")
 636
 637     @unittest.skipIf(
 638         mozinfo.isWin or not mozinfo.info.get("debug"),
 639         "We don't have a stack fixer on hand for windows.",
 640     )
 641     def testAssertStack(self):
 642         """
 643         When an assertion is hit, we should produce a useful stack.
 644         """
 645         self.writeFile(
 646             "test_assert.js",
 647             """
 648           add_test(function test_asserts_immediately() {
 649             Components.classes["@mozilla.org/xpcom/debug;1"]
 650                       .getService(Components.interfaces.nsIDebug2)
 651                       .assertion("foo", "assertion failed", "test.js", 1)
 652             run_next_test();
 653           });
 654         """,
 655         )
 656
 657         self.writeManifest(["test_assert.js"])
 658         self.assertTestResult(False)
 659
 660         self.assertInLog("###!!! ASSERTION")
 661         log_lines = self.log.getvalue().splitlines()
 662         line_pat = r"#\d\d:"
 663         unknown_pat = r"#\d\d\: \?\?\?\[.* \+0x[a-f0-9]+\]"
 664         self.assertFalse(
 665             any(re.search(unknown_pat, line) for line in log_lines),
 666             "An stack frame without symbols was found in\n%s"
 667             % pprint.pformat(log_lines),
 668         )
 669         self.assertTrue(
 670             any(re.search(line_pat, line) for line in log_lines),
 671             "No line resembling a stack frame was found in\n%s"
 672             % pprint.pformat(log_lines),
 673         )
 674
 675     def testChildPass(self):
 676         """
 677         Check that a simple test running in a child process passes.
 678         """
 679         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 680         self.writeFile("test_child_pass.js", CHILD_TEST_PASSING)
 681         self.writeManifest(["test_child_pass.js"])
 682
 683         self.assertTestResult(True, verbose=True)
 684         self.assertEqual(1, self.x.testCount)
 685         self.assertEqual(1, self.x.passCount)
 686         self.assertEqual(0, self.x.failCount)
 687         self.assertEqual(0, self.x.todoCount)
 688         self.assertInLog(TEST_PASS_STRING)
 689         self.assertInLog("CHILD-TEST-STARTED")
 690         self.assertInLog("CHILD-TEST-COMPLETED")
 691         self.assertNotInLog(TEST_FAIL_STRING)
 692
 693     def testChildFail(self):
 694         """
 695         Check that a simple failing test running in a child process fails.
 696         """
 697         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
 698         self.writeFile("test_child_fail.js", CHILD_TEST_FAILING)
 699         self.writeManifest(["test_child_fail.js"])
 700
 701         self.assertTestResult(False)
 702         self.assertEqual(1, self.x.testCount)
 703         self.assertEqual(0, self.x.passCount)
 704         self.assertEqual(1, self.x.failCount)
 705         self.assertEqual(0, self.x.todoCount)
 706         self.assertInLog(TEST_FAIL_STRING)
 707         self.assertInLog("CHILD-TEST-STARTED")
 708         self.assertInLog("CHILD-TEST-COMPLETED")
 709         self.assertNotInLog(TEST_PASS_STRING)
 710
 711     def testChildHang(self):
 712         """
 713         Check that incomplete output from a child process results in a
 714         test failure.
 715         """
 716         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 717         self.writeFile("test_child_hang.js", CHILD_TEST_HANG)
 718         self.writeManifest(["test_child_hang.js"])
 719
 720         self.assertTestResult(False)
 721         self.assertEqual(1, self.x.testCount)
 722         self.assertEqual(0, self.x.passCount)
 723         self.assertEqual(1, self.x.failCount)
 724         self.assertEqual(0, self.x.todoCount)
 725         self.assertInLog(TEST_FAIL_STRING)
 726         self.assertInLog("CHILD-TEST-STARTED")
 727         self.assertNotInLog("CHILD-TEST-COMPLETED")
 728         self.assertNotInLog(TEST_PASS_STRING)
 729
 730     def testChild(self):
 731         """
 732         Checks that calling do_load_child_test_harness without run_test_in_child
 733         results in a usable test state. This test has a spurious failure when
 734         run using |mach python-test|. See bug 1103226.
 735         """
 736         self.writeFile("test_child_assertions.js", CHILD_HARNESS_SIMPLE)
 737         self.writeManifest(["test_child_assertions.js"])
 738
 739         self.assertTestResult(True)
 740         self.assertEqual(1, self.x.testCount)
 741         self.assertEqual(1, self.x.passCount)
 742         self.assertEqual(0, self.x.failCount)
 743         self.assertEqual(0, self.x.todoCount)
 744         self.assertInLog(TEST_PASS_STRING)
 745         self.assertNotInLog(TEST_FAIL_STRING)
 746
 747     def testSkipForAddTest(self):
 748         """
 749         Check that add_test is skipped if |skip_if| condition is true
 750         """
 751         self.writeFile(
 752             "test_skip.js",
 753             """
 754 add_test({
 755   skip_if: () => true,
 756 }, function test_should_be_skipped() {
 757   Assert.ok(false);
 758   run_next_test();
 759 });
 760 """,
 761         )
 762         self.writeManifest(["test_skip.js"])
 763         self.assertTestResult(True, verbose=True)
 764         self.assertEqual(1, self.x.testCount)
 765         self.assertEqual(1, self.x.passCount)
 766         self.assertEqual(0, self.x.failCount)
 767         self.assertEqual(0, self.x.todoCount)
 768         self.assertInLog(TEST_PASS_STRING)
 769         self.assertInLog("TEST-SKIP")
 770         self.assertNotInLog(TEST_FAIL_STRING)
 771
 772     def testNotSkipForAddTask(self):
 773         """
 774         Check that add_task is not skipped if |skip_if| condition is false
 775         """
 776         self.writeFile(
 777             "test_not_skip.js",
 778             """
 779 add_task({
 780   skip_if: () => false,
 781 }, function test_should_not_be_skipped() {
 782   Assert.ok(true);
 783 });
 784 """,
 785         )
 786         self.writeManifest(["test_not_skip.js"])
 787         self.assertTestResult(True, verbose=True)
 788         self.assertEqual(1, self.x.testCount)
 789         self.assertEqual(1, self.x.passCount)
 790         self.assertEqual(0, self.x.failCount)
 791         self.assertEqual(0, self.x.todoCount)
 792         self.assertInLog(TEST_PASS_STRING)
 793         self.assertNotInLog("TEST-SKIP")
 794         self.assertNotInLog(TEST_FAIL_STRING)
 795
 796     def testSkipForAddTask(self):
 797         """
 798         Check that add_task is skipped if |skip_if| condition is true
 799         """
 800         self.writeFile(
 801             "test_skip.js",
 802             """
 803 add_task({
 804   skip_if: () => true,
 805 }, function test_should_be_skipped() {
 806   Assert.ok(false);
 807 });
 808 """,
 809         )
 810         self.writeManifest(["test_skip.js"])
 811         self.assertTestResult(True, verbose=True)
 812         self.assertEqual(1, self.x.testCount)
 813         self.assertEqual(1, self.x.passCount)
 814         self.assertEqual(0, self.x.failCount)
 815         self.assertEqual(0, self.x.todoCount)
 816         self.assertInLog(TEST_PASS_STRING)
 817         self.assertInLog("TEST-SKIP")
 818         self.assertNotInLog(TEST_FAIL_STRING)
 819
 820     def testNotSkipForAddTest(self):
 821         """
 822         Check that add_test is not skipped if |skip_if| condition is false
 823         """
 824         self.writeFile(
 825             "test_not_skip.js",
 826             """
 827 add_test({
 828   skip_if: () => false,
 829 }, function test_should_not_be_skipped() {
 830   Assert.ok(true);
 831   run_next_test();
 832 });
 833 """,
 834         )
 835         self.writeManifest(["test_not_skip.js"])
 836         self.assertTestResult(True, verbose=True)
 837         self.assertEqual(1, self.x.testCount)
 838         self.assertEqual(1, self.x.passCount)
 839         self.assertEqual(0, self.x.failCount)
 840         self.assertEqual(0, self.x.todoCount)
 841         self.assertInLog(TEST_PASS_STRING)
 842         self.assertNotInLog("TEST-SKIP")
 843         self.assertNotInLog(TEST_FAIL_STRING)
 844
 845     def testSyntaxError(self):
 846         """
 847         Check that running a test file containing a syntax error produces
 848         a test failure and expected output.
 849         """
 850         self.writeFile("test_syntax_error.js", '"')
 851         self.writeManifest(["test_syntax_error.js"])
 852
 853         self.assertTestResult(False, verbose=True)
 854         self.assertEqual(1, self.x.testCount)
 855         self.assertEqual(0, self.x.passCount)
 856         self.assertEqual(1, self.x.failCount)
 857         self.assertEqual(0, self.x.todoCount)
 858         self.assertInLog(TEST_FAIL_STRING)
 859         self.assertNotInLog(TEST_PASS_STRING)
 860
 861     def testUnicodeInAssertMethods(self):
 862         """
 863         Check that passing unicode characters through an assertion method works.
 864         """
 865         self.writeFile("test_unicode_assert.js", PASSING_TEST_UNICODE, mode="wb")
 866         self.writeManifest(["test_unicode_assert.js"])
 867
 868         self.assertTestResult(True, verbose=True)
 869
 870     @unittest.skipIf(
 871         "MOZ_AUTOMATION" in os.environ,
 872         "Timeout code path occasionally times out (bug 1098121)",
 873     )
 874     def testHangingTimeout(self):
 875         """
 876         Check that a test that never finishes results in the correct error log.
 877         """
 878         self.writeFile("test_loop.js", SIMPLE_LOOPING_TEST)
 879         self.writeManifest(["test_loop.js"])
 880
 881         old_timeout = self.x.harness_timeout
 882         self.x.harness_timeout = 1
 883
 884         self.assertTestResult(False)
 885         self.assertEqual(1, self.x.testCount)
 886         self.assertEqual(1, self.x.failCount)
 887         self.assertEqual(0, self.x.passCount)
 888         self.assertEqual(0, self.x.todoCount)
 889         self.assertInLog("TEST-UNEXPECTED-TIMEOUT")
 890
 891         self.x.harness_timeout = old_timeout
 892
 893     def testPassFail(self):
 894         """
 895         Check that running more than one test works.
 896         """
 897         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 898         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
 899         self.writeManifest(["test_pass.js", "test_fail.js"])
 900
 901         self.assertTestResult(False)
 902         self.assertEqual(2, self.x.testCount)
 903         self.assertEqual(1, self.x.passCount)
 904         self.assertEqual(1, self.x.failCount)
 905         self.assertEqual(0, self.x.todoCount)
 906         self.assertInLog(TEST_PASS_STRING)
 907         self.assertInLog(TEST_FAIL_STRING)
 908
 909     def testSkip(self):
 910         """
 911         Check that a simple failing test skipped in the manifest does
 912         not cause failure.
 913         """
 914         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 915         self.writeManifest([("test_basic.js", "skip-if = true")])
 916         self.assertTestResult(True)
 917         self.assertEqual(1, self.x.testCount)
 918         self.assertEqual(0, self.x.passCount)
 919         self.assertEqual(0, self.x.failCount)
 920         self.assertEqual(0, self.x.todoCount)
 921         self.assertNotInLog(TEST_FAIL_STRING)
 922         self.assertNotInLog(TEST_PASS_STRING)
 923
 924     def testKnownFail(self):
 925         """
 926         Check that a simple failing test marked as known-fail in the manifest
 927         does not cause failure.
 928         """
 929         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 930         self.writeManifest([("test_basic.js", "fail-if = true")])
 931         self.assertTestResult(True)
 932         self.assertEqual(1, self.x.testCount)
 933         self.assertEqual(0, self.x.passCount)
 934         self.assertEqual(0, self.x.failCount)
 935         self.assertEqual(1, self.x.todoCount)
 936         self.assertInLog("TEST-FAIL")
 937         # This should be suppressed because the harness doesn't include
 938         # the full log from the xpcshell run when things pass.
 939         self.assertNotInLog(TEST_FAIL_STRING)
 940         self.assertNotInLog(TEST_PASS_STRING)
 941
 942     def testUnexpectedPass(self):
 943         """
 944         Check that a simple failing test marked as known-fail in the manifest
 945         that passes causes an unexpected pass.
 946         """
 947         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 948         self.writeManifest([("test_basic.js", "fail-if = true")])
 949         self.assertTestResult(False)
 950         self.assertEqual(1, self.x.testCount)
 951         self.assertEqual(0, self.x.passCount)
 952         self.assertEqual(1, self.x.failCount)
 953         self.assertEqual(0, self.x.todoCount)
 954         # From the outer (Python) harness
 955         self.assertInLog("TEST-UNEXPECTED-PASS")
 956         self.assertNotInLog("TEST-KNOWN-FAIL")
 957
 958     def testReturnNonzero(self):
 959         """
 960         Check that a test where xpcshell returns nonzero fails.
 961         """
 962         self.writeFile("test_error.js", "throw 'foo'")
 963         self.writeManifest(["test_error.js"])
 964
 965         self.assertTestResult(False)
 966         self.assertEqual(1, self.x.testCount)
 967         self.assertEqual(0, self.x.passCount)
 968         self.assertEqual(1, self.x.failCount)
 969         self.assertEqual(0, self.x.todoCount)
 970         self.assertInLog(TEST_FAIL_STRING)
 971         self.assertNotInLog(TEST_PASS_STRING)
 972
 973     def testUncaughtRejection(self):
 974         """
 975         Ensure a simple test with an uncaught rejection is reported.
 976         """
 977         self.writeFile(
 978             "test_simple_uncaught_rejection.js", SIMPLE_UNCAUGHT_REJECTION_TEST
 979         )
 980         self.writeManifest(["test_simple_uncaught_rejection.js"])
 981
 982         self.assertTestResult(False)
 983         self.assertInLog(TEST_FAIL_STRING)
 984         self.assertInLog("test_simple_uncaught_rejection.js:3:18")
 985         self.assertInLog("Test rejection.")
 986         self.assertEqual(1, self.x.testCount)
 987         self.assertEqual(0, self.x.passCount)
 988         self.assertEqual(1, self.x.failCount)
 989
 990     def testAddTestSimple(self):
 991         """
 992         Ensure simple add_test() works.
 993         """
 994         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
 995         self.writeManifest(["test_add_test_simple.js"])
 996
 997         self.assertTestResult(True)
 998         self.assertEqual(1, self.x.testCount)
 999         self.assertEqual(1, self.x.passCount)
1000         self.assertEqual(0, self.x.failCount)
1001
1002     def testCrashLogging(self):
1003         """
1004         Test that a crashing test process logs a failure.
1005         """
1006         self.writeFile("test_crashes.js", TEST_CRASHING)
1007         self.writeManifest(["test_crashes.js"])
1008
1009         self.assertTestResult(False)
1010         self.assertEqual(1, self.x.testCount)
1011         self.assertEqual(0, self.x.passCount)
1012         self.assertEqual(1, self.x.failCount)
1013         if mozinfo.info.get("crashreporter"):
1014             self.assertInLog("\nPROCESS-CRASH")
1015
1016     def testLogCorrectFileName(self):
1017         """
1018         Make sure a meaningful filename and line number is logged
1019         by a passing test.
1020         """
1021         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
1022         self.writeManifest(["test_add_test_simple.js"])
1023
1024         self.assertTestResult(True, verbose=True)
1025         self.assertInLog("true == true")
1026         self.assertNotInLog("[Assert.ok :")
1027         self.assertInLog("[test_simple : 5]")
1028
1029     def testAddTestFailing(self):
1030         """
1031         Ensure add_test() with a failing test is reported.
1032         """
1033         self.writeFile("test_add_test_failing.js", ADD_TEST_FAILING)
1034         self.writeManifest(["test_add_test_failing.js"])
1035
1036         self.assertTestResult(False)
1037         self.assertEqual(1, self.x.testCount)
1038         self.assertEqual(0, self.x.passCount)
1039         self.assertEqual(1, self.x.failCount)
1040
1041     def testAddTestUncaughtRejection(self):
1042         """
1043         Ensure add_test() with an uncaught rejection is reported.
1044         """
1045         self.writeFile(
1046             "test_add_test_uncaught_rejection.js", ADD_TEST_UNCAUGHT_REJECTION
1047         )
1048         self.writeManifest(["test_add_test_uncaught_rejection.js"])
1049
1050         self.assertTestResult(False)
1051         self.assertEqual(1, self.x.testCount)
1052         self.assertEqual(0, self.x.passCount)
1053         self.assertEqual(1, self.x.failCount)
1054
1055     def testAddTaskTestSingle(self):
1056         """
1057         Ensure add_test_task() with a single passing test works.
1058         """
1059         self.writeFile("test_add_task_simple.js", ADD_TASK_SINGLE)
1060         self.writeManifest(["test_add_task_simple.js"])
1061
1062         self.assertTestResult(True)
1063         self.assertEqual(1, self.x.testCount)
1064         self.assertEqual(1, self.x.passCount)
1065         self.assertEqual(0, self.x.failCount)
1066
1067     def testAddTaskTestMultiple(self):
1068         """
1069         Ensure multiple calls to add_test_task() work as expected.
1070         """
1071         self.writeFile("test_add_task_multiple.js", ADD_TASK_MULTIPLE)
1072         self.writeManifest(["test_add_task_multiple.js"])
1073
1074         self.assertTestResult(True)
1075         self.assertEqual(1, self.x.testCount)
1076         self.assertEqual(1, self.x.passCount)
1077         self.assertEqual(0, self.x.failCount)
1078
1079     def testAddTaskTestRejected(self):
1080         """
1081         Ensure rejected task reports as failure.
1082         """
1083         self.writeFile("test_add_task_rejected.js", ADD_TASK_REJECTED)
1084         self.writeManifest(["test_add_task_rejected.js"])
1085
1086         self.assertTestResult(False)
1087         self.assertEqual(1, self.x.testCount)
1088         self.assertEqual(0, self.x.passCount)
1089         self.assertEqual(1, self.x.failCount)
1090
1091     def testAddTaskTestRejectedUndefined(self):
1092         """
1093         Ensure rejected task with undefined reason reports as failure and does not hang.
1094         """
1095         self.writeFile(
1096             "test_add_task_rejected_undefined.js", ADD_TASK_REJECTED_UNDEFINED
1097         )
1098         self.writeManifest(["test_add_task_rejected_undefined.js"])
1099
1100         self.assertTestResult(False)
1101         self.assertEqual(1, self.x.testCount)
1102         self.assertEqual(0, self.x.passCount)
1103         self.assertEqual(1, self.x.failCount)
1104         self.assertNotInLog("TEST-UNEXPECTED-TIMEOUT")
1105
1106     def testAddTaskTestFailureInside(self):
1107         """
1108         Ensure tests inside task are reported as failures.
1109         """
1110         self.writeFile("test_add_task_failure_inside.js", ADD_TASK_FAILURE_INSIDE)
1111         self.writeManifest(["test_add_task_failure_inside.js"])
1112
1113         self.assertTestResult(False)
1114         self.assertEqual(1, self.x.testCount)
1115         self.assertEqual(0, self.x.passCount)
1116         self.assertEqual(1, self.x.failCount)
1117
1118     def testAddTaskRunNextTest(self):
1119         """
1120         Calling run_next_test() from inside add_task() results in failure.
1121         """
1122         self.writeFile("test_add_task_run_next_test.js", ADD_TASK_RUN_NEXT_TEST)
1123         self.writeManifest(["test_add_task_run_next_test.js"])
1124
1125         self.assertTestResult(False)
1126         self.assertEqual(1, self.x.testCount)
1127         self.assertEqual(0, self.x.passCount)
1128         self.assertEqual(1, self.x.failCount)
1129
1130     def testAddTaskStackTrace(self):
1131         """
1132         Ensuring that calling Assert.ok(false) from inside add_task()
1133         results in a human-readable stack trace.
1134         """
1135         self.writeFile("test_add_task_stack_trace.js", ADD_TASK_STACK_TRACE)
1136         self.writeManifest(["test_add_task_stack_trace.js"])
1137
1138         self.assertTestResult(False)
1139         self.assertInLog("this_test_will_fail")
1140         self.assertInLog("run_next_test")
1141         self.assertInLog("run_test")
1142
1143     def testAddTaskSkip(self):
1144         self.writeFile("test_tasks_skip.js", ADD_TASK_SKIP)
1145         self.writeManifest(["test_tasks_skip.js"])
1146
1147         self.assertTestResult(True)
1148         self.assertEqual(1, self.x.testCount)
1149         self.assertEqual(1, self.x.passCount)
1150         self.assertEqual(0, self.x.failCount)
1151
1152     def testAddTaskSkipAll(self):
1153         self.writeFile("test_tasks_skipall.js", ADD_TASK_SKIPALL)
1154         self.writeManifest(["test_tasks_skipall.js"])
1155
1156         self.assertTestResult(True)
1157         self.assertEqual(1, self.x.testCount)
1158         self.assertEqual(1, self.x.passCount)
1159         self.assertEqual(0, self.x.failCount)
1160
1161     def testMissingHeadFile(self):
1162         """
1163         Ensure that missing head file results in fatal failure.
1164         """
1165         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
1166         self.writeManifest([("test_basic.js", 'head = "missing.js"')])
1167
1168         raised = False
1169
1170         try:
1171             # The actual return value is never checked because we raise.
1172             self.assertTestResult(True)
1173         except Exception as ex:
1174             raised = True
1175             self.assertEqual(str(ex)[0:9], "head file")
1176
1177         self.assertTrue(raised)
1178
1179     def testRandomExecution(self):
1180         """
1181         Check that random execution doesn't break.
1182         """
1183         manifest = []
1184         for i in range(0, 10):
1185             filename = "test_pass_%d.js" % i
1186             self.writeFile(filename, SIMPLE_PASSING_TEST)
1187             manifest.append(filename)
1188
1189         self.writeManifest(manifest)
1190         self.assertTestResult(True, shuffle=True)
1191         self.assertEqual(10, self.x.testCount)
1192         self.assertEqual(10, self.x.passCount)
1193
1194     def testDoThrowString(self):
1195         """
1196         Check that do_throw produces reasonable messages when the
1197         input is a string instead of an object
1198         """
1199         self.writeFile("test_error.js", ADD_TEST_THROW_STRING)
1200         self.writeManifest(["test_error.js"])
1201
1202         self.assertTestResult(False)
1203         self.assertInLog(TEST_FAIL_STRING)
1204         self.assertInLog("Passing a string to do_throw")
1205         self.assertNotInLog(TEST_PASS_STRING)
1206
1207     def testDoThrowForeignObject(self):
1208         """
1209         Check that do_throw produces reasonable messages when the
1210         input is a generic object with 'filename', 'message' and 'stack' attributes
1211         but 'object instanceof Error' returns false
1212         """
1213         self.writeFile("test_error.js", ADD_TEST_THROW_OBJECT)
1214         self.writeManifest(["test_error.js"])
1215
1216         self.assertTestResult(False)
1217         self.assertInLog(TEST_FAIL_STRING)
1218         self.assertInLog("failure.js")
1219         self.assertInLog("Error object")
1220         self.assertInLog("ERROR STACK")
1221         self.assertNotInLog(TEST_PASS_STRING)
1222
1223     def testDoReportForeignObject(self):
1224         """
1225         Check that do_report_unexpected_exception produces reasonable messages when the
1226         input is a generic object with 'filename', 'message' and 'stack' attributes
1227         but 'object instanceof Error' returns false
1228         """
1229         self.writeFile("test_error.js", ADD_TEST_REPORT_OBJECT)
1230         self.writeManifest(["test_error.js"])
1231
1232         self.assertTestResult(False)
1233         self.assertInLog(TEST_FAIL_STRING)
1234         self.assertInLog("failure.js")
1235         self.assertInLog("Error object")
1236         self.assertInLog("ERROR STACK")
1237         self.assertNotInLog(TEST_PASS_STRING)
1238
1239     def testDoReportRefError(self):
1240         """
1241         Check that do_report_unexpected_exception produces reasonable messages when the
1242         input is a JS-generated Error
1243         """
1244         self.writeFile("test_error.js", ADD_TEST_REPORT_REF_ERROR)
1245         self.writeManifest(["test_error.js"])
1246
1247         self.assertTestResult(False)
1248         self.assertInLog(TEST_FAIL_STRING)
1249         self.assertInLog("test_error.js")
1250         self.assertInLog("obj.noSuchFunction is not a function")
1251         self.assertInLog("run_test@")
1252         self.assertNotInLog(TEST_PASS_STRING)
1253
1254     def testDoReportSyntaxError(self):
1255         """
1256         Check that attempting to load a test file containing a syntax error
1257         generates details of the error in the log
1258         """
1259         self.writeFile("test_error.js", LOAD_ERROR_SYNTAX_ERROR)
1260         self.writeManifest(["test_error.js"])
1261
1262         self.assertTestResult(False)
1263         self.assertInLog(TEST_FAIL_STRING)
1264         self.assertInLog("test_error.js:3")
1265         self.assertNotInLog(TEST_PASS_STRING)
1266
1267     def testDoReportNonSyntaxError(self):
1268         """
1269         Check that attempting to load a test file containing an error other
1270         than a syntax error generates details of the error in the log
1271         """
1272         self.writeFile("test_error.js", LOAD_ERROR_OTHER_ERROR)
1273         self.writeManifest(["test_error.js"])
1274
1275         self.assertTestResult(False)
1276         self.assertInLog(TEST_FAIL_STRING)
1277         self.assertInLog("ReferenceError: assignment to undeclared variable")
1278         self.assertInLog("test_error.js:3")
1279         self.assertNotInLog(TEST_PASS_STRING)
1280
1281     def testDoPrintWhenVerboseNotExplicit(self):
1282         """
1283         Check that info() and similar calls that generate output do
1284         not have the output when not run verbosely.
1285         """
1286         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1287         self.writeManifest(["test_verbose.js"])
1288
1289         self.assertTestResult(True)
1290         self.assertNotInLog("a message from info")
1291
1292     def testDoPrintWhenVerboseExplicit(self):
1293         """
1294         Check that info() and similar calls that generate output have the
1295         output shown when run verbosely.
1296         """
1297         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1298         self.writeManifest(["test_verbose.js"])
1299         self.assertTestResult(True, verbose=True)
1300         self.assertInLog("a message from info")
1301
1302     def testDoPrintWhenVerboseInManifest(self):
1303         """
1304         Check that info() and similar calls that generate output have the
1305         output shown when 'verbose = true' is in the manifest, even when
1306         not run verbosely.
1307         """
1308         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1309         self.writeManifest([("test_verbose.js", "verbose = true")])
1310
1311         self.assertTestResult(True)
1312         self.assertInLog("a message from info")
1313
1314     def testAsyncCleanup(self):
1315         """
1316         Check that registerCleanupFunction handles nicely async cleanup tasks
1317         """
1318         self.writeFile("test_asyncCleanup.js", ASYNC_CLEANUP)
1319         self.writeManifest(["test_asyncCleanup.js"])
1320         self.assertTestResult(False)
1321         self.assertInLog('"123456" == "123456"')
1322         self.assertInLog("At this stage, the test has succeeded")
1323         self.assertInLog("Throwing an error to force displaying the log")
1324
1325     def testNoRunTestAddTest(self):
1326         """
1327         Check that add_test() works fine without run_test() in the test file.
1328         """
1329         self.writeFile("test_noRunTestAddTest.js", NO_RUN_TEST_ADD_TEST)
1330         self.writeManifest(["test_noRunTestAddTest.js"])
1331
1332         self.assertTestResult(True)
1333         self.assertEqual(1, self.x.testCount)
1334         self.assertEqual(1, self.x.passCount)
1335         self.assertEqual(0, self.x.failCount)
1336         self.assertInLog(TEST_PASS_STRING)
1337         self.assertNotInLog(TEST_FAIL_STRING)
1338
1339     def testNoRunTestAddTask(self):
1340         """
1341         Check that add_task() works fine without run_test() in the test file.
1342         """
1343         self.writeFile("test_noRunTestAddTask.js", NO_RUN_TEST_ADD_TASK)
1344         self.writeManifest(["test_noRunTestAddTask.js"])
1345
1346         self.assertTestResult(True)
1347         self.assertEqual(1, self.x.testCount)
1348         self.assertEqual(1, self.x.passCount)
1349         self.assertEqual(0, self.x.failCount)
1350         self.assertInLog(TEST_PASS_STRING)
1351         self.assertNotInLog(TEST_FAIL_STRING)
1352
1353     def testNoRunTestAddTestAddTask(self):
1354         """
1355         Check that both add_test() and add_task() work without run_test()
1356         in the test file.
1357         """
1358         self.writeFile("test_noRunTestAddTestAddTask.js", NO_RUN_TEST_ADD_TEST_ADD_TASK)
1359         self.writeManifest(["test_noRunTestAddTestAddTask.js"])
1360
1361         self.assertTestResult(True)
1362         self.assertEqual(1, self.x.testCount)
1363         self.assertEqual(1, self.x.passCount)
1364         self.assertEqual(0, self.x.failCount)
1365         self.assertInLog(TEST_PASS_STRING)
1366         self.assertNotInLog(TEST_FAIL_STRING)
1367
1368     def testNoRunTestEmptyTest(self):
1369         """
1370         Check that the test passes on an empty file that contains neither
1371         run_test() nor add_test(), add_task().
1372         """
1373         self.writeFile("test_noRunTestEmptyTest.js", NO_RUN_TEST_EMPTY_TEST)
1374         self.writeManifest(["test_noRunTestEmptyTest.js"])
1375
1376         self.assertTestResult(True)
1377         self.assertEqual(1, self.x.testCount)
1378         self.assertEqual(1, self.x.passCount)
1379         self.assertEqual(0, self.x.failCount)
1380         self.assertInLog(TEST_PASS_STRING)
1381         self.assertNotInLog(TEST_FAIL_STRING)
1382
1383     def testNoRunTestAddTestFail(self):
1384         """
1385         Check that test fails on using add_test() without run_test().
1386         """
1387         self.writeFile("test_noRunTestAddTestFail.js", NO_RUN_TEST_ADD_TEST_FAIL)
1388         self.writeManifest(["test_noRunTestAddTestFail.js"])
1389
1390         self.assertTestResult(False)
1391         self.assertEqual(1, self.x.testCount)
1392         self.assertEqual(0, self.x.passCount)
1393         self.assertEqual(1, self.x.failCount)
1394         self.assertInLog(TEST_FAIL_STRING)
1395         self.assertNotInLog(TEST_PASS_STRING)
1396
1397     def testNoRunTestAddTaskFail(self):
1398         """
1399         Check that test fails on using add_task() without run_test().
1400         """
1401         self.writeFile("test_noRunTestAddTaskFail.js", NO_RUN_TEST_ADD_TASK_FAIL)
1402         self.writeManifest(["test_noRunTestAddTaskFail.js"])
1403
1404         self.assertTestResult(False)
1405         self.assertEqual(1, self.x.testCount)
1406         self.assertEqual(0, self.x.passCount)
1407         self.assertEqual(1, self.x.failCount)
1408         self.assertInLog(TEST_FAIL_STRING)
1409         self.assertNotInLog(TEST_PASS_STRING)
1410
1411     def testNoRunTestAddTaskMultiple(self):
1412         """
1413         Check that multple add_task() tests work without run_test().
1414         """
1415         self.writeFile(
1416             "test_noRunTestAddTaskMultiple.js", NO_RUN_TEST_ADD_TASK_MULTIPLE
1417         )
1418         self.writeManifest(["test_noRunTestAddTaskMultiple.js"])
1419
1420         self.assertTestResult(True)
1421         self.assertEqual(1, self.x.testCount)
1422         self.assertEqual(1, self.x.passCount)
1423         self.assertEqual(0, self.x.failCount)
1424         self.assertInLog(TEST_PASS_STRING)
1425         self.assertNotInLog(TEST_FAIL_STRING)
1426
1427     def testMozinfo(self):
1428         """
1429         Check that mozinfo.json is loaded
1430         """
1431         self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
1432         self.writeManifest(["test_mozinfo.js"])
1433         self.assertTestResult(True)
1434         self.assertEqual(1, self.x.testCount)
1435         self.assertEqual(1, self.x.passCount)
1436         self.assertEqual(0, self.x.failCount)
1437         self.assertEqual(0, self.x.todoCount)
1438         self.assertInLog(TEST_PASS_STRING)
1439         self.assertNotInLog(TEST_FAIL_STRING)
1440
1441     def testChildMozinfo(self):
1442         """
1443         Check that mozinfo.json is loaded in child process
1444         """
1445         self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
1446         self.writeFile("test_child_mozinfo.js", CHILD_MOZINFO)
1447         self.writeManifest(["test_child_mozinfo.js"])
1448         self.assertTestResult(True)
1449         self.assertEqual(1, self.x.testCount)
1450         self.assertEqual(1, self.x.passCount)
1451         self.assertEqual(0, self.x.failCount)
1452         self.assertEqual(0, self.x.todoCount)
1453         self.assertInLog(TEST_PASS_STRING)
1454         self.assertNotInLog(TEST_FAIL_STRING)
1455
1456     def testNotHeadlessByDefault(self):
1457         """
1458         Check that the default is not headless.
1459         """
1460         self.writeFile("test_notHeadlessByDefault.js", HEADLESS_FALSE)
1461         self.writeManifest(["test_notHeadlessByDefault.js"])
1462         self.assertTestResult(True)
1463
1464     def testHeadlessWhenHeadlessExplicit(self):
1465         """
1466         Check that explicitly requesting headless works when the manifest doesn't override.
1467         """
1468         self.writeFile("test_headlessWhenExplicit.js", HEADLESS_TRUE)
1469         self.writeManifest(["test_headlessWhenExplicit.js"])
1470         self.assertTestResult(True, headless=True)
1471
1472     def testHeadlessWhenHeadlessTrueInManifest(self):
1473         """
1474         Check that enabling headless in the manifest alone works.
1475         """
1476         self.writeFile("test_headlessWhenTrueInManifest.js", HEADLESS_TRUE)
1477         self.writeManifest([("test_headlessWhenTrueInManifest.js", "headless = true")])
1478         self.assertTestResult(True)
1479
1480     def testNotHeadlessWhenHeadlessFalseInManifest(self):
1481         """
1482         Check that the manifest entry overrides the explicit default.
1483         """
1484         self.writeFile("test_notHeadlessWhenFalseInManifest.js", HEADLESS_FALSE)
1485         self.writeManifest(
1486             [("test_notHeadlessWhenFalseInManifest.js", "headless = false")]
1487         )
1488         self.assertTestResult(True, headless=True)
1489
1490
1491 if __name__ == "__main__":
1492     import mozunit
1493
1494     mozinfo.find_and_update_from_json()
1495     mozunit.main()