testing/xpcshell/selftest.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6
   7 import os
   8 import pprint
   9 import re
  10 import shutil
  11 import sys
  12 import tempfile
  13 import unittest
  14
  15 import mozinfo
  16 import six
  17 from mozlog import structured
  18 from runxpcshelltests import XPCShellTests
  19
  20 TEST_PASS_STRING = "TEST-PASS"
  21 TEST_FAIL_STRING = "TEST-UNEXPECTED-FAIL"
  22
  23 SIMPLE_PASSING_TEST = "function run_test() { Assert.ok(true); }"
  24 SIMPLE_FAILING_TEST = "function run_test() { Assert.ok(false); }"
  25 SIMPLE_PREFCHECK_TEST = """
  26 function run_test() {
  27   Assert.ok(Services.prefs.getBoolPref("fake.pref.to.test"));
  28 }
  29 """
  30
  31 SIMPLE_UNCAUGHT_REJECTION_TEST = """
  32 function run_test() {
  33   Promise.reject(new Error("Test rejection."));
  34   Assert.ok(true);
  35 }
  36 """
  37
  38 ADD_TEST_SIMPLE = """
  39 function run_test() { run_next_test(); }
  40
  41 add_test(function test_simple() {
  42   Assert.ok(true);
  43   run_next_test();
  44 });
  45 """
  46
  47 ADD_TEST_FAILING = """
  48 function run_test() { run_next_test(); }
  49
  50 add_test(function test_failing() {
  51   Assert.ok(false);
  52   run_next_test();
  53 });
  54 """
  55
  56 ADD_TEST_UNCAUGHT_REJECTION = """
  57 function run_test() { run_next_test(); }
  58
  59 add_test(function test_uncaught_rejection() {
  60   Promise.reject(new Error("Test rejection."));
  61   run_next_test();
  62 });
  63 """
  64
  65 CHILD_TEST_PASSING = """
  66 function run_test () { run_next_test(); }
  67
  68 add_test(function test_child_simple () {
  69   run_test_in_child("test_pass.js");
  70   run_next_test();
  71 });
  72 """
  73
  74 CHILD_TEST_FAILING = """
  75 function run_test () { run_next_test(); }
  76
  77 add_test(function test_child_simple () {
  78   run_test_in_child("test_fail.js");
  79   run_next_test();
  80 });
  81 """
  82
  83 CHILD_HARNESS_SIMPLE = """
  84 function run_test () { run_next_test(); }
  85
  86 add_test(function test_child_assert () {
  87   do_load_child_test_harness();
  88   do_test_pending("test child assertion");
  89   sendCommand("Assert.ok(true);", do_test_finished);
  90   run_next_test();
  91 });
  92 """
  93
  94 CHILD_TEST_HANG = """
  95 function run_test () { run_next_test(); }
  96
  97 add_test(function test_child_simple () {
  98   do_test_pending("hang test");
  99   do_load_child_test_harness();
 100   sendCommand("_testLogger.info('CHILD-TEST-STARTED'); " +
 101               + "const _TEST_FILE=['test_pass.js']; _execute_test(); ",
 102               do_test_finished);
 103   run_next_test();
 104 });
 105 """
 106
 107 SIMPLE_LOOPING_TEST = """
 108 function run_test () { run_next_test(); }
 109
 110 add_test(function test_loop () {
 111   do_test_pending()
 112 });
 113 """
 114
 115 PASSING_TEST_UNICODE = b"""
 116 function run_test () { run_next_test(); }
 117
 118 add_test(function test_unicode_print () {
 119   Assert.equal("\u201c\u201d", "\u201c\u201d");
 120   run_next_test();
 121 });
 122 """
 123
 124 ADD_TASK_SINGLE = """
 125 function run_test() { run_next_test(); }
 126
 127 add_task(async function test_task() {
 128   await Promise.resolve(true);
 129   await Promise.resolve(false);
 130 });
 131 """
 132
 133 ADD_TASK_MULTIPLE = """
 134 function run_test() { run_next_test(); }
 135
 136 add_task(async function test_task() {
 137   await Promise.resolve(true);
 138 });
 139
 140 add_task(async function test_2() {
 141   await Promise.resolve(true);
 142 });
 143 """
 144
 145 ADD_TASK_REJECTED = """
 146 function run_test() { run_next_test(); }
 147
 148 add_task(async function test_failing() {
 149   await Promise.reject(new Error("I fail."));
 150 });
 151 """
 152
 153 ADD_TASK_REJECTED_UNDEFINED = """
 154 function run_test() { run_next_test(); }
 155
 156 add_task(async function test_failing() {
 157   await Promise.reject();
 158 });
 159 """
 160
 161 ADD_TASK_FAILURE_INSIDE = """
 162 function run_test() { run_next_test(); }
 163
 164 add_task(async function test() {
 165   let result = await Promise.resolve(false);
 166
 167   Assert.ok(result);
 168 });
 169 """
 170
 171 ADD_TASK_RUN_NEXT_TEST = """
 172 function run_test() { run_next_test(); }
 173
 174 add_task(function () {
 175   Assert.ok(true);
 176
 177   run_next_test();
 178 });
 179 """
 180
 181 ADD_TASK_STACK_TRACE = """
 182 function run_test() { run_next_test(); }
 183
 184 add_task(async function this_test_will_fail() {
 185   for (let i = 0; i < 10; ++i) {
 186     await Promise.resolve();
 187   }
 188   Assert.ok(false);
 189 });
 190 """
 191
 192 ADD_TASK_SKIP = """
 193 add_task(async function skipMeNot1() {
 194   Assert.ok(true, "Well well well.");
 195 });
 196
 197 add_task(async function skipMe1() {
 198   Assert.ok(false, "Not skipped after all.");
 199 }).skip();
 200
 201 add_task(async function skipMeNot2() {
 202   Assert.ok(true, "Well well well.");
 203 });
 204
 205 add_task(async function skipMeNot3() {
 206   Assert.ok(true, "Well well well.");
 207 });
 208
 209 add_task(async function skipMe2() {
 210   Assert.ok(false, "Not skipped after all.");
 211 }).skip();
 212 """
 213
 214 ADD_TASK_SKIPALL = """
 215 add_task(async function skipMe1() {
 216   Assert.ok(false, "Not skipped after all.");
 217 });
 218
 219 add_task(async function skipMe2() {
 220   Assert.ok(false, "Not skipped after all.");
 221 }).skip();
 222
 223 add_task(async function skipMe3() {
 224   Assert.ok(false, "Not skipped after all.");
 225 }).only();
 226
 227 add_task(async function skipMeNot() {
 228   Assert.ok(true, "Well well well.");
 229 }).only();
 230
 231 add_task(async function skipMe4() {
 232   Assert.ok(false, "Not skipped after all.");
 233 });
 234 """
 235
 236 ADD_TEST_THROW_STRING = """
 237 function run_test() {do_throw("Passing a string to do_throw")};
 238 """
 239
 240 ADD_TEST_THROW_OBJECT = """
 241 let error = {
 242   message: "Error object",
 243   fileName: "failure.js",
 244   stack: "ERROR STACK",
 245   toString: function() {return this.message;}
 246 };
 247 function run_test() {do_throw(error)};
 248 """
 249
 250 ADD_TEST_REPORT_OBJECT = """
 251 let error = {
 252   message: "Error object",
 253   fileName: "failure.js",
 254   stack: "ERROR STACK",
 255   toString: function() {return this.message;}
 256 };
 257 function run_test() {do_report_unexpected_exception(error)};
 258 """
 259
 260 ADD_TEST_VERBOSE = """
 261 function run_test() {info("a message from info")};
 262 """
 263
 264 # A test for genuine JS-generated Error objects
 265 ADD_TEST_REPORT_REF_ERROR = """
 266 function run_test() {
 267   let obj = {blah: 0};
 268   try {
 269     obj.noSuchFunction();
 270   }
 271   catch (error) {
 272     do_report_unexpected_exception(error);
 273   }
 274 };
 275 """
 276
 277 # A test for failure to load a test due to a syntax error
 278 LOAD_ERROR_SYNTAX_ERROR = """
 279 function run_test(
 280 """
 281
 282 # A test for failure to load a test due to an error other than a syntax error
 283 LOAD_ERROR_OTHER_ERROR = """
 284 "use strict";
 285 no_such_var = "foo"; // assignment to undeclared variable
 286 """
 287
 288 # A test that crashes outright.
 289 TEST_CRASHING = """
 290 function run_test () {
 291   const { ctypes } = ChromeUtils.import("resource://gre/modules/ctypes.jsm");
 292   let zero = new ctypes.intptr_t(8);
 293   let badptr = ctypes.cast(zero, ctypes.PointerType(ctypes.int32_t));
 294   badptr.contents;
 295 }
 296 """
 297
 298 # A test for asynchronous cleanup functions
 299 ASYNC_CLEANUP = """
 300 function run_test() {
 301   // The list of checkpoints in the order we encounter them.
 302   let checkpoints = [];
 303
 304   // Cleanup tasks, in reverse order
 305   registerCleanupFunction(function cleanup_checkout() {
 306     Assert.equal(checkpoints.join(""), "123456");
 307     info("At this stage, the test has succeeded");
 308     do_throw("Throwing an error to force displaying the log");
 309   });
 310
 311   registerCleanupFunction(function sync_cleanup_2() {
 312     checkpoints.push(6);
 313   });
 314
 315   registerCleanupFunction(async function async_cleanup_4() {
 316     await undefined;
 317     checkpoints.push(5);
 318   });
 319
 320   registerCleanupFunction(async function async_cleanup_3() {
 321     await undefined;
 322     checkpoints.push(4);
 323   });
 324
 325   registerCleanupFunction(function async_cleanup_2() {
 326     let deferred = Promise.withResolvers();
 327     executeSoon(deferred.resolve);
 328     return deferred.promise.then(function() {
 329       checkpoints.push(3);
 330     });
 331   });
 332
 333   registerCleanupFunction(function sync_cleanup() {
 334     checkpoints.push(2);
 335   });
 336
 337   registerCleanupFunction(function async_cleanup() {
 338     let deferred = Promise.withResolvers();
 339     executeSoon(deferred.resolve);
 340     return deferred.promise.then(function() {
 341       checkpoints.push(1);
 342     });
 343   });
 344
 345 }
 346 """
 347
 348 # A test to check that add_test() tests run without run_test()
 349 NO_RUN_TEST_ADD_TEST = """
 350 add_test(function no_run_test_add_test() {
 351   Assert.ok(true);
 352   run_next_test();
 353 });
 354 """
 355
 356 # A test to check that add_task() tests run without run_test()
 357 NO_RUN_TEST_ADD_TASK = """
 358 add_task(function no_run_test_add_task() {
 359   Assert.ok(true);
 360 });
 361 """
 362
 363 # A test to check that both add_task() and add_test() work without run_test()
 364 NO_RUN_TEST_ADD_TEST_ADD_TASK = """
 365 add_test(function no_run_test_add_test() {
 366   Assert.ok(true);
 367   run_next_test();
 368 });
 369
 370 add_task(function no_run_test_add_task() {
 371   Assert.ok(true);
 372 });
 373 """
 374
 375 # A test to check that an empty test file without run_test(),
 376 # add_test() or add_task() works.
 377 NO_RUN_TEST_EMPTY_TEST = """
 378 // This is an empty test file.
 379 """
 380
 381 NO_RUN_TEST_ADD_TEST_FAIL = """
 382 add_test(function no_run_test_add_test_fail() {
 383   Assert.ok(false);
 384   run_next_test();
 385 });
 386 """
 387
 388 NO_RUN_TEST_ADD_TASK_FAIL = """
 389 add_task(function no_run_test_add_task_fail() {
 390   Assert.ok(false);
 391 });
 392 """
 393
 394 NO_RUN_TEST_ADD_TASK_MULTIPLE = """
 395 add_task(async function test_task() {
 396   await Promise.resolve(true);
 397 });
 398
 399 add_task(async function test_2() {
 400   await Promise.resolve(true);
 401 });
 402 """
 403
 404 LOAD_MOZINFO = """
 405 function run_test() {
 406   Assert.notEqual(typeof mozinfo, undefined);
 407   Assert.notEqual(typeof mozinfo.os, undefined);
 408 }
 409 """
 410
 411 CHILD_MOZINFO = """
 412 function run_test () { run_next_test(); }
 413
 414 add_test(function test_child_mozinfo () {
 415   run_test_in_child("test_mozinfo.js");
 416   run_next_test();
 417 });
 418 """
 419
 420 HEADLESS_TRUE = """
 421 add_task(function headless_true() {
 422   Assert.equal(Services.env.get("MOZ_HEADLESS"), "1", "Check MOZ_HEADLESS");
 423   Assert.equal(Services.env.get("DISPLAY"), "77", "Check DISPLAY");
 424 });
 425 """
 426
 427 HEADLESS_FALSE = """
 428 add_task(function headless_false() {
 429   Assert.notEqual(Services.env.get("MOZ_HEADLESS"), "1", "Check MOZ_HEADLESS");
 430   Assert.notEqual(Services.env.get("DISPLAY"), "77", "Check DISPLAY");
 431 });
 432 """
 433
 434
 435 class XPCShellTestsTests(unittest.TestCase):
 436     """
 437     Yes, these are unit tests for a unit test harness.
 438     """
 439
 440     def __init__(self, name):
 441         super(XPCShellTestsTests, self).__init__(name)
 442         from buildconfig import substs
 443         from mozbuild.base import MozbuildObject
 444
 445         os.environ.pop("MOZ_OBJDIR", None)
 446         self.build_obj = MozbuildObject.from_environment()
 447
 448         objdir = self.build_obj.topobjdir
 449         self.testing_modules = os.path.join(objdir, "_tests", "modules")
 450
 451         if mozinfo.isMac:
 452             self.xpcshellBin = os.path.join(
 453                 objdir,
 454                 "dist",
 455                 substs["MOZ_MACBUNDLE_NAME"],
 456                 "Contents",
 457                 "MacOS",
 458                 "xpcshell",
 459             )
 460         else:
 461             self.xpcshellBin = os.path.join(objdir, "dist", "bin", "xpcshell")
 462
 463         if sys.platform == "win32":
 464             self.xpcshellBin += ".exe"
 465         self.utility_path = os.path.join(objdir, "dist", "bin")
 466         self.symbols_path = None
 467         candidate_path = os.path.join(self.build_obj.distdir, "crashreporter-symbols")
 468         if os.path.isdir(candidate_path):
 469             self.symbols_path = candidate_path
 470
 471     def setUp(self):
 472         self.log = six.StringIO()
 473         self.tempdir = tempfile.mkdtemp()
 474         logger = structured.commandline.setup_logging(
 475             "selftest%s" % id(self), {}, {"tbpl": self.log}
 476         )
 477         self.x = XPCShellTests(logger)
 478         self.x.harness_timeout = 30 if not mozinfo.info["ccov"] else 60
 479
 480     def tearDown(self):
 481         shutil.rmtree(self.tempdir)
 482         self.x.shutdownNode()
 483
 484     def writeFile(self, name, contents, mode="w"):
 485         """
 486         Write |contents| to a file named |name| in the temp directory,
 487         and return the full path to the file.
 488         """
 489         fullpath = os.path.join(self.tempdir, name)
 490         with open(fullpath, mode) as f:
 491             f.write(contents)
 492         return fullpath
 493
 494     def writeManifest(self, tests, prefs=[]):
 495         """
 496         Write an xpcshell.ini in the temp directory and set
 497         self.manifest to its pathname. |tests| is a list containing
 498         either strings (for test names), or tuples with a test name
 499         as the first element and manifest conditions as the following
 500         elements. |prefs| is an optional list of prefs in the form of
 501         "prefname=prefvalue" strings.
 502         """
 503         testlines = []
 504         for t in tests:
 505             testlines.append(
 506                 '["%s"]' % (t if isinstance(t, six.string_types) else t[0])
 507             )
 508             if isinstance(t, tuple):
 509                 testlines.extend(t[1:])
 510         prefslines = []
 511         for p in prefs:
 512             # Append prefs lines as indented inside "prefs=" manifest option.
 513             prefslines.append('  "%s",' % p)
 514
 515         val = """
 516 [DEFAULT]
 517 head = ""
 518 tail = ""
 519 prefs = [
 520 """
 521         val += "\n".join(prefslines)
 522         val += "]\n"
 523         val += "\n".join(testlines)
 524         self.manifest = self.writeFile("xpcshell.toml", val)
 525
 526     def assertTestResult(self, expected, shuffle=False, verbose=False, headless=False):
 527         """
 528         Assert that self.x.runTests with manifest=self.manifest
 529         returns |expected|.
 530         """
 531         kwargs = {}
 532         kwargs["app_binary"] = self.app_binary
 533         kwargs["xpcshell"] = self.xpcshellBin
 534         kwargs["symbolsPath"] = self.symbols_path
 535         kwargs["manifest"] = self.manifest
 536         kwargs["mozInfo"] = mozinfo.info
 537         kwargs["shuffle"] = shuffle
 538         kwargs["verbose"] = verbose
 539         kwargs["headless"] = headless
 540         kwargs["sequential"] = True
 541         kwargs["testingModulesDir"] = self.testing_modules
 542         kwargs["utility_path"] = self.utility_path
 543         kwargs["repeat"] = 0
 544         self.assertEqual(
 545             expected,
 546             self.x.runTests(kwargs),
 547             msg="""Tests should have %s, log:
 548 ========
 549 %s
 550 ========
 551 """
 552             % ("passed" if expected else "failed", self.log.getvalue()),
 553         )
 554
 555     def _assertLog(self, s, expected):
 556         l = self.log.getvalue()
 557         self.assertEqual(
 558             expected,
 559             s in l,
 560             msg="""Value %s %s in log:
 561 ========
 562 %s
 563 ========"""
 564             % (s, "expected" if expected else "not expected", l),
 565         )
 566
 567     def assertInLog(self, s):
 568         """
 569         Assert that the string |s| is contained in self.log.
 570         """
 571         self._assertLog(s, True)
 572
 573     def assertNotInLog(self, s):
 574         """
 575         Assert that the string |s| is not contained in self.log.
 576         """
 577         self._assertLog(s, False)
 578
 579     def testPass(self):
 580         """
 581         Check that a simple test without any manifest conditions passes.
 582         """
 583         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 584         self.writeManifest(["test_basic.js"])
 585
 586         self.assertTestResult(True)
 587         self.assertEqual(1, self.x.testCount)
 588         self.assertEqual(1, self.x.passCount)
 589         self.assertEqual(0, self.x.failCount)
 590         self.assertEqual(0, self.x.todoCount)
 591         self.assertInLog(TEST_PASS_STRING)
 592         self.assertNotInLog(TEST_FAIL_STRING)
 593
 594     def testFail(self):
 595         """
 596         Check that a simple failing test without any manifest conditions fails.
 597         """
 598         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 599         self.writeManifest(["test_basic.js"])
 600
 601         self.assertTestResult(False)
 602         self.assertEqual(1, self.x.testCount)
 603         self.assertEqual(0, self.x.passCount)
 604         self.assertEqual(1, self.x.failCount)
 605         self.assertEqual(0, self.x.todoCount)
 606         self.assertInLog(TEST_FAIL_STRING)
 607         self.assertNotInLog(TEST_PASS_STRING)
 608
 609     def testPrefsInManifestVerbose(self):
 610         """
 611         Check prefs configuration option is supported in xpcshell manifests.
 612         """
 613         self.writeFile("test_prefs.js", SIMPLE_PREFCHECK_TEST)
 614         self.writeManifest(tests=["test_prefs.js"], prefs=["fake.pref.to.test=true"])
 615
 616         self.assertTestResult(True, verbose=True)
 617         self.assertInLog(TEST_PASS_STRING)
 618         self.assertNotInLog(TEST_FAIL_STRING)
 619         self.assertEqual(1, self.x.testCount)
 620         self.assertEqual(1, self.x.passCount)
 621         self.assertInLog("Per-test extra prefs will be set:")
 622         self.assertInLog("fake.pref.to.test=true")
 623
 624     def testPrefsInManifestNonVerbose(self):
 625         """
 626         Check prefs configuration are not logged in non verbose mode.
 627         """
 628         self.writeFile("test_prefs.js", SIMPLE_PREFCHECK_TEST)
 629         self.writeManifest(tests=["test_prefs.js"], prefs=["fake.pref.to.test=true"])
 630
 631         self.assertTestResult(True, verbose=False)
 632         self.assertNotInLog("Per-test extra prefs will be set:")
 633         self.assertNotInLog("fake.pref.to.test=true")
 634
 635     @unittest.skipIf(
 636         mozinfo.isWin or not mozinfo.info.get("debug"),
 637         "We don't have a stack fixer on hand for windows.",
 638     )
 639     def testAssertStack(self):
 640         """
 641         When an assertion is hit, we should produce a useful stack.
 642         """
 643         self.writeFile(
 644             "test_assert.js",
 645             """
 646           add_test(function test_asserts_immediately() {
 647             Components.classes["@mozilla.org/xpcom/debug;1"]
 648                       .getService(Components.interfaces.nsIDebug2)
 649                       .assertion("foo", "assertion failed", "test.js", 1)
 650             run_next_test();
 651           });
 652         """,
 653         )
 654
 655         self.writeManifest(["test_assert.js"])
 656         self.assertTestResult(False)
 657
 658         self.assertInLog("###!!! ASSERTION")
 659         log_lines = self.log.getvalue().splitlines()
 660         line_pat = "#\d\d:"
 661         unknown_pat = "#\d\d\: \?\?\?\[.* \+0x[a-f0-9]+\]"
 662         self.assertFalse(
 663             any(re.search(unknown_pat, line) for line in log_lines),
 664             "An stack frame without symbols was found in\n%s"
 665             % pprint.pformat(log_lines),
 666         )
 667         self.assertTrue(
 668             any(re.search(line_pat, line) for line in log_lines),
 669             "No line resembling a stack frame was found in\n%s"
 670             % pprint.pformat(log_lines),
 671         )
 672
 673     def testChildPass(self):
 674         """
 675         Check that a simple test running in a child process passes.
 676         """
 677         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 678         self.writeFile("test_child_pass.js", CHILD_TEST_PASSING)
 679         self.writeManifest(["test_child_pass.js"])
 680
 681         self.assertTestResult(True, verbose=True)
 682         self.assertEqual(1, self.x.testCount)
 683         self.assertEqual(1, self.x.passCount)
 684         self.assertEqual(0, self.x.failCount)
 685         self.assertEqual(0, self.x.todoCount)
 686         self.assertInLog(TEST_PASS_STRING)
 687         self.assertInLog("CHILD-TEST-STARTED")
 688         self.assertInLog("CHILD-TEST-COMPLETED")
 689         self.assertNotInLog(TEST_FAIL_STRING)
 690
 691     def testChildFail(self):
 692         """
 693         Check that a simple failing test running in a child process fails.
 694         """
 695         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
 696         self.writeFile("test_child_fail.js", CHILD_TEST_FAILING)
 697         self.writeManifest(["test_child_fail.js"])
 698
 699         self.assertTestResult(False)
 700         self.assertEqual(1, self.x.testCount)
 701         self.assertEqual(0, self.x.passCount)
 702         self.assertEqual(1, self.x.failCount)
 703         self.assertEqual(0, self.x.todoCount)
 704         self.assertInLog(TEST_FAIL_STRING)
 705         self.assertInLog("CHILD-TEST-STARTED")
 706         self.assertInLog("CHILD-TEST-COMPLETED")
 707         self.assertNotInLog(TEST_PASS_STRING)
 708
 709     def testChildHang(self):
 710         """
 711         Check that incomplete output from a child process results in a
 712         test failure.
 713         """
 714         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 715         self.writeFile("test_child_hang.js", CHILD_TEST_HANG)
 716         self.writeManifest(["test_child_hang.js"])
 717
 718         self.assertTestResult(False)
 719         self.assertEqual(1, self.x.testCount)
 720         self.assertEqual(0, self.x.passCount)
 721         self.assertEqual(1, self.x.failCount)
 722         self.assertEqual(0, self.x.todoCount)
 723         self.assertInLog(TEST_FAIL_STRING)
 724         self.assertInLog("CHILD-TEST-STARTED")
 725         self.assertNotInLog("CHILD-TEST-COMPLETED")
 726         self.assertNotInLog(TEST_PASS_STRING)
 727
 728     def testChild(self):
 729         """
 730         Checks that calling do_load_child_test_harness without run_test_in_child
 731         results in a usable test state. This test has a spurious failure when
 732         run using |mach python-test|. See bug 1103226.
 733         """
 734         self.writeFile("test_child_assertions.js", CHILD_HARNESS_SIMPLE)
 735         self.writeManifest(["test_child_assertions.js"])
 736
 737         self.assertTestResult(True)
 738         self.assertEqual(1, self.x.testCount)
 739         self.assertEqual(1, self.x.passCount)
 740         self.assertEqual(0, self.x.failCount)
 741         self.assertEqual(0, self.x.todoCount)
 742         self.assertInLog(TEST_PASS_STRING)
 743         self.assertNotInLog(TEST_FAIL_STRING)
 744
 745     def testSkipForAddTest(self):
 746         """
 747         Check that add_test is skipped if |skip_if| condition is true
 748         """
 749         self.writeFile(
 750             "test_skip.js",
 751             """
 752 add_test({
 753   skip_if: () => true,
 754 }, function test_should_be_skipped() {
 755   Assert.ok(false);
 756   run_next_test();
 757 });
 758 """,
 759         )
 760         self.writeManifest(["test_skip.js"])
 761         self.assertTestResult(True, verbose=True)
 762         self.assertEqual(1, self.x.testCount)
 763         self.assertEqual(1, self.x.passCount)
 764         self.assertEqual(0, self.x.failCount)
 765         self.assertEqual(0, self.x.todoCount)
 766         self.assertInLog(TEST_PASS_STRING)
 767         self.assertInLog("TEST-SKIP")
 768         self.assertNotInLog(TEST_FAIL_STRING)
 769
 770     def testNotSkipForAddTask(self):
 771         """
 772         Check that add_task is not skipped if |skip_if| condition is false
 773         """
 774         self.writeFile(
 775             "test_not_skip.js",
 776             """
 777 add_task({
 778   skip_if: () => false,
 779 }, function test_should_not_be_skipped() {
 780   Assert.ok(true);
 781 });
 782 """,
 783         )
 784         self.writeManifest(["test_not_skip.js"])
 785         self.assertTestResult(True, verbose=True)
 786         self.assertEqual(1, self.x.testCount)
 787         self.assertEqual(1, self.x.passCount)
 788         self.assertEqual(0, self.x.failCount)
 789         self.assertEqual(0, self.x.todoCount)
 790         self.assertInLog(TEST_PASS_STRING)
 791         self.assertNotInLog("TEST-SKIP")
 792         self.assertNotInLog(TEST_FAIL_STRING)
 793
 794     def testSkipForAddTask(self):
 795         """
 796         Check that add_task is skipped if |skip_if| condition is true
 797         """
 798         self.writeFile(
 799             "test_skip.js",
 800             """
 801 add_task({
 802   skip_if: () => true,
 803 }, function test_should_be_skipped() {
 804   Assert.ok(false);
 805 });
 806 """,
 807         )
 808         self.writeManifest(["test_skip.js"])
 809         self.assertTestResult(True, verbose=True)
 810         self.assertEqual(1, self.x.testCount)
 811         self.assertEqual(1, self.x.passCount)
 812         self.assertEqual(0, self.x.failCount)
 813         self.assertEqual(0, self.x.todoCount)
 814         self.assertInLog(TEST_PASS_STRING)
 815         self.assertInLog("TEST-SKIP")
 816         self.assertNotInLog(TEST_FAIL_STRING)
 817
 818     def testNotSkipForAddTest(self):
 819         """
 820         Check that add_test is not skipped if |skip_if| condition is false
 821         """
 822         self.writeFile(
 823             "test_not_skip.js",
 824             """
 825 add_test({
 826   skip_if: () => false,
 827 }, function test_should_not_be_skipped() {
 828   Assert.ok(true);
 829   run_next_test();
 830 });
 831 """,
 832         )
 833         self.writeManifest(["test_not_skip.js"])
 834         self.assertTestResult(True, verbose=True)
 835         self.assertEqual(1, self.x.testCount)
 836         self.assertEqual(1, self.x.passCount)
 837         self.assertEqual(0, self.x.failCount)
 838         self.assertEqual(0, self.x.todoCount)
 839         self.assertInLog(TEST_PASS_STRING)
 840         self.assertNotInLog("TEST-SKIP")
 841         self.assertNotInLog(TEST_FAIL_STRING)
 842
 843     def testSyntaxError(self):
 844         """
 845         Check that running a test file containing a syntax error produces
 846         a test failure and expected output.
 847         """
 848         self.writeFile("test_syntax_error.js", '"')
 849         self.writeManifest(["test_syntax_error.js"])
 850
 851         self.assertTestResult(False, verbose=True)
 852         self.assertEqual(1, self.x.testCount)
 853         self.assertEqual(0, self.x.passCount)
 854         self.assertEqual(1, self.x.failCount)
 855         self.assertEqual(0, self.x.todoCount)
 856         self.assertInLog(TEST_FAIL_STRING)
 857         self.assertNotInLog(TEST_PASS_STRING)
 858
 859     def testUnicodeInAssertMethods(self):
 860         """
 861         Check that passing unicode characters through an assertion method works.
 862         """
 863         self.writeFile("test_unicode_assert.js", PASSING_TEST_UNICODE, mode="wb")
 864         self.writeManifest(["test_unicode_assert.js"])
 865
 866         self.assertTestResult(True, verbose=True)
 867
 868     @unittest.skipIf(
 869         "MOZ_AUTOMATION" in os.environ,
 870         "Timeout code path occasionally times out (bug 1098121)",
 871     )
 872     def testHangingTimeout(self):
 873         """
 874         Check that a test that never finishes results in the correct error log.
 875         """
 876         self.writeFile("test_loop.js", SIMPLE_LOOPING_TEST)
 877         self.writeManifest(["test_loop.js"])
 878
 879         old_timeout = self.x.harness_timeout
 880         self.x.harness_timeout = 1
 881
 882         self.assertTestResult(False)
 883         self.assertEqual(1, self.x.testCount)
 884         self.assertEqual(1, self.x.failCount)
 885         self.assertEqual(0, self.x.passCount)
 886         self.assertEqual(0, self.x.todoCount)
 887         self.assertInLog("TEST-UNEXPECTED-TIMEOUT")
 888
 889         self.x.harness_timeout = old_timeout
 890
 891     def testPassFail(self):
 892         """
 893         Check that running more than one test works.
 894         """
 895         self.writeFile("test_pass.js", SIMPLE_PASSING_TEST)
 896         self.writeFile("test_fail.js", SIMPLE_FAILING_TEST)
 897         self.writeManifest(["test_pass.js", "test_fail.js"])
 898
 899         self.assertTestResult(False)
 900         self.assertEqual(2, self.x.testCount)
 901         self.assertEqual(1, self.x.passCount)
 902         self.assertEqual(1, self.x.failCount)
 903         self.assertEqual(0, self.x.todoCount)
 904         self.assertInLog(TEST_PASS_STRING)
 905         self.assertInLog(TEST_FAIL_STRING)
 906
 907     def testSkip(self):
 908         """
 909         Check that a simple failing test skipped in the manifest does
 910         not cause failure.
 911         """
 912         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 913         self.writeManifest([("test_basic.js", "skip-if = true")])
 914         self.assertTestResult(True)
 915         self.assertEqual(1, self.x.testCount)
 916         self.assertEqual(0, self.x.passCount)
 917         self.assertEqual(0, self.x.failCount)
 918         self.assertEqual(0, self.x.todoCount)
 919         self.assertNotInLog(TEST_FAIL_STRING)
 920         self.assertNotInLog(TEST_PASS_STRING)
 921
 922     def testKnownFail(self):
 923         """
 924         Check that a simple failing test marked as known-fail in the manifest
 925         does not cause failure.
 926         """
 927         self.writeFile("test_basic.js", SIMPLE_FAILING_TEST)
 928         self.writeManifest([("test_basic.js", "fail-if = true")])
 929         self.assertTestResult(True)
 930         self.assertEqual(1, self.x.testCount)
 931         self.assertEqual(0, self.x.passCount)
 932         self.assertEqual(0, self.x.failCount)
 933         self.assertEqual(1, self.x.todoCount)
 934         self.assertInLog("TEST-FAIL")
 935         # This should be suppressed because the harness doesn't include
 936         # the full log from the xpcshell run when things pass.
 937         self.assertNotInLog(TEST_FAIL_STRING)
 938         self.assertNotInLog(TEST_PASS_STRING)
 939
 940     def testUnexpectedPass(self):
 941         """
 942         Check that a simple failing test marked as known-fail in the manifest
 943         that passes causes an unexpected pass.
 944         """
 945         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
 946         self.writeManifest([("test_basic.js", "fail-if = true")])
 947         self.assertTestResult(False)
 948         self.assertEqual(1, self.x.testCount)
 949         self.assertEqual(0, self.x.passCount)
 950         self.assertEqual(1, self.x.failCount)
 951         self.assertEqual(0, self.x.todoCount)
 952         # From the outer (Python) harness
 953         self.assertInLog("TEST-UNEXPECTED-PASS")
 954         self.assertNotInLog("TEST-KNOWN-FAIL")
 955
 956     def testReturnNonzero(self):
 957         """
 958         Check that a test where xpcshell returns nonzero fails.
 959         """
 960         self.writeFile("test_error.js", "throw 'foo'")
 961         self.writeManifest(["test_error.js"])
 962
 963         self.assertTestResult(False)
 964         self.assertEqual(1, self.x.testCount)
 965         self.assertEqual(0, self.x.passCount)
 966         self.assertEqual(1, self.x.failCount)
 967         self.assertEqual(0, self.x.todoCount)
 968         self.assertInLog(TEST_FAIL_STRING)
 969         self.assertNotInLog(TEST_PASS_STRING)
 970
 971     def testUncaughtRejection(self):
 972         """
 973         Ensure a simple test with an uncaught rejection is reported.
 974         """
 975         self.writeFile(
 976             "test_simple_uncaught_rejection.js", SIMPLE_UNCAUGHT_REJECTION_TEST
 977         )
 978         self.writeManifest(["test_simple_uncaught_rejection.js"])
 979
 980         self.assertTestResult(False)
 981         self.assertInLog(TEST_FAIL_STRING)
 982         self.assertInLog("test_simple_uncaught_rejection.js:3:18")
 983         self.assertInLog("Test rejection.")
 984         self.assertEqual(1, self.x.testCount)
 985         self.assertEqual(0, self.x.passCount)
 986         self.assertEqual(1, self.x.failCount)
 987
 988     def testAddTestSimple(self):
 989         """
 990         Ensure simple add_test() works.
 991         """
 992         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
 993         self.writeManifest(["test_add_test_simple.js"])
 994
 995         self.assertTestResult(True)
 996         self.assertEqual(1, self.x.testCount)
 997         self.assertEqual(1, self.x.passCount)
 998         self.assertEqual(0, self.x.failCount)
 999
1000     def testCrashLogging(self):
1001         """
1002         Test that a crashing test process logs a failure.
1003         """
1004         self.writeFile("test_crashes.js", TEST_CRASHING)
1005         self.writeManifest(["test_crashes.js"])
1006
1007         self.assertTestResult(False)
1008         self.assertEqual(1, self.x.testCount)
1009         self.assertEqual(0, self.x.passCount)
1010         self.assertEqual(1, self.x.failCount)
1011         if mozinfo.info.get("crashreporter"):
1012             self.assertInLog("\nPROCESS-CRASH")
1013
1014     def testLogCorrectFileName(self):
1015         """
1016         Make sure a meaningful filename and line number is logged
1017         by a passing test.
1018         """
1019         self.writeFile("test_add_test_simple.js", ADD_TEST_SIMPLE)
1020         self.writeManifest(["test_add_test_simple.js"])
1021
1022         self.assertTestResult(True, verbose=True)
1023         self.assertInLog("true == true")
1024         self.assertNotInLog("[Assert.ok :")
1025         self.assertInLog("[test_simple : 5]")
1026
1027     def testAddTestFailing(self):
1028         """
1029         Ensure add_test() with a failing test is reported.
1030         """
1031         self.writeFile("test_add_test_failing.js", ADD_TEST_FAILING)
1032         self.writeManifest(["test_add_test_failing.js"])
1033
1034         self.assertTestResult(False)
1035         self.assertEqual(1, self.x.testCount)
1036         self.assertEqual(0, self.x.passCount)
1037         self.assertEqual(1, self.x.failCount)
1038
1039     def testAddTestUncaughtRejection(self):
1040         """
1041         Ensure add_test() with an uncaught rejection is reported.
1042         """
1043         self.writeFile(
1044             "test_add_test_uncaught_rejection.js", ADD_TEST_UNCAUGHT_REJECTION
1045         )
1046         self.writeManifest(["test_add_test_uncaught_rejection.js"])
1047
1048         self.assertTestResult(False)
1049         self.assertEqual(1, self.x.testCount)
1050         self.assertEqual(0, self.x.passCount)
1051         self.assertEqual(1, self.x.failCount)
1052
1053     def testAddTaskTestSingle(self):
1054         """
1055         Ensure add_test_task() with a single passing test works.
1056         """
1057         self.writeFile("test_add_task_simple.js", ADD_TASK_SINGLE)
1058         self.writeManifest(["test_add_task_simple.js"])
1059
1060         self.assertTestResult(True)
1061         self.assertEqual(1, self.x.testCount)
1062         self.assertEqual(1, self.x.passCount)
1063         self.assertEqual(0, self.x.failCount)
1064
1065     def testAddTaskTestMultiple(self):
1066         """
1067         Ensure multiple calls to add_test_task() work as expected.
1068         """
1069         self.writeFile("test_add_task_multiple.js", ADD_TASK_MULTIPLE)
1070         self.writeManifest(["test_add_task_multiple.js"])
1071
1072         self.assertTestResult(True)
1073         self.assertEqual(1, self.x.testCount)
1074         self.assertEqual(1, self.x.passCount)
1075         self.assertEqual(0, self.x.failCount)
1076
1077     def testAddTaskTestRejected(self):
1078         """
1079         Ensure rejected task reports as failure.
1080         """
1081         self.writeFile("test_add_task_rejected.js", ADD_TASK_REJECTED)
1082         self.writeManifest(["test_add_task_rejected.js"])
1083
1084         self.assertTestResult(False)
1085         self.assertEqual(1, self.x.testCount)
1086         self.assertEqual(0, self.x.passCount)
1087         self.assertEqual(1, self.x.failCount)
1088
1089     def testAddTaskTestRejectedUndefined(self):
1090         """
1091         Ensure rejected task with undefined reason reports as failure and does not hang.
1092         """
1093         self.writeFile(
1094             "test_add_task_rejected_undefined.js", ADD_TASK_REJECTED_UNDEFINED
1095         )
1096         self.writeManifest(["test_add_task_rejected_undefined.js"])
1097
1098         self.assertTestResult(False)
1099         self.assertEqual(1, self.x.testCount)
1100         self.assertEqual(0, self.x.passCount)
1101         self.assertEqual(1, self.x.failCount)
1102         self.assertNotInLog("TEST-UNEXPECTED-TIMEOUT")
1103
1104     def testAddTaskTestFailureInside(self):
1105         """
1106         Ensure tests inside task are reported as failures.
1107         """
1108         self.writeFile("test_add_task_failure_inside.js", ADD_TASK_FAILURE_INSIDE)
1109         self.writeManifest(["test_add_task_failure_inside.js"])
1110
1111         self.assertTestResult(False)
1112         self.assertEqual(1, self.x.testCount)
1113         self.assertEqual(0, self.x.passCount)
1114         self.assertEqual(1, self.x.failCount)
1115
1116     def testAddTaskRunNextTest(self):
1117         """
1118         Calling run_next_test() from inside add_task() results in failure.
1119         """
1120         self.writeFile("test_add_task_run_next_test.js", ADD_TASK_RUN_NEXT_TEST)
1121         self.writeManifest(["test_add_task_run_next_test.js"])
1122
1123         self.assertTestResult(False)
1124         self.assertEqual(1, self.x.testCount)
1125         self.assertEqual(0, self.x.passCount)
1126         self.assertEqual(1, self.x.failCount)
1127
1128     def testAddTaskStackTrace(self):
1129         """
1130         Ensuring that calling Assert.ok(false) from inside add_task()
1131         results in a human-readable stack trace.
1132         """
1133         self.writeFile("test_add_task_stack_trace.js", ADD_TASK_STACK_TRACE)
1134         self.writeManifest(["test_add_task_stack_trace.js"])
1135
1136         self.assertTestResult(False)
1137         self.assertInLog("this_test_will_fail")
1138         self.assertInLog("run_next_test")
1139         self.assertInLog("run_test")
1140         self.assertNotInLog("Task.jsm")
1141
1142     def testAddTaskSkip(self):
1143         self.writeFile("test_tasks_skip.js", ADD_TASK_SKIP)
1144         self.writeManifest(["test_tasks_skip.js"])
1145
1146         self.assertTestResult(True)
1147         self.assertEqual(1, self.x.testCount)
1148         self.assertEqual(1, self.x.passCount)
1149         self.assertEqual(0, self.x.failCount)
1150
1151     def testAddTaskSkipAll(self):
1152         self.writeFile("test_tasks_skipall.js", ADD_TASK_SKIPALL)
1153         self.writeManifest(["test_tasks_skipall.js"])
1154
1155         self.assertTestResult(True)
1156         self.assertEqual(1, self.x.testCount)
1157         self.assertEqual(1, self.x.passCount)
1158         self.assertEqual(0, self.x.failCount)
1159
1160     def testMissingHeadFile(self):
1161         """
1162         Ensure that missing head file results in fatal failure.
1163         """
1164         self.writeFile("test_basic.js", SIMPLE_PASSING_TEST)
1165         self.writeManifest([("test_basic.js", 'head = "missing.js"')])
1166
1167         raised = False
1168
1169         try:
1170             # The actual return value is never checked because we raise.
1171             self.assertTestResult(True)
1172         except Exception as ex:
1173             raised = True
1174             self.assertEqual(str(ex)[0:9], "head file")
1175
1176         self.assertTrue(raised)
1177
1178     def testRandomExecution(self):
1179         """
1180         Check that random execution doesn't break.
1181         """
1182         manifest = []
1183         for i in range(0, 10):
1184             filename = "test_pass_%d.js" % i
1185             self.writeFile(filename, SIMPLE_PASSING_TEST)
1186             manifest.append(filename)
1187
1188         self.writeManifest(manifest)
1189         self.assertTestResult(True, shuffle=True)
1190         self.assertEqual(10, self.x.testCount)
1191         self.assertEqual(10, self.x.passCount)
1192
1193     def testDoThrowString(self):
1194         """
1195         Check that do_throw produces reasonable messages when the
1196         input is a string instead of an object
1197         """
1198         self.writeFile("test_error.js", ADD_TEST_THROW_STRING)
1199         self.writeManifest(["test_error.js"])
1200
1201         self.assertTestResult(False)
1202         self.assertInLog(TEST_FAIL_STRING)
1203         self.assertInLog("Passing a string to do_throw")
1204         self.assertNotInLog(TEST_PASS_STRING)
1205
1206     def testDoThrowForeignObject(self):
1207         """
1208         Check that do_throw produces reasonable messages when the
1209         input is a generic object with 'filename', 'message' and 'stack' attributes
1210         but 'object instanceof Error' returns false
1211         """
1212         self.writeFile("test_error.js", ADD_TEST_THROW_OBJECT)
1213         self.writeManifest(["test_error.js"])
1214
1215         self.assertTestResult(False)
1216         self.assertInLog(TEST_FAIL_STRING)
1217         self.assertInLog("failure.js")
1218         self.assertInLog("Error object")
1219         self.assertInLog("ERROR STACK")
1220         self.assertNotInLog(TEST_PASS_STRING)
1221
1222     def testDoReportForeignObject(self):
1223         """
1224         Check that do_report_unexpected_exception produces reasonable messages when the
1225         input is a generic object with 'filename', 'message' and 'stack' attributes
1226         but 'object instanceof Error' returns false
1227         """
1228         self.writeFile("test_error.js", ADD_TEST_REPORT_OBJECT)
1229         self.writeManifest(["test_error.js"])
1230
1231         self.assertTestResult(False)
1232         self.assertInLog(TEST_FAIL_STRING)
1233         self.assertInLog("failure.js")
1234         self.assertInLog("Error object")
1235         self.assertInLog("ERROR STACK")
1236         self.assertNotInLog(TEST_PASS_STRING)
1237
1238     def testDoReportRefError(self):
1239         """
1240         Check that do_report_unexpected_exception produces reasonable messages when the
1241         input is a JS-generated Error
1242         """
1243         self.writeFile("test_error.js", ADD_TEST_REPORT_REF_ERROR)
1244         self.writeManifest(["test_error.js"])
1245
1246         self.assertTestResult(False)
1247         self.assertInLog(TEST_FAIL_STRING)
1248         self.assertInLog("test_error.js")
1249         self.assertInLog("obj.noSuchFunction is not a function")
1250         self.assertInLog("run_test@")
1251         self.assertNotInLog(TEST_PASS_STRING)
1252
1253     def testDoReportSyntaxError(self):
1254         """
1255         Check that attempting to load a test file containing a syntax error
1256         generates details of the error in the log
1257         """
1258         self.writeFile("test_error.js", LOAD_ERROR_SYNTAX_ERROR)
1259         self.writeManifest(["test_error.js"])
1260
1261         self.assertTestResult(False)
1262         self.assertInLog(TEST_FAIL_STRING)
1263         self.assertInLog("test_error.js:3")
1264         self.assertNotInLog(TEST_PASS_STRING)
1265
1266     def testDoReportNonSyntaxError(self):
1267         """
1268         Check that attempting to load a test file containing an error other
1269         than a syntax error generates details of the error in the log
1270         """
1271         self.writeFile("test_error.js", LOAD_ERROR_OTHER_ERROR)
1272         self.writeManifest(["test_error.js"])
1273
1274         self.assertTestResult(False)
1275         self.assertInLog(TEST_FAIL_STRING)
1276         self.assertInLog("ReferenceError: assignment to undeclared variable")
1277         self.assertInLog("test_error.js:3")
1278         self.assertNotInLog(TEST_PASS_STRING)
1279
1280     def testDoPrintWhenVerboseNotExplicit(self):
1281         """
1282         Check that info() and similar calls that generate output do
1283         not have the output when not run verbosely.
1284         """
1285         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1286         self.writeManifest(["test_verbose.js"])
1287
1288         self.assertTestResult(True)
1289         self.assertNotInLog("a message from info")
1290
1291     def testDoPrintWhenVerboseExplicit(self):
1292         """
1293         Check that info() and similar calls that generate output have the
1294         output shown when run verbosely.
1295         """
1296         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1297         self.writeManifest(["test_verbose.js"])
1298         self.assertTestResult(True, verbose=True)
1299         self.assertInLog("a message from info")
1300
1301     def testDoPrintWhenVerboseInManifest(self):
1302         """
1303         Check that info() and similar calls that generate output have the
1304         output shown when 'verbose = true' is in the manifest, even when
1305         not run verbosely.
1306         """
1307         self.writeFile("test_verbose.js", ADD_TEST_VERBOSE)
1308         self.writeManifest([("test_verbose.js", "verbose = true")])
1309
1310         self.assertTestResult(True)
1311         self.assertInLog("a message from info")
1312
1313     def testAsyncCleanup(self):
1314         """
1315         Check that registerCleanupFunction handles nicely async cleanup tasks
1316         """
1317         self.writeFile("test_asyncCleanup.js", ASYNC_CLEANUP)
1318         self.writeManifest(["test_asyncCleanup.js"])
1319         self.assertTestResult(False)
1320         self.assertInLog('"123456" == "123456"')
1321         self.assertInLog("At this stage, the test has succeeded")
1322         self.assertInLog("Throwing an error to force displaying the log")
1323
1324     def testNoRunTestAddTest(self):
1325         """
1326         Check that add_test() works fine without run_test() in the test file.
1327         """
1328         self.writeFile("test_noRunTestAddTest.js", NO_RUN_TEST_ADD_TEST)
1329         self.writeManifest(["test_noRunTestAddTest.js"])
1330
1331         self.assertTestResult(True)
1332         self.assertEqual(1, self.x.testCount)
1333         self.assertEqual(1, self.x.passCount)
1334         self.assertEqual(0, self.x.failCount)
1335         self.assertInLog(TEST_PASS_STRING)
1336         self.assertNotInLog(TEST_FAIL_STRING)
1337
1338     def testNoRunTestAddTask(self):
1339         """
1340         Check that add_task() works fine without run_test() in the test file.
1341         """
1342         self.writeFile("test_noRunTestAddTask.js", NO_RUN_TEST_ADD_TASK)
1343         self.writeManifest(["test_noRunTestAddTask.js"])
1344
1345         self.assertTestResult(True)
1346         self.assertEqual(1, self.x.testCount)
1347         self.assertEqual(1, self.x.passCount)
1348         self.assertEqual(0, self.x.failCount)
1349         self.assertInLog(TEST_PASS_STRING)
1350         self.assertNotInLog(TEST_FAIL_STRING)
1351
1352     def testNoRunTestAddTestAddTask(self):
1353         """
1354         Check that both add_test() and add_task() work without run_test()
1355         in the test file.
1356         """
1357         self.writeFile("test_noRunTestAddTestAddTask.js", NO_RUN_TEST_ADD_TEST_ADD_TASK)
1358         self.writeManifest(["test_noRunTestAddTestAddTask.js"])
1359
1360         self.assertTestResult(True)
1361         self.assertEqual(1, self.x.testCount)
1362         self.assertEqual(1, self.x.passCount)
1363         self.assertEqual(0, self.x.failCount)
1364         self.assertInLog(TEST_PASS_STRING)
1365         self.assertNotInLog(TEST_FAIL_STRING)
1366
1367     def testNoRunTestEmptyTest(self):
1368         """
1369         Check that the test passes on an empty file that contains neither
1370         run_test() nor add_test(), add_task().
1371         """
1372         self.writeFile("test_noRunTestEmptyTest.js", NO_RUN_TEST_EMPTY_TEST)
1373         self.writeManifest(["test_noRunTestEmptyTest.js"])
1374
1375         self.assertTestResult(True)
1376         self.assertEqual(1, self.x.testCount)
1377         self.assertEqual(1, self.x.passCount)
1378         self.assertEqual(0, self.x.failCount)
1379         self.assertInLog(TEST_PASS_STRING)
1380         self.assertNotInLog(TEST_FAIL_STRING)
1381
1382     def testNoRunTestAddTestFail(self):
1383         """
1384         Check that test fails on using add_test() without run_test().
1385         """
1386         self.writeFile("test_noRunTestAddTestFail.js", NO_RUN_TEST_ADD_TEST_FAIL)
1387         self.writeManifest(["test_noRunTestAddTestFail.js"])
1388
1389         self.assertTestResult(False)
1390         self.assertEqual(1, self.x.testCount)
1391         self.assertEqual(0, self.x.passCount)
1392         self.assertEqual(1, self.x.failCount)
1393         self.assertInLog(TEST_FAIL_STRING)
1394         self.assertNotInLog(TEST_PASS_STRING)
1395
1396     def testNoRunTestAddTaskFail(self):
1397         """
1398         Check that test fails on using add_task() without run_test().
1399         """
1400         self.writeFile("test_noRunTestAddTaskFail.js", NO_RUN_TEST_ADD_TASK_FAIL)
1401         self.writeManifest(["test_noRunTestAddTaskFail.js"])
1402
1403         self.assertTestResult(False)
1404         self.assertEqual(1, self.x.testCount)
1405         self.assertEqual(0, self.x.passCount)
1406         self.assertEqual(1, self.x.failCount)
1407         self.assertInLog(TEST_FAIL_STRING)
1408         self.assertNotInLog(TEST_PASS_STRING)
1409
1410     def testNoRunTestAddTaskMultiple(self):
1411         """
1412         Check that multple add_task() tests work without run_test().
1413         """
1414         self.writeFile(
1415             "test_noRunTestAddTaskMultiple.js", NO_RUN_TEST_ADD_TASK_MULTIPLE
1416         )
1417         self.writeManifest(["test_noRunTestAddTaskMultiple.js"])
1418
1419         self.assertTestResult(True)
1420         self.assertEqual(1, self.x.testCount)
1421         self.assertEqual(1, self.x.passCount)
1422         self.assertEqual(0, self.x.failCount)
1423         self.assertInLog(TEST_PASS_STRING)
1424         self.assertNotInLog(TEST_FAIL_STRING)
1425
1426     def testMozinfo(self):
1427         """
1428         Check that mozinfo.json is loaded
1429         """
1430         self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
1431         self.writeManifest(["test_mozinfo.js"])
1432         self.assertTestResult(True)
1433         self.assertEqual(1, self.x.testCount)
1434         self.assertEqual(1, self.x.passCount)
1435         self.assertEqual(0, self.x.failCount)
1436         self.assertEqual(0, self.x.todoCount)
1437         self.assertInLog(TEST_PASS_STRING)
1438         self.assertNotInLog(TEST_FAIL_STRING)
1439
1440     def testChildMozinfo(self):
1441         """
1442         Check that mozinfo.json is loaded in child process
1443         """
1444         self.writeFile("test_mozinfo.js", LOAD_MOZINFO)
1445         self.writeFile("test_child_mozinfo.js", CHILD_MOZINFO)
1446         self.writeManifest(["test_child_mozinfo.js"])
1447         self.assertTestResult(True)
1448         self.assertEqual(1, self.x.testCount)
1449         self.assertEqual(1, self.x.passCount)
1450         self.assertEqual(0, self.x.failCount)
1451         self.assertEqual(0, self.x.todoCount)
1452         self.assertInLog(TEST_PASS_STRING)
1453         self.assertNotInLog(TEST_FAIL_STRING)
1454
1455     def testNotHeadlessByDefault(self):
1456         """
1457         Check that the default is not headless.
1458         """
1459         self.writeFile("test_notHeadlessByDefault.js", HEADLESS_FALSE)
1460         self.writeManifest(["test_notHeadlessByDefault.js"])
1461         self.assertTestResult(True)
1462
1463     def testHeadlessWhenHeadlessExplicit(self):
1464         """
1465         Check that explicitly requesting headless works when the manifest doesn't override.
1466         """
1467         self.writeFile("test_headlessWhenExplicit.js", HEADLESS_TRUE)
1468         self.writeManifest(["test_headlessWhenExplicit.js"])
1469         self.assertTestResult(True, headless=True)
1470
1471     def testHeadlessWhenHeadlessTrueInManifest(self):
1472         """
1473         Check that enabling headless in the manifest alone works.
1474         """
1475         self.writeFile("test_headlessWhenTrueInManifest.js", HEADLESS_TRUE)
1476         self.writeManifest([("test_headlessWhenTrueInManifest.js", "headless = true")])
1477         self.assertTestResult(True)
1478
1479     def testNotHeadlessWhenHeadlessFalseInManifest(self):
1480         """
1481         Check that the manifest entry overrides the explicit default.
1482         """
1483         self.writeFile("test_notHeadlessWhenFalseInManifest.js", HEADLESS_FALSE)
1484         self.writeManifest(
1485             [("test_notHeadlessWhenFalseInManifest.js", "headless = false")]
1486         )
1487         self.assertTestResult(True, headless=True)
1488
1489
1490 if __name__ == "__main__":
1491     import mozunit
1492
1493     mozinfo.find_and_update_from_json()
1494     mozunit.main()