testing/mochitest/tests/python/test_mochitest_integration.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import
   6
   7 import os
   8 from functools import partial
   9
  10 from manifestparser import TestManifest
  11
  12 import mozunit
  13 import pytest
  14 from moztest.selftest.output import get_mozharness_status, filter_action
  15 from conftest import setup_args
  16
  17 from mozharness.base.log import INFO, WARNING, ERROR
  18 from mozharness.mozilla.automation import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
  19
  20
  21 here = os.path.abspath(os.path.dirname(__file__))
  22 get_mozharness_status = partial(get_mozharness_status, "mochitest")
  23
  24
  25 @pytest.fixture
  26 def test_name(request):
  27     flavor = request.getfixturevalue("flavor")
  28
  29     def inner(name):
  30         if flavor == "plain":
  31             return f"test_{name}.html"
  32         elif flavor == "browser-chrome":
  33             return f"browser_{name}.js"
  34
  35     return inner
  36
  37
  38 @pytest.fixture
  39 def test_manifest(setup_test_harness, request):
  40     flavor = request.getfixturevalue("flavor")
  41     test_root = setup_test_harness(*setup_args, flavor=flavor)
  42     assert test_root
  43
  44     def inner(manifestFileNames):
  45         return TestManifest(
  46             manifests=[os.path.join(test_root, name) for name in manifestFileNames],
  47             strict=False,
  48             rootdir=test_root,
  49         )
  50
  51     return inner
  52
  53
  54 @pytest.mark.parametrize("runFailures", ["selftest", ""])
  55 @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
  56 def test_output_pass(flavor, runFailures, runtests, test_name):
  57     extra_opts = {}
  58     results = {
  59         "status": 1 if runFailures else 0,
  60         "tbpl_status": TBPL_WARNING if runFailures else TBPL_SUCCESS,
  61         "log_level": (INFO, WARNING),
  62         "lines": 2 if runFailures else 1,
  63         "line_status": "PASS",
  64     }
  65     if runFailures:
  66         extra_opts["runFailures"] = runFailures
  67         extra_opts["crashAsPass"] = True
  68         extra_opts["timeoutAsPass"] = True
  69
  70     status, lines = runtests(test_name("pass"), **extra_opts)
  71     assert status == results["status"]
  72
  73     tbpl_status, log_level, summary = get_mozharness_status(lines, status)
  74     assert tbpl_status == results["tbpl_status"]
  75     assert log_level in results["log_level"]
  76
  77     lines = filter_action("test_status", lines)
  78     assert len(lines) == results["lines"]
  79     assert lines[0]["status"] == results["line_status"]
  80
  81
  82 @pytest.mark.parametrize("runFailures", ["selftest", ""])
  83 @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
  84 def test_output_fail(flavor, runFailures, runtests, test_name):
  85     extra_opts = {}
  86     results = {
  87         "status": 0 if runFailures else 1,
  88         "tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING,
  89         "log_level": (INFO, WARNING),
  90         "lines": 1,
  91         "line_status": "PASS" if runFailures else "FAIL",
  92     }
  93     if runFailures:
  94         extra_opts["runFailures"] = runFailures
  95         extra_opts["crashAsPass"] = True
  96         extra_opts["timeoutAsPass"] = True
  97
  98     status, lines = runtests(test_name("fail"), **extra_opts)
  99     assert status == results["status"]
 100
 101     tbpl_status, log_level, summary = get_mozharness_status(lines, status)
 102     assert tbpl_status == results["tbpl_status"]
 103     assert log_level in results["log_level"]
 104
 105     lines = filter_action("test_status", lines)
 106     assert len(lines) == results["lines"]
 107     assert lines[0]["status"] == results["line_status"]
 108
 109
 110 @pytest.mark.skip_mozinfo("!crashreporter")
 111 @pytest.mark.parametrize("runFailures", ["selftest", ""])
 112 @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
 113 def test_output_crash(flavor, runFailures, runtests, test_name):
 114     extra_opts = {}
 115     results = {
 116         "status": 0 if runFailures else 1,
 117         "tbpl_status": TBPL_FAILURE,
 118         "log_level": ERROR,
 119         "lines": 1,
 120     }
 121     if runFailures:
 122         extra_opts["runFailures"] = runFailures
 123         extra_opts["crashAsPass"] = True
 124         extra_opts["timeoutAsPass"] = True
 125         # bug 1443327 - we do not set MOZ_CRASHREPORTER_SHUTDOWN for browser-chrome
 126         # the error regex's don't pick this up as a failure
 127         if flavor == "browser-chrome":
 128             results["tbpl_status"] = TBPL_SUCCESS
 129             results["log_level"] = (INFO, WARNING)
 130
 131     status, lines = runtests(
 132         test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts
 133     )
 134     assert status == results["status"]
 135
 136     tbpl_status, log_level, summary = get_mozharness_status(lines, status)
 137     assert tbpl_status == results["tbpl_status"]
 138     assert log_level in results["log_level"]
 139
 140     if not runFailures:
 141         crash = filter_action("crash", lines)
 142         assert len(crash) == 1
 143         assert crash[0]["action"] == "crash"
 144         assert crash[0]["signature"]
 145         assert crash[0]["minidump_path"]
 146
 147     lines = filter_action("test_end", lines)
 148     assert len(lines) == results["lines"]
 149
 150
 151 @pytest.mark.skip_mozinfo("!asan")
 152 @pytest.mark.parametrize("runFailures", [""])
 153 @pytest.mark.parametrize("flavor", ["plain"])
 154 def test_output_asan(flavor, runFailures, runtests, test_name):
 155     extra_opts = {}
 156     results = {"status": 1, "tbpl_status": TBPL_FAILURE, "log_level": ERROR, "lines": 0}
 157
 158     status, lines = runtests(
 159         test_name("crash"), environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"], **extra_opts
 160     )
 161     assert status == results["status"]
 162
 163     tbpl_status, log_level, summary = get_mozharness_status(lines, status)
 164     assert tbpl_status == results["tbpl_status"]
 165     assert log_level == results["log_level"]
 166
 167     crash = filter_action("crash", lines)
 168     assert len(crash) == results["lines"]
 169
 170     process_output = filter_action("process_output", lines)
 171     assert any("ERROR: AddressSanitizer" in l["data"] for l in process_output)
 172
 173
 174 @pytest.mark.skip_mozinfo("!debug")
 175 @pytest.mark.parametrize("runFailures", [""])
 176 @pytest.mark.parametrize("flavor", ["plain"])
 177 def test_output_assertion(flavor, runFailures, runtests, test_name):
 178     extra_opts = {}
 179     results = {
 180         "status": 0,
 181         "tbpl_status": TBPL_WARNING,
 182         "log_level": WARNING,
 183         "lines": 1,
 184         "assertions": 1,
 185     }
 186
 187     status, lines = runtests(test_name("assertion"), **extra_opts)
 188     # TODO: mochitest should return non-zero here
 189     assert status == results["status"]
 190
 191     tbpl_status, log_level, summary = get_mozharness_status(lines, status)
 192     assert tbpl_status == results["tbpl_status"]
 193     assert log_level == results["log_level"]
 194
 195     test_end = filter_action("test_end", lines)
 196     assert len(test_end) == results["lines"]
 197     # TODO: this should be ASSERT, but moving the assertion check before
 198     # the test_end action caused a bunch of failures.
 199     assert test_end[0]["status"] == "OK"
 200
 201     assertions = filter_action("assertion_count", lines)
 202     assert len(assertions) == results["assertions"]
 203     assert assertions[0]["count"] == results["assertions"]
 204
 205
 206 @pytest.mark.skip_mozinfo("!debug")
 207 @pytest.mark.parametrize("runFailures", [""])
 208 @pytest.mark.parametrize("flavor", ["plain"])
 209 def test_output_leak(flavor, runFailures, runtests, test_name):
 210     extra_opts = {}
 211     results = {"status": 0, "tbpl_status": TBPL_WARNING, "log_level": WARNING}
 212
 213     status, lines = runtests(test_name("leak"), **extra_opts)
 214     # TODO: mochitest should return non-zero here
 215     assert status == results["status"]
 216
 217     tbpl_status, log_level, summary = get_mozharness_status(lines, status)
 218     assert tbpl_status == results["tbpl_status"]
 219     assert log_level == results["log_level"]
 220
 221     leak_totals = filter_action("mozleak_total", lines)
 222     found_leaks = False
 223     for lt in leak_totals:
 224         if lt["bytes"] == 0:
 225             # No leaks in this process.
 226             assert len(lt["objects"]) == 0
 227             continue
 228
 229         assert not found_leaks, "Only one process should have leaked"
 230         found_leaks = True
 231         assert lt["process"] == "tab"
 232         assert lt["bytes"] == 1
 233         assert lt["objects"] == ["IntentionallyLeakedObject"]
 234
 235     assert found_leaks, "At least one process should have leaked"
 236
 237
 238 @pytest.mark.parametrize("flavor", ["plain"])
 239 def test_output_testfile_in_dupe_manifests(flavor, runtests, test_name, test_manifest):
 240     results = {
 241         "status": 0,
 242         "tbpl_status": TBPL_SUCCESS,
 243         "log_level": (INFO, WARNING),
 244         "line_status": "PASS",
 245         # We expect the test to be executed exactly 2 times,
 246         # once for each manifest where the test file has been included.
 247         "lines": 2,
 248     }
 249
 250     # Explicitly provide a manifestFile property that includes the
 251     # two manifest files that share the same test file.
 252     extra_opts = {
 253         "manifestFile": test_manifest(
 254             [
 255                 "mochitest-dupemanifest-1.ini",
 256                 "mochitest-dupemanifest-2.ini",
 257             ]
 258         ),
 259         "runByManifest": True,
 260     }
 261
 262     # Execute mochitest by explicitly request the test file listed
 263     # in two manifest files to be executed.
 264     status, lines = runtests(test_name("pass"), **extra_opts)
 265     assert status == results["status"]
 266
 267     tbpl_status, log_level, summary = get_mozharness_status(lines, status)
 268     assert tbpl_status == results["tbpl_status"]
 269     assert log_level in results["log_level"]
 270
 271     lines = filter_action("test_status", lines)
 272     assert len(lines) == results["lines"]
 273     assert lines[0]["status"] == results["line_status"]
 274
 275
 276 if __name__ == "__main__":
 277     mozunit.main()