tools/tryselect/test/test_mozharness_integration.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import json
   6 import os
   7
   8 import mozunit
   9 import pytest
  10 from mozfile import load_source
  11 from tryselect.tasks import build, resolve_tests_by_suite
  12
  13 MOZHARNESS_SCRIPTS = {
  14     "android_emulator_unittest": {
  15         "class_name": "AndroidEmulatorTest",
  16         "configs": [
  17             "android/android_common.py",
  18         ],
  19         "xfail": [
  20             "cppunittest",
  21             "crashtest-qr",
  22             "gtest",
  23             "geckoview-junit",
  24             "jittest",
  25             "jsreftest",
  26             "reftest-qr",
  27         ],
  28     },
  29     "desktop_unittest": {
  30         "class_name": "DesktopUnittest",
  31         "configs": [
  32             "unittests/linux_unittest.py",
  33             "unittests/mac_unittest.py",
  34             "unittests/win_unittest.py",
  35         ],
  36         "xfail": [
  37             "cppunittest",
  38             "gtest",
  39             "jittest",
  40             "jittest-chunked",
  41             "jittest1",
  42             "jittest2",
  43             "jsreftest",
  44             "mochitest-valgrind-plain",
  45             "reftest-no-accel",
  46             "reftest-snapshot",
  47             "xpcshell-msix",
  48         ],
  49     },
  50 }
  51 """A suite being listed in a script's `xfail` list  means it won't work
  52 properly with MOZHARNESS_TEST_PATHS (the mechanism |mach try fuzzy <path>|
  53 uses).
  54 """
  55
  56
  57 def get_mozharness_test_paths(name):
  58     scriptdir = os.path.join(build.topsrcdir, "testing", "mozharness")
  59     mod = load_source(
  60         "scripts." + name, os.path.join(scriptdir, "scripts", name + ".py")
  61     )
  62
  63     class_name = MOZHARNESS_SCRIPTS[name]["class_name"]
  64     cls = getattr(mod, class_name)
  65     return cls(require_config_file=False)._get_mozharness_test_paths
  66
  67
  68 @pytest.fixture(scope="module")
  69 def all_suites():
  70     from moztest.resolve import _test_flavors, _test_subsuites
  71
  72     all_suites = []
  73     for flavor in _test_flavors:
  74         all_suites.append({"flavor": flavor, "srcdir_relpath": "test"})
  75
  76     for flavor, subsuite in _test_subsuites:
  77         all_suites.append(
  78             {"flavor": flavor, "subsuite": subsuite, "srcdir_relpath": "test"}
  79         )
  80
  81     return all_suites
  82
  83
  84 def generate_suites_from_config(path):
  85     parent, name = os.path.split(path)
  86     name = os.path.splitext(name)[0]
  87
  88     configdir = os.path.join(
  89         build.topsrcdir, "testing", "mozharness", "configs", parent
  90     )
  91
  92     mod = load_source(name, os.path.join(configdir, name + ".py"))
  93
  94     config = mod.config
  95
  96     for category in sorted(config["suite_definitions"]):
  97         key = "all_{}_suites".format(category)
  98         if key not in config:
  99             yield category,
 100             continue
 101
 102         for suite in sorted(config["all_{}_suites".format(category)]):
 103             yield category, suite
 104
 105
 106 def generate_suites():
 107     for name, script in MOZHARNESS_SCRIPTS.items():
 108         seen = set()
 109
 110         for path in script["configs"]:
 111             for suite in generate_suites_from_config(path):
 112                 if suite in seen:
 113                     continue
 114                 seen.add(suite)
 115
 116                 item = (name, suite)
 117
 118                 if suite[-1] in script["xfail"]:
 119                     item = pytest.param(item, marks=pytest.mark.xfail)
 120
 121                 yield item
 122
 123
 124 def idfn(item):
 125     name, suite = item
 126     return "{}/{}".format(name, suite[-1])
 127
 128
 129 @pytest.mark.parametrize("item", generate_suites(), ids=idfn)
 130 def test_suites(item, patch_resolver, all_suites):
 131     """An integration test to make sure the suites returned by
 132     `tasks.resolve_tests_by_suite` match up with the names defined in
 133     mozharness.
 134     """
 135     patch_resolver([], all_suites)
 136     suites = resolve_tests_by_suite(["test"])
 137     os.environ["MOZHARNESS_TEST_PATHS"] = json.dumps(suites)
 138
 139     name, suite = item
 140     func = get_mozharness_test_paths(name)
 141     assert func(*suite)
 142
 143
 144 if __name__ == "__main__":
 145     mozunit.main()