tools/tryselect/test/test_mozharness_integration.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import imp
   6 import json
   7 import os
   8
   9 import mozunit
  10 import pytest
  11 from tryselect.tasks import build, resolve_tests_by_suite
  12
  13 MOZHARNESS_SCRIPTS = {
  14     "android_emulator_unittest": {
  15         "class_name": "AndroidEmulatorTest",
  16         "configs": [
  17             "android/android_common.py",
  18         ],
  19         "xfail": [
  20             "cppunittest",
  21             "crashtest-qr",
  22             "gtest",
  23             "geckoview-junit",
  24             "jittest",
  25             "jsreftest",
  26             "reftest-qr",
  27         ],
  28     },
  29     "desktop_unittest": {
  30         "class_name": "DesktopUnittest",
  31         "configs": [
  32             "unittests/linux_unittest.py",
  33             "unittests/mac_unittest.py",
  34             "unittests/win_unittest.py",
  35         ],
  36         "xfail": [
  37             "cppunittest",
  38             "gtest",
  39             "jittest",
  40             "jittest-chunked",
  41             "jittest1",
  42             "jittest2",
  43             "jsreftest",
  44             "mochitest-valgrind-plain",
  45             "reftest-no-accel",
  46             "reftest-snapshot",
  47             "xpcshell-msix",
  48         ],
  49     },
  50 }
  51 """A suite being listed in a script's `xfail` list  means it won't work
  52 properly with MOZHARNESS_TEST_PATHS (the mechanism |mach try fuzzy <path>|
  53 uses).
  54 """
  55
  56
  57 def get_mozharness_test_paths(name):
  58     scriptdir = os.path.join(build.topsrcdir, "testing", "mozharness", "scripts")
  59
  60     files = imp.find_module(name, [scriptdir])
  61     mod = imp.load_module("scripts.{}".format(name), *files)
  62
  63     class_name = MOZHARNESS_SCRIPTS[name]["class_name"]
  64     cls = getattr(mod, class_name)
  65     return cls(require_config_file=False)._get_mozharness_test_paths
  66
  67
  68 @pytest.fixture(scope="module")
  69 def all_suites():
  70     from moztest.resolve import _test_flavors, _test_subsuites
  71
  72     all_suites = []
  73     for flavor in _test_flavors:
  74         all_suites.append({"flavor": flavor, "srcdir_relpath": "test"})
  75
  76     for flavor, subsuite in _test_subsuites:
  77         all_suites.append(
  78             {"flavor": flavor, "subsuite": subsuite, "srcdir_relpath": "test"}
  79         )
  80
  81     return all_suites
  82
  83
  84 def generate_suites_from_config(path):
  85     configdir = os.path.join(build.topsrcdir, "testing", "mozharness", "configs")
  86
  87     parent, name = os.path.split(path)
  88     name = os.path.splitext(name)[0]
  89
  90     files = imp.find_module("{}".format(name), [os.path.join(configdir, parent)])
  91     mod = imp.load_module("config.{}".format(name), *files)
  92     config = mod.config
  93
  94     for category in sorted(config["suite_definitions"]):
  95         key = "all_{}_suites".format(category)
  96         if key not in config:
  97             yield category,
  98             continue
  99
 100         for suite in sorted(config["all_{}_suites".format(category)]):
 101             yield category, suite
 102
 103
 104 def generate_suites():
 105     for name, script in MOZHARNESS_SCRIPTS.items():
 106         seen = set()
 107
 108         for path in script["configs"]:
 109             for suite in generate_suites_from_config(path):
 110                 if suite in seen:
 111                     continue
 112                 seen.add(suite)
 113
 114                 item = (name, suite)
 115
 116                 if suite[-1] in script["xfail"]:
 117                     item = pytest.param(item, marks=pytest.mark.xfail)
 118
 119                 yield item
 120
 121
 122 def idfn(item):
 123     name, suite = item
 124     return "{}/{}".format(name, suite[-1])
 125
 126
 127 @pytest.mark.parametrize("item", generate_suites(), ids=idfn)
 128 def test_suites(item, patch_resolver, all_suites):
 129     """An integration test to make sure the suites returned by
 130     `tasks.resolve_tests_by_suite` match up with the names defined in
 131     mozharness.
 132     """
 133     patch_resolver([], all_suites)
 134     suites = resolve_tests_by_suite(["test"])
 135     os.environ["MOZHARNESS_TEST_PATHS"] = json.dumps(suites)
 136
 137     name, suite = item
 138     func = get_mozharness_test_paths(name)
 139     assert func(*suite)
 140
 141
 142 if __name__ == "__main__":
 143     mozunit.main()