testing/tools/mach_test_package_initialize.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import json
   6 import os
   7 import sys
   8 import types
   9
  10 SEARCH_PATHS = [
  11     "gtest",
  12     "marionette/client",
  13     "marionette/harness",
  14     "mochitest",
  15     "mozbase/manifestparser",
  16     "mozbase/mozcrash",
  17     "mozbase/mozdebug",
  18     "mozbase/mozdevice",
  19     "mozbase/mozfile",
  20     "mozbase/mozgeckoprofile",
  21     "mozbase/mozhttpd",
  22     "mozbase/mozinfo",
  23     "mozbase/mozinstall",
  24     "mozbase/mozleak",
  25     "mozbase/mozlog",
  26     "mozbase/moznetwork",
  27     "mozbase/mozpower",
  28     "mozbase/mozprocess",
  29     "mozbase/mozprofile",
  30     "mozbase/mozrunner",
  31     "mozbase/mozscreenshot",
  32     "mozbase/mozserve",
  33     "mozbase/mozsystemmonitor",
  34     "mozbase/moztest",
  35     "mozbase/mozversion",
  36     "reftest",
  37     "tools/mach",
  38     "tools/mozterm",
  39     "tools/geckoprocesstypes_generator",
  40     "tools/six",
  41     "tools/wptserve",
  42     "web-platform",
  43     "web-platform/tests/tools/wptrunner",
  44     "xpcshell",
  45 ]
  46
  47
  48 CATEGORIES = {
  49     "testing": {
  50         "short": "Testing",
  51         "long": "Run tests.",
  52         "priority": 30,
  53     },
  54     "devenv": {
  55         "short": "Development Environment",
  56         "long": "Set up and configure your development environment.",
  57         "priority": 20,
  58     },
  59     "misc": {
  60         "short": "Potpourri",
  61         "long": "Potent potables and assorted snacks.",
  62         "priority": 10,
  63     },
  64     "disabled": {
  65         "short": "Disabled",
  66         "long": "The disabled commands are hidden by default. Use -v to display them. "
  67         "These commands are unavailable for your current context, "
  68         'run "mach <command>" to see why.',
  69         "priority": 0,
  70     },
  71 }
  72
  73
  74 IS_WIN = sys.platform in ("win32", "cygwin")
  75
  76
  77 def ancestors(path, depth=0):
  78     """Emit the parent directories of a path."""
  79     count = 1
  80     while path and count != depth:
  81         yield path
  82         newpath = os.path.dirname(path)
  83         if newpath == path:
  84             break
  85         path = newpath
  86         count += 1
  87
  88
  89 def activate_mozharness_venv(context):
  90     """Activate the mozharness virtualenv in-process."""
  91     venv = os.path.join(
  92         context.mozharness_workdir,
  93         context.mozharness_config.get("virtualenv_path", "venv"),
  94     )
  95
  96     if not os.path.isdir(venv):
  97         print("No mozharness virtualenv detected at '{}'.".format(venv))
  98         return 1
  99
 100     venv_bin = os.path.join(venv, "Scripts" if IS_WIN else "bin")
 101     activate_path = os.path.join(venv_bin, "activate_this.py")
 102
 103     exec(open(activate_path).read(), dict(__file__=activate_path))
 104
 105     if isinstance(os.environ["PATH"], str):
 106         os.environ["PATH"] = os.environ["PATH"].encode("utf-8")
 107
 108     # sys.executable is used by mochitest-media to start the websocketprocessbridge,
 109     # for some reason it doesn't get set when calling `activate_this.py` so set it
 110     # here instead.
 111     binary = "python"
 112     if IS_WIN:
 113         binary += ".exe"
 114     sys.executable = os.path.join(venv_bin, binary)
 115
 116
 117 def find_firefox(context):
 118     """Try to automagically find the firefox binary."""
 119     import mozinstall
 120
 121     search_paths = []
 122
 123     # Check for a mozharness setup
 124     config = context.mozharness_config
 125     if config and "binary_path" in config:
 126         return config["binary_path"]
 127     elif config:
 128         search_paths.append(os.path.join(context.mozharness_workdir, "application"))
 129
 130     # Check for test-stage setup
 131     dist_bin = os.path.join(os.path.dirname(context.package_root), "bin")
 132     if os.path.isdir(dist_bin):
 133         search_paths.append(dist_bin)
 134
 135     for path in search_paths:
 136         try:
 137             return mozinstall.get_binary(path, "firefox")
 138         except mozinstall.InvalidBinary:
 139             continue
 140
 141
 142 def find_hostutils(context):
 143     workdir = context.mozharness_workdir
 144     hostutils = os.path.join(workdir, "hostutils")
 145     for fname in os.listdir(hostutils):
 146         fpath = os.path.join(hostutils, fname)
 147         if os.path.isdir(fpath) and fname.startswith("host-utils"):
 148             return fpath
 149
 150
 151 def normalize_test_path(test_root, path):
 152     if os.path.isabs(path) or os.path.exists(path):
 153         return os.path.normpath(os.path.abspath(path))
 154
 155     for parent in ancestors(test_root):
 156         test_path = os.path.join(parent, path)
 157         if os.path.exists(test_path):
 158             return os.path.normpath(os.path.abspath(test_path))
 159     # Not a valid path? Return as is and let test harness deal with it
 160     return path
 161
 162
 163 def bootstrap(test_package_root):
 164     test_package_root = os.path.abspath(test_package_root)
 165
 166     sys.path[0:0] = [os.path.join(test_package_root, path) for path in SEARCH_PATHS]
 167     import mach.main
 168     from mach.command_util import MachCommandReference, load_commands_from_spec
 169
 170     # Centralized registry of available mach commands
 171     MACH_COMMANDS = {
 172         "gtest": MachCommandReference("gtest/mach_test_package_commands.py"),
 173         "marionette-test": MachCommandReference(
 174             "marionette/mach_test_package_commands.py"
 175         ),
 176         "mochitest": MachCommandReference("mochitest/mach_test_package_commands.py"),
 177         "geckoview-junit": MachCommandReference(
 178             "mochitest/mach_test_package_commands.py"
 179         ),
 180         "reftest": MachCommandReference("reftest/mach_test_package_commands.py"),
 181         "mach-commands": MachCommandReference(
 182             "python/mach/mach/commands/commandinfo.py"
 183         ),
 184         "mach-debug-commands": MachCommandReference(
 185             "python/mach/mach/commands/commandinfo.py"
 186         ),
 187         "mach-completion": MachCommandReference(
 188             "python/mach/mach/commands/commandinfo.py"
 189         ),
 190         "web-platform-tests": MachCommandReference(
 191             "web-platform/mach_test_package_commands.py"
 192         ),
 193         "wpt": MachCommandReference("web-platform/mach_test_package_commands.py"),
 194         "xpcshell-test": MachCommandReference("xpcshell/mach_test_package_commands.py"),
 195     }
 196
 197     def populate_context(context, key=None):
 198         # These values will be set lazily, and cached after first being invoked.
 199         if key == "package_root":
 200             return test_package_root
 201
 202         if key == "bin_dir":
 203             return os.path.join(test_package_root, "bin")
 204
 205         if key == "certs_dir":
 206             return os.path.join(test_package_root, "certs")
 207
 208         if key == "module_dir":
 209             return os.path.join(test_package_root, "modules")
 210
 211         if key == "ancestors":
 212             return ancestors
 213
 214         if key == "normalize_test_path":
 215             return normalize_test_path
 216
 217         if key == "firefox_bin":
 218             return find_firefox(context)
 219
 220         if key == "hostutils":
 221             return find_hostutils(context)
 222
 223         if key == "mozharness_config":
 224             for dir_path in ancestors(context.package_root):
 225                 mozharness_config = os.path.join(dir_path, "logs", "localconfig.json")
 226                 if os.path.isfile(mozharness_config):
 227                     with open(mozharness_config, "rb") as f:
 228                         return json.load(f)
 229             return {}
 230
 231         if key == "mozharness_workdir":
 232             config = context.mozharness_config
 233             if config:
 234                 return os.path.join(config["base_work_dir"], config["work_dir"])
 235
 236         if key == "activate_mozharness_venv":
 237             return types.MethodType(activate_mozharness_venv, context)
 238
 239     mach = mach.main.Mach(os.getcwd())
 240     mach.populate_context_handler = populate_context
 241
 242     for category, meta in CATEGORIES.items():
 243         mach.define_category(category, meta["short"], meta["long"], meta["priority"])
 244
 245     # Depending on which test zips were extracted,
 246     # the command module might not exist
 247     load_commands_from_spec(MACH_COMMANDS, test_package_root, missing_ok=True)
 248
 249     return mach