testing/tools/mach_test_package_bootstrap.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import absolute_import, print_function, unicode_literals
   6
   7 import json
   8 import os
   9 import sys
  10 import types
  11
  12
  13 SEARCH_PATHS = [
  14     "gtest",
  15     "marionette/client",
  16     "marionette/harness",
  17     "mochitest",
  18     "mozbase/manifestparser",
  19     "mozbase/mozcrash",
  20     "mozbase/mozdebug",
  21     "mozbase/mozdevice",
  22     "mozbase/mozfile",
  23     "mozbase/mozgeckoprofile",
  24     "mozbase/mozhttpd",
  25     "mozbase/mozinfo",
  26     "mozbase/mozinstall",
  27     "mozbase/mozleak",
  28     "mozbase/mozlog",
  29     "mozbase/moznetwork",
  30     "mozbase/mozpower",
  31     "mozbase/mozprocess",
  32     "mozbase/mozprofile",
  33     "mozbase/mozrunner",
  34     "mozbase/mozscreenshot",
  35     "mozbase/mozsystemmonitor",
  36     "mozbase/moztest",
  37     "mozbase/mozversion",
  38     "reftest",
  39     "tools/mach",
  40     "tools/mozterm",
  41     "tools/six",
  42     "tools/wptserve",
  43     "web-platform",
  44     "web-platform/tests/tools/wptrunner",
  45     "xpcshell",
  46 ]
  47
  48 # Individual files providing mach commands.
  49 MACH_MODULES = [
  50     "gtest/mach_test_package_commands.py",
  51     "marionette/mach_test_package_commands.py",
  52     "mochitest/mach_test_package_commands.py",
  53     "reftest/mach_test_package_commands.py",
  54     "tools/mach/mach/commands/commandinfo.py",
  55     "web-platform/mach_test_package_commands.py",
  56     "xpcshell/mach_test_package_commands.py",
  57 ]
  58
  59
  60 CATEGORIES = {
  61     "testing": {
  62         "short": "Testing",
  63         "long": "Run tests.",
  64         "priority": 30,
  65     },
  66     "devenv": {
  67         "short": "Development Environment",
  68         "long": "Set up and configure your development environment.",
  69         "priority": 20,
  70     },
  71     "misc": {
  72         "short": "Potpourri",
  73         "long": "Potent potables and assorted snacks.",
  74         "priority": 10,
  75     },
  76     "disabled": {
  77         "short": "Disabled",
  78         "long": "The disabled commands are hidden by default. Use -v to display them. "
  79         "These commands are unavailable for your current context, "
  80         'run "mach <command>" to see why.',
  81         "priority": 0,
  82     },
  83 }
  84
  85
  86 IS_WIN = sys.platform in ("win32", "cygwin")
  87
  88
  89 def ancestors(path, depth=0):
  90     """Emit the parent directories of a path."""
  91     count = 1
  92     while path and count != depth:
  93         yield path
  94         newpath = os.path.dirname(path)
  95         if newpath == path:
  96             break
  97         path = newpath
  98         count += 1
  99
 100
 101 def activate_mozharness_venv(context):
 102     """Activate the mozharness virtualenv in-process."""
 103     venv = os.path.join(
 104         context.mozharness_workdir,
 105         context.mozharness_config.get("virtualenv_path", "venv"),
 106     )
 107
 108     if not os.path.isdir(venv):
 109         print("No mozharness virtualenv detected at '{}'.".format(venv))
 110         return 1
 111
 112     venv_bin = os.path.join(venv, "Scripts" if IS_WIN else "bin")
 113     activate_path = os.path.join(venv_bin, "activate_this.py")
 114
 115     exec(open(activate_path).read(), dict(__file__=activate_path))
 116
 117     if isinstance(os.environ["PATH"], str):
 118         os.environ["PATH"] = os.environ["PATH"].encode("utf-8")
 119
 120     # sys.executable is used by mochitest-media to start the websocketprocessbridge,
 121     # for some reason it doesn't get set when calling `activate_this.py` so set it
 122     # here instead.
 123     binary = "python"
 124     if IS_WIN:
 125         binary += ".exe"
 126     sys.executable = os.path.join(venv_bin, binary)
 127
 128
 129 def find_firefox(context):
 130     """Try to automagically find the firefox binary."""
 131     import mozinstall
 132
 133     search_paths = []
 134
 135     # Check for a mozharness setup
 136     config = context.mozharness_config
 137     if config and "binary_path" in config:
 138         return config["binary_path"]
 139     elif config:
 140         search_paths.append(os.path.join(context.mozharness_workdir, "application"))
 141
 142     # Check for test-stage setup
 143     dist_bin = os.path.join(os.path.dirname(context.package_root), "bin")
 144     if os.path.isdir(dist_bin):
 145         search_paths.append(dist_bin)
 146
 147     for path in search_paths:
 148         try:
 149             return mozinstall.get_binary(path, "firefox")
 150         except mozinstall.InvalidBinary:
 151             continue
 152
 153
 154 def find_hostutils(context):
 155     workdir = context.mozharness_workdir
 156     hostutils = os.path.join(workdir, "hostutils")
 157     for fname in os.listdir(hostutils):
 158         fpath = os.path.join(hostutils, fname)
 159         if os.path.isdir(fpath) and fname.startswith("host-utils"):
 160             return fpath
 161
 162
 163 def normalize_test_path(test_root, path):
 164     if os.path.isabs(path) or os.path.exists(path):
 165         return os.path.normpath(os.path.abspath(path))
 166
 167     for parent in ancestors(test_root):
 168         test_path = os.path.join(parent, path)
 169         if os.path.exists(test_path):
 170             return os.path.normpath(os.path.abspath(test_path))
 171     # Not a valid path? Return as is and let test harness deal with it
 172     return path
 173
 174
 175 def bootstrap(test_package_root):
 176     test_package_root = os.path.abspath(test_package_root)
 177
 178     sys.path[0:0] = [os.path.join(test_package_root, path) for path in SEARCH_PATHS]
 179     import mach.main
 180
 181     def populate_context(context, key=None):
 182         # These values will be set lazily, and cached after first being invoked.
 183         if key == "package_root":
 184             return test_package_root
 185
 186         if key == "bin_dir":
 187             return os.path.join(test_package_root, "bin")
 188
 189         if key == "certs_dir":
 190             return os.path.join(test_package_root, "certs")
 191
 192         if key == "module_dir":
 193             return os.path.join(test_package_root, "modules")
 194
 195         if key == "ancestors":
 196             return ancestors
 197
 198         if key == "normalize_test_path":
 199             return normalize_test_path
 200
 201         if key == "firefox_bin":
 202             return find_firefox(context)
 203
 204         if key == "hostutils":
 205             return find_hostutils(context)
 206
 207         if key == "mozharness_config":
 208             for dir_path in ancestors(context.package_root):
 209                 mozharness_config = os.path.join(dir_path, "logs", "localconfig.json")
 210                 if os.path.isfile(mozharness_config):
 211                     with open(mozharness_config, "rb") as f:
 212                         return json.load(f)
 213             return {}
 214
 215         if key == "mozharness_workdir":
 216             config = context.mozharness_config
 217             if config:
 218                 return os.path.join(config["base_work_dir"], config["work_dir"])
 219
 220         if key == "activate_mozharness_venv":
 221             return types.MethodType(activate_mozharness_venv, context)
 222
 223     mach = mach.main.Mach(os.getcwd())
 224     mach.populate_context_handler = populate_context
 225
 226     for category, meta in CATEGORIES.items():
 227         mach.define_category(category, meta["short"], meta["long"], meta["priority"])
 228
 229     for path in MACH_MODULES:
 230         cmdfile = os.path.join(test_package_root, path)
 231
 232         # Depending on which test zips were extracted,
 233         # the command module might not exist
 234         if os.path.isfile(cmdfile):
 235             mach.load_commands_from_file(cmdfile)
 236
 237     return mach