remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import json
  12 import sys
  13 import os
  14 import tempfile
  15 import shutil
  16 import subprocess
  17
  18 from mach.decorators import (
  19     Command,
  20     CommandArgument,
  21     CommandProvider,
  22     SubCommand,
  23 )
  24
  25 from mozbuild.base import (
  26     MachCommandBase,
  27     MozbuildObject,
  28 )
  29 from mozbuild import nodeutil
  30 import mozprofile
  31
  32
  33 EX_CONFIG = 78
  34 EX_SOFTWARE = 70
  35 EX_USAGE = 64
  36
  37 DEFAULT_REPO = "https://github.com/andreastt/puppeteer.git"
  38 DEFAULT_COMMITISH = "firefox"
  39
  40
  41 def setup():
  42     # add node and npm from mozbuild to front of system path
  43     npm, _ = nodeutil.find_npm_executable()
  44     if not npm:
  45         exit(EX_CONFIG, "could not find npm executable")
  46     path = os.path.abspath(os.path.join(npm, os.pardir))
  47     os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
  48
  49
  50 @CommandProvider
  51 class RemoteCommands(MachCommandBase):
  52     def __init__(self, context):
  53         MachCommandBase.__init__(self, context)
  54         self.remotedir = os.path.join(self.topsrcdir, "remote")
  55
  56     @Command("remote", category="misc",
  57              description="Remote protocol related operations.")
  58     def remote(self):
  59         self.parser.print_usage()
  60         exit(EX_USAGE)
  61
  62     @SubCommand("remote", "vendor-puppeteer",
  63                 "Pull in latest changes of the Puppeteer client.")
  64     @CommandArgument("--repository",
  65                      metavar="REPO",
  66                      default=DEFAULT_REPO,
  67                      help="The (possibly remote) repository to clone from. "
  68                           "Defaults to {}.".format(DEFAULT_REPO))
  69     @CommandArgument("--commitish",
  70                      metavar="COMMITISH",
  71                      default=DEFAULT_COMMITISH,
  72                      help="The commit or tag object name to check out. "
  73                           "Defaults to \"{}\".".format(DEFAULT_COMMITISH))
  74     def vendor_puppeteer(self, repository, commitish):
  75         puppeteerdir = os.path.join(self.remotedir, "test", "puppeteer")
  76
  77         shutil.rmtree(puppeteerdir, ignore_errors=True)
  78         os.makedirs(puppeteerdir)
  79         with TemporaryDirectory() as tmpdir:
  80             git("clone", "-q", repository, tmpdir)
  81             git("checkout", commitish, worktree=tmpdir)
  82             git("checkout-index", "-a", "-f",
  83                 "--prefix", "{}/".format(puppeteerdir),
  84                 worktree=tmpdir)
  85
  86         # remove files which may interfere with git checkout of central
  87         try:
  88             os.remove(os.path.join(puppeteerdir, ".gitattributes"))
  89             os.remove(os.path.join(puppeteerdir, ".gitignore"))
  90         except OSError:
  91             pass
  92
  93         import yaml
  94         annotation = {
  95             "schema": 1,
  96             "bugzilla": {
  97                 "product": "Remote Protocol",
  98                 "component": "Agent",
  99             },
 100             "origin": {
 101                 "name": "puppeteer",
 102                 "description": "Headless Chrome Node API",
 103                 "url": repository,
 104                 "license": "Apache-2.0",
 105                 "release": commitish,
 106             },
 107         }
 108         with open(os.path.join(puppeteerdir, "moz.yaml"), "w") as fh:
 109             yaml.safe_dump(annotation, fh,
 110                            default_flow_style=False,
 111                            encoding="utf-8",
 112                            allow_unicode=True)
 113
 114
 115 def git(*args, **kwargs):
 116     cmd = ("git",)
 117     if kwargs.get("worktree"):
 118         cmd += ("-C", kwargs["worktree"])
 119     cmd += args
 120
 121     pipe = kwargs.get("pipe")
 122     git_p = subprocess.Popen(cmd,
 123                              env={"GIT_CONFIG_NOSYSTEM": "1"},
 124                              stdout=subprocess.PIPE,
 125                              stderr=subprocess.PIPE)
 126     pipe_p = None
 127     if pipe:
 128         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 129
 130     if pipe:
 131         _, pipe_err = pipe_p.communicate()
 132     out, git_err = git_p.communicate()
 133
 134     # use error from first program that failed
 135     if git_p.returncode > 0:
 136         exit(EX_SOFTWARE, git_err)
 137     if pipe and pipe_p.returncode > 0:
 138         exit(EX_SOFTWARE, pipe_err)
 139
 140     return out
 141
 142
 143 def npm(*args, **kwargs):
 144     env = None
 145     if kwargs.get("env"):
 146         env = os.environ.copy()
 147         env.update(kwargs["env"])
 148
 149     p = subprocess.Popen(("npm",) + args,
 150                          cwd=kwargs.get("cwd"),
 151                          env=env)
 152
 153     p.wait()
 154     if p.returncode > 0:
 155         exit(p.returncode, "npm: exit code {}".format(p.returncode))
 156
 157
 158 # tempfile.TemporaryDirectory missing from Python 2.7
 159 class TemporaryDirectory(object):
 160     def __init__(self):
 161         self.path = tempfile.mkdtemp()
 162         self._closed = False
 163
 164     def __repr__(self):
 165         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 166
 167     def __enter__(self):
 168         return self.path
 169
 170     def __exit__(self, exc, value, tb):
 171         self.clean()
 172
 173     def __del__(self):
 174         self.clean()
 175
 176     def clean(self):
 177         if self.path and not self._closed:
 178             shutil.rmtree(self.path)
 179             self._closed = True
 180
 181
 182 class PuppeteerRunner(MozbuildObject):
 183     def __init__(self, *args, **kwargs):
 184         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 185
 186         self.remotedir = os.path.join(self.topsrcdir, "remote")
 187         self.puppeteerdir = os.path.join(self.remotedir, "test", "puppeteer")
 188
 189     def run_test(self, *tests, **params):
 190         """
 191         Runs Puppeteer unit tests with npm.
 192
 193         Possible optional test parameters:
 194
 195         `binary`:
 196           Path for the browser binary to use.  Defaults to the local
 197           build.
 198         `jobs`:
 199           Number of tests to run in parallel.  Defaults to not
 200           parallelise, e.g. `-j1`.
 201         `headless`:
 202           Boolean to indicate whether to activate Firefox' headless mode.
 203         `extra_prefs`:
 204           Dictionary of extra preferences to write to the profile,
 205           before invoking npm.  Overrides default preferences.
 206         """
 207         setup()
 208
 209         binary = params.get("binary") or self.get_binary_path()
 210         product = params.get("product", "firefox")
 211
 212         env = {"DUMPIO": "1"}
 213         extra_options = {}
 214         for k, v in params.get("extra_launcher_options", {}).items():
 215             extra_options[k] = json.loads(v)
 216
 217         if product == "firefox":
 218             env["BINARY"] = binary
 219             command = ["run", "funit", "--verbose"]
 220         elif product == "chrome":
 221             command = ["run", "unit", "--verbose"]
 222
 223         if params.get("jobs"):
 224             env["PPTR_PARALLEL_TESTS"] = str(params["jobs"])
 225         env["HEADLESS"] = str(params.get("headless", False))
 226
 227         prefs = {}
 228         for k, v in params.get("extra_prefs", {}).items():
 229             prefs[k] = mozprofile.Preferences.cast(v)
 230
 231         if prefs:
 232             extra_options["extraPrefsFirefox"] = prefs
 233
 234         if extra_options:
 235             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 236
 237         return npm(*command, cwd=self.puppeteerdir, env=env)
 238
 239
 240 @CommandProvider
 241 class PuppeteerTest(MachCommandBase):
 242     @Command("puppeteer-test", category="testing",
 243              description="Run Puppeteer unit tests.")
 244     @CommandArgument("--product",
 245                      type=str,
 246                      default="firefox",
 247                      choices=["chrome", "firefox"])
 248     @CommandArgument("--binary",
 249                      type=str,
 250                      help="Path to browser binary.  Defaults to local Firefox build.")
 251     @CommandArgument("--enable-fission",
 252                      action="store_true",
 253                      help="Enable Fission (site isolation) in Gecko.")
 254     @CommandArgument("-z", "--headless",
 255                      action="store_true",
 256                      help="Run browser in headless mode.")
 257     @CommandArgument("--setpref",
 258                      action="append",
 259                      dest="extra_prefs",
 260                      metavar="<pref>=<value>",
 261                      help="Defines additional user preferences.")
 262     @CommandArgument("--setopt",
 263                      action="append",
 264                      dest="extra_options",
 265                      metavar="<option>=<value>",
 266                      help="Defines additional options for `puppeteer.launch`.")
 267     @CommandArgument("-j",
 268                      dest="jobs",
 269                      type=int,
 270                      metavar="<N>",
 271                      help="Optionally run tests in parallel.")
 272     @CommandArgument("-v",
 273                      dest="verbosity",
 274                      action="count",
 275                      default=0,
 276                      help="Increase remote agent logging verbosity to include "
 277                           "debug level messages with -v, and trace messages with -vv.")
 278     @CommandArgument("tests", nargs="*")
 279     def puppeteer_test(self, binary=None, enable_fission=False, headless=False,
 280                        extra_prefs=None, extra_options=None, jobs=1, verbosity=0,
 281                        tests=None, product="firefox", **kwargs):
 282         # moztest calls this programmatically with test objects or manifests
 283         if "test_objects" in kwargs and tests is not None:
 284             raise ValueError("Expected either 'test_objects' or 'tests'")
 285
 286         if product != "firefox" and extra_prefs is not None:
 287             raise ValueError("User preferences are not recognized by %s" % product)
 288
 289         if "test_objects" in kwargs:
 290             tests = []
 291             for test in kwargs["test_objects"]:
 292                 tests.append(test["path"])
 293
 294         prefs = {}
 295         for s in (extra_prefs or []):
 296             kv = s.split("=")
 297             if len(kv) != 2:
 298                 exit(EX_USAGE, "syntax error in --setpref={}".format(s))
 299             prefs[kv[0]] = kv[1].strip()
 300
 301         options = {}
 302         for s in (extra_options or []):
 303             kv = s.split("=")
 304             if len(kv) != 2:
 305                 exit(EX_USAGE, "syntax error in --setopt={}".format(s))
 306             options[kv[0]] = kv[1].strip()
 307
 308         if enable_fission:
 309             prefs.update({"fission.autostart": True,
 310                           "dom.serviceWorkers.parent_intercept": True,
 311                           "browser.tabs.documentchannel": True})
 312
 313         if verbosity == 1:
 314             prefs["remote.log.level"] = "Debug"
 315         elif verbosity > 1:
 316             prefs["remote.log.level"] = "Trace"
 317
 318         self.install_puppeteer(product)
 319
 320         params = {"binary": binary,
 321                   "headless": headless,
 322                   "extra_prefs": prefs,
 323                   "product": product,
 324                   "jobs": jobs,
 325                   "extra_launcher_options": options}
 326         puppeteer = self._spawn(PuppeteerRunner)
 327         try:
 328             return puppeteer.run_test(*tests, **params)
 329         except Exception as e:
 330             exit(EX_SOFTWARE, e)
 331
 332     def install_puppeteer(self, product):
 333         setup()
 334         env = {}
 335         if product != "chrome":
 336             env["PUPPETEER_SKIP_CHROMIUM_DOWNLOAD"] = "1"
 337         npm("install",
 338             cwd=os.path.join(self.topsrcdir, "remote", "test", "puppeteer"),
 339             env=env)
 340
 341
 342 def exit(code, error=None):
 343     if error is not None:
 344         if isinstance(error, Exception):
 345             import traceback
 346             traceback.print_exc()
 347         else:
 348             message = str(error).split("\n")[0].strip()
 349             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 350     sys.exit(code)