remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import sys
  12 import os
  13 import tempfile
  14 import shutil
  15 import subprocess
  16
  17 from mach.decorators import (
  18     Command,
  19     CommandArgument,
  20     CommandProvider,
  21     SubCommand,
  22 )
  23
  24 from mozbuild.base import (
  25     MachCommandBase,
  26     MozbuildObject,
  27 )
  28 from mozbuild import nodeutil
  29 import mozprofile
  30
  31
  32 EX_CONFIG = 78
  33 EX_SOFTWARE = 70
  34 EX_USAGE = 64
  35
  36 DEFAULT_REPO = "https://github.com/andreastt/puppeteer.git"
  37 DEFAULT_COMMITISH = "firefox"
  38
  39
  40 def setup():
  41     # add node and npm from mozbuild to front of system path
  42     npm, _ = nodeutil.find_npm_executable()
  43     if not npm:
  44         exit(EX_CONFIG, "could not find npm executable")
  45     path = os.path.abspath(os.path.join(npm, os.pardir))
  46     os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
  47
  48
  49 @CommandProvider
  50 class RemoteCommands(MachCommandBase):
  51     def __init__(self, context):
  52         MachCommandBase.__init__(self, context)
  53         self.remotedir = os.path.join(self.topsrcdir, "remote")
  54
  55     @Command("remote", category="misc",
  56              description="Remote protocol related operations.")
  57     def remote(self):
  58         self.parser.print_usage()
  59         exit(EX_USAGE)
  60
  61     @SubCommand("remote", "vendor-puppeteer",
  62                 "Pull in latest changes of the Puppeteer client.")
  63     @CommandArgument("--repository",
  64                      metavar="REPO",
  65                      default=DEFAULT_REPO,
  66                      help="The (possibly remote) repository to clone from. "
  67                           "Defaults to {}.".format(DEFAULT_REPO))
  68     @CommandArgument("--commitish",
  69                      metavar="COMMITISH",
  70                      default=DEFAULT_COMMITISH,
  71                      help="The commit or tag object name to check out. "
  72                           "Defaults to \"{}\".".format(DEFAULT_COMMITISH))
  73     def vendor_puppeteer(self, repository, commitish):
  74         puppeteerdir = os.path.join(self.remotedir, "test", "puppeteer")
  75
  76         shutil.rmtree(puppeteerdir, ignore_errors=True)
  77         os.makedirs(puppeteerdir)
  78         with TemporaryDirectory() as tmpdir:
  79             git("clone", "-q", repository, tmpdir)
  80             git("checkout", commitish, worktree=tmpdir)
  81             git("checkout-index", "-a", "-f",
  82                 "--prefix", "{}/".format(puppeteerdir),
  83                 worktree=tmpdir)
  84
  85         # remove files which may interfere with git checkout of central
  86         try:
  87             os.remove(os.path.join(puppeteerdir, ".gitattributes"))
  88             os.remove(os.path.join(puppeteerdir, ".gitignore"))
  89         except OSError:
  90             pass
  91
  92         import yaml
  93         annotation = {
  94             "schema": 1,
  95             "bugzilla": {
  96                 "product": "Remote Protocol",
  97                 "component": "Agent",
  98             },
  99             "origin": {
 100                 "name": "puppeteer",
 101                 "description": "Headless Chrome Node API",
 102                 "url": repository,
 103                 "license": "Apache-2.0",
 104                 "release": commitish,
 105             },
 106         }
 107         with open(os.path.join(puppeteerdir, "moz.yaml"), "w") as fh:
 108             yaml.safe_dump(annotation, fh,
 109                            default_flow_style=False,
 110                            encoding="utf-8",
 111                            allow_unicode=True)
 112
 113
 114 def git(*args, **kwargs):
 115     cmd = ("git",)
 116     if kwargs.get("worktree"):
 117         cmd += ("-C", kwargs["worktree"])
 118     cmd += args
 119
 120     pipe = kwargs.get("pipe")
 121     git_p = subprocess.Popen(cmd,
 122                              env={"GIT_CONFIG_NOSYSTEM": "1"},
 123                              stdout=subprocess.PIPE,
 124                              stderr=subprocess.PIPE)
 125     pipe_p = None
 126     if pipe:
 127         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 128
 129     if pipe:
 130         _, pipe_err = pipe_p.communicate()
 131     out, git_err = git_p.communicate()
 132
 133     # use error from first program that failed
 134     if git_p.returncode > 0:
 135         exit(EX_SOFTWARE, git_err)
 136     if pipe and pipe_p.returncode > 0:
 137         exit(EX_SOFTWARE, pipe_err)
 138
 139     return out
 140
 141
 142 def npm(*args, **kwargs):
 143     env = None
 144     if kwargs.get("env"):
 145         env = os.environ.copy()
 146         env.update(kwargs["env"])
 147
 148     p = subprocess.Popen(("npm",) + args,
 149                          cwd=kwargs.get("cwd"),
 150                          env=env)
 151
 152     p.wait()
 153     if p.returncode > 0:
 154         exit(p.returncode, "npm: exit code {}".format(p.returncode))
 155
 156
 157 # tempfile.TemporaryDirectory missing from Python 2.7
 158 class TemporaryDirectory(object):
 159     def __init__(self):
 160         self.path = tempfile.mkdtemp()
 161         self._closed = False
 162
 163     def __repr__(self):
 164         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 165
 166     def __enter__(self):
 167         return self.path
 168
 169     def __exit__(self, exc, value, tb):
 170         self.clean()
 171
 172     def __del__(self):
 173         self.clean()
 174
 175     def clean(self):
 176         if self.path and not self._closed:
 177             shutil.rmtree(self.path)
 178             self._closed = True
 179
 180
 181 class PuppeteerRunner(MozbuildObject):
 182     def __init__(self, *args, **kwargs):
 183         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 184
 185         self.profile = mozprofile.Profile()
 186
 187         self.remotedir = os.path.join(self.topsrcdir, "remote")
 188         self.puppeteerdir = os.path.join(self.remotedir, "test", "puppeteer")
 189
 190     def run_test(self, *tests, **params):
 191         """
 192         Runs Puppeteer unit tests with npm.
 193
 194         Possible optional test parameters:
 195
 196         `binary`:
 197           Path for the browser binary to use.  Defaults to the local
 198           build.
 199         `jobs`:
 200           Number of tests to run in parallel.  Defaults to not
 201           parallelise, e.g. `-j1`.
 202         `headless`:
 203           Boolean to indicate whether to activate Firefox' headless mode.
 204         `extra_prefs`:
 205           Dictionary of extra preferences to write to the profile,
 206           before invoking npm.  Overrides default preferences.
 207         """
 208         setup()
 209
 210         binary = params.get("binary") or self.get_binary_path()
 211
 212         # currently runs against puppeteer-chrome
 213         # but future intention is to run against puppeteer-firefox
 214         # when it targets the Mozilla remote agent instead of Juggler
 215         env = {"CHROME": binary,
 216                "DUMPIO": "1"}
 217
 218         if params.get("jobs"):
 219             env["PPTR_PARALLEL_TESTS"] = str(params["jobs"])
 220
 221         if params.get("headless"):
 222             env["MOZ_HEADLESS"] = "1"
 223
 224         prefs = params.get("extra_prefs", {})
 225         for k, v in params.get("extra_prefs", {}).items():
 226             prefs[k] = mozprofile.Preferences.cast(v)
 227
 228         prefs.update({
 229             # https://bugzilla.mozilla.org/show_bug.cgi?id=1544393
 230             "remote.enabled": True,
 231             # https://bugzilla.mozilla.org/show_bug.cgi?id=1543115
 232             "browser.dom.window.dump.enabled": True,
 233         })
 234
 235         self.profile.set_preferences(prefs)
 236
 237         # PROFILE is a Puppeteer workaround (see ab302d6)
 238         # for passing the --profile flag to Firefox
 239         env["PROFILE"] = self.profile.profile
 240
 241         return npm("run", "unit", "--verbose", *tests,
 242                    cwd=self.puppeteerdir,
 243                    env=env)
 244
 245
 246 @CommandProvider
 247 class PuppeteerTest(MachCommandBase):
 248     @Command("puppeteer-test", category="testing",
 249              description="Run Puppeteer unit tests.")
 250     @CommandArgument("--binary",
 251                      type=str,
 252                      help="Path to Firefox binary.  Defaults to local build.")
 253     @CommandArgument("-z", "--headless",
 254                      action="store_true",
 255                      help="Run browser in headless mode (default).")
 256     @CommandArgument("--setpref",
 257                      action="append",
 258                      dest="extra_prefs",
 259                      metavar="<pref>=<value>",
 260                      help="Defines additional user preferences.")
 261     @CommandArgument("-j",
 262                      dest="jobs",
 263                      type=int,
 264                      metavar="<N>",
 265                      help="Optionally run tests in parallel.")
 266     @CommandArgument("tests", nargs="*")
 267     def puppeteer_test(self, binary=None, headless=True, extra_prefs=None,
 268                        jobs=1, tests=None, **kwargs):
 269         # moztest calls this programmatically with test objects or manifests
 270         if "test_objects" in kwargs and tests is not None:
 271             raise ValueError("Expected either 'test_objects' or 'tests'")
 272
 273         if "test_objects" in kwargs:
 274             tests = []
 275             for test in kwargs["test_objects"]:
 276                 tests.append(test["path"])
 277
 278         prefs = {}
 279         for s in (extra_prefs or []):
 280             kv = s.split("=")
 281             if len(kv) != 2:
 282                 exit(EX_USAGE, "syntax error in --setpref={}".format(s))
 283             prefs[kv[0]] = kv[1].strip()
 284
 285         self.install_puppeteer()
 286
 287         params = {"binary": binary,
 288                   "headless": headless,
 289                   "extra_prefs": prefs,
 290                   "jobs": jobs}
 291         puppeteer = self._spawn(PuppeteerRunner)
 292         try:
 293             return puppeteer.run_test(*tests, **params)
 294         except Exception as e:
 295             exit(EX_SOFTWARE, e)
 296
 297     def install_puppeteer(self):
 298         setup()
 299         npm("install",
 300             cwd=os.path.join(self.topsrcdir, "remote", "test", "puppeteer"),
 301             env={"PUPPETEER_SKIP_CHROMIUM_DOWNLOAD": "1"})
 302
 303
 304 def exit(code, error=None):
 305     if error is not None:
 306         if isinstance(error, Exception):
 307             import traceback
 308             traceback.print_exc()
 309         else:
 310             message = str(error).split("\n")[0].strip()
 311             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 312     sys.exit(code)