remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import argparse
  12 import json
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import tempfile
  19
  20 from collections import OrderedDict
  21
  22 from six import iteritems
  23
  24 from mach.decorators import (
  25     Command,
  26     CommandArgument,
  27     CommandProvider,
  28     SubCommand,
  29 )
  30
  31 from mozbuild.base import (
  32     MachCommandBase,
  33     MozbuildObject,
  34     BinaryNotFoundException,
  35 )
  36 from mozbuild import nodeutil
  37 import mozlog
  38 import mozprofile
  39
  40
  41 EX_CONFIG = 78
  42 EX_SOFTWARE = 70
  43 EX_USAGE = 64
  44
  45
  46 def setup():
  47     # add node and npm from mozbuild to front of system path
  48     npm, _ = nodeutil.find_npm_executable()
  49     if not npm:
  50         exit(EX_CONFIG, "could not find npm executable")
  51     path = os.path.abspath(os.path.join(npm, os.pardir))
  52     os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
  53
  54
  55 @CommandProvider
  56 class RemoteCommands(MachCommandBase):
  57     def __init__(self, *args, **kwargs):
  58         super(RemoteCommands, self).__init__(*args, **kwargs)
  59         self.remotedir = os.path.join(self.topsrcdir, "remote")
  60
  61     @Command("remote", category="misc",
  62              description="Remote protocol related operations.")
  63     def remote(self):
  64         """The remote subcommands all relate to the remote protocol."""
  65         self._sub_mach(['help', 'remote'])
  66         return 1
  67
  68     @SubCommand("remote", "vendor-puppeteer",
  69                 "Pull in latest changes of the Puppeteer client.")
  70     @CommandArgument("--repository",
  71                      metavar="REPO",
  72                      required=True,
  73                      help="The (possibly remote) repository to clone from.")
  74     @CommandArgument("--commitish",
  75                      metavar="COMMITISH",
  76                      required=True,
  77                      help="The commit or tag object name to check out.")
  78     def vendor_puppeteer(self, repository, commitish):
  79         puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
  80
  81         # Preserve our custom mocha reporter
  82         shutil.move(os.path.join(puppeteer_dir, "json-mocha-reporter.js"), self.remotedir)
  83         shutil.rmtree(puppeteer_dir, ignore_errors=True)
  84         os.makedirs(puppeteer_dir)
  85         with TemporaryDirectory() as tmpdir:
  86             git("clone", "-q", repository, tmpdir)
  87             git("checkout", commitish, worktree=tmpdir)
  88             git("checkout-index", "-a", "-f",
  89                 "--prefix", "{}/".format(puppeteer_dir),
  90                 worktree=tmpdir)
  91
  92         # remove files which may interfere with git checkout of central
  93         try:
  94             os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
  95             os.remove(os.path.join(puppeteer_dir, ".gitignore"))
  96         except OSError:
  97             pass
  98
  99         experimental_dir = os.path.join(puppeteer_dir, "experimental")
 100         if os.path.isdir(experimental_dir):
 101             shutil.rmtree(experimental_dir)
 102
 103         shutil.move(os.path.join(self.remotedir, "json-mocha-reporter.js"), puppeteer_dir)
 104
 105         import yaml
 106         annotation = {
 107             "schema": 1,
 108             "bugzilla": {
 109                 "product": "Remote Protocol",
 110                 "component": "Agent",
 111             },
 112             "origin": {
 113                 "name": "puppeteer",
 114                 "description": "Headless Chrome Node API",
 115                 "url": repository,
 116                 "license": "Apache-2.0",
 117                 "release": commitish,
 118             },
 119         }
 120         with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 121             yaml.safe_dump(annotation, fh,
 122                            default_flow_style=False,
 123                            encoding="utf-8",
 124                            allow_unicode=True)
 125
 126
 127 def git(*args, **kwargs):
 128     cmd = ("git",)
 129     if kwargs.get("worktree"):
 130         cmd += ("-C", kwargs["worktree"])
 131     cmd += args
 132
 133     pipe = kwargs.get("pipe")
 134     git_p = subprocess.Popen(cmd,
 135                              env={"GIT_CONFIG_NOSYSTEM": "1"},
 136                              stdout=subprocess.PIPE,
 137                              stderr=subprocess.PIPE)
 138     pipe_p = None
 139     if pipe:
 140         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 141
 142     if pipe:
 143         _, pipe_err = pipe_p.communicate()
 144     out, git_err = git_p.communicate()
 145
 146     # use error from first program that failed
 147     if git_p.returncode > 0:
 148         exit(EX_SOFTWARE, git_err)
 149     if pipe and pipe_p.returncode > 0:
 150         exit(EX_SOFTWARE, pipe_err)
 151
 152     return out
 153
 154
 155 def npm(*args, **kwargs):
 156     from mozprocess import processhandler
 157     env = None
 158     if kwargs.get("env"):
 159         env = os.environ.copy()
 160         env.update(kwargs["env"])
 161
 162     proc_kwargs = {}
 163     if "processOutputLine" in kwargs:
 164         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 165
 166     p = processhandler.ProcessHandler(cmd="npm",
 167                                       args=list(args),
 168                                       cwd=kwargs.get("cwd"),
 169                                       env=env,
 170                                       universal_newlines=True,
 171                                       **proc_kwargs)
 172     if not kwargs.get("wait", True):
 173         return p
 174
 175     wait_proc(p, cmd="npm", exit_on_fail=kwargs.get("exit_on_fail", True))
 176
 177     return p.returncode
 178
 179
 180 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 181     try:
 182         p.run(outputTimeout=output_timeout)
 183         p.wait()
 184         if p.timedOut:
 185             # In some cases, we wait longer for a mocha timeout
 186             print("Timed out after {} seconds of no output".format(output_timeout))
 187     finally:
 188         p.kill()
 189     if exit_on_fail and p.returncode > 0:
 190         msg = ("%s: exit code %s" % (cmd, p.returncode) if cmd
 191                else "exit code %s" % p.returncode)
 192         exit(p.returncode, msg)
 193
 194
 195 class MochaOutputHandler(object):
 196     def __init__(self, logger, expected):
 197         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 198
 199         self.logger = logger
 200         self.proc = None
 201         self.test_results = OrderedDict()
 202         self.expected = expected
 203         self.unexpected_skips = set()
 204
 205         self.has_unexpected = False
 206         self.logger.suite_start([], name="puppeteer-tests")
 207         self.status_map = {
 208             "CRASHED": "CRASH",
 209             "OK": "PASS",
 210             "TERMINATED": "CRASH",
 211             "pass": "PASS",
 212             "fail": "FAIL",
 213             "pending": "SKIP"
 214         }
 215
 216     @property
 217     def pid(self):
 218         return self.proc and self.proc.pid
 219
 220     def __call__(self, line):
 221         event = None
 222         try:
 223             if line.startswith('[') and line.endswith(']'):
 224                 event = json.loads(line)
 225             self.process_event(event)
 226         except ValueError:
 227             pass
 228         finally:
 229             self.logger.process_output(self.pid, line, command="npm")
 230
 231     def process_event(self, event):
 232         if isinstance(event, list) and len(event) > 1:
 233             status = self.status_map.get(event[0])
 234             test_start = event[0] == 'test-start'
 235             if not status and not test_start:
 236                 return
 237             test_info = event[1]
 238             test_name = test_info.get("fullTitle", "")
 239             test_path = test_info.get("file", "")
 240             test_err = test_info.get("err")
 241             if status == "FAIL" and test_err:
 242                 if "timeout" in test_err.lower():
 243                     status = "TIMEOUT"
 244             if test_name and test_path:
 245                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 246             # mocha hook failures are not tracked in metadata
 247             if status != "PASS" and self.hook_re.search(test_name):
 248                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 249                 return
 250             if test_start:
 251                 self.logger.test_start(test_name)
 252                 return
 253             expected = self.expected.get(test_name, ["PASS"])
 254             # mozlog doesn't really allow unexpected skip,
 255             # so if a test is disabled just expect that and note the unexpected skip
 256             # Also, mocha doesn't log test-start for skipped tests
 257             if status == "SKIP":
 258                 self.logger.test_start(test_name)
 259                 if self.expected and status not in expected:
 260                     self.unexpected_skips.add(test_name)
 261                 expected = ["SKIP"]
 262             known_intermittent = expected[1:]
 263             expected_status = expected[0]
 264
 265             self.test_results[test_name] = status
 266             self.logger.test_end(test_name,
 267                                  status=status,
 268                                  expected=expected_status,
 269                                  known_intermittent=known_intermittent)
 270
 271             if status not in expected:
 272                 self.has_unexpected = True
 273
 274     def new_expected(self):
 275         new_expected = OrderedDict()
 276         for test_name, status in iteritems(self.test_results):
 277             if test_name not in self.expected:
 278                 new_status = [status]
 279             else:
 280                 if status in self.expected[test_name]:
 281                     new_status = self.expected[test_name]
 282                 else:
 283                     new_status = [status]
 284             new_expected[test_name] = new_status
 285         return new_expected
 286
 287     def after_end(self, subset=False):
 288         if not subset:
 289             missing = set(self.expected) - set(self.test_results)
 290             extra = set(self.test_results) - set(self.expected)
 291             if missing:
 292                 self.has_unexpected = True
 293                 for test_name in missing:
 294                     self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,))
 295             if self.expected and extra:
 296                 self.has_unexpected = True
 297                 for test_name in extra:
 298                     self.logger.error("TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,))
 299
 300         if self.unexpected_skips:
 301             self.has_unexpected = True
 302             for test_name in self.unexpected_skips:
 303                 self.logger.error("TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,))
 304         self.logger.suite_end()
 305
 306
 307 # tempfile.TemporaryDirectory missing from Python 2.7
 308 class TemporaryDirectory(object):
 309     def __init__(self):
 310         self.path = tempfile.mkdtemp()
 311         self._closed = False
 312
 313     def __repr__(self):
 314         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 315
 316     def __enter__(self):
 317         return self.path
 318
 319     def __exit__(self, exc, value, tb):
 320         self.clean()
 321
 322     def __del__(self):
 323         self.clean()
 324
 325     def clean(self):
 326         if self.path and not self._closed:
 327             shutil.rmtree(self.path)
 328             self._closed = True
 329
 330
 331 class PuppeteerRunner(MozbuildObject):
 332     def __init__(self, *args, **kwargs):
 333         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 334
 335         self.remotedir = os.path.join(self.topsrcdir, "remote")
 336         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 337
 338     def run_test(self, logger, *tests, **params):
 339         """
 340         Runs Puppeteer unit tests with npm.
 341
 342         Possible optional test parameters:
 343
 344         `binary`:
 345           Path for the browser binary to use.  Defaults to the local
 346           build.
 347         `headless`:
 348           Boolean to indicate whether to activate Firefox' headless mode.
 349         `extra_prefs`:
 350           Dictionary of extra preferences to write to the profile,
 351           before invoking npm.  Overrides default preferences.
 352         `write_results`:
 353           Path to write the results json file
 354         `subset`
 355           Indicates only a subset of tests are being run, so we should
 356           skip the check for missing results
 357         """
 358         setup()
 359
 360         binary = params.get("binary") or self.get_binary_path()
 361         product = params.get("product", "firefox")
 362
 363         env = {
 364             # Print browser process ouptut
 365             "DUMPIO": "1",
 366             # Checked by Puppeteer's custom mocha config
 367             "CI": "1",
 368             # Causes some tests to be skipped due to assumptions about install
 369             "PUPPETEER_ALT_INSTALL": "1"
 370         }
 371         extra_options = {}
 372         for k, v in params.get("extra_launcher_options", {}).items():
 373             extra_options[k] = json.loads(v)
 374
 375         # Override upstream defaults: no retries, shorter timeout
 376         mocha_options = [
 377             "--reporter", "./json-mocha-reporter.js",
 378             "--retries", "0",
 379             "--fullTrace",
 380             "--timeout", "15000",
 381             "--no-parallel",
 382         ]
 383         if product == "firefox":
 384             env["BINARY"] = binary
 385             env["PUPPETEER_PRODUCT"] = "firefox"
 386         command = ["run", "unit", "--"] + mocha_options
 387
 388         env["HEADLESS"] = str(params.get("headless", False))
 389
 390         prefs = {}
 391         for k, v in params.get("extra_prefs", {}).items():
 392             prefs[k] = mozprofile.Preferences.cast(v)
 393
 394         if prefs:
 395             extra_options["extraPrefsFirefox"] = prefs
 396
 397         if extra_options:
 398             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 399
 400         expected_path = os.path.join(os.path.dirname(__file__),
 401                                      "puppeteer-expected.json")
 402         if product == "firefox" and os.path.exists(expected_path):
 403             with open(expected_path) as f:
 404                 expected_data = json.load(f)
 405         else:
 406             expected_data = {}
 407
 408         output_handler = MochaOutputHandler(logger, expected_data)
 409         proc = npm(*command, cwd=self.puppeteer_dir, env=env,
 410                    processOutputLine=output_handler, wait=False)
 411         output_handler.proc = proc
 412
 413         # Puppeteer unit tests don't always clean-up child processes in case of
 414         # failure, so use an output_timeout as a fallback
 415         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 416
 417         output_handler.after_end(params.get("subset", False))
 418
 419         # Non-zero return codes are non-fatal for now since we have some
 420         # issues with unresolved promises that shouldn't otherwise block
 421         # running the tests
 422         if proc.returncode != 0:
 423             logger.warning("npm exited with code %s" % proc.returncode)
 424
 425         if params["write_results"]:
 426             with open(params["write_results"], "w") as f:
 427                 json.dump(output_handler.new_expected(), f, indent=2,
 428                           separators=(",", ": "))
 429
 430         if output_handler.has_unexpected:
 431             exit(1, "Got unexpected results")
 432
 433
 434 def create_parser_puppeteer():
 435     p = argparse.ArgumentParser()
 436     p.add_argument("--product",
 437                    type=str,
 438                    default="firefox",
 439                    choices=["chrome", "firefox"])
 440     p.add_argument("--binary",
 441                    type=str,
 442                    help="Path to browser binary.  Defaults to local Firefox build.")
 443     p.add_argument("--enable-fission",
 444                    action="store_true",
 445                    help="Enable Fission (site isolation) in Gecko.")
 446     p.add_argument("-z", "--headless",
 447                    action="store_true",
 448                    help="Run browser in headless mode.")
 449     p.add_argument("--setpref",
 450                    action="append",
 451                    dest="extra_prefs",
 452                    metavar="<pref>=<value>",
 453                    help="Defines additional user preferences.")
 454     p.add_argument("--setopt",
 455                    action="append",
 456                    dest="extra_options",
 457                    metavar="<option>=<value>",
 458                    help="Defines additional options for `puppeteer.launch`.")
 459     p.add_argument("-v",
 460                    dest="verbosity",
 461                    action="count",
 462                    default=0,
 463                    help="Increase remote agent logging verbosity to include "
 464                         "debug level messages with -v, trace messages with -vv,"
 465                         "and to not truncate long trace messages with -vvv")
 466     p.add_argument("--write-results",
 467                    action="store",
 468                    nargs="?",
 469                    default=None,
 470                    const=os.path.join(os.path.dirname(__file__),
 471                                       "puppeteer-expected.json"),
 472                    help="Path to write updated results to (defaults to the "
 473                         "expectations file if the argument is provided but "
 474                         "no path is passed)")
 475     p.add_argument("--subset",
 476                    action="store_true",
 477                    default=False,
 478                    help="Indicate that only a subset of the tests are running, "
 479                         "so checks for missing tests should be skipped")
 480     p.add_argument("tests", nargs="*")
 481     mozlog.commandline.add_logging_group(p)
 482     return p
 483
 484
 485 @CommandProvider
 486 class PuppeteerTest(MachCommandBase):
 487     @Command("puppeteer-test", category="testing",
 488              description="Run Puppeteer unit tests.",
 489              parser=create_parser_puppeteer)
 490     def puppeteer_test(self, binary=None, enable_fission=False, headless=False,
 491                        extra_prefs=None, extra_options=None, verbosity=0,
 492                        tests=None, product="firefox", write_results=None,
 493                        subset=False, **kwargs):
 494
 495         logger = mozlog.commandline.setup_logging("puppeteer-test",
 496                                                   kwargs,
 497                                                   {"mach": sys.stdout})
 498
 499         # moztest calls this programmatically with test objects or manifests
 500         if "test_objects" in kwargs and tests is not None:
 501             logger.error("Expected either 'test_objects' or 'tests'")
 502             exit(1)
 503
 504         if product != "firefox" and extra_prefs is not None:
 505             logger.error("User preferences are not recognized by %s" % product)
 506             exit(1)
 507
 508         if "test_objects" in kwargs:
 509             tests = []
 510             for test in kwargs["test_objects"]:
 511                 tests.append(test["path"])
 512
 513         prefs = {}
 514         for s in (extra_prefs or []):
 515             kv = s.split("=")
 516             if len(kv) != 2:
 517                 logger.error("syntax error in --setpref={}".format(s))
 518                 exit(EX_USAGE)
 519             prefs[kv[0]] = kv[1].strip()
 520
 521         options = {}
 522         for s in (extra_options or []):
 523             kv = s.split("=")
 524             if len(kv) != 2:
 525                 logger.error("syntax error in --setopt={}".format(s))
 526                 exit(EX_USAGE)
 527             options[kv[0]] = kv[1].strip()
 528
 529         if enable_fission:
 530             prefs.update({"fission.autostart": True,
 531                           "dom.serviceWorkers.parent_intercept": True})
 532
 533         if verbosity == 1:
 534             prefs["remote.log.level"] = "Debug"
 535         elif verbosity > 1:
 536             prefs["remote.log.level"] = "Trace"
 537         if verbosity > 2:
 538             prefs["remote.log.truncate"] = False
 539
 540         self.install_puppeteer(product)
 541
 542         params = {"binary": binary,
 543                   "headless": headless,
 544                   "extra_prefs": prefs,
 545                   "product": product,
 546                   "extra_launcher_options": options,
 547                   "write_results": write_results,
 548                   "subset": subset}
 549         puppeteer = self._spawn(PuppeteerRunner)
 550         try:
 551             return puppeteer.run_test(logger, *tests, **params)
 552         except BinaryNotFoundException as e:
 553             logger.error(e)
 554             logger.info(e.help())
 555             exit(1)
 556         except Exception as e:
 557             exit(EX_SOFTWARE, e)
 558
 559     def install_puppeteer(self, product):
 560         setup()
 561         env = {}
 562         from mozversioncontrol import get_repository_object
 563         repo = get_repository_object(self.topsrcdir)
 564         puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 565         changed_files = False
 566         for f in repo.get_changed_files():
 567             if f.startswith(puppeteer_dir) and f.endswith(".ts"):
 568                 changed_files = True
 569                 break
 570
 571         if product != "chrome":
 572             env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 573         lib_dir = os.path.join(self.topsrcdir, puppeteer_dir, "lib")
 574         if changed_files and os.path.isdir(lib_dir):
 575             # clobber lib to force `tsc compile` step
 576             shutil.rmtree(lib_dir)
 577         npm("install",
 578             cwd=os.path.join(self.topsrcdir, puppeteer_dir),
 579             env=env)
 580
 581
 582 def exit(code, error=None):
 583     if error is not None:
 584         if isinstance(error, Exception):
 585             import traceback
 586             traceback.print_exc()
 587         else:
 588             message = str(error).split("\n")[0].strip()
 589             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 590     sys.exit(code)