remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import argparse
  12 import json
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import tempfile
  19
  20 from collections import OrderedDict
  21
  22 from six import iteritems
  23
  24 from mach.decorators import (
  25     Command,
  26     CommandArgument,
  27     CommandProvider,
  28     SubCommand,
  29 )
  30
  31 from mozbuild.base import (
  32     MachCommandBase,
  33     MozbuildObject,
  34     BinaryNotFoundException,
  35 )
  36 from mozbuild import nodeutil
  37 import mozlog
  38 import mozprofile
  39
  40
  41 EX_CONFIG = 78
  42 EX_SOFTWARE = 70
  43 EX_USAGE = 64
  44
  45
  46 def setup():
  47     # add node and npm from mozbuild to front of system path
  48     npm, _ = nodeutil.find_npm_executable()
  49     if not npm:
  50         exit(EX_CONFIG, "could not find npm executable")
  51     path = os.path.abspath(os.path.join(npm, os.pardir))
  52     os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
  53
  54
  55 @CommandProvider
  56 class RemoteCommands(MachCommandBase):
  57     def __init__(self, *args, **kwargs):
  58         super(RemoteCommands, self).__init__(*args, **kwargs)
  59         self.remotedir = os.path.join(self.topsrcdir, "remote")
  60
  61     @Command(
  62         "remote", category="misc", description="Remote protocol related operations."
  63     )
  64     def remote(self):
  65         """The remote subcommands all relate to the remote protocol."""
  66         self._sub_mach(["help", "remote"])
  67         return 1
  68
  69     @SubCommand(
  70         "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
  71     )
  72     @CommandArgument(
  73         "--repository",
  74         metavar="REPO",
  75         required=True,
  76         help="The (possibly remote) repository to clone from.",
  77     )
  78     @CommandArgument(
  79         "--commitish",
  80         metavar="COMMITISH",
  81         required=True,
  82         help="The commit or tag object name to check out.",
  83     )
  84     @CommandArgument(
  85         "--no-install",
  86         dest="install",
  87         action="store_false",
  88         default=True,
  89         help="Do not install the just-pulled Puppeteer package,",
  90     )
  91     def vendor_puppeteer(self, repository, commitish, install):
  92         puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
  93
  94         # Preserve our custom mocha reporter
  95         shutil.move(
  96             os.path.join(puppeteer_dir, "json-mocha-reporter.js"), self.remotedir
  97         )
  98         shutil.rmtree(puppeteer_dir, ignore_errors=True)
  99         os.makedirs(puppeteer_dir)
 100         with TemporaryDirectory() as tmpdir:
 101             git("clone", "-q", repository, tmpdir)
 102             git("checkout", commitish, worktree=tmpdir)
 103             git(
 104                 "checkout-index",
 105                 "-a",
 106                 "-f",
 107                 "--prefix",
 108                 "{}/".format(puppeteer_dir),
 109                 worktree=tmpdir,
 110             )
 111
 112         # remove files which may interfere with git checkout of central
 113         try:
 114             os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
 115             os.remove(os.path.join(puppeteer_dir, ".gitignore"))
 116         except OSError:
 117             pass
 118
 119         unwanted_dirs = ["experimental", "docs"]
 120
 121         for dir in unwanted_dirs:
 122             dir_path = os.path.join(puppeteer_dir, dir)
 123             if os.path.isdir(dir_path):
 124                 shutil.rmtree(dir_path)
 125
 126         shutil.move(
 127             os.path.join(self.remotedir, "json-mocha-reporter.js"), puppeteer_dir
 128         )
 129
 130         import yaml
 131
 132         annotation = {
 133             "schema": 1,
 134             "bugzilla": {
 135                 "product": "Remote Protocol",
 136                 "component": "Agent",
 137             },
 138             "origin": {
 139                 "name": "puppeteer",
 140                 "description": "Headless Chrome Node API",
 141                 "url": repository,
 142                 "license": "Apache-2.0",
 143                 "release": commitish,
 144             },
 145         }
 146         with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 147             yaml.safe_dump(
 148                 annotation,
 149                 fh,
 150                 default_flow_style=False,
 151                 encoding="utf-8",
 152                 allow_unicode=True,
 153             )
 154
 155         if install:
 156             env = {"PUPPETEER_SKIP_DOWNLOAD": "1"}
 157             npm("install", cwd=os.path.join(self.topsrcdir, puppeteer_dir), env=env)
 158
 159
 160 def git(*args, **kwargs):
 161     cmd = ("git",)
 162     if kwargs.get("worktree"):
 163         cmd += ("-C", kwargs["worktree"])
 164     cmd += args
 165
 166     pipe = kwargs.get("pipe")
 167     git_p = subprocess.Popen(
 168         cmd,
 169         env={"GIT_CONFIG_NOSYSTEM": "1"},
 170         stdout=subprocess.PIPE,
 171         stderr=subprocess.PIPE,
 172     )
 173     pipe_p = None
 174     if pipe:
 175         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 176
 177     if pipe:
 178         _, pipe_err = pipe_p.communicate()
 179     out, git_err = git_p.communicate()
 180
 181     # use error from first program that failed
 182     if git_p.returncode > 0:
 183         exit(EX_SOFTWARE, git_err)
 184     if pipe and pipe_p.returncode > 0:
 185         exit(EX_SOFTWARE, pipe_err)
 186
 187     return out
 188
 189
 190 def npm(*args, **kwargs):
 191     from mozprocess import processhandler
 192
 193     env = None
 194     if kwargs.get("env"):
 195         env = os.environ.copy()
 196         env.update(kwargs["env"])
 197
 198     proc_kwargs = {}
 199     if "processOutputLine" in kwargs:
 200         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 201
 202     p = processhandler.ProcessHandler(
 203         cmd="npm",
 204         args=list(args),
 205         cwd=kwargs.get("cwd"),
 206         env=env,
 207         universal_newlines=True,
 208         **proc_kwargs
 209     )
 210     if not kwargs.get("wait", True):
 211         return p
 212
 213     wait_proc(p, cmd="npm", exit_on_fail=kwargs.get("exit_on_fail", True))
 214
 215     return p.returncode
 216
 217
 218 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 219     try:
 220         p.run(outputTimeout=output_timeout)
 221         p.wait()
 222         if p.timedOut:
 223             # In some cases, we wait longer for a mocha timeout
 224             print("Timed out after {} seconds of no output".format(output_timeout))
 225     finally:
 226         p.kill()
 227     if exit_on_fail and p.returncode > 0:
 228         msg = (
 229             "%s: exit code %s" % (cmd, p.returncode)
 230             if cmd
 231             else "exit code %s" % p.returncode
 232         )
 233         exit(p.returncode, msg)
 234
 235
 236 class MochaOutputHandler(object):
 237     def __init__(self, logger, expected):
 238         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 239
 240         self.logger = logger
 241         self.proc = None
 242         self.test_results = OrderedDict()
 243         self.expected = expected
 244         self.unexpected_skips = set()
 245
 246         self.has_unexpected = False
 247         self.logger.suite_start([], name="puppeteer-tests")
 248         self.status_map = {
 249             "CRASHED": "CRASH",
 250             "OK": "PASS",
 251             "TERMINATED": "CRASH",
 252             "pass": "PASS",
 253             "fail": "FAIL",
 254             "pending": "SKIP",
 255         }
 256
 257     @property
 258     def pid(self):
 259         return self.proc and self.proc.pid
 260
 261     def __call__(self, line):
 262         event = None
 263         try:
 264             if line.startswith("[") and line.endswith("]"):
 265                 event = json.loads(line)
 266             self.process_event(event)
 267         except ValueError:
 268             pass
 269         finally:
 270             self.logger.process_output(self.pid, line, command="npm")
 271
 272     def process_event(self, event):
 273         if isinstance(event, list) and len(event) > 1:
 274             status = self.status_map.get(event[0])
 275             test_start = event[0] == "test-start"
 276             if not status and not test_start:
 277                 return
 278             test_info = event[1]
 279             test_name = test_info.get("fullTitle", "")
 280             test_path = test_info.get("file", "")
 281             test_err = test_info.get("err")
 282             if status == "FAIL" and test_err:
 283                 if "timeout" in test_err.lower():
 284                     status = "TIMEOUT"
 285             if test_name and test_path:
 286                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 287             # mocha hook failures are not tracked in metadata
 288             if status != "PASS" and self.hook_re.search(test_name):
 289                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 290                 return
 291             if test_start:
 292                 self.logger.test_start(test_name)
 293                 return
 294             expected = self.expected.get(test_name, ["PASS"])
 295             # mozlog doesn't really allow unexpected skip,
 296             # so if a test is disabled just expect that and note the unexpected skip
 297             # Also, mocha doesn't log test-start for skipped tests
 298             if status == "SKIP":
 299                 self.logger.test_start(test_name)
 300                 if self.expected and status not in expected:
 301                     self.unexpected_skips.add(test_name)
 302                 expected = ["SKIP"]
 303             known_intermittent = expected[1:]
 304             expected_status = expected[0]
 305
 306             # check if we've seen a result for this test before this log line
 307             result_recorded = self.test_results.get(test_name)
 308             if result_recorded:
 309                 self.logger.warning(
 310                     "Received a second status for {}: "
 311                     "first {}, now {}".format(test_name, result_recorded, status)
 312                 )
 313             # mocha intermittently logs an additional test result after the
 314             # test has already timed out. Avoid recording this second status.
 315             if result_recorded != "TIMEOUT":
 316                 self.test_results[test_name] = status
 317                 if status not in expected:
 318                     self.has_unexpected = True
 319             self.logger.test_end(
 320                 test_name,
 321                 status=status,
 322                 expected=expected_status,
 323                 known_intermittent=known_intermittent,
 324             )
 325
 326     def new_expected(self):
 327         new_expected = OrderedDict()
 328         for test_name, status in iteritems(self.test_results):
 329             if test_name not in self.expected:
 330                 new_status = [status]
 331             else:
 332                 if status in self.expected[test_name]:
 333                     new_status = self.expected[test_name]
 334                 else:
 335                     new_status = [status]
 336             new_expected[test_name] = new_status
 337         return new_expected
 338
 339     def after_end(self, subset=False):
 340         if not subset:
 341             missing = set(self.expected) - set(self.test_results)
 342             extra = set(self.test_results) - set(self.expected)
 343             if missing:
 344                 self.has_unexpected = True
 345                 for test_name in missing:
 346                     self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,))
 347             if self.expected and extra:
 348                 self.has_unexpected = True
 349                 for test_name in extra:
 350                     self.logger.error(
 351                         "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,)
 352                     )
 353
 354         if self.unexpected_skips:
 355             self.has_unexpected = True
 356             for test_name in self.unexpected_skips:
 357                 self.logger.error(
 358                     "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
 359                 )
 360         self.logger.suite_end()
 361
 362
 363 # tempfile.TemporaryDirectory missing from Python 2.7
 364 class TemporaryDirectory(object):
 365     def __init__(self):
 366         self.path = tempfile.mkdtemp()
 367         self._closed = False
 368
 369     def __repr__(self):
 370         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 371
 372     def __enter__(self):
 373         return self.path
 374
 375     def __exit__(self, exc, value, tb):
 376         self.clean()
 377
 378     def __del__(self):
 379         self.clean()
 380
 381     def clean(self):
 382         if self.path and not self._closed:
 383             shutil.rmtree(self.path)
 384             self._closed = True
 385
 386
 387 class PuppeteerRunner(MozbuildObject):
 388     def __init__(self, *args, **kwargs):
 389         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 390
 391         self.remotedir = os.path.join(self.topsrcdir, "remote")
 392         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 393
 394     def run_test(self, logger, *tests, **params):
 395         """
 396         Runs Puppeteer unit tests with npm.
 397
 398         Possible optional test parameters:
 399
 400         `binary`:
 401           Path for the browser binary to use.  Defaults to the local
 402           build.
 403         `headless`:
 404           Boolean to indicate whether to activate Firefox' headless mode.
 405         `extra_prefs`:
 406           Dictionary of extra preferences to write to the profile,
 407           before invoking npm.  Overrides default preferences.
 408         `write_results`:
 409           Path to write the results json file
 410         `subset`
 411           Indicates only a subset of tests are being run, so we should
 412           skip the check for missing results
 413         """
 414         setup()
 415
 416         binary = params.get("binary") or self.get_binary_path()
 417         product = params.get("product", "firefox")
 418
 419         env = {
 420             # Print browser process ouptut
 421             "DUMPIO": "1",
 422             # Checked by Puppeteer's custom mocha config
 423             "CI": "1",
 424             # Causes some tests to be skipped due to assumptions about install
 425             "PUPPETEER_ALT_INSTALL": "1",
 426         }
 427         extra_options = {}
 428         for k, v in params.get("extra_launcher_options", {}).items():
 429             extra_options[k] = json.loads(v)
 430
 431         # Override upstream defaults: no retries, shorter timeout
 432         mocha_options = [
 433             "--reporter",
 434             "./json-mocha-reporter.js",
 435             "--retries",
 436             "0",
 437             "--fullTrace",
 438             "--timeout",
 439             "20000",
 440             "--no-parallel",
 441         ]
 442         if product == "firefox":
 443             env["BINARY"] = binary
 444             env["PUPPETEER_PRODUCT"] = "firefox"
 445         command = ["run", "unit", "--"] + mocha_options
 446
 447         env["HEADLESS"] = str(params.get("headless", False))
 448
 449         prefs = {}
 450         for k, v in params.get("extra_prefs", {}).items():
 451             prefs[k] = mozprofile.Preferences.cast(v)
 452
 453         if prefs:
 454             extra_options["extraPrefsFirefox"] = prefs
 455
 456         if extra_options:
 457             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 458
 459         expected_path = os.path.join(
 460             os.path.dirname(__file__), "puppeteer-expected.json"
 461         )
 462         if product == "firefox" and os.path.exists(expected_path):
 463             with open(expected_path) as f:
 464                 expected_data = json.load(f)
 465         else:
 466             expected_data = {}
 467
 468         output_handler = MochaOutputHandler(logger, expected_data)
 469         proc = npm(
 470             *command,
 471             cwd=self.puppeteer_dir,
 472             env=env,
 473             processOutputLine=output_handler,
 474             wait=False
 475         )
 476         output_handler.proc = proc
 477
 478         # Puppeteer unit tests don't always clean-up child processes in case of
 479         # failure, so use an output_timeout as a fallback
 480         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 481
 482         output_handler.after_end(params.get("subset", False))
 483
 484         # Non-zero return codes are non-fatal for now since we have some
 485         # issues with unresolved promises that shouldn't otherwise block
 486         # running the tests
 487         if proc.returncode != 0:
 488             logger.warning("npm exited with code %s" % proc.returncode)
 489
 490         if params["write_results"]:
 491             with open(params["write_results"], "w") as f:
 492                 json.dump(
 493                     output_handler.new_expected(), f, indent=2, separators=(",", ": ")
 494                 )
 495
 496         if output_handler.has_unexpected:
 497             exit(1, "Got unexpected results")
 498
 499
 500 def create_parser_puppeteer():
 501     p = argparse.ArgumentParser()
 502     p.add_argument(
 503         "--product", type=str, default="firefox", choices=["chrome", "firefox"]
 504     )
 505     p.add_argument(
 506         "--binary",
 507         type=str,
 508         help="Path to browser binary.  Defaults to local Firefox build.",
 509     )
 510     p.add_argument(
 511         "--ci",
 512         action="store_true",
 513         help="Flag that indicates that tests run in a CI environment.",
 514     )
 515     p.add_argument(
 516         "--enable-fission",
 517         action="store_true",
 518         help="Enable Fission (site isolation) in Gecko.",
 519     )
 520     p.add_argument(
 521         "-z", "--headless", action="store_true", help="Run browser in headless mode."
 522     )
 523     p.add_argument(
 524         "--setpref",
 525         action="append",
 526         dest="extra_prefs",
 527         metavar="<pref>=<value>",
 528         help="Defines additional user preferences.",
 529     )
 530     p.add_argument(
 531         "--setopt",
 532         action="append",
 533         dest="extra_options",
 534         metavar="<option>=<value>",
 535         help="Defines additional options for `puppeteer.launch`.",
 536     )
 537     p.add_argument(
 538         "-v",
 539         dest="verbosity",
 540         action="count",
 541         default=0,
 542         help="Increase remote agent logging verbosity to include "
 543         "debug level messages with -v, trace messages with -vv,"
 544         "and to not truncate long trace messages with -vvv",
 545     )
 546     p.add_argument(
 547         "--write-results",
 548         action="store",
 549         nargs="?",
 550         default=None,
 551         const=os.path.join(os.path.dirname(__file__), "puppeteer-expected.json"),
 552         help="Path to write updated results to (defaults to the "
 553         "expectations file if the argument is provided but "
 554         "no path is passed)",
 555     )
 556     p.add_argument(
 557         "--subset",
 558         action="store_true",
 559         default=False,
 560         help="Indicate that only a subset of the tests are running, "
 561         "so checks for missing tests should be skipped",
 562     )
 563     p.add_argument("tests", nargs="*")
 564     mozlog.commandline.add_logging_group(p)
 565     return p
 566
 567
 568 @CommandProvider
 569 class PuppeteerTest(MachCommandBase):
 570     @Command(
 571         "puppeteer-test",
 572         category="testing",
 573         description="Run Puppeteer unit tests.",
 574         parser=create_parser_puppeteer,
 575     )
 576     def puppeteer_test(
 577         self,
 578         binary=None,
 579         ci=False,
 580         enable_fission=False,
 581         headless=False,
 582         extra_prefs=None,
 583         extra_options=None,
 584         verbosity=0,
 585         tests=None,
 586         product="firefox",
 587         write_results=None,
 588         subset=False,
 589         **kwargs
 590     ):
 591
 592         self.ci = ci
 593
 594         logger = mozlog.commandline.setup_logging(
 595             "puppeteer-test", kwargs, {"mach": sys.stdout}
 596         )
 597
 598         # moztest calls this programmatically with test objects or manifests
 599         if "test_objects" in kwargs and tests is not None:
 600             logger.error("Expected either 'test_objects' or 'tests'")
 601             exit(1)
 602
 603         if product != "firefox" and extra_prefs is not None:
 604             logger.error("User preferences are not recognized by %s" % product)
 605             exit(1)
 606
 607         if "test_objects" in kwargs:
 608             tests = []
 609             for test in kwargs["test_objects"]:
 610                 tests.append(test["path"])
 611
 612         prefs = {}
 613         for s in extra_prefs or []:
 614             kv = s.split("=")
 615             if len(kv) != 2:
 616                 logger.error("syntax error in --setpref={}".format(s))
 617                 exit(EX_USAGE)
 618             prefs[kv[0]] = kv[1].strip()
 619
 620         options = {}
 621         for s in extra_options or []:
 622             kv = s.split("=")
 623             if len(kv) != 2:
 624                 logger.error("syntax error in --setopt={}".format(s))
 625                 exit(EX_USAGE)
 626             options[kv[0]] = kv[1].strip()
 627
 628         if enable_fission:
 629             prefs.update(
 630                 {"fission.autostart": True, "dom.serviceWorkers.parent_intercept": True}
 631             )
 632
 633         if verbosity == 1:
 634             prefs["remote.log.level"] = "Debug"
 635         elif verbosity > 1:
 636             prefs["remote.log.level"] = "Trace"
 637         if verbosity > 2:
 638             prefs["remote.log.truncate"] = False
 639
 640         self.install_puppeteer(product)
 641
 642         params = {
 643             "binary": binary,
 644             "headless": headless,
 645             "extra_prefs": prefs,
 646             "product": product,
 647             "extra_launcher_options": options,
 648             "write_results": write_results,
 649             "subset": subset,
 650         }
 651         puppeteer = self._spawn(PuppeteerRunner)
 652         try:
 653             return puppeteer.run_test(logger, *tests, **params)
 654         except BinaryNotFoundException as e:
 655             logger.error(e)
 656             logger.info(e.help())
 657             exit(1)
 658         except Exception as e:
 659             exit(EX_SOFTWARE, e)
 660
 661     def install_puppeteer(self, product):
 662         setup()
 663         env = {}
 664         from mozversioncontrol import get_repository_object
 665
 666         repo = get_repository_object(self.topsrcdir)
 667         puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 668         changed_files = False
 669         for f in repo.get_changed_files():
 670             if f.startswith(puppeteer_dir) and f.endswith(".ts"):
 671                 changed_files = True
 672                 break
 673
 674         if product != "chrome":
 675             env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 676         lib_dir = os.path.join(self.topsrcdir, puppeteer_dir, "lib")
 677         if changed_files and os.path.isdir(lib_dir):
 678             # clobber lib to force `tsc compile` step
 679             shutil.rmtree(lib_dir)
 680
 681         command = "ci" if self.ci else "install"
 682         npm(command, cwd=os.path.join(self.topsrcdir, puppeteer_dir), env=env)
 683
 684
 685 def exit(code, error=None):
 686     if error is not None:
 687         if isinstance(error, Exception):
 688             import traceback
 689
 690             traceback.print_exc()
 691         else:
 692             message = str(error).split("\n")[0].strip()
 693             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 694     sys.exit(code)