remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import argparse
  12 import json
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import tempfile
  19
  20 from collections import OrderedDict
  21
  22 from six import iteritems
  23
  24 from mach.decorators import (
  25     Command,
  26     CommandArgument,
  27     CommandProvider,
  28     SubCommand,
  29 )
  30
  31 from mozbuild.base import (
  32     MachCommandBase,
  33     MozbuildObject,
  34     BinaryNotFoundException,
  35 )
  36 from mozbuild import nodeutil
  37 import mozlog
  38 import mozprofile
  39
  40
  41 EX_CONFIG = 78
  42 EX_SOFTWARE = 70
  43 EX_USAGE = 64
  44
  45
  46 def setup():
  47     # add node and npm from mozbuild to front of system path
  48     npm, _ = nodeutil.find_npm_executable()
  49     if not npm:
  50         exit(EX_CONFIG, "could not find npm executable")
  51     path = os.path.abspath(os.path.join(npm, os.pardir))
  52     os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
  53
  54
  55 @CommandProvider
  56 class RemoteCommands(MachCommandBase):
  57     def remotedir(self):
  58         return os.path.join(self.topsrcdir, "remote")
  59
  60     @Command(
  61         "remote", category="misc", description="Remote protocol related operations."
  62     )
  63     def remote(self, command_context):
  64         """The remote subcommands all relate to the remote protocol."""
  65         self._sub_mach(["help", "remote"])
  66         return 1
  67
  68     @SubCommand(
  69         "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
  70     )
  71     @CommandArgument(
  72         "--repository",
  73         metavar="REPO",
  74         required=True,
  75         help="The (possibly remote) repository to clone from.",
  76     )
  77     @CommandArgument(
  78         "--commitish",
  79         metavar="COMMITISH",
  80         required=True,
  81         help="The commit or tag object name to check out.",
  82     )
  83     @CommandArgument(
  84         "--no-install",
  85         dest="install",
  86         action="store_false",
  87         default=True,
  88         help="Do not install the just-pulled Puppeteer package,",
  89     )
  90     def vendor_puppeteer(self, command_context, repository, commitish, install):
  91         puppeteer_dir = os.path.join(self.remotedir(), "test", "puppeteer")
  92
  93         # Preserve our custom mocha reporter
  94         shutil.move(
  95             os.path.join(puppeteer_dir, "json-mocha-reporter.js"), self.remotedir()
  96         )
  97         shutil.rmtree(puppeteer_dir, ignore_errors=True)
  98         os.makedirs(puppeteer_dir)
  99         with TemporaryDirectory() as tmpdir:
 100             git("clone", "-q", repository, tmpdir)
 101             git("checkout", commitish, worktree=tmpdir)
 102             git(
 103                 "checkout-index",
 104                 "-a",
 105                 "-f",
 106                 "--prefix",
 107                 "{}/".format(puppeteer_dir),
 108                 worktree=tmpdir,
 109             )
 110
 111         # remove files which may interfere with git checkout of central
 112         try:
 113             os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
 114             os.remove(os.path.join(puppeteer_dir, ".gitignore"))
 115         except OSError:
 116             pass
 117
 118         unwanted_dirs = ["experimental", "docs"]
 119
 120         for dir in unwanted_dirs:
 121             dir_path = os.path.join(puppeteer_dir, dir)
 122             if os.path.isdir(dir_path):
 123                 shutil.rmtree(dir_path)
 124
 125         shutil.move(
 126             os.path.join(self.remotedir(), "json-mocha-reporter.js"), puppeteer_dir
 127         )
 128
 129         import yaml
 130
 131         annotation = {
 132             "schema": 1,
 133             "bugzilla": {
 134                 "product": "Remote Protocol",
 135                 "component": "Agent",
 136             },
 137             "origin": {
 138                 "name": "puppeteer",
 139                 "description": "Headless Chrome Node API",
 140                 "url": repository,
 141                 "license": "Apache-2.0",
 142                 "release": commitish,
 143             },
 144         }
 145         with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 146             yaml.safe_dump(
 147                 annotation,
 148                 fh,
 149                 default_flow_style=False,
 150                 encoding="utf-8",
 151                 allow_unicode=True,
 152             )
 153
 154         if install:
 155             env = {"PUPPETEER_SKIP_DOWNLOAD": "1"}
 156             npm("install", cwd=os.path.join(self.topsrcdir, puppeteer_dir), env=env)
 157
 158
 159 def git(*args, **kwargs):
 160     cmd = ("git",)
 161     if kwargs.get("worktree"):
 162         cmd += ("-C", kwargs["worktree"])
 163     cmd += args
 164
 165     pipe = kwargs.get("pipe")
 166     git_p = subprocess.Popen(
 167         cmd,
 168         env={"GIT_CONFIG_NOSYSTEM": "1"},
 169         stdout=subprocess.PIPE,
 170         stderr=subprocess.PIPE,
 171     )
 172     pipe_p = None
 173     if pipe:
 174         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 175
 176     if pipe:
 177         _, pipe_err = pipe_p.communicate()
 178     out, git_err = git_p.communicate()
 179
 180     # use error from first program that failed
 181     if git_p.returncode > 0:
 182         exit(EX_SOFTWARE, git_err)
 183     if pipe and pipe_p.returncode > 0:
 184         exit(EX_SOFTWARE, pipe_err)
 185
 186     return out
 187
 188
 189 def npm(*args, **kwargs):
 190     from mozprocess import processhandler
 191
 192     env = None
 193     if kwargs.get("env"):
 194         env = os.environ.copy()
 195         env.update(kwargs["env"])
 196
 197     proc_kwargs = {}
 198     if "processOutputLine" in kwargs:
 199         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 200
 201     p = processhandler.ProcessHandler(
 202         cmd="npm",
 203         args=list(args),
 204         cwd=kwargs.get("cwd"),
 205         env=env,
 206         universal_newlines=True,
 207         **proc_kwargs
 208     )
 209     if not kwargs.get("wait", True):
 210         return p
 211
 212     wait_proc(p, cmd="npm", exit_on_fail=kwargs.get("exit_on_fail", True))
 213
 214     return p.returncode
 215
 216
 217 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 218     try:
 219         p.run(outputTimeout=output_timeout)
 220         p.wait()
 221         if p.timedOut:
 222             # In some cases, we wait longer for a mocha timeout
 223             print("Timed out after {} seconds of no output".format(output_timeout))
 224     finally:
 225         p.kill()
 226     if exit_on_fail and p.returncode > 0:
 227         msg = (
 228             "%s: exit code %s" % (cmd, p.returncode)
 229             if cmd
 230             else "exit code %s" % p.returncode
 231         )
 232         exit(p.returncode, msg)
 233
 234
 235 class MochaOutputHandler(object):
 236     def __init__(self, logger, expected):
 237         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 238
 239         self.logger = logger
 240         self.proc = None
 241         self.test_results = OrderedDict()
 242         self.expected = expected
 243         self.unexpected_skips = set()
 244
 245         self.has_unexpected = False
 246         self.logger.suite_start([], name="puppeteer-tests")
 247         self.status_map = {
 248             "CRASHED": "CRASH",
 249             "OK": "PASS",
 250             "TERMINATED": "CRASH",
 251             "pass": "PASS",
 252             "fail": "FAIL",
 253             "pending": "SKIP",
 254         }
 255
 256     @property
 257     def pid(self):
 258         return self.proc and self.proc.pid
 259
 260     def __call__(self, line):
 261         event = None
 262         try:
 263             if line.startswith("[") and line.endswith("]"):
 264                 event = json.loads(line)
 265             self.process_event(event)
 266         except ValueError:
 267             pass
 268         finally:
 269             self.logger.process_output(self.pid, line, command="npm")
 270
 271     def process_event(self, event):
 272         if isinstance(event, list) and len(event) > 1:
 273             status = self.status_map.get(event[0])
 274             test_start = event[0] == "test-start"
 275             if not status and not test_start:
 276                 return
 277             test_info = event[1]
 278             test_name = test_info.get("fullTitle", "")
 279             test_path = test_info.get("file", "")
 280             test_err = test_info.get("err")
 281             if status == "FAIL" and test_err:
 282                 if "timeout" in test_err.lower():
 283                     status = "TIMEOUT"
 284             if test_name and test_path:
 285                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 286             # mocha hook failures are not tracked in metadata
 287             if status != "PASS" and self.hook_re.search(test_name):
 288                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 289                 return
 290             if test_start:
 291                 self.logger.test_start(test_name)
 292                 return
 293             expected = self.expected.get(test_name, ["PASS"])
 294             # mozlog doesn't really allow unexpected skip,
 295             # so if a test is disabled just expect that and note the unexpected skip
 296             # Also, mocha doesn't log test-start for skipped tests
 297             if status == "SKIP":
 298                 self.logger.test_start(test_name)
 299                 if self.expected and status not in expected:
 300                     self.unexpected_skips.add(test_name)
 301                 expected = ["SKIP"]
 302             known_intermittent = expected[1:]
 303             expected_status = expected[0]
 304
 305             # check if we've seen a result for this test before this log line
 306             result_recorded = self.test_results.get(test_name)
 307             if result_recorded:
 308                 self.logger.warning(
 309                     "Received a second status for {}: "
 310                     "first {}, now {}".format(test_name, result_recorded, status)
 311                 )
 312             # mocha intermittently logs an additional test result after the
 313             # test has already timed out. Avoid recording this second status.
 314             if result_recorded != "TIMEOUT":
 315                 self.test_results[test_name] = status
 316                 if status not in expected:
 317                     self.has_unexpected = True
 318             self.logger.test_end(
 319                 test_name,
 320                 status=status,
 321                 expected=expected_status,
 322                 known_intermittent=known_intermittent,
 323             )
 324
 325     def new_expected(self):
 326         new_expected = OrderedDict()
 327         for test_name, status in iteritems(self.test_results):
 328             if test_name not in self.expected:
 329                 new_status = [status]
 330             else:
 331                 if status in self.expected[test_name]:
 332                     new_status = self.expected[test_name]
 333                 else:
 334                     new_status = [status]
 335             new_expected[test_name] = new_status
 336         return new_expected
 337
 338     def after_end(self, subset=False):
 339         if not subset:
 340             missing = set(self.expected) - set(self.test_results)
 341             extra = set(self.test_results) - set(self.expected)
 342             if missing:
 343                 self.has_unexpected = True
 344                 for test_name in missing:
 345                     self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,))
 346             if self.expected and extra:
 347                 self.has_unexpected = True
 348                 for test_name in extra:
 349                     self.logger.error(
 350                         "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,)
 351                     )
 352
 353         if self.unexpected_skips:
 354             self.has_unexpected = True
 355             for test_name in self.unexpected_skips:
 356                 self.logger.error(
 357                     "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
 358                 )
 359         self.logger.suite_end()
 360
 361
 362 # tempfile.TemporaryDirectory missing from Python 2.7
 363 class TemporaryDirectory(object):
 364     def __init__(self):
 365         self.path = tempfile.mkdtemp()
 366         self._closed = False
 367
 368     def __repr__(self):
 369         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 370
 371     def __enter__(self):
 372         return self.path
 373
 374     def __exit__(self, exc, value, tb):
 375         self.clean()
 376
 377     def __del__(self):
 378         self.clean()
 379
 380     def clean(self):
 381         if self.path and not self._closed:
 382             shutil.rmtree(self.path)
 383             self._closed = True
 384
 385
 386 class PuppeteerRunner(MozbuildObject):
 387     def __init__(self, *args, **kwargs):
 388         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 389
 390         self.remotedir = os.path.join(self.topsrcdir, "remote")
 391         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 392
 393     def run_test(self, logger, *tests, **params):
 394         """
 395         Runs Puppeteer unit tests with npm.
 396
 397         Possible optional test parameters:
 398
 399         `binary`:
 400           Path for the browser binary to use.  Defaults to the local
 401           build.
 402         `headless`:
 403           Boolean to indicate whether to activate Firefox' headless mode.
 404         `extra_prefs`:
 405           Dictionary of extra preferences to write to the profile,
 406           before invoking npm.  Overrides default preferences.
 407         `enable_webrender`:
 408           Boolean to indicate whether to enable WebRender compositor in Gecko.
 409         `write_results`:
 410           Path to write the results json file
 411         `subset`
 412           Indicates only a subset of tests are being run, so we should
 413           skip the check for missing results
 414         """
 415         setup()
 416
 417         binary = params.get("binary") or self.get_binary_path()
 418         product = params.get("product", "firefox")
 419
 420         env = {
 421             # Print browser process ouptut
 422             "DUMPIO": "1",
 423             # Checked by Puppeteer's custom mocha config
 424             "CI": "1",
 425             # Causes some tests to be skipped due to assumptions about install
 426             "PUPPETEER_ALT_INSTALL": "1",
 427         }
 428         extra_options = {}
 429         for k, v in params.get("extra_launcher_options", {}).items():
 430             extra_options[k] = json.loads(v)
 431
 432         # Override upstream defaults: no retries, shorter timeout
 433         mocha_options = [
 434             "--reporter",
 435             "./json-mocha-reporter.js",
 436             "--retries",
 437             "0",
 438             "--fullTrace",
 439             "--timeout",
 440             "20000",
 441             "--no-parallel",
 442         ]
 443         if product == "firefox":
 444             env["BINARY"] = binary
 445             env["PUPPETEER_PRODUCT"] = "firefox"
 446
 447             env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
 448
 449         command = ["run", "unit", "--"] + mocha_options
 450
 451         env["HEADLESS"] = str(params.get("headless", False))
 452
 453         prefs = {}
 454         for k, v in params.get("extra_prefs", {}).items():
 455             prefs[k] = mozprofile.Preferences.cast(v)
 456
 457         if prefs:
 458             extra_options["extraPrefsFirefox"] = prefs
 459
 460         if extra_options:
 461             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 462
 463         expected_path = os.path.join(
 464             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 465         )
 466         if product == "firefox" and os.path.exists(expected_path):
 467             with open(expected_path) as f:
 468                 expected_data = json.load(f)
 469         else:
 470             expected_data = {}
 471
 472         output_handler = MochaOutputHandler(logger, expected_data)
 473         proc = npm(
 474             *command,
 475             cwd=self.puppeteer_dir,
 476             env=env,
 477             processOutputLine=output_handler,
 478             wait=False
 479         )
 480         output_handler.proc = proc
 481
 482         # Puppeteer unit tests don't always clean-up child processes in case of
 483         # failure, so use an output_timeout as a fallback
 484         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 485
 486         output_handler.after_end(params.get("subset", False))
 487
 488         # Non-zero return codes are non-fatal for now since we have some
 489         # issues with unresolved promises that shouldn't otherwise block
 490         # running the tests
 491         if proc.returncode != 0:
 492             logger.warning("npm exited with code %s" % proc.returncode)
 493
 494         if params["write_results"]:
 495             with open(params["write_results"], "w") as f:
 496                 json.dump(
 497                     output_handler.new_expected(), f, indent=2, separators=(",", ": ")
 498                 )
 499
 500         if output_handler.has_unexpected:
 501             exit(1, "Got unexpected results")
 502
 503
 504 def create_parser_puppeteer():
 505     p = argparse.ArgumentParser()
 506     p.add_argument(
 507         "--product", type=str, default="firefox", choices=["chrome", "firefox"]
 508     )
 509     p.add_argument(
 510         "--binary",
 511         type=str,
 512         help="Path to browser binary.  Defaults to local Firefox build.",
 513     )
 514     p.add_argument(
 515         "--ci",
 516         action="store_true",
 517         help="Flag that indicates that tests run in a CI environment.",
 518     )
 519     p.add_argument(
 520         "--enable-fission",
 521         action="store_true",
 522         help="Enable Fission (site isolation) in Gecko.",
 523     )
 524     p.add_argument(
 525         "--enable-webrender",
 526         action="store_true",
 527         help="Enable the WebRender compositor in Gecko.",
 528     )
 529     p.add_argument(
 530         "-z", "--headless", action="store_true", help="Run browser in headless mode."
 531     )
 532     p.add_argument(
 533         "--setpref",
 534         action="append",
 535         dest="extra_prefs",
 536         metavar="<pref>=<value>",
 537         help="Defines additional user preferences.",
 538     )
 539     p.add_argument(
 540         "--setopt",
 541         action="append",
 542         dest="extra_options",
 543         metavar="<option>=<value>",
 544         help="Defines additional options for `puppeteer.launch`.",
 545     )
 546     p.add_argument(
 547         "-v",
 548         dest="verbosity",
 549         action="count",
 550         default=0,
 551         help="Increase remote agent logging verbosity to include "
 552         "debug level messages with -v, trace messages with -vv,"
 553         "and to not truncate long trace messages with -vvv",
 554     )
 555     p.add_argument(
 556         "--write-results",
 557         action="store",
 558         nargs="?",
 559         default=None,
 560         const=os.path.join(
 561             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 562         ),
 563         help="Path to write updated results to (defaults to the "
 564         "expectations file if the argument is provided but "
 565         "no path is passed)",
 566     )
 567     p.add_argument(
 568         "--subset",
 569         action="store_true",
 570         default=False,
 571         help="Indicate that only a subset of the tests are running, "
 572         "so checks for missing tests should be skipped",
 573     )
 574     p.add_argument("tests", nargs="*")
 575     mozlog.commandline.add_logging_group(p)
 576     return p
 577
 578
 579 @CommandProvider
 580 class PuppeteerTest(MachCommandBase):
 581     @Command(
 582         "puppeteer-test",
 583         category="testing",
 584         description="Run Puppeteer unit tests.",
 585         parser=create_parser_puppeteer,
 586     )
 587     def puppeteer_test(
 588         self,
 589         command_context,
 590         binary=None,
 591         ci=False,
 592         enable_fission=False,
 593         enable_webrender=False,
 594         headless=False,
 595         extra_prefs=None,
 596         extra_options=None,
 597         verbosity=0,
 598         tests=None,
 599         product="firefox",
 600         write_results=None,
 601         subset=False,
 602         **kwargs
 603     ):
 604
 605         self.ci = ci
 606
 607         logger = mozlog.commandline.setup_logging(
 608             "puppeteer-test", kwargs, {"mach": sys.stdout}
 609         )
 610
 611         # moztest calls this programmatically with test objects or manifests
 612         if "test_objects" in kwargs and tests is not None:
 613             logger.error("Expected either 'test_objects' or 'tests'")
 614             exit(1)
 615
 616         if product != "firefox" and extra_prefs is not None:
 617             logger.error("User preferences are not recognized by %s" % product)
 618             exit(1)
 619
 620         if "test_objects" in kwargs:
 621             tests = []
 622             for test in kwargs["test_objects"]:
 623                 tests.append(test["path"])
 624
 625         prefs = {}
 626         for s in extra_prefs or []:
 627             kv = s.split("=")
 628             if len(kv) != 2:
 629                 logger.error("syntax error in --setpref={}".format(s))
 630                 exit(EX_USAGE)
 631             prefs[kv[0]] = kv[1].strip()
 632
 633         options = {}
 634         for s in extra_options or []:
 635             kv = s.split("=")
 636             if len(kv) != 2:
 637                 logger.error("syntax error in --setopt={}".format(s))
 638                 exit(EX_USAGE)
 639             options[kv[0]] = kv[1].strip()
 640
 641         if enable_fission:
 642             prefs.update({"fission.autostart": True})
 643
 644         if verbosity == 1:
 645             prefs["remote.log.level"] = "Debug"
 646         elif verbosity > 1:
 647             prefs["remote.log.level"] = "Trace"
 648         if verbosity > 2:
 649             prefs["remote.log.truncate"] = False
 650
 651         self.install_puppeteer(product)
 652
 653         params = {
 654             "binary": binary,
 655             "headless": headless,
 656             "enable_webrender": enable_webrender,
 657             "extra_prefs": prefs,
 658             "product": product,
 659             "extra_launcher_options": options,
 660             "write_results": write_results,
 661             "subset": subset,
 662         }
 663         puppeteer = self._spawn(PuppeteerRunner)
 664         try:
 665             return puppeteer.run_test(logger, *tests, **params)
 666         except BinaryNotFoundException as e:
 667             logger.error(e)
 668             logger.info(e.help())
 669             exit(1)
 670         except Exception as e:
 671             exit(EX_SOFTWARE, e)
 672
 673     def install_puppeteer(self, product):
 674         setup()
 675         env = {}
 676         from mozversioncontrol import get_repository_object
 677
 678         repo = get_repository_object(self.topsrcdir)
 679         puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 680         changed_files = False
 681         for f in repo.get_changed_files():
 682             if f.startswith(puppeteer_dir) and f.endswith(".ts"):
 683                 changed_files = True
 684                 break
 685
 686         if product != "chrome":
 687             env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 688         lib_dir = os.path.join(self.topsrcdir, puppeteer_dir, "lib")
 689         if changed_files and os.path.isdir(lib_dir):
 690             # clobber lib to force `tsc compile` step
 691             shutil.rmtree(lib_dir)
 692
 693         command = "ci" if self.ci else "install"
 694         npm(command, cwd=os.path.join(self.topsrcdir, puppeteer_dir), env=env)
 695
 696
 697 def exit(code, error=None):
 698     if error is not None:
 699         if isinstance(error, Exception):
 700             import traceback
 701
 702             traceback.print_exc()
 703         else:
 704             message = str(error).split("\n")[0].strip()
 705             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 706     sys.exit(code)