remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import argparse
  12 import json
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import tempfile
  19
  20 from collections import OrderedDict
  21
  22 from six import iteritems
  23
  24 from mach.decorators import (
  25     Command,
  26     CommandArgument,
  27     CommandProvider,
  28     SubCommand,
  29 )
  30
  31 from mozbuild.base import (
  32     MachCommandBase,
  33     MozbuildObject,
  34     BinaryNotFoundException,
  35 )
  36 from mozbuild import nodeutil
  37 import mozlog
  38 import mozprofile
  39
  40
  41 EX_CONFIG = 78
  42 EX_SOFTWARE = 70
  43 EX_USAGE = 64
  44
  45
  46 def setup():
  47     # add node and npm from mozbuild to front of system path
  48     npm, _ = nodeutil.find_npm_executable()
  49     if not npm:
  50         exit(EX_CONFIG, "could not find npm executable")
  51     path = os.path.abspath(os.path.join(npm, os.pardir))
  52     os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
  53
  54
  55 @CommandProvider
  56 class RemoteCommands(MachCommandBase):
  57     def __init__(self, *args, **kwargs):
  58         super(RemoteCommands, self).__init__(*args, **kwargs)
  59         self.remotedir = os.path.join(self.topsrcdir, "remote")
  60
  61     @Command(
  62         "remote", category="misc", description="Remote protocol related operations."
  63     )
  64     def remote(self):
  65         """The remote subcommands all relate to the remote protocol."""
  66         self._sub_mach(["help", "remote"])
  67         return 1
  68
  69     @SubCommand(
  70         "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
  71     )
  72     @CommandArgument(
  73         "--repository",
  74         metavar="REPO",
  75         required=True,
  76         help="The (possibly remote) repository to clone from.",
  77     )
  78     @CommandArgument(
  79         "--commitish",
  80         metavar="COMMITISH",
  81         required=True,
  82         help="The commit or tag object name to check out.",
  83     )
  84     @CommandArgument(
  85         "--no-install",
  86         dest="install",
  87         action="store_false",
  88         default=True,
  89         help="Do not install the just-pulled Puppeteer package,",
  90     )
  91     def vendor_puppeteer(self, repository, commitish, install):
  92         puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
  93
  94         # Preserve our custom mocha reporter
  95         shutil.move(
  96             os.path.join(puppeteer_dir, "json-mocha-reporter.js"), self.remotedir
  97         )
  98         shutil.rmtree(puppeteer_dir, ignore_errors=True)
  99         os.makedirs(puppeteer_dir)
 100         with TemporaryDirectory() as tmpdir:
 101             git("clone", "-q", repository, tmpdir)
 102             git("checkout", commitish, worktree=tmpdir)
 103             git(
 104                 "checkout-index",
 105                 "-a",
 106                 "-f",
 107                 "--prefix",
 108                 "{}/".format(puppeteer_dir),
 109                 worktree=tmpdir,
 110             )
 111
 112         # remove files which may interfere with git checkout of central
 113         try:
 114             os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
 115             os.remove(os.path.join(puppeteer_dir, ".gitignore"))
 116         except OSError:
 117             pass
 118
 119         unwanted_dirs = ["experimental", "docs"]
 120
 121         for dir in unwanted_dirs:
 122             dir_path = os.path.join(puppeteer_dir, dir)
 123             if os.path.isdir(dir_path):
 124                 shutil.rmtree(dir_path)
 125
 126         shutil.move(
 127             os.path.join(self.remotedir, "json-mocha-reporter.js"), puppeteer_dir
 128         )
 129
 130         import yaml
 131
 132         annotation = {
 133             "schema": 1,
 134             "bugzilla": {
 135                 "product": "Remote Protocol",
 136                 "component": "Agent",
 137             },
 138             "origin": {
 139                 "name": "puppeteer",
 140                 "description": "Headless Chrome Node API",
 141                 "url": repository,
 142                 "license": "Apache-2.0",
 143                 "release": commitish,
 144             },
 145         }
 146         with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 147             yaml.safe_dump(
 148                 annotation,
 149                 fh,
 150                 default_flow_style=False,
 151                 encoding="utf-8",
 152                 allow_unicode=True,
 153             )
 154
 155         if install:
 156             env = {"PUPPETEER_SKIP_DOWNLOAD": "1"}
 157             npm("install", cwd=os.path.join(self.topsrcdir, puppeteer_dir), env=env)
 158
 159
 160 def git(*args, **kwargs):
 161     cmd = ("git",)
 162     if kwargs.get("worktree"):
 163         cmd += ("-C", kwargs["worktree"])
 164     cmd += args
 165
 166     pipe = kwargs.get("pipe")
 167     git_p = subprocess.Popen(
 168         cmd,
 169         env={"GIT_CONFIG_NOSYSTEM": "1"},
 170         stdout=subprocess.PIPE,
 171         stderr=subprocess.PIPE,
 172     )
 173     pipe_p = None
 174     if pipe:
 175         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 176
 177     if pipe:
 178         _, pipe_err = pipe_p.communicate()
 179     out, git_err = git_p.communicate()
 180
 181     # use error from first program that failed
 182     if git_p.returncode > 0:
 183         exit(EX_SOFTWARE, git_err)
 184     if pipe and pipe_p.returncode > 0:
 185         exit(EX_SOFTWARE, pipe_err)
 186
 187     return out
 188
 189
 190 def npm(*args, **kwargs):
 191     from mozprocess import processhandler
 192
 193     env = None
 194     if kwargs.get("env"):
 195         env = os.environ.copy()
 196         env.update(kwargs["env"])
 197
 198     proc_kwargs = {}
 199     if "processOutputLine" in kwargs:
 200         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 201
 202     p = processhandler.ProcessHandler(
 203         cmd="npm",
 204         args=list(args),
 205         cwd=kwargs.get("cwd"),
 206         env=env,
 207         universal_newlines=True,
 208         **proc_kwargs
 209     )
 210     if not kwargs.get("wait", True):
 211         return p
 212
 213     wait_proc(p, cmd="npm", exit_on_fail=kwargs.get("exit_on_fail", True))
 214
 215     return p.returncode
 216
 217
 218 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 219     try:
 220         p.run(outputTimeout=output_timeout)
 221         p.wait()
 222         if p.timedOut:
 223             # In some cases, we wait longer for a mocha timeout
 224             print("Timed out after {} seconds of no output".format(output_timeout))
 225     finally:
 226         p.kill()
 227     if exit_on_fail and p.returncode > 0:
 228         msg = (
 229             "%s: exit code %s" % (cmd, p.returncode)
 230             if cmd
 231             else "exit code %s" % p.returncode
 232         )
 233         exit(p.returncode, msg)
 234
 235
 236 class MochaOutputHandler(object):
 237     def __init__(self, logger, expected):
 238         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 239
 240         self.logger = logger
 241         self.proc = None
 242         self.test_results = OrderedDict()
 243         self.expected = expected
 244         self.unexpected_skips = set()
 245
 246         self.has_unexpected = False
 247         self.logger.suite_start([], name="puppeteer-tests")
 248         self.status_map = {
 249             "CRASHED": "CRASH",
 250             "OK": "PASS",
 251             "TERMINATED": "CRASH",
 252             "pass": "PASS",
 253             "fail": "FAIL",
 254             "pending": "SKIP",
 255         }
 256
 257     @property
 258     def pid(self):
 259         return self.proc and self.proc.pid
 260
 261     def __call__(self, line):
 262         event = None
 263         try:
 264             if line.startswith("[") and line.endswith("]"):
 265                 event = json.loads(line)
 266             self.process_event(event)
 267         except ValueError:
 268             pass
 269         finally:
 270             self.logger.process_output(self.pid, line, command="npm")
 271
 272     def process_event(self, event):
 273         if isinstance(event, list) and len(event) > 1:
 274             status = self.status_map.get(event[0])
 275             test_start = event[0] == "test-start"
 276             if not status and not test_start:
 277                 return
 278             test_info = event[1]
 279             test_name = test_info.get("fullTitle", "")
 280             test_path = test_info.get("file", "")
 281             test_err = test_info.get("err")
 282             if status == "FAIL" and test_err:
 283                 if "timeout" in test_err.lower():
 284                     status = "TIMEOUT"
 285             if test_name and test_path:
 286                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 287             # mocha hook failures are not tracked in metadata
 288             if status != "PASS" and self.hook_re.search(test_name):
 289                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 290                 return
 291             if test_start:
 292                 self.logger.test_start(test_name)
 293                 return
 294             expected = self.expected.get(test_name, ["PASS"])
 295             # mozlog doesn't really allow unexpected skip,
 296             # so if a test is disabled just expect that and note the unexpected skip
 297             # Also, mocha doesn't log test-start for skipped tests
 298             if status == "SKIP":
 299                 self.logger.test_start(test_name)
 300                 if self.expected and status not in expected:
 301                     self.unexpected_skips.add(test_name)
 302                 expected = ["SKIP"]
 303             known_intermittent = expected[1:]
 304             expected_status = expected[0]
 305
 306             # check if we've seen a result for this test before this log line
 307             result_recorded = self.test_results.get(test_name)
 308             if result_recorded:
 309                 self.logger.warning(
 310                     "Received a second status for {}: "
 311                     "first {}, now {}".format(test_name, result_recorded, status)
 312                 )
 313             # mocha intermittently logs an additional test result after the
 314             # test has already timed out. Avoid recording this second status.
 315             if result_recorded != "TIMEOUT":
 316                 self.test_results[test_name] = status
 317                 if status not in expected:
 318                     self.has_unexpected = True
 319             self.logger.test_end(
 320                 test_name,
 321                 status=status,
 322                 expected=expected_status,
 323                 known_intermittent=known_intermittent,
 324             )
 325
 326     def new_expected(self):
 327         new_expected = OrderedDict()
 328         for test_name, status in iteritems(self.test_results):
 329             if test_name not in self.expected:
 330                 new_status = [status]
 331             else:
 332                 if status in self.expected[test_name]:
 333                     new_status = self.expected[test_name]
 334                 else:
 335                     new_status = [status]
 336             new_expected[test_name] = new_status
 337         return new_expected
 338
 339     def after_end(self, subset=False):
 340         if not subset:
 341             missing = set(self.expected) - set(self.test_results)
 342             extra = set(self.test_results) - set(self.expected)
 343             if missing:
 344                 self.has_unexpected = True
 345                 for test_name in missing:
 346                     self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,))
 347             if self.expected and extra:
 348                 self.has_unexpected = True
 349                 for test_name in extra:
 350                     self.logger.error(
 351                         "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,)
 352                     )
 353
 354         if self.unexpected_skips:
 355             self.has_unexpected = True
 356             for test_name in self.unexpected_skips:
 357                 self.logger.error(
 358                     "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
 359                 )
 360         self.logger.suite_end()
 361
 362
 363 # tempfile.TemporaryDirectory missing from Python 2.7
 364 class TemporaryDirectory(object):
 365     def __init__(self):
 366         self.path = tempfile.mkdtemp()
 367         self._closed = False
 368
 369     def __repr__(self):
 370         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 371
 372     def __enter__(self):
 373         return self.path
 374
 375     def __exit__(self, exc, value, tb):
 376         self.clean()
 377
 378     def __del__(self):
 379         self.clean()
 380
 381     def clean(self):
 382         if self.path and not self._closed:
 383             shutil.rmtree(self.path)
 384             self._closed = True
 385
 386
 387 class PuppeteerRunner(MozbuildObject):
 388     def __init__(self, *args, **kwargs):
 389         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 390
 391         self.remotedir = os.path.join(self.topsrcdir, "remote")
 392         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 393
 394     def run_test(self, logger, *tests, **params):
 395         """
 396         Runs Puppeteer unit tests with npm.
 397
 398         Possible optional test parameters:
 399
 400         `binary`:
 401           Path for the browser binary to use.  Defaults to the local
 402           build.
 403         `headless`:
 404           Boolean to indicate whether to activate Firefox' headless mode.
 405         `extra_prefs`:
 406           Dictionary of extra preferences to write to the profile,
 407           before invoking npm.  Overrides default preferences.
 408         `enable_webrender`:
 409           Boolean to indicate whether to enable WebRender compositor in Gecko.
 410         `write_results`:
 411           Path to write the results json file
 412         `subset`
 413           Indicates only a subset of tests are being run, so we should
 414           skip the check for missing results
 415         """
 416         setup()
 417
 418         binary = params.get("binary") or self.get_binary_path()
 419         product = params.get("product", "firefox")
 420
 421         env = {
 422             # Print browser process ouptut
 423             "DUMPIO": "1",
 424             # Checked by Puppeteer's custom mocha config
 425             "CI": "1",
 426             # Causes some tests to be skipped due to assumptions about install
 427             "PUPPETEER_ALT_INSTALL": "1",
 428         }
 429         extra_options = {}
 430         for k, v in params.get("extra_launcher_options", {}).items():
 431             extra_options[k] = json.loads(v)
 432
 433         # Override upstream defaults: no retries, shorter timeout
 434         mocha_options = [
 435             "--reporter",
 436             "./json-mocha-reporter.js",
 437             "--retries",
 438             "0",
 439             "--fullTrace",
 440             "--timeout",
 441             "20000",
 442             "--no-parallel",
 443         ]
 444         if product == "firefox":
 445             env["BINARY"] = binary
 446             env["PUPPETEER_PRODUCT"] = "firefox"
 447
 448             env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
 449
 450         command = ["run", "unit", "--"] + mocha_options
 451
 452         env["HEADLESS"] = str(params.get("headless", False))
 453
 454         prefs = {}
 455         for k, v in params.get("extra_prefs", {}).items():
 456             prefs[k] = mozprofile.Preferences.cast(v)
 457
 458         if prefs:
 459             extra_options["extraPrefsFirefox"] = prefs
 460
 461         if extra_options:
 462             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 463
 464         expected_path = os.path.join(
 465             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 466         )
 467         if product == "firefox" and os.path.exists(expected_path):
 468             with open(expected_path) as f:
 469                 expected_data = json.load(f)
 470         else:
 471             expected_data = {}
 472
 473         output_handler = MochaOutputHandler(logger, expected_data)
 474         proc = npm(
 475             *command,
 476             cwd=self.puppeteer_dir,
 477             env=env,
 478             processOutputLine=output_handler,
 479             wait=False
 480         )
 481         output_handler.proc = proc
 482
 483         # Puppeteer unit tests don't always clean-up child processes in case of
 484         # failure, so use an output_timeout as a fallback
 485         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 486
 487         output_handler.after_end(params.get("subset", False))
 488
 489         # Non-zero return codes are non-fatal for now since we have some
 490         # issues with unresolved promises that shouldn't otherwise block
 491         # running the tests
 492         if proc.returncode != 0:
 493             logger.warning("npm exited with code %s" % proc.returncode)
 494
 495         if params["write_results"]:
 496             with open(params["write_results"], "w") as f:
 497                 json.dump(
 498                     output_handler.new_expected(), f, indent=2, separators=(",", ": ")
 499                 )
 500
 501         if output_handler.has_unexpected:
 502             exit(1, "Got unexpected results")
 503
 504
 505 def create_parser_puppeteer():
 506     p = argparse.ArgumentParser()
 507     p.add_argument(
 508         "--product", type=str, default="firefox", choices=["chrome", "firefox"]
 509     )
 510     p.add_argument(
 511         "--binary",
 512         type=str,
 513         help="Path to browser binary.  Defaults to local Firefox build.",
 514     )
 515     p.add_argument(
 516         "--ci",
 517         action="store_true",
 518         help="Flag that indicates that tests run in a CI environment.",
 519     )
 520     p.add_argument(
 521         "--enable-fission",
 522         action="store_true",
 523         help="Enable Fission (site isolation) in Gecko.",
 524     )
 525     p.add_argument(
 526         "--enable-webrender",
 527         action="store_true",
 528         help="Enable the WebRender compositor in Gecko.",
 529     )
 530     p.add_argument(
 531         "-z", "--headless", action="store_true", help="Run browser in headless mode."
 532     )
 533     p.add_argument(
 534         "--setpref",
 535         action="append",
 536         dest="extra_prefs",
 537         metavar="<pref>=<value>",
 538         help="Defines additional user preferences.",
 539     )
 540     p.add_argument(
 541         "--setopt",
 542         action="append",
 543         dest="extra_options",
 544         metavar="<option>=<value>",
 545         help="Defines additional options for `puppeteer.launch`.",
 546     )
 547     p.add_argument(
 548         "-v",
 549         dest="verbosity",
 550         action="count",
 551         default=0,
 552         help="Increase remote agent logging verbosity to include "
 553         "debug level messages with -v, trace messages with -vv,"
 554         "and to not truncate long trace messages with -vvv",
 555     )
 556     p.add_argument(
 557         "--write-results",
 558         action="store",
 559         nargs="?",
 560         default=None,
 561         const=os.path.join(
 562             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 563         ),
 564         help="Path to write updated results to (defaults to the "
 565         "expectations file if the argument is provided but "
 566         "no path is passed)",
 567     )
 568     p.add_argument(
 569         "--subset",
 570         action="store_true",
 571         default=False,
 572         help="Indicate that only a subset of the tests are running, "
 573         "so checks for missing tests should be skipped",
 574     )
 575     p.add_argument("tests", nargs="*")
 576     mozlog.commandline.add_logging_group(p)
 577     return p
 578
 579
 580 @CommandProvider
 581 class PuppeteerTest(MachCommandBase):
 582     @Command(
 583         "puppeteer-test",
 584         category="testing",
 585         description="Run Puppeteer unit tests.",
 586         parser=create_parser_puppeteer,
 587     )
 588     def puppeteer_test(
 589         self,
 590         binary=None,
 591         ci=False,
 592         enable_fission=False,
 593         enable_webrender=False,
 594         headless=False,
 595         extra_prefs=None,
 596         extra_options=None,
 597         verbosity=0,
 598         tests=None,
 599         product="firefox",
 600         write_results=None,
 601         subset=False,
 602         **kwargs
 603     ):
 604
 605         self.ci = ci
 606
 607         logger = mozlog.commandline.setup_logging(
 608             "puppeteer-test", kwargs, {"mach": sys.stdout}
 609         )
 610
 611         # moztest calls this programmatically with test objects or manifests
 612         if "test_objects" in kwargs and tests is not None:
 613             logger.error("Expected either 'test_objects' or 'tests'")
 614             exit(1)
 615
 616         if product != "firefox" and extra_prefs is not None:
 617             logger.error("User preferences are not recognized by %s" % product)
 618             exit(1)
 619
 620         if "test_objects" in kwargs:
 621             tests = []
 622             for test in kwargs["test_objects"]:
 623                 tests.append(test["path"])
 624
 625         prefs = {}
 626         for s in extra_prefs or []:
 627             kv = s.split("=")
 628             if len(kv) != 2:
 629                 logger.error("syntax error in --setpref={}".format(s))
 630                 exit(EX_USAGE)
 631             prefs[kv[0]] = kv[1].strip()
 632
 633         options = {}
 634         for s in extra_options or []:
 635             kv = s.split("=")
 636             if len(kv) != 2:
 637                 logger.error("syntax error in --setopt={}".format(s))
 638                 exit(EX_USAGE)
 639             options[kv[0]] = kv[1].strip()
 640
 641         if enable_fission:
 642             prefs.update(
 643                 {"fission.autostart": True, "dom.serviceWorkers.parent_intercept": True}
 644             )
 645
 646         if verbosity == 1:
 647             prefs["remote.log.level"] = "Debug"
 648         elif verbosity > 1:
 649             prefs["remote.log.level"] = "Trace"
 650         if verbosity > 2:
 651             prefs["remote.log.truncate"] = False
 652
 653         self.install_puppeteer(product)
 654
 655         params = {
 656             "binary": binary,
 657             "headless": headless,
 658             "enable_webrender": enable_webrender,
 659             "extra_prefs": prefs,
 660             "product": product,
 661             "extra_launcher_options": options,
 662             "write_results": write_results,
 663             "subset": subset,
 664         }
 665         puppeteer = self._spawn(PuppeteerRunner)
 666         try:
 667             return puppeteer.run_test(logger, *tests, **params)
 668         except BinaryNotFoundException as e:
 669             logger.error(e)
 670             logger.info(e.help())
 671             exit(1)
 672         except Exception as e:
 673             exit(EX_SOFTWARE, e)
 674
 675     def install_puppeteer(self, product):
 676         setup()
 677         env = {}
 678         from mozversioncontrol import get_repository_object
 679
 680         repo = get_repository_object(self.topsrcdir)
 681         puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 682         changed_files = False
 683         for f in repo.get_changed_files():
 684             if f.startswith(puppeteer_dir) and f.endswith(".ts"):
 685                 changed_files = True
 686                 break
 687
 688         if product != "chrome":
 689             env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 690         lib_dir = os.path.join(self.topsrcdir, puppeteer_dir, "lib")
 691         if changed_files and os.path.isdir(lib_dir):
 692             # clobber lib to force `tsc compile` step
 693             shutil.rmtree(lib_dir)
 694
 695         command = "ci" if self.ci else "install"
 696         npm(command, cwd=os.path.join(self.topsrcdir, puppeteer_dir), env=env)
 697
 698
 699 def exit(code, error=None):
 700     if error is not None:
 701         if isinstance(error, Exception):
 702             import traceback
 703
 704             traceback.print_exc()
 705         else:
 706             message = str(error).split("\n")[0].strip()
 707             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 708     sys.exit(code)