remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import argparse
  12 import json
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import tempfile
  19
  20 from collections import OrderedDict
  21
  22 from six import iteritems
  23
  24 from mach.decorators import (
  25     Command,
  26     CommandArgument,
  27     CommandProvider,
  28     SubCommand,
  29 )
  30
  31 from mozbuild.base import (
  32     MachCommandBase,
  33     MozbuildObject,
  34     BinaryNotFoundException,
  35 )
  36 from mozbuild import nodeutil
  37 import mozlog
  38 import mozprofile
  39
  40
  41 EX_CONFIG = 78
  42 EX_SOFTWARE = 70
  43 EX_USAGE = 64
  44
  45
  46 def setup():
  47     # add node and npm from mozbuild to front of system path
  48     npm, _ = nodeutil.find_npm_executable()
  49     if not npm:
  50         exit(EX_CONFIG, "could not find npm executable")
  51     path = os.path.abspath(os.path.join(npm, os.pardir))
  52     os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
  53
  54
  55 @CommandProvider
  56 class RemoteCommands(MachCommandBase):
  57     def remotedir(self, command_context):
  58         return os.path.join(command_context.topsrcdir, "remote")
  59
  60     @Command(
  61         "remote", category="misc", description="Remote protocol related operations."
  62     )
  63     def remote(self, command_context):
  64         """The remote subcommands all relate to the remote protocol."""
  65         command_context._sub_mach(["help", "remote"])
  66         return 1
  67
  68     @SubCommand(
  69         "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
  70     )
  71     @CommandArgument(
  72         "--repository",
  73         metavar="REPO",
  74         required=True,
  75         help="The (possibly remote) repository to clone from.",
  76     )
  77     @CommandArgument(
  78         "--commitish",
  79         metavar="COMMITISH",
  80         required=True,
  81         help="The commit or tag object name to check out.",
  82     )
  83     @CommandArgument(
  84         "--no-install",
  85         dest="install",
  86         action="store_false",
  87         default=True,
  88         help="Do not install the just-pulled Puppeteer package,",
  89     )
  90     def vendor_puppeteer(self, command_context, repository, commitish, install):
  91         puppeteer_dir = os.path.join(
  92             self.remotedir(command_context), "test", "puppeteer"
  93         )
  94
  95         # Preserve our custom mocha reporter
  96         shutil.move(
  97             os.path.join(puppeteer_dir, "json-mocha-reporter.js"),
  98             self.remotedir(command_context),
  99         )
 100         shutil.rmtree(puppeteer_dir, ignore_errors=True)
 101         os.makedirs(puppeteer_dir)
 102         with TemporaryDirectory() as tmpdir:
 103             git("clone", "-q", repository, tmpdir)
 104             git("checkout", commitish, worktree=tmpdir)
 105             git(
 106                 "checkout-index",
 107                 "-a",
 108                 "-f",
 109                 "--prefix",
 110                 "{}/".format(puppeteer_dir),
 111                 worktree=tmpdir,
 112             )
 113
 114         # remove files which may interfere with git checkout of central
 115         try:
 116             os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
 117             os.remove(os.path.join(puppeteer_dir, ".gitignore"))
 118         except OSError:
 119             pass
 120
 121         unwanted_dirs = ["experimental", "docs"]
 122
 123         for dir in unwanted_dirs:
 124             dir_path = os.path.join(puppeteer_dir, dir)
 125             if os.path.isdir(dir_path):
 126                 shutil.rmtree(dir_path)
 127
 128         shutil.move(
 129             os.path.join(self.remotedir(command_context), "json-mocha-reporter.js"),
 130             puppeteer_dir,
 131         )
 132
 133         import yaml
 134
 135         annotation = {
 136             "schema": 1,
 137             "bugzilla": {
 138                 "product": "Remote Protocol",
 139                 "component": "Agent",
 140             },
 141             "origin": {
 142                 "name": "puppeteer",
 143                 "description": "Headless Chrome Node API",
 144                 "url": repository,
 145                 "license": "Apache-2.0",
 146                 "release": commitish,
 147             },
 148         }
 149         with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 150             yaml.safe_dump(
 151                 annotation,
 152                 fh,
 153                 default_flow_style=False,
 154                 encoding="utf-8",
 155                 allow_unicode=True,
 156             )
 157
 158         if install:
 159             env = {"PUPPETEER_SKIP_DOWNLOAD": "1"}
 160             npm(
 161                 "install",
 162                 cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
 163                 env=env,
 164             )
 165
 166
 167 def git(*args, **kwargs):
 168     cmd = ("git",)
 169     if kwargs.get("worktree"):
 170         cmd += ("-C", kwargs["worktree"])
 171     cmd += args
 172
 173     pipe = kwargs.get("pipe")
 174     git_p = subprocess.Popen(
 175         cmd,
 176         env={"GIT_CONFIG_NOSYSTEM": "1"},
 177         stdout=subprocess.PIPE,
 178         stderr=subprocess.PIPE,
 179     )
 180     pipe_p = None
 181     if pipe:
 182         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 183
 184     if pipe:
 185         _, pipe_err = pipe_p.communicate()
 186     out, git_err = git_p.communicate()
 187
 188     # use error from first program that failed
 189     if git_p.returncode > 0:
 190         exit(EX_SOFTWARE, git_err)
 191     if pipe and pipe_p.returncode > 0:
 192         exit(EX_SOFTWARE, pipe_err)
 193
 194     return out
 195
 196
 197 def npm(*args, **kwargs):
 198     from mozprocess import processhandler
 199
 200     env = None
 201     if kwargs.get("env"):
 202         env = os.environ.copy()
 203         env.update(kwargs["env"])
 204
 205     proc_kwargs = {}
 206     if "processOutputLine" in kwargs:
 207         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 208
 209     p = processhandler.ProcessHandler(
 210         cmd="npm",
 211         args=list(args),
 212         cwd=kwargs.get("cwd"),
 213         env=env,
 214         universal_newlines=True,
 215         **proc_kwargs
 216     )
 217     if not kwargs.get("wait", True):
 218         return p
 219
 220     wait_proc(p, cmd="npm", exit_on_fail=kwargs.get("exit_on_fail", True))
 221
 222     return p.returncode
 223
 224
 225 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 226     try:
 227         p.run(outputTimeout=output_timeout)
 228         p.wait()
 229         if p.timedOut:
 230             # In some cases, we wait longer for a mocha timeout
 231             print("Timed out after {} seconds of no output".format(output_timeout))
 232     finally:
 233         p.kill()
 234     if exit_on_fail and p.returncode > 0:
 235         msg = (
 236             "%s: exit code %s" % (cmd, p.returncode)
 237             if cmd
 238             else "exit code %s" % p.returncode
 239         )
 240         exit(p.returncode, msg)
 241
 242
 243 class MochaOutputHandler(object):
 244     def __init__(self, logger, expected):
 245         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 246
 247         self.logger = logger
 248         self.proc = None
 249         self.test_results = OrderedDict()
 250         self.expected = expected
 251         self.unexpected_skips = set()
 252
 253         self.has_unexpected = False
 254         self.logger.suite_start([], name="puppeteer-tests")
 255         self.status_map = {
 256             "CRASHED": "CRASH",
 257             "OK": "PASS",
 258             "TERMINATED": "CRASH",
 259             "pass": "PASS",
 260             "fail": "FAIL",
 261             "pending": "SKIP",
 262         }
 263
 264     @property
 265     def pid(self):
 266         return self.proc and self.proc.pid
 267
 268     def __call__(self, line):
 269         event = None
 270         try:
 271             if line.startswith("[") and line.endswith("]"):
 272                 event = json.loads(line)
 273             self.process_event(event)
 274         except ValueError:
 275             pass
 276         finally:
 277             self.logger.process_output(self.pid, line, command="npm")
 278
 279     def process_event(self, event):
 280         if isinstance(event, list) and len(event) > 1:
 281             status = self.status_map.get(event[0])
 282             test_start = event[0] == "test-start"
 283             if not status and not test_start:
 284                 return
 285             test_info = event[1]
 286             test_name = test_info.get("fullTitle", "")
 287             test_path = test_info.get("file", "")
 288             test_err = test_info.get("err")
 289             if status == "FAIL" and test_err:
 290                 if "timeout" in test_err.lower():
 291                     status = "TIMEOUT"
 292             if test_name and test_path:
 293                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 294             # mocha hook failures are not tracked in metadata
 295             if status != "PASS" and self.hook_re.search(test_name):
 296                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 297                 return
 298             if test_start:
 299                 self.logger.test_start(test_name)
 300                 return
 301             expected = self.expected.get(test_name, ["PASS"])
 302             # mozlog doesn't really allow unexpected skip,
 303             # so if a test is disabled just expect that and note the unexpected skip
 304             # Also, mocha doesn't log test-start for skipped tests
 305             if status == "SKIP":
 306                 self.logger.test_start(test_name)
 307                 if self.expected and status not in expected:
 308                     self.unexpected_skips.add(test_name)
 309                 expected = ["SKIP"]
 310             known_intermittent = expected[1:]
 311             expected_status = expected[0]
 312
 313             # check if we've seen a result for this test before this log line
 314             result_recorded = self.test_results.get(test_name)
 315             if result_recorded:
 316                 self.logger.warning(
 317                     "Received a second status for {}: "
 318                     "first {}, now {}".format(test_name, result_recorded, status)
 319                 )
 320             # mocha intermittently logs an additional test result after the
 321             # test has already timed out. Avoid recording this second status.
 322             if result_recorded != "TIMEOUT":
 323                 self.test_results[test_name] = status
 324                 if status not in expected:
 325                     self.has_unexpected = True
 326             self.logger.test_end(
 327                 test_name,
 328                 status=status,
 329                 expected=expected_status,
 330                 known_intermittent=known_intermittent,
 331             )
 332
 333     def new_expected(self):
 334         new_expected = OrderedDict()
 335         for test_name, status in iteritems(self.test_results):
 336             if test_name not in self.expected:
 337                 new_status = [status]
 338             else:
 339                 if status in self.expected[test_name]:
 340                     new_status = self.expected[test_name]
 341                 else:
 342                     new_status = [status]
 343             new_expected[test_name] = new_status
 344         return new_expected
 345
 346     def after_end(self, subset=False):
 347         if not subset:
 348             missing = set(self.expected) - set(self.test_results)
 349             extra = set(self.test_results) - set(self.expected)
 350             if missing:
 351                 self.has_unexpected = True
 352                 for test_name in missing:
 353                     self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,))
 354             if self.expected and extra:
 355                 self.has_unexpected = True
 356                 for test_name in extra:
 357                     self.logger.error(
 358                         "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,)
 359                     )
 360
 361         if self.unexpected_skips:
 362             self.has_unexpected = True
 363             for test_name in self.unexpected_skips:
 364                 self.logger.error(
 365                     "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
 366                 )
 367         self.logger.suite_end()
 368
 369
 370 # tempfile.TemporaryDirectory missing from Python 2.7
 371 class TemporaryDirectory(object):
 372     def __init__(self):
 373         self.path = tempfile.mkdtemp()
 374         self._closed = False
 375
 376     def __repr__(self):
 377         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 378
 379     def __enter__(self):
 380         return self.path
 381
 382     def __exit__(self, exc, value, tb):
 383         self.clean()
 384
 385     def __del__(self):
 386         self.clean()
 387
 388     def clean(self):
 389         if self.path and not self._closed:
 390             shutil.rmtree(self.path)
 391             self._closed = True
 392
 393
 394 class PuppeteerRunner(MozbuildObject):
 395     def __init__(self, *args, **kwargs):
 396         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 397
 398         self.remotedir = os.path.join(self.topsrcdir, "remote")
 399         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 400
 401     def run_test(self, logger, *tests, **params):
 402         """
 403         Runs Puppeteer unit tests with npm.
 404
 405         Possible optional test parameters:
 406
 407         `binary`:
 408           Path for the browser binary to use.  Defaults to the local
 409           build.
 410         `headless`:
 411           Boolean to indicate whether to activate Firefox' headless mode.
 412         `extra_prefs`:
 413           Dictionary of extra preferences to write to the profile,
 414           before invoking npm.  Overrides default preferences.
 415         `enable_webrender`:
 416           Boolean to indicate whether to enable WebRender compositor in Gecko.
 417         `write_results`:
 418           Path to write the results json file
 419         `subset`
 420           Indicates only a subset of tests are being run, so we should
 421           skip the check for missing results
 422         """
 423         setup()
 424
 425         binary = params.get("binary") or self.get_binary_path()
 426         product = params.get("product", "firefox")
 427
 428         env = {
 429             # Print browser process ouptut
 430             "DUMPIO": "1",
 431             # Checked by Puppeteer's custom mocha config
 432             "CI": "1",
 433             # Causes some tests to be skipped due to assumptions about install
 434             "PUPPETEER_ALT_INSTALL": "1",
 435         }
 436         extra_options = {}
 437         for k, v in params.get("extra_launcher_options", {}).items():
 438             extra_options[k] = json.loads(v)
 439
 440         # Override upstream defaults: no retries, shorter timeout
 441         mocha_options = [
 442             "--reporter",
 443             "./json-mocha-reporter.js",
 444             "--retries",
 445             "0",
 446             "--fullTrace",
 447             "--timeout",
 448             "20000",
 449             "--no-parallel",
 450         ]
 451         if product == "firefox":
 452             env["BINARY"] = binary
 453             env["PUPPETEER_PRODUCT"] = "firefox"
 454
 455             env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
 456
 457         command = ["run", "unit", "--"] + mocha_options
 458
 459         env["HEADLESS"] = str(params.get("headless", False))
 460
 461         prefs = {}
 462         for k, v in params.get("extra_prefs", {}).items():
 463             prefs[k] = mozprofile.Preferences.cast(v)
 464
 465         if prefs:
 466             extra_options["extraPrefsFirefox"] = prefs
 467
 468         if extra_options:
 469             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 470
 471         expected_path = os.path.join(
 472             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 473         )
 474         if product == "firefox" and os.path.exists(expected_path):
 475             with open(expected_path) as f:
 476                 expected_data = json.load(f)
 477         else:
 478             expected_data = {}
 479
 480         output_handler = MochaOutputHandler(logger, expected_data)
 481         proc = npm(
 482             *command,
 483             cwd=self.puppeteer_dir,
 484             env=env,
 485             processOutputLine=output_handler,
 486             wait=False
 487         )
 488         output_handler.proc = proc
 489
 490         # Puppeteer unit tests don't always clean-up child processes in case of
 491         # failure, so use an output_timeout as a fallback
 492         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 493
 494         output_handler.after_end(params.get("subset", False))
 495
 496         # Non-zero return codes are non-fatal for now since we have some
 497         # issues with unresolved promises that shouldn't otherwise block
 498         # running the tests
 499         if proc.returncode != 0:
 500             logger.warning("npm exited with code %s" % proc.returncode)
 501
 502         if params["write_results"]:
 503             with open(params["write_results"], "w") as f:
 504                 json.dump(
 505                     output_handler.new_expected(), f, indent=2, separators=(",", ": ")
 506                 )
 507
 508         if output_handler.has_unexpected:
 509             exit(1, "Got unexpected results")
 510
 511
 512 def create_parser_puppeteer():
 513     p = argparse.ArgumentParser()
 514     p.add_argument(
 515         "--product", type=str, default="firefox", choices=["chrome", "firefox"]
 516     )
 517     p.add_argument(
 518         "--binary",
 519         type=str,
 520         help="Path to browser binary.  Defaults to local Firefox build.",
 521     )
 522     p.add_argument(
 523         "--ci",
 524         action="store_true",
 525         help="Flag that indicates that tests run in a CI environment.",
 526     )
 527     p.add_argument(
 528         "--enable-fission",
 529         action="store_true",
 530         help="Enable Fission (site isolation) in Gecko.",
 531     )
 532     p.add_argument(
 533         "--enable-webrender",
 534         action="store_true",
 535         help="Enable the WebRender compositor in Gecko.",
 536     )
 537     p.add_argument(
 538         "-z", "--headless", action="store_true", help="Run browser in headless mode."
 539     )
 540     p.add_argument(
 541         "--setpref",
 542         action="append",
 543         dest="extra_prefs",
 544         metavar="<pref>=<value>",
 545         help="Defines additional user preferences.",
 546     )
 547     p.add_argument(
 548         "--setopt",
 549         action="append",
 550         dest="extra_options",
 551         metavar="<option>=<value>",
 552         help="Defines additional options for `puppeteer.launch`.",
 553     )
 554     p.add_argument(
 555         "-v",
 556         dest="verbosity",
 557         action="count",
 558         default=0,
 559         help="Increase remote agent logging verbosity to include "
 560         "debug level messages with -v, trace messages with -vv,"
 561         "and to not truncate long trace messages with -vvv",
 562     )
 563     p.add_argument(
 564         "--write-results",
 565         action="store",
 566         nargs="?",
 567         default=None,
 568         const=os.path.join(
 569             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 570         ),
 571         help="Path to write updated results to (defaults to the "
 572         "expectations file if the argument is provided but "
 573         "no path is passed)",
 574     )
 575     p.add_argument(
 576         "--subset",
 577         action="store_true",
 578         default=False,
 579         help="Indicate that only a subset of the tests are running, "
 580         "so checks for missing tests should be skipped",
 581     )
 582     p.add_argument("tests", nargs="*")
 583     mozlog.commandline.add_logging_group(p)
 584     return p
 585
 586
 587 @CommandProvider
 588 class PuppeteerTest(MachCommandBase):
 589     @Command(
 590         "puppeteer-test",
 591         category="testing",
 592         description="Run Puppeteer unit tests.",
 593         parser=create_parser_puppeteer,
 594     )
 595     def puppeteer_test(
 596         self,
 597         command_context,
 598         binary=None,
 599         ci=False,
 600         enable_fission=False,
 601         enable_webrender=False,
 602         headless=False,
 603         extra_prefs=None,
 604         extra_options=None,
 605         verbosity=0,
 606         tests=None,
 607         product="firefox",
 608         write_results=None,
 609         subset=False,
 610         **kwargs
 611     ):
 612
 613         logger = mozlog.commandline.setup_logging(
 614             "puppeteer-test", kwargs, {"mach": sys.stdout}
 615         )
 616
 617         # moztest calls this programmatically with test objects or manifests
 618         if "test_objects" in kwargs and tests is not None:
 619             logger.error("Expected either 'test_objects' or 'tests'")
 620             exit(1)
 621
 622         if product != "firefox" and extra_prefs is not None:
 623             logger.error("User preferences are not recognized by %s" % product)
 624             exit(1)
 625
 626         if "test_objects" in kwargs:
 627             tests = []
 628             for test in kwargs["test_objects"]:
 629                 tests.append(test["path"])
 630
 631         prefs = {}
 632         for s in extra_prefs or []:
 633             kv = s.split("=")
 634             if len(kv) != 2:
 635                 logger.error("syntax error in --setpref={}".format(s))
 636                 exit(EX_USAGE)
 637             prefs[kv[0]] = kv[1].strip()
 638
 639         options = {}
 640         for s in extra_options or []:
 641             kv = s.split("=")
 642             if len(kv) != 2:
 643                 logger.error("syntax error in --setopt={}".format(s))
 644                 exit(EX_USAGE)
 645             options[kv[0]] = kv[1].strip()
 646
 647         if enable_fission:
 648             prefs.update({"fission.autostart": True})
 649
 650         if verbosity == 1:
 651             prefs["remote.log.level"] = "Debug"
 652         elif verbosity > 1:
 653             prefs["remote.log.level"] = "Trace"
 654         if verbosity > 2:
 655             prefs["remote.log.truncate"] = False
 656
 657         self.install_puppeteer(command_context, product, ci)
 658
 659         params = {
 660             "binary": binary,
 661             "headless": headless,
 662             "enable_webrender": enable_webrender,
 663             "extra_prefs": prefs,
 664             "product": product,
 665             "extra_launcher_options": options,
 666             "write_results": write_results,
 667             "subset": subset,
 668         }
 669         puppeteer = command_context._spawn(PuppeteerRunner)
 670         try:
 671             return puppeteer.run_test(logger, *tests, **params)
 672         except BinaryNotFoundException as e:
 673             logger.error(e)
 674             logger.info(e.help())
 675             exit(1)
 676         except Exception as e:
 677             exit(EX_SOFTWARE, e)
 678
 679     def install_puppeteer(self, command_context, product, ci):
 680         setup()
 681         env = {}
 682         from mozversioncontrol import get_repository_object
 683
 684         repo = get_repository_object(command_context.topsrcdir)
 685         puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 686         changed_files = False
 687         for f in repo.get_changed_files():
 688             if f.startswith(puppeteer_dir) and f.endswith(".ts"):
 689                 changed_files = True
 690                 break
 691
 692         if product != "chrome":
 693             env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 694         lib_dir = os.path.join(command_context.topsrcdir, puppeteer_dir, "lib")
 695         if changed_files and os.path.isdir(lib_dir):
 696             # clobber lib to force `tsc compile` step
 697             shutil.rmtree(lib_dir)
 698
 699         command = "ci" if ci else "install"
 700         npm(
 701             command, cwd=os.path.join(command_context.topsrcdir, puppeteer_dir), env=env
 702         )
 703
 704
 705 def exit(code, error=None):
 706     if error is not None:
 707         if isinstance(error, Exception):
 708             import traceback
 709
 710             traceback.print_exc()
 711         else:
 712             message = str(error).split("\n")[0].strip()
 713             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 714     sys.exit(code)