remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import argparse
   6 import json
   7 import os
   8 import platform
   9 import re
  10 import shutil
  11 import subprocess
  12 import sys
  13 import tempfile
  14 from collections import OrderedDict
  15
  16 import mozlog
  17 import mozprofile
  18 from mach.decorators import Command, CommandArgument, SubCommand
  19 from mozbuild import nodeutil
  20 from mozbuild.base import BinaryNotFoundException, MozbuildObject
  21
  22 EX_CONFIG = 78
  23 EX_SOFTWARE = 70
  24 EX_USAGE = 64
  25
  26
  27 def setup():
  28     # add node and npm from mozbuild to front of system path
  29     npm, _ = nodeutil.find_npm_executable()
  30     if not npm:
  31         exit(EX_CONFIG, "could not find npm executable")
  32     path = os.path.abspath(os.path.join(npm, os.pardir))
  33     os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
  34
  35
  36 def remotedir(command_context):
  37     return os.path.join(command_context.topsrcdir, "remote")
  38
  39
  40 @Command("remote", category="misc", description="Remote protocol related operations.")
  41 def remote(command_context):
  42     """The remote subcommands all relate to the remote protocol."""
  43     command_context._sub_mach(["help", "remote"])
  44     return 1
  45
  46
  47 @SubCommand(
  48     "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
  49 )
  50 @CommandArgument(
  51     "--repository",
  52     metavar="REPO",
  53     required=True,
  54     help="The (possibly remote) repository to clone from.",
  55 )
  56 @CommandArgument(
  57     "--commitish",
  58     metavar="COMMITISH",
  59     required=True,
  60     help="The commit or tag object name to check out.",
  61 )
  62 @CommandArgument(
  63     "--no-install",
  64     dest="install",
  65     action="store_false",
  66     default=True,
  67     help="Do not install the just-pulled Puppeteer package,",
  68 )
  69 def vendor_puppeteer(command_context, repository, commitish, install):
  70     puppeteer_dir = os.path.join(remotedir(command_context), "test", "puppeteer")
  71
  72     # Preserve our custom mocha reporter
  73     shutil.move(
  74         os.path.join(puppeteer_dir, "json-mocha-reporter.js"),
  75         remotedir(command_context),
  76     )
  77     shutil.rmtree(puppeteer_dir, ignore_errors=True)
  78     os.makedirs(puppeteer_dir)
  79     with TemporaryDirectory() as tmpdir:
  80         git("clone", "-q", repository, tmpdir)
  81         git("checkout", commitish, worktree=tmpdir)
  82         git(
  83             "checkout-index",
  84             "-a",
  85             "-f",
  86             "--prefix",
  87             "{}/".format(puppeteer_dir),
  88             worktree=tmpdir,
  89         )
  90
  91     # remove files which may interfere with git checkout of central
  92     try:
  93         os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
  94         os.remove(os.path.join(puppeteer_dir, ".gitignore"))
  95     except OSError:
  96         pass
  97
  98     unwanted_dirs = ["experimental", "docs"]
  99
 100     for dir in unwanted_dirs:
 101         dir_path = os.path.join(puppeteer_dir, dir)
 102         if os.path.isdir(dir_path):
 103             shutil.rmtree(dir_path)
 104
 105     shutil.move(
 106         os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
 107         puppeteer_dir,
 108     )
 109
 110     import yaml
 111
 112     annotation = {
 113         "schema": 1,
 114         "bugzilla": {
 115             "product": "Remote Protocol",
 116             "component": "Agent",
 117         },
 118         "origin": {
 119             "name": "puppeteer",
 120             "description": "Headless Chrome Node API",
 121             "url": repository,
 122             "license": "Apache-2.0",
 123             "release": commitish,
 124         },
 125     }
 126     with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 127         yaml.safe_dump(
 128             annotation,
 129             fh,
 130             default_flow_style=False,
 131             encoding="utf-8",
 132             allow_unicode=True,
 133         )
 134
 135     if install:
 136         env = {"HUSKY": "0", "PUPPETEER_SKIP_DOWNLOAD": "1"}
 137         npm(
 138             "install",
 139             cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
 140             env=env,
 141         )
 142
 143
 144 def git(*args, **kwargs):
 145     cmd = ("git",)
 146     if kwargs.get("worktree"):
 147         cmd += ("-C", kwargs["worktree"])
 148     cmd += args
 149
 150     pipe = kwargs.get("pipe")
 151     git_p = subprocess.Popen(
 152         cmd,
 153         env={"GIT_CONFIG_NOSYSTEM": "1"},
 154         stdout=subprocess.PIPE,
 155         stderr=subprocess.PIPE,
 156     )
 157     pipe_p = None
 158     if pipe:
 159         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 160
 161     if pipe:
 162         _, pipe_err = pipe_p.communicate()
 163     out, git_err = git_p.communicate()
 164
 165     # use error from first program that failed
 166     if git_p.returncode > 0:
 167         exit(EX_SOFTWARE, git_err)
 168     if pipe and pipe_p.returncode > 0:
 169         exit(EX_SOFTWARE, pipe_err)
 170
 171     return out
 172
 173
 174 def npm(*args, **kwargs):
 175     from mozprocess import processhandler
 176
 177     env = None
 178     npm, _ = nodeutil.find_npm_executable()
 179     if kwargs.get("env"):
 180         env = os.environ.copy()
 181         env.update(kwargs["env"])
 182
 183     proc_kwargs = {}
 184     if "processOutputLine" in kwargs:
 185         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 186
 187     p = processhandler.ProcessHandler(
 188         cmd=npm,
 189         args=list(args),
 190         cwd=kwargs.get("cwd"),
 191         env=env,
 192         universal_newlines=True,
 193         **proc_kwargs,
 194     )
 195     if not kwargs.get("wait", True):
 196         return p
 197
 198     wait_proc(p, cmd=npm, exit_on_fail=kwargs.get("exit_on_fail", True))
 199
 200     return p.returncode
 201
 202
 203 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 204     try:
 205         p.run(outputTimeout=output_timeout)
 206         p.wait()
 207         if p.timedOut:
 208             # In some cases, we wait longer for a mocha timeout
 209             print("Timed out after {} seconds of no output".format(output_timeout))
 210     finally:
 211         p.kill()
 212     if exit_on_fail and p.returncode > 0:
 213         msg = (
 214             "%s: exit code %s" % (cmd, p.returncode)
 215             if cmd
 216             else "exit code %s" % p.returncode
 217         )
 218         exit(p.returncode, msg)
 219
 220
 221 class MochaOutputHandler(object):
 222     def __init__(self, logger, expected):
 223         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 224
 225         self.logger = logger
 226         self.proc = None
 227         self.test_results = OrderedDict()
 228         self.expected = expected
 229         self.unexpected_skips = set()
 230
 231         self.has_unexpected = False
 232         self.logger.suite_start([], name="puppeteer-tests")
 233         self.status_map = {
 234             "CRASHED": "CRASH",
 235             "OK": "PASS",
 236             "TERMINATED": "CRASH",
 237             "pass": "PASS",
 238             "fail": "FAIL",
 239             "pending": "SKIP",
 240         }
 241
 242     @property
 243     def pid(self):
 244         return self.proc and self.proc.pid
 245
 246     def __call__(self, line):
 247         event = None
 248         try:
 249             if line.startswith("[") and line.endswith("]"):
 250                 event = json.loads(line)
 251             self.process_event(event)
 252         except ValueError:
 253             pass
 254         finally:
 255             self.logger.process_output(self.pid, line, command="npm")
 256
 257     def process_event(self, event):
 258         if isinstance(event, list) and len(event) > 1:
 259             status = self.status_map.get(event[0])
 260             test_start = event[0] == "test-start"
 261             if not status and not test_start:
 262                 return
 263             test_info = event[1]
 264             test_full_title = test_info.get("fullTitle", "")
 265             test_name = test_full_title
 266             test_path = test_info.get("file", "")
 267             test_file_name = os.path.basename(test_path).replace(".js", "")
 268             test_err = test_info.get("err")
 269             if status == "FAIL" and test_err:
 270                 if "timeout" in test_err.lower():
 271                     status = "TIMEOUT"
 272             if test_name and test_path:
 273                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 274             # mocha hook failures are not tracked in metadata
 275             if status != "PASS" and self.hook_re.search(test_name):
 276                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 277                 return
 278             if test_start:
 279                 self.logger.test_start(test_name)
 280                 return
 281             expected_name = "[{}] {}".format(test_file_name, test_full_title)
 282             expected_item = next(
 283                 (
 284                     expectation
 285                     for expectation in list(self.expected)
 286                     if expectation["testIdPattern"] == expected_name
 287                 ),
 288                 None,
 289             )
 290             if expected_item is None:
 291                 # if there is no expectation data for a specific test case,
 292                 # try to find data for a whole file.
 293                 expected_item_for_file = next(
 294                     (
 295                         expectation
 296                         for expectation in list(self.expected)
 297                         if expectation["testIdPattern"] == f"[{test_file_name}]"
 298                     ),
 299                     None,
 300                 )
 301                 if expected_item_for_file is None:
 302                     expected = ["PASS"]
 303                 else:
 304                     expected = expected_item_for_file["expectations"]
 305             else:
 306                 expected = expected_item["expectations"]
 307             # mozlog doesn't really allow unexpected skip,
 308             # so if a test is disabled just expect that and note the unexpected skip
 309             # Also, mocha doesn't log test-start for skipped tests
 310             if status == "SKIP":
 311                 self.logger.test_start(test_name)
 312                 if self.expected and status not in expected:
 313                     self.unexpected_skips.add(test_name)
 314                 expected = ["SKIP"]
 315             known_intermittent = expected[1:]
 316             expected_status = expected[0]
 317
 318             # check if we've seen a result for this test before this log line
 319             result_recorded = self.test_results.get(test_name)
 320             if result_recorded:
 321                 self.logger.warning(
 322                     "Received a second status for {}: "
 323                     "first {}, now {}".format(test_name, result_recorded, status)
 324                 )
 325             # mocha intermittently logs an additional test result after the
 326             # test has already timed out. Avoid recording this second status.
 327             if result_recorded != "TIMEOUT":
 328                 self.test_results[test_name] = status
 329                 if status not in expected:
 330                     self.has_unexpected = True
 331             self.logger.test_end(
 332                 test_name,
 333                 status=status,
 334                 expected=expected_status,
 335                 known_intermittent=known_intermittent,
 336             )
 337
 338     def after_end(self):
 339         if self.unexpected_skips:
 340             self.has_unexpected = True
 341             for test_name in self.unexpected_skips:
 342                 self.logger.error(
 343                     "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
 344                 )
 345         self.logger.suite_end()
 346
 347
 348 # tempfile.TemporaryDirectory missing from Python 2.7
 349 class TemporaryDirectory(object):
 350     def __init__(self):
 351         self.path = tempfile.mkdtemp()
 352         self._closed = False
 353
 354     def __repr__(self):
 355         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 356
 357     def __enter__(self):
 358         return self.path
 359
 360     def __exit__(self, exc, value, tb):
 361         self.clean()
 362
 363     def __del__(self):
 364         self.clean()
 365
 366     def clean(self):
 367         if self.path and not self._closed:
 368             shutil.rmtree(self.path)
 369             self._closed = True
 370
 371
 372 class PuppeteerRunner(MozbuildObject):
 373     def __init__(self, *args, **kwargs):
 374         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 375
 376         self.remotedir = os.path.join(self.topsrcdir, "remote")
 377         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 378
 379     def run_test(self, logger, *tests, **params):
 380         """
 381         Runs Puppeteer unit tests with npm.
 382
 383         Possible optional test parameters:
 384
 385         `binary`:
 386           Path for the browser binary to use.  Defaults to the local
 387           build.
 388         `headless`:
 389           Boolean to indicate whether to activate Firefox' headless mode.
 390         `extra_prefs`:
 391           Dictionary of extra preferences to write to the profile,
 392           before invoking npm.  Overrides default preferences.
 393         `enable_webrender`:
 394           Boolean to indicate whether to enable WebRender compositor in Gecko.
 395         `subset`
 396           Indicates only a subset of tests are being run, so we should
 397           skip the check for missing results
 398         """
 399         setup()
 400
 401         binary = params.get("binary") or self.get_binary_path()
 402         product = params.get("product", "firefox")
 403
 404         env = {
 405             # Print browser process ouptut
 406             "DUMPIO": "1",
 407             # Checked by Puppeteer's custom mocha config
 408             "CI": "1",
 409             # Causes some tests to be skipped due to assumptions about install
 410             "PUPPETEER_ALT_INSTALL": "1",
 411         }
 412         extra_options = {}
 413         for k, v in params.get("extra_launcher_options", {}).items():
 414             extra_options[k] = json.loads(v)
 415
 416         # Override upstream defaults: no retries, shorter timeout
 417         mocha_options = [
 418             "--reporter",
 419             "./json-mocha-reporter.js",
 420             "--retries",
 421             "0",
 422             "--fullTrace",
 423             "--timeout",
 424             "20000",
 425             "--no-parallel",
 426             "--no-coverage",
 427         ]
 428         env["HEADLESS"] = str(params.get("headless", False))
 429
 430         if product == "firefox":
 431             env["BINARY"] = binary
 432             env["PUPPETEER_PRODUCT"] = "firefox"
 433
 434             env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
 435
 436             test_command = "test:firefox"
 437         elif env["HEADLESS"] == "False":
 438             test_command = "test:chrome:headful"
 439         else:
 440             test_command = "test:chrome:headless"
 441
 442         command = ["run", test_command, "--"] + mocha_options
 443
 444         prefs = {}
 445         for k, v in params.get("extra_prefs", {}).items():
 446             print("Using extra preference: {}={}".format(k, v))
 447             prefs[k] = mozprofile.Preferences.cast(v)
 448
 449         if prefs:
 450             extra_options["extraPrefsFirefox"] = prefs
 451
 452         if extra_options:
 453             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 454
 455         expected_path = os.path.join(
 456             os.path.dirname(__file__),
 457             "test",
 458             "puppeteer",
 459             "test",
 460             "TestExpectations.json",
 461         )
 462         if os.path.exists(expected_path):
 463             with open(expected_path) as f:
 464                 expected_data = json.load(f)
 465         else:
 466             expected_data = []
 467         # Filter expectation data for the selected browser,
 468         # headless or headful mode, and the operating system.
 469         expected_platform = platform.uname().system.lower()
 470         if expected_platform == "windows":
 471             expected_platform = "win32"
 472
 473         expectations = filter(
 474             lambda el: product in el["parameters"]
 475             and (
 476                 (env["HEADLESS"] == "False" and "headless" not in el["parameters"])
 477                 or "headful" not in el["parameters"]
 478             )
 479             and expected_platform in el["platforms"],
 480             expected_data,
 481         )
 482
 483         output_handler = MochaOutputHandler(logger, list(expectations))
 484         proc = npm(
 485             *command,
 486             cwd=self.puppeteer_dir,
 487             env=env,
 488             processOutputLine=output_handler,
 489             wait=False,
 490         )
 491         output_handler.proc = proc
 492
 493         # Puppeteer unit tests don't always clean-up child processes in case of
 494         # failure, so use an output_timeout as a fallback
 495         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 496
 497         output_handler.after_end()
 498
 499         # Non-zero return codes are non-fatal for now since we have some
 500         # issues with unresolved promises that shouldn't otherwise block
 501         # running the tests
 502         if proc.returncode != 0:
 503             logger.warning("npm exited with code %s" % proc.returncode)
 504
 505         if output_handler.has_unexpected:
 506             exit(1, "Got unexpected results")
 507
 508
 509 def create_parser_puppeteer():
 510     p = argparse.ArgumentParser()
 511     p.add_argument(
 512         "--product", type=str, default="firefox", choices=["chrome", "firefox"]
 513     )
 514     p.add_argument(
 515         "--binary",
 516         type=str,
 517         help="Path to browser binary.  Defaults to local Firefox build.",
 518     )
 519     p.add_argument(
 520         "--ci",
 521         action="store_true",
 522         help="Flag that indicates that tests run in a CI environment.",
 523     )
 524     p.add_argument(
 525         "--disable-fission",
 526         action="store_true",
 527         default=False,
 528         dest="disable_fission",
 529         help="Disable Fission (site isolation) in Gecko.",
 530     )
 531     p.add_argument(
 532         "--enable-webrender",
 533         action="store_true",
 534         help="Enable the WebRender compositor in Gecko.",
 535     )
 536     p.add_argument(
 537         "-z", "--headless", action="store_true", help="Run browser in headless mode."
 538     )
 539     p.add_argument(
 540         "--setpref",
 541         action="append",
 542         dest="extra_prefs",
 543         metavar="<pref>=<value>",
 544         help="Defines additional user preferences.",
 545     )
 546     p.add_argument(
 547         "--setopt",
 548         action="append",
 549         dest="extra_options",
 550         metavar="<option>=<value>",
 551         help="Defines additional options for `puppeteer.launch`.",
 552     )
 553     p.add_argument(
 554         "-v",
 555         dest="verbosity",
 556         action="count",
 557         default=0,
 558         help="Increase remote agent logging verbosity to include "
 559         "debug level messages with -v, trace messages with -vv,"
 560         "and to not truncate long trace messages with -vvv",
 561     )
 562     p.add_argument(
 563         "--subset",
 564         action="store_true",
 565         default=False,
 566         help="Indicate that only a subset of the tests are running, "
 567         "so checks for missing tests should be skipped",
 568     )
 569     p.add_argument("tests", nargs="*")
 570     mozlog.commandline.add_logging_group(p)
 571     return p
 572
 573
 574 @Command(
 575     "puppeteer-test",
 576     category="testing",
 577     description="Run Puppeteer unit tests.",
 578     parser=create_parser_puppeteer,
 579 )
 580 @CommandArgument(
 581     "--no-install",
 582     dest="install",
 583     action="store_false",
 584     default=True,
 585     help="Do not install the Puppeteer package",
 586 )
 587 def puppeteer_test(
 588     command_context,
 589     binary=None,
 590     ci=False,
 591     disable_fission=False,
 592     enable_webrender=False,
 593     headless=False,
 594     extra_prefs=None,
 595     extra_options=None,
 596     install=False,
 597     verbosity=0,
 598     tests=None,
 599     product="firefox",
 600     subset=False,
 601     **kwargs,
 602 ):
 603
 604     logger = mozlog.commandline.setup_logging(
 605         "puppeteer-test", kwargs, {"mach": sys.stdout}
 606     )
 607
 608     # moztest calls this programmatically with test objects or manifests
 609     if "test_objects" in kwargs and tests is not None:
 610         logger.error("Expected either 'test_objects' or 'tests'")
 611         exit(1)
 612
 613     if product != "firefox" and extra_prefs is not None:
 614         logger.error("User preferences are not recognized by %s" % product)
 615         exit(1)
 616
 617     if "test_objects" in kwargs:
 618         tests = []
 619         for test in kwargs["test_objects"]:
 620             tests.append(test["path"])
 621
 622     prefs = {}
 623     for s in extra_prefs or []:
 624         kv = s.split("=")
 625         if len(kv) != 2:
 626             logger.error("syntax error in --setpref={}".format(s))
 627             exit(EX_USAGE)
 628         prefs[kv[0]] = kv[1].strip()
 629
 630     options = {}
 631     for s in extra_options or []:
 632         kv = s.split("=")
 633         if len(kv) != 2:
 634             logger.error("syntax error in --setopt={}".format(s))
 635             exit(EX_USAGE)
 636         options[kv[0]] = kv[1].strip()
 637
 638     prefs.update({"fission.autostart": True})
 639     if disable_fission:
 640         prefs.update({"fission.autostart": False})
 641
 642     if verbosity == 1:
 643         prefs["remote.log.level"] = "Debug"
 644     elif verbosity > 1:
 645         prefs["remote.log.level"] = "Trace"
 646     if verbosity > 2:
 647         prefs["remote.log.truncate"] = False
 648
 649     if install:
 650         install_puppeteer(command_context, product, ci)
 651
 652     params = {
 653         "binary": binary,
 654         "headless": headless,
 655         "enable_webrender": enable_webrender,
 656         "extra_prefs": prefs,
 657         "product": product,
 658         "extra_launcher_options": options,
 659         "subset": subset,
 660     }
 661     puppeteer = command_context._spawn(PuppeteerRunner)
 662     try:
 663         return puppeteer.run_test(logger, *tests, **params)
 664     except BinaryNotFoundException as e:
 665         logger.error(e)
 666         logger.info(e.help())
 667         exit(1)
 668     except Exception as e:
 669         exit(EX_SOFTWARE, e)
 670
 671
 672 def install_puppeteer(command_context, product, ci):
 673     setup()
 674     env = {"HUSKY": "0"}
 675
 676     puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 677
 678     if product != "chrome":
 679         env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 680
 681     if not ci:
 682         npm(
 683             "run",
 684             "clean",
 685             cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
 686             env=env,
 687             exit_on_fail=False,
 688         )
 689
 690     command = "ci" if ci else "install"
 691     npm(command, cwd=os.path.join(command_context.topsrcdir, puppeteer_dir), env=env)
 692     npm(
 693         "run",
 694         "build:dev",
 695         cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
 696         env=env,
 697     )
 698
 699
 700 def exit(code, error=None):
 701     if error is not None:
 702         if isinstance(error, Exception):
 703             import traceback
 704
 705             traceback.print_exc()
 706         else:
 707             message = str(error).split("\n")[0].strip()
 708             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 709     sys.exit(code)