remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import argparse
  12 import json
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import tempfile
  19
  20 from collections import OrderedDict
  21
  22 from six import iteritems
  23
  24 from mach.decorators import (
  25     Command,
  26     CommandArgument,
  27     SubCommand,
  28 )
  29
  30 from mozbuild.base import (
  31     MozbuildObject,
  32     BinaryNotFoundException,
  33 )
  34 from mozbuild import nodeutil
  35 import mozlog
  36 import mozprofile
  37
  38
  39 EX_CONFIG = 78
  40 EX_SOFTWARE = 70
  41 EX_USAGE = 64
  42
  43
  44 def setup():
  45     # add node and npm from mozbuild to front of system path
  46     npm, _ = nodeutil.find_npm_executable()
  47     if not npm:
  48         exit(EX_CONFIG, "could not find npm executable")
  49     path = os.path.abspath(os.path.join(npm, os.pardir))
  50     os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
  51
  52
  53 def remotedir(command_context):
  54     return os.path.join(command_context.topsrcdir, "remote")
  55
  56
  57 @Command("remote", category="misc", description="Remote protocol related operations.")
  58 def remote(command_context):
  59     """The remote subcommands all relate to the remote protocol."""
  60     command_context._sub_mach(["help", "remote"])
  61     return 1
  62
  63
  64 @SubCommand(
  65     "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
  66 )
  67 @CommandArgument(
  68     "--repository",
  69     metavar="REPO",
  70     required=True,
  71     help="The (possibly remote) repository to clone from.",
  72 )
  73 @CommandArgument(
  74     "--commitish",
  75     metavar="COMMITISH",
  76     required=True,
  77     help="The commit or tag object name to check out.",
  78 )
  79 @CommandArgument(
  80     "--no-install",
  81     dest="install",
  82     action="store_false",
  83     default=True,
  84     help="Do not install the just-pulled Puppeteer package,",
  85 )
  86 def vendor_puppeteer(command_context, repository, commitish, install):
  87     puppeteer_dir = os.path.join(remotedir(command_context), "test", "puppeteer")
  88
  89     # Preserve our custom mocha reporter
  90     shutil.move(
  91         os.path.join(puppeteer_dir, "json-mocha-reporter.js"),
  92         remotedir(command_context),
  93     )
  94     shutil.rmtree(puppeteer_dir, ignore_errors=True)
  95     os.makedirs(puppeteer_dir)
  96     with TemporaryDirectory() as tmpdir:
  97         git("clone", "-q", repository, tmpdir)
  98         git("checkout", commitish, worktree=tmpdir)
  99         git(
 100             "checkout-index",
 101             "-a",
 102             "-f",
 103             "--prefix",
 104             "{}/".format(puppeteer_dir),
 105             worktree=tmpdir,
 106         )
 107
 108     # remove files which may interfere with git checkout of central
 109     try:
 110         os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
 111         os.remove(os.path.join(puppeteer_dir, ".gitignore"))
 112     except OSError:
 113         pass
 114
 115     unwanted_dirs = ["experimental", "docs"]
 116
 117     for dir in unwanted_dirs:
 118         dir_path = os.path.join(puppeteer_dir, dir)
 119         if os.path.isdir(dir_path):
 120             shutil.rmtree(dir_path)
 121
 122     shutil.move(
 123         os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
 124         puppeteer_dir,
 125     )
 126
 127     import yaml
 128
 129     annotation = {
 130         "schema": 1,
 131         "bugzilla": {
 132             "product": "Remote Protocol",
 133             "component": "Agent",
 134         },
 135         "origin": {
 136             "name": "puppeteer",
 137             "description": "Headless Chrome Node API",
 138             "url": repository,
 139             "license": "Apache-2.0",
 140             "release": commitish,
 141         },
 142     }
 143     with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 144         yaml.safe_dump(
 145             annotation,
 146             fh,
 147             default_flow_style=False,
 148             encoding="utf-8",
 149             allow_unicode=True,
 150         )
 151
 152     if install:
 153         env = {"PUPPETEER_SKIP_DOWNLOAD": "1"}
 154         npm(
 155             "install",
 156             cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
 157             env=env,
 158         )
 159
 160
 161 def git(*args, **kwargs):
 162     cmd = ("git",)
 163     if kwargs.get("worktree"):
 164         cmd += ("-C", kwargs["worktree"])
 165     cmd += args
 166
 167     pipe = kwargs.get("pipe")
 168     git_p = subprocess.Popen(
 169         cmd,
 170         env={"GIT_CONFIG_NOSYSTEM": "1"},
 171         stdout=subprocess.PIPE,
 172         stderr=subprocess.PIPE,
 173     )
 174     pipe_p = None
 175     if pipe:
 176         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 177
 178     if pipe:
 179         _, pipe_err = pipe_p.communicate()
 180     out, git_err = git_p.communicate()
 181
 182     # use error from first program that failed
 183     if git_p.returncode > 0:
 184         exit(EX_SOFTWARE, git_err)
 185     if pipe and pipe_p.returncode > 0:
 186         exit(EX_SOFTWARE, pipe_err)
 187
 188     return out
 189
 190
 191 def npm(*args, **kwargs):
 192     from mozprocess import processhandler
 193
 194     env = None
 195     if kwargs.get("env"):
 196         env = os.environ.copy()
 197         env.update(kwargs["env"])
 198
 199     proc_kwargs = {}
 200     if "processOutputLine" in kwargs:
 201         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 202
 203     p = processhandler.ProcessHandler(
 204         cmd="npm",
 205         args=list(args),
 206         cwd=kwargs.get("cwd"),
 207         env=env,
 208         universal_newlines=True,
 209         **proc_kwargs
 210     )
 211     if not kwargs.get("wait", True):
 212         return p
 213
 214     wait_proc(p, cmd="npm", exit_on_fail=kwargs.get("exit_on_fail", True))
 215
 216     return p.returncode
 217
 218
 219 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 220     try:
 221         p.run(outputTimeout=output_timeout)
 222         p.wait()
 223         if p.timedOut:
 224             # In some cases, we wait longer for a mocha timeout
 225             print("Timed out after {} seconds of no output".format(output_timeout))
 226     finally:
 227         p.kill()
 228     if exit_on_fail and p.returncode > 0:
 229         msg = (
 230             "%s: exit code %s" % (cmd, p.returncode)
 231             if cmd
 232             else "exit code %s" % p.returncode
 233         )
 234         exit(p.returncode, msg)
 235
 236
 237 class MochaOutputHandler(object):
 238     def __init__(self, logger, expected):
 239         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 240
 241         self.logger = logger
 242         self.proc = None
 243         self.test_results = OrderedDict()
 244         self.expected = expected
 245         self.unexpected_skips = set()
 246
 247         self.has_unexpected = False
 248         self.logger.suite_start([], name="puppeteer-tests")
 249         self.status_map = {
 250             "CRASHED": "CRASH",
 251             "OK": "PASS",
 252             "TERMINATED": "CRASH",
 253             "pass": "PASS",
 254             "fail": "FAIL",
 255             "pending": "SKIP",
 256         }
 257
 258     @property
 259     def pid(self):
 260         return self.proc and self.proc.pid
 261
 262     def __call__(self, line):
 263         event = None
 264         try:
 265             if line.startswith("[") and line.endswith("]"):
 266                 event = json.loads(line)
 267             self.process_event(event)
 268         except ValueError:
 269             pass
 270         finally:
 271             self.logger.process_output(self.pid, line, command="npm")
 272
 273     def process_event(self, event):
 274         if isinstance(event, list) and len(event) > 1:
 275             status = self.status_map.get(event[0])
 276             test_start = event[0] == "test-start"
 277             if not status and not test_start:
 278                 return
 279             test_info = event[1]
 280             test_name = test_info.get("fullTitle", "")
 281             test_path = test_info.get("file", "")
 282             test_err = test_info.get("err")
 283             if status == "FAIL" and test_err:
 284                 if "timeout" in test_err.lower():
 285                     status = "TIMEOUT"
 286             if test_name and test_path:
 287                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 288             # mocha hook failures are not tracked in metadata
 289             if status != "PASS" and self.hook_re.search(test_name):
 290                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 291                 return
 292             if test_start:
 293                 self.logger.test_start(test_name)
 294                 return
 295             expected = self.expected.get(test_name, ["PASS"])
 296             # mozlog doesn't really allow unexpected skip,
 297             # so if a test is disabled just expect that and note the unexpected skip
 298             # Also, mocha doesn't log test-start for skipped tests
 299             if status == "SKIP":
 300                 self.logger.test_start(test_name)
 301                 if self.expected and status not in expected:
 302                     self.unexpected_skips.add(test_name)
 303                 expected = ["SKIP"]
 304             known_intermittent = expected[1:]
 305             expected_status = expected[0]
 306
 307             # check if we've seen a result for this test before this log line
 308             result_recorded = self.test_results.get(test_name)
 309             if result_recorded:
 310                 self.logger.warning(
 311                     "Received a second status for {}: "
 312                     "first {}, now {}".format(test_name, result_recorded, status)
 313                 )
 314             # mocha intermittently logs an additional test result after the
 315             # test has already timed out. Avoid recording this second status.
 316             if result_recorded != "TIMEOUT":
 317                 self.test_results[test_name] = status
 318                 if status not in expected:
 319                     self.has_unexpected = True
 320             self.logger.test_end(
 321                 test_name,
 322                 status=status,
 323                 expected=expected_status,
 324                 known_intermittent=known_intermittent,
 325             )
 326
 327     def new_expected(self):
 328         new_expected = OrderedDict()
 329         for test_name, status in iteritems(self.test_results):
 330             if test_name not in self.expected:
 331                 new_status = [status]
 332             else:
 333                 if status in self.expected[test_name]:
 334                     new_status = self.expected[test_name]
 335                 else:
 336                     new_status = [status]
 337             new_expected[test_name] = new_status
 338         return new_expected
 339
 340     def after_end(self, subset=False):
 341         if not subset:
 342             missing = set(self.expected) - set(self.test_results)
 343             extra = set(self.test_results) - set(self.expected)
 344             if missing:
 345                 self.has_unexpected = True
 346                 for test_name in missing:
 347                     self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,))
 348             if self.expected and extra:
 349                 self.has_unexpected = True
 350                 for test_name in extra:
 351                     self.logger.error(
 352                         "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,)
 353                     )
 354
 355         if self.unexpected_skips:
 356             self.has_unexpected = True
 357             for test_name in self.unexpected_skips:
 358                 self.logger.error(
 359                     "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
 360                 )
 361         self.logger.suite_end()
 362
 363
 364 # tempfile.TemporaryDirectory missing from Python 2.7
 365 class TemporaryDirectory(object):
 366     def __init__(self):
 367         self.path = tempfile.mkdtemp()
 368         self._closed = False
 369
 370     def __repr__(self):
 371         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 372
 373     def __enter__(self):
 374         return self.path
 375
 376     def __exit__(self, exc, value, tb):
 377         self.clean()
 378
 379     def __del__(self):
 380         self.clean()
 381
 382     def clean(self):
 383         if self.path and not self._closed:
 384             shutil.rmtree(self.path)
 385             self._closed = True
 386
 387
 388 class PuppeteerRunner(MozbuildObject):
 389     def __init__(self, *args, **kwargs):
 390         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 391
 392         self.remotedir = os.path.join(self.topsrcdir, "remote")
 393         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 394
 395     def run_test(self, logger, *tests, **params):
 396         """
 397         Runs Puppeteer unit tests with npm.
 398
 399         Possible optional test parameters:
 400
 401         `binary`:
 402           Path for the browser binary to use.  Defaults to the local
 403           build.
 404         `headless`:
 405           Boolean to indicate whether to activate Firefox' headless mode.
 406         `extra_prefs`:
 407           Dictionary of extra preferences to write to the profile,
 408           before invoking npm.  Overrides default preferences.
 409         `enable_webrender`:
 410           Boolean to indicate whether to enable WebRender compositor in Gecko.
 411         `write_results`:
 412           Path to write the results json file
 413         `subset`
 414           Indicates only a subset of tests are being run, so we should
 415           skip the check for missing results
 416         """
 417         setup()
 418
 419         binary = params.get("binary") or self.get_binary_path()
 420         product = params.get("product", "firefox")
 421
 422         env = {
 423             # Print browser process ouptut
 424             "DUMPIO": "1",
 425             # Checked by Puppeteer's custom mocha config
 426             "CI": "1",
 427             # Causes some tests to be skipped due to assumptions about install
 428             "PUPPETEER_ALT_INSTALL": "1",
 429         }
 430         extra_options = {}
 431         for k, v in params.get("extra_launcher_options", {}).items():
 432             extra_options[k] = json.loads(v)
 433
 434         # Override upstream defaults: no retries, shorter timeout
 435         mocha_options = [
 436             "--reporter",
 437             "./json-mocha-reporter.js",
 438             "--retries",
 439             "0",
 440             "--fullTrace",
 441             "--timeout",
 442             "20000",
 443             "--no-parallel",
 444         ]
 445         if product == "firefox":
 446             env["BINARY"] = binary
 447             env["PUPPETEER_PRODUCT"] = "firefox"
 448
 449             env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
 450
 451         command = ["run", "unit", "--"] + mocha_options
 452
 453         env["HEADLESS"] = str(params.get("headless", False))
 454
 455         prefs = {}
 456         for k, v in params.get("extra_prefs", {}).items():
 457             print("Using extra preference: {}={}".format(k, v))
 458             prefs[k] = mozprofile.Preferences.cast(v)
 459
 460         if prefs:
 461             extra_options["extraPrefsFirefox"] = prefs
 462
 463         if extra_options:
 464             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 465
 466         expected_path = os.path.join(
 467             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 468         )
 469         if product == "firefox" and os.path.exists(expected_path):
 470             with open(expected_path) as f:
 471                 expected_data = json.load(f)
 472         else:
 473             expected_data = {}
 474
 475         output_handler = MochaOutputHandler(logger, expected_data)
 476         proc = npm(
 477             *command,
 478             cwd=self.puppeteer_dir,
 479             env=env,
 480             processOutputLine=output_handler,
 481             wait=False
 482         )
 483         output_handler.proc = proc
 484
 485         # Puppeteer unit tests don't always clean-up child processes in case of
 486         # failure, so use an output_timeout as a fallback
 487         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 488
 489         output_handler.after_end(params.get("subset", False))
 490
 491         # Non-zero return codes are non-fatal for now since we have some
 492         # issues with unresolved promises that shouldn't otherwise block
 493         # running the tests
 494         if proc.returncode != 0:
 495             logger.warning("npm exited with code %s" % proc.returncode)
 496
 497         if params["write_results"]:
 498             with open(params["write_results"], "w") as f:
 499                 json.dump(
 500                     output_handler.new_expected(), f, indent=2, separators=(",", ": ")
 501                 )
 502
 503         if output_handler.has_unexpected:
 504             exit(1, "Got unexpected results")
 505
 506
 507 def create_parser_puppeteer():
 508     p = argparse.ArgumentParser()
 509     p.add_argument(
 510         "--product", type=str, default="firefox", choices=["chrome", "firefox"]
 511     )
 512     p.add_argument(
 513         "--binary",
 514         type=str,
 515         help="Path to browser binary.  Defaults to local Firefox build.",
 516     )
 517     p.add_argument(
 518         "--ci",
 519         action="store_true",
 520         help="Flag that indicates that tests run in a CI environment.",
 521     )
 522     p.add_argument(
 523         "--enable-fission",
 524         action="store_true",
 525         help="Enable Fission (site isolation) in Gecko.",
 526     )
 527     p.add_argument(
 528         "--enable-webrender",
 529         action="store_true",
 530         help="Enable the WebRender compositor in Gecko.",
 531     )
 532     p.add_argument(
 533         "-z", "--headless", action="store_true", help="Run browser in headless mode."
 534     )
 535     p.add_argument(
 536         "--setpref",
 537         action="append",
 538         dest="extra_prefs",
 539         metavar="<pref>=<value>",
 540         help="Defines additional user preferences.",
 541     )
 542     p.add_argument(
 543         "--setopt",
 544         action="append",
 545         dest="extra_options",
 546         metavar="<option>=<value>",
 547         help="Defines additional options for `puppeteer.launch`.",
 548     )
 549     p.add_argument(
 550         "-v",
 551         dest="verbosity",
 552         action="count",
 553         default=0,
 554         help="Increase remote agent logging verbosity to include "
 555         "debug level messages with -v, trace messages with -vv,"
 556         "and to not truncate long trace messages with -vvv",
 557     )
 558     p.add_argument(
 559         "--write-results",
 560         action="store",
 561         nargs="?",
 562         default=None,
 563         const=os.path.join(
 564             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 565         ),
 566         help="Path to write updated results to (defaults to the "
 567         "expectations file if the argument is provided but "
 568         "no path is passed)",
 569     )
 570     p.add_argument(
 571         "--subset",
 572         action="store_true",
 573         default=False,
 574         help="Indicate that only a subset of the tests are running, "
 575         "so checks for missing tests should be skipped",
 576     )
 577     p.add_argument("tests", nargs="*")
 578     mozlog.commandline.add_logging_group(p)
 579     return p
 580
 581
 582 @Command(
 583     "puppeteer-test",
 584     category="testing",
 585     description="Run Puppeteer unit tests.",
 586     parser=create_parser_puppeteer,
 587 )
 588 @CommandArgument(
 589     "--no-install",
 590     dest="install",
 591     action="store_false",
 592     default=True,
 593     help="Do not install the Puppeteer package",
 594 )
 595 def puppeteer_test(
 596     command_context,
 597     binary=None,
 598     ci=False,
 599     enable_fission=False,
 600     enable_webrender=False,
 601     headless=False,
 602     extra_prefs=None,
 603     extra_options=None,
 604     install=False,
 605     verbosity=0,
 606     tests=None,
 607     product="firefox",
 608     write_results=None,
 609     subset=False,
 610     **kwargs
 611 ):
 612
 613     logger = mozlog.commandline.setup_logging(
 614         "puppeteer-test", kwargs, {"mach": sys.stdout}
 615     )
 616
 617     # moztest calls this programmatically with test objects or manifests
 618     if "test_objects" in kwargs and tests is not None:
 619         logger.error("Expected either 'test_objects' or 'tests'")
 620         exit(1)
 621
 622     if product != "firefox" and extra_prefs is not None:
 623         logger.error("User preferences are not recognized by %s" % product)
 624         exit(1)
 625
 626     if "test_objects" in kwargs:
 627         tests = []
 628         for test in kwargs["test_objects"]:
 629             tests.append(test["path"])
 630
 631     prefs = {}
 632     for s in extra_prefs or []:
 633         kv = s.split("=")
 634         if len(kv) != 2:
 635             logger.error("syntax error in --setpref={}".format(s))
 636             exit(EX_USAGE)
 637         prefs[kv[0]] = kv[1].strip()
 638
 639     options = {}
 640     for s in extra_options or []:
 641         kv = s.split("=")
 642         if len(kv) != 2:
 643             logger.error("syntax error in --setopt={}".format(s))
 644             exit(EX_USAGE)
 645         options[kv[0]] = kv[1].strip()
 646
 647     if enable_fission:
 648         prefs.update({"fission.autostart": True})
 649     else:
 650         prefs.update({"fission.autostart": False})
 651
 652     if verbosity == 1:
 653         prefs["remote.log.level"] = "Debug"
 654     elif verbosity > 1:
 655         prefs["remote.log.level"] = "Trace"
 656     if verbosity > 2:
 657         prefs["remote.log.truncate"] = False
 658
 659     if install:
 660         install_puppeteer(command_context, product, ci)
 661
 662     params = {
 663         "binary": binary,
 664         "headless": headless,
 665         "enable_webrender": enable_webrender,
 666         "extra_prefs": prefs,
 667         "product": product,
 668         "extra_launcher_options": options,
 669         "write_results": write_results,
 670         "subset": subset,
 671     }
 672     puppeteer = command_context._spawn(PuppeteerRunner)
 673     try:
 674         return puppeteer.run_test(logger, *tests, **params)
 675     except BinaryNotFoundException as e:
 676         logger.error(e)
 677         logger.info(e.help())
 678         exit(1)
 679     except Exception as e:
 680         exit(EX_SOFTWARE, e)
 681
 682
 683 def install_puppeteer(command_context, product, ci):
 684     setup()
 685     env = {}
 686     from mozversioncontrol import get_repository_object
 687
 688     repo = get_repository_object(command_context.topsrcdir)
 689     puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 690     changed_files = False
 691     for f in repo.get_changed_files():
 692         if f.startswith(puppeteer_dir) and f.endswith(".ts"):
 693             changed_files = True
 694             break
 695
 696     if product != "chrome":
 697         env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 698     lib_dir = os.path.join(command_context.topsrcdir, puppeteer_dir, "lib")
 699     if changed_files and os.path.isdir(lib_dir):
 700         # clobber lib to force `tsc compile` step
 701         shutil.rmtree(lib_dir)
 702
 703     command = "ci" if ci else "install"
 704     npm(command, cwd=os.path.join(command_context.topsrcdir, puppeteer_dir), env=env)
 705
 706
 707 def exit(code, error=None):
 708     if error is not None:
 709         if isinstance(error, Exception):
 710             import traceback
 711
 712             traceback.print_exc()
 713         else:
 714             message = str(error).split("\n")[0].strip()
 715             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 716     sys.exit(code)