remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import argparse
  12 import json
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import tempfile
  19
  20 from collections import OrderedDict
  21
  22 from six import iteritems
  23
  24 from mach.decorators import (
  25     Command,
  26     CommandArgument,
  27     SubCommand,
  28 )
  29
  30 from mozbuild.base import (
  31     MozbuildObject,
  32     BinaryNotFoundException,
  33 )
  34 from mozbuild import nodeutil
  35 import mozlog
  36 import mozprofile
  37
  38
  39 EX_CONFIG = 78
  40 EX_SOFTWARE = 70
  41 EX_USAGE = 64
  42
  43
  44 def setup():
  45     # add node and npm from mozbuild to front of system path
  46     npm, _ = nodeutil.find_npm_executable()
  47     if not npm:
  48         exit(EX_CONFIG, "could not find npm executable")
  49     path = os.path.abspath(os.path.join(npm, os.pardir))
  50     os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
  51
  52
  53 def remotedir(command_context):
  54     return os.path.join(command_context.topsrcdir, "remote")
  55
  56
  57 @Command("remote", category="misc", description="Remote protocol related operations.")
  58 def remote(command_context):
  59     """The remote subcommands all relate to the remote protocol."""
  60     command_context._sub_mach(["help", "remote"])
  61     return 1
  62
  63
  64 @SubCommand(
  65     "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
  66 )
  67 @CommandArgument(
  68     "--repository",
  69     metavar="REPO",
  70     required=True,
  71     help="The (possibly remote) repository to clone from.",
  72 )
  73 @CommandArgument(
  74     "--commitish",
  75     metavar="COMMITISH",
  76     required=True,
  77     help="The commit or tag object name to check out.",
  78 )
  79 @CommandArgument(
  80     "--no-install",
  81     dest="install",
  82     action="store_false",
  83     default=True,
  84     help="Do not install the just-pulled Puppeteer package,",
  85 )
  86 def vendor_puppeteer(command_context, repository, commitish, install):
  87     puppeteer_dir = os.path.join(remotedir(command_context), "test", "puppeteer")
  88
  89     # Preserve our custom mocha reporter
  90     shutil.move(
  91         os.path.join(puppeteer_dir, "json-mocha-reporter.js"),
  92         remotedir(command_context),
  93     )
  94     shutil.rmtree(puppeteer_dir, ignore_errors=True)
  95     os.makedirs(puppeteer_dir)
  96     with TemporaryDirectory() as tmpdir:
  97         git("clone", "-q", repository, tmpdir)
  98         git("checkout", commitish, worktree=tmpdir)
  99         git(
 100             "checkout-index",
 101             "-a",
 102             "-f",
 103             "--prefix",
 104             "{}/".format(puppeteer_dir),
 105             worktree=tmpdir,
 106         )
 107
 108     # remove files which may interfere with git checkout of central
 109     try:
 110         os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
 111         os.remove(os.path.join(puppeteer_dir, ".gitignore"))
 112     except OSError:
 113         pass
 114
 115     unwanted_dirs = ["experimental", "docs"]
 116
 117     for dir in unwanted_dirs:
 118         dir_path = os.path.join(puppeteer_dir, dir)
 119         if os.path.isdir(dir_path):
 120             shutil.rmtree(dir_path)
 121
 122     shutil.move(
 123         os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
 124         puppeteer_dir,
 125     )
 126
 127     import yaml
 128
 129     annotation = {
 130         "schema": 1,
 131         "bugzilla": {
 132             "product": "Remote Protocol",
 133             "component": "Agent",
 134         },
 135         "origin": {
 136             "name": "puppeteer",
 137             "description": "Headless Chrome Node API",
 138             "url": repository,
 139             "license": "Apache-2.0",
 140             "release": commitish,
 141         },
 142     }
 143     with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 144         yaml.safe_dump(
 145             annotation,
 146             fh,
 147             default_flow_style=False,
 148             encoding="utf-8",
 149             allow_unicode=True,
 150         )
 151
 152     if install:
 153         env = {"PUPPETEER_SKIP_DOWNLOAD": "1"}
 154         npm(
 155             "install",
 156             cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
 157             env=env,
 158         )
 159
 160
 161 def git(*args, **kwargs):
 162     cmd = ("git",)
 163     if kwargs.get("worktree"):
 164         cmd += ("-C", kwargs["worktree"])
 165     cmd += args
 166
 167     pipe = kwargs.get("pipe")
 168     git_p = subprocess.Popen(
 169         cmd,
 170         env={"GIT_CONFIG_NOSYSTEM": "1"},
 171         stdout=subprocess.PIPE,
 172         stderr=subprocess.PIPE,
 173     )
 174     pipe_p = None
 175     if pipe:
 176         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 177
 178     if pipe:
 179         _, pipe_err = pipe_p.communicate()
 180     out, git_err = git_p.communicate()
 181
 182     # use error from first program that failed
 183     if git_p.returncode > 0:
 184         exit(EX_SOFTWARE, git_err)
 185     if pipe and pipe_p.returncode > 0:
 186         exit(EX_SOFTWARE, pipe_err)
 187
 188     return out
 189
 190
 191 def npm(*args, **kwargs):
 192     from mozprocess import processhandler
 193
 194     env = None
 195     if kwargs.get("env"):
 196         env = os.environ.copy()
 197         env.update(kwargs["env"])
 198
 199     proc_kwargs = {}
 200     if "processOutputLine" in kwargs:
 201         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 202
 203     p = processhandler.ProcessHandler(
 204         cmd="npm",
 205         args=list(args),
 206         cwd=kwargs.get("cwd"),
 207         env=env,
 208         universal_newlines=True,
 209         **proc_kwargs
 210     )
 211     if not kwargs.get("wait", True):
 212         return p
 213
 214     wait_proc(p, cmd="npm", exit_on_fail=kwargs.get("exit_on_fail", True))
 215
 216     return p.returncode
 217
 218
 219 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 220     try:
 221         p.run(outputTimeout=output_timeout)
 222         p.wait()
 223         if p.timedOut:
 224             # In some cases, we wait longer for a mocha timeout
 225             print("Timed out after {} seconds of no output".format(output_timeout))
 226     finally:
 227         p.kill()
 228     if exit_on_fail and p.returncode > 0:
 229         msg = (
 230             "%s: exit code %s" % (cmd, p.returncode)
 231             if cmd
 232             else "exit code %s" % p.returncode
 233         )
 234         exit(p.returncode, msg)
 235
 236
 237 class MochaOutputHandler(object):
 238     def __init__(self, logger, expected):
 239         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 240
 241         self.logger = logger
 242         self.proc = None
 243         self.test_results = OrderedDict()
 244         self.expected = expected
 245         self.unexpected_skips = set()
 246
 247         self.has_unexpected = False
 248         self.logger.suite_start([], name="puppeteer-tests")
 249         self.status_map = {
 250             "CRASHED": "CRASH",
 251             "OK": "PASS",
 252             "TERMINATED": "CRASH",
 253             "pass": "PASS",
 254             "fail": "FAIL",
 255             "pending": "SKIP",
 256         }
 257
 258     @property
 259     def pid(self):
 260         return self.proc and self.proc.pid
 261
 262     def __call__(self, line):
 263         event = None
 264         try:
 265             if line.startswith("[") and line.endswith("]"):
 266                 event = json.loads(line)
 267             self.process_event(event)
 268         except ValueError:
 269             pass
 270         finally:
 271             self.logger.process_output(self.pid, line, command="npm")
 272
 273     def process_event(self, event):
 274         if isinstance(event, list) and len(event) > 1:
 275             status = self.status_map.get(event[0])
 276             test_start = event[0] == "test-start"
 277             if not status and not test_start:
 278                 return
 279             test_info = event[1]
 280             test_name = test_info.get("fullTitle", "")
 281             test_path = test_info.get("file", "")
 282             test_err = test_info.get("err")
 283             if status == "FAIL" and test_err:
 284                 if "timeout" in test_err.lower():
 285                     status = "TIMEOUT"
 286             if test_name and test_path:
 287                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 288             # mocha hook failures are not tracked in metadata
 289             if status != "PASS" and self.hook_re.search(test_name):
 290                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 291                 return
 292             if test_start:
 293                 self.logger.test_start(test_name)
 294                 return
 295             expected = self.expected.get(test_name, ["PASS"])
 296             # mozlog doesn't really allow unexpected skip,
 297             # so if a test is disabled just expect that and note the unexpected skip
 298             # Also, mocha doesn't log test-start for skipped tests
 299             if status == "SKIP":
 300                 self.logger.test_start(test_name)
 301                 if self.expected and status not in expected:
 302                     self.unexpected_skips.add(test_name)
 303                 expected = ["SKIP"]
 304             known_intermittent = expected[1:]
 305             expected_status = expected[0]
 306
 307             # check if we've seen a result for this test before this log line
 308             result_recorded = self.test_results.get(test_name)
 309             if result_recorded:
 310                 self.logger.warning(
 311                     "Received a second status for {}: "
 312                     "first {}, now {}".format(test_name, result_recorded, status)
 313                 )
 314             # mocha intermittently logs an additional test result after the
 315             # test has already timed out. Avoid recording this second status.
 316             if result_recorded != "TIMEOUT":
 317                 self.test_results[test_name] = status
 318                 if status not in expected:
 319                     self.has_unexpected = True
 320             self.logger.test_end(
 321                 test_name,
 322                 status=status,
 323                 expected=expected_status,
 324                 known_intermittent=known_intermittent,
 325             )
 326
 327     def new_expected(self):
 328         new_expected = OrderedDict()
 329         for test_name, status in iteritems(self.test_results):
 330             if test_name not in self.expected:
 331                 new_status = [status]
 332             else:
 333                 if status in self.expected[test_name]:
 334                     new_status = self.expected[test_name]
 335                 else:
 336                     new_status = [status]
 337             new_expected[test_name] = new_status
 338         return new_expected
 339
 340     def after_end(self, subset=False):
 341         if not subset:
 342             missing = set(self.expected) - set(self.test_results)
 343             extra = set(self.test_results) - set(self.expected)
 344             if missing:
 345                 self.has_unexpected = True
 346                 for test_name in missing:
 347                     self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,))
 348             if self.expected and extra:
 349                 self.has_unexpected = True
 350                 for test_name in extra:
 351                     self.logger.error(
 352                         "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,)
 353                     )
 354
 355         if self.unexpected_skips:
 356             self.has_unexpected = True
 357             for test_name in self.unexpected_skips:
 358                 self.logger.error(
 359                     "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
 360                 )
 361         self.logger.suite_end()
 362
 363
 364 # tempfile.TemporaryDirectory missing from Python 2.7
 365 class TemporaryDirectory(object):
 366     def __init__(self):
 367         self.path = tempfile.mkdtemp()
 368         self._closed = False
 369
 370     def __repr__(self):
 371         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 372
 373     def __enter__(self):
 374         return self.path
 375
 376     def __exit__(self, exc, value, tb):
 377         self.clean()
 378
 379     def __del__(self):
 380         self.clean()
 381
 382     def clean(self):
 383         if self.path and not self._closed:
 384             shutil.rmtree(self.path)
 385             self._closed = True
 386
 387
 388 class PuppeteerRunner(MozbuildObject):
 389     def __init__(self, *args, **kwargs):
 390         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 391
 392         self.remotedir = os.path.join(self.topsrcdir, "remote")
 393         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 394
 395     def run_test(self, logger, *tests, **params):
 396         """
 397         Runs Puppeteer unit tests with npm.
 398
 399         Possible optional test parameters:
 400
 401         `binary`:
 402           Path for the browser binary to use.  Defaults to the local
 403           build.
 404         `headless`:
 405           Boolean to indicate whether to activate Firefox' headless mode.
 406         `extra_prefs`:
 407           Dictionary of extra preferences to write to the profile,
 408           before invoking npm.  Overrides default preferences.
 409         `enable_webrender`:
 410           Boolean to indicate whether to enable WebRender compositor in Gecko.
 411         `write_results`:
 412           Path to write the results json file
 413         `subset`
 414           Indicates only a subset of tests are being run, so we should
 415           skip the check for missing results
 416         """
 417         setup()
 418
 419         binary = params.get("binary") or self.get_binary_path()
 420         product = params.get("product", "firefox")
 421
 422         env = {
 423             # Print browser process ouptut
 424             "DUMPIO": "1",
 425             # Checked by Puppeteer's custom mocha config
 426             "CI": "1",
 427             # Causes some tests to be skipped due to assumptions about install
 428             "PUPPETEER_ALT_INSTALL": "1",
 429         }
 430         extra_options = {}
 431         for k, v in params.get("extra_launcher_options", {}).items():
 432             extra_options[k] = json.loads(v)
 433
 434         # Override upstream defaults: no retries, shorter timeout
 435         mocha_options = [
 436             "--reporter",
 437             "./json-mocha-reporter.js",
 438             "--retries",
 439             "0",
 440             "--fullTrace",
 441             "--timeout",
 442             "20000",
 443             "--no-parallel",
 444         ]
 445         if product == "firefox":
 446             env["BINARY"] = binary
 447             env["PUPPETEER_PRODUCT"] = "firefox"
 448
 449             env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
 450
 451         command = ["run", "unit", "--"] + mocha_options
 452
 453         env["HEADLESS"] = str(params.get("headless", False))
 454
 455         prefs = {}
 456         for k, v in params.get("extra_prefs", {}).items():
 457             print("Using extra preference: {}={}".format(k, v))
 458             prefs[k] = mozprofile.Preferences.cast(v)
 459
 460         if prefs:
 461             extra_options["extraPrefsFirefox"] = prefs
 462
 463         if extra_options:
 464             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 465
 466         expected_path = os.path.join(
 467             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 468         )
 469         if product == "firefox" and os.path.exists(expected_path):
 470             with open(expected_path) as f:
 471                 expected_data = json.load(f)
 472         else:
 473             expected_data = {}
 474
 475         output_handler = MochaOutputHandler(logger, expected_data)
 476         proc = npm(
 477             *command,
 478             cwd=self.puppeteer_dir,
 479             env=env,
 480             processOutputLine=output_handler,
 481             wait=False
 482         )
 483         output_handler.proc = proc
 484
 485         # Puppeteer unit tests don't always clean-up child processes in case of
 486         # failure, so use an output_timeout as a fallback
 487         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 488
 489         output_handler.after_end(params.get("subset", False))
 490
 491         # Non-zero return codes are non-fatal for now since we have some
 492         # issues with unresolved promises that shouldn't otherwise block
 493         # running the tests
 494         if proc.returncode != 0:
 495             logger.warning("npm exited with code %s" % proc.returncode)
 496
 497         if params["write_results"]:
 498             with open(params["write_results"], "w") as f:
 499                 json.dump(
 500                     output_handler.new_expected(), f, indent=2, separators=(",", ": ")
 501                 )
 502
 503         if output_handler.has_unexpected:
 504             exit(1, "Got unexpected results")
 505
 506
 507 def create_parser_puppeteer():
 508     p = argparse.ArgumentParser()
 509     p.add_argument(
 510         "--product", type=str, default="firefox", choices=["chrome", "firefox"]
 511     )
 512     p.add_argument(
 513         "--binary",
 514         type=str,
 515         help="Path to browser binary.  Defaults to local Firefox build.",
 516     )
 517     p.add_argument(
 518         "--ci",
 519         action="store_true",
 520         help="Flag that indicates that tests run in a CI environment.",
 521     )
 522     p.add_argument(
 523         "--enable-fission",
 524         action="store_true",
 525         help="Enable Fission (site isolation) in Gecko.",
 526     )
 527     p.add_argument(
 528         "--enable-webrender",
 529         action="store_true",
 530         help="Enable the WebRender compositor in Gecko.",
 531     )
 532     p.add_argument(
 533         "-z", "--headless", action="store_true", help="Run browser in headless mode."
 534     )
 535     p.add_argument(
 536         "--setpref",
 537         action="append",
 538         dest="extra_prefs",
 539         metavar="<pref>=<value>",
 540         help="Defines additional user preferences.",
 541     )
 542     p.add_argument(
 543         "--setopt",
 544         action="append",
 545         dest="extra_options",
 546         metavar="<option>=<value>",
 547         help="Defines additional options for `puppeteer.launch`.",
 548     )
 549     p.add_argument(
 550         "-v",
 551         dest="verbosity",
 552         action="count",
 553         default=0,
 554         help="Increase remote agent logging verbosity to include "
 555         "debug level messages with -v, trace messages with -vv,"
 556         "and to not truncate long trace messages with -vvv",
 557     )
 558     p.add_argument(
 559         "--write-results",
 560         action="store",
 561         nargs="?",
 562         default=None,
 563         const=os.path.join(
 564             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 565         ),
 566         help="Path to write updated results to (defaults to the "
 567         "expectations file if the argument is provided but "
 568         "no path is passed)",
 569     )
 570     p.add_argument(
 571         "--subset",
 572         action="store_true",
 573         default=False,
 574         help="Indicate that only a subset of the tests are running, "
 575         "so checks for missing tests should be skipped",
 576     )
 577     p.add_argument("tests", nargs="*")
 578     mozlog.commandline.add_logging_group(p)
 579     return p
 580
 581
 582 @Command(
 583     "puppeteer-test",
 584     category="testing",
 585     description="Run Puppeteer unit tests.",
 586     parser=create_parser_puppeteer,
 587 )
 588 def puppeteer_test(
 589     command_context,
 590     binary=None,
 591     ci=False,
 592     enable_fission=False,
 593     enable_webrender=False,
 594     headless=False,
 595     extra_prefs=None,
 596     extra_options=None,
 597     verbosity=0,
 598     tests=None,
 599     product="firefox",
 600     write_results=None,
 601     subset=False,
 602     **kwargs
 603 ):
 604
 605     logger = mozlog.commandline.setup_logging(
 606         "puppeteer-test", kwargs, {"mach": sys.stdout}
 607     )
 608
 609     # moztest calls this programmatically with test objects or manifests
 610     if "test_objects" in kwargs and tests is not None:
 611         logger.error("Expected either 'test_objects' or 'tests'")
 612         exit(1)
 613
 614     if product != "firefox" and extra_prefs is not None:
 615         logger.error("User preferences are not recognized by %s" % product)
 616         exit(1)
 617
 618     if "test_objects" in kwargs:
 619         tests = []
 620         for test in kwargs["test_objects"]:
 621             tests.append(test["path"])
 622
 623     prefs = {}
 624     for s in extra_prefs or []:
 625         kv = s.split("=")
 626         if len(kv) != 2:
 627             logger.error("syntax error in --setpref={}".format(s))
 628             exit(EX_USAGE)
 629         prefs[kv[0]] = kv[1].strip()
 630
 631     options = {}
 632     for s in extra_options or []:
 633         kv = s.split("=")
 634         if len(kv) != 2:
 635             logger.error("syntax error in --setopt={}".format(s))
 636             exit(EX_USAGE)
 637         options[kv[0]] = kv[1].strip()
 638
 639     if enable_fission:
 640         prefs.update({"fission.autostart": True})
 641
 642     if verbosity == 1:
 643         prefs["remote.log.level"] = "Debug"
 644     elif verbosity > 1:
 645         prefs["remote.log.level"] = "Trace"
 646     if verbosity > 2:
 647         prefs["remote.log.truncate"] = False
 648
 649     install_puppeteer(command_context, product, ci)
 650
 651     params = {
 652         "binary": binary,
 653         "headless": headless,
 654         "enable_webrender": enable_webrender,
 655         "extra_prefs": prefs,
 656         "product": product,
 657         "extra_launcher_options": options,
 658         "write_results": write_results,
 659         "subset": subset,
 660     }
 661     puppeteer = command_context._spawn(PuppeteerRunner)
 662     try:
 663         return puppeteer.run_test(logger, *tests, **params)
 664     except BinaryNotFoundException as e:
 665         logger.error(e)
 666         logger.info(e.help())
 667         exit(1)
 668     except Exception as e:
 669         exit(EX_SOFTWARE, e)
 670
 671
 672 def install_puppeteer(command_context, product, ci):
 673     setup()
 674     env = {}
 675     from mozversioncontrol import get_repository_object
 676
 677     repo = get_repository_object(command_context.topsrcdir)
 678     puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 679     changed_files = False
 680     for f in repo.get_changed_files():
 681         if f.startswith(puppeteer_dir) and f.endswith(".ts"):
 682             changed_files = True
 683             break
 684
 685     if product != "chrome":
 686         env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 687     lib_dir = os.path.join(command_context.topsrcdir, puppeteer_dir, "lib")
 688     if changed_files and os.path.isdir(lib_dir):
 689         # clobber lib to force `tsc compile` step
 690         shutil.rmtree(lib_dir)
 691
 692     command = "ci" if ci else "install"
 693     npm(command, cwd=os.path.join(command_context.topsrcdir, puppeteer_dir), env=env)
 694
 695
 696 def exit(code, error=None):
 697     if error is not None:
 698         if isinstance(error, Exception):
 699             import traceback
 700
 701             traceback.print_exc()
 702         else:
 703             message = str(error).split("\n")[0].strip()
 704             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 705     sys.exit(code)