remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import argparse
  12 import json
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import tempfile
  19
  20 from collections import OrderedDict
  21
  22 from six import iteritems
  23
  24 from mach.decorators import (
  25     Command,
  26     CommandArgument,
  27     SubCommand,
  28 )
  29
  30 from mozbuild.base import (
  31     MozbuildObject,
  32     BinaryNotFoundException,
  33 )
  34 from mozbuild import nodeutil
  35 import mozlog
  36 import mozprofile
  37
  38
  39 EX_CONFIG = 78
  40 EX_SOFTWARE = 70
  41 EX_USAGE = 64
  42
  43
  44 def setup():
  45     # add node and npm from mozbuild to front of system path
  46     npm, _ = nodeutil.find_npm_executable()
  47     if not npm:
  48         exit(EX_CONFIG, "could not find npm executable")
  49     path = os.path.abspath(os.path.join(npm, os.pardir))
  50     os.environ["PATH"] = "{}:{}".format(path, os.environ["PATH"])
  51
  52
  53 def remotedir(command_context):
  54     return os.path.join(command_context.topsrcdir, "remote")
  55
  56
  57 @Command("remote", category="misc", description="Remote protocol related operations.")
  58 def remote(command_context):
  59     """The remote subcommands all relate to the remote protocol."""
  60     command_context._sub_mach(["help", "remote"])
  61     return 1
  62
  63
  64 @SubCommand(
  65     "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
  66 )
  67 @CommandArgument(
  68     "--repository",
  69     metavar="REPO",
  70     required=True,
  71     help="The (possibly remote) repository to clone from.",
  72 )
  73 @CommandArgument(
  74     "--commitish",
  75     metavar="COMMITISH",
  76     required=True,
  77     help="The commit or tag object name to check out.",
  78 )
  79 @CommandArgument(
  80     "--no-install",
  81     dest="install",
  82     action="store_false",
  83     default=True,
  84     help="Do not install the just-pulled Puppeteer package,",
  85 )
  86 def vendor_puppeteer(command_context, repository, commitish, install):
  87     puppeteer_dir = os.path.join(remotedir(command_context), "test", "puppeteer")
  88
  89     # Preserve our custom mocha reporter
  90     shutil.move(
  91         os.path.join(puppeteer_dir, "json-mocha-reporter.js"),
  92         remotedir(command_context),
  93     )
  94     shutil.rmtree(puppeteer_dir, ignore_errors=True)
  95     os.makedirs(puppeteer_dir)
  96     with TemporaryDirectory() as tmpdir:
  97         git("clone", "-q", repository, tmpdir)
  98         git("checkout", commitish, worktree=tmpdir)
  99         git(
 100             "checkout-index",
 101             "-a",
 102             "-f",
 103             "--prefix",
 104             "{}/".format(puppeteer_dir),
 105             worktree=tmpdir,
 106         )
 107
 108     # remove files which may interfere with git checkout of central
 109     try:
 110         os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
 111         os.remove(os.path.join(puppeteer_dir, ".gitignore"))
 112     except OSError:
 113         pass
 114
 115     unwanted_dirs = ["experimental", "docs"]
 116
 117     for dir in unwanted_dirs:
 118         dir_path = os.path.join(puppeteer_dir, dir)
 119         if os.path.isdir(dir_path):
 120             shutil.rmtree(dir_path)
 121
 122     shutil.move(
 123         os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
 124         puppeteer_dir,
 125     )
 126
 127     import yaml
 128
 129     annotation = {
 130         "schema": 1,
 131         "bugzilla": {
 132             "product": "Remote Protocol",
 133             "component": "Agent",
 134         },
 135         "origin": {
 136             "name": "puppeteer",
 137             "description": "Headless Chrome Node API",
 138             "url": repository,
 139             "license": "Apache-2.0",
 140             "release": commitish,
 141         },
 142     }
 143     with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 144         yaml.safe_dump(
 145             annotation,
 146             fh,
 147             default_flow_style=False,
 148             encoding="utf-8",
 149             allow_unicode=True,
 150         )
 151
 152     if install:
 153         env = {"PUPPETEER_SKIP_DOWNLOAD": "1"}
 154         npm(
 155             "install",
 156             cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
 157             env=env,
 158         )
 159
 160
 161 def git(*args, **kwargs):
 162     cmd = ("git",)
 163     if kwargs.get("worktree"):
 164         cmd += ("-C", kwargs["worktree"])
 165     cmd += args
 166
 167     pipe = kwargs.get("pipe")
 168     git_p = subprocess.Popen(
 169         cmd,
 170         env={"GIT_CONFIG_NOSYSTEM": "1"},
 171         stdout=subprocess.PIPE,
 172         stderr=subprocess.PIPE,
 173     )
 174     pipe_p = None
 175     if pipe:
 176         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 177
 178     if pipe:
 179         _, pipe_err = pipe_p.communicate()
 180     out, git_err = git_p.communicate()
 181
 182     # use error from first program that failed
 183     if git_p.returncode > 0:
 184         exit(EX_SOFTWARE, git_err)
 185     if pipe and pipe_p.returncode > 0:
 186         exit(EX_SOFTWARE, pipe_err)
 187
 188     return out
 189
 190
 191 def npm(*args, **kwargs):
 192     from mozprocess import processhandler
 193
 194     env = None
 195     if kwargs.get("env"):
 196         env = os.environ.copy()
 197         env.update(kwargs["env"])
 198
 199     proc_kwargs = {}
 200     if "processOutputLine" in kwargs:
 201         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 202
 203     p = processhandler.ProcessHandler(
 204         cmd="npm",
 205         args=list(args),
 206         cwd=kwargs.get("cwd"),
 207         env=env,
 208         universal_newlines=True,
 209         **proc_kwargs
 210     )
 211     if not kwargs.get("wait", True):
 212         return p
 213
 214     wait_proc(p, cmd="npm", exit_on_fail=kwargs.get("exit_on_fail", True))
 215
 216     return p.returncode
 217
 218
 219 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 220     try:
 221         p.run(outputTimeout=output_timeout)
 222         p.wait()
 223         if p.timedOut:
 224             # In some cases, we wait longer for a mocha timeout
 225             print("Timed out after {} seconds of no output".format(output_timeout))
 226     finally:
 227         p.kill()
 228     if exit_on_fail and p.returncode > 0:
 229         msg = (
 230             "%s: exit code %s" % (cmd, p.returncode)
 231             if cmd
 232             else "exit code %s" % p.returncode
 233         )
 234         exit(p.returncode, msg)
 235
 236
 237 class MochaOutputHandler(object):
 238     def __init__(self, logger, expected):
 239         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 240
 241         self.logger = logger
 242         self.proc = None
 243         self.test_results = OrderedDict()
 244         self.expected = expected
 245         self.unexpected_skips = set()
 246
 247         self.has_unexpected = False
 248         self.logger.suite_start([], name="puppeteer-tests")
 249         self.status_map = {
 250             "CRASHED": "CRASH",
 251             "OK": "PASS",
 252             "TERMINATED": "CRASH",
 253             "pass": "PASS",
 254             "fail": "FAIL",
 255             "pending": "SKIP",
 256         }
 257
 258     @property
 259     def pid(self):
 260         return self.proc and self.proc.pid
 261
 262     def __call__(self, line):
 263         event = None
 264         try:
 265             if line.startswith("[") and line.endswith("]"):
 266                 event = json.loads(line)
 267             self.process_event(event)
 268         except ValueError:
 269             pass
 270         finally:
 271             self.logger.process_output(self.pid, line, command="npm")
 272
 273     def process_event(self, event):
 274         if isinstance(event, list) and len(event) > 1:
 275             status = self.status_map.get(event[0])
 276             test_start = event[0] == "test-start"
 277             if not status and not test_start:
 278                 return
 279             test_info = event[1]
 280             test_name = test_info.get("fullTitle", "")
 281             test_path = test_info.get("file", "")
 282             test_err = test_info.get("err")
 283             if status == "FAIL" and test_err:
 284                 if "timeout" in test_err.lower():
 285                     status = "TIMEOUT"
 286             if test_name and test_path:
 287                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 288             # mocha hook failures are not tracked in metadata
 289             if status != "PASS" and self.hook_re.search(test_name):
 290                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 291                 return
 292             if test_start:
 293                 self.logger.test_start(test_name)
 294                 return
 295             expected = self.expected.get(test_name, ["PASS"])
 296             # mozlog doesn't really allow unexpected skip,
 297             # so if a test is disabled just expect that and note the unexpected skip
 298             # Also, mocha doesn't log test-start for skipped tests
 299             if status == "SKIP":
 300                 self.logger.test_start(test_name)
 301                 if self.expected and status not in expected:
 302                     self.unexpected_skips.add(test_name)
 303                 expected = ["SKIP"]
 304             known_intermittent = expected[1:]
 305             expected_status = expected[0]
 306
 307             # check if we've seen a result for this test before this log line
 308             result_recorded = self.test_results.get(test_name)
 309             if result_recorded:
 310                 self.logger.warning(
 311                     "Received a second status for {}: "
 312                     "first {}, now {}".format(test_name, result_recorded, status)
 313                 )
 314             # mocha intermittently logs an additional test result after the
 315             # test has already timed out. Avoid recording this second status.
 316             if result_recorded != "TIMEOUT":
 317                 self.test_results[test_name] = status
 318                 if status not in expected:
 319                     self.has_unexpected = True
 320             self.logger.test_end(
 321                 test_name,
 322                 status=status,
 323                 expected=expected_status,
 324                 known_intermittent=known_intermittent,
 325             )
 326
 327     def new_expected(self):
 328         new_expected = OrderedDict()
 329         for test_name, status in iteritems(self.test_results):
 330             if test_name not in self.expected:
 331                 new_status = [status]
 332             else:
 333                 if status in self.expected[test_name]:
 334                     new_status = self.expected[test_name]
 335                 else:
 336                     new_status = [status]
 337             new_expected[test_name] = new_status
 338         return new_expected
 339
 340     def after_end(self, subset=False):
 341         if not subset:
 342             missing = set(self.expected) - set(self.test_results)
 343             extra = set(self.test_results) - set(self.expected)
 344             if missing:
 345                 self.has_unexpected = True
 346                 for test_name in missing:
 347                     self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,))
 348             if self.expected and extra:
 349                 self.has_unexpected = True
 350                 for test_name in extra:
 351                     self.logger.error(
 352                         "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,)
 353                     )
 354
 355         if self.unexpected_skips:
 356             self.has_unexpected = True
 357             for test_name in self.unexpected_skips:
 358                 self.logger.error(
 359                     "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
 360                 )
 361         self.logger.suite_end()
 362
 363
 364 # tempfile.TemporaryDirectory missing from Python 2.7
 365 class TemporaryDirectory(object):
 366     def __init__(self):
 367         self.path = tempfile.mkdtemp()
 368         self._closed = False
 369
 370     def __repr__(self):
 371         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 372
 373     def __enter__(self):
 374         return self.path
 375
 376     def __exit__(self, exc, value, tb):
 377         self.clean()
 378
 379     def __del__(self):
 380         self.clean()
 381
 382     def clean(self):
 383         if self.path and not self._closed:
 384             shutil.rmtree(self.path)
 385             self._closed = True
 386
 387
 388 class PuppeteerRunner(MozbuildObject):
 389     def __init__(self, *args, **kwargs):
 390         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 391
 392         self.remotedir = os.path.join(self.topsrcdir, "remote")
 393         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 394
 395     def run_test(self, logger, *tests, **params):
 396         """
 397         Runs Puppeteer unit tests with npm.
 398
 399         Possible optional test parameters:
 400
 401         `binary`:
 402           Path for the browser binary to use.  Defaults to the local
 403           build.
 404         `headless`:
 405           Boolean to indicate whether to activate Firefox' headless mode.
 406         `extra_prefs`:
 407           Dictionary of extra preferences to write to the profile,
 408           before invoking npm.  Overrides default preferences.
 409         `enable_webrender`:
 410           Boolean to indicate whether to enable WebRender compositor in Gecko.
 411         `write_results`:
 412           Path to write the results json file
 413         `subset`
 414           Indicates only a subset of tests are being run, so we should
 415           skip the check for missing results
 416         """
 417         setup()
 418
 419         binary = params.get("binary") or self.get_binary_path()
 420         product = params.get("product", "firefox")
 421
 422         env = {
 423             # Print browser process ouptut
 424             "DUMPIO": "1",
 425             # Checked by Puppeteer's custom mocha config
 426             "CI": "1",
 427             # Causes some tests to be skipped due to assumptions about install
 428             "PUPPETEER_ALT_INSTALL": "1",
 429         }
 430         extra_options = {}
 431         for k, v in params.get("extra_launcher_options", {}).items():
 432             extra_options[k] = json.loads(v)
 433
 434         # Override upstream defaults: no retries, shorter timeout
 435         mocha_options = [
 436             "--reporter",
 437             "./json-mocha-reporter.js",
 438             "--retries",
 439             "0",
 440             "--fullTrace",
 441             "--timeout",
 442             "20000",
 443             "--no-parallel",
 444         ]
 445         if product == "firefox":
 446             env["BINARY"] = binary
 447             env["PUPPETEER_PRODUCT"] = "firefox"
 448
 449             env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
 450
 451         command = ["run", "unit", "--"] + mocha_options
 452
 453         env["HEADLESS"] = str(params.get("headless", False))
 454
 455         prefs = {}
 456         for k, v in params.get("extra_prefs", {}).items():
 457             prefs[k] = mozprofile.Preferences.cast(v)
 458
 459         if prefs:
 460             extra_options["extraPrefsFirefox"] = prefs
 461
 462         if extra_options:
 463             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 464
 465         expected_path = os.path.join(
 466             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 467         )
 468         if product == "firefox" and os.path.exists(expected_path):
 469             with open(expected_path) as f:
 470                 expected_data = json.load(f)
 471         else:
 472             expected_data = {}
 473
 474         output_handler = MochaOutputHandler(logger, expected_data)
 475         proc = npm(
 476             *command,
 477             cwd=self.puppeteer_dir,
 478             env=env,
 479             processOutputLine=output_handler,
 480             wait=False
 481         )
 482         output_handler.proc = proc
 483
 484         # Puppeteer unit tests don't always clean-up child processes in case of
 485         # failure, so use an output_timeout as a fallback
 486         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 487
 488         output_handler.after_end(params.get("subset", False))
 489
 490         # Non-zero return codes are non-fatal for now since we have some
 491         # issues with unresolved promises that shouldn't otherwise block
 492         # running the tests
 493         if proc.returncode != 0:
 494             logger.warning("npm exited with code %s" % proc.returncode)
 495
 496         if params["write_results"]:
 497             with open(params["write_results"], "w") as f:
 498                 json.dump(
 499                     output_handler.new_expected(), f, indent=2, separators=(",", ": ")
 500                 )
 501
 502         if output_handler.has_unexpected:
 503             exit(1, "Got unexpected results")
 504
 505
 506 def create_parser_puppeteer():
 507     p = argparse.ArgumentParser()
 508     p.add_argument(
 509         "--product", type=str, default="firefox", choices=["chrome", "firefox"]
 510     )
 511     p.add_argument(
 512         "--binary",
 513         type=str,
 514         help="Path to browser binary.  Defaults to local Firefox build.",
 515     )
 516     p.add_argument(
 517         "--ci",
 518         action="store_true",
 519         help="Flag that indicates that tests run in a CI environment.",
 520     )
 521     p.add_argument(
 522         "--enable-fission",
 523         action="store_true",
 524         help="Enable Fission (site isolation) in Gecko.",
 525     )
 526     p.add_argument(
 527         "--enable-webrender",
 528         action="store_true",
 529         help="Enable the WebRender compositor in Gecko.",
 530     )
 531     p.add_argument(
 532         "-z", "--headless", action="store_true", help="Run browser in headless mode."
 533     )
 534     p.add_argument(
 535         "--setpref",
 536         action="append",
 537         dest="extra_prefs",
 538         metavar="<pref>=<value>",
 539         help="Defines additional user preferences.",
 540     )
 541     p.add_argument(
 542         "--setopt",
 543         action="append",
 544         dest="extra_options",
 545         metavar="<option>=<value>",
 546         help="Defines additional options for `puppeteer.launch`.",
 547     )
 548     p.add_argument(
 549         "-v",
 550         dest="verbosity",
 551         action="count",
 552         default=0,
 553         help="Increase remote agent logging verbosity to include "
 554         "debug level messages with -v, trace messages with -vv,"
 555         "and to not truncate long trace messages with -vvv",
 556     )
 557     p.add_argument(
 558         "--write-results",
 559         action="store",
 560         nargs="?",
 561         default=None,
 562         const=os.path.join(
 563             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 564         ),
 565         help="Path to write updated results to (defaults to the "
 566         "expectations file if the argument is provided but "
 567         "no path is passed)",
 568     )
 569     p.add_argument(
 570         "--subset",
 571         action="store_true",
 572         default=False,
 573         help="Indicate that only a subset of the tests are running, "
 574         "so checks for missing tests should be skipped",
 575     )
 576     p.add_argument("tests", nargs="*")
 577     mozlog.commandline.add_logging_group(p)
 578     return p
 579
 580
 581 @Command(
 582     "puppeteer-test",
 583     category="testing",
 584     description="Run Puppeteer unit tests.",
 585     parser=create_parser_puppeteer,
 586 )
 587 def puppeteer_test(
 588     command_context,
 589     binary=None,
 590     ci=False,
 591     enable_fission=False,
 592     enable_webrender=False,
 593     headless=False,
 594     extra_prefs=None,
 595     extra_options=None,
 596     verbosity=0,
 597     tests=None,
 598     product="firefox",
 599     write_results=None,
 600     subset=False,
 601     **kwargs
 602 ):
 603
 604     logger = mozlog.commandline.setup_logging(
 605         "puppeteer-test", kwargs, {"mach": sys.stdout}
 606     )
 607
 608     # moztest calls this programmatically with test objects or manifests
 609     if "test_objects" in kwargs and tests is not None:
 610         logger.error("Expected either 'test_objects' or 'tests'")
 611         exit(1)
 612
 613     if product != "firefox" and extra_prefs is not None:
 614         logger.error("User preferences are not recognized by %s" % product)
 615         exit(1)
 616
 617     if "test_objects" in kwargs:
 618         tests = []
 619         for test in kwargs["test_objects"]:
 620             tests.append(test["path"])
 621
 622     prefs = {}
 623     for s in extra_prefs or []:
 624         kv = s.split("=")
 625         if len(kv) != 2:
 626             logger.error("syntax error in --setpref={}".format(s))
 627             exit(EX_USAGE)
 628         prefs[kv[0]] = kv[1].strip()
 629
 630     options = {}
 631     for s in extra_options or []:
 632         kv = s.split("=")
 633         if len(kv) != 2:
 634             logger.error("syntax error in --setopt={}".format(s))
 635             exit(EX_USAGE)
 636         options[kv[0]] = kv[1].strip()
 637
 638     if enable_fission:
 639         prefs.update({"fission.autostart": True})
 640
 641     if verbosity == 1:
 642         prefs["remote.log.level"] = "Debug"
 643     elif verbosity > 1:
 644         prefs["remote.log.level"] = "Trace"
 645     if verbosity > 2:
 646         prefs["remote.log.truncate"] = False
 647
 648     install_puppeteer(command_context, product, ci)
 649
 650     params = {
 651         "binary": binary,
 652         "headless": headless,
 653         "enable_webrender": enable_webrender,
 654         "extra_prefs": prefs,
 655         "product": product,
 656         "extra_launcher_options": options,
 657         "write_results": write_results,
 658         "subset": subset,
 659     }
 660     puppeteer = command_context._spawn(PuppeteerRunner)
 661     try:
 662         return puppeteer.run_test(logger, *tests, **params)
 663     except BinaryNotFoundException as e:
 664         logger.error(e)
 665         logger.info(e.help())
 666         exit(1)
 667     except Exception as e:
 668         exit(EX_SOFTWARE, e)
 669
 670
 671 def install_puppeteer(command_context, product, ci):
 672     setup()
 673     env = {}
 674     from mozversioncontrol import get_repository_object
 675
 676     repo = get_repository_object(command_context.topsrcdir)
 677     puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 678     changed_files = False
 679     for f in repo.get_changed_files():
 680         if f.startswith(puppeteer_dir) and f.endswith(".ts"):
 681             changed_files = True
 682             break
 683
 684     if product != "chrome":
 685         env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 686     lib_dir = os.path.join(command_context.topsrcdir, puppeteer_dir, "lib")
 687     if changed_files and os.path.isdir(lib_dir):
 688         # clobber lib to force `tsc compile` step
 689         shutil.rmtree(lib_dir)
 690
 691     command = "ci" if ci else "install"
 692     npm(command, cwd=os.path.join(command_context.topsrcdir, puppeteer_dir), env=env)
 693
 694
 695 def exit(code, error=None):
 696     if error is not None:
 697         if isinstance(error, Exception):
 698             import traceback
 699
 700             traceback.print_exc()
 701         else:
 702             message = str(error).split("\n")[0].strip()
 703             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 704     sys.exit(code)