remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import argparse
  12 import json
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import tempfile
  19
  20 from collections import OrderedDict
  21
  22 from six import iteritems
  23
  24 from mach.decorators import (
  25     Command,
  26     CommandArgument,
  27     SubCommand,
  28 )
  29
  30 from mozbuild.base import (
  31     MozbuildObject,
  32     BinaryNotFoundException,
  33 )
  34 from mozbuild import nodeutil
  35 import mozlog
  36 import mozprofile
  37
  38
  39 EX_CONFIG = 78
  40 EX_SOFTWARE = 70
  41 EX_USAGE = 64
  42
  43
  44 def setup():
  45     # add node and npm from mozbuild to front of system path
  46     npm, _ = nodeutil.find_npm_executable()
  47     if not npm:
  48         exit(EX_CONFIG, "could not find npm executable")
  49     path = os.path.abspath(os.path.join(npm, os.pardir))
  50     os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
  51
  52
  53 def remotedir(command_context):
  54     return os.path.join(command_context.topsrcdir, "remote")
  55
  56
  57 @Command("remote", category="misc", description="Remote protocol related operations.")
  58 def remote(command_context):
  59     """The remote subcommands all relate to the remote protocol."""
  60     command_context._sub_mach(["help", "remote"])
  61     return 1
  62
  63
  64 @SubCommand(
  65     "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
  66 )
  67 @CommandArgument(
  68     "--repository",
  69     metavar="REPO",
  70     required=True,
  71     help="The (possibly remote) repository to clone from.",
  72 )
  73 @CommandArgument(
  74     "--commitish",
  75     metavar="COMMITISH",
  76     required=True,
  77     help="The commit or tag object name to check out.",
  78 )
  79 @CommandArgument(
  80     "--no-install",
  81     dest="install",
  82     action="store_false",
  83     default=True,
  84     help="Do not install the just-pulled Puppeteer package,",
  85 )
  86 def vendor_puppeteer(command_context, repository, commitish, install):
  87     puppeteer_dir = os.path.join(remotedir(command_context), "test", "puppeteer")
  88
  89     # Preserve our custom mocha reporter
  90     shutil.move(
  91         os.path.join(puppeteer_dir, "json-mocha-reporter.js"),
  92         remotedir(command_context),
  93     )
  94     shutil.rmtree(puppeteer_dir, ignore_errors=True)
  95     os.makedirs(puppeteer_dir)
  96     with TemporaryDirectory() as tmpdir:
  97         git("clone", "-q", repository, tmpdir)
  98         git("checkout", commitish, worktree=tmpdir)
  99         git(
 100             "checkout-index",
 101             "-a",
 102             "-f",
 103             "--prefix",
 104             "{}/".format(puppeteer_dir),
 105             worktree=tmpdir,
 106         )
 107
 108     # remove files which may interfere with git checkout of central
 109     try:
 110         os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
 111         os.remove(os.path.join(puppeteer_dir, ".gitignore"))
 112     except OSError:
 113         pass
 114
 115     unwanted_dirs = ["experimental", "docs"]
 116
 117     for dir in unwanted_dirs:
 118         dir_path = os.path.join(puppeteer_dir, dir)
 119         if os.path.isdir(dir_path):
 120             shutil.rmtree(dir_path)
 121
 122     shutil.move(
 123         os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
 124         puppeteer_dir,
 125     )
 126
 127     import yaml
 128
 129     annotation = {
 130         "schema": 1,
 131         "bugzilla": {
 132             "product": "Remote Protocol",
 133             "component": "Agent",
 134         },
 135         "origin": {
 136             "name": "puppeteer",
 137             "description": "Headless Chrome Node API",
 138             "url": repository,
 139             "license": "Apache-2.0",
 140             "release": commitish,
 141         },
 142     }
 143     with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 144         yaml.safe_dump(
 145             annotation,
 146             fh,
 147             default_flow_style=False,
 148             encoding="utf-8",
 149             allow_unicode=True,
 150         )
 151
 152     if install:
 153         env = {"HUSKY": "0", "PUPPETEER_SKIP_DOWNLOAD": "1"}
 154         npm(
 155             "install",
 156             cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
 157             env=env,
 158         )
 159
 160
 161 def git(*args, **kwargs):
 162     cmd = ("git",)
 163     if kwargs.get("worktree"):
 164         cmd += ("-C", kwargs["worktree"])
 165     cmd += args
 166
 167     pipe = kwargs.get("pipe")
 168     git_p = subprocess.Popen(
 169         cmd,
 170         env={"GIT_CONFIG_NOSYSTEM": "1"},
 171         stdout=subprocess.PIPE,
 172         stderr=subprocess.PIPE,
 173     )
 174     pipe_p = None
 175     if pipe:
 176         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 177
 178     if pipe:
 179         _, pipe_err = pipe_p.communicate()
 180     out, git_err = git_p.communicate()
 181
 182     # use error from first program that failed
 183     if git_p.returncode > 0:
 184         exit(EX_SOFTWARE, git_err)
 185     if pipe and pipe_p.returncode > 0:
 186         exit(EX_SOFTWARE, pipe_err)
 187
 188     return out
 189
 190
 191 def npm(*args, **kwargs):
 192     from mozprocess import processhandler
 193
 194     env = None
 195     npm, _ = nodeutil.find_npm_executable()
 196     if kwargs.get("env"):
 197         env = os.environ.copy()
 198         env.update(kwargs["env"])
 199
 200     proc_kwargs = {}
 201     if "processOutputLine" in kwargs:
 202         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 203
 204     p = processhandler.ProcessHandler(
 205         cmd=npm,
 206         args=list(args),
 207         cwd=kwargs.get("cwd"),
 208         env=env,
 209         universal_newlines=True,
 210         **proc_kwargs
 211     )
 212     if not kwargs.get("wait", True):
 213         return p
 214
 215     wait_proc(p, cmd=npm, exit_on_fail=kwargs.get("exit_on_fail", True))
 216
 217     return p.returncode
 218
 219
 220 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 221     try:
 222         p.run(outputTimeout=output_timeout)
 223         p.wait()
 224         if p.timedOut:
 225             # In some cases, we wait longer for a mocha timeout
 226             print("Timed out after {} seconds of no output".format(output_timeout))
 227     finally:
 228         p.kill()
 229     if exit_on_fail and p.returncode > 0:
 230         msg = (
 231             "%s: exit code %s" % (cmd, p.returncode)
 232             if cmd
 233             else "exit code %s" % p.returncode
 234         )
 235         exit(p.returncode, msg)
 236
 237
 238 class MochaOutputHandler(object):
 239     def __init__(self, logger, expected):
 240         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 241
 242         self.logger = logger
 243         self.proc = None
 244         self.test_results = OrderedDict()
 245         self.expected = expected
 246         self.unexpected_skips = set()
 247
 248         self.has_unexpected = False
 249         self.logger.suite_start([], name="puppeteer-tests")
 250         self.status_map = {
 251             "CRASHED": "CRASH",
 252             "OK": "PASS",
 253             "TERMINATED": "CRASH",
 254             "pass": "PASS",
 255             "fail": "FAIL",
 256             "pending": "SKIP",
 257         }
 258
 259     @property
 260     def pid(self):
 261         return self.proc and self.proc.pid
 262
 263     def __call__(self, line):
 264         event = None
 265         try:
 266             if line.startswith("[") and line.endswith("]"):
 267                 event = json.loads(line)
 268             self.process_event(event)
 269         except ValueError:
 270             pass
 271         finally:
 272             self.logger.process_output(self.pid, line, command="npm")
 273
 274     def process_event(self, event):
 275         if isinstance(event, list) and len(event) > 1:
 276             status = self.status_map.get(event[0])
 277             test_start = event[0] == "test-start"
 278             if not status and not test_start:
 279                 return
 280             test_info = event[1]
 281             test_name = test_info.get("fullTitle", "")
 282             test_path = test_info.get("file", "")
 283             test_err = test_info.get("err")
 284             if status == "FAIL" and test_err:
 285                 if "timeout" in test_err.lower():
 286                     status = "TIMEOUT"
 287             if test_name and test_path:
 288                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 289             # mocha hook failures are not tracked in metadata
 290             if status != "PASS" and self.hook_re.search(test_name):
 291                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 292                 return
 293             if test_start:
 294                 self.logger.test_start(test_name)
 295                 return
 296             expected = self.expected.get(test_name, ["PASS"])
 297             # mozlog doesn't really allow unexpected skip,
 298             # so if a test is disabled just expect that and note the unexpected skip
 299             # Also, mocha doesn't log test-start for skipped tests
 300             if status == "SKIP":
 301                 self.logger.test_start(test_name)
 302                 if self.expected and status not in expected:
 303                     self.unexpected_skips.add(test_name)
 304                 expected = ["SKIP"]
 305             known_intermittent = expected[1:]
 306             expected_status = expected[0]
 307
 308             # check if we've seen a result for this test before this log line
 309             result_recorded = self.test_results.get(test_name)
 310             if result_recorded:
 311                 self.logger.warning(
 312                     "Received a second status for {}: "
 313                     "first {}, now {}".format(test_name, result_recorded, status)
 314                 )
 315             # mocha intermittently logs an additional test result after the
 316             # test has already timed out. Avoid recording this second status.
 317             if result_recorded != "TIMEOUT":
 318                 self.test_results[test_name] = status
 319                 if status not in expected:
 320                     self.has_unexpected = True
 321             self.logger.test_end(
 322                 test_name,
 323                 status=status,
 324                 expected=expected_status,
 325                 known_intermittent=known_intermittent,
 326             )
 327
 328     def new_expected(self):
 329         new_expected = OrderedDict()
 330         for test_name, status in iteritems(self.test_results):
 331             if test_name not in self.expected:
 332                 new_status = [status]
 333             else:
 334                 if status in self.expected[test_name]:
 335                     new_status = self.expected[test_name]
 336                 else:
 337                     new_status = [status]
 338             new_expected[test_name] = new_status
 339         return new_expected
 340
 341     def after_end(self, subset=False):
 342         if not subset:
 343             missing = set(self.expected) - set(self.test_results)
 344             extra = set(self.test_results) - set(self.expected)
 345             if missing:
 346                 self.has_unexpected = True
 347                 for test_name in missing:
 348                     self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,))
 349             if self.expected and extra:
 350                 self.has_unexpected = True
 351                 for test_name in extra:
 352                     self.logger.error(
 353                         "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,)
 354                     )
 355
 356         if self.unexpected_skips:
 357             self.has_unexpected = True
 358             for test_name in self.unexpected_skips:
 359                 self.logger.error(
 360                     "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
 361                 )
 362         self.logger.suite_end()
 363
 364
 365 # tempfile.TemporaryDirectory missing from Python 2.7
 366 class TemporaryDirectory(object):
 367     def __init__(self):
 368         self.path = tempfile.mkdtemp()
 369         self._closed = False
 370
 371     def __repr__(self):
 372         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 373
 374     def __enter__(self):
 375         return self.path
 376
 377     def __exit__(self, exc, value, tb):
 378         self.clean()
 379
 380     def __del__(self):
 381         self.clean()
 382
 383     def clean(self):
 384         if self.path and not self._closed:
 385             shutil.rmtree(self.path)
 386             self._closed = True
 387
 388
 389 class PuppeteerRunner(MozbuildObject):
 390     def __init__(self, *args, **kwargs):
 391         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 392
 393         self.remotedir = os.path.join(self.topsrcdir, "remote")
 394         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 395
 396     def run_test(self, logger, *tests, **params):
 397         """
 398         Runs Puppeteer unit tests with npm.
 399
 400         Possible optional test parameters:
 401
 402         `binary`:
 403           Path for the browser binary to use.  Defaults to the local
 404           build.
 405         `headless`:
 406           Boolean to indicate whether to activate Firefox' headless mode.
 407         `extra_prefs`:
 408           Dictionary of extra preferences to write to the profile,
 409           before invoking npm.  Overrides default preferences.
 410         `enable_webrender`:
 411           Boolean to indicate whether to enable WebRender compositor in Gecko.
 412         `write_results`:
 413           Path to write the results json file
 414         `subset`
 415           Indicates only a subset of tests are being run, so we should
 416           skip the check for missing results
 417         """
 418         setup()
 419
 420         binary = params.get("binary") or self.get_binary_path()
 421         product = params.get("product", "firefox")
 422
 423         env = {
 424             # Print browser process ouptut
 425             "DUMPIO": "1",
 426             # Checked by Puppeteer's custom mocha config
 427             "CI": "1",
 428             # Causes some tests to be skipped due to assumptions about install
 429             "PUPPETEER_ALT_INSTALL": "1",
 430         }
 431         extra_options = {}
 432         for k, v in params.get("extra_launcher_options", {}).items():
 433             extra_options[k] = json.loads(v)
 434
 435         # Override upstream defaults: no retries, shorter timeout
 436         mocha_options = [
 437             "--reporter",
 438             "./json-mocha-reporter.js",
 439             "--retries",
 440             "0",
 441             "--fullTrace",
 442             "--timeout",
 443             "20000",
 444             "--no-parallel",
 445         ]
 446         if product == "firefox":
 447             env["BINARY"] = binary
 448             env["PUPPETEER_PRODUCT"] = "firefox"
 449
 450             env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
 451
 452         command = ["run", "unit", "--"] + mocha_options
 453
 454         env["HEADLESS"] = str(params.get("headless", False))
 455
 456         prefs = {}
 457         for k, v in params.get("extra_prefs", {}).items():
 458             print("Using extra preference: {}={}".format(k, v))
 459             prefs[k] = mozprofile.Preferences.cast(v)
 460
 461         if prefs:
 462             extra_options["extraPrefsFirefox"] = prefs
 463
 464         if extra_options:
 465             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 466
 467         expected_path = os.path.join(
 468             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 469         )
 470         if product == "firefox" and os.path.exists(expected_path):
 471             with open(expected_path) as f:
 472                 expected_data = json.load(f)
 473         else:
 474             expected_data = {}
 475
 476         output_handler = MochaOutputHandler(logger, expected_data)
 477         proc = npm(
 478             *command,
 479             cwd=self.puppeteer_dir,
 480             env=env,
 481             processOutputLine=output_handler,
 482             wait=False
 483         )
 484         output_handler.proc = proc
 485
 486         # Puppeteer unit tests don't always clean-up child processes in case of
 487         # failure, so use an output_timeout as a fallback
 488         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 489
 490         output_handler.after_end(params.get("subset", False))
 491
 492         # Non-zero return codes are non-fatal for now since we have some
 493         # issues with unresolved promises that shouldn't otherwise block
 494         # running the tests
 495         if proc.returncode != 0:
 496             logger.warning("npm exited with code %s" % proc.returncode)
 497
 498         if params["write_results"]:
 499             with open(params["write_results"], "w") as f:
 500                 json.dump(
 501                     output_handler.new_expected(), f, indent=2, separators=(",", ": ")
 502                 )
 503
 504         if output_handler.has_unexpected:
 505             exit(1, "Got unexpected results")
 506
 507
 508 def create_parser_puppeteer():
 509     p = argparse.ArgumentParser()
 510     p.add_argument(
 511         "--product", type=str, default="firefox", choices=["chrome", "firefox"]
 512     )
 513     p.add_argument(
 514         "--binary",
 515         type=str,
 516         help="Path to browser binary.  Defaults to local Firefox build.",
 517     )
 518     p.add_argument(
 519         "--ci",
 520         action="store_true",
 521         help="Flag that indicates that tests run in a CI environment.",
 522     )
 523     p.add_argument(
 524         "--disable-fission",
 525         action="store_true",
 526         default=False,
 527         dest="disable_fission",
 528         help="Disable Fission (site isolation) in Gecko.",
 529     )
 530     p.add_argument(
 531         "--enable-webrender",
 532         action="store_true",
 533         help="Enable the WebRender compositor in Gecko.",
 534     )
 535     p.add_argument(
 536         "-z", "--headless", action="store_true", help="Run browser in headless mode."
 537     )
 538     p.add_argument(
 539         "--setpref",
 540         action="append",
 541         dest="extra_prefs",
 542         metavar="<pref>=<value>",
 543         help="Defines additional user preferences.",
 544     )
 545     p.add_argument(
 546         "--setopt",
 547         action="append",
 548         dest="extra_options",
 549         metavar="<option>=<value>",
 550         help="Defines additional options for `puppeteer.launch`.",
 551     )
 552     p.add_argument(
 553         "-v",
 554         dest="verbosity",
 555         action="count",
 556         default=0,
 557         help="Increase remote agent logging verbosity to include "
 558         "debug level messages with -v, trace messages with -vv,"
 559         "and to not truncate long trace messages with -vvv",
 560     )
 561     p.add_argument(
 562         "--write-results",
 563         action="store",
 564         nargs="?",
 565         default=None,
 566         const=os.path.join(
 567             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 568         ),
 569         help="Path to write updated results to (defaults to the "
 570         "expectations file if the argument is provided but "
 571         "no path is passed)",
 572     )
 573     p.add_argument(
 574         "--subset",
 575         action="store_true",
 576         default=False,
 577         help="Indicate that only a subset of the tests are running, "
 578         "so checks for missing tests should be skipped",
 579     )
 580     p.add_argument("tests", nargs="*")
 581     mozlog.commandline.add_logging_group(p)
 582     return p
 583
 584
 585 @Command(
 586     "puppeteer-test",
 587     category="testing",
 588     description="Run Puppeteer unit tests.",
 589     parser=create_parser_puppeteer,
 590 )
 591 @CommandArgument(
 592     "--no-install",
 593     dest="install",
 594     action="store_false",
 595     default=True,
 596     help="Do not install the Puppeteer package",
 597 )
 598 def puppeteer_test(
 599     command_context,
 600     binary=None,
 601     ci=False,
 602     disable_fission=False,
 603     enable_webrender=False,
 604     headless=False,
 605     extra_prefs=None,
 606     extra_options=None,
 607     install=False,
 608     verbosity=0,
 609     tests=None,
 610     product="firefox",
 611     write_results=None,
 612     subset=False,
 613     **kwargs
 614 ):
 615
 616     logger = mozlog.commandline.setup_logging(
 617         "puppeteer-test", kwargs, {"mach": sys.stdout}
 618     )
 619
 620     # moztest calls this programmatically with test objects or manifests
 621     if "test_objects" in kwargs and tests is not None:
 622         logger.error("Expected either 'test_objects' or 'tests'")
 623         exit(1)
 624
 625     if product != "firefox" and extra_prefs is not None:
 626         logger.error("User preferences are not recognized by %s" % product)
 627         exit(1)
 628
 629     if "test_objects" in kwargs:
 630         tests = []
 631         for test in kwargs["test_objects"]:
 632             tests.append(test["path"])
 633
 634     prefs = {}
 635     for s in extra_prefs or []:
 636         kv = s.split("=")
 637         if len(kv) != 2:
 638             logger.error("syntax error in --setpref={}".format(s))
 639             exit(EX_USAGE)
 640         prefs[kv[0]] = kv[1].strip()
 641
 642     options = {}
 643     for s in extra_options or []:
 644         kv = s.split("=")
 645         if len(kv) != 2:
 646             logger.error("syntax error in --setopt={}".format(s))
 647             exit(EX_USAGE)
 648         options[kv[0]] = kv[1].strip()
 649
 650     prefs.update({"fission.autostart": True})
 651     if disable_fission:
 652         prefs.update({"fission.autostart": False})
 653
 654     if verbosity == 1:
 655         prefs["remote.log.level"] = "Debug"
 656     elif verbosity > 1:
 657         prefs["remote.log.level"] = "Trace"
 658     if verbosity > 2:
 659         prefs["remote.log.truncate"] = False
 660
 661     if install:
 662         install_puppeteer(command_context, product, ci)
 663
 664     params = {
 665         "binary": binary,
 666         "headless": headless,
 667         "enable_webrender": enable_webrender,
 668         "extra_prefs": prefs,
 669         "product": product,
 670         "extra_launcher_options": options,
 671         "write_results": write_results,
 672         "subset": subset,
 673     }
 674     puppeteer = command_context._spawn(PuppeteerRunner)
 675     try:
 676         return puppeteer.run_test(logger, *tests, **params)
 677     except BinaryNotFoundException as e:
 678         logger.error(e)
 679         logger.info(e.help())
 680         exit(1)
 681     except Exception as e:
 682         exit(EX_SOFTWARE, e)
 683
 684
 685 def install_puppeteer(command_context, product, ci):
 686     setup()
 687     env = {"HUSKY": "0"}
 688     from mozversioncontrol import get_repository_object
 689
 690     repo = get_repository_object(command_context.topsrcdir)
 691     puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 692     changed_files = False
 693     for f in repo.get_changed_files():
 694         if f.startswith(puppeteer_dir) and f.endswith(".ts"):
 695             changed_files = True
 696             break
 697
 698     if product != "chrome":
 699         env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 700     lib_dir = os.path.join(command_context.topsrcdir, puppeteer_dir, "lib")
 701     if changed_files and os.path.isdir(lib_dir):
 702         # clobber lib to force `tsc compile` step
 703         shutil.rmtree(lib_dir)
 704
 705     command = "ci" if ci else "install"
 706     npm(command, cwd=os.path.join(command_context.topsrcdir, puppeteer_dir), env=env)
 707
 708
 709 def exit(code, error=None):
 710     if error is not None:
 711         if isinstance(error, Exception):
 712             import traceback
 713
 714             traceback.print_exc()
 715         else:
 716             message = str(error).split("\n")[0].strip()
 717             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 718     sys.exit(code)