remote/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 from __future__ import (
   6     absolute_import,
   7     print_function,
   8     unicode_literals,
   9 )
  10
  11 import argparse
  12 import json
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import sys
  18 import tempfile
  19
  20 from collections import OrderedDict
  21
  22 from six import iteritems
  23
  24 from mach.decorators import (
  25     Command,
  26     CommandArgument,
  27     SubCommand,
  28 )
  29
  30 from mozbuild.base import (
  31     MozbuildObject,
  32     BinaryNotFoundException,
  33 )
  34 from mozbuild import nodeutil
  35 import mozlog
  36 import mozprofile
  37
  38
  39 EX_CONFIG = 78
  40 EX_SOFTWARE = 70
  41 EX_USAGE = 64
  42
  43
  44 def setup():
  45     # add node and npm from mozbuild to front of system path
  46     npm, _ = nodeutil.find_npm_executable()
  47     if not npm:
  48         exit(EX_CONFIG, "could not find npm executable")
  49     path = os.path.abspath(os.path.join(npm, os.pardir))
  50     os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
  51
  52
  53 def remotedir(command_context):
  54     return os.path.join(command_context.topsrcdir, "remote")
  55
  56
  57 @Command("remote", category="misc", description="Remote protocol related operations.")
  58 def remote(command_context):
  59     """The remote subcommands all relate to the remote protocol."""
  60     command_context._sub_mach(["help", "remote"])
  61     return 1
  62
  63
  64 @SubCommand(
  65     "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
  66 )
  67 @CommandArgument(
  68     "--repository",
  69     metavar="REPO",
  70     required=True,
  71     help="The (possibly remote) repository to clone from.",
  72 )
  73 @CommandArgument(
  74     "--commitish",
  75     metavar="COMMITISH",
  76     required=True,
  77     help="The commit or tag object name to check out.",
  78 )
  79 @CommandArgument(
  80     "--no-install",
  81     dest="install",
  82     action="store_false",
  83     default=True,
  84     help="Do not install the just-pulled Puppeteer package,",
  85 )
  86 def vendor_puppeteer(command_context, repository, commitish, install):
  87     puppeteer_dir = os.path.join(remotedir(command_context), "test", "puppeteer")
  88
  89     # Preserve our custom mocha reporter
  90     shutil.move(
  91         os.path.join(puppeteer_dir, "json-mocha-reporter.js"),
  92         remotedir(command_context),
  93     )
  94     shutil.rmtree(puppeteer_dir, ignore_errors=True)
  95     os.makedirs(puppeteer_dir)
  96     with TemporaryDirectory() as tmpdir:
  97         git("clone", "-q", repository, tmpdir)
  98         git("checkout", commitish, worktree=tmpdir)
  99         git(
 100             "checkout-index",
 101             "-a",
 102             "-f",
 103             "--prefix",
 104             "{}/".format(puppeteer_dir),
 105             worktree=tmpdir,
 106         )
 107
 108     # remove files which may interfere with git checkout of central
 109     try:
 110         os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
 111         os.remove(os.path.join(puppeteer_dir, ".gitignore"))
 112     except OSError:
 113         pass
 114
 115     unwanted_dirs = ["experimental", "docs"]
 116
 117     for dir in unwanted_dirs:
 118         dir_path = os.path.join(puppeteer_dir, dir)
 119         if os.path.isdir(dir_path):
 120             shutil.rmtree(dir_path)
 121
 122     shutil.move(
 123         os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
 124         puppeteer_dir,
 125     )
 126
 127     import yaml
 128
 129     annotation = {
 130         "schema": 1,
 131         "bugzilla": {
 132             "product": "Remote Protocol",
 133             "component": "Agent",
 134         },
 135         "origin": {
 136             "name": "puppeteer",
 137             "description": "Headless Chrome Node API",
 138             "url": repository,
 139             "license": "Apache-2.0",
 140             "release": commitish,
 141         },
 142     }
 143     with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
 144         yaml.safe_dump(
 145             annotation,
 146             fh,
 147             default_flow_style=False,
 148             encoding="utf-8",
 149             allow_unicode=True,
 150         )
 151
 152     if install:
 153         env = {"HUSKY": "0", "PUPPETEER_SKIP_DOWNLOAD": "1"}
 154         npm(
 155             "install",
 156             cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
 157             env=env,
 158         )
 159
 160
 161 def git(*args, **kwargs):
 162     cmd = ("git",)
 163     if kwargs.get("worktree"):
 164         cmd += ("-C", kwargs["worktree"])
 165     cmd += args
 166
 167     pipe = kwargs.get("pipe")
 168     git_p = subprocess.Popen(
 169         cmd,
 170         env={"GIT_CONFIG_NOSYSTEM": "1"},
 171         stdout=subprocess.PIPE,
 172         stderr=subprocess.PIPE,
 173     )
 174     pipe_p = None
 175     if pipe:
 176         pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
 177
 178     if pipe:
 179         _, pipe_err = pipe_p.communicate()
 180     out, git_err = git_p.communicate()
 181
 182     # use error from first program that failed
 183     if git_p.returncode > 0:
 184         exit(EX_SOFTWARE, git_err)
 185     if pipe and pipe_p.returncode > 0:
 186         exit(EX_SOFTWARE, pipe_err)
 187
 188     return out
 189
 190
 191 def npm(*args, **kwargs):
 192     from mozprocess import processhandler
 193
 194     env = None
 195     npm, _ = nodeutil.find_npm_executable()
 196     if kwargs.get("env"):
 197         env = os.environ.copy()
 198         env.update(kwargs["env"])
 199
 200     proc_kwargs = {}
 201     if "processOutputLine" in kwargs:
 202         proc_kwargs["processOutputLine"] = kwargs["processOutputLine"]
 203
 204     p = processhandler.ProcessHandler(
 205         cmd=npm,
 206         args=list(args),
 207         cwd=kwargs.get("cwd"),
 208         env=env,
 209         universal_newlines=True,
 210         **proc_kwargs,
 211     )
 212     if not kwargs.get("wait", True):
 213         return p
 214
 215     wait_proc(p, cmd=npm, exit_on_fail=kwargs.get("exit_on_fail", True))
 216
 217     return p.returncode
 218
 219
 220 def wait_proc(p, cmd=None, exit_on_fail=True, output_timeout=None):
 221     try:
 222         p.run(outputTimeout=output_timeout)
 223         p.wait()
 224         if p.timedOut:
 225             # In some cases, we wait longer for a mocha timeout
 226             print("Timed out after {} seconds of no output".format(output_timeout))
 227     finally:
 228         p.kill()
 229     if exit_on_fail and p.returncode > 0:
 230         msg = (
 231             "%s: exit code %s" % (cmd, p.returncode)
 232             if cmd
 233             else "exit code %s" % p.returncode
 234         )
 235         exit(p.returncode, msg)
 236
 237
 238 class MochaOutputHandler(object):
 239     def __init__(self, logger, expected):
 240         self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
 241
 242         self.logger = logger
 243         self.proc = None
 244         self.test_results = OrderedDict()
 245         self.expected = expected
 246         self.unexpected_skips = set()
 247
 248         self.has_unexpected = False
 249         self.logger.suite_start([], name="puppeteer-tests")
 250         self.status_map = {
 251             "CRASHED": "CRASH",
 252             "OK": "PASS",
 253             "TERMINATED": "CRASH",
 254             "pass": "PASS",
 255             "fail": "FAIL",
 256             "pending": "SKIP",
 257         }
 258
 259     @property
 260     def pid(self):
 261         return self.proc and self.proc.pid
 262
 263     def __call__(self, line):
 264         event = None
 265         try:
 266             if line.startswith("[") and line.endswith("]"):
 267                 event = json.loads(line)
 268             self.process_event(event)
 269         except ValueError:
 270             pass
 271         finally:
 272             self.logger.process_output(self.pid, line, command="npm")
 273
 274     def process_event(self, event):
 275         if isinstance(event, list) and len(event) > 1:
 276             status = self.status_map.get(event[0])
 277             test_start = event[0] == "test-start"
 278             if not status and not test_start:
 279                 return
 280             test_info = event[1]
 281             test_name = test_info.get("fullTitle", "")
 282             test_path = test_info.get("file", "")
 283             test_err = test_info.get("err")
 284             if status == "FAIL" and test_err:
 285                 if "timeout" in test_err.lower():
 286                     status = "TIMEOUT"
 287             if test_name and test_path:
 288                 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
 289             # mocha hook failures are not tracked in metadata
 290             if status != "PASS" and self.hook_re.search(test_name):
 291                 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
 292                 return
 293             if test_start:
 294                 self.logger.test_start(test_name)
 295                 return
 296             expected = self.expected.get(test_name, ["PASS"])
 297             # mozlog doesn't really allow unexpected skip,
 298             # so if a test is disabled just expect that and note the unexpected skip
 299             # Also, mocha doesn't log test-start for skipped tests
 300             if status == "SKIP":
 301                 self.logger.test_start(test_name)
 302                 if self.expected and status not in expected:
 303                     self.unexpected_skips.add(test_name)
 304                 expected = ["SKIP"]
 305             known_intermittent = expected[1:]
 306             expected_status = expected[0]
 307
 308             # check if we've seen a result for this test before this log line
 309             result_recorded = self.test_results.get(test_name)
 310             if result_recorded:
 311                 self.logger.warning(
 312                     "Received a second status for {}: "
 313                     "first {}, now {}".format(test_name, result_recorded, status)
 314                 )
 315             # mocha intermittently logs an additional test result after the
 316             # test has already timed out. Avoid recording this second status.
 317             if result_recorded != "TIMEOUT":
 318                 self.test_results[test_name] = status
 319                 if status not in expected:
 320                     self.has_unexpected = True
 321             self.logger.test_end(
 322                 test_name,
 323                 status=status,
 324                 expected=expected_status,
 325                 known_intermittent=known_intermittent,
 326             )
 327
 328     def new_expected(self):
 329         new_expected = OrderedDict()
 330         for test_name, status in iteritems(self.test_results):
 331             if test_name not in self.expected:
 332                 new_status = [status]
 333             else:
 334                 if status in self.expected[test_name]:
 335                     new_status = self.expected[test_name]
 336                 else:
 337                     new_status = [status]
 338             new_expected[test_name] = new_status
 339         return new_expected
 340
 341     def after_end(self, subset=False):
 342         if not subset:
 343             missing = set(self.expected) - set(self.test_results)
 344             extra = set(self.test_results) - set(self.expected)
 345             if missing:
 346                 self.has_unexpected = True
 347                 for test_name in missing:
 348                     self.logger.error("TEST-UNEXPECTED-MISSING %s" % (test_name,))
 349             if self.expected and extra:
 350                 self.has_unexpected = True
 351                 for test_name in extra:
 352                     self.logger.error(
 353                         "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name,)
 354                     )
 355
 356         if self.unexpected_skips:
 357             self.has_unexpected = True
 358             for test_name in self.unexpected_skips:
 359                 self.logger.error(
 360                     "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
 361                 )
 362         self.logger.suite_end()
 363
 364
 365 # tempfile.TemporaryDirectory missing from Python 2.7
 366 class TemporaryDirectory(object):
 367     def __init__(self):
 368         self.path = tempfile.mkdtemp()
 369         self._closed = False
 370
 371     def __repr__(self):
 372         return "<{} {!r}>".format(self.__class__.__name__, self.path)
 373
 374     def __enter__(self):
 375         return self.path
 376
 377     def __exit__(self, exc, value, tb):
 378         self.clean()
 379
 380     def __del__(self):
 381         self.clean()
 382
 383     def clean(self):
 384         if self.path and not self._closed:
 385             shutil.rmtree(self.path)
 386             self._closed = True
 387
 388
 389 class PuppeteerRunner(MozbuildObject):
 390     def __init__(self, *args, **kwargs):
 391         super(PuppeteerRunner, self).__init__(*args, **kwargs)
 392
 393         self.remotedir = os.path.join(self.topsrcdir, "remote")
 394         self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
 395
 396     def run_test(self, logger, *tests, **params):
 397         """
 398         Runs Puppeteer unit tests with npm.
 399
 400         Possible optional test parameters:
 401
 402         `binary`:
 403           Path for the browser binary to use.  Defaults to the local
 404           build.
 405         `headless`:
 406           Boolean to indicate whether to activate Firefox' headless mode.
 407         `extra_prefs`:
 408           Dictionary of extra preferences to write to the profile,
 409           before invoking npm.  Overrides default preferences.
 410         `enable_webrender`:
 411           Boolean to indicate whether to enable WebRender compositor in Gecko.
 412         `write_results`:
 413           Path to write the results json file
 414         `subset`
 415           Indicates only a subset of tests are being run, so we should
 416           skip the check for missing results
 417         """
 418         setup()
 419
 420         binary = params.get("binary") or self.get_binary_path()
 421         product = params.get("product", "firefox")
 422
 423         env = {
 424             # Print browser process ouptut
 425             "DUMPIO": "1",
 426             # Checked by Puppeteer's custom mocha config
 427             "CI": "1",
 428             # Causes some tests to be skipped due to assumptions about install
 429             "PUPPETEER_ALT_INSTALL": "1",
 430         }
 431         extra_options = {}
 432         for k, v in params.get("extra_launcher_options", {}).items():
 433             extra_options[k] = json.loads(v)
 434
 435         # Override upstream defaults: no retries, shorter timeout
 436         mocha_options = [
 437             "--reporter",
 438             "./json-mocha-reporter.js",
 439             "--retries",
 440             "0",
 441             "--fullTrace",
 442             "--timeout",
 443             "20000",
 444             "--no-parallel",
 445         ]
 446         env["HEADLESS"] = str(params.get("headless", False))
 447
 448         if product == "firefox":
 449             env["BINARY"] = binary
 450             env["PUPPETEER_PRODUCT"] = "firefox"
 451
 452             env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
 453
 454             test_command = "test:firefox"
 455         elif env["HEADLESS"] == "False":
 456             test_command = "test:chrome:headful"
 457         else:
 458             test_command = "test:chrome:headless"
 459
 460         command = ["run", test_command, "--"] + mocha_options
 461
 462         prefs = {}
 463         for k, v in params.get("extra_prefs", {}).items():
 464             print("Using extra preference: {}={}".format(k, v))
 465             prefs[k] = mozprofile.Preferences.cast(v)
 466
 467         if prefs:
 468             extra_options["extraPrefsFirefox"] = prefs
 469
 470         if extra_options:
 471             env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
 472
 473         expected_path = os.path.join(
 474             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 475         )
 476         if product == "firefox" and os.path.exists(expected_path):
 477             with open(expected_path) as f:
 478                 expected_data = json.load(f)
 479         else:
 480             expected_data = {}
 481
 482         output_handler = MochaOutputHandler(logger, expected_data)
 483         proc = npm(
 484             *command,
 485             cwd=self.puppeteer_dir,
 486             env=env,
 487             processOutputLine=output_handler,
 488             wait=False,
 489         )
 490         output_handler.proc = proc
 491
 492         # Puppeteer unit tests don't always clean-up child processes in case of
 493         # failure, so use an output_timeout as a fallback
 494         wait_proc(proc, "npm", output_timeout=60, exit_on_fail=False)
 495
 496         output_handler.after_end(params.get("subset", False))
 497
 498         # Non-zero return codes are non-fatal for now since we have some
 499         # issues with unresolved promises that shouldn't otherwise block
 500         # running the tests
 501         if proc.returncode != 0:
 502             logger.warning("npm exited with code %s" % proc.returncode)
 503
 504         if params["write_results"]:
 505             with open(params["write_results"], "w") as f:
 506                 json.dump(
 507                     output_handler.new_expected(), f, indent=2, separators=(",", ": ")
 508                 )
 509
 510         if output_handler.has_unexpected:
 511             exit(1, "Got unexpected results")
 512
 513
 514 def create_parser_puppeteer():
 515     p = argparse.ArgumentParser()
 516     p.add_argument(
 517         "--product", type=str, default="firefox", choices=["chrome", "firefox"]
 518     )
 519     p.add_argument(
 520         "--binary",
 521         type=str,
 522         help="Path to browser binary.  Defaults to local Firefox build.",
 523     )
 524     p.add_argument(
 525         "--ci",
 526         action="store_true",
 527         help="Flag that indicates that tests run in a CI environment.",
 528     )
 529     p.add_argument(
 530         "--disable-fission",
 531         action="store_true",
 532         default=False,
 533         dest="disable_fission",
 534         help="Disable Fission (site isolation) in Gecko.",
 535     )
 536     p.add_argument(
 537         "--enable-webrender",
 538         action="store_true",
 539         help="Enable the WebRender compositor in Gecko.",
 540     )
 541     p.add_argument(
 542         "-z", "--headless", action="store_true", help="Run browser in headless mode."
 543     )
 544     p.add_argument(
 545         "--setpref",
 546         action="append",
 547         dest="extra_prefs",
 548         metavar="<pref>=<value>",
 549         help="Defines additional user preferences.",
 550     )
 551     p.add_argument(
 552         "--setopt",
 553         action="append",
 554         dest="extra_options",
 555         metavar="<option>=<value>",
 556         help="Defines additional options for `puppeteer.launch`.",
 557     )
 558     p.add_argument(
 559         "-v",
 560         dest="verbosity",
 561         action="count",
 562         default=0,
 563         help="Increase remote agent logging verbosity to include "
 564         "debug level messages with -v, trace messages with -vv,"
 565         "and to not truncate long trace messages with -vvv",
 566     )
 567     p.add_argument(
 568         "--write-results",
 569         action="store",
 570         nargs="?",
 571         default=None,
 572         const=os.path.join(
 573             os.path.dirname(__file__), "test", "puppeteer-expected.json"
 574         ),
 575         help="Path to write updated results to (defaults to the "
 576         "expectations file if the argument is provided but "
 577         "no path is passed)",
 578     )
 579     p.add_argument(
 580         "--subset",
 581         action="store_true",
 582         default=False,
 583         help="Indicate that only a subset of the tests are running, "
 584         "so checks for missing tests should be skipped",
 585     )
 586     p.add_argument("tests", nargs="*")
 587     mozlog.commandline.add_logging_group(p)
 588     return p
 589
 590
 591 @Command(
 592     "puppeteer-test",
 593     category="testing",
 594     description="Run Puppeteer unit tests.",
 595     parser=create_parser_puppeteer,
 596 )
 597 @CommandArgument(
 598     "--no-install",
 599     dest="install",
 600     action="store_false",
 601     default=True,
 602     help="Do not install the Puppeteer package",
 603 )
 604 def puppeteer_test(
 605     command_context,
 606     binary=None,
 607     ci=False,
 608     disable_fission=False,
 609     enable_webrender=False,
 610     headless=False,
 611     extra_prefs=None,
 612     extra_options=None,
 613     install=False,
 614     verbosity=0,
 615     tests=None,
 616     product="firefox",
 617     write_results=None,
 618     subset=False,
 619     **kwargs,
 620 ):
 621
 622     logger = mozlog.commandline.setup_logging(
 623         "puppeteer-test", kwargs, {"mach": sys.stdout}
 624     )
 625
 626     # moztest calls this programmatically with test objects or manifests
 627     if "test_objects" in kwargs and tests is not None:
 628         logger.error("Expected either 'test_objects' or 'tests'")
 629         exit(1)
 630
 631     if product != "firefox" and extra_prefs is not None:
 632         logger.error("User preferences are not recognized by %s" % product)
 633         exit(1)
 634
 635     if "test_objects" in kwargs:
 636         tests = []
 637         for test in kwargs["test_objects"]:
 638             tests.append(test["path"])
 639
 640     prefs = {}
 641     for s in extra_prefs or []:
 642         kv = s.split("=")
 643         if len(kv) != 2:
 644             logger.error("syntax error in --setpref={}".format(s))
 645             exit(EX_USAGE)
 646         prefs[kv[0]] = kv[1].strip()
 647
 648     options = {}
 649     for s in extra_options or []:
 650         kv = s.split("=")
 651         if len(kv) != 2:
 652             logger.error("syntax error in --setopt={}".format(s))
 653             exit(EX_USAGE)
 654         options[kv[0]] = kv[1].strip()
 655
 656     prefs.update({"fission.autostart": True})
 657     if disable_fission:
 658         prefs.update({"fission.autostart": False})
 659
 660     if verbosity == 1:
 661         prefs["remote.log.level"] = "Debug"
 662     elif verbosity > 1:
 663         prefs["remote.log.level"] = "Trace"
 664     if verbosity > 2:
 665         prefs["remote.log.truncate"] = False
 666
 667     if install:
 668         install_puppeteer(command_context, product, ci)
 669
 670     params = {
 671         "binary": binary,
 672         "headless": headless,
 673         "enable_webrender": enable_webrender,
 674         "extra_prefs": prefs,
 675         "product": product,
 676         "extra_launcher_options": options,
 677         "write_results": write_results,
 678         "subset": subset,
 679     }
 680     puppeteer = command_context._spawn(PuppeteerRunner)
 681     try:
 682         return puppeteer.run_test(logger, *tests, **params)
 683     except BinaryNotFoundException as e:
 684         logger.error(e)
 685         logger.info(e.help())
 686         exit(1)
 687     except Exception as e:
 688         exit(EX_SOFTWARE, e)
 689
 690
 691 def install_puppeteer(command_context, product, ci):
 692     setup()
 693     env = {"HUSKY": "0"}
 694     from mozversioncontrol import get_repository_object
 695
 696     repo = get_repository_object(command_context.topsrcdir)
 697     puppeteer_dir = os.path.join("remote", "test", "puppeteer")
 698     changed_files = False
 699     for f in repo.get_changed_files():
 700         if f.startswith(puppeteer_dir) and f.endswith(".ts"):
 701             changed_files = True
 702             break
 703
 704     if product != "chrome":
 705         env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
 706     lib_dir = os.path.join(command_context.topsrcdir, puppeteer_dir, "lib")
 707     if changed_files and os.path.isdir(lib_dir):
 708         # clobber lib to force `tsc compile` step
 709         shutil.rmtree(lib_dir)
 710
 711     command = "ci" if ci else "install"
 712     npm(command, cwd=os.path.join(command_context.topsrcdir, puppeteer_dir), env=env)
 713     npm(
 714         "run",
 715         "build:dev",
 716         cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
 717         env=env,
 718     )
 719
 720
 721 def exit(code, error=None):
 722     if error is not None:
 723         if isinstance(error, Exception):
 724             import traceback
 725
 726             traceback.print_exc()
 727         else:
 728             message = str(error).split("\n")[0].strip()
 729             print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
 730     sys.exit(code)