no bug - Bumping Firefox l10n changesets r=release a=l10n-bump DONTBUILD CLOSED TREE
[gecko.git] / remote / mach_commands.py
blob797f313ddabb528803caf23d42251f9f019a79e0
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
5 import argparse
6 import json
7 import os
8 import platform
9 import re
10 import shutil
11 import subprocess
12 import sys
13 import tempfile
14 from collections import OrderedDict
16 import mozlog
17 import mozprofile
18 from mach.decorators import Command, CommandArgument, SubCommand
19 from mozbuild import nodeutil
20 from mozbuild.base import BinaryNotFoundException, MozbuildObject
22 EX_CONFIG = 78
23 EX_SOFTWARE = 70
24 EX_USAGE = 64
27 def setup():
28 # add node and npm from mozbuild to front of system path
29 npm, _ = nodeutil.find_npm_executable()
30 if not npm:
31 exit(EX_CONFIG, "could not find npm executable")
32 path = os.path.abspath(os.path.join(npm, os.pardir))
33 os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
36 def remotedir(command_context):
37 return os.path.join(command_context.topsrcdir, "remote")
40 @Command("remote", category="misc", description="Remote protocol related operations.")
41 def remote(command_context):
42 """The remote subcommands all relate to the remote protocol."""
43 command_context._sub_mach(["help", "remote"])
44 return 1
47 @SubCommand(
48 "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
50 @CommandArgument(
51 "--repository",
52 metavar="REPO",
53 default="https://github.com/puppeteer/puppeteer.git",
54 help="The (possibly local) repository to clone from.",
56 @CommandArgument(
57 "--commitish",
58 metavar="COMMITISH",
59 required=True,
60 help="The commit or tag object name to check out.",
62 @CommandArgument(
63 "--no-install",
64 dest="install",
65 action="store_false",
66 default=True,
67 help="Do not install the just-pulled Puppeteer package,",
69 def vendor_puppeteer(command_context, repository, commitish, install):
70 puppeteer_dir = os.path.join(remotedir(command_context), "test", "puppeteer")
72 # Preserve our custom mocha reporter
73 shutil.move(
74 os.path.join(puppeteer_dir, "json-mocha-reporter.js"),
75 os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
77 shutil.rmtree(puppeteer_dir, ignore_errors=True)
78 os.makedirs(puppeteer_dir)
79 with TemporaryDirectory() as tmpdir:
80 git("clone", "-q", repository, tmpdir)
81 git("checkout", commitish, worktree=tmpdir)
82 git(
83 "checkout-index",
84 "-a",
85 "-f",
86 "--prefix",
87 "{}/".format(puppeteer_dir),
88 worktree=tmpdir,
91 # remove files which may interfere with git checkout of central
92 try:
93 os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
94 os.remove(os.path.join(puppeteer_dir, ".gitignore"))
95 except OSError:
96 pass
98 unwanted_dirs = ["experimental", "docs"]
100 for dir in unwanted_dirs:
101 dir_path = os.path.join(puppeteer_dir, dir)
102 if os.path.isdir(dir_path):
103 shutil.rmtree(dir_path)
105 shutil.move(
106 os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
107 puppeteer_dir,
110 import yaml
112 annotation = {
113 "schema": 1,
114 "bugzilla": {
115 "product": "Remote Protocol",
116 "component": "Agent",
118 "origin": {
119 "name": "puppeteer",
120 "description": "Headless Chrome Node API",
121 "url": repository,
122 "license": "Apache-2.0",
123 "release": commitish,
126 with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
127 yaml.safe_dump(
128 annotation,
130 default_flow_style=False,
131 encoding="utf-8",
132 allow_unicode=True,
135 if install:
136 env = {
137 "CI": "1", # Force the quiet logger of wireit
138 "HUSKY": "0", # Disable any hook checks
139 "PUPPETEER_SKIP_DOWNLOAD": "1", # Don't download any build
142 run_npm(
143 "run",
144 "clean",
145 cwd=puppeteer_dir,
146 env=env,
147 exit_on_fail=False,
150 run_npm(
151 "install",
152 cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
153 env=env,
157 def git(*args, **kwargs):
158 cmd = ("git",)
159 if kwargs.get("worktree"):
160 cmd += ("-C", kwargs["worktree"])
161 cmd += args
163 pipe = kwargs.get("pipe")
164 git_p = subprocess.Popen(
165 cmd,
166 env={"GIT_CONFIG_NOSYSTEM": "1"},
167 stdout=subprocess.PIPE,
168 stderr=subprocess.PIPE,
170 pipe_p = None
171 if pipe:
172 pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
174 if pipe:
175 _, pipe_err = pipe_p.communicate()
176 out, git_err = git_p.communicate()
178 # use error from first program that failed
179 if git_p.returncode > 0:
180 exit(EX_SOFTWARE, git_err)
181 if pipe and pipe_p.returncode > 0:
182 exit(EX_SOFTWARE, pipe_err)
184 return out
187 def run_npm(*args, **kwargs):
188 from mozprocess import run_and_wait
190 def output_timeout_handler(proc):
191 # In some cases, we wait longer for a mocha timeout
192 print(
193 "Timed out after {} seconds of no output".format(kwargs["output_timeout"])
196 env = os.environ.copy()
197 npm, _ = nodeutil.find_npm_executable()
198 if kwargs.get("env"):
199 env.update(kwargs["env"])
201 proc_kwargs = {"output_timeout_handler": output_timeout_handler}
202 for kw in ["output_line_handler", "output_timeout"]:
203 if kw in kwargs:
204 proc_kwargs[kw] = kwargs[kw]
206 cmd = [npm]
207 cmd.extend(list(args))
209 p = run_and_wait(
210 args=cmd,
211 cwd=kwargs.get("cwd"),
212 env=env,
213 text=True,
214 **proc_kwargs,
216 post_wait_proc(p, cmd=npm, exit_on_fail=kwargs.get("exit_on_fail", True))
218 return p.returncode
221 def post_wait_proc(p, cmd=None, exit_on_fail=True):
222 if p.poll() is None:
223 p.kill()
224 if exit_on_fail and p.returncode > 0:
225 msg = (
226 "%s: exit code %s" % (cmd, p.returncode)
227 if cmd
228 else "exit code %s" % p.returncode
230 exit(p.returncode, msg)
233 class MochaOutputHandler(object):
234 def __init__(self, logger, expected):
235 self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
237 self.logger = logger
238 self.proc = None
239 self.test_results = OrderedDict()
240 self.expected = expected
241 self.unexpected_skips = set()
243 self.has_unexpected = False
244 self.logger.suite_start([], name="puppeteer-tests")
245 self.status_map = {
246 "CRASHED": "CRASH",
247 "OK": "PASS",
248 "TERMINATED": "CRASH",
249 "pass": "PASS",
250 "fail": "FAIL",
251 "pending": "SKIP",
254 @property
255 def pid(self):
256 return self.proc and self.proc.pid
258 def __call__(self, proc, line):
259 self.proc = proc
260 line = line.rstrip("\r\n")
261 event = None
262 try:
263 if line.startswith("[") and line.endswith("]"):
264 event = json.loads(line)
265 self.process_event(event)
266 except ValueError:
267 pass
268 finally:
269 self.logger.process_output(self.pid, line, command="npm")
271 def testExpectation(self, testIdPattern, expected_name):
272 if testIdPattern.find("*") == -1:
273 return expected_name == testIdPattern
274 else:
275 return re.compile(re.escape(testIdPattern).replace(r"\*", ".*")).search(
276 expected_name
279 def process_event(self, event):
280 if isinstance(event, list) and len(event) > 1:
281 status = self.status_map.get(event[0])
282 test_start = event[0] == "test-start"
283 if not status and not test_start:
284 return
285 test_info = event[1]
286 test_full_title = test_info.get("fullTitle", "")
287 test_name = test_full_title
288 test_path = test_info.get("file", "")
289 test_file_name = os.path.basename(test_path).replace(".js", "")
290 test_err = test_info.get("err")
291 if status == "FAIL" and test_err:
292 if "timeout" in test_err.lower():
293 status = "TIMEOUT"
294 if test_name and test_path:
295 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
296 # mocha hook failures are not tracked in metadata
297 if status != "PASS" and self.hook_re.search(test_name):
298 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
299 return
300 if test_start:
301 self.logger.test_start(test_name)
302 return
303 expected_name = "[{}] {}".format(test_file_name, test_full_title)
304 expected_item = next(
306 expectation
307 for expectation in reversed(list(self.expected))
308 if self.testExpectation(expectation["testIdPattern"], expected_name)
310 None,
312 if expected_item is None:
313 expected = ["PASS"]
314 else:
315 expected = expected_item["expectations"]
316 # mozlog doesn't really allow unexpected skip,
317 # so if a test is disabled just expect that and note the unexpected skip
318 # Also, mocha doesn't log test-start for skipped tests
319 if status == "SKIP":
320 self.logger.test_start(test_name)
321 if self.expected and status not in expected:
322 self.unexpected_skips.add(test_name)
323 expected = ["SKIP"]
324 known_intermittent = expected[1:]
325 expected_status = expected[0]
327 # check if we've seen a result for this test before this log line
328 result_recorded = self.test_results.get(test_name)
329 if result_recorded:
330 self.logger.warning(
331 "Received a second status for {}: "
332 "first {}, now {}".format(test_name, result_recorded, status)
334 # mocha intermittently logs an additional test result after the
335 # test has already timed out. Avoid recording this second status.
336 if result_recorded != "TIMEOUT":
337 self.test_results[test_name] = status
338 if status not in expected:
339 self.has_unexpected = True
340 self.logger.test_end(
341 test_name,
342 status=status,
343 expected=expected_status,
344 known_intermittent=known_intermittent,
347 def after_end(self):
348 if self.unexpected_skips:
349 self.has_unexpected = True
350 for test_name in self.unexpected_skips:
351 self.logger.error(
352 "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
354 self.logger.suite_end()
357 # tempfile.TemporaryDirectory missing from Python 2.7
358 class TemporaryDirectory(object):
359 def __init__(self):
360 self.path = tempfile.mkdtemp()
361 self._closed = False
363 def __repr__(self):
364 return "<{} {!r}>".format(self.__class__.__name__, self.path)
366 def __enter__(self):
367 return self.path
369 def __exit__(self, exc, value, tb):
370 self.clean()
372 def __del__(self):
373 self.clean()
375 def clean(self):
376 if self.path and not self._closed:
377 shutil.rmtree(self.path)
378 self._closed = True
381 class PuppeteerRunner(MozbuildObject):
382 def __init__(self, *args, **kwargs):
383 super(PuppeteerRunner, self).__init__(*args, **kwargs)
385 self.remotedir = os.path.join(self.topsrcdir, "remote")
386 self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
388 def run_test(self, logger, *tests, **params):
390 Runs Puppeteer unit tests with npm.
392 Possible optional test parameters:
394 `binary`:
395 Path for the browser binary to use. Defaults to the local
396 build.
397 `cdp`:
398 Boolean to indicate whether to test Firefox with CDP protocol.
399 `headless`:
400 Boolean to indicate whether to activate Firefox' headless mode.
401 `extra_prefs`:
402 Dictionary of extra preferences to write to the profile,
403 before invoking npm. Overrides default preferences.
404 `enable_webrender`:
405 Boolean to indicate whether to enable WebRender compositor in Gecko.
407 setup()
409 binary = params.get("binary")
410 headless = params.get("headless", False)
411 product = params.get("product", "firefox")
412 with_cdp = params.get("cdp", False)
414 extra_options = {}
415 for k, v in params.get("extra_launcher_options", {}).items():
416 extra_options[k] = json.loads(v)
418 # Override upstream defaults: no retries, shorter timeout
419 mocha_options = [
420 "--reporter",
421 "./json-mocha-reporter.js",
422 "--retries",
423 "0",
424 "--fullTrace",
425 "--timeout",
426 "20000",
427 "--no-parallel",
428 "--no-coverage",
431 env = {
432 # Checked by Puppeteer's custom mocha config
433 "CI": "1",
434 # Print browser process ouptut
435 "DUMPIO": "1",
436 # Run in headless mode if trueish, otherwise use headful
437 "HEADLESS": str(headless),
438 # Causes some tests to be skipped due to assumptions about install
439 "PUPPETEER_ALT_INSTALL": "1",
442 if product == "firefox":
443 env["BINARY"] = binary or self.get_binary_path()
444 env["PUPPETEER_PRODUCT"] = "firefox"
445 env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
446 else:
447 if binary:
448 env["BINARY"] = binary
449 env["PUPPETEER_CACHE_DIR"] = os.path.join(
450 self.topobjdir,
451 "_tests",
452 "remote",
453 "test",
454 "puppeteer",
455 ".cache",
458 test_command = "test:" + product
460 if with_cdp:
461 if headless:
462 test_command = test_command + ":headless"
463 else:
464 test_command = test_command + ":headful"
465 else:
466 if headless:
467 test_command = test_command + ":bidi"
468 else:
469 if product == "chrome":
470 raise Exception(
471 "Chrome doesn't support headful mode with the WebDriver BiDi protocol"
474 test_command = test_command + ":bidi:headful"
476 command = ["run", test_command, "--"] + mocha_options
478 prefs = {}
479 for k, v in params.get("extra_prefs", {}).items():
480 print("Using extra preference: {}={}".format(k, v))
481 prefs[k] = mozprofile.Preferences.cast(v)
483 if prefs:
484 extra_options["extraPrefsFirefox"] = prefs
486 if extra_options:
487 env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
489 expected_path = os.path.join(
490 os.path.dirname(__file__),
491 "test",
492 "puppeteer",
493 "test",
494 "TestExpectations.json",
496 if os.path.exists(expected_path):
497 with open(expected_path) as f:
498 expected_data = json.load(f)
499 else:
500 expected_data = []
502 expected_platform = platform.uname().system.lower()
503 if expected_platform == "windows":
504 expected_platform = "win32"
506 # Filter expectation data for the selected browser,
507 # headless or headful mode, the operating system,
508 # run in BiDi mode or not.
509 expectations = [
510 expectation
511 for expectation in expected_data
512 if is_relevant_expectation(
513 expectation, product, with_cdp, env["HEADLESS"], expected_platform
517 output_handler = MochaOutputHandler(logger, expectations)
518 run_npm(
519 *command,
520 cwd=self.puppeteer_dir,
521 env=env,
522 output_line_handler=output_handler,
523 # Puppeteer unit tests don't always clean-up child processes in case of
524 # failure, so use an output_timeout as a fallback
525 output_timeout=60,
526 exit_on_fail=True,
529 output_handler.after_end()
531 if output_handler.has_unexpected:
532 logger.error("Got unexpected results")
533 exit(1)
536 def create_parser_puppeteer():
537 p = argparse.ArgumentParser()
538 p.add_argument(
539 "--product", type=str, default="firefox", choices=["chrome", "firefox"]
541 p.add_argument(
542 "--binary",
543 type=str,
544 help="Path to browser binary. Defaults to local Firefox build.",
546 p.add_argument(
547 "--cdp",
548 action="store_true",
549 help="Flag that indicates whether to test Firefox with the CDP protocol.",
551 p.add_argument(
552 "--ci",
553 action="store_true",
554 help="Flag that indicates that tests run in a CI environment.",
556 p.add_argument(
557 "--disable-fission",
558 action="store_true",
559 default=False,
560 dest="disable_fission",
561 help="Disable Fission (site isolation) in Gecko.",
563 p.add_argument(
564 "--enable-webrender",
565 action="store_true",
566 help="Enable the WebRender compositor in Gecko.",
568 p.add_argument(
569 "-z", "--headless", action="store_true", help="Run browser in headless mode."
571 p.add_argument(
572 "--setpref",
573 action="append",
574 dest="extra_prefs",
575 metavar="<pref>=<value>",
576 help="Defines additional user preferences.",
578 p.add_argument(
579 "--setopt",
580 action="append",
581 dest="extra_options",
582 metavar="<option>=<value>",
583 help="Defines additional options for `puppeteer.launch`.",
585 p.add_argument(
586 "-v",
587 dest="verbosity",
588 action="count",
589 default=0,
590 help="Increase remote agent logging verbosity to include "
591 "debug level messages with -v, trace messages with -vv,"
592 "and to not truncate long trace messages with -vvv",
594 p.add_argument("tests", nargs="*")
595 mozlog.commandline.add_logging_group(p)
596 return p
599 def is_relevant_expectation(
600 expectation, expected_product, with_cdp, is_headless, expected_platform
602 parameters = expectation["parameters"]
604 if expected_product == "firefox":
605 is_expected_product = "chrome" not in parameters
606 else:
607 is_expected_product = "firefox" not in parameters
609 if with_cdp:
610 is_expected_protocol = "webDriverBiDi" not in parameters
611 else:
612 is_expected_protocol = "cdp" not in parameters
613 is_headless = "True"
615 if is_headless == "True":
616 is_expected_mode = "headful" not in parameters
617 else:
618 is_expected_mode = "headless" not in parameters
620 is_expected_platform = expected_platform in expectation["platforms"]
622 return (
623 is_expected_product
624 and is_expected_protocol
625 and is_expected_mode
626 and is_expected_platform
630 @Command(
631 "puppeteer-test",
632 category="testing",
633 description="Run Puppeteer unit tests.",
634 parser=create_parser_puppeteer,
636 @CommandArgument(
637 "--no-install",
638 dest="install",
639 action="store_false",
640 default=True,
641 help="Do not install the Puppeteer package",
643 def puppeteer_test(
644 command_context,
645 binary=None,
646 cdp=False,
647 ci=False,
648 disable_fission=False,
649 enable_webrender=False,
650 headless=False,
651 extra_prefs=None,
652 extra_options=None,
653 install=False,
654 verbosity=0,
655 tests=None,
656 product="firefox",
657 **kwargs,
659 logger = mozlog.commandline.setup_logging(
660 "puppeteer-test", kwargs, {"mach": sys.stdout}
663 # moztest calls this programmatically with test objects or manifests
664 if "test_objects" in kwargs and tests is not None:
665 logger.error("Expected either 'test_objects' or 'tests'")
666 exit(1)
668 if product != "firefox" and extra_prefs is not None:
669 logger.error("User preferences are not recognized by %s" % product)
670 exit(1)
672 if "test_objects" in kwargs:
673 tests = []
674 for test in kwargs["test_objects"]:
675 tests.append(test["path"])
677 prefs = {}
678 for s in extra_prefs or []:
679 kv = s.split("=")
680 if len(kv) != 2:
681 logger.error("syntax error in --setpref={}".format(s))
682 exit(EX_USAGE)
683 prefs[kv[0]] = kv[1].strip()
685 options = {}
686 for s in extra_options or []:
687 kv = s.split("=")
688 if len(kv) != 2:
689 logger.error("syntax error in --setopt={}".format(s))
690 exit(EX_USAGE)
691 options[kv[0]] = kv[1].strip()
693 prefs.update({"fission.autostart": True})
694 if disable_fission:
695 prefs.update({"fission.autostart": False})
697 if verbosity == 1:
698 prefs["remote.log.level"] = "Debug"
699 elif verbosity > 1:
700 prefs["remote.log.level"] = "Trace"
701 if verbosity > 2:
702 prefs["remote.log.truncate"] = False
704 if install:
705 install_puppeteer(command_context, product, ci)
707 params = {
708 "binary": binary,
709 "cdp": cdp,
710 "headless": headless,
711 "enable_webrender": enable_webrender,
712 "extra_prefs": prefs,
713 "product": product,
714 "extra_launcher_options": options,
716 puppeteer = command_context._spawn(PuppeteerRunner)
717 try:
718 return puppeteer.run_test(logger, *tests, **params)
719 except BinaryNotFoundException as e:
720 logger.error(e)
721 logger.info(e.help())
722 exit(1)
723 except Exception as e:
724 exit(EX_SOFTWARE, e)
727 def install_puppeteer(command_context, product, ci):
728 setup()
730 env = {
731 "CI": "1", # Force the quiet logger of wireit
732 "HUSKY": "0", # Disable any hook checks
735 puppeteer_dir = os.path.join("remote", "test", "puppeteer")
736 puppeteer_dir_full_path = os.path.join(command_context.topsrcdir, puppeteer_dir)
737 puppeteer_test_dir = os.path.join(puppeteer_dir, "test")
739 if product == "chrome":
740 env["PUPPETEER_PRODUCT"] = "chrome"
741 env["PUPPETEER_CACHE_DIR"] = os.path.join(
742 command_context.topobjdir, "_tests", puppeteer_dir, ".cache"
744 else:
745 env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
747 if not ci:
748 run_npm(
749 "run",
750 "clean",
751 cwd=puppeteer_dir_full_path,
752 env=env,
753 exit_on_fail=False,
756 # Always use the `ci` command to not get updated sub-dependencies installed.
757 run_npm("ci", cwd=puppeteer_dir_full_path, env=env)
759 # Build Puppeteer and the code to download browsers.
760 run_npm(
761 "run",
762 "build",
763 cwd=os.path.join(command_context.topsrcdir, puppeteer_test_dir),
764 env=env,
767 # Run post install steps, including downloading the Chrome browser if requested
768 run_npm("run", "postinstall", cwd=puppeteer_dir_full_path, env=env)
771 def exit(code, error=None):
772 if error is not None:
773 if isinstance(error, Exception):
774 import traceback
776 traceback.print_exc()
777 else:
778 message = str(error).split("\n")[0].strip()
779 print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
780 sys.exit(code)