Bug 1869647 - Mark hasStorageAccess.sub.https.window.html as intermittent after wpt...
[gecko.git] / remote / mach_commands.py
blobfb2923737aa85a25aa4dea554de7e46ae80a59b6
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
5 import argparse
6 import json
7 import os
8 import platform
9 import re
10 import shutil
11 import subprocess
12 import sys
13 import tempfile
14 from collections import OrderedDict
16 import mozlog
17 import mozprofile
18 from mach.decorators import Command, CommandArgument, SubCommand
19 from mozbuild import nodeutil
20 from mozbuild.base import BinaryNotFoundException, MozbuildObject
22 EX_CONFIG = 78
23 EX_SOFTWARE = 70
24 EX_USAGE = 64
27 def setup():
28 # add node and npm from mozbuild to front of system path
29 npm, _ = nodeutil.find_npm_executable()
30 if not npm:
31 exit(EX_CONFIG, "could not find npm executable")
32 path = os.path.abspath(os.path.join(npm, os.pardir))
33 os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
36 def remotedir(command_context):
37 return os.path.join(command_context.topsrcdir, "remote")
40 @Command("remote", category="misc", description="Remote protocol related operations.")
41 def remote(command_context):
42 """The remote subcommands all relate to the remote protocol."""
43 command_context._sub_mach(["help", "remote"])
44 return 1
47 @SubCommand(
48 "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
50 @CommandArgument(
51 "--repository",
52 metavar="REPO",
53 default="https://github.com/puppeteer/puppeteer.git",
54 help="The (possibly local) repository to clone from.",
56 @CommandArgument(
57 "--commitish",
58 metavar="COMMITISH",
59 required=True,
60 help="The commit or tag object name to check out.",
62 @CommandArgument(
63 "--no-install",
64 dest="install",
65 action="store_false",
66 default=True,
67 help="Do not install the just-pulled Puppeteer package,",
69 def vendor_puppeteer(command_context, repository, commitish, install):
70 puppeteer_dir = os.path.join(remotedir(command_context), "test", "puppeteer")
72 # Preserve our custom mocha reporter
73 shutil.move(
74 os.path.join(puppeteer_dir, "json-mocha-reporter.js"),
75 os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
77 shutil.rmtree(puppeteer_dir, ignore_errors=True)
78 os.makedirs(puppeteer_dir)
79 with TemporaryDirectory() as tmpdir:
80 git("clone", "-q", repository, tmpdir)
81 git("checkout", commitish, worktree=tmpdir)
82 git(
83 "checkout-index",
84 "-a",
85 "-f",
86 "--prefix",
87 "{}/".format(puppeteer_dir),
88 worktree=tmpdir,
91 # remove files which may interfere with git checkout of central
92 try:
93 os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
94 os.remove(os.path.join(puppeteer_dir, ".gitignore"))
95 except OSError:
96 pass
98 unwanted_dirs = ["experimental", "docs"]
100 for dir in unwanted_dirs:
101 dir_path = os.path.join(puppeteer_dir, dir)
102 if os.path.isdir(dir_path):
103 shutil.rmtree(dir_path)
105 shutil.move(
106 os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
107 puppeteer_dir,
110 import yaml
112 annotation = {
113 "schema": 1,
114 "bugzilla": {
115 "product": "Remote Protocol",
116 "component": "Agent",
118 "origin": {
119 "name": "puppeteer",
120 "description": "Headless Chrome Node API",
121 "url": repository,
122 "license": "Apache-2.0",
123 "release": commitish,
126 with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
127 yaml.safe_dump(
128 annotation,
130 default_flow_style=False,
131 encoding="utf-8",
132 allow_unicode=True,
135 if install:
136 env = {
137 "CI": "1", # Force the quiet logger of wireit
138 "HUSKY": "0", # Disable any hook checks
139 "PUPPETEER_SKIP_DOWNLOAD": "1", # Don't download any build
142 run_npm(
143 "install",
144 cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
145 env=env,
149 def git(*args, **kwargs):
150 cmd = ("git",)
151 if kwargs.get("worktree"):
152 cmd += ("-C", kwargs["worktree"])
153 cmd += args
155 pipe = kwargs.get("pipe")
156 git_p = subprocess.Popen(
157 cmd,
158 env={"GIT_CONFIG_NOSYSTEM": "1"},
159 stdout=subprocess.PIPE,
160 stderr=subprocess.PIPE,
162 pipe_p = None
163 if pipe:
164 pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
166 if pipe:
167 _, pipe_err = pipe_p.communicate()
168 out, git_err = git_p.communicate()
170 # use error from first program that failed
171 if git_p.returncode > 0:
172 exit(EX_SOFTWARE, git_err)
173 if pipe and pipe_p.returncode > 0:
174 exit(EX_SOFTWARE, pipe_err)
176 return out
179 def run_npm(*args, **kwargs):
180 from mozprocess import run_and_wait
182 def output_timeout_handler(proc):
183 # In some cases, we wait longer for a mocha timeout
184 print(
185 "Timed out after {} seconds of no output".format(kwargs["output_timeout"])
188 env = None
189 npm, _ = nodeutil.find_npm_executable()
190 if kwargs.get("env"):
191 env = os.environ.copy()
192 env.update(kwargs["env"])
194 proc_kwargs = {"output_timeout_handler": output_timeout_handler}
195 for kw in ["output_line_handler", "output_timeout"]:
196 if kw in kwargs:
197 proc_kwargs[kw] = kwargs[kw]
199 cmd = [npm]
200 cmd.extend(list(args))
202 p = run_and_wait(
203 args=cmd,
204 cwd=kwargs.get("cwd"),
205 env=env,
206 text=True,
207 **proc_kwargs,
209 post_wait_proc(p, cmd=npm, exit_on_fail=kwargs.get("exit_on_fail", True))
211 return p.returncode
214 def post_wait_proc(p, cmd=None, exit_on_fail=True):
215 if p.poll() is None:
216 p.kill()
217 if exit_on_fail and p.returncode > 0:
218 msg = (
219 "%s: exit code %s" % (cmd, p.returncode)
220 if cmd
221 else "exit code %s" % p.returncode
223 exit(p.returncode, msg)
226 class MochaOutputHandler(object):
227 def __init__(self, logger, expected):
228 self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
230 self.logger = logger
231 self.proc = None
232 self.test_results = OrderedDict()
233 self.expected = expected
234 self.unexpected_skips = set()
236 self.has_unexpected = False
237 self.logger.suite_start([], name="puppeteer-tests")
238 self.status_map = {
239 "CRASHED": "CRASH",
240 "OK": "PASS",
241 "TERMINATED": "CRASH",
242 "pass": "PASS",
243 "fail": "FAIL",
244 "pending": "SKIP",
247 @property
248 def pid(self):
249 return self.proc and self.proc.pid
251 def __call__(self, proc, line):
252 self.proc = proc
253 line = line.rstrip("\r\n")
254 event = None
255 try:
256 if line.startswith("[") and line.endswith("]"):
257 event = json.loads(line)
258 self.process_event(event)
259 except ValueError:
260 pass
261 finally:
262 self.logger.process_output(self.pid, line, command="npm")
264 def testExpectation(self, testIdPattern, expected_name):
265 if testIdPattern.find("*") == -1:
266 return expected_name == testIdPattern
267 else:
268 return re.compile(re.escape(testIdPattern).replace(r"\*", ".*")).search(
269 expected_name
272 def process_event(self, event):
273 if isinstance(event, list) and len(event) > 1:
274 status = self.status_map.get(event[0])
275 test_start = event[0] == "test-start"
276 if not status and not test_start:
277 return
278 test_info = event[1]
279 test_full_title = test_info.get("fullTitle", "")
280 test_name = test_full_title
281 test_path = test_info.get("file", "")
282 test_file_name = os.path.basename(test_path).replace(".js", "")
283 test_err = test_info.get("err")
284 if status == "FAIL" and test_err:
285 if "timeout" in test_err.lower():
286 status = "TIMEOUT"
287 if test_name and test_path:
288 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
289 # mocha hook failures are not tracked in metadata
290 if status != "PASS" and self.hook_re.search(test_name):
291 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
292 return
293 if test_start:
294 self.logger.test_start(test_name)
295 return
296 expected_name = "[{}] {}".format(test_file_name, test_full_title)
297 expected_item = next(
299 expectation
300 for expectation in reversed(list(self.expected))
301 if self.testExpectation(expectation["testIdPattern"], expected_name)
303 None,
305 if expected_item is None:
306 expected = ["PASS"]
307 else:
308 expected = expected_item["expectations"]
309 # mozlog doesn't really allow unexpected skip,
310 # so if a test is disabled just expect that and note the unexpected skip
311 # Also, mocha doesn't log test-start for skipped tests
312 if status == "SKIP":
313 self.logger.test_start(test_name)
314 if self.expected and status not in expected:
315 self.unexpected_skips.add(test_name)
316 expected = ["SKIP"]
317 known_intermittent = expected[1:]
318 expected_status = expected[0]
320 # check if we've seen a result for this test before this log line
321 result_recorded = self.test_results.get(test_name)
322 if result_recorded:
323 self.logger.warning(
324 "Received a second status for {}: "
325 "first {}, now {}".format(test_name, result_recorded, status)
327 # mocha intermittently logs an additional test result after the
328 # test has already timed out. Avoid recording this second status.
329 if result_recorded != "TIMEOUT":
330 self.test_results[test_name] = status
331 if status not in expected:
332 self.has_unexpected = True
333 self.logger.test_end(
334 test_name,
335 status=status,
336 expected=expected_status,
337 known_intermittent=known_intermittent,
340 def after_end(self):
341 if self.unexpected_skips:
342 self.has_unexpected = True
343 for test_name in self.unexpected_skips:
344 self.logger.error(
345 "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
347 self.logger.suite_end()
350 # tempfile.TemporaryDirectory missing from Python 2.7
351 class TemporaryDirectory(object):
352 def __init__(self):
353 self.path = tempfile.mkdtemp()
354 self._closed = False
356 def __repr__(self):
357 return "<{} {!r}>".format(self.__class__.__name__, self.path)
359 def __enter__(self):
360 return self.path
362 def __exit__(self, exc, value, tb):
363 self.clean()
365 def __del__(self):
366 self.clean()
368 def clean(self):
369 if self.path and not self._closed:
370 shutil.rmtree(self.path)
371 self._closed = True
374 class PuppeteerRunner(MozbuildObject):
375 def __init__(self, *args, **kwargs):
376 super(PuppeteerRunner, self).__init__(*args, **kwargs)
378 self.remotedir = os.path.join(self.topsrcdir, "remote")
379 self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
381 def run_test(self, logger, *tests, **params):
383 Runs Puppeteer unit tests with npm.
385 Possible optional test parameters:
387 `bidi`:
388 Boolean to indicate whether to test Firefox with BiDi protocol.
389 `binary`:
390 Path for the browser binary to use. Defaults to the local
391 build.
392 `headless`:
393 Boolean to indicate whether to activate Firefox' headless mode.
394 `extra_prefs`:
395 Dictionary of extra preferences to write to the profile,
396 before invoking npm. Overrides default preferences.
397 `enable_webrender`:
398 Boolean to indicate whether to enable WebRender compositor in Gecko.
400 setup()
402 with_bidi = params.get("bidi", False)
403 binary = params.get("binary") or self.get_binary_path()
404 product = params.get("product", "firefox")
406 env = {
407 # Print browser process ouptut
408 "DUMPIO": "1",
409 # Checked by Puppeteer's custom mocha config
410 "CI": "1",
411 # Causes some tests to be skipped due to assumptions about install
412 "PUPPETEER_ALT_INSTALL": "1",
414 extra_options = {}
415 for k, v in params.get("extra_launcher_options", {}).items():
416 extra_options[k] = json.loads(v)
418 # Override upstream defaults: no retries, shorter timeout
419 mocha_options = [
420 "--reporter",
421 "./json-mocha-reporter.js",
422 "--retries",
423 "0",
424 "--fullTrace",
425 "--timeout",
426 "20000",
427 "--no-parallel",
428 "--no-coverage",
430 env["HEADLESS"] = str(params.get("headless", False))
431 test_command = "test:" + product
433 if product == "firefox":
434 env["BINARY"] = binary
435 env["PUPPETEER_PRODUCT"] = "firefox"
436 env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
437 else:
438 env["PUPPETEER_CACHE_DIR"] = os.path.join(
439 self.topobjdir,
440 "_tests",
441 "remote",
442 "test",
443 "puppeteer",
444 ".cache",
447 if with_bidi is True:
448 test_command = test_command + ":bidi"
449 elif env["HEADLESS"] == "True":
450 test_command = test_command + ":headless"
451 else:
452 test_command = test_command + ":headful"
454 command = ["run", test_command, "--"] + mocha_options
456 prefs = {}
457 for k, v in params.get("extra_prefs", {}).items():
458 print("Using extra preference: {}={}".format(k, v))
459 prefs[k] = mozprofile.Preferences.cast(v)
461 if prefs:
462 extra_options["extraPrefsFirefox"] = prefs
464 if extra_options:
465 env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
467 expected_path = os.path.join(
468 os.path.dirname(__file__),
469 "test",
470 "puppeteer",
471 "test",
472 "TestExpectations.json",
474 if os.path.exists(expected_path):
475 with open(expected_path) as f:
476 expected_data = json.load(f)
477 else:
478 expected_data = []
480 expected_platform = platform.uname().system.lower()
481 if expected_platform == "windows":
482 expected_platform = "win32"
484 # Filter expectation data for the selected browser,
485 # headless or headful mode, the operating system,
486 # run in BiDi mode or not.
487 expectations = [
488 expectation
489 for expectation in expected_data
490 if is_relevant_expectation(
491 expectation, product, with_bidi, env["HEADLESS"], expected_platform
495 output_handler = MochaOutputHandler(logger, expectations)
496 run_npm(
497 *command,
498 cwd=self.puppeteer_dir,
499 env=env,
500 output_line_handler=output_handler,
501 # Puppeteer unit tests don't always clean-up child processes in case of
502 # failure, so use an output_timeout as a fallback
503 output_timeout=60,
504 exit_on_fail=True,
507 output_handler.after_end()
509 if output_handler.has_unexpected:
510 logger.error("Got unexpected results")
511 exit(1)
514 def create_parser_puppeteer():
515 p = argparse.ArgumentParser()
516 p.add_argument(
517 "--product", type=str, default="firefox", choices=["chrome", "firefox"]
519 p.add_argument(
520 "--bidi",
521 action="store_true",
522 help="Flag that indicates whether to test Firefox with BiDi protocol.",
524 p.add_argument(
525 "--binary",
526 type=str,
527 help="Path to browser binary. Defaults to local Firefox build.",
529 p.add_argument(
530 "--ci",
531 action="store_true",
532 help="Flag that indicates that tests run in a CI environment.",
534 p.add_argument(
535 "--disable-fission",
536 action="store_true",
537 default=False,
538 dest="disable_fission",
539 help="Disable Fission (site isolation) in Gecko.",
541 p.add_argument(
542 "--enable-webrender",
543 action="store_true",
544 help="Enable the WebRender compositor in Gecko.",
546 p.add_argument(
547 "-z", "--headless", action="store_true", help="Run browser in headless mode."
549 p.add_argument(
550 "--setpref",
551 action="append",
552 dest="extra_prefs",
553 metavar="<pref>=<value>",
554 help="Defines additional user preferences.",
556 p.add_argument(
557 "--setopt",
558 action="append",
559 dest="extra_options",
560 metavar="<option>=<value>",
561 help="Defines additional options for `puppeteer.launch`.",
563 p.add_argument(
564 "-v",
565 dest="verbosity",
566 action="count",
567 default=0,
568 help="Increase remote agent logging verbosity to include "
569 "debug level messages with -v, trace messages with -vv,"
570 "and to not truncate long trace messages with -vvv",
572 p.add_argument("tests", nargs="*")
573 mozlog.commandline.add_logging_group(p)
574 return p
577 def is_relevant_expectation(
578 expectation, expected_product, with_bidi, is_headless, expected_platform
580 parameters = expectation["parameters"]
582 if expected_product == "firefox":
583 is_expected_product = "chrome" not in parameters
584 else:
585 is_expected_product = "firefox" not in parameters
587 if with_bidi is True:
588 is_expected_protocol = "cdp" not in parameters
589 is_headless = "True"
590 else:
591 is_expected_protocol = "webDriverBiDi" not in parameters
593 if is_headless == "True":
594 is_expected_mode = "headful" not in parameters
595 else:
596 is_expected_mode = "headless" not in parameters
598 is_expected_platform = expected_platform in expectation["platforms"]
600 return (
601 is_expected_product
602 and is_expected_protocol
603 and is_expected_mode
604 and is_expected_platform
608 @Command(
609 "puppeteer-test",
610 category="testing",
611 description="Run Puppeteer unit tests.",
612 parser=create_parser_puppeteer,
614 @CommandArgument(
615 "--no-install",
616 dest="install",
617 action="store_false",
618 default=True,
619 help="Do not install the Puppeteer package",
621 def puppeteer_test(
622 command_context,
623 bidi=None,
624 binary=None,
625 ci=False,
626 disable_fission=False,
627 enable_webrender=False,
628 headless=False,
629 extra_prefs=None,
630 extra_options=None,
631 install=False,
632 verbosity=0,
633 tests=None,
634 product="firefox",
635 **kwargs,
637 logger = mozlog.commandline.setup_logging(
638 "puppeteer-test", kwargs, {"mach": sys.stdout}
641 # moztest calls this programmatically with test objects or manifests
642 if "test_objects" in kwargs and tests is not None:
643 logger.error("Expected either 'test_objects' or 'tests'")
644 exit(1)
646 if product != "firefox" and extra_prefs is not None:
647 logger.error("User preferences are not recognized by %s" % product)
648 exit(1)
650 if "test_objects" in kwargs:
651 tests = []
652 for test in kwargs["test_objects"]:
653 tests.append(test["path"])
655 prefs = {}
656 for s in extra_prefs or []:
657 kv = s.split("=")
658 if len(kv) != 2:
659 logger.error("syntax error in --setpref={}".format(s))
660 exit(EX_USAGE)
661 prefs[kv[0]] = kv[1].strip()
663 options = {}
664 for s in extra_options or []:
665 kv = s.split("=")
666 if len(kv) != 2:
667 logger.error("syntax error in --setopt={}".format(s))
668 exit(EX_USAGE)
669 options[kv[0]] = kv[1].strip()
671 prefs.update({"fission.autostart": True})
672 if disable_fission:
673 prefs.update({"fission.autostart": False})
675 if verbosity == 1:
676 prefs["remote.log.level"] = "Debug"
677 elif verbosity > 1:
678 prefs["remote.log.level"] = "Trace"
679 if verbosity > 2:
680 prefs["remote.log.truncate"] = False
682 if install:
683 install_puppeteer(command_context, product, ci)
685 params = {
686 "bidi": bidi,
687 "binary": binary,
688 "headless": headless,
689 "enable_webrender": enable_webrender,
690 "extra_prefs": prefs,
691 "product": product,
692 "extra_launcher_options": options,
694 puppeteer = command_context._spawn(PuppeteerRunner)
695 try:
696 return puppeteer.run_test(logger, *tests, **params)
697 except BinaryNotFoundException as e:
698 logger.error(e)
699 logger.info(e.help())
700 exit(1)
701 except Exception as e:
702 exit(EX_SOFTWARE, e)
705 def install_puppeteer(command_context, product, ci):
706 setup()
708 env = {
709 "CI": "1", # Force the quiet logger of wireit
710 "HUSKY": "0", # Disable any hook checks
713 puppeteer_dir = os.path.join("remote", "test", "puppeteer")
714 puppeteer_dir_full_path = os.path.join(command_context.topsrcdir, puppeteer_dir)
715 puppeteer_test_dir = os.path.join(puppeteer_dir, "test")
717 if product == "chrome":
718 env["PUPPETEER_CACHE_DIR"] = os.path.join(
719 command_context.topobjdir, "_tests", puppeteer_dir, ".cache"
721 else:
722 env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
724 if not ci:
725 run_npm(
726 "run",
727 "clean",
728 cwd=puppeteer_dir_full_path,
729 env=env,
730 exit_on_fail=False,
733 # Always use the `ci` command to not get updated sub-dependencies installed.
734 run_npm("ci", cwd=puppeteer_dir_full_path, env=env)
735 run_npm(
736 "run",
737 "build",
738 cwd=os.path.join(command_context.topsrcdir, puppeteer_test_dir),
739 env=env,
743 def exit(code, error=None):
744 if error is not None:
745 if isinstance(error, Exception):
746 import traceback
748 traceback.print_exc()
749 else:
750 message = str(error).split("\n")[0].strip()
751 print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
752 sys.exit(code)