Backed out 2 changesets (bug 1855992) for causing talos failures @ mozilla::net:...
[gecko.git] / remote / mach_commands.py
blobf5b31a449d27f38b5cdf7b63d191c528d39abd18
1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
5 import argparse
6 import json
7 import os
8 import platform
9 import re
10 import shutil
11 import subprocess
12 import sys
13 import tempfile
14 from collections import OrderedDict
16 import mozlog
17 import mozprofile
18 from mach.decorators import Command, CommandArgument, SubCommand
19 from mozbuild import nodeutil
20 from mozbuild.base import BinaryNotFoundException, MozbuildObject
22 EX_CONFIG = 78
23 EX_SOFTWARE = 70
24 EX_USAGE = 64
27 def setup():
28 # add node and npm from mozbuild to front of system path
29 npm, _ = nodeutil.find_npm_executable()
30 if not npm:
31 exit(EX_CONFIG, "could not find npm executable")
32 path = os.path.abspath(os.path.join(npm, os.pardir))
33 os.environ["PATH"] = "{}{}{}".format(path, os.pathsep, os.environ["PATH"])
36 def remotedir(command_context):
37 return os.path.join(command_context.topsrcdir, "remote")
40 @Command("remote", category="misc", description="Remote protocol related operations.")
41 def remote(command_context):
42 """The remote subcommands all relate to the remote protocol."""
43 command_context._sub_mach(["help", "remote"])
44 return 1
47 @SubCommand(
48 "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
50 @CommandArgument(
51 "--repository",
52 metavar="REPO",
53 default="https://github.com/puppeteer/puppeteer.git",
54 help="The (possibly local) repository to clone from.",
56 @CommandArgument(
57 "--commitish",
58 metavar="COMMITISH",
59 required=True,
60 help="The commit or tag object name to check out.",
62 @CommandArgument(
63 "--no-install",
64 dest="install",
65 action="store_false",
66 default=True,
67 help="Do not install the just-pulled Puppeteer package,",
69 def vendor_puppeteer(command_context, repository, commitish, install):
70 puppeteer_dir = os.path.join(remotedir(command_context), "test", "puppeteer")
72 # Preserve our custom mocha reporter
73 shutil.move(
74 os.path.join(puppeteer_dir, "json-mocha-reporter.js"),
75 remotedir(command_context),
77 shutil.rmtree(puppeteer_dir, ignore_errors=True)
78 os.makedirs(puppeteer_dir)
79 with TemporaryDirectory() as tmpdir:
80 git("clone", "-q", repository, tmpdir)
81 git("checkout", commitish, worktree=tmpdir)
82 git(
83 "checkout-index",
84 "-a",
85 "-f",
86 "--prefix",
87 "{}/".format(puppeteer_dir),
88 worktree=tmpdir,
91 # remove files which may interfere with git checkout of central
92 try:
93 os.remove(os.path.join(puppeteer_dir, ".gitattributes"))
94 os.remove(os.path.join(puppeteer_dir, ".gitignore"))
95 except OSError:
96 pass
98 unwanted_dirs = ["experimental", "docs"]
100 for dir in unwanted_dirs:
101 dir_path = os.path.join(puppeteer_dir, dir)
102 if os.path.isdir(dir_path):
103 shutil.rmtree(dir_path)
105 shutil.move(
106 os.path.join(remotedir(command_context), "json-mocha-reporter.js"),
107 puppeteer_dir,
110 import yaml
112 annotation = {
113 "schema": 1,
114 "bugzilla": {
115 "product": "Remote Protocol",
116 "component": "Agent",
118 "origin": {
119 "name": "puppeteer",
120 "description": "Headless Chrome Node API",
121 "url": repository,
122 "license": "Apache-2.0",
123 "release": commitish,
126 with open(os.path.join(puppeteer_dir, "moz.yaml"), "w") as fh:
127 yaml.safe_dump(
128 annotation,
130 default_flow_style=False,
131 encoding="utf-8",
132 allow_unicode=True,
135 if install:
136 env = {"HUSKY": "0", "PUPPETEER_SKIP_DOWNLOAD": "1"}
137 run_npm(
138 "install",
139 cwd=os.path.join(command_context.topsrcdir, puppeteer_dir),
140 env=env,
144 def git(*args, **kwargs):
145 cmd = ("git",)
146 if kwargs.get("worktree"):
147 cmd += ("-C", kwargs["worktree"])
148 cmd += args
150 pipe = kwargs.get("pipe")
151 git_p = subprocess.Popen(
152 cmd,
153 env={"GIT_CONFIG_NOSYSTEM": "1"},
154 stdout=subprocess.PIPE,
155 stderr=subprocess.PIPE,
157 pipe_p = None
158 if pipe:
159 pipe_p = subprocess.Popen(pipe, stdin=git_p.stdout, stderr=subprocess.PIPE)
161 if pipe:
162 _, pipe_err = pipe_p.communicate()
163 out, git_err = git_p.communicate()
165 # use error from first program that failed
166 if git_p.returncode > 0:
167 exit(EX_SOFTWARE, git_err)
168 if pipe and pipe_p.returncode > 0:
169 exit(EX_SOFTWARE, pipe_err)
171 return out
174 def run_npm(*args, **kwargs):
175 from mozprocess import run_and_wait
177 def output_timeout_handler(proc):
178 # In some cases, we wait longer for a mocha timeout
179 print(
180 "Timed out after {} seconds of no output".format(kwargs["output_timeout"])
183 env = None
184 npm, _ = nodeutil.find_npm_executable()
185 if kwargs.get("env"):
186 env = os.environ.copy()
187 env.update(kwargs["env"])
189 proc_kwargs = {"output_timeout_handler": output_timeout_handler}
190 for kw in ["output_line_handler", "output_timeout"]:
191 if kw in kwargs:
192 proc_kwargs[kw] = kwargs[kw]
194 cmd = [npm]
195 cmd.extend(list(args))
197 p = run_and_wait(
198 args=cmd,
199 cwd=kwargs.get("cwd"),
200 env=env,
201 text=True,
202 **proc_kwargs,
204 post_wait_proc(p, cmd=npm, exit_on_fail=kwargs.get("exit_on_fail", True))
206 return p.returncode
209 def post_wait_proc(p, cmd=None, exit_on_fail=True):
210 if p.poll() is None:
211 p.kill()
212 if exit_on_fail and p.returncode > 0:
213 msg = (
214 "%s: exit code %s" % (cmd, p.returncode)
215 if cmd
216 else "exit code %s" % p.returncode
218 exit(p.returncode, msg)
221 class MochaOutputHandler(object):
222 def __init__(self, logger, expected):
223 self.hook_re = re.compile('"before\b?.*" hook|"after\b?.*" hook')
225 self.logger = logger
226 self.proc = None
227 self.test_results = OrderedDict()
228 self.expected = expected
229 self.unexpected_skips = set()
231 self.has_unexpected = False
232 self.logger.suite_start([], name="puppeteer-tests")
233 self.status_map = {
234 "CRASHED": "CRASH",
235 "OK": "PASS",
236 "TERMINATED": "CRASH",
237 "pass": "PASS",
238 "fail": "FAIL",
239 "pending": "SKIP",
242 @property
243 def pid(self):
244 return self.proc and self.proc.pid
246 def __call__(self, proc, line):
247 self.proc = proc
248 line = line.rstrip("\r\n")
249 event = None
250 try:
251 if line.startswith("[") and line.endswith("]"):
252 event = json.loads(line)
253 self.process_event(event)
254 except ValueError:
255 pass
256 finally:
257 self.logger.process_output(self.pid, line, command="npm")
259 def testExpectation(self, testIdPattern, expected_name):
260 if testIdPattern.find("*") == -1:
261 return expected_name == testIdPattern
262 else:
263 return re.compile(re.escape(testIdPattern).replace(r"\*", ".*")).search(
264 expected_name
267 def process_event(self, event):
268 if isinstance(event, list) and len(event) > 1:
269 status = self.status_map.get(event[0])
270 test_start = event[0] == "test-start"
271 if not status and not test_start:
272 return
273 test_info = event[1]
274 test_full_title = test_info.get("fullTitle", "")
275 test_name = test_full_title
276 test_path = test_info.get("file", "")
277 test_file_name = os.path.basename(test_path).replace(".js", "")
278 test_err = test_info.get("err")
279 if status == "FAIL" and test_err:
280 if "timeout" in test_err.lower():
281 status = "TIMEOUT"
282 if test_name and test_path:
283 test_name = "{} ({})".format(test_name, os.path.basename(test_path))
284 # mocha hook failures are not tracked in metadata
285 if status != "PASS" and self.hook_re.search(test_name):
286 self.logger.error("TEST-UNEXPECTED-ERROR %s" % (test_name,))
287 return
288 if test_start:
289 self.logger.test_start(test_name)
290 return
291 expected_name = "[{}] {}".format(test_file_name, test_full_title)
292 expected_item = next(
294 expectation
295 for expectation in reversed(list(self.expected))
296 if self.testExpectation(expectation["testIdPattern"], expected_name)
298 None,
300 if expected_item is None:
301 expected = ["PASS"]
302 else:
303 expected = expected_item["expectations"]
304 # mozlog doesn't really allow unexpected skip,
305 # so if a test is disabled just expect that and note the unexpected skip
306 # Also, mocha doesn't log test-start for skipped tests
307 if status == "SKIP":
308 self.logger.test_start(test_name)
309 if self.expected and status not in expected:
310 self.unexpected_skips.add(test_name)
311 expected = ["SKIP"]
312 known_intermittent = expected[1:]
313 expected_status = expected[0]
315 # check if we've seen a result for this test before this log line
316 result_recorded = self.test_results.get(test_name)
317 if result_recorded:
318 self.logger.warning(
319 "Received a second status for {}: "
320 "first {}, now {}".format(test_name, result_recorded, status)
322 # mocha intermittently logs an additional test result after the
323 # test has already timed out. Avoid recording this second status.
324 if result_recorded != "TIMEOUT":
325 self.test_results[test_name] = status
326 if status not in expected:
327 self.has_unexpected = True
328 self.logger.test_end(
329 test_name,
330 status=status,
331 expected=expected_status,
332 known_intermittent=known_intermittent,
335 def after_end(self):
336 if self.unexpected_skips:
337 self.has_unexpected = True
338 for test_name in self.unexpected_skips:
339 self.logger.error(
340 "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name,)
342 self.logger.suite_end()
345 # tempfile.TemporaryDirectory missing from Python 2.7
346 class TemporaryDirectory(object):
347 def __init__(self):
348 self.path = tempfile.mkdtemp()
349 self._closed = False
351 def __repr__(self):
352 return "<{} {!r}>".format(self.__class__.__name__, self.path)
354 def __enter__(self):
355 return self.path
357 def __exit__(self, exc, value, tb):
358 self.clean()
360 def __del__(self):
361 self.clean()
363 def clean(self):
364 if self.path and not self._closed:
365 shutil.rmtree(self.path)
366 self._closed = True
369 class PuppeteerRunner(MozbuildObject):
370 def __init__(self, *args, **kwargs):
371 super(PuppeteerRunner, self).__init__(*args, **kwargs)
373 self.remotedir = os.path.join(self.topsrcdir, "remote")
374 self.puppeteer_dir = os.path.join(self.remotedir, "test", "puppeteer")
376 def run_test(self, logger, *tests, **params):
378 Runs Puppeteer unit tests with npm.
380 Possible optional test parameters:
382 `bidi`:
383 Boolean to indicate whether to test Firefox with BiDi protocol.
384 `binary`:
385 Path for the browser binary to use. Defaults to the local
386 build.
387 `headless`:
388 Boolean to indicate whether to activate Firefox' headless mode.
389 `extra_prefs`:
390 Dictionary of extra preferences to write to the profile,
391 before invoking npm. Overrides default preferences.
392 `enable_webrender`:
393 Boolean to indicate whether to enable WebRender compositor in Gecko.
395 setup()
397 with_bidi = params.get("bidi", False)
398 binary = params.get("binary") or self.get_binary_path()
399 product = params.get("product", "firefox")
401 env = {
402 # Print browser process ouptut
403 "DUMPIO": "1",
404 # Checked by Puppeteer's custom mocha config
405 "CI": "1",
406 # Causes some tests to be skipped due to assumptions about install
407 "PUPPETEER_ALT_INSTALL": "1",
409 extra_options = {}
410 for k, v in params.get("extra_launcher_options", {}).items():
411 extra_options[k] = json.loads(v)
413 # Override upstream defaults: no retries, shorter timeout
414 mocha_options = [
415 "--reporter",
416 "./json-mocha-reporter.js",
417 "--retries",
418 "0",
419 "--fullTrace",
420 "--timeout",
421 "20000",
422 "--no-parallel",
423 "--no-coverage",
425 env["HEADLESS"] = str(params.get("headless", False))
426 test_command = "test:" + product
428 if product == "firefox":
429 env["BINARY"] = binary
430 env["PUPPETEER_PRODUCT"] = "firefox"
431 env["MOZ_WEBRENDER"] = "%d" % params.get("enable_webrender", False)
432 else:
433 env["PUPPETEER_CACHE_DIR"] = os.path.join(
434 self.topobjdir,
435 "_tests",
436 "remote",
437 "test",
438 "puppeteer",
439 ".cache",
442 if with_bidi is True:
443 test_command = test_command + ":bidi"
444 elif env["HEADLESS"] == "True":
445 test_command = test_command + ":headless"
446 else:
447 test_command = test_command + ":headful"
449 command = ["run", test_command, "--"] + mocha_options
451 prefs = {}
452 for k, v in params.get("extra_prefs", {}).items():
453 print("Using extra preference: {}={}".format(k, v))
454 prefs[k] = mozprofile.Preferences.cast(v)
456 if prefs:
457 extra_options["extraPrefsFirefox"] = prefs
459 if extra_options:
460 env["EXTRA_LAUNCH_OPTIONS"] = json.dumps(extra_options)
462 expected_path = os.path.join(
463 os.path.dirname(__file__),
464 "test",
465 "puppeteer",
466 "test",
467 "TestExpectations.json",
469 if os.path.exists(expected_path):
470 with open(expected_path) as f:
471 expected_data = json.load(f)
472 else:
473 expected_data = []
475 expected_platform = platform.uname().system.lower()
476 if expected_platform == "windows":
477 expected_platform = "win32"
479 # Filter expectation data for the selected browser,
480 # headless or headful mode, the operating system,
481 # run in BiDi mode or not.
482 expectations = [
483 expectation
484 for expectation in expected_data
485 if is_relevant_expectation(
486 expectation, product, with_bidi, env["HEADLESS"], expected_platform
490 output_handler = MochaOutputHandler(logger, expectations)
491 return_code = run_npm(
492 *command,
493 cwd=self.puppeteer_dir,
494 env=env,
495 output_line_handler=output_handler,
496 # Puppeteer unit tests don't always clean-up child processes in case of
497 # failure, so use an output_timeout as a fallback
498 output_timeout=60,
499 exit_on_fail=False,
502 output_handler.after_end()
504 # Non-zero return codes are non-fatal for now since we have some
505 # issues with unresolved promises that shouldn't otherwise block
506 # running the tests
507 if return_code != 0:
508 logger.warning("npm exited with code %s" % return_code)
510 if output_handler.has_unexpected:
511 exit(1, "Got unexpected results")
514 def create_parser_puppeteer():
515 p = argparse.ArgumentParser()
516 p.add_argument(
517 "--product", type=str, default="firefox", choices=["chrome", "firefox"]
519 p.add_argument(
520 "--bidi",
521 action="store_true",
522 help="Flag that indicates whether to test Firefox with BiDi protocol.",
524 p.add_argument(
525 "--binary",
526 type=str,
527 help="Path to browser binary. Defaults to local Firefox build.",
529 p.add_argument(
530 "--ci",
531 action="store_true",
532 help="Flag that indicates that tests run in a CI environment.",
534 p.add_argument(
535 "--disable-fission",
536 action="store_true",
537 default=False,
538 dest="disable_fission",
539 help="Disable Fission (site isolation) in Gecko.",
541 p.add_argument(
542 "--enable-webrender",
543 action="store_true",
544 help="Enable the WebRender compositor in Gecko.",
546 p.add_argument(
547 "-z", "--headless", action="store_true", help="Run browser in headless mode."
549 p.add_argument(
550 "--setpref",
551 action="append",
552 dest="extra_prefs",
553 metavar="<pref>=<value>",
554 help="Defines additional user preferences.",
556 p.add_argument(
557 "--setopt",
558 action="append",
559 dest="extra_options",
560 metavar="<option>=<value>",
561 help="Defines additional options for `puppeteer.launch`.",
563 p.add_argument(
564 "-v",
565 dest="verbosity",
566 action="count",
567 default=0,
568 help="Increase remote agent logging verbosity to include "
569 "debug level messages with -v, trace messages with -vv,"
570 "and to not truncate long trace messages with -vvv",
572 p.add_argument("tests", nargs="*")
573 mozlog.commandline.add_logging_group(p)
574 return p
577 def is_relevant_expectation(
578 expectation, expected_product, with_bidi, is_headless, expected_platform
580 parameters = expectation["parameters"]
582 if expected_product == "firefox":
583 is_expected_product = "chrome" not in parameters
584 else:
585 is_expected_product = "firefox" not in parameters
587 if with_bidi is True:
588 is_expected_protocol = "cdp" not in parameters
589 is_headless = "True"
590 else:
591 is_expected_protocol = "webDriverBiDi" not in parameters
593 if is_headless == "True":
594 is_expected_mode = "headful" not in parameters
595 else:
596 is_expected_mode = "headless" not in parameters
598 is_expected_platform = expected_platform in expectation["platforms"]
600 return (
601 is_expected_product
602 and is_expected_protocol
603 and is_expected_mode
604 and is_expected_platform
608 @Command(
609 "puppeteer-test",
610 category="testing",
611 description="Run Puppeteer unit tests.",
612 parser=create_parser_puppeteer,
614 @CommandArgument(
615 "--no-install",
616 dest="install",
617 action="store_false",
618 default=True,
619 help="Do not install the Puppeteer package",
621 def puppeteer_test(
622 command_context,
623 bidi=None,
624 binary=None,
625 ci=False,
626 disable_fission=False,
627 enable_webrender=False,
628 headless=False,
629 extra_prefs=None,
630 extra_options=None,
631 install=False,
632 verbosity=0,
633 tests=None,
634 product="firefox",
635 **kwargs,
637 logger = mozlog.commandline.setup_logging(
638 "puppeteer-test", kwargs, {"mach": sys.stdout}
641 # moztest calls this programmatically with test objects or manifests
642 if "test_objects" in kwargs and tests is not None:
643 logger.error("Expected either 'test_objects' or 'tests'")
644 exit(1)
646 if product != "firefox" and extra_prefs is not None:
647 logger.error("User preferences are not recognized by %s" % product)
648 exit(1)
650 if "test_objects" in kwargs:
651 tests = []
652 for test in kwargs["test_objects"]:
653 tests.append(test["path"])
655 prefs = {}
656 for s in extra_prefs or []:
657 kv = s.split("=")
658 if len(kv) != 2:
659 logger.error("syntax error in --setpref={}".format(s))
660 exit(EX_USAGE)
661 prefs[kv[0]] = kv[1].strip()
663 options = {}
664 for s in extra_options or []:
665 kv = s.split("=")
666 if len(kv) != 2:
667 logger.error("syntax error in --setopt={}".format(s))
668 exit(EX_USAGE)
669 options[kv[0]] = kv[1].strip()
671 prefs.update({"fission.autostart": True})
672 if disable_fission:
673 prefs.update({"fission.autostart": False})
675 if verbosity == 1:
676 prefs["remote.log.level"] = "Debug"
677 elif verbosity > 1:
678 prefs["remote.log.level"] = "Trace"
679 if verbosity > 2:
680 prefs["remote.log.truncate"] = False
682 if install:
683 install_puppeteer(command_context, product, ci)
685 params = {
686 "bidi": bidi,
687 "binary": binary,
688 "headless": headless,
689 "enable_webrender": enable_webrender,
690 "extra_prefs": prefs,
691 "product": product,
692 "extra_launcher_options": options,
694 puppeteer = command_context._spawn(PuppeteerRunner)
695 try:
696 return puppeteer.run_test(logger, *tests, **params)
697 except BinaryNotFoundException as e:
698 logger.error(e)
699 logger.info(e.help())
700 exit(1)
701 except Exception as e:
702 exit(EX_SOFTWARE, e)
705 def install_puppeteer(command_context, product, ci):
706 setup()
707 env = {"HUSKY": "0"}
709 puppeteer_dir = os.path.join("remote", "test", "puppeteer")
710 puppeteer_dir_full_path = os.path.join(command_context.topsrcdir, puppeteer_dir)
711 puppeteer_test_dir = os.path.join(puppeteer_dir, "test")
713 if product == "chrome":
714 env["PUPPETEER_CACHE_DIR"] = os.path.join(
715 command_context.topobjdir, "_tests", puppeteer_dir, ".cache"
717 else:
718 env["PUPPETEER_SKIP_DOWNLOAD"] = "1"
720 if not ci:
721 run_npm(
722 "run",
723 "clean",
724 cwd=puppeteer_dir_full_path,
725 env=env,
726 exit_on_fail=False,
729 command = "ci" if ci else "install"
730 run_npm(command, cwd=puppeteer_dir_full_path, env=env)
731 run_npm(
732 "run",
733 "build",
734 cwd=os.path.join(command_context.topsrcdir, puppeteer_test_dir),
735 env=env,
739 def exit(code, error=None):
740 if error is not None:
741 if isinstance(error, Exception):
742 import traceback
744 traceback.print_exc()
745 else:
746 message = str(error).split("\n")[0].strip()
747 print("{}: {}".format(sys.argv[0], message), file=sys.stderr)
748 sys.exit(code)