1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
14 from collections
import OrderedDict
18 from mach
.decorators
import Command
, CommandArgument
, SubCommand
19 from mozbuild
import nodeutil
20 from mozbuild
.base
import BinaryNotFoundException
, MozbuildObject
28 # add node and npm from mozbuild to front of system path
29 npm
, _
= nodeutil
.find_npm_executable()
31 exit(EX_CONFIG
, "could not find npm executable")
32 path
= os
.path
.abspath(os
.path
.join(npm
, os
.pardir
))
33 os
.environ
["PATH"] = "{}{}{}".format(path
, os
.pathsep
, os
.environ
["PATH"])
36 def remotedir(command_context
):
37 return os
.path
.join(command_context
.topsrcdir
, "remote")
40 @Command("remote", category
="misc", description
="Remote protocol related operations.")
41 def remote(command_context
):
42 """The remote subcommands all relate to the remote protocol."""
43 command_context
._sub
_mach
(["help", "remote"])
48 "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
53 default
="https://github.com/puppeteer/puppeteer.git",
54 help="The (possibly local) repository to clone from.",
60 help="The commit or tag object name to check out.",
67 help="Do not install the just-pulled Puppeteer package,",
69 def vendor_puppeteer(command_context
, repository
, commitish
, install
):
70 puppeteer_dir
= os
.path
.join(remotedir(command_context
), "test", "puppeteer")
72 # Preserve our custom mocha reporter
74 os
.path
.join(puppeteer_dir
, "json-mocha-reporter.js"),
75 os
.path
.join(remotedir(command_context
), "json-mocha-reporter.js"),
77 shutil
.rmtree(puppeteer_dir
, ignore_errors
=True)
78 os
.makedirs(puppeteer_dir
)
79 with
TemporaryDirectory() as tmpdir
:
80 git("clone", "-q", repository
, tmpdir
)
81 git("checkout", commitish
, worktree
=tmpdir
)
87 "{}/".format(puppeteer_dir
),
91 # remove files which may interfere with git checkout of central
93 os
.remove(os
.path
.join(puppeteer_dir
, ".gitattributes"))
94 os
.remove(os
.path
.join(puppeteer_dir
, ".gitignore"))
98 unwanted_dirs
= ["experimental", "docs"]
100 for dir in unwanted_dirs
:
101 dir_path
= os
.path
.join(puppeteer_dir
, dir)
102 if os
.path
.isdir(dir_path
):
103 shutil
.rmtree(dir_path
)
106 os
.path
.join(remotedir(command_context
), "json-mocha-reporter.js"),
115 "product": "Remote Protocol",
116 "component": "Agent",
120 "description": "Headless Chrome Node API",
122 "license": "Apache-2.0",
123 "release": commitish
,
126 with
open(os
.path
.join(puppeteer_dir
, "moz.yaml"), "w") as fh
:
130 default_flow_style
=False,
137 "CI": "1", # Force the quiet logger of wireit
138 "HUSKY": "0", # Disable any hook checks
139 "PUPPETEER_SKIP_DOWNLOAD": "1", # Don't download any build
144 cwd
=os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
),
149 def git(*args
, **kwargs
):
151 if kwargs
.get("worktree"):
152 cmd
+= ("-C", kwargs
["worktree"])
155 pipe
= kwargs
.get("pipe")
156 git_p
= subprocess
.Popen(
158 env
={"GIT_CONFIG_NOSYSTEM": "1"},
159 stdout
=subprocess
.PIPE
,
160 stderr
=subprocess
.PIPE
,
164 pipe_p
= subprocess
.Popen(pipe
, stdin
=git_p
.stdout
, stderr
=subprocess
.PIPE
)
167 _
, pipe_err
= pipe_p
.communicate()
168 out
, git_err
= git_p
.communicate()
170 # use error from first program that failed
171 if git_p
.returncode
> 0:
172 exit(EX_SOFTWARE
, git_err
)
173 if pipe
and pipe_p
.returncode
> 0:
174 exit(EX_SOFTWARE
, pipe_err
)
179 def run_npm(*args
, **kwargs
):
180 from mozprocess
import run_and_wait
182 def output_timeout_handler(proc
):
183 # In some cases, we wait longer for a mocha timeout
185 "Timed out after {} seconds of no output".format(kwargs
["output_timeout"])
189 npm
, _
= nodeutil
.find_npm_executable()
190 if kwargs
.get("env"):
191 env
= os
.environ
.copy()
192 env
.update(kwargs
["env"])
194 proc_kwargs
= {"output_timeout_handler": output_timeout_handler
}
195 for kw
in ["output_line_handler", "output_timeout"]:
197 proc_kwargs
[kw
] = kwargs
[kw
]
200 cmd
.extend(list(args
))
204 cwd
=kwargs
.get("cwd"),
209 post_wait_proc(p
, cmd
=npm
, exit_on_fail
=kwargs
.get("exit_on_fail", True))
214 def post_wait_proc(p
, cmd
=None, exit_on_fail
=True):
217 if exit_on_fail
and p
.returncode
> 0:
219 "%s: exit code %s" % (cmd
, p
.returncode
)
221 else "exit code %s" % p
.returncode
223 exit(p
.returncode
, msg
)
226 class MochaOutputHandler(object):
227 def __init__(self
, logger
, expected
):
228 self
.hook_re
= re
.compile('"before\b?.*" hook|"after\b?.*" hook')
232 self
.test_results
= OrderedDict()
233 self
.expected
= expected
234 self
.unexpected_skips
= set()
236 self
.has_unexpected
= False
237 self
.logger
.suite_start([], name
="puppeteer-tests")
241 "TERMINATED": "CRASH",
249 return self
.proc
and self
.proc
.pid
251 def __call__(self
, proc
, line
):
253 line
= line
.rstrip("\r\n")
256 if line
.startswith("[") and line
.endswith("]"):
257 event
= json
.loads(line
)
258 self
.process_event(event
)
262 self
.logger
.process_output(self
.pid
, line
, command
="npm")
264 def testExpectation(self
, testIdPattern
, expected_name
):
265 if testIdPattern
.find("*") == -1:
266 return expected_name
== testIdPattern
268 return re
.compile(re
.escape(testIdPattern
).replace(r
"\*", ".*")).search(
272 def process_event(self
, event
):
273 if isinstance(event
, list) and len(event
) > 1:
274 status
= self
.status_map
.get(event
[0])
275 test_start
= event
[0] == "test-start"
276 if not status
and not test_start
:
279 test_full_title
= test_info
.get("fullTitle", "")
280 test_name
= test_full_title
281 test_path
= test_info
.get("file", "")
282 test_file_name
= os
.path
.basename(test_path
).replace(".js", "")
283 test_err
= test_info
.get("err")
284 if status
== "FAIL" and test_err
:
285 if "timeout" in test_err
.lower():
287 if test_name
and test_path
:
288 test_name
= "{} ({})".format(test_name
, os
.path
.basename(test_path
))
289 # mocha hook failures are not tracked in metadata
290 if status
!= "PASS" and self
.hook_re
.search(test_name
):
291 self
.logger
.error("TEST-UNEXPECTED-ERROR %s" % (test_name
,))
294 self
.logger
.test_start(test_name
)
296 expected_name
= "[{}] {}".format(test_file_name
, test_full_title
)
297 expected_item
= next(
300 for expectation
in reversed(list(self
.expected
))
301 if self
.testExpectation(expectation
["testIdPattern"], expected_name
)
305 if expected_item
is None:
308 expected
= expected_item
["expectations"]
309 # mozlog doesn't really allow unexpected skip,
310 # so if a test is disabled just expect that and note the unexpected skip
311 # Also, mocha doesn't log test-start for skipped tests
313 self
.logger
.test_start(test_name
)
314 if self
.expected
and status
not in expected
:
315 self
.unexpected_skips
.add(test_name
)
317 known_intermittent
= expected
[1:]
318 expected_status
= expected
[0]
320 # check if we've seen a result for this test before this log line
321 result_recorded
= self
.test_results
.get(test_name
)
324 "Received a second status for {}: "
325 "first {}, now {}".format(test_name
, result_recorded
, status
)
327 # mocha intermittently logs an additional test result after the
328 # test has already timed out. Avoid recording this second status.
329 if result_recorded
!= "TIMEOUT":
330 self
.test_results
[test_name
] = status
331 if status
not in expected
:
332 self
.has_unexpected
= True
333 self
.logger
.test_end(
336 expected
=expected_status
,
337 known_intermittent
=known_intermittent
,
341 if self
.unexpected_skips
:
342 self
.has_unexpected
= True
343 for test_name
in self
.unexpected_skips
:
345 "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name
,)
347 self
.logger
.suite_end()
350 # tempfile.TemporaryDirectory missing from Python 2.7
351 class TemporaryDirectory(object):
353 self
.path
= tempfile
.mkdtemp()
357 return "<{} {!r}>".format(self
.__class
__.__name
__, self
.path
)
362 def __exit__(self
, exc
, value
, tb
):
369 if self
.path
and not self
._closed
:
370 shutil
.rmtree(self
.path
)
374 class PuppeteerRunner(MozbuildObject
):
375 def __init__(self
, *args
, **kwargs
):
376 super(PuppeteerRunner
, self
).__init
__(*args
, **kwargs
)
378 self
.remotedir
= os
.path
.join(self
.topsrcdir
, "remote")
379 self
.puppeteer_dir
= os
.path
.join(self
.remotedir
, "test", "puppeteer")
381 def run_test(self
, logger
, *tests
, **params
):
383 Runs Puppeteer unit tests with npm.
385 Possible optional test parameters:
388 Boolean to indicate whether to test Firefox with BiDi protocol.
390 Path for the browser binary to use. Defaults to the local
393 Boolean to indicate whether to activate Firefox' headless mode.
395 Dictionary of extra preferences to write to the profile,
396 before invoking npm. Overrides default preferences.
398 Boolean to indicate whether to enable WebRender compositor in Gecko.
402 with_bidi
= params
.get("bidi", False)
403 binary
= params
.get("binary") or self
.get_binary_path()
404 product
= params
.get("product", "firefox")
407 # Print browser process ouptut
409 # Checked by Puppeteer's custom mocha config
411 # Causes some tests to be skipped due to assumptions about install
412 "PUPPETEER_ALT_INSTALL": "1",
415 for k
, v
in params
.get("extra_launcher_options", {}).items():
416 extra_options
[k
] = json
.loads(v
)
418 # Override upstream defaults: no retries, shorter timeout
421 "./json-mocha-reporter.js",
430 env
["HEADLESS"] = str(params
.get("headless", False))
431 test_command
= "test:" + product
433 if product
== "firefox":
434 env
["BINARY"] = binary
435 env
["PUPPETEER_PRODUCT"] = "firefox"
436 env
["MOZ_WEBRENDER"] = "%d" % params
.get("enable_webrender", False)
438 env
["PUPPETEER_CACHE_DIR"] = os
.path
.join(
447 if with_bidi
is True:
448 test_command
= test_command
+ ":bidi"
449 elif env
["HEADLESS"] == "True":
450 test_command
= test_command
+ ":headless"
452 test_command
= test_command
+ ":headful"
454 command
= ["run", test_command
, "--"] + mocha_options
457 for k
, v
in params
.get("extra_prefs", {}).items():
458 print("Using extra preference: {}={}".format(k
, v
))
459 prefs
[k
] = mozprofile
.Preferences
.cast(v
)
462 extra_options
["extraPrefsFirefox"] = prefs
465 env
["EXTRA_LAUNCH_OPTIONS"] = json
.dumps(extra_options
)
467 expected_path
= os
.path
.join(
468 os
.path
.dirname(__file__
),
472 "TestExpectations.json",
474 if os
.path
.exists(expected_path
):
475 with
open(expected_path
) as f
:
476 expected_data
= json
.load(f
)
480 expected_platform
= platform
.uname().system
.lower()
481 if expected_platform
== "windows":
482 expected_platform
= "win32"
484 # Filter expectation data for the selected browser,
485 # headless or headful mode, the operating system,
486 # run in BiDi mode or not.
489 for expectation
in expected_data
490 if is_relevant_expectation(
491 expectation
, product
, with_bidi
, env
["HEADLESS"], expected_platform
495 output_handler
= MochaOutputHandler(logger
, expectations
)
498 cwd
=self
.puppeteer_dir
,
500 output_line_handler
=output_handler
,
501 # Puppeteer unit tests don't always clean-up child processes in case of
502 # failure, so use an output_timeout as a fallback
507 output_handler
.after_end()
509 if output_handler
.has_unexpected
:
510 logger
.error("Got unexpected results")
514 def create_parser_puppeteer():
515 p
= argparse
.ArgumentParser()
517 "--product", type=str, default
="firefox", choices
=["chrome", "firefox"]
522 help="Flag that indicates whether to test Firefox with BiDi protocol.",
527 help="Path to browser binary. Defaults to local Firefox build.",
532 help="Flag that indicates that tests run in a CI environment.",
538 dest
="disable_fission",
539 help="Disable Fission (site isolation) in Gecko.",
542 "--enable-webrender",
544 help="Enable the WebRender compositor in Gecko.",
547 "-z", "--headless", action
="store_true", help="Run browser in headless mode."
553 metavar
="<pref>=<value>",
554 help="Defines additional user preferences.",
559 dest
="extra_options",
560 metavar
="<option>=<value>",
561 help="Defines additional options for `puppeteer.launch`.",
568 help="Increase remote agent logging verbosity to include "
569 "debug level messages with -v, trace messages with -vv,"
570 "and to not truncate long trace messages with -vvv",
572 p
.add_argument("tests", nargs
="*")
573 mozlog
.commandline
.add_logging_group(p
)
577 def is_relevant_expectation(
578 expectation
, expected_product
, with_bidi
, is_headless
, expected_platform
580 parameters
= expectation
["parameters"]
582 if expected_product
== "firefox":
583 is_expected_product
= "chrome" not in parameters
585 is_expected_product
= "firefox" not in parameters
587 if with_bidi
is True:
588 is_expected_protocol
= "cdp" not in parameters
591 is_expected_protocol
= "webDriverBiDi" not in parameters
593 if is_headless
== "True":
594 is_expected_mode
= "headful" not in parameters
596 is_expected_mode
= "headless" not in parameters
598 is_expected_platform
= expected_platform
in expectation
["platforms"]
602 and is_expected_protocol
604 and is_expected_platform
611 description
="Run Puppeteer unit tests.",
612 parser
=create_parser_puppeteer
,
617 action
="store_false",
619 help="Do not install the Puppeteer package",
626 disable_fission
=False,
627 enable_webrender
=False,
637 logger
= mozlog
.commandline
.setup_logging(
638 "puppeteer-test", kwargs
, {"mach": sys
.stdout
}
641 # moztest calls this programmatically with test objects or manifests
642 if "test_objects" in kwargs
and tests
is not None:
643 logger
.error("Expected either 'test_objects' or 'tests'")
646 if product
!= "firefox" and extra_prefs
is not None:
647 logger
.error("User preferences are not recognized by %s" % product
)
650 if "test_objects" in kwargs
:
652 for test
in kwargs
["test_objects"]:
653 tests
.append(test
["path"])
656 for s
in extra_prefs
or []:
659 logger
.error("syntax error in --setpref={}".format(s
))
661 prefs
[kv
[0]] = kv
[1].strip()
664 for s
in extra_options
or []:
667 logger
.error("syntax error in --setopt={}".format(s
))
669 options
[kv
[0]] = kv
[1].strip()
671 prefs
.update({"fission.autostart": True})
673 prefs
.update({"fission.autostart": False})
676 prefs
["remote.log.level"] = "Debug"
678 prefs
["remote.log.level"] = "Trace"
680 prefs
["remote.log.truncate"] = False
683 install_puppeteer(command_context
, product
, ci
)
688 "headless": headless
,
689 "enable_webrender": enable_webrender
,
690 "extra_prefs": prefs
,
692 "extra_launcher_options": options
,
694 puppeteer
= command_context
._spawn
(PuppeteerRunner
)
696 return puppeteer
.run_test(logger
, *tests
, **params
)
697 except BinaryNotFoundException
as e
:
699 logger
.info(e
.help())
701 except Exception as e
:
705 def install_puppeteer(command_context
, product
, ci
):
709 "CI": "1", # Force the quiet logger of wireit
710 "HUSKY": "0", # Disable any hook checks
713 puppeteer_dir
= os
.path
.join("remote", "test", "puppeteer")
714 puppeteer_dir_full_path
= os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
)
715 puppeteer_test_dir
= os
.path
.join(puppeteer_dir
, "test")
717 if product
== "chrome":
718 env
["PUPPETEER_CACHE_DIR"] = os
.path
.join(
719 command_context
.topobjdir
, "_tests", puppeteer_dir
, ".cache"
722 env
["PUPPETEER_SKIP_DOWNLOAD"] = "1"
728 cwd
=puppeteer_dir_full_path
,
733 # Always use the `ci` command to not get updated sub-dependencies installed.
734 run_npm("ci", cwd
=puppeteer_dir_full_path
, env
=env
)
738 cwd
=os
.path
.join(command_context
.topsrcdir
, puppeteer_test_dir
),
743 def exit(code
, error
=None):
744 if error
is not None:
745 if isinstance(error
, Exception):
748 traceback
.print_exc()
750 message
= str(error
).split("\n")[0].strip()
751 print("{}: {}".format(sys
.argv
[0], message
), file=sys
.stderr
)