1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
14 from collections
import OrderedDict
18 from mach
.decorators
import Command
, CommandArgument
, SubCommand
19 from mozbuild
import nodeutil
20 from mozbuild
.base
import BinaryNotFoundException
, MozbuildObject
28 # add node and npm from mozbuild to front of system path
29 npm
, _
= nodeutil
.find_npm_executable()
31 exit(EX_CONFIG
, "could not find npm executable")
32 path
= os
.path
.abspath(os
.path
.join(npm
, os
.pardir
))
33 os
.environ
["PATH"] = "{}{}{}".format(path
, os
.pathsep
, os
.environ
["PATH"])
36 def remotedir(command_context
):
37 return os
.path
.join(command_context
.topsrcdir
, "remote")
40 @Command("remote", category
="misc", description
="Remote protocol related operations.")
41 def remote(command_context
):
42 """The remote subcommands all relate to the remote protocol."""
43 command_context
._sub
_mach
(["help", "remote"])
48 "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
53 default
="https://github.com/puppeteer/puppeteer.git",
54 help="The (possibly local) repository to clone from.",
60 help="The commit or tag object name to check out.",
67 help="Do not install the just-pulled Puppeteer package,",
69 def vendor_puppeteer(command_context
, repository
, commitish
, install
):
70 puppeteer_dir
= os
.path
.join(remotedir(command_context
), "test", "puppeteer")
72 # Preserve our custom mocha reporter
74 os
.path
.join(puppeteer_dir
, "json-mocha-reporter.js"),
75 remotedir(command_context
),
77 shutil
.rmtree(puppeteer_dir
, ignore_errors
=True)
78 os
.makedirs(puppeteer_dir
)
79 with
TemporaryDirectory() as tmpdir
:
80 git("clone", "-q", repository
, tmpdir
)
81 git("checkout", commitish
, worktree
=tmpdir
)
87 "{}/".format(puppeteer_dir
),
91 # remove files which may interfere with git checkout of central
93 os
.remove(os
.path
.join(puppeteer_dir
, ".gitattributes"))
94 os
.remove(os
.path
.join(puppeteer_dir
, ".gitignore"))
98 unwanted_dirs
= ["experimental", "docs"]
100 for dir in unwanted_dirs
:
101 dir_path
= os
.path
.join(puppeteer_dir
, dir)
102 if os
.path
.isdir(dir_path
):
103 shutil
.rmtree(dir_path
)
106 os
.path
.join(remotedir(command_context
), "json-mocha-reporter.js"),
115 "product": "Remote Protocol",
116 "component": "Agent",
120 "description": "Headless Chrome Node API",
122 "license": "Apache-2.0",
123 "release": commitish
,
126 with
open(os
.path
.join(puppeteer_dir
, "moz.yaml"), "w") as fh
:
130 default_flow_style
=False,
136 env
= {"HUSKY": "0", "PUPPETEER_SKIP_DOWNLOAD": "1"}
139 cwd
=os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
),
144 def git(*args
, **kwargs
):
146 if kwargs
.get("worktree"):
147 cmd
+= ("-C", kwargs
["worktree"])
150 pipe
= kwargs
.get("pipe")
151 git_p
= subprocess
.Popen(
153 env
={"GIT_CONFIG_NOSYSTEM": "1"},
154 stdout
=subprocess
.PIPE
,
155 stderr
=subprocess
.PIPE
,
159 pipe_p
= subprocess
.Popen(pipe
, stdin
=git_p
.stdout
, stderr
=subprocess
.PIPE
)
162 _
, pipe_err
= pipe_p
.communicate()
163 out
, git_err
= git_p
.communicate()
165 # use error from first program that failed
166 if git_p
.returncode
> 0:
167 exit(EX_SOFTWARE
, git_err
)
168 if pipe
and pipe_p
.returncode
> 0:
169 exit(EX_SOFTWARE
, pipe_err
)
174 def npm(*args
, **kwargs
):
175 from mozprocess
import processhandler
178 npm
, _
= nodeutil
.find_npm_executable()
179 if kwargs
.get("env"):
180 env
= os
.environ
.copy()
181 env
.update(kwargs
["env"])
184 if "processOutputLine" in kwargs
:
185 proc_kwargs
["processOutputLine"] = kwargs
["processOutputLine"]
187 p
= processhandler
.ProcessHandler(
190 cwd
=kwargs
.get("cwd"),
192 universal_newlines
=True,
195 if not kwargs
.get("wait", True):
198 wait_proc(p
, cmd
=npm
, exit_on_fail
=kwargs
.get("exit_on_fail", True))
203 def wait_proc(p
, cmd
=None, exit_on_fail
=True, output_timeout
=None):
205 p
.run(outputTimeout
=output_timeout
)
208 # In some cases, we wait longer for a mocha timeout
209 print("Timed out after {} seconds of no output".format(output_timeout
))
212 if exit_on_fail
and p
.returncode
> 0:
214 "%s: exit code %s" % (cmd
, p
.returncode
)
216 else "exit code %s" % p
.returncode
218 exit(p
.returncode
, msg
)
221 class MochaOutputHandler(object):
222 def __init__(self
, logger
, expected
):
223 self
.hook_re
= re
.compile('"before\b?.*" hook|"after\b?.*" hook')
227 self
.test_results
= OrderedDict()
228 self
.expected
= expected
229 self
.unexpected_skips
= set()
231 self
.has_unexpected
= False
232 self
.logger
.suite_start([], name
="puppeteer-tests")
236 "TERMINATED": "CRASH",
244 return self
.proc
and self
.proc
.pid
246 def __call__(self
, line
):
249 if line
.startswith("[") and line
.endswith("]"):
250 event
= json
.loads(line
)
251 self
.process_event(event
)
255 self
.logger
.process_output(self
.pid
, line
, command
="npm")
257 def testExpectation(self
, testIdPattern
, expected_name
):
258 if testIdPattern
.find("*") == -1:
259 return expected_name
== testIdPattern
261 return re
.compile(re
.escape(testIdPattern
).replace("\*", ".*")).search(
265 def process_event(self
, event
):
266 if isinstance(event
, list) and len(event
) > 1:
267 status
= self
.status_map
.get(event
[0])
268 test_start
= event
[0] == "test-start"
269 if not status
and not test_start
:
272 test_full_title
= test_info
.get("fullTitle", "")
273 test_name
= test_full_title
274 test_path
= test_info
.get("file", "")
275 test_file_name
= os
.path
.basename(test_path
).replace(".js", "")
276 test_err
= test_info
.get("err")
277 if status
== "FAIL" and test_err
:
278 if "timeout" in test_err
.lower():
280 if test_name
and test_path
:
281 test_name
= "{} ({})".format(test_name
, os
.path
.basename(test_path
))
282 # mocha hook failures are not tracked in metadata
283 if status
!= "PASS" and self
.hook_re
.search(test_name
):
284 self
.logger
.error("TEST-UNEXPECTED-ERROR %s" % (test_name
,))
287 self
.logger
.test_start(test_name
)
289 expected_name
= "[{}] {}".format(test_file_name
, test_full_title
)
290 expected_item
= next(
293 for expectation
in reversed(list(self
.expected
))
294 if self
.testExpectation(expectation
["testIdPattern"], expected_name
)
298 if expected_item
is None:
301 expected
= expected_item
["expectations"]
302 # mozlog doesn't really allow unexpected skip,
303 # so if a test is disabled just expect that and note the unexpected skip
304 # Also, mocha doesn't log test-start for skipped tests
306 self
.logger
.test_start(test_name
)
307 if self
.expected
and status
not in expected
:
308 self
.unexpected_skips
.add(test_name
)
310 known_intermittent
= expected
[1:]
311 expected_status
= expected
[0]
313 # check if we've seen a result for this test before this log line
314 result_recorded
= self
.test_results
.get(test_name
)
317 "Received a second status for {}: "
318 "first {}, now {}".format(test_name
, result_recorded
, status
)
320 # mocha intermittently logs an additional test result after the
321 # test has already timed out. Avoid recording this second status.
322 if result_recorded
!= "TIMEOUT":
323 self
.test_results
[test_name
] = status
324 if status
not in expected
:
325 self
.has_unexpected
= True
326 self
.logger
.test_end(
329 expected
=expected_status
,
330 known_intermittent
=known_intermittent
,
334 if self
.unexpected_skips
:
335 self
.has_unexpected
= True
336 for test_name
in self
.unexpected_skips
:
338 "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name
,)
340 self
.logger
.suite_end()
343 # tempfile.TemporaryDirectory missing from Python 2.7
344 class TemporaryDirectory(object):
346 self
.path
= tempfile
.mkdtemp()
350 return "<{} {!r}>".format(self
.__class
__.__name
__, self
.path
)
355 def __exit__(self
, exc
, value
, tb
):
362 if self
.path
and not self
._closed
:
363 shutil
.rmtree(self
.path
)
367 class PuppeteerRunner(MozbuildObject
):
368 def __init__(self
, *args
, **kwargs
):
369 super(PuppeteerRunner
, self
).__init
__(*args
, **kwargs
)
371 self
.remotedir
= os
.path
.join(self
.topsrcdir
, "remote")
372 self
.puppeteer_dir
= os
.path
.join(self
.remotedir
, "test", "puppeteer")
374 def run_test(self
, logger
, *tests
, **params
):
376 Runs Puppeteer unit tests with npm.
378 Possible optional test parameters:
381 Boolean to indicate whether to test Firefox with BiDi protocol.
383 Path for the browser binary to use. Defaults to the local
386 Boolean to indicate whether to activate Firefox' headless mode.
388 Dictionary of extra preferences to write to the profile,
389 before invoking npm. Overrides default preferences.
391 Boolean to indicate whether to enable WebRender compositor in Gecko.
395 with_bidi
= params
.get("bidi", False)
396 binary
= params
.get("binary") or self
.get_binary_path()
397 product
= params
.get("product", "firefox")
400 # Print browser process ouptut
402 # Checked by Puppeteer's custom mocha config
404 # Causes some tests to be skipped due to assumptions about install
405 "PUPPETEER_ALT_INSTALL": "1",
408 for k
, v
in params
.get("extra_launcher_options", {}).items():
409 extra_options
[k
] = json
.loads(v
)
411 # Override upstream defaults: no retries, shorter timeout
414 "./json-mocha-reporter.js",
423 env
["HEADLESS"] = str(params
.get("headless", False))
424 test_command
= "test:" + product
426 if product
== "firefox":
427 env
["BINARY"] = binary
428 env
["PUPPETEER_PRODUCT"] = "firefox"
429 env
["MOZ_WEBRENDER"] = "%d" % params
.get("enable_webrender", False)
431 env
["PUPPETEER_CACHE_DIR"] = os
.path
.join(
440 if with_bidi
is True:
441 test_command
= test_command
+ ":bidi"
442 elif env
["HEADLESS"] == "True":
443 test_command
= test_command
+ ":headless"
445 test_command
= test_command
+ ":headful"
447 command
= ["run", test_command
, "--"] + mocha_options
450 for k
, v
in params
.get("extra_prefs", {}).items():
451 print("Using extra preference: {}={}".format(k
, v
))
452 prefs
[k
] = mozprofile
.Preferences
.cast(v
)
455 extra_options
["extraPrefsFirefox"] = prefs
458 env
["EXTRA_LAUNCH_OPTIONS"] = json
.dumps(extra_options
)
460 expected_path
= os
.path
.join(
461 os
.path
.dirname(__file__
),
465 "TestExpectations.json",
467 if os
.path
.exists(expected_path
):
468 with
open(expected_path
) as f
:
469 expected_data
= json
.load(f
)
473 expected_platform
= platform
.uname().system
.lower()
474 if expected_platform
== "windows":
475 expected_platform
= "win32"
477 # Filter expectation data for the selected browser,
478 # headless or headful mode, the operating system,
479 # run in BiDi mode or not.
482 for expectation
in expected_data
483 if is_relevant_expectation(
484 expectation
, product
, with_bidi
, env
["HEADLESS"], expected_platform
488 output_handler
= MochaOutputHandler(logger
, expectations
)
491 cwd
=self
.puppeteer_dir
,
493 processOutputLine
=output_handler
,
496 output_handler
.proc
= proc
498 # Puppeteer unit tests don't always clean-up child processes in case of
499 # failure, so use an output_timeout as a fallback
500 wait_proc(proc
, "npm", output_timeout
=60, exit_on_fail
=False)
502 output_handler
.after_end()
504 # Non-zero return codes are non-fatal for now since we have some
505 # issues with unresolved promises that shouldn't otherwise block
507 if proc
.returncode
!= 0:
508 logger
.warning("npm exited with code %s" % proc
.returncode
)
510 if output_handler
.has_unexpected
:
511 exit(1, "Got unexpected results")
514 def create_parser_puppeteer():
515 p
= argparse
.ArgumentParser()
517 "--product", type=str, default
="firefox", choices
=["chrome", "firefox"]
522 help="Flag that indicates whether to test Firefox with BiDi protocol.",
527 help="Path to browser binary. Defaults to local Firefox build.",
532 help="Flag that indicates that tests run in a CI environment.",
538 dest
="disable_fission",
539 help="Disable Fission (site isolation) in Gecko.",
542 "--enable-webrender",
544 help="Enable the WebRender compositor in Gecko.",
547 "-z", "--headless", action
="store_true", help="Run browser in headless mode."
553 metavar
="<pref>=<value>",
554 help="Defines additional user preferences.",
559 dest
="extra_options",
560 metavar
="<option>=<value>",
561 help="Defines additional options for `puppeteer.launch`.",
568 help="Increase remote agent logging verbosity to include "
569 "debug level messages with -v, trace messages with -vv,"
570 "and to not truncate long trace messages with -vvv",
572 p
.add_argument("tests", nargs
="*")
573 mozlog
.commandline
.add_logging_group(p
)
577 def is_relevant_expectation(
578 expectation
, expected_product
, with_bidi
, is_headless
, expected_platform
580 parameters
= expectation
["parameters"]
582 if expected_product
== "firefox":
583 is_expected_product
= "chrome" not in parameters
585 is_expected_product
= "firefox" not in parameters
587 if with_bidi
is True:
588 is_expected_protocol
= "cdp" not in parameters
591 is_expected_protocol
= "webDriverBiDi" not in parameters
593 if is_headless
== "True":
594 is_expected_mode
= "headful" not in parameters
596 is_expected_mode
= "headless" not in parameters
598 is_expected_platform
= expected_platform
in expectation
["platforms"]
602 and is_expected_protocol
604 and is_expected_platform
611 description
="Run Puppeteer unit tests.",
612 parser
=create_parser_puppeteer
,
617 action
="store_false",
619 help="Do not install the Puppeteer package",
626 disable_fission
=False,
627 enable_webrender
=False,
637 logger
= mozlog
.commandline
.setup_logging(
638 "puppeteer-test", kwargs
, {"mach": sys
.stdout
}
641 # moztest calls this programmatically with test objects or manifests
642 if "test_objects" in kwargs
and tests
is not None:
643 logger
.error("Expected either 'test_objects' or 'tests'")
646 if product
!= "firefox" and extra_prefs
is not None:
647 logger
.error("User preferences are not recognized by %s" % product
)
650 if "test_objects" in kwargs
:
652 for test
in kwargs
["test_objects"]:
653 tests
.append(test
["path"])
656 for s
in extra_prefs
or []:
659 logger
.error("syntax error in --setpref={}".format(s
))
661 prefs
[kv
[0]] = kv
[1].strip()
664 for s
in extra_options
or []:
667 logger
.error("syntax error in --setopt={}".format(s
))
669 options
[kv
[0]] = kv
[1].strip()
671 prefs
.update({"fission.autostart": True})
673 prefs
.update({"fission.autostart": False})
676 prefs
["remote.log.level"] = "Debug"
678 prefs
["remote.log.level"] = "Trace"
680 prefs
["remote.log.truncate"] = False
683 install_puppeteer(command_context
, product
, ci
)
688 "headless": headless
,
689 "enable_webrender": enable_webrender
,
690 "extra_prefs": prefs
,
692 "extra_launcher_options": options
,
694 puppeteer
= command_context
._spawn
(PuppeteerRunner
)
696 return puppeteer
.run_test(logger
, *tests
, **params
)
697 except BinaryNotFoundException
as e
:
699 logger
.info(e
.help())
701 except Exception as e
:
705 def install_puppeteer(command_context
, product
, ci
):
709 puppeteer_dir
= os
.path
.join("remote", "test", "puppeteer")
710 puppeteer_dir_full_path
= os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
)
711 puppeteer_test_dir
= os
.path
.join(puppeteer_dir
, "test")
713 if product
== "chrome":
714 env
["PUPPETEER_CACHE_DIR"] = os
.path
.join(
715 command_context
.topobjdir
, "_tests", puppeteer_dir
, ".cache"
718 env
["PUPPETEER_SKIP_DOWNLOAD"] = "1"
724 cwd
=puppeteer_dir_full_path
,
729 command
= "ci" if ci
else "install"
730 npm(command
, cwd
=puppeteer_dir_full_path
, env
=env
)
734 cwd
=os
.path
.join(command_context
.topsrcdir
, puppeteer_test_dir
),
739 def exit(code
, error
=None):
740 if error
is not None:
741 if isinstance(error
, Exception):
744 traceback
.print_exc()
746 message
= str(error
).split("\n")[0].strip()
747 print("{}: {}".format(sys
.argv
[0], message
), file=sys
.stderr
)