1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
14 from collections
import OrderedDict
18 from mach
.decorators
import Command
, CommandArgument
, SubCommand
19 from mozbuild
import nodeutil
20 from mozbuild
.base
import BinaryNotFoundException
, MozbuildObject
28 # add node and npm from mozbuild to front of system path
29 npm
, _
= nodeutil
.find_npm_executable()
31 exit(EX_CONFIG
, "could not find npm executable")
32 path
= os
.path
.abspath(os
.path
.join(npm
, os
.pardir
))
33 os
.environ
["PATH"] = "{}{}{}".format(path
, os
.pathsep
, os
.environ
["PATH"])
36 def remotedir(command_context
):
37 return os
.path
.join(command_context
.topsrcdir
, "remote")
40 @Command("remote", category
="misc", description
="Remote protocol related operations.")
41 def remote(command_context
):
42 """The remote subcommands all relate to the remote protocol."""
43 command_context
._sub
_mach
(["help", "remote"])
48 "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
53 default
="https://github.com/puppeteer/puppeteer.git",
54 help="The (possibly local) repository to clone from.",
60 help="The commit or tag object name to check out.",
67 help="Do not install the just-pulled Puppeteer package,",
69 def vendor_puppeteer(command_context
, repository
, commitish
, install
):
70 puppeteer_dir
= os
.path
.join(remotedir(command_context
), "test", "puppeteer")
72 # Preserve our custom mocha reporter
74 os
.path
.join(puppeteer_dir
, "json-mocha-reporter.js"),
75 remotedir(command_context
),
77 shutil
.rmtree(puppeteer_dir
, ignore_errors
=True)
78 os
.makedirs(puppeteer_dir
)
79 with
TemporaryDirectory() as tmpdir
:
80 git("clone", "-q", repository
, tmpdir
)
81 git("checkout", commitish
, worktree
=tmpdir
)
87 "{}/".format(puppeteer_dir
),
91 # remove files which may interfere with git checkout of central
93 os
.remove(os
.path
.join(puppeteer_dir
, ".gitattributes"))
94 os
.remove(os
.path
.join(puppeteer_dir
, ".gitignore"))
98 unwanted_dirs
= ["experimental", "docs"]
100 for dir in unwanted_dirs
:
101 dir_path
= os
.path
.join(puppeteer_dir
, dir)
102 if os
.path
.isdir(dir_path
):
103 shutil
.rmtree(dir_path
)
106 os
.path
.join(remotedir(command_context
), "json-mocha-reporter.js"),
115 "product": "Remote Protocol",
116 "component": "Agent",
120 "description": "Headless Chrome Node API",
122 "license": "Apache-2.0",
123 "release": commitish
,
126 with
open(os
.path
.join(puppeteer_dir
, "moz.yaml"), "w") as fh
:
130 default_flow_style
=False,
136 env
= {"HUSKY": "0", "PUPPETEER_SKIP_DOWNLOAD": "1"}
139 cwd
=os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
),
144 def git(*args
, **kwargs
):
146 if kwargs
.get("worktree"):
147 cmd
+= ("-C", kwargs
["worktree"])
150 pipe
= kwargs
.get("pipe")
151 git_p
= subprocess
.Popen(
153 env
={"GIT_CONFIG_NOSYSTEM": "1"},
154 stdout
=subprocess
.PIPE
,
155 stderr
=subprocess
.PIPE
,
159 pipe_p
= subprocess
.Popen(pipe
, stdin
=git_p
.stdout
, stderr
=subprocess
.PIPE
)
162 _
, pipe_err
= pipe_p
.communicate()
163 out
, git_err
= git_p
.communicate()
165 # use error from first program that failed
166 if git_p
.returncode
> 0:
167 exit(EX_SOFTWARE
, git_err
)
168 if pipe
and pipe_p
.returncode
> 0:
169 exit(EX_SOFTWARE
, pipe_err
)
174 def run_npm(*args
, **kwargs
):
175 from mozprocess
import run_and_wait
177 def output_timeout_handler(proc
):
178 # In some cases, we wait longer for a mocha timeout
180 "Timed out after {} seconds of no output".format(kwargs
["output_timeout"])
184 npm
, _
= nodeutil
.find_npm_executable()
185 if kwargs
.get("env"):
186 env
= os
.environ
.copy()
187 env
.update(kwargs
["env"])
189 proc_kwargs
= {"output_timeout_handler": output_timeout_handler
}
190 for kw
in ["output_line_handler", "output_timeout"]:
192 proc_kwargs
[kw
] = kwargs
[kw
]
195 cmd
.extend(list(args
))
199 cwd
=kwargs
.get("cwd"),
204 post_wait_proc(p
, cmd
=npm
, exit_on_fail
=kwargs
.get("exit_on_fail", True))
209 def post_wait_proc(p
, cmd
=None, exit_on_fail
=True):
212 if exit_on_fail
and p
.returncode
> 0:
214 "%s: exit code %s" % (cmd
, p
.returncode
)
216 else "exit code %s" % p
.returncode
218 exit(p
.returncode
, msg
)
221 class MochaOutputHandler(object):
222 def __init__(self
, logger
, expected
):
223 self
.hook_re
= re
.compile('"before\b?.*" hook|"after\b?.*" hook')
227 self
.test_results
= OrderedDict()
228 self
.expected
= expected
229 self
.unexpected_skips
= set()
231 self
.has_unexpected
= False
232 self
.logger
.suite_start([], name
="puppeteer-tests")
236 "TERMINATED": "CRASH",
244 return self
.proc
and self
.proc
.pid
246 def __call__(self
, proc
, line
):
248 line
= line
.rstrip("\r\n")
251 if line
.startswith("[") and line
.endswith("]"):
252 event
= json
.loads(line
)
253 self
.process_event(event
)
257 self
.logger
.process_output(self
.pid
, line
, command
="npm")
259 def testExpectation(self
, testIdPattern
, expected_name
):
260 if testIdPattern
.find("*") == -1:
261 return expected_name
== testIdPattern
263 return re
.compile(re
.escape(testIdPattern
).replace(r
"\*", ".*")).search(
267 def process_event(self
, event
):
268 if isinstance(event
, list) and len(event
) > 1:
269 status
= self
.status_map
.get(event
[0])
270 test_start
= event
[0] == "test-start"
271 if not status
and not test_start
:
274 test_full_title
= test_info
.get("fullTitle", "")
275 test_name
= test_full_title
276 test_path
= test_info
.get("file", "")
277 test_file_name
= os
.path
.basename(test_path
).replace(".js", "")
278 test_err
= test_info
.get("err")
279 if status
== "FAIL" and test_err
:
280 if "timeout" in test_err
.lower():
282 if test_name
and test_path
:
283 test_name
= "{} ({})".format(test_name
, os
.path
.basename(test_path
))
284 # mocha hook failures are not tracked in metadata
285 if status
!= "PASS" and self
.hook_re
.search(test_name
):
286 self
.logger
.error("TEST-UNEXPECTED-ERROR %s" % (test_name
,))
289 self
.logger
.test_start(test_name
)
291 expected_name
= "[{}] {}".format(test_file_name
, test_full_title
)
292 expected_item
= next(
295 for expectation
in reversed(list(self
.expected
))
296 if self
.testExpectation(expectation
["testIdPattern"], expected_name
)
300 if expected_item
is None:
303 expected
= expected_item
["expectations"]
304 # mozlog doesn't really allow unexpected skip,
305 # so if a test is disabled just expect that and note the unexpected skip
306 # Also, mocha doesn't log test-start for skipped tests
308 self
.logger
.test_start(test_name
)
309 if self
.expected
and status
not in expected
:
310 self
.unexpected_skips
.add(test_name
)
312 known_intermittent
= expected
[1:]
313 expected_status
= expected
[0]
315 # check if we've seen a result for this test before this log line
316 result_recorded
= self
.test_results
.get(test_name
)
319 "Received a second status for {}: "
320 "first {}, now {}".format(test_name
, result_recorded
, status
)
322 # mocha intermittently logs an additional test result after the
323 # test has already timed out. Avoid recording this second status.
324 if result_recorded
!= "TIMEOUT":
325 self
.test_results
[test_name
] = status
326 if status
not in expected
:
327 self
.has_unexpected
= True
328 self
.logger
.test_end(
331 expected
=expected_status
,
332 known_intermittent
=known_intermittent
,
336 if self
.unexpected_skips
:
337 self
.has_unexpected
= True
338 for test_name
in self
.unexpected_skips
:
340 "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name
,)
342 self
.logger
.suite_end()
345 # tempfile.TemporaryDirectory missing from Python 2.7
346 class TemporaryDirectory(object):
348 self
.path
= tempfile
.mkdtemp()
352 return "<{} {!r}>".format(self
.__class
__.__name
__, self
.path
)
357 def __exit__(self
, exc
, value
, tb
):
364 if self
.path
and not self
._closed
:
365 shutil
.rmtree(self
.path
)
369 class PuppeteerRunner(MozbuildObject
):
370 def __init__(self
, *args
, **kwargs
):
371 super(PuppeteerRunner
, self
).__init
__(*args
, **kwargs
)
373 self
.remotedir
= os
.path
.join(self
.topsrcdir
, "remote")
374 self
.puppeteer_dir
= os
.path
.join(self
.remotedir
, "test", "puppeteer")
376 def run_test(self
, logger
, *tests
, **params
):
378 Runs Puppeteer unit tests with npm.
380 Possible optional test parameters:
383 Boolean to indicate whether to test Firefox with BiDi protocol.
385 Path for the browser binary to use. Defaults to the local
388 Boolean to indicate whether to activate Firefox' headless mode.
390 Dictionary of extra preferences to write to the profile,
391 before invoking npm. Overrides default preferences.
393 Boolean to indicate whether to enable WebRender compositor in Gecko.
397 with_bidi
= params
.get("bidi", False)
398 binary
= params
.get("binary") or self
.get_binary_path()
399 product
= params
.get("product", "firefox")
402 # Print browser process ouptut
404 # Checked by Puppeteer's custom mocha config
406 # Causes some tests to be skipped due to assumptions about install
407 "PUPPETEER_ALT_INSTALL": "1",
410 for k
, v
in params
.get("extra_launcher_options", {}).items():
411 extra_options
[k
] = json
.loads(v
)
413 # Override upstream defaults: no retries, shorter timeout
416 "./json-mocha-reporter.js",
425 env
["HEADLESS"] = str(params
.get("headless", False))
426 test_command
= "test:" + product
428 if product
== "firefox":
429 env
["BINARY"] = binary
430 env
["PUPPETEER_PRODUCT"] = "firefox"
431 env
["MOZ_WEBRENDER"] = "%d" % params
.get("enable_webrender", False)
433 env
["PUPPETEER_CACHE_DIR"] = os
.path
.join(
442 if with_bidi
is True:
443 test_command
= test_command
+ ":bidi"
444 elif env
["HEADLESS"] == "True":
445 test_command
= test_command
+ ":headless"
447 test_command
= test_command
+ ":headful"
449 command
= ["run", test_command
, "--"] + mocha_options
452 for k
, v
in params
.get("extra_prefs", {}).items():
453 print("Using extra preference: {}={}".format(k
, v
))
454 prefs
[k
] = mozprofile
.Preferences
.cast(v
)
457 extra_options
["extraPrefsFirefox"] = prefs
460 env
["EXTRA_LAUNCH_OPTIONS"] = json
.dumps(extra_options
)
462 expected_path
= os
.path
.join(
463 os
.path
.dirname(__file__
),
467 "TestExpectations.json",
469 if os
.path
.exists(expected_path
):
470 with
open(expected_path
) as f
:
471 expected_data
= json
.load(f
)
475 expected_platform
= platform
.uname().system
.lower()
476 if expected_platform
== "windows":
477 expected_platform
= "win32"
479 # Filter expectation data for the selected browser,
480 # headless or headful mode, the operating system,
481 # run in BiDi mode or not.
484 for expectation
in expected_data
485 if is_relevant_expectation(
486 expectation
, product
, with_bidi
, env
["HEADLESS"], expected_platform
490 output_handler
= MochaOutputHandler(logger
, expectations
)
491 return_code
= run_npm(
493 cwd
=self
.puppeteer_dir
,
495 output_line_handler
=output_handler
,
496 # Puppeteer unit tests don't always clean-up child processes in case of
497 # failure, so use an output_timeout as a fallback
502 output_handler
.after_end()
504 # Non-zero return codes are non-fatal for now since we have some
505 # issues with unresolved promises that shouldn't otherwise block
508 logger
.warning("npm exited with code %s" % return_code
)
510 if output_handler
.has_unexpected
:
511 exit(1, "Got unexpected results")
514 def create_parser_puppeteer():
515 p
= argparse
.ArgumentParser()
517 "--product", type=str, default
="firefox", choices
=["chrome", "firefox"]
522 help="Flag that indicates whether to test Firefox with BiDi protocol.",
527 help="Path to browser binary. Defaults to local Firefox build.",
532 help="Flag that indicates that tests run in a CI environment.",
538 dest
="disable_fission",
539 help="Disable Fission (site isolation) in Gecko.",
542 "--enable-webrender",
544 help="Enable the WebRender compositor in Gecko.",
547 "-z", "--headless", action
="store_true", help="Run browser in headless mode."
553 metavar
="<pref>=<value>",
554 help="Defines additional user preferences.",
559 dest
="extra_options",
560 metavar
="<option>=<value>",
561 help="Defines additional options for `puppeteer.launch`.",
568 help="Increase remote agent logging verbosity to include "
569 "debug level messages with -v, trace messages with -vv,"
570 "and to not truncate long trace messages with -vvv",
572 p
.add_argument("tests", nargs
="*")
573 mozlog
.commandline
.add_logging_group(p
)
577 def is_relevant_expectation(
578 expectation
, expected_product
, with_bidi
, is_headless
, expected_platform
580 parameters
= expectation
["parameters"]
582 if expected_product
== "firefox":
583 is_expected_product
= "chrome" not in parameters
585 is_expected_product
= "firefox" not in parameters
587 if with_bidi
is True:
588 is_expected_protocol
= "cdp" not in parameters
591 is_expected_protocol
= "webDriverBiDi" not in parameters
593 if is_headless
== "True":
594 is_expected_mode
= "headful" not in parameters
596 is_expected_mode
= "headless" not in parameters
598 is_expected_platform
= expected_platform
in expectation
["platforms"]
602 and is_expected_protocol
604 and is_expected_platform
611 description
="Run Puppeteer unit tests.",
612 parser
=create_parser_puppeteer
,
617 action
="store_false",
619 help="Do not install the Puppeteer package",
626 disable_fission
=False,
627 enable_webrender
=False,
637 logger
= mozlog
.commandline
.setup_logging(
638 "puppeteer-test", kwargs
, {"mach": sys
.stdout
}
641 # moztest calls this programmatically with test objects or manifests
642 if "test_objects" in kwargs
and tests
is not None:
643 logger
.error("Expected either 'test_objects' or 'tests'")
646 if product
!= "firefox" and extra_prefs
is not None:
647 logger
.error("User preferences are not recognized by %s" % product
)
650 if "test_objects" in kwargs
:
652 for test
in kwargs
["test_objects"]:
653 tests
.append(test
["path"])
656 for s
in extra_prefs
or []:
659 logger
.error("syntax error in --setpref={}".format(s
))
661 prefs
[kv
[0]] = kv
[1].strip()
664 for s
in extra_options
or []:
667 logger
.error("syntax error in --setopt={}".format(s
))
669 options
[kv
[0]] = kv
[1].strip()
671 prefs
.update({"fission.autostart": True})
673 prefs
.update({"fission.autostart": False})
676 prefs
["remote.log.level"] = "Debug"
678 prefs
["remote.log.level"] = "Trace"
680 prefs
["remote.log.truncate"] = False
683 install_puppeteer(command_context
, product
, ci
)
688 "headless": headless
,
689 "enable_webrender": enable_webrender
,
690 "extra_prefs": prefs
,
692 "extra_launcher_options": options
,
694 puppeteer
= command_context
._spawn
(PuppeteerRunner
)
696 return puppeteer
.run_test(logger
, *tests
, **params
)
697 except BinaryNotFoundException
as e
:
699 logger
.info(e
.help())
701 except Exception as e
:
705 def install_puppeteer(command_context
, product
, ci
):
709 puppeteer_dir
= os
.path
.join("remote", "test", "puppeteer")
710 puppeteer_dir_full_path
= os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
)
711 puppeteer_test_dir
= os
.path
.join(puppeteer_dir
, "test")
713 if product
== "chrome":
714 env
["PUPPETEER_CACHE_DIR"] = os
.path
.join(
715 command_context
.topobjdir
, "_tests", puppeteer_dir
, ".cache"
718 env
["PUPPETEER_SKIP_DOWNLOAD"] = "1"
724 cwd
=puppeteer_dir_full_path
,
729 command
= "ci" if ci
else "install"
730 run_npm(command
, cwd
=puppeteer_dir_full_path
, env
=env
)
734 cwd
=os
.path
.join(command_context
.topsrcdir
, puppeteer_test_dir
),
739 def exit(code
, error
=None):
740 if error
is not None:
741 if isinstance(error
, Exception):
744 traceback
.print_exc()
746 message
= str(error
).split("\n")[0].strip()
747 print("{}: {}".format(sys
.argv
[0], message
), file=sys
.stderr
)