1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
5 from __future__
import (
20 from collections
import OrderedDict
22 from six
import iteritems
24 from mach
.decorators
import (
31 from mozbuild
.base
import (
34 BinaryNotFoundException
,
36 from mozbuild
import nodeutil
47 # add node and npm from mozbuild to front of system path
48 npm
, _
= nodeutil
.find_npm_executable()
50 exit(EX_CONFIG
, "could not find npm executable")
51 path
= os
.path
.abspath(os
.path
.join(npm
, os
.pardir
))
52 os
.environ
["PATH"] = "{}:{}".format(path
, os
.environ
["PATH"])
56 class RemoteCommands(MachCommandBase
):
57 def __init__(self
, *args
, **kwargs
):
58 super(RemoteCommands
, self
).__init
__(*args
, **kwargs
)
59 self
.remotedir
= os
.path
.join(self
.topsrcdir
, "remote")
62 "remote", category
="misc", description
="Remote protocol related operations."
65 """The remote subcommands all relate to the remote protocol."""
66 self
._sub
_mach
(["help", "remote"])
70 "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
76 help="The (possibly remote) repository to clone from.",
82 help="The commit or tag object name to check out.",
89 help="Do not install the just-pulled Puppeteer package,",
91 def vendor_puppeteer(self
, repository
, commitish
, install
):
92 puppeteer_dir
= os
.path
.join(self
.remotedir
, "test", "puppeteer")
94 # Preserve our custom mocha reporter
96 os
.path
.join(puppeteer_dir
, "json-mocha-reporter.js"), self
.remotedir
98 shutil
.rmtree(puppeteer_dir
, ignore_errors
=True)
99 os
.makedirs(puppeteer_dir
)
100 with
TemporaryDirectory() as tmpdir
:
101 git("clone", "-q", repository
, tmpdir
)
102 git("checkout", commitish
, worktree
=tmpdir
)
108 "{}/".format(puppeteer_dir
),
112 # remove files which may interfere with git checkout of central
114 os
.remove(os
.path
.join(puppeteer_dir
, ".gitattributes"))
115 os
.remove(os
.path
.join(puppeteer_dir
, ".gitignore"))
119 unwanted_dirs
= ["experimental", "docs"]
121 for dir in unwanted_dirs
:
122 dir_path
= os
.path
.join(puppeteer_dir
, dir)
123 if os
.path
.isdir(dir_path
):
124 shutil
.rmtree(dir_path
)
127 os
.path
.join(self
.remotedir
, "json-mocha-reporter.js"), puppeteer_dir
135 "product": "Remote Protocol",
136 "component": "Agent",
140 "description": "Headless Chrome Node API",
142 "license": "Apache-2.0",
143 "release": commitish
,
146 with
open(os
.path
.join(puppeteer_dir
, "moz.yaml"), "w") as fh
:
150 default_flow_style
=False,
156 env
= {"PUPPETEER_SKIP_DOWNLOAD": "1"}
157 npm("install", cwd
=os
.path
.join(self
.topsrcdir
, puppeteer_dir
), env
=env
)
160 def git(*args
, **kwargs
):
162 if kwargs
.get("worktree"):
163 cmd
+= ("-C", kwargs
["worktree"])
166 pipe
= kwargs
.get("pipe")
167 git_p
= subprocess
.Popen(
169 env
={"GIT_CONFIG_NOSYSTEM": "1"},
170 stdout
=subprocess
.PIPE
,
171 stderr
=subprocess
.PIPE
,
175 pipe_p
= subprocess
.Popen(pipe
, stdin
=git_p
.stdout
, stderr
=subprocess
.PIPE
)
178 _
, pipe_err
= pipe_p
.communicate()
179 out
, git_err
= git_p
.communicate()
181 # use error from first program that failed
182 if git_p
.returncode
> 0:
183 exit(EX_SOFTWARE
, git_err
)
184 if pipe
and pipe_p
.returncode
> 0:
185 exit(EX_SOFTWARE
, pipe_err
)
190 def npm(*args
, **kwargs
):
191 from mozprocess
import processhandler
194 if kwargs
.get("env"):
195 env
= os
.environ
.copy()
196 env
.update(kwargs
["env"])
199 if "processOutputLine" in kwargs
:
200 proc_kwargs
["processOutputLine"] = kwargs
["processOutputLine"]
202 p
= processhandler
.ProcessHandler(
205 cwd
=kwargs
.get("cwd"),
207 universal_newlines
=True,
210 if not kwargs
.get("wait", True):
213 wait_proc(p
, cmd
="npm", exit_on_fail
=kwargs
.get("exit_on_fail", True))
218 def wait_proc(p
, cmd
=None, exit_on_fail
=True, output_timeout
=None):
220 p
.run(outputTimeout
=output_timeout
)
223 # In some cases, we wait longer for a mocha timeout
224 print("Timed out after {} seconds of no output".format(output_timeout
))
227 if exit_on_fail
and p
.returncode
> 0:
229 "%s: exit code %s" % (cmd
, p
.returncode
)
231 else "exit code %s" % p
.returncode
233 exit(p
.returncode
, msg
)
236 class MochaOutputHandler(object):
237 def __init__(self
, logger
, expected
):
238 self
.hook_re
= re
.compile('"before\b?.*" hook|"after\b?.*" hook')
242 self
.test_results
= OrderedDict()
243 self
.expected
= expected
244 self
.unexpected_skips
= set()
246 self
.has_unexpected
= False
247 self
.logger
.suite_start([], name
="puppeteer-tests")
251 "TERMINATED": "CRASH",
259 return self
.proc
and self
.proc
.pid
261 def __call__(self
, line
):
264 if line
.startswith("[") and line
.endswith("]"):
265 event
= json
.loads(line
)
266 self
.process_event(event
)
270 self
.logger
.process_output(self
.pid
, line
, command
="npm")
272 def process_event(self
, event
):
273 if isinstance(event
, list) and len(event
) > 1:
274 status
= self
.status_map
.get(event
[0])
275 test_start
= event
[0] == "test-start"
276 if not status
and not test_start
:
279 test_name
= test_info
.get("fullTitle", "")
280 test_path
= test_info
.get("file", "")
281 test_err
= test_info
.get("err")
282 if status
== "FAIL" and test_err
:
283 if "timeout" in test_err
.lower():
285 if test_name
and test_path
:
286 test_name
= "{} ({})".format(test_name
, os
.path
.basename(test_path
))
287 # mocha hook failures are not tracked in metadata
288 if status
!= "PASS" and self
.hook_re
.search(test_name
):
289 self
.logger
.error("TEST-UNEXPECTED-ERROR %s" % (test_name
,))
292 self
.logger
.test_start(test_name
)
294 expected
= self
.expected
.get(test_name
, ["PASS"])
295 # mozlog doesn't really allow unexpected skip,
296 # so if a test is disabled just expect that and note the unexpected skip
297 # Also, mocha doesn't log test-start for skipped tests
299 self
.logger
.test_start(test_name
)
300 if self
.expected
and status
not in expected
:
301 self
.unexpected_skips
.add(test_name
)
303 known_intermittent
= expected
[1:]
304 expected_status
= expected
[0]
306 # check if we've seen a result for this test before this log line
307 result_recorded
= self
.test_results
.get(test_name
)
310 "Received a second status for {}: "
311 "first {}, now {}".format(test_name
, result_recorded
, status
)
313 # mocha intermittently logs an additional test result after the
314 # test has already timed out. Avoid recording this second status.
315 if result_recorded
!= "TIMEOUT":
316 self
.test_results
[test_name
] = status
317 if status
not in expected
:
318 self
.has_unexpected
= True
319 self
.logger
.test_end(
322 expected
=expected_status
,
323 known_intermittent
=known_intermittent
,
326 def new_expected(self
):
327 new_expected
= OrderedDict()
328 for test_name
, status
in iteritems(self
.test_results
):
329 if test_name
not in self
.expected
:
330 new_status
= [status
]
332 if status
in self
.expected
[test_name
]:
333 new_status
= self
.expected
[test_name
]
335 new_status
= [status
]
336 new_expected
[test_name
] = new_status
339 def after_end(self
, subset
=False):
341 missing
= set(self
.expected
) - set(self
.test_results
)
342 extra
= set(self
.test_results
) - set(self
.expected
)
344 self
.has_unexpected
= True
345 for test_name
in missing
:
346 self
.logger
.error("TEST-UNEXPECTED-MISSING %s" % (test_name
,))
347 if self
.expected
and extra
:
348 self
.has_unexpected
= True
349 for test_name
in extra
:
351 "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name
,)
354 if self
.unexpected_skips
:
355 self
.has_unexpected
= True
356 for test_name
in self
.unexpected_skips
:
358 "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name
,)
360 self
.logger
.suite_end()
363 # tempfile.TemporaryDirectory missing from Python 2.7
364 class TemporaryDirectory(object):
366 self
.path
= tempfile
.mkdtemp()
370 return "<{} {!r}>".format(self
.__class
__.__name
__, self
.path
)
375 def __exit__(self
, exc
, value
, tb
):
382 if self
.path
and not self
._closed
:
383 shutil
.rmtree(self
.path
)
387 class PuppeteerRunner(MozbuildObject
):
388 def __init__(self
, *args
, **kwargs
):
389 super(PuppeteerRunner
, self
).__init
__(*args
, **kwargs
)
391 self
.remotedir
= os
.path
.join(self
.topsrcdir
, "remote")
392 self
.puppeteer_dir
= os
.path
.join(self
.remotedir
, "test", "puppeteer")
394 def run_test(self
, logger
, *tests
, **params
):
396 Runs Puppeteer unit tests with npm.
398 Possible optional test parameters:
401 Path for the browser binary to use. Defaults to the local
404 Boolean to indicate whether to activate Firefox' headless mode.
406 Dictionary of extra preferences to write to the profile,
407 before invoking npm. Overrides default preferences.
409 Boolean to indicate whether to enable WebRender compositor in Gecko.
411 Path to write the results json file
413 Indicates only a subset of tests are being run, so we should
414 skip the check for missing results
418 binary
= params
.get("binary") or self
.get_binary_path()
419 product
= params
.get("product", "firefox")
422 # Print browser process ouptut
424 # Checked by Puppeteer's custom mocha config
426 # Causes some tests to be skipped due to assumptions about install
427 "PUPPETEER_ALT_INSTALL": "1",
430 for k
, v
in params
.get("extra_launcher_options", {}).items():
431 extra_options
[k
] = json
.loads(v
)
433 # Override upstream defaults: no retries, shorter timeout
436 "./json-mocha-reporter.js",
444 if product
== "firefox":
445 env
["BINARY"] = binary
446 env
["PUPPETEER_PRODUCT"] = "firefox"
448 env
["MOZ_WEBRENDER"] = "%d" % params
.get("enable_webrender", False)
450 command
= ["run", "unit", "--"] + mocha_options
452 env
["HEADLESS"] = str(params
.get("headless", False))
455 for k
, v
in params
.get("extra_prefs", {}).items():
456 prefs
[k
] = mozprofile
.Preferences
.cast(v
)
459 extra_options
["extraPrefsFirefox"] = prefs
462 env
["EXTRA_LAUNCH_OPTIONS"] = json
.dumps(extra_options
)
464 expected_path
= os
.path
.join(
465 os
.path
.dirname(__file__
), "test", "puppeteer-expected.json"
467 if product
== "firefox" and os
.path
.exists(expected_path
):
468 with
open(expected_path
) as f
:
469 expected_data
= json
.load(f
)
473 output_handler
= MochaOutputHandler(logger
, expected_data
)
476 cwd
=self
.puppeteer_dir
,
478 processOutputLine
=output_handler
,
481 output_handler
.proc
= proc
483 # Puppeteer unit tests don't always clean-up child processes in case of
484 # failure, so use an output_timeout as a fallback
485 wait_proc(proc
, "npm", output_timeout
=60, exit_on_fail
=False)
487 output_handler
.after_end(params
.get("subset", False))
489 # Non-zero return codes are non-fatal for now since we have some
490 # issues with unresolved promises that shouldn't otherwise block
492 if proc
.returncode
!= 0:
493 logger
.warning("npm exited with code %s" % proc
.returncode
)
495 if params
["write_results"]:
496 with
open(params
["write_results"], "w") as f
:
498 output_handler
.new_expected(), f
, indent
=2, separators
=(",", ": ")
501 if output_handler
.has_unexpected
:
502 exit(1, "Got unexpected results")
505 def create_parser_puppeteer():
506 p
= argparse
.ArgumentParser()
508 "--product", type=str, default
="firefox", choices
=["chrome", "firefox"]
513 help="Path to browser binary. Defaults to local Firefox build.",
518 help="Flag that indicates that tests run in a CI environment.",
523 help="Enable Fission (site isolation) in Gecko.",
526 "--enable-webrender",
528 help="Enable the WebRender compositor in Gecko.",
531 "-z", "--headless", action
="store_true", help="Run browser in headless mode."
537 metavar
="<pref>=<value>",
538 help="Defines additional user preferences.",
543 dest
="extra_options",
544 metavar
="<option>=<value>",
545 help="Defines additional options for `puppeteer.launch`.",
552 help="Increase remote agent logging verbosity to include "
553 "debug level messages with -v, trace messages with -vv,"
554 "and to not truncate long trace messages with -vvv",
562 os
.path
.dirname(__file__
), "test", "puppeteer-expected.json"
564 help="Path to write updated results to (defaults to the "
565 "expectations file if the argument is provided but "
566 "no path is passed)",
572 help="Indicate that only a subset of the tests are running, "
573 "so checks for missing tests should be skipped",
575 p
.add_argument("tests", nargs
="*")
576 mozlog
.commandline
.add_logging_group(p
)
581 class PuppeteerTest(MachCommandBase
):
585 description
="Run Puppeteer unit tests.",
586 parser
=create_parser_puppeteer
,
592 enable_fission
=False,
593 enable_webrender
=False,
607 logger
= mozlog
.commandline
.setup_logging(
608 "puppeteer-test", kwargs
, {"mach": sys
.stdout
}
611 # moztest calls this programmatically with test objects or manifests
612 if "test_objects" in kwargs
and tests
is not None:
613 logger
.error("Expected either 'test_objects' or 'tests'")
616 if product
!= "firefox" and extra_prefs
is not None:
617 logger
.error("User preferences are not recognized by %s" % product
)
620 if "test_objects" in kwargs
:
622 for test
in kwargs
["test_objects"]:
623 tests
.append(test
["path"])
626 for s
in extra_prefs
or []:
629 logger
.error("syntax error in --setpref={}".format(s
))
631 prefs
[kv
[0]] = kv
[1].strip()
634 for s
in extra_options
or []:
637 logger
.error("syntax error in --setopt={}".format(s
))
639 options
[kv
[0]] = kv
[1].strip()
643 {"fission.autostart": True, "dom.serviceWorkers.parent_intercept": True}
647 prefs
["remote.log.level"] = "Debug"
649 prefs
["remote.log.level"] = "Trace"
651 prefs
["remote.log.truncate"] = False
653 self
.install_puppeteer(product
)
657 "headless": headless
,
658 "enable_webrender": enable_webrender
,
659 "extra_prefs": prefs
,
661 "extra_launcher_options": options
,
662 "write_results": write_results
,
665 puppeteer
= self
._spawn
(PuppeteerRunner
)
667 return puppeteer
.run_test(logger
, *tests
, **params
)
668 except BinaryNotFoundException
as e
:
670 logger
.info(e
.help())
672 except Exception as e
:
675 def install_puppeteer(self
, product
):
678 from mozversioncontrol
import get_repository_object
680 repo
= get_repository_object(self
.topsrcdir
)
681 puppeteer_dir
= os
.path
.join("remote", "test", "puppeteer")
682 changed_files
= False
683 for f
in repo
.get_changed_files():
684 if f
.startswith(puppeteer_dir
) and f
.endswith(".ts"):
688 if product
!= "chrome":
689 env
["PUPPETEER_SKIP_DOWNLOAD"] = "1"
690 lib_dir
= os
.path
.join(self
.topsrcdir
, puppeteer_dir
, "lib")
691 if changed_files
and os
.path
.isdir(lib_dir
):
692 # clobber lib to force `tsc compile` step
693 shutil
.rmtree(lib_dir
)
695 command
= "ci" if self
.ci
else "install"
696 npm(command
, cwd
=os
.path
.join(self
.topsrcdir
, puppeteer_dir
), env
=env
)
699 def exit(code
, error
=None):
700 if error
is not None:
701 if isinstance(error
, Exception):
704 traceback
.print_exc()
706 message
= str(error
).split("\n")[0].strip()
707 print("{}: {}".format(sys
.argv
[0], message
), file=sys
.stderr
)