1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
5 from __future__
import (
20 from collections
import OrderedDict
22 from six
import iteritems
24 from mach
.decorators
import (
30 from mozbuild
.base
import (
32 BinaryNotFoundException
,
34 from mozbuild
import nodeutil
45 # add node and npm from mozbuild to front of system path
46 npm
, _
= nodeutil
.find_npm_executable()
48 exit(EX_CONFIG
, "could not find npm executable")
49 path
= os
.path
.abspath(os
.path
.join(npm
, os
.pardir
))
50 os
.environ
["PATH"] = "{}:{}".format(path
, os
.environ
["PATH"])
53 def remotedir(command_context
):
54 return os
.path
.join(command_context
.topsrcdir
, "remote")
57 @Command("remote", category
="misc", description
="Remote protocol related operations.")
58 def remote(command_context
):
59 """The remote subcommands all relate to the remote protocol."""
60 command_context
._sub
_mach
(["help", "remote"])
65 "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
71 help="The (possibly remote) repository to clone from.",
77 help="The commit or tag object name to check out.",
84 help="Do not install the just-pulled Puppeteer package,",
86 def vendor_puppeteer(command_context
, repository
, commitish
, install
):
87 puppeteer_dir
= os
.path
.join(remotedir(command_context
), "test", "puppeteer")
89 # Preserve our custom mocha reporter
91 os
.path
.join(puppeteer_dir
, "json-mocha-reporter.js"),
92 remotedir(command_context
),
94 shutil
.rmtree(puppeteer_dir
, ignore_errors
=True)
95 os
.makedirs(puppeteer_dir
)
96 with
TemporaryDirectory() as tmpdir
:
97 git("clone", "-q", repository
, tmpdir
)
98 git("checkout", commitish
, worktree
=tmpdir
)
104 "{}/".format(puppeteer_dir
),
108 # remove files which may interfere with git checkout of central
110 os
.remove(os
.path
.join(puppeteer_dir
, ".gitattributes"))
111 os
.remove(os
.path
.join(puppeteer_dir
, ".gitignore"))
115 unwanted_dirs
= ["experimental", "docs"]
117 for dir in unwanted_dirs
:
118 dir_path
= os
.path
.join(puppeteer_dir
, dir)
119 if os
.path
.isdir(dir_path
):
120 shutil
.rmtree(dir_path
)
123 os
.path
.join(remotedir(command_context
), "json-mocha-reporter.js"),
132 "product": "Remote Protocol",
133 "component": "Agent",
137 "description": "Headless Chrome Node API",
139 "license": "Apache-2.0",
140 "release": commitish
,
143 with
open(os
.path
.join(puppeteer_dir
, "moz.yaml"), "w") as fh
:
147 default_flow_style
=False,
153 env
= {"PUPPETEER_SKIP_DOWNLOAD": "1"}
156 cwd
=os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
),
161 def git(*args
, **kwargs
):
163 if kwargs
.get("worktree"):
164 cmd
+= ("-C", kwargs
["worktree"])
167 pipe
= kwargs
.get("pipe")
168 git_p
= subprocess
.Popen(
170 env
={"GIT_CONFIG_NOSYSTEM": "1"},
171 stdout
=subprocess
.PIPE
,
172 stderr
=subprocess
.PIPE
,
176 pipe_p
= subprocess
.Popen(pipe
, stdin
=git_p
.stdout
, stderr
=subprocess
.PIPE
)
179 _
, pipe_err
= pipe_p
.communicate()
180 out
, git_err
= git_p
.communicate()
182 # use error from first program that failed
183 if git_p
.returncode
> 0:
184 exit(EX_SOFTWARE
, git_err
)
185 if pipe
and pipe_p
.returncode
> 0:
186 exit(EX_SOFTWARE
, pipe_err
)
191 def npm(*args
, **kwargs
):
192 from mozprocess
import processhandler
195 if kwargs
.get("env"):
196 env
= os
.environ
.copy()
197 env
.update(kwargs
["env"])
200 if "processOutputLine" in kwargs
:
201 proc_kwargs
["processOutputLine"] = kwargs
["processOutputLine"]
203 p
= processhandler
.ProcessHandler(
206 cwd
=kwargs
.get("cwd"),
208 universal_newlines
=True,
211 if not kwargs
.get("wait", True):
214 wait_proc(p
, cmd
="npm", exit_on_fail
=kwargs
.get("exit_on_fail", True))
219 def wait_proc(p
, cmd
=None, exit_on_fail
=True, output_timeout
=None):
221 p
.run(outputTimeout
=output_timeout
)
224 # In some cases, we wait longer for a mocha timeout
225 print("Timed out after {} seconds of no output".format(output_timeout
))
228 if exit_on_fail
and p
.returncode
> 0:
230 "%s: exit code %s" % (cmd
, p
.returncode
)
232 else "exit code %s" % p
.returncode
234 exit(p
.returncode
, msg
)
237 class MochaOutputHandler(object):
238 def __init__(self
, logger
, expected
):
239 self
.hook_re
= re
.compile('"before\b?.*" hook|"after\b?.*" hook')
243 self
.test_results
= OrderedDict()
244 self
.expected
= expected
245 self
.unexpected_skips
= set()
247 self
.has_unexpected
= False
248 self
.logger
.suite_start([], name
="puppeteer-tests")
252 "TERMINATED": "CRASH",
260 return self
.proc
and self
.proc
.pid
262 def __call__(self
, line
):
265 if line
.startswith("[") and line
.endswith("]"):
266 event
= json
.loads(line
)
267 self
.process_event(event
)
271 self
.logger
.process_output(self
.pid
, line
, command
="npm")
273 def process_event(self
, event
):
274 if isinstance(event
, list) and len(event
) > 1:
275 status
= self
.status_map
.get(event
[0])
276 test_start
= event
[0] == "test-start"
277 if not status
and not test_start
:
280 test_name
= test_info
.get("fullTitle", "")
281 test_path
= test_info
.get("file", "")
282 test_err
= test_info
.get("err")
283 if status
== "FAIL" and test_err
:
284 if "timeout" in test_err
.lower():
286 if test_name
and test_path
:
287 test_name
= "{} ({})".format(test_name
, os
.path
.basename(test_path
))
288 # mocha hook failures are not tracked in metadata
289 if status
!= "PASS" and self
.hook_re
.search(test_name
):
290 self
.logger
.error("TEST-UNEXPECTED-ERROR %s" % (test_name
,))
293 self
.logger
.test_start(test_name
)
295 expected
= self
.expected
.get(test_name
, ["PASS"])
296 # mozlog doesn't really allow unexpected skip,
297 # so if a test is disabled just expect that and note the unexpected skip
298 # Also, mocha doesn't log test-start for skipped tests
300 self
.logger
.test_start(test_name
)
301 if self
.expected
and status
not in expected
:
302 self
.unexpected_skips
.add(test_name
)
304 known_intermittent
= expected
[1:]
305 expected_status
= expected
[0]
307 # check if we've seen a result for this test before this log line
308 result_recorded
= self
.test_results
.get(test_name
)
311 "Received a second status for {}: "
312 "first {}, now {}".format(test_name
, result_recorded
, status
)
314 # mocha intermittently logs an additional test result after the
315 # test has already timed out. Avoid recording this second status.
316 if result_recorded
!= "TIMEOUT":
317 self
.test_results
[test_name
] = status
318 if status
not in expected
:
319 self
.has_unexpected
= True
320 self
.logger
.test_end(
323 expected
=expected_status
,
324 known_intermittent
=known_intermittent
,
327 def new_expected(self
):
328 new_expected
= OrderedDict()
329 for test_name
, status
in iteritems(self
.test_results
):
330 if test_name
not in self
.expected
:
331 new_status
= [status
]
333 if status
in self
.expected
[test_name
]:
334 new_status
= self
.expected
[test_name
]
336 new_status
= [status
]
337 new_expected
[test_name
] = new_status
340 def after_end(self
, subset
=False):
342 missing
= set(self
.expected
) - set(self
.test_results
)
343 extra
= set(self
.test_results
) - set(self
.expected
)
345 self
.has_unexpected
= True
346 for test_name
in missing
:
347 self
.logger
.error("TEST-UNEXPECTED-MISSING %s" % (test_name
,))
348 if self
.expected
and extra
:
349 self
.has_unexpected
= True
350 for test_name
in extra
:
352 "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name
,)
355 if self
.unexpected_skips
:
356 self
.has_unexpected
= True
357 for test_name
in self
.unexpected_skips
:
359 "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name
,)
361 self
.logger
.suite_end()
364 # tempfile.TemporaryDirectory missing from Python 2.7
365 class TemporaryDirectory(object):
367 self
.path
= tempfile
.mkdtemp()
371 return "<{} {!r}>".format(self
.__class
__.__name
__, self
.path
)
376 def __exit__(self
, exc
, value
, tb
):
383 if self
.path
and not self
._closed
:
384 shutil
.rmtree(self
.path
)
388 class PuppeteerRunner(MozbuildObject
):
389 def __init__(self
, *args
, **kwargs
):
390 super(PuppeteerRunner
, self
).__init
__(*args
, **kwargs
)
392 self
.remotedir
= os
.path
.join(self
.topsrcdir
, "remote")
393 self
.puppeteer_dir
= os
.path
.join(self
.remotedir
, "test", "puppeteer")
395 def run_test(self
, logger
, *tests
, **params
):
397 Runs Puppeteer unit tests with npm.
399 Possible optional test parameters:
402 Path for the browser binary to use. Defaults to the local
405 Boolean to indicate whether to activate Firefox' headless mode.
407 Dictionary of extra preferences to write to the profile,
408 before invoking npm. Overrides default preferences.
410 Boolean to indicate whether to enable WebRender compositor in Gecko.
412 Path to write the results json file
414 Indicates only a subset of tests are being run, so we should
415 skip the check for missing results
419 binary
= params
.get("binary") or self
.get_binary_path()
420 product
= params
.get("product", "firefox")
423 # Print browser process ouptut
425 # Checked by Puppeteer's custom mocha config
427 # Causes some tests to be skipped due to assumptions about install
428 "PUPPETEER_ALT_INSTALL": "1",
431 for k
, v
in params
.get("extra_launcher_options", {}).items():
432 extra_options
[k
] = json
.loads(v
)
434 # Override upstream defaults: no retries, shorter timeout
437 "./json-mocha-reporter.js",
445 if product
== "firefox":
446 env
["BINARY"] = binary
447 env
["PUPPETEER_PRODUCT"] = "firefox"
449 env
["MOZ_WEBRENDER"] = "%d" % params
.get("enable_webrender", False)
451 command
= ["run", "unit", "--"] + mocha_options
453 env
["HEADLESS"] = str(params
.get("headless", False))
456 for k
, v
in params
.get("extra_prefs", {}).items():
457 prefs
[k
] = mozprofile
.Preferences
.cast(v
)
460 extra_options
["extraPrefsFirefox"] = prefs
463 env
["EXTRA_LAUNCH_OPTIONS"] = json
.dumps(extra_options
)
465 expected_path
= os
.path
.join(
466 os
.path
.dirname(__file__
), "test", "puppeteer-expected.json"
468 if product
== "firefox" and os
.path
.exists(expected_path
):
469 with
open(expected_path
) as f
:
470 expected_data
= json
.load(f
)
474 output_handler
= MochaOutputHandler(logger
, expected_data
)
477 cwd
=self
.puppeteer_dir
,
479 processOutputLine
=output_handler
,
482 output_handler
.proc
= proc
484 # Puppeteer unit tests don't always clean-up child processes in case of
485 # failure, so use an output_timeout as a fallback
486 wait_proc(proc
, "npm", output_timeout
=60, exit_on_fail
=False)
488 output_handler
.after_end(params
.get("subset", False))
490 # Non-zero return codes are non-fatal for now since we have some
491 # issues with unresolved promises that shouldn't otherwise block
493 if proc
.returncode
!= 0:
494 logger
.warning("npm exited with code %s" % proc
.returncode
)
496 if params
["write_results"]:
497 with
open(params
["write_results"], "w") as f
:
499 output_handler
.new_expected(), f
, indent
=2, separators
=(",", ": ")
502 if output_handler
.has_unexpected
:
503 exit(1, "Got unexpected results")
506 def create_parser_puppeteer():
507 p
= argparse
.ArgumentParser()
509 "--product", type=str, default
="firefox", choices
=["chrome", "firefox"]
514 help="Path to browser binary. Defaults to local Firefox build.",
519 help="Flag that indicates that tests run in a CI environment.",
524 help="Enable Fission (site isolation) in Gecko.",
527 "--enable-webrender",
529 help="Enable the WebRender compositor in Gecko.",
532 "-z", "--headless", action
="store_true", help="Run browser in headless mode."
538 metavar
="<pref>=<value>",
539 help="Defines additional user preferences.",
544 dest
="extra_options",
545 metavar
="<option>=<value>",
546 help="Defines additional options for `puppeteer.launch`.",
553 help="Increase remote agent logging verbosity to include "
554 "debug level messages with -v, trace messages with -vv,"
555 "and to not truncate long trace messages with -vvv",
563 os
.path
.dirname(__file__
), "test", "puppeteer-expected.json"
565 help="Path to write updated results to (defaults to the "
566 "expectations file if the argument is provided but "
567 "no path is passed)",
573 help="Indicate that only a subset of the tests are running, "
574 "so checks for missing tests should be skipped",
576 p
.add_argument("tests", nargs
="*")
577 mozlog
.commandline
.add_logging_group(p
)
584 description
="Run Puppeteer unit tests.",
585 parser
=create_parser_puppeteer
,
591 enable_fission
=False,
592 enable_webrender
=False,
604 logger
= mozlog
.commandline
.setup_logging(
605 "puppeteer-test", kwargs
, {"mach": sys
.stdout
}
608 # moztest calls this programmatically with test objects or manifests
609 if "test_objects" in kwargs
and tests
is not None:
610 logger
.error("Expected either 'test_objects' or 'tests'")
613 if product
!= "firefox" and extra_prefs
is not None:
614 logger
.error("User preferences are not recognized by %s" % product
)
617 if "test_objects" in kwargs
:
619 for test
in kwargs
["test_objects"]:
620 tests
.append(test
["path"])
623 for s
in extra_prefs
or []:
626 logger
.error("syntax error in --setpref={}".format(s
))
628 prefs
[kv
[0]] = kv
[1].strip()
631 for s
in extra_options
or []:
634 logger
.error("syntax error in --setopt={}".format(s
))
636 options
[kv
[0]] = kv
[1].strip()
639 prefs
.update({"fission.autostart": True})
642 prefs
["remote.log.level"] = "Debug"
644 prefs
["remote.log.level"] = "Trace"
646 prefs
["remote.log.truncate"] = False
648 install_puppeteer(command_context
, product
, ci
)
652 "headless": headless
,
653 "enable_webrender": enable_webrender
,
654 "extra_prefs": prefs
,
656 "extra_launcher_options": options
,
657 "write_results": write_results
,
660 puppeteer
= command_context
._spawn
(PuppeteerRunner
)
662 return puppeteer
.run_test(logger
, *tests
, **params
)
663 except BinaryNotFoundException
as e
:
665 logger
.info(e
.help())
667 except Exception as e
:
671 def install_puppeteer(command_context
, product
, ci
):
674 from mozversioncontrol
import get_repository_object
676 repo
= get_repository_object(command_context
.topsrcdir
)
677 puppeteer_dir
= os
.path
.join("remote", "test", "puppeteer")
678 changed_files
= False
679 for f
in repo
.get_changed_files():
680 if f
.startswith(puppeteer_dir
) and f
.endswith(".ts"):
684 if product
!= "chrome":
685 env
["PUPPETEER_SKIP_DOWNLOAD"] = "1"
686 lib_dir
= os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
, "lib")
687 if changed_files
and os
.path
.isdir(lib_dir
):
688 # clobber lib to force `tsc compile` step
689 shutil
.rmtree(lib_dir
)
691 command
= "ci" if ci
else "install"
692 npm(command
, cwd
=os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
), env
=env
)
695 def exit(code
, error
=None):
696 if error
is not None:
697 if isinstance(error
, Exception):
700 traceback
.print_exc()
702 message
= str(error
).split("\n")[0].strip()
703 print("{}: {}".format(sys
.argv
[0], message
), file=sys
.stderr
)