1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
5 from __future__
import (
20 from collections
import OrderedDict
22 from six
import iteritems
24 from mach
.decorators
import (
30 from mozbuild
.base
import (
32 BinaryNotFoundException
,
34 from mozbuild
import nodeutil
45 # add node and npm from mozbuild to front of system path
46 npm
, _
= nodeutil
.find_npm_executable()
48 exit(EX_CONFIG
, "could not find npm executable")
49 path
= os
.path
.abspath(os
.path
.join(npm
, os
.pardir
))
50 os
.environ
["PATH"] = "{}{}{}".format(path
, os
.pathsep
, os
.environ
["PATH"])
53 def remotedir(command_context
):
54 return os
.path
.join(command_context
.topsrcdir
, "remote")
57 @Command("remote", category
="misc", description
="Remote protocol related operations.")
58 def remote(command_context
):
59 """The remote subcommands all relate to the remote protocol."""
60 command_context
._sub
_mach
(["help", "remote"])
65 "remote", "vendor-puppeteer", "Pull in latest changes of the Puppeteer client."
71 help="The (possibly remote) repository to clone from.",
77 help="The commit or tag object name to check out.",
84 help="Do not install the just-pulled Puppeteer package,",
86 def vendor_puppeteer(command_context
, repository
, commitish
, install
):
87 puppeteer_dir
= os
.path
.join(remotedir(command_context
), "test", "puppeteer")
89 # Preserve our custom mocha reporter
91 os
.path
.join(puppeteer_dir
, "json-mocha-reporter.js"),
92 remotedir(command_context
),
94 shutil
.rmtree(puppeteer_dir
, ignore_errors
=True)
95 os
.makedirs(puppeteer_dir
)
96 with
TemporaryDirectory() as tmpdir
:
97 git("clone", "-q", repository
, tmpdir
)
98 git("checkout", commitish
, worktree
=tmpdir
)
104 "{}/".format(puppeteer_dir
),
108 # remove files which may interfere with git checkout of central
110 os
.remove(os
.path
.join(puppeteer_dir
, ".gitattributes"))
111 os
.remove(os
.path
.join(puppeteer_dir
, ".gitignore"))
115 unwanted_dirs
= ["experimental", "docs"]
117 for dir in unwanted_dirs
:
118 dir_path
= os
.path
.join(puppeteer_dir
, dir)
119 if os
.path
.isdir(dir_path
):
120 shutil
.rmtree(dir_path
)
123 os
.path
.join(remotedir(command_context
), "json-mocha-reporter.js"),
132 "product": "Remote Protocol",
133 "component": "Agent",
137 "description": "Headless Chrome Node API",
139 "license": "Apache-2.0",
140 "release": commitish
,
143 with
open(os
.path
.join(puppeteer_dir
, "moz.yaml"), "w") as fh
:
147 default_flow_style
=False,
153 env
= {"PUPPETEER_SKIP_DOWNLOAD": "1"}
156 cwd
=os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
),
161 def git(*args
, **kwargs
):
163 if kwargs
.get("worktree"):
164 cmd
+= ("-C", kwargs
["worktree"])
167 pipe
= kwargs
.get("pipe")
168 git_p
= subprocess
.Popen(
170 env
={"GIT_CONFIG_NOSYSTEM": "1"},
171 stdout
=subprocess
.PIPE
,
172 stderr
=subprocess
.PIPE
,
176 pipe_p
= subprocess
.Popen(pipe
, stdin
=git_p
.stdout
, stderr
=subprocess
.PIPE
)
179 _
, pipe_err
= pipe_p
.communicate()
180 out
, git_err
= git_p
.communicate()
182 # use error from first program that failed
183 if git_p
.returncode
> 0:
184 exit(EX_SOFTWARE
, git_err
)
185 if pipe
and pipe_p
.returncode
> 0:
186 exit(EX_SOFTWARE
, pipe_err
)
191 def npm(*args
, **kwargs
):
192 from mozprocess
import processhandler
195 if kwargs
.get("env"):
196 env
= os
.environ
.copy()
197 env
.update(kwargs
["env"])
200 if "processOutputLine" in kwargs
:
201 proc_kwargs
["processOutputLine"] = kwargs
["processOutputLine"]
203 p
= processhandler
.ProcessHandler(
206 cwd
=kwargs
.get("cwd"),
208 universal_newlines
=True,
211 if not kwargs
.get("wait", True):
214 wait_proc(p
, cmd
="npm", exit_on_fail
=kwargs
.get("exit_on_fail", True))
219 def wait_proc(p
, cmd
=None, exit_on_fail
=True, output_timeout
=None):
221 p
.run(outputTimeout
=output_timeout
)
224 # In some cases, we wait longer for a mocha timeout
225 print("Timed out after {} seconds of no output".format(output_timeout
))
228 if exit_on_fail
and p
.returncode
> 0:
230 "%s: exit code %s" % (cmd
, p
.returncode
)
232 else "exit code %s" % p
.returncode
234 exit(p
.returncode
, msg
)
237 class MochaOutputHandler(object):
238 def __init__(self
, logger
, expected
):
239 self
.hook_re
= re
.compile('"before\b?.*" hook|"after\b?.*" hook')
243 self
.test_results
= OrderedDict()
244 self
.expected
= expected
245 self
.unexpected_skips
= set()
247 self
.has_unexpected
= False
248 self
.logger
.suite_start([], name
="puppeteer-tests")
252 "TERMINATED": "CRASH",
260 return self
.proc
and self
.proc
.pid
262 def __call__(self
, line
):
265 if line
.startswith("[") and line
.endswith("]"):
266 event
= json
.loads(line
)
267 self
.process_event(event
)
271 self
.logger
.process_output(self
.pid
, line
, command
="npm")
273 def process_event(self
, event
):
274 if isinstance(event
, list) and len(event
) > 1:
275 status
= self
.status_map
.get(event
[0])
276 test_start
= event
[0] == "test-start"
277 if not status
and not test_start
:
280 test_name
= test_info
.get("fullTitle", "")
281 test_path
= test_info
.get("file", "")
282 test_err
= test_info
.get("err")
283 if status
== "FAIL" and test_err
:
284 if "timeout" in test_err
.lower():
286 if test_name
and test_path
:
287 test_name
= "{} ({})".format(test_name
, os
.path
.basename(test_path
))
288 # mocha hook failures are not tracked in metadata
289 if status
!= "PASS" and self
.hook_re
.search(test_name
):
290 self
.logger
.error("TEST-UNEXPECTED-ERROR %s" % (test_name
,))
293 self
.logger
.test_start(test_name
)
295 expected
= self
.expected
.get(test_name
, ["PASS"])
296 # mozlog doesn't really allow unexpected skip,
297 # so if a test is disabled just expect that and note the unexpected skip
298 # Also, mocha doesn't log test-start for skipped tests
300 self
.logger
.test_start(test_name
)
301 if self
.expected
and status
not in expected
:
302 self
.unexpected_skips
.add(test_name
)
304 known_intermittent
= expected
[1:]
305 expected_status
= expected
[0]
307 # check if we've seen a result for this test before this log line
308 result_recorded
= self
.test_results
.get(test_name
)
311 "Received a second status for {}: "
312 "first {}, now {}".format(test_name
, result_recorded
, status
)
314 # mocha intermittently logs an additional test result after the
315 # test has already timed out. Avoid recording this second status.
316 if result_recorded
!= "TIMEOUT":
317 self
.test_results
[test_name
] = status
318 if status
not in expected
:
319 self
.has_unexpected
= True
320 self
.logger
.test_end(
323 expected
=expected_status
,
324 known_intermittent
=known_intermittent
,
327 def new_expected(self
):
328 new_expected
= OrderedDict()
329 for test_name
, status
in iteritems(self
.test_results
):
330 if test_name
not in self
.expected
:
331 new_status
= [status
]
333 if status
in self
.expected
[test_name
]:
334 new_status
= self
.expected
[test_name
]
336 new_status
= [status
]
337 new_expected
[test_name
] = new_status
340 def after_end(self
, subset
=False):
342 missing
= set(self
.expected
) - set(self
.test_results
)
343 extra
= set(self
.test_results
) - set(self
.expected
)
345 self
.has_unexpected
= True
346 for test_name
in missing
:
347 self
.logger
.error("TEST-UNEXPECTED-MISSING %s" % (test_name
,))
348 if self
.expected
and extra
:
349 self
.has_unexpected
= True
350 for test_name
in extra
:
352 "TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name
,)
355 if self
.unexpected_skips
:
356 self
.has_unexpected
= True
357 for test_name
in self
.unexpected_skips
:
359 "TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name
,)
361 self
.logger
.suite_end()
364 # tempfile.TemporaryDirectory missing from Python 2.7
365 class TemporaryDirectory(object):
367 self
.path
= tempfile
.mkdtemp()
371 return "<{} {!r}>".format(self
.__class
__.__name
__, self
.path
)
376 def __exit__(self
, exc
, value
, tb
):
383 if self
.path
and not self
._closed
:
384 shutil
.rmtree(self
.path
)
388 class PuppeteerRunner(MozbuildObject
):
389 def __init__(self
, *args
, **kwargs
):
390 super(PuppeteerRunner
, self
).__init
__(*args
, **kwargs
)
392 self
.remotedir
= os
.path
.join(self
.topsrcdir
, "remote")
393 self
.puppeteer_dir
= os
.path
.join(self
.remotedir
, "test", "puppeteer")
395 def run_test(self
, logger
, *tests
, **params
):
397 Runs Puppeteer unit tests with npm.
399 Possible optional test parameters:
402 Path for the browser binary to use. Defaults to the local
405 Boolean to indicate whether to activate Firefox' headless mode.
407 Dictionary of extra preferences to write to the profile,
408 before invoking npm. Overrides default preferences.
410 Boolean to indicate whether to enable WebRender compositor in Gecko.
412 Path to write the results json file
414 Indicates only a subset of tests are being run, so we should
415 skip the check for missing results
419 binary
= params
.get("binary") or self
.get_binary_path()
420 product
= params
.get("product", "firefox")
423 # Print browser process ouptut
425 # Checked by Puppeteer's custom mocha config
427 # Causes some tests to be skipped due to assumptions about install
428 "PUPPETEER_ALT_INSTALL": "1",
431 for k
, v
in params
.get("extra_launcher_options", {}).items():
432 extra_options
[k
] = json
.loads(v
)
434 # Override upstream defaults: no retries, shorter timeout
437 "./json-mocha-reporter.js",
445 if product
== "firefox":
446 env
["BINARY"] = binary
447 env
["PUPPETEER_PRODUCT"] = "firefox"
449 env
["MOZ_WEBRENDER"] = "%d" % params
.get("enable_webrender", False)
451 command
= ["run", "unit", "--"] + mocha_options
453 env
["HEADLESS"] = str(params
.get("headless", False))
456 for k
, v
in params
.get("extra_prefs", {}).items():
457 print("Using extra preference: {}={}".format(k
, v
))
458 prefs
[k
] = mozprofile
.Preferences
.cast(v
)
461 extra_options
["extraPrefsFirefox"] = prefs
464 env
["EXTRA_LAUNCH_OPTIONS"] = json
.dumps(extra_options
)
466 expected_path
= os
.path
.join(
467 os
.path
.dirname(__file__
), "test", "puppeteer-expected.json"
469 if product
== "firefox" and os
.path
.exists(expected_path
):
470 with
open(expected_path
) as f
:
471 expected_data
= json
.load(f
)
475 output_handler
= MochaOutputHandler(logger
, expected_data
)
478 cwd
=self
.puppeteer_dir
,
480 processOutputLine
=output_handler
,
483 output_handler
.proc
= proc
485 # Puppeteer unit tests don't always clean-up child processes in case of
486 # failure, so use an output_timeout as a fallback
487 wait_proc(proc
, "npm", output_timeout
=60, exit_on_fail
=False)
489 output_handler
.after_end(params
.get("subset", False))
491 # Non-zero return codes are non-fatal for now since we have some
492 # issues with unresolved promises that shouldn't otherwise block
494 if proc
.returncode
!= 0:
495 logger
.warning("npm exited with code %s" % proc
.returncode
)
497 if params
["write_results"]:
498 with
open(params
["write_results"], "w") as f
:
500 output_handler
.new_expected(), f
, indent
=2, separators
=(",", ": ")
503 if output_handler
.has_unexpected
:
504 exit(1, "Got unexpected results")
507 def create_parser_puppeteer():
508 p
= argparse
.ArgumentParser()
510 "--product", type=str, default
="firefox", choices
=["chrome", "firefox"]
515 help="Path to browser binary. Defaults to local Firefox build.",
520 help="Flag that indicates that tests run in a CI environment.",
525 help="Enable Fission (site isolation) in Gecko.",
528 "--enable-webrender",
530 help="Enable the WebRender compositor in Gecko.",
533 "-z", "--headless", action
="store_true", help="Run browser in headless mode."
539 metavar
="<pref>=<value>",
540 help="Defines additional user preferences.",
545 dest
="extra_options",
546 metavar
="<option>=<value>",
547 help="Defines additional options for `puppeteer.launch`.",
554 help="Increase remote agent logging verbosity to include "
555 "debug level messages with -v, trace messages with -vv,"
556 "and to not truncate long trace messages with -vvv",
564 os
.path
.dirname(__file__
), "test", "puppeteer-expected.json"
566 help="Path to write updated results to (defaults to the "
567 "expectations file if the argument is provided but "
568 "no path is passed)",
574 help="Indicate that only a subset of the tests are running, "
575 "so checks for missing tests should be skipped",
577 p
.add_argument("tests", nargs
="*")
578 mozlog
.commandline
.add_logging_group(p
)
585 description
="Run Puppeteer unit tests.",
586 parser
=create_parser_puppeteer
,
591 action
="store_false",
593 help="Do not install the Puppeteer package",
599 enable_fission
=False,
600 enable_webrender
=False,
613 logger
= mozlog
.commandline
.setup_logging(
614 "puppeteer-test", kwargs
, {"mach": sys
.stdout
}
617 # moztest calls this programmatically with test objects or manifests
618 if "test_objects" in kwargs
and tests
is not None:
619 logger
.error("Expected either 'test_objects' or 'tests'")
622 if product
!= "firefox" and extra_prefs
is not None:
623 logger
.error("User preferences are not recognized by %s" % product
)
626 if "test_objects" in kwargs
:
628 for test
in kwargs
["test_objects"]:
629 tests
.append(test
["path"])
632 for s
in extra_prefs
or []:
635 logger
.error("syntax error in --setpref={}".format(s
))
637 prefs
[kv
[0]] = kv
[1].strip()
640 for s
in extra_options
or []:
643 logger
.error("syntax error in --setopt={}".format(s
))
645 options
[kv
[0]] = kv
[1].strip()
648 prefs
.update({"fission.autostart": True})
650 prefs
.update({"fission.autostart": False})
653 prefs
["remote.log.level"] = "Debug"
655 prefs
["remote.log.level"] = "Trace"
657 prefs
["remote.log.truncate"] = False
660 install_puppeteer(command_context
, product
, ci
)
664 "headless": headless
,
665 "enable_webrender": enable_webrender
,
666 "extra_prefs": prefs
,
668 "extra_launcher_options": options
,
669 "write_results": write_results
,
672 puppeteer
= command_context
._spawn
(PuppeteerRunner
)
674 return puppeteer
.run_test(logger
, *tests
, **params
)
675 except BinaryNotFoundException
as e
:
677 logger
.info(e
.help())
679 except Exception as e
:
683 def install_puppeteer(command_context
, product
, ci
):
686 from mozversioncontrol
import get_repository_object
688 repo
= get_repository_object(command_context
.topsrcdir
)
689 puppeteer_dir
= os
.path
.join("remote", "test", "puppeteer")
690 changed_files
= False
691 for f
in repo
.get_changed_files():
692 if f
.startswith(puppeteer_dir
) and f
.endswith(".ts"):
696 if product
!= "chrome":
697 env
["PUPPETEER_SKIP_DOWNLOAD"] = "1"
698 lib_dir
= os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
, "lib")
699 if changed_files
and os
.path
.isdir(lib_dir
):
700 # clobber lib to force `tsc compile` step
701 shutil
.rmtree(lib_dir
)
703 command
= "ci" if ci
else "install"
704 npm(command
, cwd
=os
.path
.join(command_context
.topsrcdir
, puppeteer_dir
), env
=env
)
707 def exit(code
, error
=None):
708 if error
is not None:
709 if isinstance(error
, Exception):
712 traceback
.print_exc()
714 message
= str(error
).split("\n")[0].strip()
715 print("{}: {}".format(sys
.argv
[0], message
), file=sys
.stderr
)