1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
5 from __future__
import (
20 from collections
import OrderedDict
22 from six
import iteritems
24 from mach
.decorators
import (
31 from mozbuild
.base
import (
34 BinaryNotFoundException
,
36 from mozbuild
import nodeutil
47 # add node and npm from mozbuild to front of system path
48 npm
, _
= nodeutil
.find_npm_executable()
50 exit(EX_CONFIG
, "could not find npm executable")
51 path
= os
.path
.abspath(os
.path
.join(npm
, os
.pardir
))
52 os
.environ
["PATH"] = "{}:{}".format(path
, os
.environ
["PATH"])
56 class RemoteCommands(MachCommandBase
):
57 def __init__(self
, *args
, **kwargs
):
58 super(RemoteCommands
, self
).__init
__(*args
, **kwargs
)
59 self
.remotedir
= os
.path
.join(self
.topsrcdir
, "remote")
61 @Command("remote", category
="misc",
62 description
="Remote protocol related operations.")
64 """The remote subcommands all relate to the remote protocol."""
65 self
._sub
_mach
(['help', 'remote'])
68 @SubCommand("remote", "vendor-puppeteer",
69 "Pull in latest changes of the Puppeteer client.")
70 @CommandArgument("--repository",
73 help="The (possibly remote) repository to clone from.")
74 @CommandArgument("--commitish",
77 help="The commit or tag object name to check out.")
78 def vendor_puppeteer(self
, repository
, commitish
):
79 puppeteer_dir
= os
.path
.join(self
.remotedir
, "test", "puppeteer")
81 # Preserve our custom mocha reporter
82 shutil
.move(os
.path
.join(puppeteer_dir
, "json-mocha-reporter.js"), self
.remotedir
)
83 shutil
.rmtree(puppeteer_dir
, ignore_errors
=True)
84 os
.makedirs(puppeteer_dir
)
85 with
TemporaryDirectory() as tmpdir
:
86 git("clone", "-q", repository
, tmpdir
)
87 git("checkout", commitish
, worktree
=tmpdir
)
88 git("checkout-index", "-a", "-f",
89 "--prefix", "{}/".format(puppeteer_dir
),
92 # remove files which may interfere with git checkout of central
94 os
.remove(os
.path
.join(puppeteer_dir
, ".gitattributes"))
95 os
.remove(os
.path
.join(puppeteer_dir
, ".gitignore"))
99 experimental_dir
= os
.path
.join(puppeteer_dir
, "experimental")
100 if os
.path
.isdir(experimental_dir
):
101 shutil
.rmtree(experimental_dir
)
103 shutil
.move(os
.path
.join(self
.remotedir
, "json-mocha-reporter.js"), puppeteer_dir
)
109 "product": "Remote Protocol",
110 "component": "Agent",
114 "description": "Headless Chrome Node API",
116 "license": "Apache-2.0",
117 "release": commitish
,
120 with
open(os
.path
.join(puppeteer_dir
, "moz.yaml"), "w") as fh
:
121 yaml
.safe_dump(annotation
, fh
,
122 default_flow_style
=False,
127 def git(*args
, **kwargs
):
129 if kwargs
.get("worktree"):
130 cmd
+= ("-C", kwargs
["worktree"])
133 pipe
= kwargs
.get("pipe")
134 git_p
= subprocess
.Popen(cmd
,
135 env
={"GIT_CONFIG_NOSYSTEM": "1"},
136 stdout
=subprocess
.PIPE
,
137 stderr
=subprocess
.PIPE
)
140 pipe_p
= subprocess
.Popen(pipe
, stdin
=git_p
.stdout
, stderr
=subprocess
.PIPE
)
143 _
, pipe_err
= pipe_p
.communicate()
144 out
, git_err
= git_p
.communicate()
146 # use error from first program that failed
147 if git_p
.returncode
> 0:
148 exit(EX_SOFTWARE
, git_err
)
149 if pipe
and pipe_p
.returncode
> 0:
150 exit(EX_SOFTWARE
, pipe_err
)
155 def npm(*args
, **kwargs
):
156 from mozprocess
import processhandler
158 if kwargs
.get("env"):
159 env
= os
.environ
.copy()
160 env
.update(kwargs
["env"])
163 if "processOutputLine" in kwargs
:
164 proc_kwargs
["processOutputLine"] = kwargs
["processOutputLine"]
166 p
= processhandler
.ProcessHandler(cmd
="npm",
168 cwd
=kwargs
.get("cwd"),
170 universal_newlines
=True,
172 if not kwargs
.get("wait", True):
175 wait_proc(p
, cmd
="npm", exit_on_fail
=kwargs
.get("exit_on_fail", True))
180 def wait_proc(p
, cmd
=None, exit_on_fail
=True, output_timeout
=None):
182 p
.run(outputTimeout
=output_timeout
)
185 # In some cases, we wait longer for a mocha timeout
186 print("Timed out after {} seconds of no output".format(output_timeout
))
189 if exit_on_fail
and p
.returncode
> 0:
190 msg
= ("%s: exit code %s" % (cmd
, p
.returncode
) if cmd
191 else "exit code %s" % p
.returncode
)
192 exit(p
.returncode
, msg
)
195 class MochaOutputHandler(object):
196 def __init__(self
, logger
, expected
):
197 self
.hook_re
= re
.compile('"before\b?.*" hook|"after\b?.*" hook')
201 self
.test_results
= OrderedDict()
202 self
.expected
= expected
203 self
.unexpected_skips
= set()
205 self
.has_unexpected
= False
206 self
.logger
.suite_start([], name
="puppeteer-tests")
210 "TERMINATED": "CRASH",
218 return self
.proc
and self
.proc
.pid
220 def __call__(self
, line
):
223 if line
.startswith('[') and line
.endswith(']'):
224 event
= json
.loads(line
)
225 self
.process_event(event
)
229 self
.logger
.process_output(self
.pid
, line
, command
="npm")
231 def process_event(self
, event
):
232 if isinstance(event
, list) and len(event
) > 1:
233 status
= self
.status_map
.get(event
[0])
234 test_start
= event
[0] == 'test-start'
235 if not status
and not test_start
:
238 test_name
= test_info
.get("fullTitle", "")
239 test_path
= test_info
.get("file", "")
240 test_err
= test_info
.get("err")
241 if status
== "FAIL" and test_err
:
242 if "timeout" in test_err
.lower():
244 if test_name
and test_path
:
245 test_name
= "{} ({})".format(test_name
, os
.path
.basename(test_path
))
246 # mocha hook failures are not tracked in metadata
247 if status
!= "PASS" and self
.hook_re
.search(test_name
):
248 self
.logger
.error("TEST-UNEXPECTED-ERROR %s" % (test_name
,))
251 self
.logger
.test_start(test_name
)
253 expected
= self
.expected
.get(test_name
, ["PASS"])
254 # mozlog doesn't really allow unexpected skip,
255 # so if a test is disabled just expect that and note the unexpected skip
256 # Also, mocha doesn't log test-start for skipped tests
258 self
.logger
.test_start(test_name
)
259 if self
.expected
and status
not in expected
:
260 self
.unexpected_skips
.add(test_name
)
262 known_intermittent
= expected
[1:]
263 expected_status
= expected
[0]
265 self
.test_results
[test_name
] = status
266 self
.logger
.test_end(test_name
,
268 expected
=expected_status
,
269 known_intermittent
=known_intermittent
)
271 if status
not in expected
:
272 self
.has_unexpected
= True
274 def new_expected(self
):
275 new_expected
= OrderedDict()
276 for test_name
, status
in iteritems(self
.test_results
):
277 if test_name
not in self
.expected
:
278 new_status
= [status
]
280 if status
in self
.expected
[test_name
]:
281 new_status
= self
.expected
[test_name
]
283 new_status
= [status
]
284 new_expected
[test_name
] = new_status
287 def after_end(self
, subset
=False):
289 missing
= set(self
.expected
) - set(self
.test_results
)
290 extra
= set(self
.test_results
) - set(self
.expected
)
292 self
.has_unexpected
= True
293 for test_name
in missing
:
294 self
.logger
.error("TEST-UNEXPECTED-MISSING %s" % (test_name
,))
295 if self
.expected
and extra
:
296 self
.has_unexpected
= True
297 for test_name
in extra
:
298 self
.logger
.error("TEST-UNEXPECTED-MISSING Unknown new test %s" % (test_name
,))
300 if self
.unexpected_skips
:
301 self
.has_unexpected
= True
302 for test_name
in self
.unexpected_skips
:
303 self
.logger
.error("TEST-UNEXPECTED-MISSING Unexpected skipped %s" % (test_name
,))
304 self
.logger
.suite_end()
307 # tempfile.TemporaryDirectory missing from Python 2.7
308 class TemporaryDirectory(object):
310 self
.path
= tempfile
.mkdtemp()
314 return "<{} {!r}>".format(self
.__class
__.__name
__, self
.path
)
319 def __exit__(self
, exc
, value
, tb
):
326 if self
.path
and not self
._closed
:
327 shutil
.rmtree(self
.path
)
331 class PuppeteerRunner(MozbuildObject
):
332 def __init__(self
, *args
, **kwargs
):
333 super(PuppeteerRunner
, self
).__init
__(*args
, **kwargs
)
335 self
.remotedir
= os
.path
.join(self
.topsrcdir
, "remote")
336 self
.puppeteer_dir
= os
.path
.join(self
.remotedir
, "test", "puppeteer")
338 def run_test(self
, logger
, *tests
, **params
):
340 Runs Puppeteer unit tests with npm.
342 Possible optional test parameters:
345 Path for the browser binary to use. Defaults to the local
348 Boolean to indicate whether to activate Firefox' headless mode.
350 Dictionary of extra preferences to write to the profile,
351 before invoking npm. Overrides default preferences.
353 Path to write the results json file
355 Indicates only a subset of tests are being run, so we should
356 skip the check for missing results
360 binary
= params
.get("binary") or self
.get_binary_path()
361 product
= params
.get("product", "firefox")
364 # Print browser process ouptut
366 # Checked by Puppeteer's custom mocha config
368 # Causes some tests to be skipped due to assumptions about install
369 "PUPPETEER_ALT_INSTALL": "1"
372 for k
, v
in params
.get("extra_launcher_options", {}).items():
373 extra_options
[k
] = json
.loads(v
)
375 # Override upstream defaults: no retries, shorter timeout
377 "--reporter", "./json-mocha-reporter.js",
380 "--timeout", "15000",
383 if product
== "firefox":
384 env
["BINARY"] = binary
385 env
["PUPPETEER_PRODUCT"] = "firefox"
386 command
= ["run", "unit", "--"] + mocha_options
388 env
["HEADLESS"] = str(params
.get("headless", False))
391 for k
, v
in params
.get("extra_prefs", {}).items():
392 prefs
[k
] = mozprofile
.Preferences
.cast(v
)
395 extra_options
["extraPrefsFirefox"] = prefs
398 env
["EXTRA_LAUNCH_OPTIONS"] = json
.dumps(extra_options
)
400 expected_path
= os
.path
.join(os
.path
.dirname(__file__
),
401 "puppeteer-expected.json")
402 if product
== "firefox" and os
.path
.exists(expected_path
):
403 with
open(expected_path
) as f
:
404 expected_data
= json
.load(f
)
408 output_handler
= MochaOutputHandler(logger
, expected_data
)
409 proc
= npm(*command
, cwd
=self
.puppeteer_dir
, env
=env
,
410 processOutputLine
=output_handler
, wait
=False)
411 output_handler
.proc
= proc
413 # Puppeteer unit tests don't always clean-up child processes in case of
414 # failure, so use an output_timeout as a fallback
415 wait_proc(proc
, "npm", output_timeout
=60, exit_on_fail
=False)
417 output_handler
.after_end(params
.get("subset", False))
419 # Non-zero return codes are non-fatal for now since we have some
420 # issues with unresolved promises that shouldn't otherwise block
422 if proc
.returncode
!= 0:
423 logger
.warning("npm exited with code %s" % proc
.returncode
)
425 if params
["write_results"]:
426 with
open(params
["write_results"], "w") as f
:
427 json
.dump(output_handler
.new_expected(), f
, indent
=2,
428 separators
=(",", ": "))
430 if output_handler
.has_unexpected
:
431 exit(1, "Got unexpected results")
434 def create_parser_puppeteer():
435 p
= argparse
.ArgumentParser()
436 p
.add_argument("--product",
439 choices
=["chrome", "firefox"])
440 p
.add_argument("--binary",
442 help="Path to browser binary. Defaults to local Firefox build.")
443 p
.add_argument("--enable-fission",
445 help="Enable Fission (site isolation) in Gecko.")
446 p
.add_argument("-z", "--headless",
448 help="Run browser in headless mode.")
449 p
.add_argument("--setpref",
452 metavar
="<pref>=<value>",
453 help="Defines additional user preferences.")
454 p
.add_argument("--setopt",
456 dest
="extra_options",
457 metavar
="<option>=<value>",
458 help="Defines additional options for `puppeteer.launch`.")
463 help="Increase remote agent logging verbosity to include "
464 "debug level messages with -v, trace messages with -vv,"
465 "and to not truncate long trace messages with -vvv")
466 p
.add_argument("--write-results",
470 const
=os
.path
.join(os
.path
.dirname(__file__
),
471 "puppeteer-expected.json"),
472 help="Path to write updated results to (defaults to the "
473 "expectations file if the argument is provided but "
474 "no path is passed)")
475 p
.add_argument("--subset",
478 help="Indicate that only a subset of the tests are running, "
479 "so checks for missing tests should be skipped")
480 p
.add_argument("tests", nargs
="*")
481 mozlog
.commandline
.add_logging_group(p
)
486 class PuppeteerTest(MachCommandBase
):
487 @Command("puppeteer-test", category
="testing",
488 description
="Run Puppeteer unit tests.",
489 parser
=create_parser_puppeteer
)
490 def puppeteer_test(self
, binary
=None, enable_fission
=False, headless
=False,
491 extra_prefs
=None, extra_options
=None, verbosity
=0,
492 tests
=None, product
="firefox", write_results
=None,
493 subset
=False, **kwargs
):
495 logger
= mozlog
.commandline
.setup_logging("puppeteer-test",
497 {"mach": sys
.stdout
})
499 # moztest calls this programmatically with test objects or manifests
500 if "test_objects" in kwargs
and tests
is not None:
501 logger
.error("Expected either 'test_objects' or 'tests'")
504 if product
!= "firefox" and extra_prefs
is not None:
505 logger
.error("User preferences are not recognized by %s" % product
)
508 if "test_objects" in kwargs
:
510 for test
in kwargs
["test_objects"]:
511 tests
.append(test
["path"])
514 for s
in (extra_prefs
or []):
517 logger
.error("syntax error in --setpref={}".format(s
))
519 prefs
[kv
[0]] = kv
[1].strip()
522 for s
in (extra_options
or []):
525 logger
.error("syntax error in --setopt={}".format(s
))
527 options
[kv
[0]] = kv
[1].strip()
530 prefs
.update({"fission.autostart": True,
531 "dom.serviceWorkers.parent_intercept": True})
534 prefs
["remote.log.level"] = "Debug"
536 prefs
["remote.log.level"] = "Trace"
538 prefs
["remote.log.truncate"] = False
540 self
.install_puppeteer(product
)
542 params
= {"binary": binary
,
543 "headless": headless
,
544 "extra_prefs": prefs
,
546 "extra_launcher_options": options
,
547 "write_results": write_results
,
549 puppeteer
= self
._spawn
(PuppeteerRunner
)
551 return puppeteer
.run_test(logger
, *tests
, **params
)
552 except BinaryNotFoundException
as e
:
554 logger
.info(e
.help())
556 except Exception as e
:
559 def install_puppeteer(self
, product
):
562 from mozversioncontrol
import get_repository_object
563 repo
= get_repository_object(self
.topsrcdir
)
564 puppeteer_dir
= os
.path
.join("remote", "test", "puppeteer")
565 changed_files
= False
566 for f
in repo
.get_changed_files():
567 if f
.startswith(puppeteer_dir
) and f
.endswith(".ts"):
571 if product
!= "chrome":
572 env
["PUPPETEER_SKIP_DOWNLOAD"] = "1"
573 lib_dir
= os
.path
.join(self
.topsrcdir
, puppeteer_dir
, "lib")
574 if changed_files
and os
.path
.isdir(lib_dir
):
575 # clobber lib to force `tsc compile` step
576 shutil
.rmtree(lib_dir
)
578 cwd
=os
.path
.join(self
.topsrcdir
, puppeteer_dir
),
582 def exit(code
, error
=None):
583 if error
is not None:
584 if isinstance(error
, Exception):
586 traceback
.print_exc()
588 message
= str(error
).split("\n")[0].strip()
589 print("{}: {}".format(sys
.argv
[0], message
), file=sys
.stderr
)