1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 # Integrates the web-platform-tests test runner with mach.
7 from __future__
import absolute_import
, unicode_literals
, print_function
12 from six
import iteritems
14 from mozbuild
.base
import (
15 MachCommandConditions
as conditions
,
19 from mach
.decorators
import (
23 from mach_commands_base
import WebPlatformTestsRunner
, create_parser_wpt
26 class WebPlatformTestsRunnerSetup(MozbuildObject
):
27 default_log_type
= "mach"
29 def __init__(self
, *args
, **kwargs
):
30 super(WebPlatformTestsRunnerSetup
, self
).__init
__(*args
, **kwargs
)
31 self
._here
= os
.path
.join(self
.topsrcdir
, "testing", "web-platform")
32 kwargs
["tests_root"] = os
.path
.join(self
._here
, "tests")
33 sys
.path
.insert(0, kwargs
["tests_root"])
34 build_path
= os
.path
.join(self
.topobjdir
, "build")
35 if build_path
not in sys
.path
:
36 sys
.path
.append(build_path
)
38 def kwargs_common(self
, kwargs
):
39 """Setup kwargs relevant for all browser products"""
41 tests_src_path
= os
.path
.join(self
._here
, "tests")
44 kwargs
["product"] in {"firefox", "firefox_android"}
45 and kwargs
["specialpowers_path"] is None
47 kwargs
["specialpowers_path"] = os
.path
.join(
48 self
.distdir
, "xpi-stage", "specialpowers@mozilla.org.xpi"
51 if kwargs
["product"] == "firefox_android":
52 # package_name may be different in the future
53 package_name
= kwargs
["package_name"]
57 ] = package_name
= "org.mozilla.geckoview.test_runner"
59 # Note that this import may fail in non-firefox-for-android trees
60 from mozrunner
.devices
.android_device
import (
62 verify_android_device
,
66 kwargs
["adb_binary"] = get_adb_path(self
)
68 InstallIntent
.NO
if kwargs
.pop("no_install") else InstallIntent
.YES
70 verify_android_device(
71 self
, install
=install
, verbose
=False, xre
=True, app
=package_name
74 if kwargs
["certutil_binary"] is None:
75 kwargs
["certutil_binary"] = os
.path
.join(
76 os
.environ
.get("MOZ_HOST_BIN"), "certutil"
79 if kwargs
["install_fonts"] is None:
80 kwargs
["install_fonts"] = True
82 if not kwargs
["device_serial"]:
83 kwargs
["device_serial"] = ["emulator-5554"]
85 if kwargs
["config"] is None:
86 kwargs
["config"] = os
.path
.join(
87 self
.topobjdir
, "_tests", "web-platform", "wptrunner.local.ini"
91 kwargs
["exclude"] is None
92 and kwargs
["include"] is None
93 and not sys
.platform
.startswith("linux")
95 kwargs
["exclude"] = ["css"]
97 if kwargs
["ssl_type"] in (None, "pregenerated"):
98 cert_root
= os
.path
.join(tests_src_path
, "tools", "certs")
99 if kwargs
["ca_cert_path"] is None:
100 kwargs
["ca_cert_path"] = os
.path
.join(cert_root
, "cacert.pem")
102 if kwargs
["host_key_path"] is None:
103 kwargs
["host_key_path"] = os
.path
.join(
104 cert_root
, "web-platform.test.key"
107 if kwargs
["host_cert_path"] is None:
108 kwargs
["host_cert_path"] = os
.path
.join(
109 cert_root
, "web-platform.test.pem"
112 if kwargs
["reftest_screenshot"] is None:
113 kwargs
["reftest_screenshot"] = "fail"
115 kwargs
["capture_stdio"] = True
119 def kwargs_firefox(self
, kwargs
):
120 """Setup kwargs specific to running Firefox and other gecko browsers"""
122 from wptrunner
import wptcommandline
124 kwargs
= self
.kwargs_common(kwargs
)
126 if kwargs
["binary"] is None:
127 kwargs
["binary"] = self
.get_binary_path()
129 if kwargs
["certutil_binary"] is None:
130 kwargs
["certutil_binary"] = self
.get_binary_path("certutil")
132 if kwargs
["webdriver_binary"] is None:
133 kwargs
["webdriver_binary"] = self
.get_binary_path(
134 "geckodriver", validate_exists
=False
137 if kwargs
["install_fonts"] is None:
138 kwargs
["install_fonts"] = True
141 kwargs
["install_fonts"]
142 and mozinfo
.info
["os"] == "win"
143 and mozinfo
.info
["os_version"] == "6.1"
145 # On Windows 7 --install-fonts fails, so fall back to a Firefox-specific codepath
146 self
.setup_fonts_firefox()
147 kwargs
["install_fonts"] = False
149 if kwargs
["preload_browser"] is None:
150 kwargs
["preload_browser"] = False
152 if kwargs
["prefs_root"] is None:
153 kwargs
["prefs_root"] = os
.path
.join(self
.topsrcdir
, "testing", "profiles")
155 if kwargs
["stackfix_dir"] is None:
156 kwargs
["stackfix_dir"] = self
.bindir
158 kwargs
= wptcommandline
.check_args(kwargs
)
162 def kwargs_wptrun(self
, kwargs
):
163 """Setup kwargs for wpt-run which is only used for non-gecko browser products"""
164 from tools
.wpt
import run
, virtualenv
166 kwargs
= self
.kwargs_common(kwargs
)
168 # Add additional kwargs consumed by the run frontend. Currently we don't
169 # have a way to set these through mach
170 kwargs
["channel"] = None
171 kwargs
["prompt"] = True
172 kwargs
["install_browser"] = False
173 kwargs
["install_webdriver"] = None
174 kwargs
["affected"] = None
177 # We do this explicitly to avoid calling pip with options that aren't
178 # supported in the in-tree version
179 wptrunner_path
= os
.path
.join(self
._here
, "tests", "tools", "wptrunner")
180 browser_cls
= run
.product_setup
[kwargs
["product"]].browser_cls
181 requirements
= ["requirements.txt"]
182 if hasattr(browser_cls
, "requirements"):
183 requirements
.append(browser_cls
.requirements
)
185 for filename
in requirements
:
186 path
= os
.path
.join(wptrunner_path
, filename
)
187 if os
.path
.exists(path
):
188 self
.virtualenv_manager
.install_pip_requirements(
189 path
, require_hashes
=False
192 venv
= virtualenv
.Virtualenv(
193 self
.virtualenv_manager
.virtualenv_root
, skip_virtualenv_setup
=True
196 kwargs
= run
.setup_wptrunner(venv
, **kwargs
)
197 except run
.WptrunError
as e
:
198 print(e
, file=sys
.stderr
)
201 # This is kind of a hack; override the metadata paths so we don't use
202 # gecko metadata for non-gecko products
203 for key
, value
in list(iteritems(kwargs
["test_paths"])):
204 meta_suffix
= key
.strip("/")
205 meta_dir
= os
.path
.join(
206 self
._here
, "products", kwargs
["product"], meta_suffix
208 value
["metadata_path"] = meta_dir
209 if not os
.path
.exists(meta_dir
):
210 os
.makedirs(meta_dir
)
213 def setup_fonts_firefox(self
):
214 # Ensure the Ahem font is available
215 if not sys
.platform
.startswith("darwin"):
216 font_path
= os
.path
.join(os
.path
.dirname(self
.get_binary_path()), "fonts")
218 font_path
= os
.path
.join(
219 os
.path
.dirname(self
.get_binary_path()),
225 ahem_src
= os
.path
.join(
226 self
.topsrcdir
, "testing", "web-platform", "tests", "fonts", "Ahem.ttf"
228 ahem_dest
= os
.path
.join(font_path
, "Ahem.ttf")
229 if not os
.path
.exists(ahem_dest
) and os
.path
.exists(ahem_src
):
230 with
open(ahem_src
, "rb") as src
, open(ahem_dest
, "wb") as dest
:
231 dest
.write(src
.read())
234 class WebPlatformTestsServeRunner(MozbuildObject
):
235 def run(self
, **kwargs
):
238 os
.path
.abspath(os
.path
.join(os
.path
.dirname(__file__
), "tests", "tools")),
240 from serve
import serve
241 from wptrunner
import wptcommandline
242 import manifestupdate
246 logger
= logging
.getLogger("web-platform-tests")
248 src_root
= self
.topsrcdir
249 obj_root
= self
.topobjdir
250 src_wpt_dir
= os
.path
.join(src_root
, "testing", "web-platform")
252 config_path
= manifestupdate
.generate_config(
256 os
.path
.join(obj_root
, "_tests", "web-platform"),
260 test_paths
= wptcommandline
.get_test_paths(
261 wptcommandline
.config
.read(config_path
)
264 def get_route_builder(*args
, **kwargs
):
265 route_builder
= serve
.get_route_builder(*args
, **kwargs
)
267 for url_base
, paths
in iteritems(test_paths
):
269 route_builder
.add_mount_point(url_base
, paths
["tests_path"])
273 return 0 if serve
.run(route_builder
=get_route_builder
, **kwargs
) else 1
276 class WebPlatformTestsUpdater(MozbuildObject
):
277 """Update web platform tests."""
279 def setup_logging(self
, **kwargs
):
282 return update
.setup_logging(kwargs
, {"mach": sys
.stdout
})
284 def update_manifest(self
, logger
, **kwargs
):
285 import manifestupdate
287 return manifestupdate
.run(
288 logger
=logger
, src_root
=self
.topsrcdir
, obj_root
=self
.topobjdir
, **kwargs
291 def run_update(self
, logger
, **kwargs
):
293 from update
import updatecommandline
295 self
.update_manifest(logger
, **kwargs
)
297 if kwargs
["config"] is None:
298 kwargs
["config"] = os
.path
.join(
299 self
.topobjdir
, "_tests", "web-platform", "wptrunner.local.ini"
301 if kwargs
["product"] is None:
302 kwargs
["product"] = "firefox"
304 kwargs
["store_state"] = False
306 kwargs
= updatecommandline
.check_args(kwargs
)
309 update
.run_update(logger
, **kwargs
)
313 traceback
.print_exc()
316 class WebPlatformTestsUnittestRunner(MozbuildObject
):
317 def run(self
, **kwargs
):
318 import unittestrunner
320 return unittestrunner
.run(self
.topsrcdir
, **kwargs
)
323 class WebPlatformTestsTestPathsRunner(MozbuildObject
):
324 """Update web platform tests."""
326 def run(self
, **kwargs
):
329 os
.path
.abspath(os
.path
.join(os
.path
.dirname(__file__
), "tests", "tools")),
331 from wptrunner
import wptcommandline
332 from manifest
import testpaths
333 import manifestupdate
337 logger
= logging
.getLogger("web-platform-tests")
339 src_root
= self
.topsrcdir
340 obj_root
= self
.topobjdir
341 src_wpt_dir
= os
.path
.join(src_root
, "testing", "web-platform")
343 config_path
= manifestupdate
.generate_config(
347 os
.path
.join(obj_root
, "_tests", "web-platform"),
351 test_paths
= wptcommandline
.get_test_paths(
352 wptcommandline
.config
.read(config_path
)
355 for url_base
, paths
in iteritems(test_paths
):
356 if "manifest_path" not in paths
:
357 paths
["manifest_path"] = os
.path
.join(
358 paths
["metadata_path"], "MANIFEST.json"
362 path
=paths
["manifest_path"],
364 tests_root
=paths
["tests_path"],
365 update
=kwargs
["update"],
366 rebuild
=kwargs
["rebuild"],
368 cache_root
=kwargs
["cache_root"],
369 test_ids
=kwargs
["test_ids"],
372 testpaths
.write_output(results
, kwargs
["json"])
376 class WebPlatformTestsFissionRegressionsRunner(MozbuildObject
):
377 def run(self
, **kwargs
):
379 import fissionregressions
381 src_root
= self
.topsrcdir
382 obj_root
= self
.topobjdir
383 logger
= mozlog
.structuredlog
.StructuredLogger("web-platform-tests")
386 return fissionregressions
.run(logger
, src_root
, obj_root
, **kwargs
)
391 traceback
.print_exc()
395 def create_parser_update():
396 from update
import updatecommandline
398 return updatecommandline
.create_parser()
401 def create_parser_manifest_update():
402 import manifestupdate
404 return manifestupdate
.create_parser()
407 def create_parser_metadata_summary():
410 return metasummary
.create_parser()
413 def create_parser_metadata_merge():
416 return metamerge
.get_parser()
419 def create_parser_serve():
421 0, os
.path
.abspath(os
.path
.join(os
.path
.dirname(__file__
), "tests", "tools"))
425 return serve
.serve
.get_parser()
428 def create_parser_unittest():
429 import unittestrunner
431 return unittestrunner
.get_parser()
434 def create_parser_fission_regressions():
435 import fissionregressions
437 return fissionregressions
.get_parser()
440 def create_parser_testpaths():
442 from mach
.util
import get_state_dir
444 parser
= argparse
.ArgumentParser()
448 action
="store_false",
450 help="Don't update manifest before continuing",
457 help="Force a full rebuild of the manifest.",
462 default
=os
.path
.join(get_state_dir(), "cache", "wpt"),
463 help="Path in which to store any caches (default <tests_root>/.wptcache/)",
466 "test_ids", action
="store", nargs
="+", help="Test ids for which to get paths"
469 "--json", action
="store_true", default
=False, help="Output as JSON"
475 "web-platform-tests",
477 conditions
=[conditions
.is_firefox_or_android
],
478 description
="Run web-platform-tests.",
479 parser
=create_parser_wpt
,
480 virtualenv_name
="wpt",
482 def run_web_platform_tests(command_context
, **params
):
483 if params
["product"] is None:
484 if conditions
.is_android(command_context
):
485 params
["product"] = "firefox_android"
487 params
["product"] = "firefox"
488 if "test_objects" in params
:
491 for item
in params
["test_objects"]:
492 include
.append(item
["name"])
493 test_types
.add(item
.get("subsuite"))
494 if None not in test_types
:
495 params
["test_types"] = list(test_types
)
496 params
["include"] = include
497 del params
["test_objects"]
498 if params
.get("debugger", None):
501 if not mozdebug
.get_debugger_info(params
.get("debugger")):
504 wpt_setup
= command_context
._spawn
(WebPlatformTestsRunnerSetup
)
505 wpt_setup
._mach
_context
= command_context
._mach
_context
506 wpt_runner
= WebPlatformTestsRunner(wpt_setup
)
508 logger
= wpt_runner
.setup_logging(**params
)
509 # wptrunner already handles setting any log parameter from
510 # mach test to the logger, so it's OK to remove that argument now
515 conditions
.is_android(command_context
)
516 and params
["product"] != "firefox_android"
518 logger
.warning("Must specify --product=firefox_android in Android environment.")
520 return wpt_runner
.run(logger
, **params
)
526 conditions
=[conditions
.is_firefox_or_android
],
527 description
="Run web-platform-tests.",
528 parser
=create_parser_wpt
,
530 def run_wpt(command_context
, **params
):
531 return run_web_platform_tests(command_context
, **params
)
535 "web-platform-tests-update",
537 description
="Update web-platform-test metadata.",
538 parser
=create_parser_update
,
539 virtualenv_name
="wpt",
541 def update_web_platform_tests(command_context
, **params
):
542 wpt_updater
= command_context
._spawn
(WebPlatformTestsUpdater
)
543 logger
= wpt_updater
.setup_logging(**params
)
544 return wpt_updater
.run_update(logger
, **params
)
550 description
="Update web-platform-test metadata.",
551 parser
=create_parser_update
,
553 def update_wpt(command_context
, **params
):
554 return update_web_platform_tests(command_context
, **params
)
558 "wpt-manifest-update",
560 description
="Update web-platform-test manifests.",
561 parser
=create_parser_manifest_update
,
562 virtualenv_name
="wpt",
564 def wpt_manifest_update(command_context
, **params
):
565 wpt_setup
= command_context
._spawn
(WebPlatformTestsRunnerSetup
)
566 wpt_runner
= WebPlatformTestsRunner(wpt_setup
)
567 logger
= wpt_runner
.setup_logging(**params
)
569 "The wpt manifest is now automatically updated, "
570 "so running this command is usually unnecessary"
572 return 0 if wpt_runner
.update_manifest(logger
, **params
) else 1
578 description
="Run the wpt server",
579 parser
=create_parser_serve
,
580 virtualenv_name
="wpt",
582 def wpt_serve(command_context
, **params
):
585 logger
= logging
.getLogger("web-platform-tests")
586 logger
.addHandler(logging
.StreamHandler(sys
.stdout
))
587 wpt_serve
= command_context
._spawn
(WebPlatformTestsServeRunner
)
588 return wpt_serve
.run(**params
)
592 "wpt-metadata-summary",
594 description
="Create a json summary of the wpt metadata",
595 parser
=create_parser_metadata_summary
,
597 def wpt_summary(command_context
, **params
):
600 wpt_setup
= command_context
._spawn
(WebPlatformTestsRunnerSetup
)
601 return metasummary
.run(wpt_setup
.topsrcdir
, wpt_setup
.topobjdir
, **params
)
604 @Command("wpt-metadata-merge", category
="testing", parser
=create_parser_metadata_merge
)
605 def wpt_meta_merge(command_context
, **params
):
608 if params
["dest"] is None:
609 params
["dest"] = params
["current"]
610 return metamerge
.run(**params
)
616 description
="Run the wpt tools and wptrunner unit tests",
617 parser
=create_parser_unittest
,
618 virtualenv_name
="wpt",
620 def wpt_unittest(command_context
, **params
):
621 runner
= command_context
._spawn
(WebPlatformTestsUnittestRunner
)
622 return 0 if runner
.run(**params
) else 1
628 description
="Get a mapping from test ids to files",
629 parser
=create_parser_testpaths
,
631 def wpt_test_paths(command_context
, **params
):
632 runner
= command_context
._spawn
(WebPlatformTestsTestPathsRunner
)
638 "wpt-fission-regressions",
640 description
="Dump a list of fission-specific regressions",
641 parser
=create_parser_fission_regressions
,
643 def wpt_fission_regressions(command_context
, **params
):
644 runner
= command_context
._spawn
(WebPlatformTestsFissionRegressionsRunner
)