testing/web-platform/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Integrates the web-platform-tests test runner with mach.
   6
   7 from __future__ import absolute_import, unicode_literals, print_function
   8
   9 import os
  10 import sys
  11
  12 from six import iteritems
  13
  14 from mozbuild.base import (
  15     MachCommandBase,
  16     MachCommandConditions as conditions,
  17     MozbuildObject,
  18 )
  19
  20 from mach.decorators import (
  21     CommandProvider,
  22     Command,
  23 )
  24
  25 from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
  26
  27
  28 class WebPlatformTestsRunnerSetup(MozbuildObject):
  29     default_log_type = "mach"
  30
  31     def __init__(self, *args, **kwargs):
  32         super(WebPlatformTestsRunnerSetup, self).__init__(*args, **kwargs)
  33         self._here = os.path.join(self.topsrcdir, "testing", "web-platform")
  34         kwargs["tests_root"] = os.path.join(self._here, "tests")
  35         sys.path.insert(0, kwargs["tests_root"])
  36         build_path = os.path.join(self.topobjdir, "build")
  37         if build_path not in sys.path:
  38             sys.path.append(build_path)
  39
  40     def kwargs_common(self, kwargs):
  41         """Setup kwargs relevant for all browser products"""
  42
  43         tests_src_path = os.path.join(self._here, "tests")
  44
  45         if (
  46             kwargs["product"] in {"firefox", "firefox_android"}
  47             and kwargs["specialpowers_path"] is None
  48         ):
  49             kwargs["specialpowers_path"] = os.path.join(
  50                 self.distdir, "xpi-stage", "specialpowers@mozilla.org.xpi"
  51             )
  52
  53         if kwargs["product"] == "firefox_android":
  54             # package_name may be different in the future
  55             package_name = kwargs["package_name"]
  56             if not package_name:
  57                 kwargs["package_name"] = package_name = "org.mozilla.geckoview.test"
  58
  59             # Note that this import may fail in non-firefox-for-android trees
  60             from mozrunner.devices.android_device import (
  61                 get_adb_path,
  62                 verify_android_device,
  63                 InstallIntent,
  64             )
  65
  66             kwargs["adb_binary"] = get_adb_path(self)
  67             install = (
  68                 InstallIntent.NO if kwargs.pop("no_install") else InstallIntent.YES
  69             )
  70             verify_android_device(
  71                 self, install=install, verbose=False, xre=True, app=package_name
  72             )
  73
  74             if kwargs["certutil_binary"] is None:
  75                 kwargs["certutil_binary"] = os.path.join(
  76                     os.environ.get("MOZ_HOST_BIN"), "certutil"
  77                 )
  78
  79             if kwargs["install_fonts"] is None:
  80                 kwargs["install_fonts"] = True
  81
  82         if kwargs["config"] is None:
  83             kwargs["config"] = os.path.join(
  84                 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini"
  85             )
  86
  87         if (
  88             kwargs["exclude"] is None
  89             and kwargs["include"] is None
  90             and not sys.platform.startswith("linux")
  91         ):
  92             kwargs["exclude"] = ["css"]
  93
  94         if kwargs["ssl_type"] in (None, "pregenerated"):
  95             cert_root = os.path.join(tests_src_path, "tools", "certs")
  96             if kwargs["ca_cert_path"] is None:
  97                 kwargs["ca_cert_path"] = os.path.join(cert_root, "cacert.pem")
  98
  99             if kwargs["host_key_path"] is None:
 100                 kwargs["host_key_path"] = os.path.join(
 101                     cert_root, "web-platform.test.key"
 102                 )
 103
 104             if kwargs["host_cert_path"] is None:
 105                 kwargs["host_cert_path"] = os.path.join(
 106                     cert_root, "web-platform.test.pem"
 107                 )
 108
 109         if kwargs["reftest_screenshot"] is None:
 110             kwargs["reftest_screenshot"] = "fail"
 111
 112         kwargs["capture_stdio"] = True
 113
 114         return kwargs
 115
 116     def kwargs_firefox(self, kwargs):
 117         """Setup kwargs specific to running Firefox and other gecko browsers"""
 118         import mozinfo
 119         from wptrunner import wptcommandline
 120
 121         kwargs = self.kwargs_common(kwargs)
 122
 123         if kwargs["binary"] is None:
 124             kwargs["binary"] = self.get_binary_path()
 125
 126         if kwargs["certutil_binary"] is None:
 127             kwargs["certutil_binary"] = self.get_binary_path("certutil")
 128
 129         if kwargs["webdriver_binary"] is None:
 130             kwargs["webdriver_binary"] = self.get_binary_path(
 131                 "geckodriver", validate_exists=False
 132             )
 133
 134         if kwargs["install_fonts"] is None:
 135             kwargs["install_fonts"] = True
 136
 137         if (
 138             kwargs["install_fonts"]
 139             and mozinfo.info["os"] == "win"
 140             and mozinfo.info["os_version"] == "6.1"
 141         ):
 142             # On Windows 7 --install-fonts fails, so fall back to a Firefox-specific codepath
 143             self.setup_fonts_firefox()
 144             kwargs["install_fonts"] = False
 145
 146         if kwargs["preload_browser"] is None:
 147             kwargs["preload_browser"] = False
 148
 149         if kwargs["prefs_root"] is None:
 150             kwargs["prefs_root"] = os.path.join(self.topsrcdir, "testing", "profiles")
 151
 152         if kwargs["stackfix_dir"] is None:
 153             kwargs["stackfix_dir"] = self.bindir
 154
 155         kwargs = wptcommandline.check_args(kwargs)
 156
 157         return kwargs
 158
 159     def kwargs_wptrun(self, kwargs):
 160         """Setup kwargs for wpt-run which is only used for non-gecko browser products"""
 161         from tools.wpt import run
 162
 163         kwargs = self.kwargs_common(kwargs)
 164
 165         # Add additional kwargs consumed by the run frontend. Currently we don't
 166         # have a way to set these through mach
 167         kwargs["channel"] = None
 168         kwargs["prompt"] = True
 169         kwargs["install_browser"] = False
 170         kwargs["install_webdriver"] = None
 171         kwargs["affected"] = None
 172
 173         # Install the deps
 174         # We do this explicitly to avoid calling pip with options that aren't
 175         # supported in the in-tree version
 176         wptrunner_path = os.path.join(self._here, "tests", "tools", "wptrunner")
 177         browser_cls = run.product_setup[kwargs["product"]].browser_cls
 178         requirements = ["requirements.txt"]
 179         if hasattr(browser_cls, "requirements"):
 180             requirements.append(browser_cls.requirements)
 181
 182         for filename in requirements:
 183             path = os.path.join(wptrunner_path, filename)
 184             if os.path.exists(path):
 185                 self.virtualenv_manager.install_pip_requirements(
 186                     path, require_hashes=False
 187                 )
 188
 189         venv = run.virtualenv.Virtualenv(
 190             self.virtualenv_manager.virtualenv_root, skip_virtualenv_setup=True
 191         )
 192         try:
 193             kwargs = run.setup_wptrunner(venv, **kwargs)
 194         except run.WptrunError as e:
 195             print(e.message, file=sys.stderr)
 196             sys.exit(1)
 197
 198         # This is kind of a hack; override the metadata paths so we don't use
 199         # gecko metadata for non-gecko products
 200         for key, value in list(iteritems(kwargs["test_paths"])):
 201             meta_suffix = key.strip("/")
 202             meta_dir = os.path.join(
 203                 self._here, "products", kwargs["product"], meta_suffix
 204             )
 205             value["metadata_path"] = meta_dir
 206             if not os.path.exists(meta_dir):
 207                 os.makedirs(meta_dir)
 208         return kwargs
 209
 210     def setup_fonts_firefox(self):
 211         # Ensure the Ahem font is available
 212         if not sys.platform.startswith("darwin"):
 213             font_path = os.path.join(os.path.dirname(self.get_binary_path()), "fonts")
 214         else:
 215             font_path = os.path.join(
 216                 os.path.dirname(self.get_binary_path()),
 217                 os.pardir,
 218                 "Resources",
 219                 "res",
 220                 "fonts",
 221             )
 222         ahem_src = os.path.join(
 223             self.topsrcdir, "testing", "web-platform", "tests", "fonts", "Ahem.ttf"
 224         )
 225         ahem_dest = os.path.join(font_path, "Ahem.ttf")
 226         if not os.path.exists(ahem_dest) and os.path.exists(ahem_src):
 227             with open(ahem_src, "rb") as src, open(ahem_dest, "wb") as dest:
 228                 dest.write(src.read())
 229
 230
 231 class WebPlatformTestsServeRunner(MozbuildObject):
 232     def run(self, **kwargs):
 233         sys.path.insert(
 234             0,
 235             os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 236         )
 237         from serve import serve
 238         from wptrunner import wptcommandline
 239         import manifestupdate
 240
 241         import logging
 242
 243         logger = logging.getLogger("web-platform-tests")
 244
 245         src_root = self.topsrcdir
 246         obj_root = self.topobjdir
 247         src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
 248
 249         config_path = manifestupdate.generate_config(
 250             logger,
 251             src_root,
 252             src_wpt_dir,
 253             os.path.join(obj_root, "_tests", "web-platform"),
 254             False,
 255         )
 256
 257         test_paths = wptcommandline.get_test_paths(
 258             wptcommandline.config.read(config_path)
 259         )
 260
 261         def get_route_builder(*args, **kwargs):
 262             route_builder = serve.get_route_builder(*args, **kwargs)
 263
 264             for url_base, paths in iteritems(test_paths):
 265                 if url_base != "/":
 266                     route_builder.add_mount_point(url_base, paths["tests_path"])
 267
 268             return route_builder
 269
 270         return 0 if serve.run(route_builder=get_route_builder, **kwargs) else 1
 271
 272
 273 class WebPlatformTestsUpdater(MozbuildObject):
 274     """Update web platform tests."""
 275
 276     def setup_logging(self, **kwargs):
 277         import update
 278
 279         return update.setup_logging(kwargs, {"mach": sys.stdout})
 280
 281     def update_manifest(self, logger, **kwargs):
 282         import manifestupdate
 283
 284         return manifestupdate.run(
 285             logger=logger, src_root=self.topsrcdir, obj_root=self.topobjdir, **kwargs
 286         )
 287
 288     def run_update(self, logger, **kwargs):
 289         import update
 290         from update import updatecommandline
 291
 292         self.update_manifest(logger, **kwargs)
 293
 294         if kwargs["config"] is None:
 295             kwargs["config"] = os.path.join(
 296                 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini"
 297             )
 298         if kwargs["product"] is None:
 299             kwargs["product"] = "firefox"
 300
 301         kwargs["store_state"] = False
 302
 303         kwargs = updatecommandline.check_args(kwargs)
 304
 305         try:
 306             update.run_update(logger, **kwargs)
 307         except Exception:
 308             import traceback
 309
 310             traceback.print_exc()
 311
 312
 313 class WebPlatformTestsUnittestRunner(MozbuildObject):
 314     def run(self, **kwargs):
 315         import unittestrunner
 316
 317         return unittestrunner.run(self.topsrcdir, **kwargs)
 318
 319
 320 class WebPlatformTestsTestPathsRunner(MozbuildObject):
 321     """Update web platform tests."""
 322
 323     def run(self, **kwargs):
 324         sys.path.insert(
 325             0,
 326             os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 327         )
 328         from wptrunner import wptcommandline
 329         from manifest import testpaths
 330         import manifestupdate
 331
 332         import logging
 333
 334         logger = logging.getLogger("web-platform-tests")
 335
 336         src_root = self.topsrcdir
 337         obj_root = self.topobjdir
 338         src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
 339
 340         config_path = manifestupdate.generate_config(
 341             logger,
 342             src_root,
 343             src_wpt_dir,
 344             os.path.join(obj_root, "_tests", "web-platform"),
 345             False,
 346         )
 347
 348         test_paths = wptcommandline.get_test_paths(
 349             wptcommandline.config.read(config_path)
 350         )
 351         results = {}
 352         for url_base, paths in iteritems(test_paths):
 353             if "manifest_path" not in paths:
 354                 paths["manifest_path"] = os.path.join(
 355                     paths["metadata_path"], "MANIFEST.json"
 356                 )
 357             results.update(
 358                 testpaths.get_paths(
 359                     path=paths["manifest_path"],
 360                     src_root=src_root,
 361                     tests_root=paths["tests_path"],
 362                     update=kwargs["update"],
 363                     rebuild=kwargs["rebuild"],
 364                     url_base=url_base,
 365                     cache_root=kwargs["cache_root"],
 366                     test_ids=kwargs["test_ids"],
 367                 )
 368             )
 369         testpaths.write_output(results, kwargs["json"])
 370         return True
 371
 372
 373 class WebPlatformTestsFissionRegressionsRunner(MozbuildObject):
 374     def run(self, **kwargs):
 375         import mozlog
 376         import fissionregressions
 377
 378         src_root = self.topsrcdir
 379         obj_root = self.topobjdir
 380         logger = mozlog.structuredlog.StructuredLogger("web-platform-tests")
 381
 382         try:
 383             return fissionregressions.run(logger, src_root, obj_root, **kwargs)
 384         except Exception:
 385             import traceback
 386             import pdb
 387
 388             traceback.print_exc()
 389             pdb.post_mortem()
 390
 391
 392 def create_parser_update():
 393     from update import updatecommandline
 394
 395     return updatecommandline.create_parser()
 396
 397
 398 def create_parser_manifest_update():
 399     import manifestupdate
 400
 401     return manifestupdate.create_parser()
 402
 403
 404 def create_parser_metadata_summary():
 405     import metasummary
 406
 407     return metasummary.create_parser()
 408
 409
 410 def create_parser_metadata_merge():
 411     import metamerge
 412
 413     return metamerge.get_parser()
 414
 415
 416 def create_parser_serve():
 417     sys.path.insert(
 418         0, os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools"))
 419     )
 420     import serve
 421
 422     return serve.serve.get_parser()
 423
 424
 425 def create_parser_unittest():
 426     import unittestrunner
 427
 428     return unittestrunner.get_parser()
 429
 430
 431 def create_parser_fission_regressions():
 432     import fissionregressions
 433
 434     return fissionregressions.get_parser()
 435
 436
 437 def create_parser_testpaths():
 438     import argparse
 439     from mozboot.util import get_state_dir
 440
 441     parser = argparse.ArgumentParser()
 442     parser.add_argument(
 443         "--no-update",
 444         dest="update",
 445         action="store_false",
 446         default=True,
 447         help="Don't update manifest before continuing",
 448     )
 449     parser.add_argument(
 450         "-r",
 451         "--rebuild",
 452         action="store_true",
 453         default=False,
 454         help="Force a full rebuild of the manifest.",
 455     )
 456     parser.add_argument(
 457         "--cache-root",
 458         action="store",
 459         default=os.path.join(get_state_dir(), "cache", "wpt"),
 460         help="Path in which to store any caches (default <tests_root>/.wptcache/)",
 461     )
 462     parser.add_argument(
 463         "test_ids", action="store", nargs="+", help="Test ids for which to get paths"
 464     )
 465     parser.add_argument(
 466         "--json", action="store_true", default=False, help="Output as JSON"
 467     )
 468     return parser
 469
 470
 471 @CommandProvider
 472 class MachCommands(MachCommandBase):
 473     def setup(self):
 474         self.activate_virtualenv()
 475
 476     @Command(
 477         "web-platform-tests",
 478         category="testing",
 479         conditions=[conditions.is_firefox_or_android],
 480         description="Run web-platform-tests.",
 481         parser=create_parser_wpt,
 482     )
 483     def run_web_platform_tests(self, command_context, **params):
 484         self.setup()
 485         if params["product"] is None:
 486             if conditions.is_android(self):
 487                 params["product"] = "firefox_android"
 488             else:
 489                 params["product"] = "firefox"
 490         if "test_objects" in params:
 491             include = []
 492             test_types = set()
 493             for item in params["test_objects"]:
 494                 include.append(item["name"])
 495                 test_types.add(item.get("subsuite"))
 496             if None not in test_types:
 497                 params["test_types"] = list(test_types)
 498             params["include"] = include
 499             del params["test_objects"]
 500         if params.get("debugger", None):
 501             import mozdebug
 502
 503             if not mozdebug.get_debugger_info(params.get("debugger")):
 504                 sys.exit(1)
 505
 506         wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
 507         wpt_setup._mach_context = self._mach_context
 508         wpt_runner = WebPlatformTestsRunner(wpt_setup)
 509
 510         logger = wpt_runner.setup_logging(**params)
 511
 512         if conditions.is_android(self) and params["product"] != "firefox_android":
 513             logger.warning(
 514                 "Must specify --product=firefox_android in Android environment."
 515             )
 516
 517         return wpt_runner.run(logger, **params)
 518
 519     @Command(
 520         "wpt",
 521         category="testing",
 522         conditions=[conditions.is_firefox_or_android],
 523         description="Run web-platform-tests.",
 524         parser=create_parser_wpt,
 525     )
 526     def run_wpt(self, command_context, **params):
 527         return self.run_web_platform_tests(command_context, **params)
 528
 529     @Command(
 530         "web-platform-tests-update",
 531         category="testing",
 532         description="Update web-platform-test metadata.",
 533         parser=create_parser_update,
 534     )
 535     def update_web_platform_tests(self, command_context, **params):
 536         self.setup()
 537         self.virtualenv_manager.install_pip_package("html5lib==1.0.1")
 538         self.virtualenv_manager.install_pip_package("ujson")
 539         self.virtualenv_manager.install_pip_package("requests")
 540
 541         wpt_updater = self._spawn(WebPlatformTestsUpdater)
 542         logger = wpt_updater.setup_logging(**params)
 543         return wpt_updater.run_update(logger, **params)
 544
 545     @Command(
 546         "wpt-update",
 547         category="testing",
 548         description="Update web-platform-test metadata.",
 549         parser=create_parser_update,
 550     )
 551     def update_wpt(self, command_context, **params):
 552         return self.update_web_platform_tests(command_context, **params)
 553
 554     @Command(
 555         "wpt-manifest-update",
 556         category="testing",
 557         description="Update web-platform-test manifests.",
 558         parser=create_parser_manifest_update,
 559     )
 560     def wpt_manifest_update(self, command_context, **params):
 561         self.setup()
 562         wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
 563         wpt_runner = WebPlatformTestsRunner(wpt_setup)
 564         logger = wpt_runner.setup_logging(**params)
 565         logger.warning(
 566             "The wpt manifest is now automatically updated, "
 567             "so running this command is usually unnecessary"
 568         )
 569         return 0 if wpt_runner.update_manifest(logger, **params) else 1
 570
 571     @Command(
 572         "wpt-serve",
 573         category="testing",
 574         description="Run the wpt server",
 575         parser=create_parser_serve,
 576     )
 577     def wpt_serve(self, command_context, **params):
 578         self.setup()
 579         import logging
 580
 581         logger = logging.getLogger("web-platform-tests")
 582         logger.addHandler(logging.StreamHandler(sys.stdout))
 583         wpt_serve = self._spawn(WebPlatformTestsServeRunner)
 584         return wpt_serve.run(**params)
 585
 586     @Command(
 587         "wpt-metadata-summary",
 588         category="testing",
 589         description="Create a json summary of the wpt metadata",
 590         parser=create_parser_metadata_summary,
 591     )
 592     def wpt_summary(self, command_context, **params):
 593         import metasummary
 594
 595         wpt_setup = self._spawn(WebPlatformTestsRunnerSetup)
 596         return metasummary.run(wpt_setup.topsrcdir, wpt_setup.topobjdir, **params)
 597
 598     @Command(
 599         "wpt-metadata-merge", category="testing", parser=create_parser_metadata_merge
 600     )
 601     def wpt_meta_merge(self, command_context, **params):
 602         import metamerge
 603
 604         if params["dest"] is None:
 605             params["dest"] = params["current"]
 606         return metamerge.run(**params)
 607
 608     @Command(
 609         "wpt-unittest",
 610         category="testing",
 611         description="Run the wpt tools and wptrunner unit tests",
 612         parser=create_parser_unittest,
 613     )
 614     def wpt_unittest(self, command_context, **params):
 615         self.setup()
 616         self.virtualenv_manager.install_pip_package("tox")
 617         runner = self._spawn(WebPlatformTestsUnittestRunner)
 618         return 0 if runner.run(**params) else 1
 619
 620     @Command(
 621         "wpt-test-paths",
 622         category="testing",
 623         description="Get a mapping from test ids to files",
 624         parser=create_parser_testpaths,
 625     )
 626     def wpt_test_paths(self, command_context, **params):
 627         runner = self._spawn(WebPlatformTestsTestPathsRunner)
 628         runner.run(**params)
 629         return 0
 630
 631     @Command(
 632         "wpt-fission-regressions",
 633         category="testing",
 634         description="Dump a list of fission-specific regressions",
 635         parser=create_parser_fission_regressions,
 636     )
 637     def wpt_fission_regressions(self, command_context, **params):
 638         runner = self._spawn(WebPlatformTestsFissionRegressionsRunner)
 639         runner.run(**params)
 640         return 0