testing/web-platform/mach_commands.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 # Integrates the web-platform-tests test runner with mach.
   6
   7 from __future__ import absolute_import, unicode_literals, print_function
   8
   9 import os
  10 import sys
  11
  12 from six import iteritems
  13
  14 from mozbuild.base import (
  15     MachCommandConditions as conditions,
  16     MozbuildObject,
  17 )
  18
  19 from mach.decorators import (
  20     Command,
  21 )
  22
  23 from mach_commands_base import WebPlatformTestsRunner, create_parser_wpt
  24
  25
  26 class WebPlatformTestsRunnerSetup(MozbuildObject):
  27     default_log_type = "mach"
  28
  29     def __init__(self, *args, **kwargs):
  30         super(WebPlatformTestsRunnerSetup, self).__init__(*args, **kwargs)
  31         self._here = os.path.join(self.topsrcdir, "testing", "web-platform")
  32         kwargs["tests_root"] = os.path.join(self._here, "tests")
  33         sys.path.insert(0, kwargs["tests_root"])
  34         build_path = os.path.join(self.topobjdir, "build")
  35         if build_path not in sys.path:
  36             sys.path.append(build_path)
  37
  38     def kwargs_common(self, kwargs):
  39         """Setup kwargs relevant for all browser products"""
  40
  41         tests_src_path = os.path.join(self._here, "tests")
  42
  43         if (
  44             kwargs["product"] in {"firefox", "firefox_android"}
  45             and kwargs["specialpowers_path"] is None
  46         ):
  47             kwargs["specialpowers_path"] = os.path.join(
  48                 self.distdir, "xpi-stage", "specialpowers@mozilla.org.xpi"
  49             )
  50
  51         if kwargs["product"] == "firefox_android":
  52             # package_name may be different in the future
  53             package_name = kwargs["package_name"]
  54             if not package_name:
  55                 kwargs[
  56                     "package_name"
  57                 ] = package_name = "org.mozilla.geckoview.test_runner"
  58
  59             # Note that this import may fail in non-firefox-for-android trees
  60             from mozrunner.devices.android_device import (
  61                 get_adb_path,
  62                 verify_android_device,
  63                 InstallIntent,
  64             )
  65
  66             kwargs["adb_binary"] = get_adb_path(self)
  67             install = (
  68                 InstallIntent.NO if kwargs.pop("no_install") else InstallIntent.YES
  69             )
  70             verify_android_device(
  71                 self, install=install, verbose=False, xre=True, app=package_name
  72             )
  73
  74             if kwargs["certutil_binary"] is None:
  75                 kwargs["certutil_binary"] = os.path.join(
  76                     os.environ.get("MOZ_HOST_BIN"), "certutil"
  77                 )
  78
  79             if kwargs["install_fonts"] is None:
  80                 kwargs["install_fonts"] = True
  81
  82             if not kwargs["device_serial"]:
  83                 kwargs["device_serial"] = ["emulator-5554"]
  84
  85         if kwargs["config"] is None:
  86             kwargs["config"] = os.path.join(
  87                 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini"
  88             )
  89
  90         if (
  91             kwargs["exclude"] is None
  92             and kwargs["include"] is None
  93             and not sys.platform.startswith("linux")
  94         ):
  95             kwargs["exclude"] = ["css"]
  96
  97         if kwargs["ssl_type"] in (None, "pregenerated"):
  98             cert_root = os.path.join(tests_src_path, "tools", "certs")
  99             if kwargs["ca_cert_path"] is None:
 100                 kwargs["ca_cert_path"] = os.path.join(cert_root, "cacert.pem")
 101
 102             if kwargs["host_key_path"] is None:
 103                 kwargs["host_key_path"] = os.path.join(
 104                     cert_root, "web-platform.test.key"
 105                 )
 106
 107             if kwargs["host_cert_path"] is None:
 108                 kwargs["host_cert_path"] = os.path.join(
 109                     cert_root, "web-platform.test.pem"
 110                 )
 111
 112         if kwargs["reftest_screenshot"] is None:
 113             kwargs["reftest_screenshot"] = "fail"
 114
 115         kwargs["capture_stdio"] = True
 116
 117         return kwargs
 118
 119     def kwargs_firefox(self, kwargs):
 120         """Setup kwargs specific to running Firefox and other gecko browsers"""
 121         import mozinfo
 122         from wptrunner import wptcommandline
 123
 124         kwargs = self.kwargs_common(kwargs)
 125
 126         if kwargs["binary"] is None:
 127             kwargs["binary"] = self.get_binary_path()
 128
 129         if kwargs["certutil_binary"] is None:
 130             kwargs["certutil_binary"] = self.get_binary_path("certutil")
 131
 132         if kwargs["webdriver_binary"] is None:
 133             kwargs["webdriver_binary"] = self.get_binary_path(
 134                 "geckodriver", validate_exists=False
 135             )
 136
 137         if kwargs["install_fonts"] is None:
 138             kwargs["install_fonts"] = True
 139
 140         if (
 141             kwargs["install_fonts"]
 142             and mozinfo.info["os"] == "win"
 143             and mozinfo.info["os_version"] == "6.1"
 144         ):
 145             # On Windows 7 --install-fonts fails, so fall back to a Firefox-specific codepath
 146             self.setup_fonts_firefox()
 147             kwargs["install_fonts"] = False
 148
 149         if kwargs["preload_browser"] is None:
 150             kwargs["preload_browser"] = False
 151
 152         if kwargs["prefs_root"] is None:
 153             kwargs["prefs_root"] = os.path.join(self.topsrcdir, "testing", "profiles")
 154
 155         if kwargs["stackfix_dir"] is None:
 156             kwargs["stackfix_dir"] = self.bindir
 157
 158         kwargs = wptcommandline.check_args(kwargs)
 159
 160         return kwargs
 161
 162     def kwargs_wptrun(self, kwargs):
 163         """Setup kwargs for wpt-run which is only used for non-gecko browser products"""
 164         from tools.wpt import run, virtualenv
 165
 166         kwargs = self.kwargs_common(kwargs)
 167
 168         # Add additional kwargs consumed by the run frontend. Currently we don't
 169         # have a way to set these through mach
 170         kwargs["channel"] = None
 171         kwargs["prompt"] = True
 172         kwargs["install_browser"] = False
 173         kwargs["install_webdriver"] = None
 174         kwargs["affected"] = None
 175
 176         # Install the deps
 177         # We do this explicitly to avoid calling pip with options that aren't
 178         # supported in the in-tree version
 179         wptrunner_path = os.path.join(self._here, "tests", "tools", "wptrunner")
 180         browser_cls = run.product_setup[kwargs["product"]].browser_cls
 181         requirements = ["requirements.txt"]
 182         if hasattr(browser_cls, "requirements"):
 183             requirements.append(browser_cls.requirements)
 184
 185         for filename in requirements:
 186             path = os.path.join(wptrunner_path, filename)
 187             if os.path.exists(path):
 188                 self.virtualenv_manager.install_pip_requirements(
 189                     path, require_hashes=False
 190                 )
 191
 192         venv = virtualenv.Virtualenv(
 193             self.virtualenv_manager.virtualenv_root, skip_virtualenv_setup=True
 194         )
 195         try:
 196             kwargs = run.setup_wptrunner(venv, **kwargs)
 197         except run.WptrunError as e:
 198             print(e, file=sys.stderr)
 199             sys.exit(1)
 200
 201         # This is kind of a hack; override the metadata paths so we don't use
 202         # gecko metadata for non-gecko products
 203         for key, value in list(iteritems(kwargs["test_paths"])):
 204             meta_suffix = key.strip("/")
 205             meta_dir = os.path.join(
 206                 self._here, "products", kwargs["product"], meta_suffix
 207             )
 208             value["metadata_path"] = meta_dir
 209             if not os.path.exists(meta_dir):
 210                 os.makedirs(meta_dir)
 211         return kwargs
 212
 213     def setup_fonts_firefox(self):
 214         # Ensure the Ahem font is available
 215         if not sys.platform.startswith("darwin"):
 216             font_path = os.path.join(os.path.dirname(self.get_binary_path()), "fonts")
 217         else:
 218             font_path = os.path.join(
 219                 os.path.dirname(self.get_binary_path()),
 220                 os.pardir,
 221                 "Resources",
 222                 "res",
 223                 "fonts",
 224             )
 225         ahem_src = os.path.join(
 226             self.topsrcdir, "testing", "web-platform", "tests", "fonts", "Ahem.ttf"
 227         )
 228         ahem_dest = os.path.join(font_path, "Ahem.ttf")
 229         if not os.path.exists(ahem_dest) and os.path.exists(ahem_src):
 230             with open(ahem_src, "rb") as src, open(ahem_dest, "wb") as dest:
 231                 dest.write(src.read())
 232
 233
 234 class WebPlatformTestsServeRunner(MozbuildObject):
 235     def run(self, **kwargs):
 236         sys.path.insert(
 237             0,
 238             os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 239         )
 240         from serve import serve
 241         from wptrunner import wptcommandline
 242         import manifestupdate
 243
 244         import logging
 245
 246         logger = logging.getLogger("web-platform-tests")
 247
 248         src_root = self.topsrcdir
 249         obj_root = self.topobjdir
 250         src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
 251
 252         config_path = manifestupdate.generate_config(
 253             logger,
 254             src_root,
 255             src_wpt_dir,
 256             os.path.join(obj_root, "_tests", "web-platform"),
 257             False,
 258         )
 259
 260         test_paths = wptcommandline.get_test_paths(
 261             wptcommandline.config.read(config_path)
 262         )
 263
 264         def get_route_builder(*args, **kwargs):
 265             route_builder = serve.get_route_builder(*args, **kwargs)
 266
 267             for url_base, paths in iteritems(test_paths):
 268                 if url_base != "/":
 269                     route_builder.add_mount_point(url_base, paths["tests_path"])
 270
 271             return route_builder
 272
 273         return 0 if serve.run(route_builder=get_route_builder, **kwargs) else 1
 274
 275
 276 class WebPlatformTestsUpdater(MozbuildObject):
 277     """Update web platform tests."""
 278
 279     def setup_logging(self, **kwargs):
 280         import update
 281
 282         return update.setup_logging(kwargs, {"mach": sys.stdout})
 283
 284     def update_manifest(self, logger, **kwargs):
 285         import manifestupdate
 286
 287         return manifestupdate.run(
 288             logger=logger, src_root=self.topsrcdir, obj_root=self.topobjdir, **kwargs
 289         )
 290
 291     def run_update(self, logger, **kwargs):
 292         import update
 293         from update import updatecommandline
 294
 295         self.update_manifest(logger, **kwargs)
 296
 297         if kwargs["config"] is None:
 298             kwargs["config"] = os.path.join(
 299                 self.topobjdir, "_tests", "web-platform", "wptrunner.local.ini"
 300             )
 301         if kwargs["product"] is None:
 302             kwargs["product"] = "firefox"
 303
 304         kwargs["store_state"] = False
 305
 306         kwargs = updatecommandline.check_args(kwargs)
 307
 308         try:
 309             update.run_update(logger, **kwargs)
 310         except Exception:
 311             import traceback
 312
 313             traceback.print_exc()
 314
 315
 316 class WebPlatformTestsUnittestRunner(MozbuildObject):
 317     def run(self, **kwargs):
 318         import unittestrunner
 319
 320         return unittestrunner.run(self.topsrcdir, **kwargs)
 321
 322
 323 class WebPlatformTestsTestPathsRunner(MozbuildObject):
 324     """Update web platform tests."""
 325
 326     def run(self, **kwargs):
 327         sys.path.insert(
 328             0,
 329             os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools")),
 330         )
 331         from wptrunner import wptcommandline
 332         from manifest import testpaths
 333         import manifestupdate
 334
 335         import logging
 336
 337         logger = logging.getLogger("web-platform-tests")
 338
 339         src_root = self.topsrcdir
 340         obj_root = self.topobjdir
 341         src_wpt_dir = os.path.join(src_root, "testing", "web-platform")
 342
 343         config_path = manifestupdate.generate_config(
 344             logger,
 345             src_root,
 346             src_wpt_dir,
 347             os.path.join(obj_root, "_tests", "web-platform"),
 348             False,
 349         )
 350
 351         test_paths = wptcommandline.get_test_paths(
 352             wptcommandline.config.read(config_path)
 353         )
 354         results = {}
 355         for url_base, paths in iteritems(test_paths):
 356             if "manifest_path" not in paths:
 357                 paths["manifest_path"] = os.path.join(
 358                     paths["metadata_path"], "MANIFEST.json"
 359                 )
 360             results.update(
 361                 testpaths.get_paths(
 362                     path=paths["manifest_path"],
 363                     src_root=src_root,
 364                     tests_root=paths["tests_path"],
 365                     update=kwargs["update"],
 366                     rebuild=kwargs["rebuild"],
 367                     url_base=url_base,
 368                     cache_root=kwargs["cache_root"],
 369                     test_ids=kwargs["test_ids"],
 370                 )
 371             )
 372         testpaths.write_output(results, kwargs["json"])
 373         return True
 374
 375
 376 class WebPlatformTestsFissionRegressionsRunner(MozbuildObject):
 377     def run(self, **kwargs):
 378         import mozlog
 379         import fissionregressions
 380
 381         src_root = self.topsrcdir
 382         obj_root = self.topobjdir
 383         logger = mozlog.structuredlog.StructuredLogger("web-platform-tests")
 384
 385         try:
 386             return fissionregressions.run(logger, src_root, obj_root, **kwargs)
 387         except Exception:
 388             import traceback
 389             import pdb
 390
 391             traceback.print_exc()
 392             pdb.post_mortem()
 393
 394
 395 def create_parser_update():
 396     from update import updatecommandline
 397
 398     return updatecommandline.create_parser()
 399
 400
 401 def create_parser_manifest_update():
 402     import manifestupdate
 403
 404     return manifestupdate.create_parser()
 405
 406
 407 def create_parser_metadata_summary():
 408     import metasummary
 409
 410     return metasummary.create_parser()
 411
 412
 413 def create_parser_metadata_merge():
 414     import metamerge
 415
 416     return metamerge.get_parser()
 417
 418
 419 def create_parser_serve():
 420     sys.path.insert(
 421         0, os.path.abspath(os.path.join(os.path.dirname(__file__), "tests", "tools"))
 422     )
 423     import serve
 424
 425     return serve.serve.get_parser()
 426
 427
 428 def create_parser_unittest():
 429     import unittestrunner
 430
 431     return unittestrunner.get_parser()
 432
 433
 434 def create_parser_fission_regressions():
 435     import fissionregressions
 436
 437     return fissionregressions.get_parser()
 438
 439
 440 def create_parser_testpaths():
 441     import argparse
 442     from mach.util import get_state_dir
 443
 444     parser = argparse.ArgumentParser()
 445     parser.add_argument(
 446         "--no-update",
 447         dest="update",
 448         action="store_false",
 449         default=True,
 450         help="Don't update manifest before continuing",
 451     )
 452     parser.add_argument(
 453         "-r",
 454         "--rebuild",
 455         action="store_true",
 456         default=False,
 457         help="Force a full rebuild of the manifest.",
 458     )
 459     parser.add_argument(
 460         "--cache-root",
 461         action="store",
 462         default=os.path.join(get_state_dir(), "cache", "wpt"),
 463         help="Path in which to store any caches (default <tests_root>/.wptcache/)",
 464     )
 465     parser.add_argument(
 466         "test_ids", action="store", nargs="+", help="Test ids for which to get paths"
 467     )
 468     parser.add_argument(
 469         "--json", action="store_true", default=False, help="Output as JSON"
 470     )
 471     return parser
 472
 473
 474 @Command(
 475     "web-platform-tests",
 476     category="testing",
 477     conditions=[conditions.is_firefox_or_android],
 478     description="Run web-platform-tests.",
 479     parser=create_parser_wpt,
 480     virtualenv_name="wpt",
 481 )
 482 def run_web_platform_tests(command_context, **params):
 483     if params["product"] is None:
 484         if conditions.is_android(command_context):
 485             params["product"] = "firefox_android"
 486         else:
 487             params["product"] = "firefox"
 488     if "test_objects" in params:
 489         include = []
 490         test_types = set()
 491         for item in params["test_objects"]:
 492             include.append(item["name"])
 493             test_types.add(item.get("subsuite"))
 494         if None not in test_types:
 495             params["test_types"] = list(test_types)
 496         params["include"] = include
 497         del params["test_objects"]
 498     if params.get("debugger", None):
 499         import mozdebug
 500
 501         if not mozdebug.get_debugger_info(params.get("debugger")):
 502             sys.exit(1)
 503
 504     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 505     wpt_setup._mach_context = command_context._mach_context
 506     wpt_runner = WebPlatformTestsRunner(wpt_setup)
 507
 508     logger = wpt_runner.setup_logging(**params)
 509     # wptrunner already handles setting any log parameter from
 510     # mach test to the logger, so it's OK to remove that argument now
 511     if "log" in params:
 512         del params["log"]
 513
 514     if (
 515         conditions.is_android(command_context)
 516         and params["product"] != "firefox_android"
 517     ):
 518         logger.warning("Must specify --product=firefox_android in Android environment.")
 519
 520     return wpt_runner.run(logger, **params)
 521
 522
 523 @Command(
 524     "wpt",
 525     category="testing",
 526     conditions=[conditions.is_firefox_or_android],
 527     description="Run web-platform-tests.",
 528     parser=create_parser_wpt,
 529 )
 530 def run_wpt(command_context, **params):
 531     return run_web_platform_tests(command_context, **params)
 532
 533
 534 @Command(
 535     "web-platform-tests-update",
 536     category="testing",
 537     description="Update web-platform-test metadata.",
 538     parser=create_parser_update,
 539     virtualenv_name="wpt",
 540 )
 541 def update_web_platform_tests(command_context, **params):
 542     wpt_updater = command_context._spawn(WebPlatformTestsUpdater)
 543     logger = wpt_updater.setup_logging(**params)
 544     return wpt_updater.run_update(logger, **params)
 545
 546
 547 @Command(
 548     "wpt-update",
 549     category="testing",
 550     description="Update web-platform-test metadata.",
 551     parser=create_parser_update,
 552 )
 553 def update_wpt(command_context, **params):
 554     return update_web_platform_tests(command_context, **params)
 555
 556
 557 @Command(
 558     "wpt-manifest-update",
 559     category="testing",
 560     description="Update web-platform-test manifests.",
 561     parser=create_parser_manifest_update,
 562     virtualenv_name="wpt",
 563 )
 564 def wpt_manifest_update(command_context, **params):
 565     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 566     wpt_runner = WebPlatformTestsRunner(wpt_setup)
 567     logger = wpt_runner.setup_logging(**params)
 568     logger.warning(
 569         "The wpt manifest is now automatically updated, "
 570         "so running this command is usually unnecessary"
 571     )
 572     return 0 if wpt_runner.update_manifest(logger, **params) else 1
 573
 574
 575 @Command(
 576     "wpt-serve",
 577     category="testing",
 578     description="Run the wpt server",
 579     parser=create_parser_serve,
 580     virtualenv_name="wpt",
 581 )
 582 def wpt_serve(command_context, **params):
 583     import logging
 584
 585     logger = logging.getLogger("web-platform-tests")
 586     logger.addHandler(logging.StreamHandler(sys.stdout))
 587     wpt_serve = command_context._spawn(WebPlatformTestsServeRunner)
 588     return wpt_serve.run(**params)
 589
 590
 591 @Command(
 592     "wpt-metadata-summary",
 593     category="testing",
 594     description="Create a json summary of the wpt metadata",
 595     parser=create_parser_metadata_summary,
 596 )
 597 def wpt_summary(command_context, **params):
 598     import metasummary
 599
 600     wpt_setup = command_context._spawn(WebPlatformTestsRunnerSetup)
 601     return metasummary.run(wpt_setup.topsrcdir, wpt_setup.topobjdir, **params)
 602
 603
 604 @Command("wpt-metadata-merge", category="testing", parser=create_parser_metadata_merge)
 605 def wpt_meta_merge(command_context, **params):
 606     import metamerge
 607
 608     if params["dest"] is None:
 609         params["dest"] = params["current"]
 610     return metamerge.run(**params)
 611
 612
 613 @Command(
 614     "wpt-unittest",
 615     category="testing",
 616     description="Run the wpt tools and wptrunner unit tests",
 617     parser=create_parser_unittest,
 618     virtualenv_name="wpt",
 619 )
 620 def wpt_unittest(command_context, **params):
 621     runner = command_context._spawn(WebPlatformTestsUnittestRunner)
 622     return 0 if runner.run(**params) else 1
 623
 624
 625 @Command(
 626     "wpt-test-paths",
 627     category="testing",
 628     description="Get a mapping from test ids to files",
 629     parser=create_parser_testpaths,
 630 )
 631 def wpt_test_paths(command_context, **params):
 632     runner = command_context._spawn(WebPlatformTestsTestPathsRunner)
 633     runner.run(**params)
 634     return 0
 635
 636
 637 @Command(
 638     "wpt-fission-regressions",
 639     category="testing",
 640     description="Dump a list of fission-specific regressions",
 641     parser=create_parser_fission_regressions,
 642 )
 643 def wpt_fission_regressions(command_context, **params):
 644     runner = command_context._spawn(WebPlatformTestsFissionRegressionsRunner)
 645     runner.run(**params)
 646     return 0